diff --git a/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometScanExec.scala b/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometScanExec.scala index 18871b814..499c089b0 100644 --- a/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometScanExec.scala +++ b/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometScanExec.scala @@ -69,6 +69,8 @@ trait ShimCometScanExec { readSchema: StructType, options: ParquetOptions): FileScanRDD = classOf[FileScanRDD].getDeclaredConstructors + // workaround for aws spark 3.4 implementation + .filter(c => List(3,5,6).contains(c.getParameterCount()) ) .map { c => c.getParameterCount match { case 3 => c.newInstance(sparkSession, readFunction, filePartitions)