[FIX] - workaround for aws emr spark 3.4

workaround for Amazon EMR version: emr-6.15.0 and Spark 3.4.1 custom implementation
apache · May 10, 2024 · cc2cd52 · cc2cd52
1 parent bc6b2cd
commit cc2cd52
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometScanExec.scala b/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometScanExec.scala
@@ -69,6 +69,8 @@ trait ShimCometScanExec {
       readSchema: StructType,
       options: ParquetOptions): FileScanRDD =
     classOf[FileScanRDD].getDeclaredConstructors
+      // workaround for aws spark 3.4 implementation
+      .filter(c => List(3,5,6).contains(c.getParameterCount()) )
       .map { c =>
         c.getParameterCount match {
           case 3 => c.newInstance(sparkSession, readFunction, filePartitions)