fix: newFileScanRDD should not take constructor from custom Spark ver…

…sions (#412) * [FIX] - workaround for aws emr spark 3.4 workaround for Amazon EMR version: emr-6.15.0 and Spark 3.4.1 custom implementation * Update spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometScanExec.scala * Update spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometScanExec.scala --------- Co-authored-by: Liang-Chi Hsieh <[email protected]>
apache · May 18, 2024 · 1f23c18 · 1f23c18
1 parent ec8da30
commit 1f23c18
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometScanExec.scala b/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometScanExec.scala
@@ -69,6 +69,8 @@ trait ShimCometScanExec {
       readSchema: StructType,
       options: ParquetOptions): FileScanRDD =
     classOf[FileScanRDD].getDeclaredConstructors
+      // Prevent to pick up incorrect constructors from any custom Spark forks.
+      .filter(c => List(3, 5, 6).contains(c.getParameterCount()) )
       .map { c =>
         c.getParameterCount match {
           case 3 => c.newInstance(sparkSession, readFunction, filePartitions)