From 1f23c18841808819d102e6c9ba44d13ce6211be3 Mon Sep 17 00:00:00 2001 From: ceppelli Date: Sat, 18 May 2024 19:43:06 +0200 Subject: [PATCH] fix: newFileScanRDD should not take constructor from custom Spark versions (#412) * [FIX] - workaround for aws emr spark 3.4 workaround for Amazon EMR version: emr-6.15.0 and Spark 3.4.1 custom implementation * Update spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometScanExec.scala * Update spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometScanExec.scala --------- Co-authored-by: Liang-Chi Hsieh --- .../spark-3.x/org/apache/comet/shims/ShimCometScanExec.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometScanExec.scala b/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometScanExec.scala index 18871b814..544a67385 100644 --- a/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometScanExec.scala +++ b/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometScanExec.scala @@ -69,6 +69,8 @@ trait ShimCometScanExec { readSchema: StructType, options: ParquetOptions): FileScanRDD = classOf[FileScanRDD].getDeclaredConstructors + // Prevent to pick up incorrect constructors from any custom Spark forks. + .filter(c => List(3, 5, 6).contains(c.getParameterCount()) ) .map { c => c.getParameterCount match { case 3 => c.newInstance(sparkSession, readFunction, filePartitions)