Save

apache · Dec 5, 2024 · 0602e24 · 0602e24
1 parent 0b43b23
commit 0602e24
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 2 deletions.
diff --git a/native/core/src/execution/datafusion/schema_adapter.rs b/native/core/src/execution/datafusion/schema_adapter.rs
@@ -127,6 +127,10 @@ pub fn spark_can_cast_types(from_type: &DataType, to_type: &DataType) -> bool {
             true
         }
         (_, DataType::Timestamp(TimeUnit::Nanosecond, _)) => false,
+        // Native cast invoked for unsupported cast from FixedSizeBinary(3) to Binary.
+        (DataType::FixedSizeBinary(_), _) => false,
+        // Native cast invoked for unsupported cast from UInt32 to Int64.
+        (DataType::UInt8 | DataType::UInt16 | DataType::UInt32 | DataType::UInt64, _) => false,
         _ => can_cast_types(from_type, to_type),
     }
 }

diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
@@ -2496,8 +2496,9 @@ object QueryPlanSerde extends Logging with ShimQueryPlanSerde with CometExprShim
           // Sink operators don't have children
           result.clearChildren()
 
-          val dataFilters = scan.dataFilters.map(exprToProto(_, scan.output))
-          nativeScanBuilder.addAllDataFilters(dataFilters.map(_.get).asJava)
+          // TODO remove flatMap and add error handling for unsupported data filters
+          val dataFilters = scan.dataFilters.flatMap(exprToProto(_, scan.output))
+          nativeScanBuilder.addAllDataFilters(dataFilters.asJava)
 
           // TODO: modify CometNativeScan to generate the file partitions without instantiating RDD.
           scan.inputRDD match {