From 0602e24c349c05e0b6980762ca769155af480f2a Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 5 Dec 2024 10:40:29 -0700 Subject: [PATCH] Save --- native/core/src/execution/datafusion/schema_adapter.rs | 4 ++++ .../main/scala/org/apache/comet/serde/QueryPlanSerde.scala | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/native/core/src/execution/datafusion/schema_adapter.rs b/native/core/src/execution/datafusion/schema_adapter.rs index c5d714564..ff325f647 100644 --- a/native/core/src/execution/datafusion/schema_adapter.rs +++ b/native/core/src/execution/datafusion/schema_adapter.rs @@ -127,6 +127,10 @@ pub fn spark_can_cast_types(from_type: &DataType, to_type: &DataType) -> bool { true } (_, DataType::Timestamp(TimeUnit::Nanosecond, _)) => false, + // Native cast invoked for unsupported cast from FixedSizeBinary(3) to Binary. + (DataType::FixedSizeBinary(_), _) => false, + // Native cast invoked for unsupported cast from UInt32 to Int64. + (DataType::UInt8 | DataType::UInt16 | DataType::UInt32 | DataType::UInt64, _) => false, _ => can_cast_types(from_type, to_type), } } diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 29e50d734..b30ad1396 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -2496,8 +2496,9 @@ object QueryPlanSerde extends Logging with ShimQueryPlanSerde with CometExprShim // Sink operators don't have children result.clearChildren() - val dataFilters = scan.dataFilters.map(exprToProto(_, scan.output)) - nativeScanBuilder.addAllDataFilters(dataFilters.map(_.get).asJava) + // TODO remove flatMap and add error handling for unsupported data filters + val dataFilters = scan.dataFilters.flatMap(exprToProto(_, scan.output)) + nativeScanBuilder.addAllDataFilters(dataFilters.asJava) // TODO: modify CometNativeScan to generate the file partitions without instantiating RDD. scan.inputRDD match {