diff --git a/dev/diffs/4.0.0-preview1.diff b/dev/diffs/4.0.0-preview1.diff index 1a6810a7a..ba68d2a7b 100644 --- a/dev/diffs/4.0.0-preview1.diff +++ b/dev/diffs/4.0.0-preview1.diff @@ -2088,7 +2088,7 @@ index cd6f41b4ef4..4b6a17344bc 100644 ParquetOutputFormat.WRITER_VERSION -> ParquetProperties.WriterVersion.PARQUET_2_0.toString ) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala -index 795e9f46a8d..6285a1e388b 100644 +index 795e9f46a8d..5306c94a686 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala @@ -1100,7 +1100,11 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared @@ -2150,17 +2150,7 @@ index 795e9f46a8d..6285a1e388b 100644 // block 1: // null count min max // page-0 0 0 99 -@@ -2211,7 +2225,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared - } - } - -- test("SPARK-47120: subquery literal filter pushdown") { -+ test("SPARK-47120: subquery literal filter pushdown", -+ IgnoreComet("TODO: https://github.com/apache/datafusion-comet/issues/551")) { - withTable("t1", "t2") { - sql("create table t1(d date) using parquet") - sql("create table t2(d date) using parquet") -@@ -2301,7 +2316,11 @@ class ParquetV1FilterSuite extends ParquetFilterSuite { +@@ -2301,7 +2315,11 @@ class ParquetV1FilterSuite extends ParquetFilterSuite { assert(pushedParquetFilters.exists(_.getClass === filterClass), s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.") @@ -2173,7 +2163,7 @@ index 795e9f46a8d..6285a1e388b 100644 } else { assert(selectedFilters.isEmpty, "There is filter pushed down") } -@@ -2362,7 +2381,11 @@ class ParquetV2FilterSuite extends ParquetFilterSuite { +@@ -2362,7 +2380,11 @@ class ParquetV2FilterSuite extends ParquetFilterSuite { assert(pushedParquetFilters.exists(_.getClass === filterClass), s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.") diff --git a/docs/source/user-guide/source.md b/docs/source/user-guide/source.md index 71c9060cb..5f5d03d90 100644 --- a/docs/source/user-guide/source.md +++ b/docs/source/user-guide/source.md @@ -27,7 +27,7 @@ Official source releases can be downloaded from https://dist.apache.org/repos/di ```console # Pick the latest version -export COMET_VERSION=0.3.0 +export COMET_VERSION=0.4.0 # Download the tarball curl -O "https://dist.apache.org/repos/dist/release/datafusion/datafusion-comet-$COMET_VERSION/apache-datafusion-comet-$COMET_VERSION.tar.gz" # Unpack diff --git a/spark/src/main/scala/org/apache/comet/parquet/ParquetFilters.scala b/spark/src/main/scala/org/apache/comet/parquet/ParquetFilters.scala index 17844aba8..bcb23986f 100644 --- a/spark/src/main/scala/org/apache/comet/parquet/ParquetFilters.scala +++ b/spark/src/main/scala/org/apache/comet/parquet/ParquetFilters.scala @@ -723,20 +723,22 @@ class ParquetFilters( .lift(nameToParquetField(name).fieldType) .map(_(nameToParquetField(name).fieldNames, value)) - case sources.LessThan(name, value) if canMakeFilterOn(name, value) => + case sources.LessThan(name, value) if (value != null) && canMakeFilterOn(name, value) => makeLt .lift(nameToParquetField(name).fieldType) .map(_(nameToParquetField(name).fieldNames, value)) - case sources.LessThanOrEqual(name, value) if canMakeFilterOn(name, value) => + case sources.LessThanOrEqual(name, value) + if (value != null) && canMakeFilterOn(name, value) => makeLtEq .lift(nameToParquetField(name).fieldType) .map(_(nameToParquetField(name).fieldNames, value)) - case sources.GreaterThan(name, value) if canMakeFilterOn(name, value) => + case sources.GreaterThan(name, value) if (value != null) && canMakeFilterOn(name, value) => makeGt .lift(nameToParquetField(name).fieldType) .map(_(nameToParquetField(name).fieldNames, value)) - case sources.GreaterThanOrEqual(name, value) if canMakeFilterOn(name, value) => + case sources.GreaterThanOrEqual(name, value) + if (value != null) && canMakeFilterOn(name, value) => makeGtEq .lift(nameToParquetField(name).fieldType) .map(_(nameToParquetField(name).fieldNames, value))