Skip to content

Commit

Permalink
add tests for Map and Array
Browse files Browse the repository at this point in the history
  • Loading branch information
andygrove committed Dec 12, 2024
1 parent 27d3f80 commit b0ad61c
Showing 1 changed file with 25 additions and 2 deletions.
27 changes: 25 additions & 2 deletions spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala
Original file line number Diff line number Diff line change
Expand Up @@ -2276,15 +2276,16 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper {
}
}

test("read map[int, int] from parquet") {
ignore("read map[int, int] from parquet") {
withTempPath { dir =>
// create input file with Comet disabled
withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
val df = spark
.range(5)
// Spark does not allow null as a key but does allow null as a
// value, and the entire map be null
.select(when(col("id") > 1, map(col("id"), when(col("id") > 2, col("id")))).alias("map1"))
.select(
when(col("id") > 1, map(col("id"), when(col("id") > 2, col("id")))).alias("map1"))
df.write.parquet(dir.toString())
}

Expand All @@ -2299,6 +2300,28 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper {
}
}

ignore("read array[int] from parquet") {
withTempPath { dir =>
// create input file with Comet disabled
withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
val df = spark
.range(5)
// Spark does not allow null as a key but does allow null as a
// value, and the entire map be null
.select(when(col("id") > 1, sequence(lit(0), col("id") * 2)).alias("array1"))
df.write.parquet(dir.toString())
}

Seq("", "parquet").foreach { v1List =>
withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> v1List) {
val df = spark.read.parquet(dir.toString())
checkSparkAnswerAndOperator(df.select("array1"))
checkSparkAnswerAndOperator(df.select(element_at(col("array1"), lit(1))))
}
}
}
}

test("CreateArray") {
Seq(true, false).foreach { dictionaryEnabled =>
withTempDir { dir =>
Expand Down

0 comments on commit b0ad61c

Please sign in to comment.