From 3e3daea189d159a74aad75be90c72b17bf0a8142 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Thu, 11 Apr 2024 13:01:47 -0700 Subject: [PATCH] Add Comet memoryoverhead for Spark SQL tests --- dev/diffs/3.4.2.diff | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dev/diffs/3.4.2.diff b/dev/diffs/3.4.2.diff index e8a8764a5..1fe914a12 100644 --- a/dev/diffs/3.4.2.diff +++ b/dev/diffs/3.4.2.diff @@ -1404,10 +1404,10 @@ index dd55fcfe42c..cc18147d17a 100644 spark.internalCreateDataFrame(withoutFilters.execute(), schema) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala -index ed2e309fa07..4cfe0093da7 100644 +index ed2e309fa07..f51a2395a53 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala -@@ -74,6 +74,21 @@ trait SharedSparkSessionBase +@@ -74,6 +74,22 @@ trait SharedSparkSessionBase // this rule may potentially block testing of other optimization rules such as // ConstantPropagation etc. .set(SQLConf.OPTIMIZER_EXCLUDED_RULES.key, ConvertToLocalRelation.ruleName) @@ -1424,6 +1424,7 @@ index ed2e309fa07..4cfe0093da7 100644 + .set("spark.shuffle.manager", + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .set("spark.comet.exec.shuffle.enabled", "true") ++ .set("spark.comet.memoryOverhead", "10g") + } + } conf.set(