Remove config spark.comet.xxhash64.enabled

apache · Jun 17, 2024 · fe471a6 · fe471a6
1 parent 741d00c
commit fe471a6
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 28 deletions.
diff --git a/common/src/main/scala/org/apache/comet/CometConf.scala b/common/src/main/scala/org/apache/comet/CometConf.scala
@@ -401,12 +401,6 @@ object CometConf extends ShimCometConf {
       .booleanConf
       .createWithDefault(false)
 
-  val COMET_XXHASH64_ENABLED: ConfigEntry[Boolean] =
-    conf("spark.comet.xxhash64.enabled")
-      .doc("The xxhash64 implementation is not optimized yet and may cause performance issues.")
-      .booleanConf
-      .createWithDefault(false)
-
 }
 
 object ConfigHelpers {

diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
@@ -2101,27 +2101,19 @@ object QueryPlanSerde extends Logging with ShimQueryPlanSerde with CometExprShim
           scalarExprToProtoWithReturnType("murmur3_hash", IntegerType, exprs :+ seedExpr: _*)
 
         case XxHash64(children, seed) =>
-          if (CometConf.COMET_XXHASH64_ENABLED.get()) {
-            val firstUnSupportedInput = children.find(c => !supportedDataType(c.dataType))
-            if (firstUnSupportedInput.isDefined) {
-              withInfo(expr, s"Unsupported datatype ${firstUnSupportedInput.get.dataType}")
-              return None
-            }
-            val exprs = children.map(exprToProtoInternal(_, inputs))
-            val seedBuilder = ExprOuterClass.Literal
-              .newBuilder()
-              .setDatatype(serializeDataType(LongType).get)
-              .setLongVal(seed)
-            val seedExpr = Some(ExprOuterClass.Expr.newBuilder().setLiteral(seedBuilder).build())
-            // the seed is put at the end of the arguments
-            scalarExprToProtoWithReturnType("xxhash64", LongType, exprs :+ seedExpr: _*)
-          } else {
-            withInfo(
-              expr,
-              "xxhash64 is disabled by default. " +
-                s"Set ${CometConf.COMET_XXHASH64_ENABLED.key}=true to enable it.")
-            None
+          val firstUnSupportedInput = children.find(c => !supportedDataType(c.dataType))
+          if (firstUnSupportedInput.isDefined) {
+            withInfo(expr, s"Unsupported datatype ${firstUnSupportedInput.get.dataType}")
+            return None
           }
+          val exprs = children.map(exprToProtoInternal(_, inputs))
+          val seedBuilder = ExprOuterClass.Literal
+            .newBuilder()
+            .setDatatype(serializeDataType(LongType).get)
+            .setLongVal(seed)
+          val seedExpr = Some(ExprOuterClass.Expr.newBuilder().setLiteral(seedBuilder).build())
+          // the seed is put at the end of the arguments
+          scalarExprToProtoWithReturnType("xxhash64", LongType, exprs :+ seedExpr: _*)
 
         case Sha2(left, numBits) =>
           if (!numBits.foldable) {

diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala
@@ -1587,7 +1587,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper {
     Seq(true, false).foreach { dictionary =>
       withSQLConf(
         "parquet.enable.dictionary" -> dictionary.toString,
-        CometConf.COMET_XXHASH64_ENABLED.key -> "true",
         CometConf.COMET_CAST_ALLOW_INCOMPATIBLE.key -> "true") {
         val table = "test"
         withTable(table) {
@@ -1620,7 +1619,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper {
     Seq(true, false).foreach { dictionary =>
       withSQLConf(
         "parquet.enable.dictionary" -> dictionary.toString,
-        CometConf.COMET_XXHASH64_ENABLED.key -> "true",
         CometConf.COMET_CAST_ALLOW_INCOMPATIBLE.key -> "true") {
         val table = "test"
         withTable(table) {