feat: Use specified branch of arrow-rs with workaround to invalid off…

…set buffers from Java Arrow
apache · Mar 31, 2024 · 9e6f9b2 · 9e6f9b2
1 parent aa6ddc5
commit 9e6f9b2
Show file tree

Hide file tree

Showing 14 changed files with 661 additions and 528 deletions.
diff --git a/common/src/main/scala/org/apache/comet/CometConf.scala b/common/src/main/scala/org/apache/comet/CometConf.scala
@@ -130,14 +130,14 @@ object CometConf {
       .booleanConf
       .createWithDefault(false)
 
-  val COMET_COLUMNAR_SHUFFLE_ENABLED: ConfigEntry[Boolean] = conf(
-    "spark.comet.columnar.shuffle.enabled")
-    .doc(
-      "Force Comet to only use columnar shuffle for CometScan and Spark regular operators. " +
-        "If this is enabled, Comet native shuffle will not be enabled but only Arrow shuffle. " +
-        "By default, this config is false.")
-    .booleanConf
-    .createWithDefault(false)
+  val COMET_COLUMNAR_SHUFFLE_ENABLED: ConfigEntry[Boolean] =
+    conf("spark.comet.columnar.shuffle.enabled")
+      .doc(
+        "Whether to enable Arrow-based columnar shuffle for Comet and Spark regular operators. " +
+          "If this is enabled, Comet prefers columnar shuffle than native shuffle. " +
+          "By default, this config is true.")
+      .booleanConf
+      .createWithDefault(true)
 
   val COMET_EXEC_BROADCAST_ENABLED: ConfigEntry[Boolean] =
     conf(s"$COMET_EXEC_CONFIG_PREFIX.broadcast.enabled")