Merge remote-tracking branch 'upstream/main' into broadcast_hash_join

apache · Mar 21, 2024 · ba4bada · ba4bada
2 parents 64240fa + eb6c704
commit ba4bada
Show file tree

Hide file tree

Showing 25 changed files with 20,255 additions and 1 deletion.
diff --git a/Makefile b/Makefile
@@ -33,7 +33,7 @@ test-jvm: core
 test: test-rust test-jvm
 clean:
 	cd core && cargo clean
-	./mvnw clean
+	./mvnw clean $(PROFILES)
 	rm -rf .dist
 bench:
 	cd core && RUSTFLAGS="-Ctarget-cpu=native" cargo bench $(filter-out $@,$(MAKECMDGOALS))

diff --git a/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala b/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala
@@ -335,6 +335,27 @@ class CometSparkSessionExtensions
               op
           }
 
+        case op: ShuffledHashJoinExec
+            if isCometOperatorEnabled(conf, "hash_join") &&
+              op.children.forall(isCometNative(_)) =>
+          val newOp = transform1(op)
+          newOp match {
+            case Some(nativeOp) =>
+              CometHashJoinExec(
+                nativeOp,
+                op,
+                op.leftKeys,
+                op.rightKeys,
+                op.joinType,
+                op.condition,
+                op.buildSide,
+                op.left,
+                op.right,
+                SerializedPlan(None))
+            case None =>
+              op
+          }
+
         case op: SortMergeJoinExec
             if isCometOperatorEnabled(conf, "sort_merge_join") &&
               op.children.forall(isCometNative(_)) =>

diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala
@@ -589,6 +589,43 @@ case class CometHashAggregateExec(
     Objects.hashCode(groupingExpressions, aggregateExpressions, input, mode, child)
 }
 
+case class CometHashJoinExec(
+    override val nativeOp: Operator,
+    override val originalPlan: SparkPlan,
+    leftKeys: Seq[Expression],
+    rightKeys: Seq[Expression],
+    joinType: JoinType,
+    condition: Option[Expression],
+    buildSide: BuildSide,
+    override val left: SparkPlan,
+    override val right: SparkPlan,
+    override val serializedPlanOpt: SerializedPlan)
+    extends CometBinaryExec {
+  override def withNewChildrenInternal(newLeft: SparkPlan, newRight: SparkPlan): SparkPlan =
+    this.copy(left = newLeft, right = newRight)
+
+  override def stringArgs: Iterator[Any] =
+    Iterator(leftKeys, rightKeys, joinType, condition, left, right)
+
+  override def equals(obj: Any): Boolean = {
+    obj match {
+      case other: CometHashJoinExec =>
+        this.leftKeys == other.leftKeys &&
+        this.rightKeys == other.rightKeys &&
+        this.condition == other.condition &&
+        this.buildSide == other.buildSide &&
+        this.left == other.left &&
+        this.right == other.right &&
+        this.serializedPlanOpt == other.serializedPlanOpt
+      case _ =>
+        false
+    }
+  }
+
+  override def hashCode(): Int =
+    Objects.hashCode(leftKeys, rightKeys, condition, left, right)
+}
+
 case class CometSortMergeJoinExec(
     override val nativeOp: Operator,
     override val originalPlan: SparkPlan,

diff --git a/spark/src/test/resources/tpch-query-results/q1.sql.out b/spark/src/test/resources/tpch-query-results/q1.sql.out
@@ -0,0 +1,9 @@
+-- Automatically generated by CometTPCHQuerySuite
+
+-- !query schema
+struct<l_returnflag:string,l_linestatus:string,sum_qty:decimal(22,2),sum_base_price:decimal(22,2),sum_disc_price:decimal(36,4),sum_charge:decimal(38,6),avg_qty:decimal(16,6),avg_price:decimal(16,6),avg_disc:decimal(16,6),count_order:bigint>
+-- !query output
+A	F	37734107.00	56586554400.73	53758257134.8700	55909065222.827692	25.522006	38273.129735	0.049985	1478493
+N	F	991417.00	1487504710.38	1413082168.0541	1469649223.194375	25.516472	38284.467761	0.050093	38854
+N	O	74476040.00	111701729697.74	106118230307.6056	110367043872.497010	25.502227	38249.117989	0.049997	2920374
+R	F	37719753.00	56568041380.90	53741292684.6040	55889619119.831932	25.505794	38250.854626	0.050009	1478870
diff --git a/spark/src/test/resources/tpch-query-results/q10.sql.out b/spark/src/test/resources/tpch-query-results/q10.sql.out
@@ -0,0 +1,25 @@
+-- Automatically generated by CometTPCHQuerySuite
+
+-- !query schema
+struct<c_custkey:bigint,c_name:string,revenue:decimal(36,4),c_acctbal:decimal(12,2),n_name:string,c_address:string,c_phone:string,c_comment:string>
+-- !query output
+57040	Customer#x	734235.2455	632.87	JAPAN	Eioyzjf4pp	22-895-641-3466	sits. slyly regular requests sleep alongside of the regular inst
+143347	Customer#x	721002.6948	2557.47	EGYPT	1aReFYv,Kw4	14-742-935-3718	ggle carefully enticing requests. final deposits use bold, bold pinto beans. ironic, idle re
+60838	Customer#x	679127.3077	2454.77	BRAZIL	64EaJ5vMAHWJlBOxJklpNc2RJiWE	12-913-494-9813	 need to boost against the slyly regular account
+101998	Customer#x	637029.5667	3790.89	UNITED KINGDOM	01c9CILnNtfOQYmZj	33-593-865-6378	ress foxes wake slyly after the bold excuses. ironic platelets are furiously carefully bold theodolites
+125341	Customer#x	633508.0860	4983.51	GERMANY	S29ODD6bceU8QSuuEJznkNaK	17-582-695-5962	arefully even depths. blithely even excuses sleep furiously. foxes use except the dependencies. ca
+25501	Customer#x	620269.7849	7725.04	ETHIOPIA	  W556MXuoiaYCCZamJI,Rn0B4ACUGdkQ8DZ	15-874-808-6793	he pending instructions wake carefully at the pinto beans. regular, final instructions along the slyly fina
+115831	Customer#x	596423.8672	5098.10	FRANCE	rFeBbEEyk dl ne7zV5fDrmiq1oK09wV7pxqCgIc	16-715-386-3788	l somas sleep. furiously final deposits wake blithely regular pinto b
+84223	Customer#x	594998.0239	528.65	UNITED KINGDOM	nAVZCs6BaWap rrM27N 2qBnzc5WBauxbA	33-442-824-8191	 slyly final deposits haggle regular, pending dependencies. pending escapades wake 
+54289	Customer#x	585603.3918	5583.02	IRAN	vXCxoCsU0Bad5JQI ,oobkZ	20-834-292-4707	ely special foxes are quickly finally ironic p
+39922	Customer#x	584878.1134	7321.11	GERMANY	Zgy4s50l2GKN4pLDPBU8m342gIw6R	17-147-757-8036	y final requests. furiously final foxes cajole blithely special platelets. f
+6226	Customer#x	576783.7606	2230.09	UNITED KINGDOM	8gPu8,NPGkfyQQ0hcIYUGPIBWc,ybP5g,	33-657-701-3391	ending platelets along the express deposits cajole carefully final 
+922	Customer#x	576767.5333	3869.25	GERMANY	Az9RFaut7NkPnc5zSD2PwHgVwr4jRzq	17-945-916-9648	luffily fluffy deposits. packages c
+147946	Customer#x	576455.1320	2030.13	ALGERIA	iANyZHjqhyy7Ajah0pTrYyhJ	10-886-956-3143	ithely ironic deposits haggle blithely ironic requests. quickly regu
+115640	Customer#x	569341.1933	6436.10	ARGENTINA	Vtgfia9qI 7EpHgecU1X	11-411-543-4901	ost slyly along the patterns; pinto be
+73606	Customer#x	568656.8578	1785.67	JAPAN	xuR0Tro5yChDfOCrjkd2ol	22-437-653-6966	he furiously regular ideas. slowly
+110246	Customer#x	566842.9815	7763.35	VIETNAM	7KzflgX MDOq7sOkI	31-943-426-9837	egular deposits serve blithely above the fl
+142549	Customer#x	563537.2368	5085.99	INDONESIA	ChqEoK43OysjdHbtKCp6dKqjNyvvi9	19-955-562-2398	sleep pending courts. ironic deposits against the carefully unusual platelets cajole carefully express accounts.
+146149	Customer#x	557254.9865	1791.55	ROMANIA	s87fvzFQpU	29-744-164-6487	 of the slyly silent accounts. quickly final accounts across the 
+52528	Customer#x	556397.3509	551.79	ARGENTINA	NFztyTOR10UOJ	11-208-192-3205	 deposits hinder. blithely pending asymptotes breach slyly regular re
+23431	Customer#x	554269.5360	3381.86	ROMANIA	HgiV0phqhaIa9aydNoIlb	29-915-458-2654	nusual, even instructions: furiously stealthy n