diff --git a/common/src/main/scala/org/apache/comet/CometConf.scala b/common/src/main/scala/org/apache/comet/CometConf.scala index 5aee02f11..42fb5fb4c 100644 --- a/common/src/main/scala/org/apache/comet/CometConf.scala +++ b/common/src/main/scala/org/apache/comet/CometConf.scala @@ -29,6 +29,8 @@ import org.apache.spark.network.util.JavaUtils import org.apache.spark.sql.comet.util.Utils import org.apache.spark.sql.internal.SQLConf +import org.apache.comet.shims.ShimCometConf + /** * Configurations for a Comet application. Mostly inspired by [[SQLConf]] in Spark. * @@ -41,7 +43,7 @@ import org.apache.spark.sql.internal.SQLConf * which retrieves the config value from the thread-local [[SQLConf]] object. Alternatively, you * can also explicitly pass a [[SQLConf]] object to the `get` method. */ -object CometConf { +object CometConf extends ShimCometConf { /** List of all configs that is used for generating documentation */ val allConfs = new ListBuffer[ConfigEntry[_]] @@ -361,7 +363,7 @@ object CometConf { "column to a long column, a float column to a double column, etc. This is automatically" + "enabled when reading from Iceberg tables.") .booleanConf - .createWithDefault(false) + .createWithDefault(COMET_SCHEMA_EVOLUTION_ENABLED_DEFAULT) val COMET_ROW_TO_COLUMNAR_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.rowToColumnar.enabled") @@ -382,12 +384,13 @@ object CometConf { .createWithDefault(Seq("Range,InMemoryTableScan")) val COMET_ANSI_MODE_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.ansi.enabled") + .internal() .doc( "Comet does not respect ANSI mode in most cases and by default will not accelerate " + "queries when ansi mode is enabled. Enable this setting to test Comet's experimental " + "support for ANSI mode. This should not be used in production.") .booleanConf - .createWithDefault(false) + .createWithDefault(COMET_ANSI_MODE_ENABLED_DEFAULT) val COMET_CAST_ALLOW_INCOMPATIBLE: ConfigEntry[Boolean] = conf("spark.comet.cast.allowIncompatible") diff --git a/common/src/main/spark-3.x/org/apache/comet/shims/ShimCometConf.scala b/common/src/main/spark-3.x/org/apache/comet/shims/ShimCometConf.scala new file mode 100644 index 000000000..dc84a7525 --- /dev/null +++ b/common/src/main/spark-3.x/org/apache/comet/shims/ShimCometConf.scala @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.shims + +trait ShimCometConf { + protected val COMET_SCHEMA_EVOLUTION_ENABLED_DEFAULT = false + protected val COMET_ANSI_MODE_ENABLED_DEFAULT = false +} diff --git a/common/src/main/spark-4.0/org/apache/comet/shims/ShimCometConf.scala b/common/src/main/spark-4.0/org/apache/comet/shims/ShimCometConf.scala new file mode 100644 index 000000000..13da6bc10 --- /dev/null +++ b/common/src/main/spark-4.0/org/apache/comet/shims/ShimCometConf.scala @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.shims + +trait ShimCometConf { + protected val COMET_SCHEMA_EVOLUTION_ENABLED_DEFAULT = true + protected val COMET_ANSI_MODE_ENABLED_DEFAULT = true +} diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index eb349b349..104f29ce8 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -23,7 +23,6 @@ Comet provides the following configuration settings. | Config | Description | Default Value | |--------|-------------|---------------| -| spark.comet.ansi.enabled | Comet does not respect ANSI mode in most cases and by default will not accelerate queries when ansi mode is enabled. Enable this setting to test Comet's experimental support for ANSI mode. This should not be used in production. | false | | spark.comet.batchSize | The columnar batch size, i.e., the maximum number of rows that a batch can contain. | 8192 | | spark.comet.cast.allowIncompatible | Comet is not currently fully compatible with Spark for all cast operations. Set this config to true to allow them anyway. See compatibility guide for more information. | false | | spark.comet.columnar.shuffle.async.enabled | Whether to enable asynchronous shuffle for Arrow-based shuffle. By default, this config is false. | false | diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/explain.txt index d4d5ff424..762f3a4f6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/explain.txt @@ -12,8 +12,8 @@ TakeOrderedAndProject (40) : : : +- * HashAggregate (7) : : : +- * Project (6) : : : +- * BroadcastHashJoin Inner BuildRight (5) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_returns (1) : : : +- ReusedExchange (4) : : +- BroadcastExchange (24) @@ -26,18 +26,18 @@ TakeOrderedAndProject (40) : : +- * HashAggregate (17) : : +- * Project (16) : : +- * BroadcastHashJoin Inner BuildRight (15) - : : :- * Filter (13) - : : : +- * ColumnarToRow (12) + : : :- * ColumnarToRow (13) + : : : +- CometFilter (12) : : : +- CometScan parquet spark_catalog.default.store_returns (11) : : +- ReusedExchange (14) : +- BroadcastExchange (31) - : +- * Project (30) - : +- * Filter (29) - : +- * ColumnarToRow (28) + : +- * ColumnarToRow (30) + : +- CometProject (29) + : +- CometFilter (28) : +- CometScan parquet spark_catalog.default.store (27) +- BroadcastExchange (37) - +- * Filter (36) - +- * ColumnarToRow (35) + +- * ColumnarToRow (36) + +- CometFilter (35) +- CometScan parquet spark_catalog.default.customer (34) @@ -49,12 +49,12 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#4), dynamicpruningexpression(sr PushedFilters: [IsNotNull(sr_store_sk), IsNotNull(sr_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 2] +(2) CometFilter Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] +Condition : (isnotnull(sr_store_sk#2) AND isnotnull(sr_customer_sk#1)) -(3) Filter [codegen id : 2] +(3) ColumnarToRow [codegen id : 2] Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] -Condition : (isnotnull(sr_store_sk#2) AND isnotnull(sr_customer_sk#1)) (4) ReusedExchange [Reuses operator id: 45] Output [1]: [d_date_sk#6] @@ -99,12 +99,12 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#16), dynamicpruningexpression(s PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 4] +(12) CometFilter Input [4]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15, sr_returned_date_sk#16] +Condition : isnotnull(sr_store_sk#14) -(13) Filter [codegen id : 4] +(13) ColumnarToRow [codegen id : 4] Input [4]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15, sr_returned_date_sk#16] -Condition : isnotnull(sr_store_sk#14) (14) ReusedExchange [Reuses operator id: 45] Output [1]: [d_date_sk#18] @@ -180,16 +180,16 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 7] -Input [2]: [s_store_sk#29, s_state#30] - -(29) Filter [codegen id : 7] +(28) CometFilter Input [2]: [s_store_sk#29, s_state#30] Condition : ((isnotnull(s_state#30) AND (s_state#30 = TN)) AND isnotnull(s_store_sk#29)) -(30) Project [codegen id : 7] -Output [1]: [s_store_sk#29] +(29) CometProject Input [2]: [s_store_sk#29, s_state#30] +Arguments: [s_store_sk#29], [s_store_sk#29] + +(30) ColumnarToRow [codegen id : 7] +Input [1]: [s_store_sk#29] (31) BroadcastExchange Input [1]: [s_store_sk#29] @@ -212,12 +212,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 8] +(35) CometFilter Input [2]: [c_customer_sk#31, c_customer_id#32] +Condition : isnotnull(c_customer_sk#31) -(36) Filter [codegen id : 8] +(36) ColumnarToRow [codegen id : 8] Input [2]: [c_customer_sk#31, c_customer_id#32] -Condition : isnotnull(c_customer_sk#31) (37) BroadcastExchange Input [2]: [c_customer_sk#31, c_customer_id#32] @@ -241,9 +241,9 @@ Arguments: 100, [c_customer_id#32 ASC NULLS FIRST], [c_customer_id#32] Subquery:1 Hosting operator id = 1 Hosting Expression = sr_returned_date_sk#4 IN dynamicpruning#5 BroadcastExchange (45) -+- * Project (44) - +- * Filter (43) - +- * ColumnarToRow (42) ++- * ColumnarToRow (44) + +- CometProject (43) + +- CometFilter (42) +- CometScan parquet spark_catalog.default.date_dim (41) @@ -254,16 +254,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(42) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#6, d_year#33] - -(43) Filter [codegen id : 1] +(42) CometFilter Input [2]: [d_date_sk#6, d_year#33] Condition : ((isnotnull(d_year#33) AND (d_year#33 = 2000)) AND isnotnull(d_date_sk#6)) -(44) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(43) CometProject Input [2]: [d_date_sk#6, d_year#33] +Arguments: [d_date_sk#6], [d_date_sk#6] + +(44) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#6] (45) BroadcastExchange Input [1]: [d_date_sk#6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/simplified.txt index 1475a3732..688fb69a8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/simplified.txt @@ -14,17 +14,17 @@ TakeOrderedAndProject [c_customer_id] HashAggregate [sr_customer_sk,sr_store_sk,sr_return_amt] [sum,sum] Project [sr_customer_sk,sr_store_sk,sr_return_amt] BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Filter [sr_store_sk,sr_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [sr_store_sk,sr_customer_sk] CometScan parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [d_date_sk] #2 @@ -44,9 +44,9 @@ TakeOrderedAndProject [c_customer_id] HashAggregate [sr_customer_sk,sr_store_sk,sr_return_amt] [sum,sum] Project [sr_customer_sk,sr_store_sk,sr_return_amt] BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Filter [sr_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [sr_store_sk] CometScan parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -54,15 +54,15 @@ TakeOrderedAndProject [c_customer_id] InputAdapter BroadcastExchange #6 WholeStageCodegen (7) - Project [s_store_sk] - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [s_store_sk] + CometFilter [s_state,s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_state] InputAdapter BroadcastExchange #7 WholeStageCodegen (8) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/explain.txt index 5c21b8b9b..15490b87d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/explain.txt @@ -12,8 +12,8 @@ TakeOrderedAndProject (43) : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (24) : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (17) : : : :- * BroadcastHashJoin LeftSemi BuildRight (10) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.customer (1) : : : : +- BroadcastExchange (9) : : : : +- * Project (8) @@ -34,13 +34,13 @@ TakeOrderedAndProject (43) : : : +- CometScan parquet spark_catalog.default.catalog_sales (18) : : +- ReusedExchange (20) : +- BroadcastExchange (31) - : +- * Project (30) - : +- * Filter (29) - : +- * ColumnarToRow (28) + : +- * ColumnarToRow (30) + : +- CometProject (29) + : +- CometFilter (28) : +- CometScan parquet spark_catalog.default.customer_address (27) +- BroadcastExchange (37) - +- * Filter (36) - +- * ColumnarToRow (35) + +- * ColumnarToRow (36) + +- CometFilter (35) +- CometScan parquet spark_catalog.default.customer_demographics (34) @@ -51,12 +51,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 9] +(2) CometFilter Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) -(3) Filter [codegen id : 9] +(3) ColumnarToRow [codegen id : 9] Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) (4) Scan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] @@ -172,16 +172,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [In(ca_county, [Dona Ana County,Jefferson County,La Porte County,Rush County,Toole County]), IsNotNull(ca_address_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 7] -Input [2]: [ca_address_sk#18, ca_county#19] - -(29) Filter [codegen id : 7] +(28) CometFilter Input [2]: [ca_address_sk#18, ca_county#19] Condition : (ca_county#19 IN (Rush County,Toole County,Jefferson County,Dona Ana County,La Porte County) AND isnotnull(ca_address_sk#18)) -(30) Project [codegen id : 7] -Output [1]: [ca_address_sk#18] +(29) CometProject Input [2]: [ca_address_sk#18, ca_county#19] +Arguments: [ca_address_sk#18], [ca_address_sk#18] + +(30) ColumnarToRow [codegen id : 7] +Input [1]: [ca_address_sk#18] (31) BroadcastExchange Input [1]: [ca_address_sk#18] @@ -204,12 +204,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 8] +(35) CometFilter Input [9]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Condition : isnotnull(cd_demo_sk#20) -(36) Filter [codegen id : 8] +(36) ColumnarToRow [codegen id : 8] Input [9]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Condition : isnotnull(cd_demo_sk#20) (37) BroadcastExchange Input [9]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] @@ -251,9 +251,9 @@ Arguments: 100, [cd_gender#21 ASC NULLS FIRST, cd_marital_status#22 ASC NULLS FI Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 BroadcastExchange (48) -+- * Project (47) - +- * Filter (46) - +- * ColumnarToRow (45) ++- * ColumnarToRow (47) + +- CometProject (46) + +- CometFilter (45) +- CometScan parquet spark_catalog.default.date_dim (44) @@ -264,16 +264,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,1), LessThanOrEqual(d_moy,4), IsNotNull(d_date_sk)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#9, d_year#38, d_moy#39] - -(46) Filter [codegen id : 1] +(45) CometFilter Input [3]: [d_date_sk#9, d_year#38, d_moy#39] Condition : (((((isnotnull(d_year#38) AND isnotnull(d_moy#39)) AND (d_year#38 = 2002)) AND (d_moy#39 >= 1)) AND (d_moy#39 <= 4)) AND isnotnull(d_date_sk#9)) -(47) Project [codegen id : 1] -Output [1]: [d_date_sk#9] +(46) CometProject Input [3]: [d_date_sk#9, d_year#38, d_moy#39] +Arguments: [d_date_sk#9], [d_date_sk#9] + +(47) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#9] (48) BroadcastExchange Input [1]: [d_date_sk#9] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/simplified.txt index e31b38b47..89893c831 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/simplified.txt @@ -14,9 +14,9 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_current_addr_sk,c_current_cdemo_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] InputAdapter BroadcastExchange #2 @@ -29,10 +29,10 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_moy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter ReusedExchange [d_date_sk] #3 @@ -61,15 +61,15 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter BroadcastExchange #6 WholeStageCodegen (7) - Project [ca_address_sk] - Filter [ca_county,ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_county,ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] InputAdapter BroadcastExchange #7 WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/explain.txt index f821d2a67..c663d4688 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/explain.txt @@ -14,12 +14,12 @@ TakeOrderedAndProject (72) : : : +- * BroadcastHashJoin Inner BuildRight (11) : : : :- * Project (9) : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.customer (1) : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) + : : : : +- * ColumnarToRow (6) + : : : : +- CometFilter (5) : : : : +- CometScan parquet spark_catalog.default.store_sales (4) : : : +- ReusedExchange (10) : : +- BroadcastExchange (32) @@ -30,12 +30,12 @@ TakeOrderedAndProject (72) : : +- * BroadcastHashJoin Inner BuildRight (27) : : :- * Project (25) : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : :- * Filter (19) - : : : : +- * ColumnarToRow (18) + : : : :- * ColumnarToRow (19) + : : : : +- CometFilter (18) : : : : +- CometScan parquet spark_catalog.default.customer (17) : : : +- BroadcastExchange (23) - : : : +- * Filter (22) - : : : +- * ColumnarToRow (21) + : : : +- * ColumnarToRow (22) + : : : +- CometFilter (21) : : : +- CometScan parquet spark_catalog.default.store_sales (20) : : +- ReusedExchange (26) : +- BroadcastExchange (51) @@ -47,12 +47,12 @@ TakeOrderedAndProject (72) : +- * BroadcastHashJoin Inner BuildRight (45) : :- * Project (43) : : +- * BroadcastHashJoin Inner BuildRight (42) - : : :- * Filter (37) - : : : +- * ColumnarToRow (36) + : : :- * ColumnarToRow (37) + : : : +- CometFilter (36) : : : +- CometScan parquet spark_catalog.default.customer (35) : : +- BroadcastExchange (41) - : : +- * Filter (40) - : : +- * ColumnarToRow (39) + : : +- * ColumnarToRow (40) + : : +- CometFilter (39) : : +- CometScan parquet spark_catalog.default.web_sales (38) : +- ReusedExchange (44) +- BroadcastExchange (69) @@ -63,12 +63,12 @@ TakeOrderedAndProject (72) +- * BroadcastHashJoin Inner BuildRight (64) :- * Project (62) : +- * BroadcastHashJoin Inner BuildRight (61) - : :- * Filter (56) - : : +- * ColumnarToRow (55) + : :- * ColumnarToRow (56) + : : +- CometFilter (55) : : +- CometScan parquet spark_catalog.default.customer (54) : +- BroadcastExchange (60) - : +- * Filter (59) - : +- * ColumnarToRow (58) + : +- * ColumnarToRow (59) + : +- CometFilter (58) : +- CometScan parquet spark_catalog.default.web_sales (57) +- ReusedExchange (63) @@ -80,12 +80,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) -(3) Filter [codegen id : 3] +(3) ColumnarToRow [codegen id : 3] Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) (4) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] @@ -95,12 +95,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] +Condition : isnotnull(ss_customer_sk#9) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_customer_sk#9) (7) BroadcastExchange Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] @@ -158,12 +158,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 6] +(18) CometFilter Input [8]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] +Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_customer_id#22)) -(19) Filter [codegen id : 6] +(19) ColumnarToRow [codegen id : 6] Input [8]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_customer_id#22)) (20) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] @@ -173,12 +173,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#32), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 4] +(21) CometFilter Input [4]: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] +Condition : isnotnull(ss_customer_sk#29) -(22) Filter [codegen id : 4] +(22) ColumnarToRow [codegen id : 4] Input [4]: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] -Condition : isnotnull(ss_customer_sk#29) (23) BroadcastExchange Input [4]: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] @@ -246,12 +246,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(36) ColumnarToRow [codegen id : 10] +(36) CometFilter Input [8]: [c_customer_sk#41, c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48] +Condition : (isnotnull(c_customer_sk#41) AND isnotnull(c_customer_id#42)) -(37) Filter [codegen id : 10] +(37) ColumnarToRow [codegen id : 10] Input [8]: [c_customer_sk#41, c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48] -Condition : (isnotnull(c_customer_sk#41) AND isnotnull(c_customer_id#42)) (38) Scan parquet spark_catalog.default.web_sales Output [4]: [ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] @@ -261,12 +261,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#52), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(39) ColumnarToRow [codegen id : 8] +(39) CometFilter Input [4]: [ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] +Condition : isnotnull(ws_bill_customer_sk#49) -(40) Filter [codegen id : 8] +(40) ColumnarToRow [codegen id : 8] Input [4]: [ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] -Condition : isnotnull(ws_bill_customer_sk#49) (41) BroadcastExchange Input [4]: [ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] @@ -338,12 +338,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(55) ColumnarToRow [codegen id : 14] +(55) CometFilter Input [8]: [c_customer_sk#61, c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68] +Condition : (isnotnull(c_customer_sk#61) AND isnotnull(c_customer_id#62)) -(56) Filter [codegen id : 14] +(56) ColumnarToRow [codegen id : 14] Input [8]: [c_customer_sk#61, c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68] -Condition : (isnotnull(c_customer_sk#61) AND isnotnull(c_customer_id#62)) (57) Scan parquet spark_catalog.default.web_sales Output [4]: [ws_bill_customer_sk#69, ws_ext_discount_amt#70, ws_ext_list_price#71, ws_sold_date_sk#72] @@ -353,12 +353,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#72), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(58) ColumnarToRow [codegen id : 12] +(58) CometFilter Input [4]: [ws_bill_customer_sk#69, ws_ext_discount_amt#70, ws_ext_list_price#71, ws_sold_date_sk#72] +Condition : isnotnull(ws_bill_customer_sk#69) -(59) Filter [codegen id : 12] +(59) ColumnarToRow [codegen id : 12] Input [4]: [ws_bill_customer_sk#69, ws_ext_discount_amt#70, ws_ext_list_price#71, ws_sold_date_sk#72] -Condition : isnotnull(ws_bill_customer_sk#69) (60) BroadcastExchange Input [4]: [ws_bill_customer_sk#69, ws_ext_discount_amt#70, ws_ext_list_price#71, ws_sold_date_sk#72] @@ -427,8 +427,8 @@ Arguments: 100, [customer_preferred_cust_flag#39 ASC NULLS FIRST], [customer_pre Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13 BroadcastExchange (76) -+- * Filter (75) - +- * ColumnarToRow (74) ++- * ColumnarToRow (75) + +- CometFilter (74) +- CometScan parquet spark_catalog.default.date_dim (73) @@ -439,12 +439,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(74) ColumnarToRow [codegen id : 1] +(74) CometFilter Input [2]: [d_date_sk#14, d_year#15] +Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2001)) AND isnotnull(d_date_sk#14)) -(75) Filter [codegen id : 1] +(75) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2001)) AND isnotnull(d_date_sk#14)) (76) BroadcastExchange Input [2]: [d_date_sk#14, d_year#15] @@ -452,8 +452,8 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint) Subquery:2 Hosting operator id = 20 Hosting Expression = ss_sold_date_sk#32 IN dynamicpruning#33 BroadcastExchange (80) -+- * Filter (79) - +- * ColumnarToRow (78) ++- * ColumnarToRow (79) + +- CometFilter (78) +- CometScan parquet spark_catalog.default.date_dim (77) @@ -464,12 +464,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(78) ColumnarToRow [codegen id : 1] +(78) CometFilter Input [2]: [d_date_sk#34, d_year#35] +Condition : ((isnotnull(d_year#35) AND (d_year#35 = 2002)) AND isnotnull(d_date_sk#34)) -(79) Filter [codegen id : 1] +(79) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#34, d_year#35] -Condition : ((isnotnull(d_year#35) AND (d_year#35 = 2002)) AND isnotnull(d_date_sk#34)) (80) BroadcastExchange Input [2]: [d_date_sk#34, d_year#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/simplified.txt index 55c995970..562b5fdf2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/simplified.txt @@ -16,23 +16,23 @@ TakeOrderedAndProject [customer_preferred_cust_flag] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_customer_id] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] InputAdapter BroadcastExchange #2 WholeStageCodegen (1) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [d_date_sk,d_year] #3 @@ -48,23 +48,23 @@ TakeOrderedAndProject [customer_preferred_cust_flag] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_customer_id] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] InputAdapter BroadcastExchange #6 WholeStageCodegen (4) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #7 WholeStageCodegen (1) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [d_date_sk,d_year] #7 @@ -81,16 +81,16 @@ TakeOrderedAndProject [customer_preferred_cust_flag] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_customer_id] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] InputAdapter BroadcastExchange #10 WholeStageCodegen (8) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_bill_customer_sk] CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -107,16 +107,16 @@ TakeOrderedAndProject [customer_preferred_cust_flag] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_customer_id] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] InputAdapter BroadcastExchange #13 WholeStageCodegen (12) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_bill_customer_sk] CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/explain.txt index 49faad274..6cf7f4b08 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/explain.txt @@ -11,12 +11,12 @@ TakeOrderedAndProject (20) +- * BroadcastHashJoin Inner BuildRight (11) :- * Project (9) : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) + : :- * ColumnarToRow (3) + : : +- CometFilter (2) : : +- CometScan parquet spark_catalog.default.web_sales (1) : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) + : +- * ColumnarToRow (6) + : +- CometFilter (5) : +- CometScan parquet spark_catalog.default.item (4) +- ReusedExchange (10) @@ -29,12 +29,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sol PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] +Condition : isnotnull(ws_item_sk#1) -(3) Filter [codegen id : 3] +(3) ColumnarToRow [codegen id : 3] Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Condition : isnotnull(ws_item_sk#1) (4) Scan parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] @@ -43,12 +43,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Condition : (i_category#10 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (i_category#10 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) (7) BroadcastExchange Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] @@ -119,9 +119,9 @@ Arguments: 100, [i_category#10 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_ite Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 BroadcastExchange (25) -+- * Project (24) - +- * Filter (23) - +- * ColumnarToRow (22) ++- * ColumnarToRow (24) + +- CometProject (23) + +- CometFilter (22) +- CometScan parquet spark_catalog.default.date_dim (21) @@ -132,16 +132,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(22) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#11, d_date#19] - -(23) Filter [codegen id : 1] +(22) CometFilter Input [2]: [d_date_sk#11, d_date#19] Condition : (((isnotnull(d_date#19) AND (d_date#19 >= 1999-02-22)) AND (d_date#19 <= 1999-03-24)) AND isnotnull(d_date_sk#11)) -(24) Project [codegen id : 1] -Output [1]: [d_date_sk#11] +(23) CometProject Input [2]: [d_date_sk#11, d_date#19] +Arguments: [d_date_sk#11], [d_date_sk#11] + +(24) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#11] (25) BroadcastExchange Input [1]: [d_date_sk#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/simplified.txt index 59f27988d..fae1c6dba 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/simplified.txt @@ -17,24 +17,24 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter BroadcastExchange #4 WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_category,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] InputAdapter ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/explain.txt index 238d32e1b..a647b1f05 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/explain.txt @@ -12,26 +12,26 @@ : : : +- * BroadcastHashJoin Inner BuildRight (15) : : : :- * Project (9) : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) + : : : : +- * ColumnarToRow (6) + : : : : +- CometFilter (5) : : : : +- CometScan parquet spark_catalog.default.store (4) : : : +- BroadcastExchange (14) - : : : +- * Project (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) + : : : +- * ColumnarToRow (13) + : : : +- CometProject (12) + : : : +- CometFilter (11) : : : +- CometScan parquet spark_catalog.default.customer_address (10) : : +- ReusedExchange (17) : +- BroadcastExchange (23) - : +- * Filter (22) - : +- * ColumnarToRow (21) + : +- * ColumnarToRow (22) + : +- CometFilter (21) : +- CometScan parquet spark_catalog.default.customer_demographics (20) +- BroadcastExchange (29) - +- * Filter (28) - +- * ColumnarToRow (27) + +- * ColumnarToRow (28) + +- CometFilter (27) +- CometScan parquet spark_catalog.default.household_demographics (26) @@ -43,12 +43,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#10), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_hdemo_sk), Or(Or(And(GreaterThanOrEqual(ss_net_profit,100.00),LessThanOrEqual(ss_net_profit,200.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,300.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,250.00))), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00)))] ReadSchema: struct -(2) ColumnarToRow [codegen id : 6] +(2) CometFilter Input [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] +Condition : (((((isnotnull(ss_store_sk#4) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_hdemo_sk#2)) AND ((((ss_net_profit#9 >= 100.00) AND (ss_net_profit#9 <= 200.00)) OR ((ss_net_profit#9 >= 150.00) AND (ss_net_profit#9 <= 300.00))) OR ((ss_net_profit#9 >= 50.00) AND (ss_net_profit#9 <= 250.00)))) AND ((((ss_sales_price#6 >= 100.00) AND (ss_sales_price#6 <= 150.00)) OR ((ss_sales_price#6 >= 50.00) AND (ss_sales_price#6 <= 100.00))) OR ((ss_sales_price#6 >= 150.00) AND (ss_sales_price#6 <= 200.00)))) -(3) Filter [codegen id : 6] +(3) ColumnarToRow [codegen id : 6] Input [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -Condition : (((((isnotnull(ss_store_sk#4) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_hdemo_sk#2)) AND ((((ss_net_profit#9 >= 100.00) AND (ss_net_profit#9 <= 200.00)) OR ((ss_net_profit#9 >= 150.00) AND (ss_net_profit#9 <= 300.00))) OR ((ss_net_profit#9 >= 50.00) AND (ss_net_profit#9 <= 250.00)))) AND ((((ss_sales_price#6 >= 100.00) AND (ss_sales_price#6 <= 150.00)) OR ((ss_sales_price#6 >= 50.00) AND (ss_sales_price#6 <= 100.00))) OR ((ss_sales_price#6 >= 150.00) AND (ss_sales_price#6 <= 200.00)))) (4) Scan parquet spark_catalog.default.store Output [1]: [s_store_sk#12] @@ -57,12 +57,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [1]: [s_store_sk#12] +Condition : isnotnull(s_store_sk#12) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [1]: [s_store_sk#12] -Condition : isnotnull(s_store_sk#12) (7) BroadcastExchange Input [1]: [s_store_sk#12] @@ -85,16 +85,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [OH,TX]),In(ca_state, [KY,NM,OR])),In(ca_state, [MS,TX,VA]))] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [ca_address_sk#13, ca_state#14, ca_country#15] - -(12) Filter [codegen id : 2] +(11) CometFilter Input [3]: [ca_address_sk#13, ca_state#14, ca_country#15] Condition : (((isnotnull(ca_country#15) AND (ca_country#15 = United States)) AND isnotnull(ca_address_sk#13)) AND ((ca_state#14 IN (TX,OH) OR ca_state#14 IN (OR,NM,KY)) OR ca_state#14 IN (VA,TX,MS))) -(13) Project [codegen id : 2] -Output [2]: [ca_address_sk#13, ca_state#14] +(12) CometProject Input [3]: [ca_address_sk#13, ca_state#14, ca_country#15] +Arguments: [ca_address_sk#13, ca_state#14], [ca_address_sk#13, ca_state#14] + +(13) ColumnarToRow [codegen id : 2] +Input [2]: [ca_address_sk#13, ca_state#14] (14) BroadcastExchange Input [2]: [ca_address_sk#13, ca_state#14] @@ -130,12 +130,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Advanced Degree )),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College ))),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,2 yr Degree )))] ReadSchema: struct -(21) ColumnarToRow [codegen id : 4] +(21) CometFilter Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] +Condition : (isnotnull(cd_demo_sk#17) AND ((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) OR ((cd_marital_status#18 = S) AND (cd_education_status#19 = College ))) OR ((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )))) -(22) Filter [codegen id : 4] +(22) ColumnarToRow [codegen id : 4] Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Condition : (isnotnull(cd_demo_sk#17) AND ((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) OR ((cd_marital_status#18 = S) AND (cd_education_status#19 = College ))) OR ((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )))) (23) BroadcastExchange Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] @@ -158,12 +158,12 @@ Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), Or(EqualTo(hd_dep_count,3),EqualTo(hd_dep_count,1))] ReadSchema: struct -(27) ColumnarToRow [codegen id : 5] +(27) CometFilter Input [2]: [hd_demo_sk#20, hd_dep_count#21] +Condition : (isnotnull(hd_demo_sk#20) AND ((hd_dep_count#21 = 3) OR (hd_dep_count#21 = 1))) -(28) Filter [codegen id : 5] +(28) ColumnarToRow [codegen id : 5] Input [2]: [hd_demo_sk#20, hd_dep_count#21] -Condition : (isnotnull(hd_demo_sk#20) AND ((hd_dep_count#21 = 3) OR (hd_dep_count#21 = 1))) (29) BroadcastExchange Input [2]: [hd_demo_sk#20, hd_dep_count#21] @@ -201,9 +201,9 @@ Results [4]: [avg(ss_quantity#5)#36 AS avg(ss_quantity)#40, cast((avg(UnscaledVa Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#10 IN dynamicpruning#11 BroadcastExchange (39) -+- * Project (38) - +- * Filter (37) - +- * ColumnarToRow (36) ++- * ColumnarToRow (38) + +- CometProject (37) + +- CometFilter (36) +- CometScan parquet spark_catalog.default.date_dim (35) @@ -214,16 +214,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(36) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#16, d_year#44] - -(37) Filter [codegen id : 1] +(36) CometFilter Input [2]: [d_date_sk#16, d_year#44] Condition : ((isnotnull(d_year#44) AND (d_year#44 = 2001)) AND isnotnull(d_date_sk#16)) -(38) Project [codegen id : 1] -Output [1]: [d_date_sk#16] +(37) CometProject Input [2]: [d_date_sk#16, d_year#44] +Arguments: [d_date_sk#16], [d_date_sk#16] + +(38) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#16] (39) BroadcastExchange Input [1]: [d_date_sk#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/simplified.txt index 5e9dda7b9..5e5fc41f8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/simplified.txt @@ -14,46 +14,46 @@ WholeStageCodegen (7) BroadcastHashJoin [ss_addr_sk,ca_address_sk,ca_state,ss_net_profit] Project [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk,ss_addr_sk,ss_cdemo_sk,ss_hdemo_sk,ss_net_profit,ss_sales_price] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk,ss_addr_sk,ss_cdemo_sk,ss_hdemo_sk,ss_net_profit,ss_sales_price] CometScan parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Filter [s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk] InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - Project [ca_address_sk,ca_state] - Filter [ca_country,ca_address_sk,ca_state] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ca_address_sk,ca_state] + CometFilter [ca_country,ca_address_sk,ca_state] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] InputAdapter ReusedExchange [d_date_sk] #2 InputAdapter BroadcastExchange #5 WholeStageCodegen (4) - Filter [cd_demo_sk,cd_marital_status,cd_education_status] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] InputAdapter BroadcastExchange #6 WholeStageCodegen (5) - Filter [hd_demo_sk,hd_dep_count] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [hd_demo_sk,hd_dep_count] CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/explain.txt index fc94f648f..565cb97da 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/explain.txt @@ -15,14 +15,14 @@ TakeOrderedAndProject (105) : :- * Project (59) : : +- * BroadcastHashJoin Inner BuildRight (58) : : :- * BroadcastHashJoin LeftSemi BuildRight (51) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- BroadcastExchange (50) : : : +- * Project (49) : : : +- * BroadcastHashJoin Inner BuildRight (48) - : : : :- * Filter (6) - : : : : +- * ColumnarToRow (5) + : : : :- * ColumnarToRow (6) + : : : : +- CometFilter (5) : : : : +- CometScan parquet spark_catalog.default.item (4) : : : +- BroadcastExchange (47) : : : +- * BroadcastHashJoin LeftSemi BuildRight (46) @@ -33,25 +33,25 @@ TakeOrderedAndProject (105) : : : : +- * BroadcastHashJoin Inner BuildRight (31) : : : : :- * Project (29) : : : : : +- * BroadcastHashJoin Inner BuildRight (28) - : : : : : :- * Filter (9) - : : : : : : +- * ColumnarToRow (8) + : : : : : :- * ColumnarToRow (9) + : : : : : : +- CometFilter (8) : : : : : : +- CometScan parquet spark_catalog.default.store_sales (7) : : : : : +- BroadcastExchange (27) : : : : : +- * BroadcastHashJoin LeftSemi BuildRight (26) - : : : : : :- * Filter (12) - : : : : : : +- * ColumnarToRow (11) + : : : : : :- * ColumnarToRow (12) + : : : : : : +- CometFilter (11) : : : : : : +- CometScan parquet spark_catalog.default.item (10) : : : : : +- BroadcastExchange (25) : : : : : +- * Project (24) : : : : : +- * BroadcastHashJoin Inner BuildRight (23) : : : : : :- * Project (21) : : : : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : : : : :- * Filter (15) - : : : : : : : +- * ColumnarToRow (14) + : : : : : : :- * ColumnarToRow (15) + : : : : : : : +- CometFilter (14) : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (13) : : : : : : +- BroadcastExchange (19) - : : : : : : +- * Filter (18) - : : : : : : +- * ColumnarToRow (17) + : : : : : : +- * ColumnarToRow (18) + : : : : : : +- CometFilter (17) : : : : : : +- CometScan parquet spark_catalog.default.item (16) : : : : : +- ReusedExchange (22) : : : : +- ReusedExchange (30) @@ -60,15 +60,15 @@ TakeOrderedAndProject (105) : : : +- * BroadcastHashJoin Inner BuildRight (43) : : : :- * Project (41) : : : : +- * BroadcastHashJoin Inner BuildRight (40) - : : : : :- * Filter (38) - : : : : : +- * ColumnarToRow (37) + : : : : :- * ColumnarToRow (38) + : : : : : +- CometFilter (37) : : : : : +- CometScan parquet spark_catalog.default.web_sales (36) : : : : +- ReusedExchange (39) : : : +- ReusedExchange (42) : : +- BroadcastExchange (57) : : +- * BroadcastHashJoin LeftSemi BuildRight (56) - : : :- * Filter (54) - : : : +- * ColumnarToRow (53) + : : :- * ColumnarToRow (54) + : : : +- CometFilter (53) : : : +- CometScan parquet spark_catalog.default.item (52) : : +- ReusedExchange (55) : +- ReusedExchange (60) @@ -82,8 +82,8 @@ TakeOrderedAndProject (105) : :- * Project (75) : : +- * BroadcastHashJoin Inner BuildRight (74) : : :- * BroadcastHashJoin LeftSemi BuildRight (72) - : : : :- * Filter (70) - : : : : +- * ColumnarToRow (69) + : : : :- * ColumnarToRow (70) + : : : : +- CometFilter (69) : : : : +- CometScan parquet spark_catalog.default.catalog_sales (68) : : : +- ReusedExchange (71) : : +- ReusedExchange (73) @@ -98,8 +98,8 @@ TakeOrderedAndProject (105) :- * Project (91) : +- * BroadcastHashJoin Inner BuildRight (90) : :- * BroadcastHashJoin LeftSemi BuildRight (88) - : : :- * Filter (86) - : : : +- * ColumnarToRow (85) + : : :- * ColumnarToRow (86) + : : : +- CometFilter (85) : : : +- CometScan parquet spark_catalog.default.web_sales (84) : : +- ReusedExchange (87) : +- ReusedExchange (89) @@ -114,12 +114,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 25] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Condition : isnotnull(ss_item_sk#1) -(3) Filter [codegen id : 25] +(3) ColumnarToRow [codegen id : 25] Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_item_sk#1) (4) Scan parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] @@ -128,12 +128,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 11] +(5) CometFilter Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] +Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) -(6) Filter [codegen id : 11] +(6) ColumnarToRow [codegen id : 11] Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) (7) Scan parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#10, ss_sold_date_sk#11] @@ -143,12 +143,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 6] +(8) CometFilter Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] +Condition : isnotnull(ss_item_sk#10) -(9) Filter [codegen id : 6] +(9) ColumnarToRow [codegen id : 6] Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Condition : isnotnull(ss_item_sk#10) (10) Scan parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] @@ -157,12 +157,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 4] +(11) CometFilter Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16)) -(12) Filter [codegen id : 4] +(12) ColumnarToRow [codegen id : 4] Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16)) (13) Scan parquet spark_catalog.default.catalog_sales Output [2]: [cs_item_sk#17, cs_sold_date_sk#18] @@ -172,12 +172,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] +(14) CometFilter Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] +Condition : isnotnull(cs_item_sk#17) -(15) Filter [codegen id : 3] +(15) ColumnarToRow [codegen id : 3] Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Condition : isnotnull(cs_item_sk#17) (16) Scan parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] @@ -186,12 +186,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 1] +(17) CometFilter Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Condition : isnotnull(i_item_sk#20) -(18) Filter [codegen id : 1] +(18) ColumnarToRow [codegen id : 1] Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Condition : isnotnull(i_item_sk#20) (19) BroadcastExchange Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] @@ -283,12 +283,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#30), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 9] +(37) CometFilter Input [2]: [ws_item_sk#29, ws_sold_date_sk#30] +Condition : isnotnull(ws_item_sk#29) -(38) Filter [codegen id : 9] +(38) ColumnarToRow [codegen id : 9] Input [2]: [ws_item_sk#29, ws_sold_date_sk#30] -Condition : isnotnull(ws_item_sk#29) (39) ReusedExchange [Reuses operator id: 19] Output [4]: [i_item_sk#32, i_brand_id#33, i_class_id#34, i_category_id#35] @@ -357,12 +357,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(53) ColumnarToRow [codegen id : 23] +(53) CometFilter Input [4]: [i_item_sk#38, i_brand_id#39, i_class_id#40, i_category_id#41] +Condition : isnotnull(i_item_sk#38) -(54) Filter [codegen id : 23] +(54) ColumnarToRow [codegen id : 23] Input [4]: [i_item_sk#38, i_brand_id#39, i_class_id#40, i_category_id#41] -Condition : isnotnull(i_item_sk#38) (55) ReusedExchange [Reuses operator id: 50] Output [1]: [ss_item_sk#37] @@ -434,12 +434,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#62), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(69) ColumnarToRow [codegen id : 51] +(69) CometFilter Input [4]: [cs_item_sk#59, cs_quantity#60, cs_list_price#61, cs_sold_date_sk#62] +Condition : isnotnull(cs_item_sk#59) -(70) Filter [codegen id : 51] +(70) ColumnarToRow [codegen id : 51] Input [4]: [cs_item_sk#59, cs_quantity#60, cs_list_price#61, cs_sold_date_sk#62] -Condition : isnotnull(cs_item_sk#59) (71) ReusedExchange [Reuses operator id: 50] Output [1]: [ss_item_sk#64] @@ -510,12 +510,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#84), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(85) ColumnarToRow [codegen id : 77] +(85) CometFilter Input [4]: [ws_item_sk#81, ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] +Condition : isnotnull(ws_item_sk#81) -(86) Filter [codegen id : 77] +(86) ColumnarToRow [codegen id : 77] Input [4]: [ws_item_sk#81, ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] -Condition : isnotnull(ws_item_sk#81) (87) ReusedExchange [Reuses operator id: 50] Output [1]: [ss_item_sk#86] @@ -727,9 +727,9 @@ Subquery:4 Hosting operator id = 116 Hosting Expression = ws_sold_date_sk#134 IN Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 BroadcastExchange (129) -+- * Project (128) - +- * Filter (127) - +- * ColumnarToRow (126) ++- * ColumnarToRow (128) + +- CometProject (127) + +- CometFilter (126) +- CometScan parquet spark_catalog.default.date_dim (125) @@ -740,16 +740,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct -(126) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#42, d_year#145, d_moy#146] - -(127) Filter [codegen id : 1] +(126) CometFilter Input [3]: [d_date_sk#42, d_year#145, d_moy#146] Condition : ((((isnotnull(d_year#145) AND isnotnull(d_moy#146)) AND (d_year#145 = 2001)) AND (d_moy#146 = 11)) AND isnotnull(d_date_sk#42)) -(128) Project [codegen id : 1] -Output [1]: [d_date_sk#42] +(127) CometProject Input [3]: [d_date_sk#42, d_year#145, d_moy#146] +Arguments: [d_date_sk#42], [d_date_sk#42] + +(128) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#42] (129) BroadcastExchange Input [1]: [d_date_sk#42] @@ -757,9 +757,9 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Subquery:6 Hosting operator id = 7 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12 BroadcastExchange (134) -+- * Project (133) - +- * Filter (132) - +- * ColumnarToRow (131) ++- * ColumnarToRow (133) + +- CometProject (132) + +- CometFilter (131) +- CometScan parquet spark_catalog.default.date_dim (130) @@ -770,16 +770,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(131) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#25, d_year#147] - -(132) Filter [codegen id : 1] +(131) CometFilter Input [2]: [d_date_sk#25, d_year#147] Condition : (((isnotnull(d_year#147) AND (d_year#147 >= 1999)) AND (d_year#147 <= 2001)) AND isnotnull(d_date_sk#25)) -(133) Project [codegen id : 1] -Output [1]: [d_date_sk#25] +(132) CometProject Input [2]: [d_date_sk#25, d_year#147] +Arguments: [d_date_sk#25], [d_date_sk#25] + +(133) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#25] (134) BroadcastExchange Input [1]: [d_date_sk#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/simplified.txt index a76997516..cf688c448 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/simplified.txt @@ -57,26 +57,26 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),su Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] BroadcastHashJoin [ss_item_sk,i_item_sk] BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_moy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #4 WholeStageCodegen (11) Project [i_item_sk] BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - Filter [i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_brand_id,i_class_id,i_category_id] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] InputAdapter BroadcastExchange #5 @@ -91,25 +91,25 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),su BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #7 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #8 WholeStageCodegen (4) BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] InputAdapter BroadcastExchange #9 @@ -118,17 +118,17 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),su BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #2 InputAdapter BroadcastExchange #10 WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] InputAdapter ReusedExchange [d_date_sk] #7 @@ -141,9 +141,9 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),su BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 InputAdapter @@ -154,9 +154,9 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),su BroadcastExchange #12 WholeStageCodegen (23) BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] InputAdapter ReusedExchange [ss_item_sk] #4 @@ -176,9 +176,9 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),su Project [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] BroadcastHashJoin [cs_item_sk,i_item_sk] BroadcastHashJoin [cs_item_sk,ss_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -201,9 +201,9 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),su Project [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] BroadcastHashJoin [ws_item_sk,i_item_sk] BroadcastHashJoin [ws_item_sk,ss_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/explain.txt index c6b42f19c..265909ec2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/explain.txt @@ -10,14 +10,14 @@ TakeOrderedAndProject (84) : :- * Project (59) : : +- * BroadcastHashJoin Inner BuildRight (58) : : :- * BroadcastHashJoin LeftSemi BuildRight (51) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- BroadcastExchange (50) : : : +- * Project (49) : : : +- * BroadcastHashJoin Inner BuildRight (48) - : : : :- * Filter (6) - : : : : +- * ColumnarToRow (5) + : : : :- * ColumnarToRow (6) + : : : : +- CometFilter (5) : : : : +- CometScan parquet spark_catalog.default.item (4) : : : +- BroadcastExchange (47) : : : +- * BroadcastHashJoin LeftSemi BuildRight (46) @@ -28,25 +28,25 @@ TakeOrderedAndProject (84) : : : : +- * BroadcastHashJoin Inner BuildRight (31) : : : : :- * Project (29) : : : : : +- * BroadcastHashJoin Inner BuildRight (28) - : : : : : :- * Filter (9) - : : : : : : +- * ColumnarToRow (8) + : : : : : :- * ColumnarToRow (9) + : : : : : : +- CometFilter (8) : : : : : : +- CometScan parquet spark_catalog.default.store_sales (7) : : : : : +- BroadcastExchange (27) : : : : : +- * BroadcastHashJoin LeftSemi BuildRight (26) - : : : : : :- * Filter (12) - : : : : : : +- * ColumnarToRow (11) + : : : : : :- * ColumnarToRow (12) + : : : : : : +- CometFilter (11) : : : : : : +- CometScan parquet spark_catalog.default.item (10) : : : : : +- BroadcastExchange (25) : : : : : +- * Project (24) : : : : : +- * BroadcastHashJoin Inner BuildRight (23) : : : : : :- * Project (21) : : : : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : : : : :- * Filter (15) - : : : : : : : +- * ColumnarToRow (14) + : : : : : : :- * ColumnarToRow (15) + : : : : : : : +- CometFilter (14) : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (13) : : : : : : +- BroadcastExchange (19) - : : : : : : +- * Filter (18) - : : : : : : +- * ColumnarToRow (17) + : : : : : : +- * ColumnarToRow (18) + : : : : : : +- CometFilter (17) : : : : : : +- CometScan parquet spark_catalog.default.item (16) : : : : : +- ReusedExchange (22) : : : : +- ReusedExchange (30) @@ -55,15 +55,15 @@ TakeOrderedAndProject (84) : : : +- * BroadcastHashJoin Inner BuildRight (43) : : : :- * Project (41) : : : : +- * BroadcastHashJoin Inner BuildRight (40) - : : : : :- * Filter (38) - : : : : : +- * ColumnarToRow (37) + : : : : :- * ColumnarToRow (38) + : : : : : +- CometFilter (37) : : : : : +- CometScan parquet spark_catalog.default.web_sales (36) : : : : +- ReusedExchange (39) : : : +- ReusedExchange (42) : : +- BroadcastExchange (57) : : +- * BroadcastHashJoin LeftSemi BuildRight (56) - : : :- * Filter (54) - : : : +- * ColumnarToRow (53) + : : :- * ColumnarToRow (54) + : : : +- CometFilter (53) : : : +- CometScan parquet spark_catalog.default.item (52) : : +- ReusedExchange (55) : +- ReusedExchange (60) @@ -77,8 +77,8 @@ TakeOrderedAndProject (84) :- * Project (74) : +- * BroadcastHashJoin Inner BuildRight (73) : :- * BroadcastHashJoin LeftSemi BuildRight (71) - : : :- * Filter (69) - : : : +- * ColumnarToRow (68) + : : :- * ColumnarToRow (69) + : : : +- CometFilter (68) : : : +- CometScan parquet spark_catalog.default.store_sales (67) : : +- ReusedExchange (70) : +- ReusedExchange (72) @@ -93,12 +93,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 25] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Condition : isnotnull(ss_item_sk#1) -(3) Filter [codegen id : 25] +(3) ColumnarToRow [codegen id : 25] Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_item_sk#1) (4) Scan parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] @@ -107,12 +107,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 11] +(5) CometFilter Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] +Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) -(6) Filter [codegen id : 11] +(6) ColumnarToRow [codegen id : 11] Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) (7) Scan parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#10, ss_sold_date_sk#11] @@ -122,12 +122,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 6] +(8) CometFilter Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] +Condition : isnotnull(ss_item_sk#10) -(9) Filter [codegen id : 6] +(9) ColumnarToRow [codegen id : 6] Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Condition : isnotnull(ss_item_sk#10) (10) Scan parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] @@ -136,12 +136,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 4] +(11) CometFilter Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16)) -(12) Filter [codegen id : 4] +(12) ColumnarToRow [codegen id : 4] Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16)) (13) Scan parquet spark_catalog.default.catalog_sales Output [2]: [cs_item_sk#17, cs_sold_date_sk#18] @@ -151,12 +151,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] +(14) CometFilter Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] +Condition : isnotnull(cs_item_sk#17) -(15) Filter [codegen id : 3] +(15) ColumnarToRow [codegen id : 3] Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Condition : isnotnull(cs_item_sk#17) (16) Scan parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] @@ -165,12 +165,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 1] +(17) CometFilter Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Condition : isnotnull(i_item_sk#20) -(18) Filter [codegen id : 1] +(18) ColumnarToRow [codegen id : 1] Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Condition : isnotnull(i_item_sk#20) (19) BroadcastExchange Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] @@ -262,12 +262,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#30), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 9] +(37) CometFilter Input [2]: [ws_item_sk#29, ws_sold_date_sk#30] +Condition : isnotnull(ws_item_sk#29) -(38) Filter [codegen id : 9] +(38) ColumnarToRow [codegen id : 9] Input [2]: [ws_item_sk#29, ws_sold_date_sk#30] -Condition : isnotnull(ws_item_sk#29) (39) ReusedExchange [Reuses operator id: 19] Output [4]: [i_item_sk#32, i_brand_id#33, i_class_id#34, i_category_id#35] @@ -336,12 +336,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(53) ColumnarToRow [codegen id : 23] +(53) CometFilter Input [4]: [i_item_sk#38, i_brand_id#39, i_class_id#40, i_category_id#41] +Condition : (((isnotnull(i_item_sk#38) AND isnotnull(i_brand_id#39)) AND isnotnull(i_class_id#40)) AND isnotnull(i_category_id#41)) -(54) Filter [codegen id : 23] +(54) ColumnarToRow [codegen id : 23] Input [4]: [i_item_sk#38, i_brand_id#39, i_class_id#40, i_category_id#41] -Condition : (((isnotnull(i_item_sk#38) AND isnotnull(i_brand_id#39)) AND isnotnull(i_class_id#40)) AND isnotnull(i_category_id#41)) (55) ReusedExchange [Reuses operator id: 50] Output [1]: [ss_item_sk#37] @@ -409,12 +409,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#59), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(68) ColumnarToRow [codegen id : 50] +(68) CometFilter Input [4]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59] +Condition : isnotnull(ss_item_sk#56) -(69) Filter [codegen id : 50] +(69) ColumnarToRow [codegen id : 50] Input [4]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59] -Condition : isnotnull(ss_item_sk#56) (70) ReusedExchange [Reuses operator id: 50] Output [1]: [ss_item_sk#61] @@ -608,9 +608,9 @@ Subquery:4 Hosting operator id = 95 Hosting Expression = ws_sold_date_sk#94 IN d Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 BroadcastExchange (108) -+- * Project (107) - +- * Filter (106) - +- * ColumnarToRow (105) ++- * ColumnarToRow (107) + +- CometProject (106) + +- CometFilter (105) +- CometScan parquet spark_catalog.default.date_dim (104) @@ -621,27 +621,27 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#106), IsNotNull(d_date_sk)] ReadSchema: struct -(105) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#42, d_week_seq#105] - -(106) Filter [codegen id : 1] +(105) CometFilter Input [2]: [d_date_sk#42, d_week_seq#105] Condition : ((isnotnull(d_week_seq#105) AND (d_week_seq#105 = ReusedSubquery Subquery scalar-subquery#106, [id=#107])) AND isnotnull(d_date_sk#42)) -(107) Project [codegen id : 1] -Output [1]: [d_date_sk#42] +(106) CometProject Input [2]: [d_date_sk#42, d_week_seq#105] +Arguments: [d_date_sk#42], [d_date_sk#42] + +(107) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#42] (108) BroadcastExchange Input [1]: [d_date_sk#42] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] -Subquery:6 Hosting operator id = 106 Hosting Expression = ReusedSubquery Subquery scalar-subquery#106, [id=#107] +Subquery:6 Hosting operator id = 105 Hosting Expression = ReusedSubquery Subquery scalar-subquery#106, [id=#107] Subquery:7 Hosting operator id = 104 Hosting Expression = Subquery scalar-subquery#106, [id=#107] -* Project (112) -+- * Filter (111) - +- * ColumnarToRow (110) +* ColumnarToRow (112) ++- CometProject (111) + +- CometFilter (110) +- CometScan parquet spark_catalog.default.date_dim (109) @@ -652,22 +652,22 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,2000), EqualTo(d_moy,12), EqualTo(d_dom,11)] ReadSchema: struct -(110) ColumnarToRow [codegen id : 1] -Input [4]: [d_week_seq#108, d_year#109, d_moy#110, d_dom#111] - -(111) Filter [codegen id : 1] +(110) CometFilter Input [4]: [d_week_seq#108, d_year#109, d_moy#110, d_dom#111] Condition : (((((isnotnull(d_year#109) AND isnotnull(d_moy#110)) AND isnotnull(d_dom#111)) AND (d_year#109 = 2000)) AND (d_moy#110 = 12)) AND (d_dom#111 = 11)) -(112) Project [codegen id : 1] -Output [1]: [d_week_seq#108] +(111) CometProject Input [4]: [d_week_seq#108, d_year#109, d_moy#110, d_dom#111] +Arguments: [d_week_seq#108], [d_week_seq#108] + +(112) ColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#108] Subquery:8 Hosting operator id = 7 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12 BroadcastExchange (117) -+- * Project (116) - +- * Filter (115) - +- * ColumnarToRow (114) ++- * ColumnarToRow (116) + +- CometProject (115) + +- CometFilter (114) +- CometScan parquet spark_catalog.default.date_dim (113) @@ -678,16 +678,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(114) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#25, d_year#112] - -(115) Filter [codegen id : 1] +(114) CometFilter Input [2]: [d_date_sk#25, d_year#112] Condition : (((isnotnull(d_year#112) AND (d_year#112 >= 1999)) AND (d_year#112 <= 2001)) AND isnotnull(d_date_sk#25)) -(116) Project [codegen id : 1] -Output [1]: [d_date_sk#25] +(115) CometProject Input [2]: [d_date_sk#25, d_year#112] +Arguments: [d_date_sk#25], [d_date_sk#25] + +(116) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#25] (117) BroadcastExchange Input [1]: [d_date_sk#25] @@ -701,9 +701,9 @@ Subquery:11 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquer Subquery:12 Hosting operator id = 67 Hosting Expression = ss_sold_date_sk#59 IN dynamicpruning#60 BroadcastExchange (122) -+- * Project (121) - +- * Filter (120) - +- * ColumnarToRow (119) ++- * ColumnarToRow (121) + +- CometProject (120) + +- CometFilter (119) +- CometScan parquet spark_catalog.default.date_dim (118) @@ -714,27 +714,27 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#114), IsNotNull(d_date_sk)] ReadSchema: struct -(119) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#66, d_week_seq#113] - -(120) Filter [codegen id : 1] +(119) CometFilter Input [2]: [d_date_sk#66, d_week_seq#113] Condition : ((isnotnull(d_week_seq#113) AND (d_week_seq#113 = ReusedSubquery Subquery scalar-subquery#114, [id=#115])) AND isnotnull(d_date_sk#66)) -(121) Project [codegen id : 1] -Output [1]: [d_date_sk#66] +(120) CometProject Input [2]: [d_date_sk#66, d_week_seq#113] +Arguments: [d_date_sk#66], [d_date_sk#66] + +(121) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#66] (122) BroadcastExchange Input [1]: [d_date_sk#66] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=15] -Subquery:13 Hosting operator id = 120 Hosting Expression = ReusedSubquery Subquery scalar-subquery#114, [id=#115] +Subquery:13 Hosting operator id = 119 Hosting Expression = ReusedSubquery Subquery scalar-subquery#114, [id=#115] Subquery:14 Hosting operator id = 118 Hosting Expression = Subquery scalar-subquery#114, [id=#115] -* Project (126) -+- * Filter (125) - +- * ColumnarToRow (124) +* ColumnarToRow (126) ++- CometProject (125) + +- CometFilter (124) +- CometScan parquet spark_catalog.default.date_dim (123) @@ -745,15 +745,15 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,11)] ReadSchema: struct -(124) ColumnarToRow [codegen id : 1] -Input [4]: [d_week_seq#116, d_year#117, d_moy#118, d_dom#119] - -(125) Filter [codegen id : 1] +(124) CometFilter Input [4]: [d_week_seq#116, d_year#117, d_moy#118, d_dom#119] Condition : (((((isnotnull(d_year#117) AND isnotnull(d_moy#118)) AND isnotnull(d_dom#119)) AND (d_year#117 = 1999)) AND (d_moy#118 = 12)) AND (d_dom#119 = 11)) -(126) Project [codegen id : 1] -Output [1]: [d_week_seq#116] +(125) CometProject Input [4]: [d_week_seq#116, d_year#117, d_moy#118, d_dom#119] +Arguments: [d_week_seq#116], [d_week_seq#116] + +(126) ColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#116] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/simplified.txt index 64646b717..55aa823ab 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/simplified.txt @@ -48,34 +48,34 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] BroadcastHashJoin [ss_item_sk,i_item_sk] BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_week_seq,d_date_sk] - ReusedSubquery [d_week_seq] #2 - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_week_seq,d_date_sk] + ReusedSubquery [d_week_seq] #2 CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] Subquery #2 WholeStageCodegen (1) - Project [d_week_seq] - Filter [d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_year,d_moy,d_dom] CometScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] InputAdapter BroadcastExchange #3 WholeStageCodegen (11) Project [i_item_sk] BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - Filter [i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_brand_id,i_class_id,i_category_id] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] InputAdapter BroadcastExchange #4 @@ -90,25 +90,25 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #3 BroadcastExchange #6 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #7 WholeStageCodegen (4) BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] InputAdapter BroadcastExchange #8 @@ -117,17 +117,17 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #3 InputAdapter BroadcastExchange #9 WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] InputAdapter ReusedExchange [d_date_sk] #6 @@ -140,9 +140,9 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #3 InputAdapter @@ -153,9 +153,9 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ BroadcastExchange #11 WholeStageCodegen (23) BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] InputAdapter ReusedExchange [ss_item_sk] #3 @@ -176,25 +176,25 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] BroadcastHashJoin [ss_item_sk,i_item_sk] BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #5 BroadcastExchange #15 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_week_seq,d_date_sk] - ReusedSubquery [d_week_seq] #6 - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_week_seq,d_date_sk] + ReusedSubquery [d_week_seq] #6 CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] Subquery #6 WholeStageCodegen (1) - Project [d_week_seq] - Filter [d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_year,d_moy,d_dom] CometScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] InputAdapter ReusedExchange [ss_item_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/explain.txt index 10ae4b11b..13a621f77 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/explain.txt @@ -9,16 +9,16 @@ TakeOrderedAndProject (22) : +- * BroadcastHashJoin Inner BuildRight (14) : :- * Project (9) : : +- * BroadcastHashJoin Inner BuildRight (8) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) + : : :- * ColumnarToRow (3) + : : : +- CometFilter (2) : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : : +- BroadcastExchange (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) + : : +- * ColumnarToRow (6) + : : +- CometFilter (5) : : +- CometScan parquet spark_catalog.default.customer (4) : +- BroadcastExchange (13) - : +- * Filter (12) - : +- * ColumnarToRow (11) + : +- * ColumnarToRow (12) + : +- CometFilter (11) : +- CometScan parquet spark_catalog.default.customer_address (10) +- ReusedExchange (16) @@ -31,12 +31,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sol PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] +Condition : isnotnull(cs_bill_customer_sk#1) -(3) Filter [codegen id : 4] +(3) ColumnarToRow [codegen id : 4] Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] -Condition : isnotnull(cs_bill_customer_sk#1) (4) Scan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#5, c_current_addr_sk#6] @@ -45,12 +45,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [2]: [c_customer_sk#5, c_current_addr_sk#6] +Condition : (isnotnull(c_customer_sk#5) AND isnotnull(c_current_addr_sk#6)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [2]: [c_customer_sk#5, c_current_addr_sk#6] -Condition : (isnotnull(c_customer_sk#5) AND isnotnull(c_current_addr_sk#6)) (7) BroadcastExchange Input [2]: [c_customer_sk#5, c_current_addr_sk#6] @@ -73,12 +73,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] +(11) CometFilter Input [3]: [ca_address_sk#7, ca_state#8, ca_zip#9] +Condition : isnotnull(ca_address_sk#7) -(12) Filter [codegen id : 2] +(12) ColumnarToRow [codegen id : 2] Input [3]: [ca_address_sk#7, ca_state#8, ca_zip#9] -Condition : isnotnull(ca_address_sk#7) (13) BroadcastExchange Input [3]: [ca_address_sk#7, ca_state#8, ca_zip#9] @@ -133,9 +133,9 @@ Arguments: 100, [ca_zip#9 ASC NULLS FIRST], [ca_zip#9, sum(cs_sales_price)#14] Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 BroadcastExchange (27) -+- * Project (26) - +- * Filter (25) - +- * ColumnarToRow (24) ++- * ColumnarToRow (26) + +- CometProject (25) + +- CometFilter (24) +- CometScan parquet spark_catalog.default.date_dim (23) @@ -146,16 +146,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#10, d_year#15, d_qoy#16] - -(25) Filter [codegen id : 1] +(24) CometFilter Input [3]: [d_date_sk#10, d_year#15, d_qoy#16] Condition : ((((isnotnull(d_qoy#16) AND isnotnull(d_year#15)) AND (d_qoy#16 = 2)) AND (d_year#15 = 2001)) AND isnotnull(d_date_sk#10)) -(26) Project [codegen id : 1] -Output [1]: [d_date_sk#10] +(25) CometProject Input [3]: [d_date_sk#10, d_year#15, d_qoy#16] +Arguments: [d_date_sk#10], [d_date_sk#10] + +(26) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#10] (27) BroadcastExchange Input [1]: [d_date_sk#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/simplified.txt index 4b916d78f..5c750b2db 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/simplified.txt @@ -11,31 +11,31 @@ TakeOrderedAndProject [ca_zip,sum(cs_sales_price)] BroadcastHashJoin [c_current_addr_sk,ca_address_sk,ca_zip,ca_state,cs_sales_price] Project [cs_sales_price,cs_sold_date_sk,c_current_addr_sk] BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Filter [cs_bill_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_bill_customer_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_qoy,d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_current_addr_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] InputAdapter ReusedExchange [d_date_sk] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/explain.txt index d21c24281..ccec341ad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/explain.txt @@ -15,34 +15,34 @@ : : : : +- * SortMergeJoin LeftSemi (12) : : : : :- * Sort (6) : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : : +- * ColumnarToRow (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : : : : +- * Sort (11) : : : : +- Exchange (10) - : : : : +- * Project (9) - : : : : +- * ColumnarToRow (8) + : : : : +- * ColumnarToRow (9) + : : : : +- CometProject (8) : : : : +- CometScan parquet spark_catalog.default.catalog_sales (7) : : : +- * Sort (18) : : : +- Exchange (17) - : : : +- * Project (16) - : : : +- * ColumnarToRow (15) + : : : +- * ColumnarToRow (16) + : : : +- CometProject (15) : : : +- CometScan parquet spark_catalog.default.catalog_returns (14) : : +- BroadcastExchange (24) - : : +- * Project (23) - : : +- * Filter (22) - : : +- * ColumnarToRow (21) + : : +- * ColumnarToRow (23) + : : +- CometProject (22) + : : +- CometFilter (21) : : +- CometScan parquet spark_catalog.default.date_dim (20) : +- BroadcastExchange (31) - : +- * Project (30) - : +- * Filter (29) - : +- * ColumnarToRow (28) + : +- * ColumnarToRow (30) + : +- CometProject (29) + : +- CometFilter (28) : +- CometScan parquet spark_catalog.default.customer_address (27) +- BroadcastExchange (38) - +- * Project (37) - +- * Filter (36) - +- * ColumnarToRow (35) + +- * ColumnarToRow (37) + +- CometProject (36) + +- CometFilter (35) +- CometScan parquet spark_catalog.default.call_center (34) @@ -53,16 +53,16 @@ Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_ship_date_sk), IsNotNull(cs_ship_addr_sk), IsNotNull(cs_call_center_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] Condition : ((isnotnull(cs_ship_date_sk#1) AND isnotnull(cs_ship_addr_sk#2)) AND isnotnull(cs_call_center_sk#3)) -(4) Project [codegen id : 1] -Output [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +(3) CometProject Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] +Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] + +(4) ColumnarToRow [codegen id : 1] +Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] (5) Exchange Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] @@ -78,12 +78,12 @@ Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] ReadSchema: struct -(8) ColumnarToRow [codegen id : 3] +(8) CometProject Input [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] +Arguments: [cs_warehouse_sk#9, cs_order_number#10], [cs_warehouse_sk#9, cs_order_number#10] -(9) Project [codegen id : 3] -Output [2]: [cs_warehouse_sk#9, cs_order_number#10] -Input [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] +(9) ColumnarToRow [codegen id : 3] +Input [2]: [cs_warehouse_sk#9, cs_order_number#10] (10) Exchange Input [2]: [cs_warehouse_sk#9, cs_order_number#10] @@ -109,12 +109,12 @@ Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] ReadSchema: struct -(15) ColumnarToRow [codegen id : 6] +(15) CometProject Input [2]: [cr_order_number#12, cr_returned_date_sk#13] +Arguments: [cr_order_number#12], [cr_order_number#12] -(16) Project [codegen id : 6] -Output [1]: [cr_order_number#12] -Input [2]: [cr_order_number#12, cr_returned_date_sk#13] +(16) ColumnarToRow [codegen id : 6] +Input [1]: [cr_order_number#12] (17) Exchange Input [1]: [cr_order_number#12] @@ -137,16 +137,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2002-02-01), LessThanOrEqual(d_date,2002-04-02), IsNotNull(d_date_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 8] -Input [2]: [d_date_sk#14, d_date#15] - -(22) Filter [codegen id : 8] +(21) CometFilter Input [2]: [d_date_sk#14, d_date#15] Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 2002-02-01)) AND (d_date#15 <= 2002-04-02)) AND isnotnull(d_date_sk#14)) -(23) Project [codegen id : 8] -Output [1]: [d_date_sk#14] +(22) CometProject Input [2]: [d_date_sk#14, d_date#15] +Arguments: [d_date_sk#14], [d_date_sk#14] + +(23) ColumnarToRow [codegen id : 8] +Input [1]: [d_date_sk#14] (24) BroadcastExchange Input [1]: [d_date_sk#14] @@ -169,16 +169,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 9] -Input [2]: [ca_address_sk#16, ca_state#17] - -(29) Filter [codegen id : 9] +(28) CometFilter Input [2]: [ca_address_sk#16, ca_state#17] Condition : ((isnotnull(ca_state#17) AND (ca_state#17 = GA)) AND isnotnull(ca_address_sk#16)) -(30) Project [codegen id : 9] -Output [1]: [ca_address_sk#16] +(29) CometProject Input [2]: [ca_address_sk#16, ca_state#17] +Arguments: [ca_address_sk#16], [ca_address_sk#16] + +(30) ColumnarToRow [codegen id : 9] +Input [1]: [ca_address_sk#16] (31) BroadcastExchange Input [1]: [ca_address_sk#16] @@ -201,16 +201,16 @@ Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_county), EqualTo(cc_county,Williamson County), IsNotNull(cc_call_center_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 10] -Input [2]: [cc_call_center_sk#18, cc_county#19] - -(36) Filter [codegen id : 10] +(35) CometFilter Input [2]: [cc_call_center_sk#18, cc_county#19] Condition : ((isnotnull(cc_county#19) AND (cc_county#19 = Williamson County)) AND isnotnull(cc_call_center_sk#18)) -(37) Project [codegen id : 10] -Output [1]: [cc_call_center_sk#18] +(36) CometProject Input [2]: [cc_call_center_sk#18, cc_county#19] +Arguments: [cc_call_center_sk#18], [cc_call_center_sk#18] + +(37) ColumnarToRow [codegen id : 10] +Input [1]: [cc_call_center_sk#18] (38) BroadcastExchange Input [1]: [cc_call_center_sk#18] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/simplified.txt index 5feaa702a..a55c182be 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/simplified.txt @@ -23,10 +23,10 @@ WholeStageCodegen (12) InputAdapter Exchange [cs_order_number] #2 WholeStageCodegen (1) - Project [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - Filter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometFilter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] InputAdapter WholeStageCodegen (4) @@ -34,9 +34,9 @@ WholeStageCodegen (12) InputAdapter Exchange [cs_order_number] #3 WholeStageCodegen (3) - Project [cs_warehouse_sk,cs_order_number] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cs_warehouse_sk,cs_order_number] CometScan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] InputAdapter WholeStageCodegen (7) @@ -44,31 +44,31 @@ WholeStageCodegen (12) InputAdapter Exchange [cr_order_number] #4 WholeStageCodegen (6) - Project [cr_order_number] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cr_order_number] CometScan parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] InputAdapter BroadcastExchange #5 WholeStageCodegen (8) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter BroadcastExchange #6 WholeStageCodegen (9) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_state,ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #7 WholeStageCodegen (10) - Project [cc_call_center_sk] - Filter [cc_county,cc_call_center_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cc_call_center_sk] + CometFilter [cc_county,cc_call_center_sk] CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/explain.txt index 4548704d0..0de98cfb0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/explain.txt @@ -17,27 +17,27 @@ TakeOrderedAndProject (40) : : : : : +- * BroadcastHashJoin Inner BuildRight (14) : : : : : :- * Project (9) : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) + : : : : : : :- * ColumnarToRow (3) + : : : : : : : +- CometFilter (2) : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : : : : +- BroadcastExchange (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) + : : : : : : +- * ColumnarToRow (6) + : : : : : : +- CometFilter (5) : : : : : : +- CometScan parquet spark_catalog.default.store_returns (4) : : : : : +- BroadcastExchange (13) - : : : : : +- * Filter (12) - : : : : : +- * ColumnarToRow (11) + : : : : : +- * ColumnarToRow (12) + : : : : : +- CometFilter (11) : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (10) : : : : +- ReusedExchange (16) : : : +- ReusedExchange (19) : : +- ReusedExchange (22) : +- BroadcastExchange (28) - : +- * Filter (27) - : +- * ColumnarToRow (26) + : +- * ColumnarToRow (27) + : +- CometFilter (26) : +- CometScan parquet spark_catalog.default.store (25) +- BroadcastExchange (34) - +- * Filter (33) - +- * ColumnarToRow (32) + +- * ColumnarToRow (33) + +- CometFilter (32) +- CometScan parquet spark_catalog.default.item (31) @@ -49,12 +49,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 8] +(2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] +Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) -(3) Filter [codegen id : 8] +(3) ColumnarToRow [codegen id : 8] Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) (4) Scan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] @@ -64,12 +64,12 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#12), dynamicpruningexpression(s PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] +Condition : ((isnotnull(sr_customer_sk#9) AND isnotnull(sr_item_sk#8)) AND isnotnull(sr_ticket_number#10)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Condition : ((isnotnull(sr_customer_sk#9) AND isnotnull(sr_item_sk#8)) AND isnotnull(sr_ticket_number#10)) (7) BroadcastExchange Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] @@ -93,12 +93,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] +(11) CometFilter Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] +Condition : (isnotnull(cs_bill_customer_sk#14) AND isnotnull(cs_item_sk#15)) -(12) Filter [codegen id : 2] +(12) ColumnarToRow [codegen id : 2] Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Condition : (isnotnull(cs_bill_customer_sk#14) AND isnotnull(cs_item_sk#15)) (13) BroadcastExchange Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] @@ -160,12 +160,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 6] +(26) CometFilter Input [2]: [s_store_sk#22, s_state#23] +Condition : isnotnull(s_store_sk#22) -(27) Filter [codegen id : 6] +(27) ColumnarToRow [codegen id : 6] Input [2]: [s_store_sk#22, s_state#23] -Condition : isnotnull(s_store_sk#22) (28) BroadcastExchange Input [2]: [s_store_sk#22, s_state#23] @@ -188,12 +188,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 7] +(32) CometFilter Input [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] +Condition : isnotnull(i_item_sk#24) -(33) Filter [codegen id : 7] +(33) ColumnarToRow [codegen id : 7] Input [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] -Condition : isnotnull(i_item_sk#24) (34) BroadcastExchange Input [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] @@ -235,9 +235,9 @@ Arguments: 100, [i_item_id#25 ASC NULLS FIRST, i_item_desc#26 ASC NULLS FIRST, s Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 BroadcastExchange (45) -+- * Project (44) - +- * Filter (43) - +- * ColumnarToRow (42) ++- * ColumnarToRow (44) + +- CometProject (43) + +- CometFilter (42) +- CometScan parquet spark_catalog.default.date_dim (41) @@ -248,16 +248,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_quarter_name), EqualTo(d_quarter_name,2001Q1), IsNotNull(d_date_sk)] ReadSchema: struct -(42) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#19, d_quarter_name#84] - -(43) Filter [codegen id : 1] +(42) CometFilter Input [2]: [d_date_sk#19, d_quarter_name#84] Condition : ((isnotnull(d_quarter_name#84) AND (d_quarter_name#84 = 2001Q1)) AND isnotnull(d_date_sk#19)) -(44) Project [codegen id : 1] -Output [1]: [d_date_sk#19] +(43) CometProject Input [2]: [d_date_sk#19, d_quarter_name#84] +Arguments: [d_date_sk#19], [d_date_sk#19] + +(44) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#19] (45) BroadcastExchange Input [1]: [d_date_sk#19] @@ -265,9 +265,9 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Subquery:2 Hosting operator id = 4 Hosting Expression = sr_returned_date_sk#12 IN dynamicpruning#13 BroadcastExchange (50) -+- * Project (49) - +- * Filter (48) - +- * ColumnarToRow (47) ++- * ColumnarToRow (49) + +- CometProject (48) + +- CometFilter (47) +- CometScan parquet spark_catalog.default.date_dim (46) @@ -278,16 +278,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_quarter_name, [2001Q1,2001Q2,2001Q3]), IsNotNull(d_date_sk)] ReadSchema: struct -(47) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#20, d_quarter_name#85] - -(48) Filter [codegen id : 1] +(47) CometFilter Input [2]: [d_date_sk#20, d_quarter_name#85] Condition : (d_quarter_name#85 IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#20)) -(49) Project [codegen id : 1] -Output [1]: [d_date_sk#20] +(48) CometProject Input [2]: [d_date_sk#20, d_quarter_name#85] +Arguments: [d_date_sk#20], [d_date_sk#20] + +(49) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#20] (50) BroadcastExchange Input [1]: [d_date_sk#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/simplified.txt index 2e1e780a7..9f4d67dec 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/simplified.txt @@ -19,39 +19,39 @@ TakeOrderedAndProject [i_item_id,i_item_desc,s_state,store_sales_quantitycount,s BroadcastHashJoin [sr_customer_sk,sr_item_sk,cs_bill_customer_sk,cs_item_sk] Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] BroadcastHashJoin [ss_customer_sk,ss_item_sk,ss_ticket_number,sr_customer_sk,sr_item_sk,sr_ticket_number] - Filter [ss_customer_sk,ss_item_sk,ss_ticket_number,ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_customer_sk,ss_item_sk,ss_ticket_number,ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_quarter_name,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_quarter_name,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Filter [sr_customer_sk,sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [sr_customer_sk,sr_item_sk,sr_ticket_number] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #4 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_quarter_name,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_quarter_name,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] InputAdapter BroadcastExchange #5 WholeStageCodegen (2) - Filter [cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_bill_customer_sk,cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] ReusedSubquery [d_date_sk] #2 InputAdapter @@ -63,14 +63,14 @@ TakeOrderedAndProject [i_item_id,i_item_desc,s_state,store_sales_quantitycount,s InputAdapter BroadcastExchange #6 WholeStageCodegen (6) - Filter [s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_state] InputAdapter BroadcastExchange #7 WholeStageCodegen (7) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/explain.txt index 34421b640..1e9c660c5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/explain.txt @@ -16,31 +16,31 @@ TakeOrderedAndProject (43) : : : : +- * BroadcastHashJoin Inner BuildRight (16) : : : : :- * Project (10) : : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : : :- * Filter (3) - : : : : : : +- * ColumnarToRow (2) + : : : : : :- * ColumnarToRow (3) + : : : : : : +- CometFilter (2) : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : : : : : +- BroadcastExchange (8) - : : : : : +- * Project (7) - : : : : : +- * Filter (6) - : : : : : +- * ColumnarToRow (5) + : : : : : +- * ColumnarToRow (7) + : : : : : +- CometProject (6) + : : : : : +- CometFilter (5) : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (4) : : : : +- BroadcastExchange (15) - : : : : +- * Project (14) - : : : : +- * Filter (13) - : : : : +- * ColumnarToRow (12) + : : : : +- * ColumnarToRow (14) + : : : : +- CometProject (13) + : : : : +- CometFilter (12) : : : : +- CometScan parquet spark_catalog.default.customer (11) : : : +- BroadcastExchange (21) - : : : +- * Filter (20) - : : : +- * ColumnarToRow (19) + : : : +- * ColumnarToRow (20) + : : : +- CometFilter (19) : : : +- CometScan parquet spark_catalog.default.customer_demographics (18) : : +- BroadcastExchange (27) - : : +- * Filter (26) - : : +- * ColumnarToRow (25) + : : +- * ColumnarToRow (26) + : : +- CometFilter (25) : : +- CometScan parquet spark_catalog.default.customer_address (24) : +- ReusedExchange (30) +- BroadcastExchange (36) - +- * Filter (35) - +- * ColumnarToRow (34) + +- * ColumnarToRow (35) + +- CometFilter (34) +- CometScan parquet spark_catalog.default.item (33) @@ -52,12 +52,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#9), dynamicpruningexpression(cs_sol PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 7] +(2) CometFilter Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) -(3) Filter [codegen id : 7] +(3) ColumnarToRow [codegen id : 7] Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) (4) Scan parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] @@ -66,16 +66,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_education_status), EqualTo(cd_gender,F), EqualTo(cd_education_status,Unknown ), IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] - -(6) Filter [codegen id : 1] +(5) CometFilter Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] Condition : ((((isnotnull(cd_gender#12) AND isnotnull(cd_education_status#13)) AND (cd_gender#12 = F)) AND (cd_education_status#13 = Unknown )) AND isnotnull(cd_demo_sk#11)) -(7) Project [codegen id : 1] -Output [2]: [cd_demo_sk#11, cd_dep_count#14] +(6) CometProject Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] +Arguments: [cd_demo_sk#11, cd_dep_count#14], [cd_demo_sk#11, cd_dep_count#14] + +(7) ColumnarToRow [codegen id : 1] +Input [2]: [cd_demo_sk#11, cd_dep_count#14] (8) BroadcastExchange Input [2]: [cd_demo_sk#11, cd_dep_count#14] @@ -98,16 +98,16 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [In(c_birth_month, [1,12,2,6,8,9]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] - -(13) Filter [codegen id : 2] +(12) CometFilter Input [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] Condition : (((c_birth_month#18 IN (1,6,8,9,12,2) AND isnotnull(c_customer_sk#15)) AND isnotnull(c_current_cdemo_sk#16)) AND isnotnull(c_current_addr_sk#17)) -(14) Project [codegen id : 2] -Output [4]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] +(13) CometProject Input [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] +Arguments: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19], [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] + +(14) ColumnarToRow [codegen id : 2] +Input [4]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] (15) BroadcastExchange Input [4]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] @@ -130,12 +130,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] +(19) CometFilter Input [1]: [cd_demo_sk#20] +Condition : isnotnull(cd_demo_sk#20) -(20) Filter [codegen id : 3] +(20) ColumnarToRow [codegen id : 3] Input [1]: [cd_demo_sk#20] -Condition : isnotnull(cd_demo_sk#20) (21) BroadcastExchange Input [1]: [cd_demo_sk#20] @@ -158,12 +158,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [In(ca_state, [IN,MS,ND,NM,OK,VA]), IsNotNull(ca_address_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 4] +(25) CometFilter Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] +Condition : (ca_state#23 IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#21)) -(26) Filter [codegen id : 4] +(26) ColumnarToRow [codegen id : 4] Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] -Condition : (ca_state#23 IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#21)) (27) BroadcastExchange Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] @@ -199,12 +199,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 6] +(34) CometFilter Input [2]: [i_item_sk#26, i_item_id#27] +Condition : isnotnull(i_item_sk#26) -(35) Filter [codegen id : 6] +(35) ColumnarToRow [codegen id : 6] Input [2]: [i_item_sk#26, i_item_id#27] -Condition : isnotnull(i_item_sk#26) (36) BroadcastExchange Input [2]: [i_item_sk#26, i_item_id#27] @@ -250,9 +250,9 @@ Arguments: 100, [ca_country#29 ASC NULLS FIRST, ca_state#30 ASC NULLS FIRST, ca_ Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#9 IN dynamicpruning#10 BroadcastExchange (48) -+- * Project (47) - +- * Filter (46) - +- * ColumnarToRow (45) ++- * ColumnarToRow (47) + +- CometProject (46) + +- CometFilter (45) +- CometScan parquet spark_catalog.default.date_dim (44) @@ -263,16 +263,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#25, d_year#75] - -(46) Filter [codegen id : 1] +(45) CometFilter Input [2]: [d_date_sk#25, d_year#75] Condition : ((isnotnull(d_year#75) AND (d_year#75 = 1998)) AND isnotnull(d_date_sk#25)) -(47) Project [codegen id : 1] -Output [1]: [d_date_sk#25] +(46) CometProject Input [2]: [d_date_sk#25, d_year#75] +Arguments: [d_date_sk#25], [d_date_sk#25] + +(47) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#25] (48) BroadcastExchange Input [1]: [d_date_sk#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/simplified.txt index 723e30a44..47911b9ba 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/simplified.txt @@ -18,54 +18,54 @@ TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,ag BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Project [cd_demo_sk,cd_dep_count] - Filter [cd_gender,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cd_demo_sk,cd_dep_count] + CometFilter [cd_gender,cd_education_status,cd_demo_sk] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - Project [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - Filter [c_birth_month,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometFilter [c_birth_month,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk] InputAdapter BroadcastExchange #6 WholeStageCodegen (4) - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ca_state,ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] InputAdapter ReusedExchange [d_date_sk] #2 InputAdapter BroadcastExchange #7 WholeStageCodegen (6) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19/explain.txt index 8a13a8dc7..999fec838 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19/explain.txt @@ -13,30 +13,30 @@ TakeOrderedAndProject (39) : : : +- * BroadcastHashJoin Inner BuildRight (16) : : : :- * Project (10) : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.date_dim (1) : : : : +- BroadcastExchange (8) - : : : : +- * Filter (7) - : : : : +- * ColumnarToRow (6) + : : : : +- * ColumnarToRow (7) + : : : : +- CometFilter (6) : : : : +- CometScan parquet spark_catalog.default.store_sales (5) : : : +- BroadcastExchange (15) - : : : +- * Project (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) + : : : +- * ColumnarToRow (14) + : : : +- CometProject (13) + : : : +- CometFilter (12) : : : +- CometScan parquet spark_catalog.default.item (11) : : +- BroadcastExchange (21) - : : +- * Filter (20) - : : +- * ColumnarToRow (19) + : : +- * ColumnarToRow (20) + : : +- CometFilter (19) : : +- CometScan parquet spark_catalog.default.customer (18) : +- BroadcastExchange (27) - : +- * Filter (26) - : +- * ColumnarToRow (25) + : +- * ColumnarToRow (26) + : +- CometFilter (25) : +- CometScan parquet spark_catalog.default.customer_address (24) +- BroadcastExchange (33) - +- * Filter (32) - +- * ColumnarToRow (31) + +- * ColumnarToRow (32) + +- CometFilter (31) +- CometScan parquet spark_catalog.default.store (30) @@ -47,16 +47,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 6] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(3) Filter [codegen id : 6] +(2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 1998)) AND isnotnull(d_date_sk#1)) -(4) Project [codegen id : 6] -Output [1]: [d_date_sk#1] +(3) CometProject Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Arguments: [d_date_sk#1], [d_date_sk#1] + +(4) ColumnarToRow [codegen id : 6] +Input [1]: [d_date_sk#1] (5) Scan parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] @@ -66,12 +66,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(true)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] +(6) CometFilter Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] +Condition : ((isnotnull(ss_item_sk#4) AND isnotnull(ss_customer_sk#5)) AND isnotnull(ss_store_sk#6)) -(7) Filter [codegen id : 1] +(7) ColumnarToRow [codegen id : 1] Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_item_sk#4) AND isnotnull(ss_customer_sk#5)) AND isnotnull(ss_store_sk#6)) (8) BroadcastExchange Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] @@ -94,16 +94,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,8), IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] - -(13) Filter [codegen id : 2] +(12) CometFilter Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] Condition : ((isnotnull(i_manager_id#14) AND (i_manager_id#14 = 8)) AND isnotnull(i_item_sk#9)) -(14) Project [codegen id : 2] -Output [5]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] +(13) CometProject Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] +Arguments: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13], [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] + +(14) ColumnarToRow [codegen id : 2] +Input [5]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] (15) BroadcastExchange Input [5]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] @@ -126,12 +126,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] +(19) CometFilter Input [2]: [c_customer_sk#15, c_current_addr_sk#16] +Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_current_addr_sk#16)) -(20) Filter [codegen id : 3] +(20) ColumnarToRow [codegen id : 3] Input [2]: [c_customer_sk#15, c_current_addr_sk#16] -Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_current_addr_sk#16)) (21) BroadcastExchange Input [2]: [c_customer_sk#15, c_current_addr_sk#16] @@ -154,12 +154,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_zip)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 4] +(25) CometFilter Input [2]: [ca_address_sk#17, ca_zip#18] +Condition : (isnotnull(ca_address_sk#17) AND isnotnull(ca_zip#18)) -(26) Filter [codegen id : 4] +(26) ColumnarToRow [codegen id : 4] Input [2]: [ca_address_sk#17, ca_zip#18] -Condition : (isnotnull(ca_address_sk#17) AND isnotnull(ca_zip#18)) (27) BroadcastExchange Input [2]: [ca_address_sk#17, ca_zip#18] @@ -182,12 +182,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_zip), IsNotNull(s_store_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 5] +(31) CometFilter Input [2]: [s_store_sk#19, s_zip#20] +Condition : (isnotnull(s_zip#20) AND isnotnull(s_store_sk#19)) -(32) Filter [codegen id : 5] +(32) ColumnarToRow [codegen id : 5] Input [2]: [s_store_sk#19, s_zip#20] -Condition : (isnotnull(s_zip#20) AND isnotnull(s_store_sk#19)) (33) BroadcastExchange Input [2]: [s_store_sk#19, s_zip#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19/simplified.txt index 85e4886d7..c2f5d1a87 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19/simplified.txt @@ -15,44 +15,44 @@ TakeOrderedAndProject [ext_price,brand,brand_id,i_manufact_id,i_manufact] BroadcastHashJoin [ss_item_sk,i_item_sk] Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price] BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_moy,d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #2 WholeStageCodegen (1) - Filter [ss_item_sk,ss_customer_sk,ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Project [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] - Filter [i_manager_id,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] + CometFilter [i_manager_id,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] InputAdapter BroadcastExchange #4 WholeStageCodegen (3) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_current_addr_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] InputAdapter BroadcastExchange #5 WholeStageCodegen (4) - Filter [ca_address_sk,ca_zip] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ca_address_sk,ca_zip] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] InputAdapter BroadcastExchange #6 WholeStageCodegen (5) - Filter [s_zip,s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_zip,s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_zip] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/explain.txt index 4bfb19a38..1215adf3f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/explain.txt @@ -1,41 +1,40 @@ == Physical Plan == -* Sort (37) -+- Exchange (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Project (23) - : +- * BroadcastHashJoin Inner BuildRight (22) - : :- * HashAggregate (16) - : : +- Exchange (15) - : : +- * HashAggregate (14) - : : +- * Project (13) - : : +- * BroadcastHashJoin Inner BuildRight (12) - : : :- Union (7) - : : : :- * Project (3) - : : : : +- * ColumnarToRow (2) - : : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : : +- * Project (6) - : : : +- * ColumnarToRow (5) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (4) - : : +- BroadcastExchange (11) - : : +- * Filter (10) - : : +- * ColumnarToRow (9) - : : +- CometScan parquet spark_catalog.default.date_dim (8) - : +- BroadcastExchange (21) - : +- * Project (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) - : +- CometScan parquet spark_catalog.default.date_dim (17) - +- BroadcastExchange (33) - +- * Project (32) - +- * BroadcastHashJoin Inner BuildRight (31) - :- * HashAggregate (25) - : +- ReusedExchange (24) - +- BroadcastExchange (30) - +- * Project (29) - +- * Filter (28) - +- * ColumnarToRow (27) - +- CometScan parquet spark_catalog.default.date_dim (26) +* Sort (36) ++- Exchange (35) + +- * Project (34) + +- * BroadcastHashJoin Inner BuildRight (33) + :- * Project (22) + : +- * BroadcastHashJoin Inner BuildRight (21) + : :- * HashAggregate (15) + : : +- Exchange (14) + : : +- * HashAggregate (13) + : : +- * Project (12) + : : +- * BroadcastHashJoin Inner BuildRight (11) + : : :- * ColumnarToRow (6) + : : : +- CometUnion (5) + : : : :- CometProject (2) + : : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : : +- CometProject (4) + : : : +- CometScan parquet spark_catalog.default.catalog_sales (3) + : : +- BroadcastExchange (10) + : : +- * ColumnarToRow (9) + : : +- CometFilter (8) + : : +- CometScan parquet spark_catalog.default.date_dim (7) + : +- BroadcastExchange (20) + : +- * ColumnarToRow (19) + : +- CometProject (18) + : +- CometFilter (17) + : +- CometScan parquet spark_catalog.default.date_dim (16) + +- BroadcastExchange (32) + +- * Project (31) + +- * BroadcastHashJoin Inner BuildRight (30) + :- * HashAggregate (24) + : +- ReusedExchange (23) + +- BroadcastExchange (29) + +- * ColumnarToRow (28) + +- CometProject (27) + +- CometFilter (26) + +- CometScan parquet spark_catalog.default.date_dim (25) (1) Scan parquet spark_catalog.default.web_sales @@ -45,168 +44,167 @@ Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ws_sold_date_sk#2)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] +(2) CometProject Input [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] +Arguments: [sold_date_sk#3, sales_price#4], [ws_sold_date_sk#2 AS sold_date_sk#3, ws_ext_sales_price#1 AS sales_price#4] -(3) Project [codegen id : 1] -Output [2]: [ws_sold_date_sk#2 AS sold_date_sk#3, ws_ext_sales_price#1 AS sales_price#4] -Input [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] - -(4) Scan parquet spark_catalog.default.catalog_sales +(3) Scan parquet spark_catalog.default.catalog_sales Output [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(cs_sold_date_sk#6)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 2] +(4) CometProject Input [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] +Arguments: [sold_date_sk#7, sales_price#8], [cs_sold_date_sk#6 AS sold_date_sk#7, cs_ext_sales_price#5 AS sales_price#8] -(6) Project [codegen id : 2] -Output [2]: [cs_sold_date_sk#6 AS sold_date_sk#7, cs_ext_sales_price#5 AS sales_price#8] -Input [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] +(5) CometUnion +Child 0 Input [2]: [sold_date_sk#3, sales_price#4] +Child 1 Input [2]: [sold_date_sk#7, sales_price#8] -(7) Union +(6) ColumnarToRow [codegen id : 2] +Input [2]: [sold_date_sk#3, sales_price#4] -(8) Scan parquet spark_catalog.default.date_dim +(7) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk), IsNotNull(d_week_seq)] ReadSchema: struct -(9) ColumnarToRow [codegen id : 3] +(8) CometFilter Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] +Condition : (isnotnull(d_date_sk#9) AND isnotnull(d_week_seq#10)) -(10) Filter [codegen id : 3] +(9) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] -Condition : (isnotnull(d_date_sk#9) AND isnotnull(d_week_seq#10)) -(11) BroadcastExchange +(10) BroadcastExchange Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] -(12) BroadcastHashJoin [codegen id : 4] +(11) BroadcastHashJoin [codegen id : 2] Left keys [1]: [sold_date_sk#3] Right keys [1]: [d_date_sk#9] Join type: Inner Join condition: None -(13) Project [codegen id : 4] +(12) Project [codegen id : 2] Output [3]: [sales_price#4, d_week_seq#10, d_day_name#11] Input [5]: [sold_date_sk#3, sales_price#4, d_date_sk#9, d_week_seq#10, d_day_name#11] -(14) HashAggregate [codegen id : 4] +(13) HashAggregate [codegen id : 2] Input [3]: [sales_price#4, d_week_seq#10, d_day_name#11] Keys [1]: [d_week_seq#10] Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] Aggregate Attributes [7]: [sum#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18] Results [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] -(15) Exchange +(14) Exchange Input [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] Arguments: hashpartitioning(d_week_seq#10, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(16) HashAggregate [codegen id : 12] +(15) HashAggregate [codegen id : 8] Input [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] Keys [1]: [d_week_seq#10] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))#32] Results [8]: [d_week_seq#10, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END))#26,17,2) AS sun_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END))#27,17,2) AS mon_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END))#28,17,2) AS tue_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#29,17,2) AS wed_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END))#30,17,2) AS thu_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END))#31,17,2) AS fri_sales#38, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))#32,17,2) AS sat_sales#39] -(17) Scan parquet spark_catalog.default.date_dim +(16) Scan parquet spark_catalog.default.date_dim Output [2]: [d_week_seq#40, d_year#41] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_week_seq)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 5] -Input [2]: [d_week_seq#40, d_year#41] - -(19) Filter [codegen id : 5] +(17) CometFilter Input [2]: [d_week_seq#40, d_year#41] Condition : ((isnotnull(d_year#41) AND (d_year#41 = 2001)) AND isnotnull(d_week_seq#40)) -(20) Project [codegen id : 5] -Output [1]: [d_week_seq#40] +(18) CometProject Input [2]: [d_week_seq#40, d_year#41] +Arguments: [d_week_seq#40], [d_week_seq#40] -(21) BroadcastExchange +(19) ColumnarToRow [codegen id : 3] +Input [1]: [d_week_seq#40] + +(20) BroadcastExchange Input [1]: [d_week_seq#40] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(22) BroadcastHashJoin [codegen id : 12] +(21) BroadcastHashJoin [codegen id : 8] Left keys [1]: [d_week_seq#10] Right keys [1]: [d_week_seq#40] Join type: Inner Join condition: None -(23) Project [codegen id : 12] +(22) Project [codegen id : 8] Output [8]: [d_week_seq#10 AS d_week_seq1#42, sun_sales#33 AS sun_sales1#43, mon_sales#34 AS mon_sales1#44, tue_sales#35 AS tue_sales1#45, wed_sales#36 AS wed_sales1#46, thu_sales#37 AS thu_sales1#47, fri_sales#38 AS fri_sales1#48, sat_sales#39 AS sat_sales1#49] Input [9]: [d_week_seq#10, sun_sales#33, mon_sales#34, tue_sales#35, wed_sales#36, thu_sales#37, fri_sales#38, sat_sales#39, d_week_seq#40] -(24) ReusedExchange [Reuses operator id: 15] +(23) ReusedExchange [Reuses operator id: 14] Output [8]: [d_week_seq#50, sum#51, sum#52, sum#53, sum#54, sum#55, sum#56, sum#57] -(25) HashAggregate [codegen id : 11] +(24) HashAggregate [codegen id : 7] Input [8]: [d_week_seq#50, sum#51, sum#52, sum#53, sum#54, sum#55, sum#56, sum#57] Keys [1]: [d_week_seq#50] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#58 = Sunday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Monday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Tuesday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Wednesday) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Thursday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Friday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Saturday ) THEN sales_price#59 END))] Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#58 = Sunday ) THEN sales_price#59 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Monday ) THEN sales_price#59 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Tuesday ) THEN sales_price#59 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Wednesday) THEN sales_price#59 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Thursday ) THEN sales_price#59 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Friday ) THEN sales_price#59 END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Saturday ) THEN sales_price#59 END))#32] Results [8]: [d_week_seq#50, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Sunday ) THEN sales_price#59 END))#26,17,2) AS sun_sales#60, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Monday ) THEN sales_price#59 END))#27,17,2) AS mon_sales#61, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Tuesday ) THEN sales_price#59 END))#28,17,2) AS tue_sales#62, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Wednesday) THEN sales_price#59 END))#29,17,2) AS wed_sales#63, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Thursday ) THEN sales_price#59 END))#30,17,2) AS thu_sales#64, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Friday ) THEN sales_price#59 END))#31,17,2) AS fri_sales#65, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Saturday ) THEN sales_price#59 END))#32,17,2) AS sat_sales#66] -(26) Scan parquet spark_catalog.default.date_dim +(25) Scan parquet spark_catalog.default.date_dim Output [2]: [d_week_seq#67, d_year#68] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_week_seq)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 10] -Input [2]: [d_week_seq#67, d_year#68] - -(28) Filter [codegen id : 10] +(26) CometFilter Input [2]: [d_week_seq#67, d_year#68] Condition : ((isnotnull(d_year#68) AND (d_year#68 = 2002)) AND isnotnull(d_week_seq#67)) -(29) Project [codegen id : 10] -Output [1]: [d_week_seq#67] +(27) CometProject Input [2]: [d_week_seq#67, d_year#68] +Arguments: [d_week_seq#67], [d_week_seq#67] + +(28) ColumnarToRow [codegen id : 6] +Input [1]: [d_week_seq#67] -(30) BroadcastExchange +(29) BroadcastExchange Input [1]: [d_week_seq#67] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] -(31) BroadcastHashJoin [codegen id : 11] +(30) BroadcastHashJoin [codegen id : 7] Left keys [1]: [d_week_seq#50] Right keys [1]: [d_week_seq#67] Join type: Inner Join condition: None -(32) Project [codegen id : 11] +(31) Project [codegen id : 7] Output [8]: [d_week_seq#50 AS d_week_seq2#69, sun_sales#60 AS sun_sales2#70, mon_sales#61 AS mon_sales2#71, tue_sales#62 AS tue_sales2#72, wed_sales#63 AS wed_sales2#73, thu_sales#64 AS thu_sales2#74, fri_sales#65 AS fri_sales2#75, sat_sales#66 AS sat_sales2#76] Input [9]: [d_week_seq#50, sun_sales#60, mon_sales#61, tue_sales#62, wed_sales#63, thu_sales#64, fri_sales#65, sat_sales#66, d_week_seq#67] -(33) BroadcastExchange +(32) BroadcastExchange Input [8]: [d_week_seq2#69, sun_sales2#70, mon_sales2#71, tue_sales2#72, wed_sales2#73, thu_sales2#74, fri_sales2#75, sat_sales2#76] Arguments: HashedRelationBroadcastMode(List(cast((input[0, int, true] - 53) as bigint)),false), [plan_id=5] -(34) BroadcastHashJoin [codegen id : 12] +(33) BroadcastHashJoin [codegen id : 8] Left keys [1]: [d_week_seq1#42] Right keys [1]: [(d_week_seq2#69 - 53)] Join type: Inner Join condition: None -(35) Project [codegen id : 12] +(34) Project [codegen id : 8] Output [8]: [d_week_seq1#42, round((sun_sales1#43 / sun_sales2#70), 2) AS round((sun_sales1 / sun_sales2), 2)#77, round((mon_sales1#44 / mon_sales2#71), 2) AS round((mon_sales1 / mon_sales2), 2)#78, round((tue_sales1#45 / tue_sales2#72), 2) AS round((tue_sales1 / tue_sales2), 2)#79, round((wed_sales1#46 / wed_sales2#73), 2) AS round((wed_sales1 / wed_sales2), 2)#80, round((thu_sales1#47 / thu_sales2#74), 2) AS round((thu_sales1 / thu_sales2), 2)#81, round((fri_sales1#48 / fri_sales2#75), 2) AS round((fri_sales1 / fri_sales2), 2)#82, round((sat_sales1#49 / sat_sales2#76), 2) AS round((sat_sales1 / sat_sales2), 2)#83] Input [16]: [d_week_seq1#42, sun_sales1#43, mon_sales1#44, tue_sales1#45, wed_sales1#46, thu_sales1#47, fri_sales1#48, sat_sales1#49, d_week_seq2#69, sun_sales2#70, mon_sales2#71, tue_sales2#72, wed_sales2#73, thu_sales2#74, fri_sales2#75, sat_sales2#76] -(36) Exchange +(35) Exchange Input [8]: [d_week_seq1#42, round((sun_sales1 / sun_sales2), 2)#77, round((mon_sales1 / mon_sales2), 2)#78, round((tue_sales1 / tue_sales2), 2)#79, round((wed_sales1 / wed_sales2), 2)#80, round((thu_sales1 / thu_sales2), 2)#81, round((fri_sales1 / fri_sales2), 2)#82, round((sat_sales1 / sat_sales2), 2)#83] Arguments: rangepartitioning(d_week_seq1#42 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(37) Sort [codegen id : 13] +(36) Sort [codegen id : 9] Input [8]: [d_week_seq1#42, round((sun_sales1 / sun_sales2), 2)#77, round((mon_sales1 / mon_sales2), 2)#78, round((tue_sales1 / tue_sales2), 2)#79, round((wed_sales1 / wed_sales2), 2)#80, round((thu_sales1 / thu_sales2), 2)#81, round((fri_sales1 / fri_sales2), 2)#82, round((sat_sales1 / sat_sales2), 2)#83] Arguments: [d_week_seq1#42 ASC NULLS FIRST], true, 0 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/simplified.txt index 0008304e8..8856ce80d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/simplified.txt @@ -1,8 +1,8 @@ -WholeStageCodegen (13) +WholeStageCodegen (9) Sort [d_week_seq1] InputAdapter Exchange [d_week_seq1] #1 - WholeStageCodegen (12) + WholeStageCodegen (8) Project [d_week_seq1,sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2] BroadcastHashJoin [d_week_seq1,d_week_seq2] Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] @@ -10,40 +10,35 @@ WholeStageCodegen (13) HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] InputAdapter Exchange [d_week_seq] #2 - WholeStageCodegen (4) + WholeStageCodegen (2) HashAggregate [d_week_seq,d_day_name,sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] Project [sales_price,d_week_seq,d_day_name] BroadcastHashJoin [sold_date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (1) - Project [ws_sold_date_sk,ws_ext_sales_price] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] - WholeStageCodegen (2) - Project [cs_sold_date_sk,cs_ext_sales_price] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] + ColumnarToRow + InputAdapter + CometUnion + CometProject [ws_sold_date_sk,ws_ext_sales_price] [sold_date_sk,sales_price] + CometScan parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] + CometProject [cs_sold_date_sk,cs_ext_sales_price] [sold_date_sk,sales_price] + CometScan parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] InputAdapter BroadcastExchange #3 - WholeStageCodegen (3) - Filter [d_date_sk,d_week_seq] - ColumnarToRow - InputAdapter + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_week_seq] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] InputAdapter BroadcastExchange #4 - WholeStageCodegen (5) - Project [d_week_seq] - Filter [d_year,d_week_seq] - ColumnarToRow - InputAdapter + WholeStageCodegen (3) + ColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_year,d_week_seq] CometScan parquet spark_catalog.default.date_dim [d_week_seq,d_year] InputAdapter BroadcastExchange #5 - WholeStageCodegen (11) + WholeStageCodegen (7) Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] BroadcastHashJoin [d_week_seq,d_week_seq] HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] @@ -51,9 +46,9 @@ WholeStageCodegen (13) ReusedExchange [d_week_seq,sum,sum,sum,sum,sum,sum,sum] #2 InputAdapter BroadcastExchange #6 - WholeStageCodegen (10) - Project [d_week_seq] - Filter [d_year,d_week_seq] - ColumnarToRow - InputAdapter + WholeStageCodegen (6) + ColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_year,d_week_seq] CometScan parquet spark_catalog.default.date_dim [d_week_seq,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/explain.txt index f5455e62f..333ef218c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/explain.txt @@ -11,12 +11,12 @@ TakeOrderedAndProject (20) +- * BroadcastHashJoin Inner BuildRight (11) :- * Project (9) : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) + : :- * ColumnarToRow (3) + : : +- CometFilter (2) : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) + : +- * ColumnarToRow (6) + : +- CometFilter (5) : +- CometScan parquet spark_catalog.default.item (4) +- ReusedExchange (10) @@ -29,12 +29,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sol PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] +Condition : isnotnull(cs_item_sk#1) -(3) Filter [codegen id : 3] +(3) ColumnarToRow [codegen id : 3] Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Condition : isnotnull(cs_item_sk#1) (4) Scan parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] @@ -43,12 +43,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Condition : (i_category#10 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (i_category#10 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) (7) BroadcastExchange Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] @@ -119,9 +119,9 @@ Arguments: 100, [i_category#10 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_ite Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 BroadcastExchange (25) -+- * Project (24) - +- * Filter (23) - +- * ColumnarToRow (22) ++- * ColumnarToRow (24) + +- CometProject (23) + +- CometFilter (22) +- CometScan parquet spark_catalog.default.date_dim (21) @@ -132,16 +132,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(22) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#11, d_date#19] - -(23) Filter [codegen id : 1] +(22) CometFilter Input [2]: [d_date_sk#11, d_date#19] Condition : (((isnotnull(d_date#19) AND (d_date#19 >= 1999-02-22)) AND (d_date#19 <= 1999-03-24)) AND isnotnull(d_date_sk#11)) -(24) Project [codegen id : 1] -Output [1]: [d_date_sk#11] +(23) CometProject Input [2]: [d_date_sk#11, d_date#19] +Arguments: [d_date_sk#11], [d_date_sk#11] + +(24) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#11] (25) BroadcastExchange Input [1]: [d_date_sk#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/simplified.txt index fbbf149b5..52c42bdf2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/simplified.txt @@ -17,24 +17,24 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter BroadcastExchange #4 WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_category,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] InputAdapter ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/explain.txt index a5417a5b6..77f17c7f3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/explain.txt @@ -10,17 +10,17 @@ TakeOrderedAndProject (24) : +- * BroadcastHashJoin Inner BuildRight (15) : :- * Project (9) : : +- * BroadcastHashJoin Inner BuildRight (8) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) + : : :- * ColumnarToRow (3) + : : : +- CometFilter (2) : : : +- CometScan parquet spark_catalog.default.inventory (1) : : +- BroadcastExchange (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) + : : +- * ColumnarToRow (6) + : : +- CometFilter (5) : : +- CometScan parquet spark_catalog.default.warehouse (4) : +- BroadcastExchange (14) - : +- * Project (13) - : +- * Filter (12) - : +- * ColumnarToRow (11) + : +- * ColumnarToRow (13) + : +- CometProject (12) + : +- CometFilter (11) : +- CometScan parquet spark_catalog.default.item (10) +- ReusedExchange (17) @@ -33,12 +33,12 @@ PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_s PushedFilters: [IsNotNull(inv_warehouse_sk), IsNotNull(inv_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Condition : (isnotnull(inv_warehouse_sk#2) AND isnotnull(inv_item_sk#1)) -(3) Filter [codegen id : 4] +(3) ColumnarToRow [codegen id : 4] Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Condition : (isnotnull(inv_warehouse_sk#2) AND isnotnull(inv_item_sk#1)) (4) Scan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] @@ -47,12 +47,12 @@ Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Condition : isnotnull(w_warehouse_sk#6) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Condition : isnotnull(w_warehouse_sk#6) (7) BroadcastExchange Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] @@ -75,16 +75,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [i_item_sk#8, i_item_id#9, i_current_price#10] - -(12) Filter [codegen id : 2] +(11) CometFilter Input [3]: [i_item_sk#8, i_item_id#9, i_current_price#10] Condition : (((isnotnull(i_current_price#10) AND (i_current_price#10 >= 0.99)) AND (i_current_price#10 <= 1.49)) AND isnotnull(i_item_sk#8)) -(13) Project [codegen id : 2] -Output [2]: [i_item_sk#8, i_item_id#9] +(12) CometProject Input [3]: [i_item_sk#8, i_item_id#9, i_current_price#10] +Arguments: [i_item_sk#8, i_item_id#9], [i_item_sk#8, i_item_id#9] + +(13) ColumnarToRow [codegen id : 2] +Input [2]: [i_item_sk#8, i_item_id#9] (14) BroadcastExchange Input [2]: [i_item_sk#8, i_item_id#9] @@ -143,8 +143,8 @@ Arguments: 100, [w_warehouse_name#7 ASC NULLS FIRST, i_item_id#9 ASC NULLS FIRST Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 BroadcastExchange (28) -+- * Filter (27) - +- * ColumnarToRow (26) ++- * ColumnarToRow (27) + +- CometFilter (26) +- CometScan parquet spark_catalog.default.date_dim (25) @@ -155,12 +155,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 1] +(26) CometFilter Input [2]: [d_date_sk#11, d_date#12] +Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-02-10)) AND (d_date#12 <= 2000-04-10)) AND isnotnull(d_date_sk#11)) -(27) Filter [codegen id : 1] +(27) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#11, d_date#12] -Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-02-10)) AND (d_date#12 <= 2000-04-10)) AND isnotnull(d_date_sk#11)) (28) BroadcastExchange Input [2]: [d_date_sk#11, d_date#12] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/simplified.txt index 912965257..e20755e12 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/simplified.txt @@ -12,31 +12,31 @@ TakeOrderedAndProject [w_warehouse_name,i_item_id,inv_before,inv_after] BroadcastHashJoin [inv_item_sk,i_item_sk] Project [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name] BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Filter [inv_warehouse_sk,inv_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [inv_warehouse_sk,inv_item_sk] CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_date,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [w_warehouse_sk] CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - Project [i_item_sk,i_item_id] - Filter [i_current_price,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_item_sk,i_item_id] + CometFilter [i_current_price,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] InputAdapter ReusedExchange [d_date_sk,d_date] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/explain.txt index c7f0a8f49..9f5771fed 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/explain.txt @@ -10,17 +10,17 @@ TakeOrderedAndProject (23) : +- * BroadcastHashJoin Inner BuildRight (11) : :- * Project (6) : : +- * BroadcastHashJoin Inner BuildRight (5) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) + : : :- * ColumnarToRow (3) + : : : +- CometFilter (2) : : : +- CometScan parquet spark_catalog.default.inventory (1) : : +- ReusedExchange (4) : +- BroadcastExchange (10) - : +- * Filter (9) - : +- * ColumnarToRow (8) + : +- * ColumnarToRow (9) + : +- CometFilter (8) : +- CometScan parquet spark_catalog.default.item (7) +- BroadcastExchange (16) - +- * Filter (15) - +- * ColumnarToRow (14) + +- * ColumnarToRow (15) + +- CometFilter (14) +- CometScan parquet spark_catalog.default.warehouse (13) @@ -32,12 +32,12 @@ PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_s PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) -(3) Filter [codegen id : 4] +(3) ColumnarToRow [codegen id : 4] Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) (4) ReusedExchange [Reuses operator id: 28] Output [1]: [d_date_sk#6] @@ -59,12 +59,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] +(8) CometFilter Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Condition : isnotnull(i_item_sk#7) -(9) Filter [codegen id : 2] +(9) ColumnarToRow [codegen id : 2] Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Condition : isnotnull(i_item_sk#7) (10) BroadcastExchange Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] @@ -87,12 +87,12 @@ Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] +(14) CometFilter Input [1]: [w_warehouse_sk#12] +Condition : isnotnull(w_warehouse_sk#12) -(15) Filter [codegen id : 3] +(15) ColumnarToRow [codegen id : 3] Input [1]: [w_warehouse_sk#12] -Condition : isnotnull(w_warehouse_sk#12) (16) BroadcastExchange Input [1]: [w_warehouse_sk#12] @@ -138,9 +138,9 @@ Arguments: 100, [qoh#23 ASC NULLS FIRST, i_product_name#13 ASC NULLS FIRST, i_br Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 BroadcastExchange (28) -+- * Project (27) - +- * Filter (26) - +- * ColumnarToRow (25) ++- * ColumnarToRow (27) + +- CometProject (26) + +- CometFilter (25) +- CometScan parquet spark_catalog.default.date_dim (24) @@ -151,16 +151,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#6, d_month_seq#24] - -(26) Filter [codegen id : 1] +(25) CometFilter Input [2]: [d_date_sk#6, d_month_seq#24] Condition : (((isnotnull(d_month_seq#24) AND (d_month_seq#24 >= 1200)) AND (d_month_seq#24 <= 1211)) AND isnotnull(d_date_sk#6)) -(27) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(26) CometProject Input [2]: [d_date_sk#6, d_month_seq#24] +Arguments: [d_date_sk#6], [d_date_sk#6] + +(27) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#6] (28) BroadcastExchange Input [1]: [d_date_sk#6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/simplified.txt index 495490277..92714bb02 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/simplified.txt @@ -12,31 +12,31 @@ TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] BroadcastHashJoin [inv_item_sk,i_item_sk] Project [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] BroadcastHashJoin [inv_date_sk,d_date_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [inv_item_sk,inv_warehouse_sk] CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_month_seq,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter ReusedExchange [d_date_sk] #2 InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] InputAdapter BroadcastExchange #4 WholeStageCodegen (3) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [w_warehouse_sk] CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/explain.txt index 2f5140f7f..328a8d353 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/explain.txt @@ -23,13 +23,13 @@ : : : +- * BroadcastHashJoin Inner BuildRight (13) : : : :- * Project (8) : : : : +- * BroadcastHashJoin Inner BuildRight (7) - : : : : :- * Filter (5) - : : : : : +- * ColumnarToRow (4) + : : : : :- * ColumnarToRow (5) + : : : : : +- CometFilter (4) : : : : : +- CometScan parquet spark_catalog.default.store_sales (3) : : : : +- ReusedExchange (6) : : : +- BroadcastExchange (12) - : : : +- * Filter (11) - : : : +- * ColumnarToRow (10) + : : : +- * ColumnarToRow (11) + : : : +- CometFilter (10) : : : +- CometScan parquet spark_catalog.default.item (9) : : +- * Sort (40) : : +- * Project (39) @@ -39,13 +39,13 @@ : : +- * HashAggregate (35) : : +- * Project (34) : : +- * BroadcastHashJoin Inner BuildRight (33) - : : :- * Project (28) - : : : +- * Filter (27) - : : : +- * ColumnarToRow (26) + : : :- * ColumnarToRow (28) + : : : +- CometProject (27) + : : : +- CometFilter (26) : : : +- CometScan parquet spark_catalog.default.store_sales (25) : : +- BroadcastExchange (32) - : : +- * Filter (31) - : : +- * ColumnarToRow (30) + : : +- * ColumnarToRow (31) + : : +- CometFilter (30) : : +- CometScan parquet spark_catalog.default.customer (29) : +- ReusedExchange (43) +- * Project (62) @@ -85,12 +85,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(4) ColumnarToRow [codegen id : 3] +(4) CometFilter Input [2]: [ss_item_sk#7, ss_sold_date_sk#8] +Condition : isnotnull(ss_item_sk#7) -(5) Filter [codegen id : 3] +(5) ColumnarToRow [codegen id : 3] Input [2]: [ss_item_sk#7, ss_sold_date_sk#8] -Condition : isnotnull(ss_item_sk#7) (6) ReusedExchange [Reuses operator id: 76] Output [2]: [d_date_sk#10, d_date#11] @@ -112,12 +112,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(10) ColumnarToRow [codegen id : 2] +(10) CometFilter Input [2]: [i_item_sk#12, i_item_desc#13] +Condition : isnotnull(i_item_sk#12) -(11) Filter [codegen id : 2] +(11) ColumnarToRow [codegen id : 2] Input [2]: [i_item_sk#12, i_item_desc#13] -Condition : isnotnull(i_item_sk#12) (12) BroadcastExchange Input [2]: [i_item_sk#12, i_item_desc#13] @@ -188,16 +188,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 8] -Input [4]: [ss_customer_sk#20, ss_quantity#21, ss_sales_price#22, ss_sold_date_sk#23] - -(27) Filter [codegen id : 8] +(26) CometFilter Input [4]: [ss_customer_sk#20, ss_quantity#21, ss_sales_price#22, ss_sold_date_sk#23] Condition : isnotnull(ss_customer_sk#20) -(28) Project [codegen id : 8] -Output [3]: [ss_customer_sk#20, ss_quantity#21, ss_sales_price#22] +(27) CometProject Input [4]: [ss_customer_sk#20, ss_quantity#21, ss_sales_price#22, ss_sold_date_sk#23] +Arguments: [ss_customer_sk#20, ss_quantity#21, ss_sales_price#22], [ss_customer_sk#20, ss_quantity#21, ss_sales_price#22] + +(28) ColumnarToRow [codegen id : 8] +Input [3]: [ss_customer_sk#20, ss_quantity#21, ss_sales_price#22] (29) Scan parquet spark_catalog.default.customer Output [1]: [c_customer_sk#24] @@ -206,12 +206,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 7] +(30) CometFilter Input [1]: [c_customer_sk#24] +Condition : isnotnull(c_customer_sk#24) -(31) Filter [codegen id : 7] +(31) ColumnarToRow [codegen id : 7] Input [1]: [c_customer_sk#24] -Condition : isnotnull(c_customer_sk#24) (32) BroadcastExchange Input [1]: [c_customer_sk#24] @@ -380,9 +380,9 @@ Results [1]: [sum(sales#34)#54 AS sum(sales)#55] Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 BroadcastExchange (71) -+- * Project (70) - +- * Filter (69) - +- * ColumnarToRow (68) ++- * ColumnarToRow (70) + +- CometProject (69) + +- CometFilter (68) +- CometScan parquet spark_catalog.default.date_dim (67) @@ -393,16 +393,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(68) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#33, d_year#56, d_moy#57] - -(69) Filter [codegen id : 1] +(68) CometFilter Input [3]: [d_date_sk#33, d_year#56, d_moy#57] Condition : ((((isnotnull(d_year#56) AND isnotnull(d_moy#57)) AND (d_year#56 = 2000)) AND (d_moy#57 = 2)) AND isnotnull(d_date_sk#33)) -(70) Project [codegen id : 1] -Output [1]: [d_date_sk#33] +(69) CometProject Input [3]: [d_date_sk#33, d_year#56, d_moy#57] +Arguments: [d_date_sk#33], [d_date_sk#33] + +(70) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#33] (71) BroadcastExchange Input [1]: [d_date_sk#33] @@ -410,9 +410,9 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Subquery:2 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 BroadcastExchange (76) -+- * Project (75) - +- * Filter (74) - +- * ColumnarToRow (73) ++- * ColumnarToRow (75) + +- CometProject (74) + +- CometFilter (73) +- CometScan parquet spark_catalog.default.date_dim (72) @@ -423,16 +423,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(73) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#10, d_date#11, d_year#58] - -(74) Filter [codegen id : 1] +(73) CometFilter Input [3]: [d_date_sk#10, d_date#11, d_year#58] Condition : (d_year#58 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#10)) -(75) Project [codegen id : 1] -Output [2]: [d_date_sk#10, d_date#11] +(74) CometProject Input [3]: [d_date_sk#10, d_date#11, d_year#58] +Arguments: [d_date_sk#10, d_date#11], [d_date_sk#10, d_date#11] + +(75) ColumnarToRow [codegen id : 1] +Input [2]: [d_date_sk#10, d_date#11] (76) BroadcastExchange Input [2]: [d_date_sk#10, d_date#11] @@ -449,8 +449,8 @@ Subquery:3 Hosting operator id = 38 Hosting Expression = Subquery scalar-subquer +- * BroadcastHashJoin Inner BuildRight (84) :- * Project (82) : +- * BroadcastHashJoin Inner BuildRight (81) - : :- * Filter (79) - : : +- * ColumnarToRow (78) + : :- * ColumnarToRow (79) + : : +- CometFilter (78) : : +- CometScan parquet spark_catalog.default.store_sales (77) : +- ReusedExchange (80) +- ReusedExchange (83) @@ -464,12 +464,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#62), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(78) ColumnarToRow [codegen id : 3] +(78) CometFilter Input [4]: [ss_customer_sk#59, ss_quantity#60, ss_sales_price#61, ss_sold_date_sk#62] +Condition : isnotnull(ss_customer_sk#59) -(79) Filter [codegen id : 3] +(79) ColumnarToRow [codegen id : 3] Input [4]: [ss_customer_sk#59, ss_quantity#60, ss_sales_price#61, ss_sold_date_sk#62] -Condition : isnotnull(ss_customer_sk#59) (80) ReusedExchange [Reuses operator id: 32] Output [1]: [c_customer_sk#64] @@ -535,9 +535,9 @@ Results [1]: [max(csales#71)#74 AS tpcds_cmax#75] Subquery:4 Hosting operator id = 77 Hosting Expression = ss_sold_date_sk#62 IN dynamicpruning#63 BroadcastExchange (96) -+- * Project (95) - +- * Filter (94) - +- * ColumnarToRow (93) ++- * ColumnarToRow (95) + +- CometProject (94) + +- CometFilter (93) +- CometScan parquet spark_catalog.default.date_dim (92) @@ -548,16 +548,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(93) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#65, d_year#76] - -(94) Filter [codegen id : 1] +(93) CometFilter Input [2]: [d_date_sk#65, d_year#76] Condition : (d_year#76 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#65)) -(95) Project [codegen id : 1] -Output [1]: [d_date_sk#65] +(94) CometProject Input [2]: [d_date_sk#65, d_year#76] +Arguments: [d_date_sk#65], [d_date_sk#65] + +(95) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#65] (96) BroadcastExchange Input [1]: [d_date_sk#65] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/simplified.txt index 38098d6a6..0ec56d0e7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/simplified.txt @@ -25,10 +25,10 @@ WholeStageCodegen (24) SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_moy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #4 @@ -44,26 +44,26 @@ WholeStageCodegen (24) BroadcastHashJoin [ss_item_sk,i_item_sk] Project [ss_item_sk,d_date] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #6 WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_date] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] InputAdapter ReusedExchange [d_date_sk,d_date] #6 InputAdapter BroadcastExchange #7 WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] InputAdapter WholeStageCodegen (9) @@ -86,17 +86,17 @@ WholeStageCodegen (24) BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #4 BroadcastExchange #12 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [c_customer_sk] #9 @@ -109,17 +109,17 @@ WholeStageCodegen (24) HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] Project [ss_quantity,ss_sales_price,c_customer_sk] BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_quantity,ss_sales_price] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_customer_sk,ss_quantity,ss_sales_price] + CometFilter [ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] InputAdapter BroadcastExchange #9 WholeStageCodegen (7) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk] InputAdapter ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/explain.txt index 7ee05d453..840f9734a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/explain.txt @@ -13,8 +13,8 @@ TakeOrderedAndProject (87) : : : : +- Exchange (24) : : : : +- * Project (23) : : : : +- * BroadcastHashJoin LeftSemi BuildRight (22) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : : : : +- BroadcastExchange (21) : : : : +- * Project (20) @@ -26,13 +26,13 @@ TakeOrderedAndProject (87) : : : : +- * BroadcastHashJoin Inner BuildRight (14) : : : : :- * Project (9) : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : :- * Filter (6) - : : : : : : +- * ColumnarToRow (5) + : : : : : :- * ColumnarToRow (6) + : : : : : : +- CometFilter (5) : : : : : : +- CometScan parquet spark_catalog.default.store_sales (4) : : : : : +- ReusedExchange (7) : : : : +- BroadcastExchange (13) - : : : : +- * Filter (12) - : : : : +- * ColumnarToRow (11) + : : : : +- * ColumnarToRow (12) + : : : : +- CometFilter (11) : : : : +- CometScan parquet spark_catalog.default.item (10) : : : +- * Sort (41) : : : +- * Project (40) @@ -42,20 +42,20 @@ TakeOrderedAndProject (87) : : : +- * HashAggregate (36) : : : +- * Project (35) : : : +- * BroadcastHashJoin Inner BuildRight (34) - : : : :- * Project (29) - : : : : +- * Filter (28) - : : : : +- * ColumnarToRow (27) + : : : :- * ColumnarToRow (29) + : : : : +- CometProject (28) + : : : : +- CometFilter (27) : : : : +- CometScan parquet spark_catalog.default.store_sales (26) : : : +- BroadcastExchange (33) - : : : +- * Filter (32) - : : : +- * ColumnarToRow (31) + : : : +- * ColumnarToRow (32) + : : : +- CometFilter (31) : : : +- CometScan parquet spark_catalog.default.customer (30) : : +- BroadcastExchange (54) : : +- * SortMergeJoin LeftSemi (53) : : :- * Sort (47) : : : +- Exchange (46) - : : : +- * Filter (45) - : : : +- * ColumnarToRow (44) + : : : +- * ColumnarToRow (45) + : : : +- CometFilter (44) : : : +- CometScan parquet spark_catalog.default.customer (43) : : +- * Sort (52) : : +- * Project (51) @@ -75,8 +75,8 @@ TakeOrderedAndProject (87) : : : +- Exchange (69) : : : +- * Project (68) : : : +- * BroadcastHashJoin LeftSemi BuildRight (67) - : : : :- * Filter (65) - : : : : +- * ColumnarToRow (64) + : : : :- * ColumnarToRow (65) + : : : : +- CometFilter (64) : : : : +- CometScan parquet spark_catalog.default.web_sales (63) : : : +- ReusedExchange (66) : : +- * Sort (75) @@ -96,12 +96,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sol PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] +(2) CometFilter Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Condition : isnotnull(cs_bill_customer_sk#1) -(3) Filter [codegen id : 5] +(3) ColumnarToRow [codegen id : 5] Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Condition : isnotnull(cs_bill_customer_sk#1) (4) Scan parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#7, ss_sold_date_sk#8] @@ -111,12 +111,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 3] +(5) CometFilter Input [2]: [ss_item_sk#7, ss_sold_date_sk#8] +Condition : isnotnull(ss_item_sk#7) -(6) Filter [codegen id : 3] +(6) ColumnarToRow [codegen id : 3] Input [2]: [ss_item_sk#7, ss_sold_date_sk#8] -Condition : isnotnull(ss_item_sk#7) (7) ReusedExchange [Reuses operator id: 97] Output [2]: [d_date_sk#10, d_date#11] @@ -138,12 +138,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] +(11) CometFilter Input [2]: [i_item_sk#12, i_item_desc#13] +Condition : isnotnull(i_item_sk#12) -(12) Filter [codegen id : 2] +(12) ColumnarToRow [codegen id : 2] Input [2]: [i_item_sk#12, i_item_desc#13] -Condition : isnotnull(i_item_sk#12) (13) BroadcastExchange Input [2]: [i_item_sk#12, i_item_desc#13] @@ -214,16 +214,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 8] -Input [4]: [ss_customer_sk#20, ss_quantity#21, ss_sales_price#22, ss_sold_date_sk#23] - -(28) Filter [codegen id : 8] +(27) CometFilter Input [4]: [ss_customer_sk#20, ss_quantity#21, ss_sales_price#22, ss_sold_date_sk#23] Condition : isnotnull(ss_customer_sk#20) -(29) Project [codegen id : 8] -Output [3]: [ss_customer_sk#20, ss_quantity#21, ss_sales_price#22] +(28) CometProject Input [4]: [ss_customer_sk#20, ss_quantity#21, ss_sales_price#22, ss_sold_date_sk#23] +Arguments: [ss_customer_sk#20, ss_quantity#21, ss_sales_price#22], [ss_customer_sk#20, ss_quantity#21, ss_sales_price#22] + +(29) ColumnarToRow [codegen id : 8] +Input [3]: [ss_customer_sk#20, ss_quantity#21, ss_sales_price#22] (30) Scan parquet spark_catalog.default.customer Output [1]: [c_customer_sk#24] @@ -232,12 +232,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 7] +(31) CometFilter Input [1]: [c_customer_sk#24] +Condition : isnotnull(c_customer_sk#24) -(32) Filter [codegen id : 7] +(32) ColumnarToRow [codegen id : 7] Input [1]: [c_customer_sk#24] -Condition : isnotnull(c_customer_sk#24) (33) BroadcastExchange Input [1]: [c_customer_sk#24] @@ -296,12 +296,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(44) ColumnarToRow [codegen id : 10] +(44) CometFilter Input [3]: [c_customer_sk#33, c_first_name#34, c_last_name#35] +Condition : isnotnull(c_customer_sk#33) -(45) Filter [codegen id : 10] +(45) ColumnarToRow [codegen id : 10] Input [3]: [c_customer_sk#33, c_first_name#34, c_last_name#35] -Condition : isnotnull(c_customer_sk#33) (46) Exchange Input [3]: [c_customer_sk#33, c_first_name#34, c_last_name#35] @@ -392,12 +392,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#47), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(64) ColumnarToRow [codegen id : 23] +(64) CometFilter Input [5]: [ws_item_sk#43, ws_bill_customer_sk#44, ws_quantity#45, ws_list_price#46, ws_sold_date_sk#47] +Condition : isnotnull(ws_bill_customer_sk#44) -(65) Filter [codegen id : 23] +(65) ColumnarToRow [codegen id : 23] Input [5]: [ws_item_sk#43, ws_bill_customer_sk#44, ws_quantity#45, ws_list_price#46, ws_sold_date_sk#47] -Condition : isnotnull(ws_bill_customer_sk#44) (66) ReusedExchange [Reuses operator id: 21] Output [1]: [item_sk#49] @@ -502,9 +502,9 @@ Arguments: 100, [c_last_name#35 ASC NULLS FIRST, c_first_name#34 ASC NULLS FIRST Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 BroadcastExchange (92) -+- * Project (91) - +- * Filter (90) - +- * ColumnarToRow (89) ++- * ColumnarToRow (91) + +- CometProject (90) + +- CometFilter (89) +- CometScan parquet spark_catalog.default.date_dim (88) @@ -515,16 +515,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(89) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#36, d_year#66, d_moy#67] - -(90) Filter [codegen id : 1] +(89) CometFilter Input [3]: [d_date_sk#36, d_year#66, d_moy#67] Condition : ((((isnotnull(d_year#66) AND isnotnull(d_moy#67)) AND (d_year#66 = 2000)) AND (d_moy#67 = 2)) AND isnotnull(d_date_sk#36)) -(91) Project [codegen id : 1] -Output [1]: [d_date_sk#36] +(90) CometProject Input [3]: [d_date_sk#36, d_year#66, d_moy#67] +Arguments: [d_date_sk#36], [d_date_sk#36] + +(91) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#36] (92) BroadcastExchange Input [1]: [d_date_sk#36] @@ -532,9 +532,9 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Subquery:2 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 BroadcastExchange (97) -+- * Project (96) - +- * Filter (95) - +- * ColumnarToRow (94) ++- * ColumnarToRow (96) + +- CometProject (95) + +- CometFilter (94) +- CometScan parquet spark_catalog.default.date_dim (93) @@ -545,16 +545,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(94) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#10, d_date#11, d_year#68] - -(95) Filter [codegen id : 1] +(94) CometFilter Input [3]: [d_date_sk#10, d_date#11, d_year#68] Condition : (d_year#68 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#10)) -(96) Project [codegen id : 1] -Output [2]: [d_date_sk#10, d_date#11] +(95) CometProject Input [3]: [d_date_sk#10, d_date#11, d_year#68] +Arguments: [d_date_sk#10, d_date#11], [d_date_sk#10, d_date#11] + +(96) ColumnarToRow [codegen id : 1] +Input [2]: [d_date_sk#10, d_date#11] (97) BroadcastExchange Input [2]: [d_date_sk#10, d_date#11] @@ -571,8 +571,8 @@ Subquery:3 Hosting operator id = 39 Hosting Expression = Subquery scalar-subquer +- * BroadcastHashJoin Inner BuildRight (105) :- * Project (103) : +- * BroadcastHashJoin Inner BuildRight (102) - : :- * Filter (100) - : : +- * ColumnarToRow (99) + : :- * ColumnarToRow (100) + : : +- CometFilter (99) : : +- CometScan parquet spark_catalog.default.store_sales (98) : +- ReusedExchange (101) +- ReusedExchange (104) @@ -586,12 +586,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#72), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(99) ColumnarToRow [codegen id : 3] +(99) CometFilter Input [4]: [ss_customer_sk#69, ss_quantity#70, ss_sales_price#71, ss_sold_date_sk#72] +Condition : isnotnull(ss_customer_sk#69) -(100) Filter [codegen id : 3] +(100) ColumnarToRow [codegen id : 3] Input [4]: [ss_customer_sk#69, ss_quantity#70, ss_sales_price#71, ss_sold_date_sk#72] -Condition : isnotnull(ss_customer_sk#69) (101) ReusedExchange [Reuses operator id: 33] Output [1]: [c_customer_sk#74] @@ -657,9 +657,9 @@ Results [1]: [max(csales#81)#84 AS tpcds_cmax#85] Subquery:4 Hosting operator id = 98 Hosting Expression = ss_sold_date_sk#72 IN dynamicpruning#73 BroadcastExchange (117) -+- * Project (116) - +- * Filter (115) - +- * ColumnarToRow (114) ++- * ColumnarToRow (116) + +- CometProject (115) + +- CometFilter (114) +- CometScan parquet spark_catalog.default.date_dim (113) @@ -670,16 +670,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(114) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#75, d_year#86] - -(115) Filter [codegen id : 1] +(114) CometFilter Input [2]: [d_date_sk#75, d_year#86] Condition : (d_year#86 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#75)) -(116) Project [codegen id : 1] -Output [1]: [d_date_sk#75] +(115) CometProject Input [2]: [d_date_sk#75, d_year#86] +Arguments: [d_date_sk#75], [d_date_sk#75] + +(116) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#75] (117) BroadcastExchange Input [1]: [d_date_sk#75] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/simplified.txt index cfb8acd14..49ddeaef8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/simplified.txt @@ -19,17 +19,17 @@ TakeOrderedAndProject [c_last_name,c_first_name,sales] WholeStageCodegen (5) Project [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] BroadcastHashJoin [cs_item_sk,item_sk] - Filter [cs_bill_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_bill_customer_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_moy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #4 @@ -45,26 +45,26 @@ TakeOrderedAndProject [c_last_name,c_first_name,sales] BroadcastHashJoin [ss_item_sk,i_item_sk] Project [ss_item_sk,d_date] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #6 WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_date] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] InputAdapter ReusedExchange [d_date_sk,d_date] #6 InputAdapter BroadcastExchange #7 WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] InputAdapter WholeStageCodegen (9) @@ -87,17 +87,17 @@ TakeOrderedAndProject [c_last_name,c_first_name,sales] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #4 BroadcastExchange #12 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [c_customer_sk] #9 @@ -110,17 +110,17 @@ TakeOrderedAndProject [c_last_name,c_first_name,sales] HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] Project [ss_quantity,ss_sales_price,c_customer_sk] BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_quantity,ss_sales_price] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_customer_sk,ss_quantity,ss_sales_price] + CometFilter [ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] InputAdapter BroadcastExchange #9 WholeStageCodegen (7) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk] InputAdapter BroadcastExchange #13 @@ -132,9 +132,9 @@ TakeOrderedAndProject [c_last_name,c_first_name,sales] InputAdapter Exchange [c_customer_sk] #14 WholeStageCodegen (10) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] InputAdapter WholeStageCodegen (14) @@ -166,9 +166,9 @@ TakeOrderedAndProject [c_last_name,c_first_name,sales] WholeStageCodegen (23) Project [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] BroadcastHashJoin [ws_item_sk,item_sk] - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_bill_customer_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/explain.txt index b27379f24..7241b5ea0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/explain.txt @@ -18,32 +18,32 @@ : : : : +- * SortMergeJoin Inner (13) : : : : :- * Sort (6) : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : : +- * ColumnarToRow (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : : +- * Sort (12) : : : : +- Exchange (11) - : : : : +- * Project (10) - : : : : +- * Filter (9) - : : : : +- * ColumnarToRow (8) + : : : : +- * ColumnarToRow (10) + : : : : +- CometProject (9) + : : : : +- CometFilter (8) : : : : +- CometScan parquet spark_catalog.default.store_returns (7) : : : +- BroadcastExchange (19) - : : : +- * Project (18) - : : : +- * Filter (17) - : : : +- * ColumnarToRow (16) + : : : +- * ColumnarToRow (18) + : : : +- CometProject (17) + : : : +- CometFilter (16) : : : +- CometScan parquet spark_catalog.default.store (15) : : +- BroadcastExchange (25) - : : +- * Filter (24) - : : +- * ColumnarToRow (23) + : : +- * ColumnarToRow (24) + : : +- CometFilter (23) : : +- CometScan parquet spark_catalog.default.item (22) : +- BroadcastExchange (31) - : +- * Filter (30) - : +- * ColumnarToRow (29) + : +- * ColumnarToRow (30) + : +- CometFilter (29) : +- CometScan parquet spark_catalog.default.customer (28) +- BroadcastExchange (37) - +- * Filter (36) - +- * ColumnarToRow (35) + +- * ColumnarToRow (36) + +- CometFilter (35) +- CometScan parquet spark_catalog.default.customer_address (34) @@ -54,16 +54,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2)) -(4) Project [codegen id : 1] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +(3) CometProject Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] + +(4) ColumnarToRow [codegen id : 1] +Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] (5) Exchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] @@ -80,16 +80,16 @@ Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 3] -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(9) Filter [codegen id : 3] +(8) CometFilter Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) -(10) Project [codegen id : 3] -Output [2]: [sr_item_sk#7, sr_ticket_number#8] +(9) CometProject Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] + +(10) ColumnarToRow [codegen id : 3] +Input [2]: [sr_item_sk#7, sr_ticket_number#8] (11) Exchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] @@ -116,16 +116,16 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 5] -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] - -(17) Filter [codegen id : 5] +(16) CometFilter Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) -(18) Project [codegen id : 5] -Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +(17) CometProject Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14], [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] + +(18) ColumnarToRow [codegen id : 5] +Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] (19) BroadcastExchange Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] @@ -148,12 +148,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_color), EqualTo(i_color,pale ), IsNotNull(i_item_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 6] +(23) CometFilter Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +Condition : ((isnotnull(i_color#18) AND (i_color#18 = pale )) AND isnotnull(i_item_sk#15)) -(24) Filter [codegen id : 6] +(24) ColumnarToRow [codegen id : 6] Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = pale )) AND isnotnull(i_item_sk#15)) (25) BroadcastExchange Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] @@ -176,12 +176,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)] ReadSchema: struct -(29) ColumnarToRow [codegen id : 7] +(29) CometFilter Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_birth_country#24)) -(30) Filter [codegen id : 7] +(30) ColumnarToRow [codegen id : 7] Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_birth_country#24)) (31) BroadcastExchange Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] @@ -204,12 +204,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_country), IsNotNull(ca_zip)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 8] +(35) CometFilter Input [3]: [ca_state#25, ca_zip#26, ca_country#27] +Condition : (isnotnull(ca_country#27) AND isnotnull(ca_zip#26)) -(36) Filter [codegen id : 8] +(36) ColumnarToRow [codegen id : 8] Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Condition : (isnotnull(ca_country#27) AND isnotnull(ca_zip#26)) (37) BroadcastExchange Input [3]: [ca_state#25, ca_zip#26, ca_country#27] @@ -290,8 +290,8 @@ Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquer : : : : +- ReusedExchange (49) : : : +- ReusedExchange (53) : : +- BroadcastExchange (59) - : : +- * Filter (58) - : : +- * ColumnarToRow (57) + : : +- * ColumnarToRow (58) + : : +- CometFilter (57) : : +- CometScan parquet spark_catalog.default.item (56) : +- ReusedExchange (62) +- ReusedExchange (65) @@ -341,12 +341,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 6] +(57) CometFilter Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] +Condition : isnotnull(i_item_sk#51) -(58) Filter [codegen id : 6] +(58) ColumnarToRow [codegen id : 6] Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Condition : isnotnull(i_item_sk#51) (59) BroadcastExchange Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/simplified.txt index 4cdde1b6f..8ebd45fd1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/simplified.txt @@ -37,9 +37,9 @@ WholeStageCodegen (11) InputAdapter BroadcastExchange #11 WholeStageCodegen (6) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] InputAdapter ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 @@ -71,10 +71,10 @@ WholeStageCodegen (11) InputAdapter Exchange [ss_ticket_number,ss_item_sk] #3 WholeStageCodegen (1) - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] InputAdapter WholeStageCodegen (4) @@ -82,37 +82,37 @@ WholeStageCodegen (11) InputAdapter Exchange [sr_ticket_number,sr_item_sk] #4 WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_ticket_number,sr_item_sk] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] InputAdapter BroadcastExchange #5 WholeStageCodegen (5) - Project [s_store_sk,s_store_name,s_state,s_zip] - Filter [s_market_id,s_store_sk,s_zip] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_market_id,s_store_sk,s_zip] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] InputAdapter BroadcastExchange #6 WholeStageCodegen (6) - Filter [i_color,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_color,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] InputAdapter BroadcastExchange #7 WholeStageCodegen (7) - Filter [c_customer_sk,c_birth_country] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_birth_country] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] InputAdapter BroadcastExchange #8 WholeStageCodegen (8) - Filter [ca_country,ca_zip] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ca_country,ca_zip] CometScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/explain.txt index a198578da..0ac5639b7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/explain.txt @@ -18,32 +18,32 @@ : : : : +- * SortMergeJoin Inner (13) : : : : :- * Sort (6) : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : : +- * ColumnarToRow (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : : +- * Sort (12) : : : : +- Exchange (11) - : : : : +- * Project (10) - : : : : +- * Filter (9) - : : : : +- * ColumnarToRow (8) + : : : : +- * ColumnarToRow (10) + : : : : +- CometProject (9) + : : : : +- CometFilter (8) : : : : +- CometScan parquet spark_catalog.default.store_returns (7) : : : +- BroadcastExchange (19) - : : : +- * Project (18) - : : : +- * Filter (17) - : : : +- * ColumnarToRow (16) + : : : +- * ColumnarToRow (18) + : : : +- CometProject (17) + : : : +- CometFilter (16) : : : +- CometScan parquet spark_catalog.default.store (15) : : +- BroadcastExchange (25) - : : +- * Filter (24) - : : +- * ColumnarToRow (23) + : : +- * ColumnarToRow (24) + : : +- CometFilter (23) : : +- CometScan parquet spark_catalog.default.item (22) : +- BroadcastExchange (31) - : +- * Filter (30) - : +- * ColumnarToRow (29) + : +- * ColumnarToRow (30) + : +- CometFilter (29) : +- CometScan parquet spark_catalog.default.customer (28) +- BroadcastExchange (37) - +- * Filter (36) - +- * ColumnarToRow (35) + +- * ColumnarToRow (36) + +- CometFilter (35) +- CometScan parquet spark_catalog.default.customer_address (34) @@ -54,16 +54,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2)) -(4) Project [codegen id : 1] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +(3) CometProject Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] + +(4) ColumnarToRow [codegen id : 1] +Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] (5) Exchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] @@ -80,16 +80,16 @@ Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 3] -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(9) Filter [codegen id : 3] +(8) CometFilter Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) -(10) Project [codegen id : 3] -Output [2]: [sr_item_sk#7, sr_ticket_number#8] +(9) CometProject Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] + +(10) ColumnarToRow [codegen id : 3] +Input [2]: [sr_item_sk#7, sr_ticket_number#8] (11) Exchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] @@ -116,16 +116,16 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 5] -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] - -(17) Filter [codegen id : 5] +(16) CometFilter Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) -(18) Project [codegen id : 5] -Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +(17) CometProject Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14], [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] + +(18) ColumnarToRow [codegen id : 5] +Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] (19) BroadcastExchange Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] @@ -148,12 +148,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_color), EqualTo(i_color,chiffon ), IsNotNull(i_item_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 6] +(23) CometFilter Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +Condition : ((isnotnull(i_color#18) AND (i_color#18 = chiffon )) AND isnotnull(i_item_sk#15)) -(24) Filter [codegen id : 6] +(24) ColumnarToRow [codegen id : 6] Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = chiffon )) AND isnotnull(i_item_sk#15)) (25) BroadcastExchange Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] @@ -176,12 +176,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)] ReadSchema: struct -(29) ColumnarToRow [codegen id : 7] +(29) CometFilter Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_birth_country#24)) -(30) Filter [codegen id : 7] +(30) ColumnarToRow [codegen id : 7] Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_birth_country#24)) (31) BroadcastExchange Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] @@ -204,12 +204,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_country), IsNotNull(ca_zip)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 8] +(35) CometFilter Input [3]: [ca_state#25, ca_zip#26, ca_country#27] +Condition : (isnotnull(ca_country#27) AND isnotnull(ca_zip#26)) -(36) Filter [codegen id : 8] +(36) ColumnarToRow [codegen id : 8] Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Condition : (isnotnull(ca_country#27) AND isnotnull(ca_zip#26)) (37) BroadcastExchange Input [3]: [ca_state#25, ca_zip#26, ca_country#27] @@ -290,8 +290,8 @@ Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquer : : : : +- ReusedExchange (49) : : : +- ReusedExchange (53) : : +- BroadcastExchange (59) - : : +- * Filter (58) - : : +- * ColumnarToRow (57) + : : +- * ColumnarToRow (58) + : : +- CometFilter (57) : : +- CometScan parquet spark_catalog.default.item (56) : +- ReusedExchange (62) +- ReusedExchange (65) @@ -341,12 +341,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 6] +(57) CometFilter Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] +Condition : isnotnull(i_item_sk#51) -(58) Filter [codegen id : 6] +(58) ColumnarToRow [codegen id : 6] Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Condition : isnotnull(i_item_sk#51) (59) BroadcastExchange Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/simplified.txt index 4cdde1b6f..8ebd45fd1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/simplified.txt @@ -37,9 +37,9 @@ WholeStageCodegen (11) InputAdapter BroadcastExchange #11 WholeStageCodegen (6) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] InputAdapter ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 @@ -71,10 +71,10 @@ WholeStageCodegen (11) InputAdapter Exchange [ss_ticket_number,ss_item_sk] #3 WholeStageCodegen (1) - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] InputAdapter WholeStageCodegen (4) @@ -82,37 +82,37 @@ WholeStageCodegen (11) InputAdapter Exchange [sr_ticket_number,sr_item_sk] #4 WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_ticket_number,sr_item_sk] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] InputAdapter BroadcastExchange #5 WholeStageCodegen (5) - Project [s_store_sk,s_store_name,s_state,s_zip] - Filter [s_market_id,s_store_sk,s_zip] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_market_id,s_store_sk,s_zip] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] InputAdapter BroadcastExchange #6 WholeStageCodegen (6) - Filter [i_color,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_color,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] InputAdapter BroadcastExchange #7 WholeStageCodegen (7) - Filter [c_customer_sk,c_birth_country] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_birth_country] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] InputAdapter BroadcastExchange #8 WholeStageCodegen (8) - Filter [ca_country,ca_zip] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ca_country,ca_zip] CometScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/explain.txt index f2e9a4eaf..2635546e4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/explain.txt @@ -17,27 +17,27 @@ TakeOrderedAndProject (40) : : : : : +- * BroadcastHashJoin Inner BuildRight (14) : : : : : :- * Project (9) : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) + : : : : : : :- * ColumnarToRow (3) + : : : : : : : +- CometFilter (2) : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : : : : +- BroadcastExchange (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) + : : : : : : +- * ColumnarToRow (6) + : : : : : : +- CometFilter (5) : : : : : : +- CometScan parquet spark_catalog.default.store_returns (4) : : : : : +- BroadcastExchange (13) - : : : : : +- * Filter (12) - : : : : : +- * ColumnarToRow (11) + : : : : : +- * ColumnarToRow (12) + : : : : : +- CometFilter (11) : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (10) : : : : +- ReusedExchange (16) : : : +- ReusedExchange (19) : : +- ReusedExchange (22) : +- BroadcastExchange (28) - : +- * Filter (27) - : +- * ColumnarToRow (26) + : +- * ColumnarToRow (27) + : +- CometFilter (26) : +- CometScan parquet spark_catalog.default.store (25) +- BroadcastExchange (34) - +- * Filter (33) - +- * ColumnarToRow (32) + +- * ColumnarToRow (33) + +- CometFilter (32) +- CometScan parquet spark_catalog.default.item (31) @@ -49,12 +49,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 8] +(2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] +Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) -(3) Filter [codegen id : 8] +(3) ColumnarToRow [codegen id : 8] Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) (4) Scan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12] @@ -64,12 +64,12 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#12), dynamicpruningexpression(s PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12] +Condition : ((isnotnull(sr_customer_sk#9) AND isnotnull(sr_item_sk#8)) AND isnotnull(sr_ticket_number#10)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12] -Condition : ((isnotnull(sr_customer_sk#9) AND isnotnull(sr_item_sk#8)) AND isnotnull(sr_ticket_number#10)) (7) BroadcastExchange Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12] @@ -93,12 +93,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] +(11) CometFilter Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_net_profit#16, cs_sold_date_sk#17] +Condition : (isnotnull(cs_bill_customer_sk#14) AND isnotnull(cs_item_sk#15)) -(12) Filter [codegen id : 2] +(12) ColumnarToRow [codegen id : 2] Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_net_profit#16, cs_sold_date_sk#17] -Condition : (isnotnull(cs_bill_customer_sk#14) AND isnotnull(cs_item_sk#15)) (13) BroadcastExchange Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_net_profit#16, cs_sold_date_sk#17] @@ -160,12 +160,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 6] +(26) CometFilter Input [3]: [s_store_sk#22, s_store_id#23, s_store_name#24] +Condition : isnotnull(s_store_sk#22) -(27) Filter [codegen id : 6] +(27) ColumnarToRow [codegen id : 6] Input [3]: [s_store_sk#22, s_store_id#23, s_store_name#24] -Condition : isnotnull(s_store_sk#22) (28) BroadcastExchange Input [3]: [s_store_sk#22, s_store_id#23, s_store_name#24] @@ -188,12 +188,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 7] +(32) CometFilter Input [3]: [i_item_sk#25, i_item_id#26, i_item_desc#27] +Condition : isnotnull(i_item_sk#25) -(33) Filter [codegen id : 7] +(33) ColumnarToRow [codegen id : 7] Input [3]: [i_item_sk#25, i_item_id#26, i_item_desc#27] -Condition : isnotnull(i_item_sk#25) (34) BroadcastExchange Input [3]: [i_item_sk#25, i_item_id#26, i_item_desc#27] @@ -235,9 +235,9 @@ Arguments: 100, [i_item_id#26 ASC NULLS FIRST, i_item_desc#27 ASC NULLS FIRST, s Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 BroadcastExchange (45) -+- * Project (44) - +- * Filter (43) - +- * ColumnarToRow (42) ++- * ColumnarToRow (44) + +- CometProject (43) + +- CometFilter (42) +- CometScan parquet spark_catalog.default.date_dim (41) @@ -248,16 +248,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,4), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(42) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#19, d_year#40, d_moy#41] - -(43) Filter [codegen id : 1] +(42) CometFilter Input [3]: [d_date_sk#19, d_year#40, d_moy#41] Condition : ((((isnotnull(d_moy#41) AND isnotnull(d_year#40)) AND (d_moy#41 = 4)) AND (d_year#40 = 2001)) AND isnotnull(d_date_sk#19)) -(44) Project [codegen id : 1] -Output [1]: [d_date_sk#19] +(43) CometProject Input [3]: [d_date_sk#19, d_year#40, d_moy#41] +Arguments: [d_date_sk#19], [d_date_sk#19] + +(44) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#19] (45) BroadcastExchange Input [1]: [d_date_sk#19] @@ -265,9 +265,9 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Subquery:2 Hosting operator id = 4 Hosting Expression = sr_returned_date_sk#12 IN dynamicpruning#13 BroadcastExchange (50) -+- * Project (49) - +- * Filter (48) - +- * ColumnarToRow (47) ++- * ColumnarToRow (49) + +- CometProject (48) + +- CometFilter (47) +- CometScan parquet spark_catalog.default.date_dim (46) @@ -278,16 +278,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,10), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(47) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#20, d_year#42, d_moy#43] - -(48) Filter [codegen id : 1] +(47) CometFilter Input [3]: [d_date_sk#20, d_year#42, d_moy#43] Condition : (((((isnotnull(d_moy#43) AND isnotnull(d_year#42)) AND (d_moy#43 >= 4)) AND (d_moy#43 <= 10)) AND (d_year#42 = 2001)) AND isnotnull(d_date_sk#20)) -(49) Project [codegen id : 1] -Output [1]: [d_date_sk#20] +(48) CometProject Input [3]: [d_date_sk#20, d_year#42, d_moy#43] +Arguments: [d_date_sk#20], [d_date_sk#20] + +(49) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#20] (50) BroadcastExchange Input [1]: [d_date_sk#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/simplified.txt index f45de0daf..eda7f6b64 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/simplified.txt @@ -19,39 +19,39 @@ TakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales BroadcastHashJoin [sr_customer_sk,sr_item_sk,cs_bill_customer_sk,cs_item_sk] Project [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk] BroadcastHashJoin [ss_customer_sk,ss_item_sk,ss_ticket_number,sr_customer_sk,sr_item_sk,sr_ticket_number] - Filter [ss_customer_sk,ss_item_sk,ss_ticket_number,ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_customer_sk,ss_item_sk,ss_ticket_number,ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_moy,d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Filter [sr_customer_sk,sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [sr_customer_sk,sr_item_sk,sr_ticket_number] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #4 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_moy,d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #5 WholeStageCodegen (2) - Filter [cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_bill_customer_sk,cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] ReusedSubquery [d_date_sk] #2 InputAdapter @@ -63,14 +63,14 @@ TakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales InputAdapter BroadcastExchange #6 WholeStageCodegen (6) - Filter [s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] InputAdapter BroadcastExchange #7 WholeStageCodegen (7) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/explain.txt index 8782d99fb..c90dcd024 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/explain.txt @@ -11,23 +11,23 @@ TakeOrderedAndProject (30) : : +- * BroadcastHashJoin Inner BuildRight (12) : : :- * Project (10) : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) + : : : +- * ColumnarToRow (7) + : : : +- CometProject (6) + : : : +- CometFilter (5) : : : +- CometScan parquet spark_catalog.default.customer_demographics (4) : : +- ReusedExchange (11) : +- BroadcastExchange (17) - : +- * Filter (16) - : +- * ColumnarToRow (15) + : +- * ColumnarToRow (16) + : +- CometFilter (15) : +- CometScan parquet spark_catalog.default.item (14) +- BroadcastExchange (24) - +- * Project (23) - +- * Filter (22) - +- * ColumnarToRow (21) + +- * ColumnarToRow (23) + +- CometProject (22) + +- CometFilter (21) +- CometScan parquet spark_catalog.default.promotion (20) @@ -39,12 +39,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#8), dynamicpruningexpression(cs_sol PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] +(2) CometFilter Input [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] +Condition : ((isnotnull(cs_bill_cdemo_sk#1) AND isnotnull(cs_item_sk#2)) AND isnotnull(cs_promo_sk#3)) -(3) Filter [codegen id : 5] +(3) ColumnarToRow [codegen id : 5] Input [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -Condition : ((isnotnull(cs_bill_cdemo_sk#1) AND isnotnull(cs_item_sk#2)) AND isnotnull(cs_promo_sk#3)) (4) Scan parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] @@ -53,16 +53,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_marital_status,S), EqualTo(cd_education_status,College ), IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] - -(6) Filter [codegen id : 1] +(5) CometFilter Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (cd_gender#11 = M)) AND (cd_marital_status#12 = S)) AND (cd_education_status#13 = College )) AND isnotnull(cd_demo_sk#10)) -(7) Project [codegen id : 1] -Output [1]: [cd_demo_sk#10] +(6) CometProject Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] +Arguments: [cd_demo_sk#10], [cd_demo_sk#10] + +(7) ColumnarToRow [codegen id : 1] +Input [1]: [cd_demo_sk#10] (8) BroadcastExchange Input [1]: [cd_demo_sk#10] @@ -98,12 +98,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 3] +(15) CometFilter Input [2]: [i_item_sk#15, i_item_id#16] +Condition : isnotnull(i_item_sk#15) -(16) Filter [codegen id : 3] +(16) ColumnarToRow [codegen id : 3] Input [2]: [i_item_sk#15, i_item_id#16] -Condition : isnotnull(i_item_sk#15) (17) BroadcastExchange Input [2]: [i_item_sk#15, i_item_id#16] @@ -126,16 +126,16 @@ Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [Or(EqualTo(p_channel_email,N),EqualTo(p_channel_event,N)), IsNotNull(p_promo_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 4] -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] - -(22) Filter [codegen id : 4] +(21) CometFilter Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] Condition : (((p_channel_email#18 = N) OR (p_channel_event#19 = N)) AND isnotnull(p_promo_sk#17)) -(23) Project [codegen id : 4] -Output [1]: [p_promo_sk#17] +(22) CometProject Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] +Arguments: [p_promo_sk#17], [p_promo_sk#17] + +(23) ColumnarToRow [codegen id : 4] +Input [1]: [p_promo_sk#17] (24) BroadcastExchange Input [1]: [p_promo_sk#17] @@ -177,9 +177,9 @@ Arguments: 100, [i_item_id#16 ASC NULLS FIRST], [i_item_id#16, agg1#40, agg2#41, Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9 BroadcastExchange (35) -+- * Project (34) - +- * Filter (33) - +- * ColumnarToRow (32) ++- * ColumnarToRow (34) + +- CometProject (33) + +- CometFilter (32) +- CometScan parquet spark_catalog.default.date_dim (31) @@ -190,16 +190,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#14, d_year#44] - -(33) Filter [codegen id : 1] +(32) CometFilter Input [2]: [d_date_sk#14, d_year#44] Condition : ((isnotnull(d_year#44) AND (d_year#44 = 2000)) AND isnotnull(d_date_sk#14)) -(34) Project [codegen id : 1] -Output [1]: [d_date_sk#14] +(33) CometProject Input [2]: [d_date_sk#14, d_year#44] +Arguments: [d_date_sk#14], [d_date_sk#14] + +(34) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#14] (35) BroadcastExchange Input [1]: [d_date_sk#14] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/simplified.txt index 41f3e4a7a..7d3893624 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/simplified.txt @@ -13,40 +13,40 @@ TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Project [cd_demo_sk] - Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cd_demo_sk] + CometFilter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] InputAdapter ReusedExchange [d_date_sk] #2 InputAdapter BroadcastExchange #4 WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] InputAdapter BroadcastExchange #5 WholeStageCodegen (4) - Project [p_promo_sk] - Filter [p_channel_email,p_channel_event,p_promo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [p_promo_sk] + CometFilter [p_channel_email,p_channel_event,p_promo_sk] CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/explain.txt index 6c2819610..e41077ed3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/explain.txt @@ -12,22 +12,22 @@ TakeOrderedAndProject (30) : : +- * BroadcastHashJoin Inner BuildRight (12) : : :- * Project (10) : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) + : : : +- * ColumnarToRow (7) + : : : +- CometProject (6) + : : : +- CometFilter (5) : : : +- CometScan parquet spark_catalog.default.customer_demographics (4) : : +- ReusedExchange (11) : +- BroadcastExchange (17) - : +- * Filter (16) - : +- * ColumnarToRow (15) + : +- * ColumnarToRow (16) + : +- CometFilter (15) : +- CometScan parquet spark_catalog.default.store (14) +- BroadcastExchange (23) - +- * Filter (22) - +- * ColumnarToRow (21) + +- * ColumnarToRow (22) + +- CometFilter (21) +- CometScan parquet spark_catalog.default.item (20) @@ -39,12 +39,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] +(2) CometFilter Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) -(3) Filter [codegen id : 5] +(3) ColumnarToRow [codegen id : 5] Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) (4) Scan parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] @@ -53,16 +53,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_marital_status,S), EqualTo(cd_education_status,College ), IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] - -(6) Filter [codegen id : 1] +(5) CometFilter Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (cd_gender#11 = M)) AND (cd_marital_status#12 = S)) AND (cd_education_status#13 = College )) AND isnotnull(cd_demo_sk#10)) -(7) Project [codegen id : 1] -Output [1]: [cd_demo_sk#10] +(6) CometProject Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] +Arguments: [cd_demo_sk#10], [cd_demo_sk#10] + +(7) ColumnarToRow [codegen id : 1] +Input [1]: [cd_demo_sk#10] (8) BroadcastExchange Input [1]: [cd_demo_sk#10] @@ -98,12 +98,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 3] +(15) CometFilter Input [2]: [s_store_sk#15, s_state#16] +Condition : ((isnotnull(s_state#16) AND (s_state#16 = TN)) AND isnotnull(s_store_sk#15)) -(16) Filter [codegen id : 3] +(16) ColumnarToRow [codegen id : 3] Input [2]: [s_store_sk#15, s_state#16] -Condition : ((isnotnull(s_state#16) AND (s_state#16 = TN)) AND isnotnull(s_store_sk#15)) (17) BroadcastExchange Input [2]: [s_store_sk#15, s_state#16] @@ -126,12 +126,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 4] +(21) CometFilter Input [2]: [i_item_sk#17, i_item_id#18] +Condition : isnotnull(i_item_sk#17) -(22) Filter [codegen id : 4] +(22) ColumnarToRow [codegen id : 4] Input [2]: [i_item_sk#17, i_item_id#18] -Condition : isnotnull(i_item_sk#17) (23) BroadcastExchange Input [2]: [i_item_sk#17, i_item_id#18] @@ -177,9 +177,9 @@ Arguments: 100, [i_item_id#19 ASC NULLS FIRST, s_state#20 ASC NULLS FIRST], [i_i Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 BroadcastExchange (35) -+- * Project (34) - +- * Filter (33) - +- * ColumnarToRow (32) ++- * ColumnarToRow (34) + +- CometProject (33) + +- CometFilter (32) +- CometScan parquet spark_catalog.default.date_dim (31) @@ -190,16 +190,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#14, d_year#47] - -(33) Filter [codegen id : 1] +(32) CometFilter Input [2]: [d_date_sk#14, d_year#47] Condition : ((isnotnull(d_year#47) AND (d_year#47 = 2002)) AND isnotnull(d_date_sk#14)) -(34) Project [codegen id : 1] -Output [1]: [d_date_sk#14] +(33) CometProject Input [2]: [d_date_sk#14, d_year#47] +Arguments: [d_date_sk#14], [d_date_sk#14] + +(34) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#14] (35) BroadcastExchange Input [1]: [d_date_sk#14] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/simplified.txt index 9cecdee29..9d073ff67 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/simplified.txt @@ -14,39 +14,39 @@ TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_cdemo_sk,ss_store_sk,ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Project [cd_demo_sk] - Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cd_demo_sk] + CometFilter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] InputAdapter ReusedExchange [d_date_sk] #2 InputAdapter BroadcastExchange #4 WholeStageCodegen (3) - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_state,s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_state] InputAdapter BroadcastExchange #5 WholeStageCodegen (4) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28/explain.txt index ccab165e2..de4ab3a2c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28/explain.txt @@ -10,9 +10,9 @@ : : : : : +- * HashAggregate (7) : : : : : +- Exchange (6) : : : : : +- * HashAggregate (5) -: : : : : +- * Project (4) -: : : : : +- * Filter (3) -: : : : : +- * ColumnarToRow (2) +: : : : : +- * ColumnarToRow (4) +: : : : : +- CometProject (3) +: : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : : +- BroadcastExchange (21) : : : : +- * HashAggregate (20) @@ -21,9 +21,9 @@ : : : : +- * HashAggregate (17) : : : : +- Exchange (16) : : : : +- * HashAggregate (15) -: : : : +- * Project (14) -: : : : +- * Filter (13) -: : : : +- * ColumnarToRow (12) +: : : : +- * ColumnarToRow (14) +: : : : +- CometProject (13) +: : : : +- CometFilter (12) : : : : +- CometScan parquet spark_catalog.default.store_sales (11) : : : +- BroadcastExchange (33) : : : +- * HashAggregate (32) @@ -32,9 +32,9 @@ : : : +- * HashAggregate (29) : : : +- Exchange (28) : : : +- * HashAggregate (27) -: : : +- * Project (26) -: : : +- * Filter (25) -: : : +- * ColumnarToRow (24) +: : : +- * ColumnarToRow (26) +: : : +- CometProject (25) +: : : +- CometFilter (24) : : : +- CometScan parquet spark_catalog.default.store_sales (23) : : +- BroadcastExchange (45) : : +- * HashAggregate (44) @@ -43,9 +43,9 @@ : : +- * HashAggregate (41) : : +- Exchange (40) : : +- * HashAggregate (39) -: : +- * Project (38) -: : +- * Filter (37) -: : +- * ColumnarToRow (36) +: : +- * ColumnarToRow (38) +: : +- CometProject (37) +: : +- CometFilter (36) : : +- CometScan parquet spark_catalog.default.store_sales (35) : +- BroadcastExchange (57) : +- * HashAggregate (56) @@ -54,9 +54,9 @@ : +- * HashAggregate (53) : +- Exchange (52) : +- * HashAggregate (51) -: +- * Project (50) -: +- * Filter (49) -: +- * ColumnarToRow (48) +: +- * ColumnarToRow (50) +: +- CometProject (49) +: +- CometFilter (48) : +- CometScan parquet spark_catalog.default.store_sales (47) +- BroadcastExchange (69) +- * HashAggregate (68) @@ -65,9 +65,9 @@ +- * HashAggregate (65) +- Exchange (64) +- * HashAggregate (63) - +- * Project (62) - +- * Filter (61) - +- * ColumnarToRow (60) + +- * ColumnarToRow (62) + +- CometProject (61) + +- CometFilter (60) +- CometScan parquet spark_catalog.default.store_sales (59) @@ -78,16 +78,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,0), LessThanOrEqual(ss_quantity,5), Or(Or(And(GreaterThanOrEqual(ss_list_price,8.00),LessThanOrEqual(ss_list_price,18.00)),And(GreaterThanOrEqual(ss_coupon_amt,459.00),LessThanOrEqual(ss_coupon_amt,1459.00))),And(GreaterThanOrEqual(ss_wholesale_cost,57.00),LessThanOrEqual(ss_wholesale_cost,77.00)))] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] Condition : (((isnotnull(ss_quantity#1) AND (ss_quantity#1 >= 0)) AND (ss_quantity#1 <= 5)) AND ((((ss_list_price#3 >= 8.00) AND (ss_list_price#3 <= 18.00)) OR ((ss_coupon_amt#4 >= 459.00) AND (ss_coupon_amt#4 <= 1459.00))) OR ((ss_wholesale_cost#2 >= 57.00) AND (ss_wholesale_cost#2 <= 77.00)))) -(4) Project [codegen id : 1] -Output [1]: [ss_list_price#3] +(3) CometProject Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] +Arguments: [ss_list_price#3], [ss_list_price#3] + +(4) ColumnarToRow [codegen id : 1] +Input [1]: [ss_list_price#3] (5) HashAggregate [codegen id : 1] Input [1]: [ss_list_price#3] @@ -132,16 +132,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,6), LessThanOrEqual(ss_quantity,10), Or(Or(And(GreaterThanOrEqual(ss_list_price,90.00),LessThanOrEqual(ss_list_price,100.00)),And(GreaterThanOrEqual(ss_coupon_amt,2323.00),LessThanOrEqual(ss_coupon_amt,3323.00))),And(GreaterThanOrEqual(ss_wholesale_cost,31.00),LessThanOrEqual(ss_wholesale_cost,51.00)))] ReadSchema: struct -(12) ColumnarToRow [codegen id : 3] -Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] - -(13) Filter [codegen id : 3] +(12) CometFilter Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] Condition : (((isnotnull(ss_quantity#16) AND (ss_quantity#16 >= 6)) AND (ss_quantity#16 <= 10)) AND ((((ss_list_price#18 >= 90.00) AND (ss_list_price#18 <= 100.00)) OR ((ss_coupon_amt#19 >= 2323.00) AND (ss_coupon_amt#19 <= 3323.00))) OR ((ss_wholesale_cost#17 >= 31.00) AND (ss_wholesale_cost#17 <= 51.00)))) -(14) Project [codegen id : 3] -Output [1]: [ss_list_price#18] +(13) CometProject Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] +Arguments: [ss_list_price#18], [ss_list_price#18] + +(14) ColumnarToRow [codegen id : 3] +Input [1]: [ss_list_price#18] (15) HashAggregate [codegen id : 3] Input [1]: [ss_list_price#18] @@ -194,16 +194,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,11), LessThanOrEqual(ss_quantity,15), Or(Or(And(GreaterThanOrEqual(ss_list_price,142.00),LessThanOrEqual(ss_list_price,152.00)),And(GreaterThanOrEqual(ss_coupon_amt,12214.00),LessThanOrEqual(ss_coupon_amt,13214.00))),And(GreaterThanOrEqual(ss_wholesale_cost,79.00),LessThanOrEqual(ss_wholesale_cost,99.00)))] ReadSchema: struct -(24) ColumnarToRow [codegen id : 6] -Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] - -(25) Filter [codegen id : 6] +(24) CometFilter Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] Condition : (((isnotnull(ss_quantity#31) AND (ss_quantity#31 >= 11)) AND (ss_quantity#31 <= 15)) AND ((((ss_list_price#33 >= 142.00) AND (ss_list_price#33 <= 152.00)) OR ((ss_coupon_amt#34 >= 12214.00) AND (ss_coupon_amt#34 <= 13214.00))) OR ((ss_wholesale_cost#32 >= 79.00) AND (ss_wholesale_cost#32 <= 99.00)))) -(26) Project [codegen id : 6] -Output [1]: [ss_list_price#33] +(25) CometProject Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] +Arguments: [ss_list_price#33], [ss_list_price#33] + +(26) ColumnarToRow [codegen id : 6] +Input [1]: [ss_list_price#33] (27) HashAggregate [codegen id : 6] Input [1]: [ss_list_price#33] @@ -256,16 +256,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,16), LessThanOrEqual(ss_quantity,20), Or(Or(And(GreaterThanOrEqual(ss_list_price,135.00),LessThanOrEqual(ss_list_price,145.00)),And(GreaterThanOrEqual(ss_coupon_amt,6071.00),LessThanOrEqual(ss_coupon_amt,7071.00))),And(GreaterThanOrEqual(ss_wholesale_cost,38.00),LessThanOrEqual(ss_wholesale_cost,58.00)))] ReadSchema: struct -(36) ColumnarToRow [codegen id : 9] -Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] - -(37) Filter [codegen id : 9] +(36) CometFilter Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] Condition : (((isnotnull(ss_quantity#46) AND (ss_quantity#46 >= 16)) AND (ss_quantity#46 <= 20)) AND ((((ss_list_price#48 >= 135.00) AND (ss_list_price#48 <= 145.00)) OR ((ss_coupon_amt#49 >= 6071.00) AND (ss_coupon_amt#49 <= 7071.00))) OR ((ss_wholesale_cost#47 >= 38.00) AND (ss_wholesale_cost#47 <= 58.00)))) -(38) Project [codegen id : 9] -Output [1]: [ss_list_price#48] +(37) CometProject Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] +Arguments: [ss_list_price#48], [ss_list_price#48] + +(38) ColumnarToRow [codegen id : 9] +Input [1]: [ss_list_price#48] (39) HashAggregate [codegen id : 9] Input [1]: [ss_list_price#48] @@ -318,16 +318,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,25), Or(Or(And(GreaterThanOrEqual(ss_list_price,122.00),LessThanOrEqual(ss_list_price,132.00)),And(GreaterThanOrEqual(ss_coupon_amt,836.00),LessThanOrEqual(ss_coupon_amt,1836.00))),And(GreaterThanOrEqual(ss_wholesale_cost,17.00),LessThanOrEqual(ss_wholesale_cost,37.00)))] ReadSchema: struct -(48) ColumnarToRow [codegen id : 12] -Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] - -(49) Filter [codegen id : 12] +(48) CometFilter Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] Condition : (((isnotnull(ss_quantity#61) AND (ss_quantity#61 >= 21)) AND (ss_quantity#61 <= 25)) AND ((((ss_list_price#63 >= 122.00) AND (ss_list_price#63 <= 132.00)) OR ((ss_coupon_amt#64 >= 836.00) AND (ss_coupon_amt#64 <= 1836.00))) OR ((ss_wholesale_cost#62 >= 17.00) AND (ss_wholesale_cost#62 <= 37.00)))) -(50) Project [codegen id : 12] -Output [1]: [ss_list_price#63] +(49) CometProject Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] +Arguments: [ss_list_price#63], [ss_list_price#63] + +(50) ColumnarToRow [codegen id : 12] +Input [1]: [ss_list_price#63] (51) HashAggregate [codegen id : 12] Input [1]: [ss_list_price#63] @@ -380,16 +380,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,26), LessThanOrEqual(ss_quantity,30), Or(Or(And(GreaterThanOrEqual(ss_list_price,154.00),LessThanOrEqual(ss_list_price,164.00)),And(GreaterThanOrEqual(ss_coupon_amt,7326.00),LessThanOrEqual(ss_coupon_amt,8326.00))),And(GreaterThanOrEqual(ss_wholesale_cost,7.00),LessThanOrEqual(ss_wholesale_cost,27.00)))] ReadSchema: struct -(60) ColumnarToRow [codegen id : 15] -Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] - -(61) Filter [codegen id : 15] +(60) CometFilter Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] Condition : (((isnotnull(ss_quantity#76) AND (ss_quantity#76 >= 26)) AND (ss_quantity#76 <= 30)) AND ((((ss_list_price#78 >= 154.00) AND (ss_list_price#78 <= 164.00)) OR ((ss_coupon_amt#79 >= 7326.00) AND (ss_coupon_amt#79 <= 8326.00))) OR ((ss_wholesale_cost#77 >= 7.00) AND (ss_wholesale_cost#77 <= 27.00)))) -(62) Project [codegen id : 15] -Output [1]: [ss_list_price#78] +(61) CometProject Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] +Arguments: [ss_list_price#78], [ss_list_price#78] + +(62) ColumnarToRow [codegen id : 15] +Input [1]: [ss_list_price#78] (63) HashAggregate [codegen id : 15] Input [1]: [ss_list_price#78] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28/simplified.txt index 7f5641ec2..a7a3f9537 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28/simplified.txt @@ -14,10 +14,10 @@ WholeStageCodegen (18) Exchange [ss_list_price] #2 WholeStageCodegen (1) HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #3 @@ -32,10 +32,10 @@ WholeStageCodegen (18) Exchange [ss_list_price] #5 WholeStageCodegen (3) HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #6 @@ -50,10 +50,10 @@ WholeStageCodegen (18) Exchange [ss_list_price] #8 WholeStageCodegen (6) HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #9 @@ -68,10 +68,10 @@ WholeStageCodegen (18) Exchange [ss_list_price] #11 WholeStageCodegen (9) HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #12 @@ -86,10 +86,10 @@ WholeStageCodegen (18) Exchange [ss_list_price] #14 WholeStageCodegen (12) HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #15 @@ -104,8 +104,8 @@ WholeStageCodegen (18) Exchange [ss_list_price] #17 WholeStageCodegen (15) HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/explain.txt index 9a5be8c4d..522754cbc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/explain.txt @@ -17,27 +17,27 @@ TakeOrderedAndProject (40) : : : : : +- * BroadcastHashJoin Inner BuildRight (14) : : : : : :- * Project (9) : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) + : : : : : : :- * ColumnarToRow (3) + : : : : : : : +- CometFilter (2) : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : : : : +- BroadcastExchange (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) + : : : : : : +- * ColumnarToRow (6) + : : : : : : +- CometFilter (5) : : : : : : +- CometScan parquet spark_catalog.default.store_returns (4) : : : : : +- BroadcastExchange (13) - : : : : : +- * Filter (12) - : : : : : +- * ColumnarToRow (11) + : : : : : +- * ColumnarToRow (12) + : : : : : +- CometFilter (11) : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (10) : : : : +- ReusedExchange (16) : : : +- ReusedExchange (19) : : +- ReusedExchange (22) : +- BroadcastExchange (28) - : +- * Filter (27) - : +- * ColumnarToRow (26) + : +- * ColumnarToRow (27) + : +- CometFilter (26) : +- CometScan parquet spark_catalog.default.store (25) +- BroadcastExchange (34) - +- * Filter (33) - +- * ColumnarToRow (32) + +- * ColumnarToRow (33) + +- CometFilter (32) +- CometScan parquet spark_catalog.default.item (31) @@ -49,12 +49,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 8] +(2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] +Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) -(3) Filter [codegen id : 8] +(3) ColumnarToRow [codegen id : 8] Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) (4) Scan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] @@ -64,12 +64,12 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#12), dynamicpruningexpression(s PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] +Condition : ((isnotnull(sr_customer_sk#9) AND isnotnull(sr_item_sk#8)) AND isnotnull(sr_ticket_number#10)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Condition : ((isnotnull(sr_customer_sk#9) AND isnotnull(sr_item_sk#8)) AND isnotnull(sr_ticket_number#10)) (7) BroadcastExchange Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] @@ -93,12 +93,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] +(11) CometFilter Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] +Condition : (isnotnull(cs_bill_customer_sk#14) AND isnotnull(cs_item_sk#15)) -(12) Filter [codegen id : 2] +(12) ColumnarToRow [codegen id : 2] Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Condition : (isnotnull(cs_bill_customer_sk#14) AND isnotnull(cs_item_sk#15)) (13) BroadcastExchange Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] @@ -160,12 +160,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 6] +(26) CometFilter Input [3]: [s_store_sk#22, s_store_id#23, s_store_name#24] +Condition : isnotnull(s_store_sk#22) -(27) Filter [codegen id : 6] +(27) ColumnarToRow [codegen id : 6] Input [3]: [s_store_sk#22, s_store_id#23, s_store_name#24] -Condition : isnotnull(s_store_sk#22) (28) BroadcastExchange Input [3]: [s_store_sk#22, s_store_id#23, s_store_name#24] @@ -188,12 +188,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 7] +(32) CometFilter Input [3]: [i_item_sk#25, i_item_id#26, i_item_desc#27] +Condition : isnotnull(i_item_sk#25) -(33) Filter [codegen id : 7] +(33) ColumnarToRow [codegen id : 7] Input [3]: [i_item_sk#25, i_item_id#26, i_item_desc#27] -Condition : isnotnull(i_item_sk#25) (34) BroadcastExchange Input [3]: [i_item_sk#25, i_item_id#26, i_item_desc#27] @@ -235,9 +235,9 @@ Arguments: 100, [i_item_id#26 ASC NULLS FIRST, i_item_desc#27 ASC NULLS FIRST, s Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 BroadcastExchange (45) -+- * Project (44) - +- * Filter (43) - +- * ColumnarToRow (42) ++- * ColumnarToRow (44) + +- CometProject (43) + +- CometFilter (42) +- CometScan parquet spark_catalog.default.date_dim (41) @@ -248,16 +248,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,9), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(42) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#19, d_year#40, d_moy#41] - -(43) Filter [codegen id : 1] +(42) CometFilter Input [3]: [d_date_sk#19, d_year#40, d_moy#41] Condition : ((((isnotnull(d_moy#41) AND isnotnull(d_year#40)) AND (d_moy#41 = 9)) AND (d_year#40 = 1999)) AND isnotnull(d_date_sk#19)) -(44) Project [codegen id : 1] -Output [1]: [d_date_sk#19] +(43) CometProject Input [3]: [d_date_sk#19, d_year#40, d_moy#41] +Arguments: [d_date_sk#19], [d_date_sk#19] + +(44) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#19] (45) BroadcastExchange Input [1]: [d_date_sk#19] @@ -265,9 +265,9 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Subquery:2 Hosting operator id = 4 Hosting Expression = sr_returned_date_sk#12 IN dynamicpruning#13 BroadcastExchange (50) -+- * Project (49) - +- * Filter (48) - +- * ColumnarToRow (47) ++- * ColumnarToRow (49) + +- CometProject (48) + +- CometFilter (47) +- CometScan parquet spark_catalog.default.date_dim (46) @@ -278,16 +278,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,9), LessThanOrEqual(d_moy,12), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(47) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#20, d_year#42, d_moy#43] - -(48) Filter [codegen id : 1] +(47) CometFilter Input [3]: [d_date_sk#20, d_year#42, d_moy#43] Condition : (((((isnotnull(d_moy#43) AND isnotnull(d_year#42)) AND (d_moy#43 >= 9)) AND (d_moy#43 <= 12)) AND (d_year#42 = 1999)) AND isnotnull(d_date_sk#20)) -(49) Project [codegen id : 1] -Output [1]: [d_date_sk#20] +(48) CometProject Input [3]: [d_date_sk#20, d_year#42, d_moy#43] +Arguments: [d_date_sk#20], [d_date_sk#20] + +(49) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#20] (50) BroadcastExchange Input [1]: [d_date_sk#20] @@ -295,9 +295,9 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Subquery:3 Hosting operator id = 10 Hosting Expression = cs_sold_date_sk#17 IN dynamicpruning#18 BroadcastExchange (55) -+- * Project (54) - +- * Filter (53) - +- * ColumnarToRow (52) ++- * ColumnarToRow (54) + +- CometProject (53) + +- CometFilter (52) +- CometScan parquet spark_catalog.default.date_dim (51) @@ -308,16 +308,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(52) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#21, d_year#44] - -(53) Filter [codegen id : 1] +(52) CometFilter Input [2]: [d_date_sk#21, d_year#44] Condition : (d_year#44 IN (1999,2000,2001) AND isnotnull(d_date_sk#21)) -(54) Project [codegen id : 1] -Output [1]: [d_date_sk#21] +(53) CometProject Input [2]: [d_date_sk#21, d_year#44] +Arguments: [d_date_sk#21], [d_date_sk#21] + +(54) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#21] (55) BroadcastExchange Input [1]: [d_date_sk#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/simplified.txt index 101893d07..68a127d35 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/simplified.txt @@ -19,47 +19,47 @@ TakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales BroadcastHashJoin [sr_customer_sk,sr_item_sk,cs_bill_customer_sk,cs_item_sk] Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] BroadcastHashJoin [ss_customer_sk,ss_item_sk,ss_ticket_number,sr_customer_sk,sr_item_sk,sr_ticket_number] - Filter [ss_customer_sk,ss_item_sk,ss_ticket_number,ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_customer_sk,ss_item_sk,ss_ticket_number,ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_moy,d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Filter [sr_customer_sk,sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [sr_customer_sk,sr_item_sk,sr_ticket_number] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #4 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_moy,d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #5 WholeStageCodegen (2) - Filter [cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_bill_customer_sk,cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #3 BroadcastExchange #6 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [d_date_sk] #2 @@ -70,14 +70,14 @@ TakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales InputAdapter BroadcastExchange #7 WholeStageCodegen (6) - Filter [s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] InputAdapter BroadcastExchange #8 WholeStageCodegen (7) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3/explain.txt index 6fdc0d461..e89cfe1ff 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3/explain.txt @@ -7,18 +7,18 @@ TakeOrderedAndProject (21) +- * BroadcastHashJoin Inner BuildRight (16) :- * Project (10) : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) + : :- * ColumnarToRow (4) + : : +- CometProject (3) + : : +- CometFilter (2) : : +- CometScan parquet spark_catalog.default.date_dim (1) : +- BroadcastExchange (8) - : +- * Filter (7) - : +- * ColumnarToRow (6) + : +- * ColumnarToRow (7) + : +- CometFilter (6) : +- CometScan parquet spark_catalog.default.store_sales (5) +- BroadcastExchange (15) - +- * Project (14) - +- * Filter (13) - +- * ColumnarToRow (12) + +- * ColumnarToRow (14) + +- CometProject (13) + +- CometFilter (12) +- CometScan parquet spark_catalog.default.item (11) @@ -29,16 +29,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Condition : ((isnotnull(d_moy#3) AND (d_moy#3 = 11)) AND isnotnull(d_date_sk#1)) -(4) Project [codegen id : 3] -Output [2]: [d_date_sk#1, d_year#2] +(3) CometProject Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] + +(4) ColumnarToRow [codegen id : 3] +Input [2]: [d_date_sk#1, d_year#2] (5) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] @@ -48,12 +48,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] +(6) CometFilter Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Condition : isnotnull(ss_item_sk#4) -(7) Filter [codegen id : 1] +(7) ColumnarToRow [codegen id : 1] Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_item_sk#4) (8) BroadcastExchange Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] @@ -76,16 +76,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,128), IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] - -(13) Filter [codegen id : 2] +(12) CometFilter Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] Condition : ((isnotnull(i_manufact_id#10) AND (i_manufact_id#10 = 128)) AND isnotnull(i_item_sk#7)) -(14) Project [codegen id : 2] -Output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] +(13) CometProject Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9], [i_item_sk#7, i_brand_id#8, i_brand#9] + +(14) ColumnarToRow [codegen id : 2] +Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] (15) BroadcastExchange Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3/simplified.txt index a869b17dd..3946c0cd8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3/simplified.txt @@ -9,23 +9,23 @@ TakeOrderedAndProject [d_year,sum_agg,brand_id,brand] BroadcastHashJoin [ss_item_sk,i_item_sk] Project [d_year,ss_item_sk,ss_ext_sales_price] BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk,d_year] - Filter [d_moy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_year] + CometFilter [d_moy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #2 WholeStageCodegen (1) - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Project [i_item_sk,i_brand_id,i_brand] - Filter [i_manufact_id,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_item_sk,i_brand_id,i_brand] + CometFilter [i_manufact_id,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/explain.txt index 344c57efd..098d00824 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/explain.txt @@ -14,13 +14,13 @@ TakeOrderedAndProject (49) : : : +- * BroadcastHashJoin Inner BuildRight (11) : : : :- * Project (6) : : : : +- * BroadcastHashJoin Inner BuildRight (5) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.web_returns (1) : : : : +- ReusedExchange (4) : : : +- BroadcastExchange (10) - : : : +- * Filter (9) - : : : +- * ColumnarToRow (8) + : : : +- * ColumnarToRow (9) + : : : +- CometFilter (8) : : : +- CometScan parquet spark_catalog.default.customer_address (7) : : +- BroadcastExchange (33) : : +- * Filter (32) @@ -34,19 +34,19 @@ TakeOrderedAndProject (49) : : +- * BroadcastHashJoin Inner BuildRight (24) : : :- * Project (22) : : : +- * BroadcastHashJoin Inner BuildRight (21) - : : : :- * Filter (19) - : : : : +- * ColumnarToRow (18) + : : : :- * ColumnarToRow (19) + : : : : +- CometFilter (18) : : : : +- CometScan parquet spark_catalog.default.web_returns (17) : : : +- ReusedExchange (20) : : +- ReusedExchange (23) : +- BroadcastExchange (39) - : +- * Filter (38) - : +- * ColumnarToRow (37) + : +- * ColumnarToRow (38) + : +- CometFilter (37) : +- CometScan parquet spark_catalog.default.customer (36) +- BroadcastExchange (46) - +- * Project (45) - +- * Filter (44) - +- * ColumnarToRow (43) + +- * ColumnarToRow (45) + +- CometProject (44) + +- CometFilter (43) +- CometScan parquet spark_catalog.default.customer_address (42) @@ -58,12 +58,12 @@ PartitionFilters: [isnotnull(wr_returned_date_sk#4), dynamicpruningexpression(wr PushedFilters: [IsNotNull(wr_returning_addr_sk), IsNotNull(wr_returning_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] +Condition : (isnotnull(wr_returning_addr_sk#2) AND isnotnull(wr_returning_customer_sk#1)) -(3) Filter [codegen id : 3] +(3) ColumnarToRow [codegen id : 3] Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] -Condition : (isnotnull(wr_returning_addr_sk#2) AND isnotnull(wr_returning_customer_sk#1)) (4) ReusedExchange [Reuses operator id: 54] Output [1]: [d_date_sk#6] @@ -85,12 +85,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_state)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] +(8) CometFilter Input [2]: [ca_address_sk#7, ca_state#8] +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_state#8)) -(9) Filter [codegen id : 2] +(9) ColumnarToRow [codegen id : 2] Input [2]: [ca_address_sk#7, ca_state#8] -Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_state#8)) (10) BroadcastExchange Input [2]: [ca_address_sk#7, ca_state#8] @@ -136,12 +136,12 @@ PartitionFilters: [isnotnull(wr_returned_date_sk#18), dynamicpruningexpression(w PushedFilters: [IsNotNull(wr_returning_addr_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 6] +(18) CometFilter Input [4]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18] +Condition : isnotnull(wr_returning_addr_sk#16) -(19) Filter [codegen id : 6] +(19) ColumnarToRow [codegen id : 6] Input [4]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18] -Condition : isnotnull(wr_returning_addr_sk#16) (20) ReusedExchange [Reuses operator id: 54] Output [1]: [d_date_sk#20] @@ -230,12 +230,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 9] +(37) CometFilter Input [14]: [c_customer_sk#33, c_customer_id#34, c_current_addr_sk#35, c_salutation#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_day#40, c_birth_month#41, c_birth_year#42, c_birth_country#43, c_login#44, c_email_address#45, c_last_review_date#46] +Condition : (isnotnull(c_customer_sk#33) AND isnotnull(c_current_addr_sk#35)) -(38) Filter [codegen id : 9] +(38) ColumnarToRow [codegen id : 9] Input [14]: [c_customer_sk#33, c_customer_id#34, c_current_addr_sk#35, c_salutation#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_day#40, c_birth_month#41, c_birth_year#42, c_birth_country#43, c_login#44, c_email_address#45, c_last_review_date#46] -Condition : (isnotnull(c_customer_sk#33) AND isnotnull(c_current_addr_sk#35)) (39) BroadcastExchange Input [14]: [c_customer_sk#33, c_customer_id#34, c_current_addr_sk#35, c_salutation#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_day#40, c_birth_month#41, c_birth_year#42, c_birth_country#43, c_login#44, c_email_address#45, c_last_review_date#46] @@ -258,16 +258,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)] ReadSchema: struct -(43) ColumnarToRow [codegen id : 10] -Input [2]: [ca_address_sk#47, ca_state#48] - -(44) Filter [codegen id : 10] +(43) CometFilter Input [2]: [ca_address_sk#47, ca_state#48] Condition : ((isnotnull(ca_state#48) AND (ca_state#48 = GA)) AND isnotnull(ca_address_sk#47)) -(45) Project [codegen id : 10] -Output [1]: [ca_address_sk#47] +(44) CometProject Input [2]: [ca_address_sk#47, ca_state#48] +Arguments: [ca_address_sk#47], [ca_address_sk#47] + +(45) ColumnarToRow [codegen id : 10] +Input [1]: [ca_address_sk#47] (46) BroadcastExchange Input [1]: [ca_address_sk#47] @@ -291,9 +291,9 @@ Arguments: 100, [c_customer_id#34 ASC NULLS FIRST, c_salutation#36 ASC NULLS FIR Subquery:1 Hosting operator id = 1 Hosting Expression = wr_returned_date_sk#4 IN dynamicpruning#5 BroadcastExchange (54) -+- * Project (53) - +- * Filter (52) - +- * ColumnarToRow (51) ++- * ColumnarToRow (53) + +- CometProject (52) + +- CometFilter (51) +- CometScan parquet spark_catalog.default.date_dim (50) @@ -304,16 +304,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(51) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#6, d_year#49] - -(52) Filter [codegen id : 1] +(51) CometFilter Input [2]: [d_date_sk#6, d_year#49] Condition : ((isnotnull(d_year#49) AND (d_year#49 = 2002)) AND isnotnull(d_date_sk#6)) -(53) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(52) CometProject Input [2]: [d_date_sk#6, d_year#49] +Arguments: [d_date_sk#6], [d_date_sk#6] + +(53) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#6] (54) BroadcastExchange Input [1]: [d_date_sk#6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/simplified.txt index b0935ef25..365f7f973 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/simplified.txt @@ -16,26 +16,26 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_pre BroadcastHashJoin [wr_returning_addr_sk,ca_address_sk] Project [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] BroadcastHashJoin [wr_returned_date_sk,d_date_sk] - Filter [wr_returning_addr_sk,wr_returning_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [wr_returning_addr_sk,wr_returning_customer_sk] CometScan parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [d_date_sk] #2 InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Filter [ca_address_sk,ca_state] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ca_address_sk,ca_state] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #4 @@ -55,9 +55,9 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_pre BroadcastHashJoin [wr_returning_addr_sk,ca_address_sk] Project [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] BroadcastHashJoin [wr_returned_date_sk,d_date_sk] - Filter [wr_returning_addr_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [wr_returning_addr_sk] CometScan parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -67,15 +67,15 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_pre InputAdapter BroadcastExchange #7 WholeStageCodegen (9) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_current_addr_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] InputAdapter BroadcastExchange #8 WholeStageCodegen (10) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_state,ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/explain.txt index 4810517ca..e20d45486 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/explain.txt @@ -16,13 +16,13 @@ : : : : : +- * BroadcastHashJoin Inner BuildRight (11) : : : : : :- * Project (6) : : : : : : +- * BroadcastHashJoin Inner BuildRight (5) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) + : : : : : : :- * ColumnarToRow (3) + : : : : : : : +- CometFilter (2) : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : : : : +- ReusedExchange (4) : : : : : +- BroadcastExchange (10) - : : : : : +- * Filter (9) - : : : : : +- * ColumnarToRow (8) + : : : : : +- * ColumnarToRow (9) + : : : : : +- CometFilter (8) : : : : : +- CometScan parquet spark_catalog.default.customer_address (7) : : : : +- BroadcastExchange (28) : : : : +- * HashAggregate (27) @@ -32,8 +32,8 @@ : : : : +- * BroadcastHashJoin Inner BuildRight (23) : : : : :- * Project (21) : : : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : : : :- * Filter (18) - : : : : : : +- * ColumnarToRow (17) + : : : : : :- * ColumnarToRow (18) + : : : : : : +- CometFilter (17) : : : : : : +- CometScan parquet spark_catalog.default.store_sales (16) : : : : : +- ReusedExchange (19) : : : : +- ReusedExchange (22) @@ -45,8 +45,8 @@ : : : +- * BroadcastHashJoin Inner BuildRight (37) : : : :- * Project (35) : : : : +- * BroadcastHashJoin Inner BuildRight (34) - : : : : :- * Filter (32) - : : : : : +- * ColumnarToRow (31) + : : : : :- * ColumnarToRow (32) + : : : : : +- CometFilter (31) : : : : : +- CometScan parquet spark_catalog.default.store_sales (30) : : : : +- ReusedExchange (33) : : : +- ReusedExchange (36) @@ -58,8 +58,8 @@ : : +- * BroadcastHashJoin Inner BuildRight (52) : : :- * Project (50) : : : +- * BroadcastHashJoin Inner BuildRight (49) - : : : :- * Filter (47) - : : : : +- * ColumnarToRow (46) + : : : :- * ColumnarToRow (47) + : : : : +- CometFilter (46) : : : : +- CometScan parquet spark_catalog.default.web_sales (45) : : : +- ReusedExchange (48) : : +- ReusedExchange (51) @@ -71,8 +71,8 @@ : +- * BroadcastHashJoin Inner BuildRight (66) : :- * Project (64) : : +- * BroadcastHashJoin Inner BuildRight (63) - : : :- * Filter (61) - : : : +- * ColumnarToRow (60) + : : :- * ColumnarToRow (61) + : : : +- CometFilter (60) : : : +- CometScan parquet spark_catalog.default.web_sales (59) : : +- ReusedExchange (62) : +- ReusedExchange (65) @@ -84,8 +84,8 @@ +- * BroadcastHashJoin Inner BuildRight (81) :- * Project (79) : +- * BroadcastHashJoin Inner BuildRight (78) - : :- * Filter (76) - : : +- * ColumnarToRow (75) + : :- * ColumnarToRow (76) + : : +- CometFilter (75) : : +- CometScan parquet spark_catalog.default.web_sales (74) : +- ReusedExchange (77) +- ReusedExchange (80) @@ -99,12 +99,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_addr_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Condition : isnotnull(ss_addr_sk#1) -(3) Filter [codegen id : 3] +(3) ColumnarToRow [codegen id : 3] Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_addr_sk#1) (4) ReusedExchange [Reuses operator id: 94] Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] @@ -126,12 +126,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] +(8) CometFilter Input [2]: [ca_address_sk#8, ca_county#9] +Condition : (isnotnull(ca_address_sk#8) AND isnotnull(ca_county#9)) -(9) Filter [codegen id : 2] +(9) ColumnarToRow [codegen id : 2] Input [2]: [ca_address_sk#8, ca_county#9] -Condition : (isnotnull(ca_address_sk#8) AND isnotnull(ca_county#9)) (10) BroadcastExchange Input [2]: [ca_address_sk#8, ca_county#9] @@ -173,12 +173,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#16), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_addr_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 6] +(17) CometFilter Input [3]: [ss_addr_sk#14, ss_ext_sales_price#15, ss_sold_date_sk#16] +Condition : isnotnull(ss_addr_sk#14) -(18) Filter [codegen id : 6] +(18) ColumnarToRow [codegen id : 6] Input [3]: [ss_addr_sk#14, ss_ext_sales_price#15, ss_sold_date_sk#16] -Condition : isnotnull(ss_addr_sk#14) (19) ReusedExchange [Reuses operator id: 98] Output [3]: [d_date_sk#18, d_year#19, d_qoy#20] @@ -242,12 +242,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#28), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_addr_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 10] +(31) CometFilter Input [3]: [ss_addr_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28] +Condition : isnotnull(ss_addr_sk#26) -(32) Filter [codegen id : 10] +(32) ColumnarToRow [codegen id : 10] Input [3]: [ss_addr_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28] -Condition : isnotnull(ss_addr_sk#26) (33) ReusedExchange [Reuses operator id: 102] Output [3]: [d_date_sk#30, d_year#31, d_qoy#32] @@ -315,12 +315,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#40), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_addr_sk)] ReadSchema: struct -(46) ColumnarToRow [codegen id : 14] +(46) CometFilter Input [3]: [ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40] +Condition : isnotnull(ws_bill_addr_sk#38) -(47) Filter [codegen id : 14] +(47) ColumnarToRow [codegen id : 14] Input [3]: [ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40] -Condition : isnotnull(ws_bill_addr_sk#38) (48) ReusedExchange [Reuses operator id: 94] Output [3]: [d_date_sk#42, d_year#43, d_qoy#44] @@ -384,12 +384,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#53), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_addr_sk)] ReadSchema: struct -(60) ColumnarToRow [codegen id : 18] +(60) CometFilter Input [3]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53] +Condition : isnotnull(ws_bill_addr_sk#51) -(61) Filter [codegen id : 18] +(61) ColumnarToRow [codegen id : 18] Input [3]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53] -Condition : isnotnull(ws_bill_addr_sk#51) (62) ReusedExchange [Reuses operator id: 98] Output [3]: [d_date_sk#55, d_year#56, d_qoy#57] @@ -457,12 +457,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#65), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_addr_sk)] ReadSchema: struct -(75) ColumnarToRow [codegen id : 22] +(75) CometFilter Input [3]: [ws_bill_addr_sk#63, ws_ext_sales_price#64, ws_sold_date_sk#65] +Condition : isnotnull(ws_bill_addr_sk#63) -(76) Filter [codegen id : 22] +(76) ColumnarToRow [codegen id : 22] Input [3]: [ws_bill_addr_sk#63, ws_ext_sales_price#64, ws_sold_date_sk#65] -Condition : isnotnull(ws_bill_addr_sk#63) (77) ReusedExchange [Reuses operator id: 102] Output [3]: [d_date_sk#67, d_year#68, d_qoy#69] @@ -534,8 +534,8 @@ Arguments: [ca_county#9 ASC NULLS FIRST], true, 0 Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 BroadcastExchange (94) -+- * Filter (93) - +- * ColumnarToRow (92) ++- * ColumnarToRow (93) + +- CometFilter (92) +- CometScan parquet spark_catalog.default.date_dim (91) @@ -546,12 +546,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,1), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(92) ColumnarToRow [codegen id : 1] +(92) CometFilter Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] +Condition : ((((isnotnull(d_qoy#7) AND isnotnull(d_year#6)) AND (d_qoy#7 = 1)) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) -(93) Filter [codegen id : 1] +(93) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Condition : ((((isnotnull(d_qoy#7) AND isnotnull(d_year#6)) AND (d_qoy#7 = 1)) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) (94) BroadcastExchange Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] @@ -559,8 +559,8 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint) Subquery:2 Hosting operator id = 16 Hosting Expression = ss_sold_date_sk#16 IN dynamicpruning#17 BroadcastExchange (98) -+- * Filter (97) - +- * ColumnarToRow (96) ++- * ColumnarToRow (97) + +- CometFilter (96) +- CometScan parquet spark_catalog.default.date_dim (95) @@ -571,12 +571,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(96) ColumnarToRow [codegen id : 1] +(96) CometFilter Input [3]: [d_date_sk#18, d_year#19, d_qoy#20] +Condition : ((((isnotnull(d_qoy#20) AND isnotnull(d_year#19)) AND (d_qoy#20 = 2)) AND (d_year#19 = 2000)) AND isnotnull(d_date_sk#18)) -(97) Filter [codegen id : 1] +(97) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#18, d_year#19, d_qoy#20] -Condition : ((((isnotnull(d_qoy#20) AND isnotnull(d_year#19)) AND (d_qoy#20 = 2)) AND (d_year#19 = 2000)) AND isnotnull(d_date_sk#18)) (98) BroadcastExchange Input [3]: [d_date_sk#18, d_year#19, d_qoy#20] @@ -584,8 +584,8 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint) Subquery:3 Hosting operator id = 30 Hosting Expression = ss_sold_date_sk#28 IN dynamicpruning#29 BroadcastExchange (102) -+- * Filter (101) - +- * ColumnarToRow (100) ++- * ColumnarToRow (101) + +- CometFilter (100) +- CometScan parquet spark_catalog.default.date_dim (99) @@ -596,12 +596,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,3), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(100) ColumnarToRow [codegen id : 1] +(100) CometFilter Input [3]: [d_date_sk#30, d_year#31, d_qoy#32] +Condition : ((((isnotnull(d_qoy#32) AND isnotnull(d_year#31)) AND (d_qoy#32 = 3)) AND (d_year#31 = 2000)) AND isnotnull(d_date_sk#30)) -(101) Filter [codegen id : 1] +(101) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#30, d_year#31, d_qoy#32] -Condition : ((((isnotnull(d_qoy#32) AND isnotnull(d_year#31)) AND (d_qoy#32 = 3)) AND (d_year#31 = 2000)) AND isnotnull(d_date_sk#30)) (102) BroadcastExchange Input [3]: [d_date_sk#30, d_year#31, d_qoy#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/simplified.txt index 67bc75cd0..f4bf6a89d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/simplified.txt @@ -20,25 +20,25 @@ WholeStageCodegen (25) BroadcastHashJoin [ss_addr_sk,ca_address_sk] Project [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_addr_sk] CometScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_qoy,d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter ReusedExchange [d_date_sk,d_year,d_qoy] #3 InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - Filter [ca_address_sk,ca_county] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ca_address_sk,ca_county] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] InputAdapter BroadcastExchange #5 @@ -52,16 +52,16 @@ WholeStageCodegen (25) BroadcastHashJoin [ss_addr_sk,ca_address_sk] Project [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_addr_sk] CometScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #7 WholeStageCodegen (1) - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_qoy,d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter ReusedExchange [d_date_sk,d_year,d_qoy] #7 @@ -79,16 +79,16 @@ WholeStageCodegen (25) BroadcastHashJoin [ss_addr_sk,ca_address_sk] Project [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_addr_sk] CometScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #3 BroadcastExchange #10 WholeStageCodegen (1) - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_qoy,d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter ReusedExchange [d_date_sk,d_year,d_qoy] #10 @@ -106,9 +106,9 @@ WholeStageCodegen (25) BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] Project [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_bill_addr_sk] CometScan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -127,9 +127,9 @@ WholeStageCodegen (25) BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] Project [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_bill_addr_sk] CometScan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 InputAdapter @@ -148,9 +148,9 @@ WholeStageCodegen (25) BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] Project [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_bill_addr_sk] CometScan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #3 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/explain.txt index 6fbaa6fab..bd7caaaa1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/explain.txt @@ -8,13 +8,13 @@ : +- * BroadcastHashJoin Inner BuildRight (22) : :- * Project (10) : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) + : : :- * ColumnarToRow (3) + : : : +- CometFilter (2) : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) + : : +- * ColumnarToRow (7) + : : +- CometProject (6) + : : +- CometFilter (5) : : +- CometScan parquet spark_catalog.default.item (4) : +- BroadcastExchange (21) : +- * Filter (20) @@ -23,8 +23,8 @@ : +- * HashAggregate (17) : +- * Project (16) : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Filter (13) - : : +- * ColumnarToRow (12) + : :- * ColumnarToRow (13) + : : +- CometFilter (12) : : +- CometScan parquet spark_catalog.default.catalog_sales (11) : +- ReusedExchange (14) +- ReusedExchange (24) @@ -38,12 +38,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sol PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_ext_discount_amt)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 6] +(2) CometFilter Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] +Condition : (isnotnull(cs_item_sk#1) AND isnotnull(cs_ext_discount_amt#2)) -(3) Filter [codegen id : 6] +(3) ColumnarToRow [codegen id : 6] Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] -Condition : (isnotnull(cs_item_sk#1) AND isnotnull(cs_ext_discount_amt#2)) (4) Scan parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_manufact_id#6] @@ -52,16 +52,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,977), IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [i_item_sk#5, i_manufact_id#6] - -(6) Filter [codegen id : 1] +(5) CometFilter Input [2]: [i_item_sk#5, i_manufact_id#6] Condition : ((isnotnull(i_manufact_id#6) AND (i_manufact_id#6 = 977)) AND isnotnull(i_item_sk#5)) -(7) Project [codegen id : 1] -Output [1]: [i_item_sk#5] +(6) CometProject Input [2]: [i_item_sk#5, i_manufact_id#6] +Arguments: [i_item_sk#5], [i_item_sk#5] + +(7) ColumnarToRow [codegen id : 1] +Input [1]: [i_item_sk#5] (8) BroadcastExchange Input [1]: [i_item_sk#5] @@ -85,12 +85,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#9), dynamicpruningexpression(cs_sol PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 3] +(12) CometFilter Input [3]: [cs_item_sk#7, cs_ext_discount_amt#8, cs_sold_date_sk#9] +Condition : isnotnull(cs_item_sk#7) -(13) Filter [codegen id : 3] +(13) ColumnarToRow [codegen id : 3] Input [3]: [cs_item_sk#7, cs_ext_discount_amt#8, cs_sold_date_sk#9] -Condition : isnotnull(cs_item_sk#7) (14) ReusedExchange [Reuses operator id: 34] Output [1]: [d_date_sk#11] @@ -176,9 +176,9 @@ Results [1]: [MakeDecimal(sum(UnscaledValue(cs_ext_discount_amt#2))#21,17,2) AS Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 BroadcastExchange (34) -+- * Project (33) - +- * Filter (32) - +- * ColumnarToRow (31) ++- * ColumnarToRow (33) + +- CometProject (32) + +- CometFilter (31) +- CometScan parquet spark_catalog.default.date_dim (30) @@ -189,16 +189,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#18, d_date#23] - -(32) Filter [codegen id : 1] +(31) CometFilter Input [2]: [d_date_sk#18, d_date#23] Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 2000-01-27)) AND (d_date#23 <= 2000-04-26)) AND isnotnull(d_date_sk#18)) -(33) Project [codegen id : 1] -Output [1]: [d_date_sk#18] +(32) CometProject Input [2]: [d_date_sk#18, d_date#23] +Arguments: [d_date_sk#18], [d_date_sk#18] + +(33) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#18] (34) BroadcastExchange Input [1]: [d_date_sk#18] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/simplified.txt index a87a0a7e1..146a33fdd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/simplified.txt @@ -10,25 +10,25 @@ WholeStageCodegen (7) BroadcastHashJoin [i_item_sk,cs_item_sk,cs_ext_discount_amt,(1.3 * avg(cs_ext_discount_amt))] Project [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk,cs_ext_discount_amt] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_item_sk,cs_ext_discount_amt] CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Project [i_item_sk] - Filter [i_manufact_id,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_item_sk] + CometFilter [i_manufact_id,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] InputAdapter BroadcastExchange #4 @@ -41,9 +41,9 @@ WholeStageCodegen (7) HashAggregate [cs_item_sk,cs_ext_discount_amt] [sum,count,sum,count] Project [cs_item_sk,cs_ext_discount_amt] BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/explain.txt index 940b542a2..f9541481c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/explain.txt @@ -13,24 +13,24 @@ TakeOrderedAndProject (63) : : +- * BroadcastHashJoin Inner BuildRight (12) : : :- * Project (6) : : : +- * BroadcastHashJoin Inner BuildRight (5) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- ReusedExchange (4) : : +- BroadcastExchange (11) - : : +- * Project (10) - : : +- * Filter (9) - : : +- * ColumnarToRow (8) + : : +- * ColumnarToRow (10) + : : +- CometProject (9) + : : +- CometFilter (8) : : +- CometScan parquet spark_catalog.default.customer_address (7) : +- BroadcastExchange (23) : +- * BroadcastHashJoin LeftSemi BuildRight (22) - : :- * Filter (16) - : : +- * ColumnarToRow (15) + : :- * ColumnarToRow (16) + : : +- CometFilter (15) : : +- CometScan parquet spark_catalog.default.item (14) : +- BroadcastExchange (21) - : +- * Project (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) + : +- * ColumnarToRow (20) + : +- CometProject (19) + : +- CometFilter (18) : +- CometScan parquet spark_catalog.default.item (17) :- * HashAggregate (43) : +- Exchange (42) @@ -41,8 +41,8 @@ TakeOrderedAndProject (63) : : +- * BroadcastHashJoin Inner BuildRight (36) : : :- * Project (34) : : : +- * BroadcastHashJoin Inner BuildRight (33) - : : : :- * Filter (31) - : : : : +- * ColumnarToRow (30) + : : : :- * ColumnarToRow (31) + : : : : +- CometFilter (30) : : : : +- CometScan parquet spark_catalog.default.catalog_sales (29) : : : +- ReusedExchange (32) : : +- ReusedExchange (35) @@ -56,8 +56,8 @@ TakeOrderedAndProject (63) : +- * BroadcastHashJoin Inner BuildRight (51) : :- * Project (49) : : +- * BroadcastHashJoin Inner BuildRight (48) - : : :- * Filter (46) - : : : +- * ColumnarToRow (45) + : : :- * ColumnarToRow (46) + : : : +- CometFilter (45) : : : +- CometScan parquet spark_catalog.default.web_sales (44) : : +- ReusedExchange (47) : +- ReusedExchange (50) @@ -72,12 +72,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) -(3) Filter [codegen id : 5] +(3) ColumnarToRow [codegen id : 5] Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) (4) ReusedExchange [Reuses operator id: 68] Output [1]: [d_date_sk#6] @@ -99,16 +99,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] -Input [2]: [ca_address_sk#7, ca_gmt_offset#8] - -(9) Filter [codegen id : 2] +(8) CometFilter Input [2]: [ca_address_sk#7, ca_gmt_offset#8] Condition : ((isnotnull(ca_gmt_offset#8) AND (ca_gmt_offset#8 = -5.00)) AND isnotnull(ca_address_sk#7)) -(10) Project [codegen id : 2] -Output [1]: [ca_address_sk#7] +(9) CometProject Input [2]: [ca_address_sk#7, ca_gmt_offset#8] +Arguments: [ca_address_sk#7], [ca_address_sk#7] + +(10) ColumnarToRow [codegen id : 2] +Input [1]: [ca_address_sk#7] (11) BroadcastExchange Input [1]: [ca_address_sk#7] @@ -131,12 +131,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 4] +(15) CometFilter Input [2]: [i_item_sk#9, i_manufact_id#10] +Condition : isnotnull(i_item_sk#9) -(16) Filter [codegen id : 4] +(16) ColumnarToRow [codegen id : 4] Input [2]: [i_item_sk#9, i_manufact_id#10] -Condition : isnotnull(i_item_sk#9) (17) Scan parquet spark_catalog.default.item Output [2]: [i_category#11, i_manufact_id#12] @@ -145,16 +145,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Electronics )] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [2]: [i_category#11, i_manufact_id#12] - -(19) Filter [codegen id : 3] +(18) CometFilter Input [2]: [i_category#11, i_manufact_id#12] Condition : (isnotnull(i_category#11) AND (i_category#11 = Electronics )) -(20) Project [codegen id : 3] -Output [1]: [i_manufact_id#12] +(19) CometProject Input [2]: [i_category#11, i_manufact_id#12] +Arguments: [i_manufact_id#12], [i_manufact_id#12] + +(20) ColumnarToRow [codegen id : 3] +Input [1]: [i_manufact_id#12] (21) BroadcastExchange Input [1]: [i_manufact_id#12] @@ -206,12 +206,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#20), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 11] +(30) CometFilter Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Condition : (isnotnull(cs_bill_addr_sk#17) AND isnotnull(cs_item_sk#18)) -(31) Filter [codegen id : 11] +(31) ColumnarToRow [codegen id : 11] Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] -Condition : (isnotnull(cs_bill_addr_sk#17) AND isnotnull(cs_item_sk#18)) (32) ReusedExchange [Reuses operator id: 68] Output [1]: [d_date_sk#22] @@ -278,12 +278,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#33), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 17] +(45) CometFilter Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] +Condition : (isnotnull(ws_bill_addr_sk#31) AND isnotnull(ws_item_sk#30)) -(46) Filter [codegen id : 17] +(46) ColumnarToRow [codegen id : 17] Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] -Condition : (isnotnull(ws_bill_addr_sk#31) AND isnotnull(ws_item_sk#30)) (47) ReusedExchange [Reuses operator id: 68] Output [1]: [d_date_sk#35] @@ -370,9 +370,9 @@ Arguments: 100, [total_sales#48 ASC NULLS FIRST], [i_manufact_id#10, total_sales Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 BroadcastExchange (68) -+- * Project (67) - +- * Filter (66) - +- * ColumnarToRow (65) ++- * ColumnarToRow (67) + +- CometProject (66) + +- CometFilter (65) +- CometScan parquet spark_catalog.default.date_dim (64) @@ -383,16 +383,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,5), IsNotNull(d_date_sk)] ReadSchema: struct -(65) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#49, d_moy#50] - -(66) Filter [codegen id : 1] +(65) CometFilter Input [3]: [d_date_sk#6, d_year#49, d_moy#50] Condition : ((((isnotnull(d_year#49) AND isnotnull(d_moy#50)) AND (d_year#49 = 1998)) AND (d_moy#50 = 5)) AND isnotnull(d_date_sk#6)) -(67) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(66) CometProject Input [3]: [d_date_sk#6, d_year#49, d_moy#50] +Arguments: [d_date_sk#6], [d_date_sk#6] + +(67) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#6] (68) BroadcastExchange Input [1]: [d_date_sk#6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/simplified.txt index 007a3be47..4ab82379f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/simplified.txt @@ -19,43 +19,43 @@ TakeOrderedAndProject [total_sales,i_manufact_id] BroadcastHashJoin [ss_addr_sk,ca_address_sk] Project [ss_item_sk,ss_addr_sk,ss_ext_sales_price] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk,ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_addr_sk,ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_moy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter ReusedExchange [d_date_sk] #3 InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - Project [ca_address_sk] - Filter [ca_gmt_offset,ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_gmt_offset,ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] InputAdapter BroadcastExchange #5 WholeStageCodegen (4) BroadcastHashJoin [i_manufact_id,i_manufact_id] - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] InputAdapter BroadcastExchange #6 WholeStageCodegen (3) - Project [i_manufact_id] - Filter [i_category] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_manufact_id] + CometFilter [i_category] CometScan parquet spark_catalog.default.item [i_category,i_manufact_id] WholeStageCodegen (12) HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] @@ -69,9 +69,9 @@ TakeOrderedAndProject [total_sales,i_manufact_id] BroadcastHashJoin [cs_bill_addr_sk,ca_address_sk] Project [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_bill_addr_sk,cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_bill_addr_sk,cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -92,9 +92,9 @@ TakeOrderedAndProject [total_sales,i_manufact_id] BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] Project [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk,ws_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_bill_addr_sk,ws_item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/explain.txt index 63dacf973..d0f166fe9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/explain.txt @@ -13,23 +13,23 @@ : : +- * BroadcastHashJoin Inner BuildRight (12) : : :- * Project (6) : : : +- * BroadcastHashJoin Inner BuildRight (5) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- ReusedExchange (4) : : +- BroadcastExchange (11) - : : +- * Project (10) - : : +- * Filter (9) - : : +- * ColumnarToRow (8) + : : +- * ColumnarToRow (10) + : : +- CometProject (9) + : : +- CometFilter (8) : : +- CometScan parquet spark_catalog.default.store (7) : +- BroadcastExchange (18) - : +- * Project (17) - : +- * Filter (16) - : +- * ColumnarToRow (15) + : +- * ColumnarToRow (17) + : +- CometProject (16) + : +- CometFilter (15) : +- CometScan parquet spark_catalog.default.household_demographics (14) +- BroadcastExchange (28) - +- * Filter (27) - +- * ColumnarToRow (26) + +- * ColumnarToRow (27) + +- CometFilter (26) +- CometScan parquet spark_catalog.default.customer (25) @@ -41,12 +41,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) -(3) Filter [codegen id : 4] +(3) ColumnarToRow [codegen id : 4] Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) (4) ReusedExchange [Reuses operator id: 37] Output [1]: [d_date_sk#7] @@ -68,16 +68,16 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#8, s_county#9] - -(9) Filter [codegen id : 2] +(8) CometFilter Input [2]: [s_store_sk#8, s_county#9] Condition : ((isnotnull(s_county#9) AND (s_county#9 = Williamson County)) AND isnotnull(s_store_sk#8)) -(10) Project [codegen id : 2] -Output [1]: [s_store_sk#8] +(9) CometProject Input [2]: [s_store_sk#8, s_county#9] +Arguments: [s_store_sk#8], [s_store_sk#8] + +(10) ColumnarToRow [codegen id : 2] +Input [1]: [s_store_sk#8] (11) BroadcastExchange Input [1]: [s_store_sk#8] @@ -100,16 +100,16 @@ Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_vehicle_count), Or(EqualTo(hd_buy_potential,>10000 ),EqualTo(hd_buy_potential,unknown )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 3] -Input [4]: [hd_demo_sk#10, hd_buy_potential#11, hd_dep_count#12, hd_vehicle_count#13] - -(16) Filter [codegen id : 3] +(15) CometFilter Input [4]: [hd_demo_sk#10, hd_buy_potential#11, hd_dep_count#12, hd_vehicle_count#13] Condition : ((((isnotnull(hd_vehicle_count#13) AND ((hd_buy_potential#11 = >10000 ) OR (hd_buy_potential#11 = unknown ))) AND (hd_vehicle_count#13 > 0)) AND CASE WHEN (hd_vehicle_count#13 > 0) THEN ((cast(hd_dep_count#12 as double) / cast(hd_vehicle_count#13 as double)) > 1.2) END) AND isnotnull(hd_demo_sk#10)) -(17) Project [codegen id : 3] -Output [1]: [hd_demo_sk#10] +(16) CometProject Input [4]: [hd_demo_sk#10, hd_buy_potential#11, hd_dep_count#12, hd_vehicle_count#13] +Arguments: [hd_demo_sk#10], [hd_demo_sk#10] + +(17) ColumnarToRow [codegen id : 3] +Input [1]: [hd_demo_sk#10] (18) BroadcastExchange Input [1]: [hd_demo_sk#10] @@ -154,12 +154,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 5] +(26) CometFilter Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] +Condition : isnotnull(c_customer_sk#18) -(27) Filter [codegen id : 5] +(27) ColumnarToRow [codegen id : 5] Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] -Condition : isnotnull(c_customer_sk#18) (28) BroadcastExchange Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] @@ -187,9 +187,9 @@ Arguments: [c_last_name#21 ASC NULLS FIRST, c_first_name#20 ASC NULLS FIRST, c_s Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 BroadcastExchange (37) -+- * Project (36) - +- * Filter (35) - +- * ColumnarToRow (34) ++- * ColumnarToRow (36) + +- CometProject (35) + +- CometFilter (34) +- CometScan parquet spark_catalog.default.date_dim (33) @@ -200,16 +200,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#7, d_year#23, d_dom#24] - -(35) Filter [codegen id : 1] +(34) CometFilter Input [3]: [d_date_sk#7, d_year#23, d_dom#24] Condition : (((((d_dom#24 >= 1) AND (d_dom#24 <= 3)) OR ((d_dom#24 >= 25) AND (d_dom#24 <= 28))) AND d_year#23 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7)) -(36) Project [codegen id : 1] -Output [1]: [d_date_sk#7] +(35) CometProject Input [3]: [d_date_sk#7, d_year#23, d_dom#24] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(36) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#7] (37) BroadcastExchange Input [1]: [d_date_sk#7] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/simplified.txt index 91e0cbf9a..80405a784 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/simplified.txt @@ -17,40 +17,40 @@ WholeStageCodegen (7) BroadcastHashJoin [ss_store_sk,s_store_sk] Project [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk,ss_hdemo_sk,ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dom,d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_dom,d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] InputAdapter ReusedExchange [d_date_sk] #3 InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - Project [s_store_sk] - Filter [s_county,s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [s_store_sk] + CometFilter [s_county,s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_county] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_vehicle_count,hd_buy_potential,hd_dep_count,hd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [hd_demo_sk] + CometFilter [hd_vehicle_count,hd_buy_potential,hd_dep_count,hd_demo_sk] CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] InputAdapter BroadcastExchange #6 WholeStageCodegen (5) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/explain.txt index c4a94f735..a46018cdd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/explain.txt @@ -12,8 +12,8 @@ TakeOrderedAndProject (42) : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (24) : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (17) : : : :- * BroadcastHashJoin LeftSemi BuildRight (10) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.customer (1) : : : : +- BroadcastExchange (9) : : : : +- * Project (8) @@ -34,12 +34,12 @@ TakeOrderedAndProject (42) : : : +- CometScan parquet spark_catalog.default.catalog_sales (18) : : +- ReusedExchange (20) : +- BroadcastExchange (30) - : +- * Filter (29) - : +- * ColumnarToRow (28) + : +- * ColumnarToRow (29) + : +- CometFilter (28) : +- CometScan parquet spark_catalog.default.customer_address (27) +- BroadcastExchange (36) - +- * Filter (35) - +- * ColumnarToRow (34) + +- * ColumnarToRow (35) + +- CometFilter (34) +- CometScan parquet spark_catalog.default.customer_demographics (33) @@ -50,12 +50,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 9] +(2) CometFilter Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) -(3) Filter [codegen id : 9] +(3) ColumnarToRow [codegen id : 9] Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) (4) Scan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] @@ -171,12 +171,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 7] +(28) CometFilter Input [2]: [ca_address_sk#18, ca_state#19] +Condition : isnotnull(ca_address_sk#18) -(29) Filter [codegen id : 7] +(29) ColumnarToRow [codegen id : 7] Input [2]: [ca_address_sk#18, ca_state#19] -Condition : isnotnull(ca_address_sk#18) (30) BroadcastExchange Input [2]: [ca_address_sk#18, ca_state#19] @@ -199,12 +199,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 8] +(34) CometFilter Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#20) -(35) Filter [codegen id : 8] +(35) ColumnarToRow [codegen id : 8] Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Condition : isnotnull(cd_demo_sk#20) (36) BroadcastExchange Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] @@ -246,9 +246,9 @@ Arguments: 100, [ca_state#19 ASC NULLS FIRST, cd_gender#21 ASC NULLS FIRST, cd_m Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 BroadcastExchange (47) -+- * Project (46) - +- * Filter (45) - +- * ColumnarToRow (44) ++- * ColumnarToRow (46) + +- CometProject (45) + +- CometFilter (44) +- CometScan parquet spark_catalog.default.date_dim (43) @@ -259,16 +259,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct -(44) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#9, d_year#74, d_qoy#75] - -(45) Filter [codegen id : 1] +(44) CometFilter Input [3]: [d_date_sk#9, d_year#74, d_qoy#75] Condition : ((((isnotnull(d_year#74) AND isnotnull(d_qoy#75)) AND (d_year#74 = 2002)) AND (d_qoy#75 < 4)) AND isnotnull(d_date_sk#9)) -(46) Project [codegen id : 1] -Output [1]: [d_date_sk#9] +(45) CometProject Input [3]: [d_date_sk#9, d_year#74, d_qoy#75] +Arguments: [d_date_sk#9], [d_date_sk#9] + +(46) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#9] (47) BroadcastExchange Input [1]: [d_date_sk#9] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/simplified.txt index 67a9e3d11..ea0ef274e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/simplified.txt @@ -14,9 +14,9 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_current_addr_sk,c_current_cdemo_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] InputAdapter BroadcastExchange #2 @@ -29,10 +29,10 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_qoy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_qoy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter ReusedExchange [d_date_sk] #3 @@ -61,14 +61,14 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter BroadcastExchange #6 WholeStageCodegen (7) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #7 WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/explain.txt index b82dfae23..39b838157 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/explain.txt @@ -14,18 +14,18 @@ TakeOrderedAndProject (28) : +- * BroadcastHashJoin Inner BuildRight (11) : :- * Project (6) : : +- * BroadcastHashJoin Inner BuildRight (5) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) + : : :- * ColumnarToRow (3) + : : : +- CometFilter (2) : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : +- ReusedExchange (4) : +- BroadcastExchange (10) - : +- * Filter (9) - : +- * ColumnarToRow (8) + : +- * ColumnarToRow (9) + : +- CometFilter (8) : +- CometScan parquet spark_catalog.default.item (7) +- BroadcastExchange (17) - +- * Project (16) - +- * Filter (15) - +- * ColumnarToRow (14) + +- * ColumnarToRow (16) + +- CometProject (15) + +- CometFilter (14) +- CometScan parquet spark_catalog.default.store (13) @@ -37,12 +37,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] +Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) -(3) Filter [codegen id : 4] +(3) ColumnarToRow [codegen id : 4] Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) (4) ReusedExchange [Reuses operator id: 33] Output [1]: [d_date_sk#7] @@ -64,12 +64,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] +(8) CometFilter Input [3]: [i_item_sk#8, i_class#9, i_category#10] +Condition : isnotnull(i_item_sk#8) -(9) Filter [codegen id : 2] +(9) ColumnarToRow [codegen id : 2] Input [3]: [i_item_sk#8, i_class#9, i_category#10] -Condition : isnotnull(i_item_sk#8) (10) BroadcastExchange Input [3]: [i_item_sk#8, i_class#9, i_category#10] @@ -92,16 +92,16 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] -Input [2]: [s_store_sk#11, s_state#12] - -(15) Filter [codegen id : 3] +(14) CometFilter Input [2]: [s_store_sk#11, s_state#12] Condition : ((isnotnull(s_state#12) AND (s_state#12 = TN)) AND isnotnull(s_store_sk#11)) -(16) Project [codegen id : 3] -Output [1]: [s_store_sk#11] +(15) CometProject Input [2]: [s_store_sk#11, s_state#12] +Arguments: [s_store_sk#11], [s_store_sk#11] + +(16) ColumnarToRow [codegen id : 3] +Input [1]: [s_store_sk#11] (17) BroadcastExchange Input [1]: [s_store_sk#11] @@ -163,9 +163,9 @@ Arguments: 100, [lochierarchy#23 DESC NULLS LAST, CASE WHEN (lochierarchy#23 = 0 Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 BroadcastExchange (33) -+- * Project (32) - +- * Filter (31) - +- * ColumnarToRow (30) ++- * ColumnarToRow (32) + +- CometProject (31) + +- CometFilter (30) +- CometScan parquet spark_catalog.default.date_dim (29) @@ -176,16 +176,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#7, d_year#28] - -(31) Filter [codegen id : 1] +(30) CometFilter Input [2]: [d_date_sk#7, d_year#28] Condition : ((isnotnull(d_year#28) AND (d_year#28 = 2001)) AND isnotnull(d_date_sk#7)) -(32) Project [codegen id : 1] -Output [1]: [d_date_sk#7] +(31) CometProject Input [2]: [d_date_sk#7, d_year#28] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(32) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#7] (33) BroadcastExchange Input [1]: [d_date_sk#7] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/simplified.txt index acd050607..7eeb607c3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/simplified.txt @@ -20,32 +20,32 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i BroadcastHashJoin [ss_item_sk,i_item_sk] Project [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk,ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [d_date_sk] #3 InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - Project [s_store_sk] - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [s_store_sk] + CometFilter [s_state,s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/explain.txt index eeb841412..23c4ae742 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/explain.txt @@ -10,19 +10,19 @@ TakeOrderedAndProject (25) : +- * BroadcastHashJoin Inner BuildRight (13) : :- * Project (11) : : +- * BroadcastHashJoin Inner BuildRight (10) - : : :- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) + : : :- * ColumnarToRow (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) : : : +- CometScan parquet spark_catalog.default.item (1) : : +- BroadcastExchange (9) - : : +- * Project (8) - : : +- * Filter (7) - : : +- * ColumnarToRow (6) + : : +- * ColumnarToRow (8) + : : +- CometProject (7) + : : +- CometFilter (6) : : +- CometScan parquet spark_catalog.default.inventory (5) : +- ReusedExchange (12) - +- * Project (19) - +- * Filter (18) - +- * ColumnarToRow (17) + +- * ColumnarToRow (19) + +- CometProject (18) + +- CometFilter (17) +- CometScan parquet spark_catalog.default.catalog_sales (16) @@ -33,16 +33,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,68.00), LessThanOrEqual(i_current_price,98.00), In(i_manufact_id, [677,694,808,940]), IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 68.00)) AND (i_current_price#4 <= 98.00)) AND i_manufact_id#5 IN (677,940,694,808)) AND isnotnull(i_item_sk#1)) -(4) Project [codegen id : 3] -Output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +(3) CometProject Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] +Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] + +(4) ColumnarToRow [codegen id : 3] +Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] (5) Scan parquet spark_catalog.default.inventory Output [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] @@ -52,16 +52,16 @@ PartitionFilters: [isnotnull(inv_date_sk#8), dynamicpruningexpression(inv_date_s PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] - -(7) Filter [codegen id : 1] +(6) CometFilter Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] Condition : (((isnotnull(inv_quantity_on_hand#7) AND (inv_quantity_on_hand#7 >= 100)) AND (inv_quantity_on_hand#7 <= 500)) AND isnotnull(inv_item_sk#6)) -(8) Project [codegen id : 1] -Output [2]: [inv_item_sk#6, inv_date_sk#8] +(7) CometProject Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] +Arguments: [inv_item_sk#6, inv_date_sk#8], [inv_item_sk#6, inv_date_sk#8] + +(8) ColumnarToRow [codegen id : 1] +Input [2]: [inv_item_sk#6, inv_date_sk#8] (9) BroadcastExchange Input [2]: [inv_item_sk#6, inv_date_sk#8] @@ -101,16 +101,16 @@ Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(17) ColumnarToRow -Input [2]: [cs_item_sk#11, cs_sold_date_sk#12] - -(18) Filter +(17) CometFilter Input [2]: [cs_item_sk#11, cs_sold_date_sk#12] Condition : isnotnull(cs_item_sk#11) -(19) Project -Output [1]: [cs_item_sk#11] +(18) CometProject Input [2]: [cs_item_sk#11, cs_sold_date_sk#12] +Arguments: [cs_item_sk#11], [cs_item_sk#11] + +(19) ColumnarToRow +Input [1]: [cs_item_sk#11] (20) BroadcastHashJoin [codegen id : 4] Left keys [1]: [i_item_sk#1] @@ -148,9 +148,9 @@ Arguments: 100, [i_item_id#2 ASC NULLS FIRST], [i_item_id#2, i_item_desc#3, i_cu Subquery:1 Hosting operator id = 5 Hosting Expression = inv_date_sk#8 IN dynamicpruning#9 BroadcastExchange (30) -+- * Project (29) - +- * Filter (28) - +- * ColumnarToRow (27) ++- * ColumnarToRow (29) + +- CometProject (28) + +- CometFilter (27) +- CometScan parquet spark_catalog.default.date_dim (26) @@ -161,16 +161,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-01), LessThanOrEqual(d_date,2000-04-01), IsNotNull(d_date_sk)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#10, d_date#13] - -(28) Filter [codegen id : 1] +(27) CometFilter Input [2]: [d_date_sk#10, d_date#13] Condition : (((isnotnull(d_date#13) AND (d_date#13 >= 2000-02-01)) AND (d_date#13 <= 2000-04-01)) AND isnotnull(d_date_sk#10)) -(29) Project [codegen id : 1] -Output [1]: [d_date_sk#10] +(28) CometProject Input [2]: [d_date_sk#10, d_date#13] +Arguments: [d_date_sk#10], [d_date_sk#10] + +(29) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#10] (30) BroadcastExchange Input [1]: [d_date_sk#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/simplified.txt index 1b469ce06..65bb06348 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/simplified.txt @@ -14,31 +14,31 @@ TakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] BroadcastHashJoin [inv_date_sk,d_date_sk] Project [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] BroadcastHashJoin [i_item_sk,inv_item_sk] - Project [i_item_sk,i_item_id,i_item_desc,i_current_price] - Filter [i_current_price,i_manufact_id,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price] + CometFilter [i_current_price,i_manufact_id,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Project [inv_item_sk,inv_date_sk] - Filter [inv_quantity_on_hand,inv_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [inv_item_sk,inv_date_sk] + CometFilter [inv_quantity_on_hand,inv_item_sk] CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter ReusedExchange [d_date_sk] #4 - Project [cs_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cs_item_sk] + CometFilter [cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/explain.txt index 66dd23ca6..5503439ee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/explain.txt @@ -12,13 +12,13 @@ : : +- * BroadcastHashJoin Inner BuildRight (11) : : :- * Project (6) : : : +- * BroadcastHashJoin Inner BuildRight (5) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- ReusedExchange (4) : : +- BroadcastExchange (10) - : : +- * Filter (9) - : : +- * ColumnarToRow (8) + : : +- * ColumnarToRow (9) + : : +- CometFilter (8) : : +- CometScan parquet spark_catalog.default.customer (7) : +- BroadcastExchange (28) : +- * HashAggregate (27) @@ -28,8 +28,8 @@ : +- * BroadcastHashJoin Inner BuildRight (23) : :- * Project (21) : : +- * BroadcastHashJoin Inner BuildRight (20) - : : :- * Filter (18) - : : : +- * ColumnarToRow (17) + : : :- * ColumnarToRow (18) + : : : +- CometFilter (17) : : : +- CometScan parquet spark_catalog.default.catalog_sales (16) : : +- ReusedExchange (19) : +- ReusedExchange (22) @@ -41,8 +41,8 @@ +- * BroadcastHashJoin Inner BuildRight (37) :- * Project (35) : +- * BroadcastHashJoin Inner BuildRight (34) - : :- * Filter (32) - : : +- * ColumnarToRow (31) + : :- * ColumnarToRow (32) + : : +- CometFilter (31) : : +- CometScan parquet spark_catalog.default.web_sales (30) : +- ReusedExchange (33) +- ReusedExchange (36) @@ -56,12 +56,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#2), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] +Condition : isnotnull(ss_customer_sk#1) -(3) Filter [codegen id : 3] +(3) ColumnarToRow [codegen id : 3] Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Condition : isnotnull(ss_customer_sk#1) (4) ReusedExchange [Reuses operator id: 52] Output [2]: [d_date_sk#4, d_date#5] @@ -83,12 +83,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] +(8) CometFilter Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Condition : isnotnull(c_customer_sk#6) -(9) Filter [codegen id : 2] +(9) ColumnarToRow [codegen id : 2] Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Condition : isnotnull(c_customer_sk#6) (10) BroadcastExchange Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] @@ -130,12 +130,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#10), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 6] +(17) CometFilter Input [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] +Condition : isnotnull(cs_bill_customer_sk#9) -(18) Filter [codegen id : 6] +(18) ColumnarToRow [codegen id : 6] Input [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] -Condition : isnotnull(cs_bill_customer_sk#9) (19) ReusedExchange [Reuses operator id: 52] Output [2]: [d_date_sk#12, d_date#13] @@ -199,12 +199,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#18), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 10] +(31) CometFilter Input [2]: [ws_bill_customer_sk#17, ws_sold_date_sk#18] +Condition : isnotnull(ws_bill_customer_sk#17) -(32) Filter [codegen id : 10] +(32) ColumnarToRow [codegen id : 10] Input [2]: [ws_bill_customer_sk#17, ws_sold_date_sk#18] -Condition : isnotnull(ws_bill_customer_sk#17) (33) ReusedExchange [Reuses operator id: 52] Output [2]: [d_date_sk#20, d_date#21] @@ -286,9 +286,9 @@ Results [1]: [count(1)#27 AS count(1)#28] Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#2 IN dynamicpruning#3 BroadcastExchange (52) -+- * Project (51) - +- * Filter (50) - +- * ColumnarToRow (49) ++- * ColumnarToRow (51) + +- CometProject (50) + +- CometFilter (49) +- CometScan parquet spark_catalog.default.date_dim (48) @@ -299,16 +299,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(49) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#29] - -(50) Filter [codegen id : 1] +(49) CometFilter Input [3]: [d_date_sk#4, d_date#5, d_month_seq#29] Condition : (((isnotnull(d_month_seq#29) AND (d_month_seq#29 >= 1200)) AND (d_month_seq#29 <= 1211)) AND isnotnull(d_date_sk#4)) -(51) Project [codegen id : 1] -Output [2]: [d_date_sk#4, d_date#5] +(50) CometProject Input [3]: [d_date_sk#4, d_date#5, d_month_seq#29] +Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] + +(51) ColumnarToRow [codegen id : 1] +Input [2]: [d_date_sk#4, d_date#5] (52) BroadcastExchange Input [2]: [d_date_sk#4, d_date#5] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/simplified.txt index 27489c0a9..315afe660 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/simplified.txt @@ -16,26 +16,26 @@ WholeStageCodegen (13) BroadcastHashJoin [ss_customer_sk,c_customer_sk] Project [ss_customer_sk,d_date] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_date] + CometFilter [d_month_seq,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] InputAdapter ReusedExchange [d_date_sk,d_date] #3 InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] InputAdapter BroadcastExchange #5 @@ -49,9 +49,9 @@ WholeStageCodegen (13) BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] Project [cs_bill_customer_sk,d_date] BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_bill_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_bill_customer_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -70,9 +70,9 @@ WholeStageCodegen (13) BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk] Project [ws_bill_customer_sk,d_date] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_bill_customer_sk] CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/explain.txt index 7dd8fcc08..d851f61bf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/explain.txt @@ -13,16 +13,16 @@ : : +- * BroadcastHashJoin Inner BuildRight (14) : : :- * Project (9) : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.inventory (1) : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) + : : : +- * ColumnarToRow (6) + : : : +- CometFilter (5) : : : +- CometScan parquet spark_catalog.default.item (4) : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) + : : +- * ColumnarToRow (12) + : : +- CometFilter (11) : : +- CometScan parquet spark_catalog.default.warehouse (10) : +- ReusedExchange (16) +- BroadcastExchange (41) @@ -37,8 +37,8 @@ : +- * BroadcastHashJoin Inner BuildRight (31) : :- * Project (29) : : +- * BroadcastHashJoin Inner BuildRight (28) - : : :- * Filter (26) - : : : +- * ColumnarToRow (25) + : : :- * ColumnarToRow (26) + : : : +- CometFilter (25) : : : +- CometScan parquet spark_catalog.default.inventory (24) : : +- ReusedExchange (27) : +- ReusedExchange (30) @@ -53,12 +53,12 @@ PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_s PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) -(3) Filter [codegen id : 4] +(3) ColumnarToRow [codegen id : 4] Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) (4) Scan parquet spark_catalog.default.item Output [1]: [i_item_sk#6] @@ -67,12 +67,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [1]: [i_item_sk#6] +Condition : isnotnull(i_item_sk#6) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [1]: [i_item_sk#6] -Condition : isnotnull(i_item_sk#6) (7) BroadcastExchange Input [1]: [i_item_sk#6] @@ -95,12 +95,12 @@ Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] +(11) CometFilter Input [2]: [w_warehouse_sk#7, w_warehouse_name#8] +Condition : isnotnull(w_warehouse_sk#7) -(12) Filter [codegen id : 2] +(12) ColumnarToRow [codegen id : 2] Input [2]: [w_warehouse_sk#7, w_warehouse_name#8] -Condition : isnotnull(w_warehouse_sk#7) (13) BroadcastExchange Input [2]: [w_warehouse_sk#7, w_warehouse_name#8] @@ -163,12 +163,12 @@ PartitionFilters: [isnotnull(inv_date_sk#29), dynamicpruningexpression(inv_date_ PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 8] +(25) CometFilter Input [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] +Condition : (isnotnull(inv_item_sk#26) AND isnotnull(inv_warehouse_sk#27)) -(26) Filter [codegen id : 8] +(26) ColumnarToRow [codegen id : 8] Input [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] -Condition : (isnotnull(inv_item_sk#26) AND isnotnull(inv_warehouse_sk#27)) (27) ReusedExchange [Reuses operator id: 7] Output [1]: [i_item_sk#31] @@ -257,9 +257,9 @@ Arguments: [w_warehouse_sk#7 ASC NULLS FIRST, i_item_sk#6 ASC NULLS FIRST, d_moy Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 BroadcastExchange (49) -+- * Project (48) - +- * Filter (47) - +- * ColumnarToRow (46) ++- * ColumnarToRow (48) + +- CometProject (47) + +- CometFilter (46) +- CometScan parquet spark_catalog.default.date_dim (45) @@ -270,16 +270,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] ReadSchema: struct -(46) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#9, d_year#49, d_moy#10] - -(47) Filter [codegen id : 1] +(46) CometFilter Input [3]: [d_date_sk#9, d_year#49, d_moy#10] Condition : ((((isnotnull(d_year#49) AND isnotnull(d_moy#10)) AND (d_year#49 = 2001)) AND (d_moy#10 = 1)) AND isnotnull(d_date_sk#9)) -(48) Project [codegen id : 1] -Output [2]: [d_date_sk#9, d_moy#10] +(47) CometProject Input [3]: [d_date_sk#9, d_year#49, d_moy#10] +Arguments: [d_date_sk#9, d_moy#10], [d_date_sk#9, d_moy#10] + +(48) ColumnarToRow [codegen id : 1] +Input [2]: [d_date_sk#9, d_moy#10] (49) BroadcastExchange Input [2]: [d_date_sk#9, d_moy#10] @@ -287,9 +287,9 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Subquery:2 Hosting operator id = 24 Hosting Expression = inv_date_sk#29 IN dynamicpruning#30 BroadcastExchange (54) -+- * Project (53) - +- * Filter (52) - +- * ColumnarToRow (51) ++- * ColumnarToRow (53) + +- CometProject (52) + +- CometFilter (51) +- CometScan parquet spark_catalog.default.date_dim (50) @@ -300,16 +300,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(51) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#34, d_year#50, d_moy#35] - -(52) Filter [codegen id : 1] +(51) CometFilter Input [3]: [d_date_sk#34, d_year#50, d_moy#35] Condition : ((((isnotnull(d_year#50) AND isnotnull(d_moy#35)) AND (d_year#50 = 2001)) AND (d_moy#35 = 2)) AND isnotnull(d_date_sk#34)) -(53) Project [codegen id : 1] -Output [2]: [d_date_sk#34, d_moy#35] +(52) CometProject Input [3]: [d_date_sk#34, d_year#50, d_moy#35] +Arguments: [d_date_sk#34, d_moy#35], [d_date_sk#34, d_moy#35] + +(53) ColumnarToRow [codegen id : 1] +Input [2]: [d_date_sk#34, d_moy#35] (54) BroadcastExchange Input [2]: [d_date_sk#34, d_moy#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/simplified.txt index 138e386cb..002266e76 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/simplified.txt @@ -17,31 +17,31 @@ WholeStageCodegen (11) BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] Project [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] BroadcastHashJoin [inv_item_sk,i_item_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [inv_item_sk,inv_warehouse_sk] CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk,d_moy] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_moy] + CometFilter [d_year,d_moy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #4 WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk] InputAdapter BroadcastExchange #5 WholeStageCodegen (2) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [w_warehouse_sk] CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] InputAdapter ReusedExchange [d_date_sk,d_moy] #3 @@ -61,17 +61,17 @@ WholeStageCodegen (11) BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] Project [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] BroadcastHashJoin [inv_item_sk,i_item_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [inv_item_sk,inv_warehouse_sk] CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) - Project [d_date_sk,d_moy] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_moy] + CometFilter [d_year,d_moy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter ReusedExchange [i_item_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/explain.txt index 4befb027a..a4184150e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/explain.txt @@ -13,16 +13,16 @@ : : +- * BroadcastHashJoin Inner BuildRight (14) : : :- * Project (9) : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.inventory (1) : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) + : : : +- * ColumnarToRow (6) + : : : +- CometFilter (5) : : : +- CometScan parquet spark_catalog.default.item (4) : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) + : : +- * ColumnarToRow (12) + : : +- CometFilter (11) : : +- CometScan parquet spark_catalog.default.warehouse (10) : +- ReusedExchange (16) +- BroadcastExchange (41) @@ -37,8 +37,8 @@ : +- * BroadcastHashJoin Inner BuildRight (31) : :- * Project (29) : : +- * BroadcastHashJoin Inner BuildRight (28) - : : :- * Filter (26) - : : : +- * ColumnarToRow (25) + : : :- * ColumnarToRow (26) + : : : +- CometFilter (25) : : : +- CometScan parquet spark_catalog.default.inventory (24) : : +- ReusedExchange (27) : +- ReusedExchange (30) @@ -53,12 +53,12 @@ PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_s PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) -(3) Filter [codegen id : 4] +(3) ColumnarToRow [codegen id : 4] Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) (4) Scan parquet spark_catalog.default.item Output [1]: [i_item_sk#6] @@ -67,12 +67,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [1]: [i_item_sk#6] +Condition : isnotnull(i_item_sk#6) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [1]: [i_item_sk#6] -Condition : isnotnull(i_item_sk#6) (7) BroadcastExchange Input [1]: [i_item_sk#6] @@ -95,12 +95,12 @@ Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] +(11) CometFilter Input [2]: [w_warehouse_sk#7, w_warehouse_name#8] +Condition : isnotnull(w_warehouse_sk#7) -(12) Filter [codegen id : 2] +(12) ColumnarToRow [codegen id : 2] Input [2]: [w_warehouse_sk#7, w_warehouse_name#8] -Condition : isnotnull(w_warehouse_sk#7) (13) BroadcastExchange Input [2]: [w_warehouse_sk#7, w_warehouse_name#8] @@ -163,12 +163,12 @@ PartitionFilters: [isnotnull(inv_date_sk#29), dynamicpruningexpression(inv_date_ PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 8] +(25) CometFilter Input [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] +Condition : (isnotnull(inv_item_sk#26) AND isnotnull(inv_warehouse_sk#27)) -(26) Filter [codegen id : 8] +(26) ColumnarToRow [codegen id : 8] Input [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] -Condition : (isnotnull(inv_item_sk#26) AND isnotnull(inv_warehouse_sk#27)) (27) ReusedExchange [Reuses operator id: 7] Output [1]: [i_item_sk#31] @@ -257,9 +257,9 @@ Arguments: [w_warehouse_sk#7 ASC NULLS FIRST, i_item_sk#6 ASC NULLS FIRST, d_moy Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 BroadcastExchange (49) -+- * Project (48) - +- * Filter (47) - +- * ColumnarToRow (46) ++- * ColumnarToRow (48) + +- CometProject (47) + +- CometFilter (46) +- CometScan parquet spark_catalog.default.date_dim (45) @@ -270,16 +270,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] ReadSchema: struct -(46) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#9, d_year#49, d_moy#10] - -(47) Filter [codegen id : 1] +(46) CometFilter Input [3]: [d_date_sk#9, d_year#49, d_moy#10] Condition : ((((isnotnull(d_year#49) AND isnotnull(d_moy#10)) AND (d_year#49 = 2001)) AND (d_moy#10 = 1)) AND isnotnull(d_date_sk#9)) -(48) Project [codegen id : 1] -Output [2]: [d_date_sk#9, d_moy#10] +(47) CometProject Input [3]: [d_date_sk#9, d_year#49, d_moy#10] +Arguments: [d_date_sk#9, d_moy#10], [d_date_sk#9, d_moy#10] + +(48) ColumnarToRow [codegen id : 1] +Input [2]: [d_date_sk#9, d_moy#10] (49) BroadcastExchange Input [2]: [d_date_sk#9, d_moy#10] @@ -287,9 +287,9 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Subquery:2 Hosting operator id = 24 Hosting Expression = inv_date_sk#29 IN dynamicpruning#30 BroadcastExchange (54) -+- * Project (53) - +- * Filter (52) - +- * ColumnarToRow (51) ++- * ColumnarToRow (53) + +- CometProject (52) + +- CometFilter (51) +- CometScan parquet spark_catalog.default.date_dim (50) @@ -300,16 +300,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(51) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#34, d_year#50, d_moy#35] - -(52) Filter [codegen id : 1] +(51) CometFilter Input [3]: [d_date_sk#34, d_year#50, d_moy#35] Condition : ((((isnotnull(d_year#50) AND isnotnull(d_moy#35)) AND (d_year#50 = 2001)) AND (d_moy#35 = 2)) AND isnotnull(d_date_sk#34)) -(53) Project [codegen id : 1] -Output [2]: [d_date_sk#34, d_moy#35] +(52) CometProject Input [3]: [d_date_sk#34, d_year#50, d_moy#35] +Arguments: [d_date_sk#34, d_moy#35], [d_date_sk#34, d_moy#35] + +(53) ColumnarToRow [codegen id : 1] +Input [2]: [d_date_sk#34, d_moy#35] (54) BroadcastExchange Input [2]: [d_date_sk#34, d_moy#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/simplified.txt index 138e386cb..002266e76 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/simplified.txt @@ -17,31 +17,31 @@ WholeStageCodegen (11) BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] Project [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] BroadcastHashJoin [inv_item_sk,i_item_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [inv_item_sk,inv_warehouse_sk] CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk,d_moy] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_moy] + CometFilter [d_year,d_moy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #4 WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk] InputAdapter BroadcastExchange #5 WholeStageCodegen (2) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [w_warehouse_sk] CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] InputAdapter ReusedExchange [d_date_sk,d_moy] #3 @@ -61,17 +61,17 @@ WholeStageCodegen (11) BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] Project [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] BroadcastHashJoin [inv_item_sk,i_item_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [inv_item_sk,inv_warehouse_sk] CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) - Project [d_date_sk,d_moy] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_moy] + CometFilter [d_year,d_moy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter ReusedExchange [i_item_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/explain.txt index 3bc53a09a..a854a1041 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/explain.txt @@ -17,12 +17,12 @@ TakeOrderedAndProject (108) : : : : : +- * BroadcastHashJoin Inner BuildRight (11) : : : : : :- * Project (9) : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) + : : : : : : :- * ColumnarToRow (3) + : : : : : : : +- CometFilter (2) : : : : : : : +- CometScan parquet spark_catalog.default.customer (1) : : : : : : +- BroadcastExchange (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) + : : : : : : +- * ColumnarToRow (6) + : : : : : : +- CometFilter (5) : : : : : : +- CometScan parquet spark_catalog.default.store_sales (4) : : : : : +- ReusedExchange (10) : : : : +- BroadcastExchange (32) @@ -33,12 +33,12 @@ TakeOrderedAndProject (108) : : : : +- * BroadcastHashJoin Inner BuildRight (27) : : : : :- * Project (25) : : : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : : : :- * Filter (19) - : : : : : : +- * ColumnarToRow (18) + : : : : : :- * ColumnarToRow (19) + : : : : : : +- CometFilter (18) : : : : : : +- CometScan parquet spark_catalog.default.customer (17) : : : : : +- BroadcastExchange (23) - : : : : : +- * Filter (22) - : : : : : +- * ColumnarToRow (21) + : : : : : +- * ColumnarToRow (22) + : : : : : +- CometFilter (21) : : : : : +- CometScan parquet spark_catalog.default.store_sales (20) : : : : +- ReusedExchange (26) : : : +- BroadcastExchange (50) @@ -50,12 +50,12 @@ TakeOrderedAndProject (108) : : : +- * BroadcastHashJoin Inner BuildRight (44) : : : :- * Project (42) : : : : +- * BroadcastHashJoin Inner BuildRight (41) - : : : : :- * Filter (36) - : : : : : +- * ColumnarToRow (35) + : : : : :- * ColumnarToRow (36) + : : : : : +- CometFilter (35) : : : : : +- CometScan parquet spark_catalog.default.customer (34) : : : : +- BroadcastExchange (40) - : : : : +- * Filter (39) - : : : : +- * ColumnarToRow (38) + : : : : +- * ColumnarToRow (39) + : : : : +- CometFilter (38) : : : : +- CometScan parquet spark_catalog.default.catalog_sales (37) : : : +- ReusedExchange (43) : : +- BroadcastExchange (68) @@ -66,12 +66,12 @@ TakeOrderedAndProject (108) : : +- * BroadcastHashJoin Inner BuildRight (63) : : :- * Project (61) : : : +- * BroadcastHashJoin Inner BuildRight (60) - : : : :- * Filter (55) - : : : : +- * ColumnarToRow (54) + : : : :- * ColumnarToRow (55) + : : : : +- CometFilter (54) : : : : +- CometScan parquet spark_catalog.default.customer (53) : : : +- BroadcastExchange (59) - : : : +- * Filter (58) - : : : +- * ColumnarToRow (57) + : : : +- * ColumnarToRow (58) + : : : +- CometFilter (57) : : : +- CometScan parquet spark_catalog.default.catalog_sales (56) : : +- ReusedExchange (62) : +- BroadcastExchange (87) @@ -83,12 +83,12 @@ TakeOrderedAndProject (108) : +- * BroadcastHashJoin Inner BuildRight (81) : :- * Project (79) : : +- * BroadcastHashJoin Inner BuildRight (78) - : : :- * Filter (73) - : : : +- * ColumnarToRow (72) + : : :- * ColumnarToRow (73) + : : : +- CometFilter (72) : : : +- CometScan parquet spark_catalog.default.customer (71) : : +- BroadcastExchange (77) - : : +- * Filter (76) - : : +- * ColumnarToRow (75) + : : +- * ColumnarToRow (76) + : : +- CometFilter (75) : : +- CometScan parquet spark_catalog.default.web_sales (74) : +- ReusedExchange (80) +- BroadcastExchange (105) @@ -99,12 +99,12 @@ TakeOrderedAndProject (108) +- * BroadcastHashJoin Inner BuildRight (100) :- * Project (98) : +- * BroadcastHashJoin Inner BuildRight (97) - : :- * Filter (92) - : : +- * ColumnarToRow (91) + : :- * ColumnarToRow (92) + : : +- CometFilter (91) : : +- CometScan parquet spark_catalog.default.customer (90) : +- BroadcastExchange (96) - : +- * Filter (95) - : +- * ColumnarToRow (94) + : +- * ColumnarToRow (95) + : +- CometFilter (94) : +- CometScan parquet spark_catalog.default.web_sales (93) +- ReusedExchange (99) @@ -116,12 +116,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) -(3) Filter [codegen id : 3] +(3) ColumnarToRow [codegen id : 3] Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) (4) Scan parquet spark_catalog.default.store_sales Output [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] @@ -131,12 +131,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#14), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] +Condition : isnotnull(ss_customer_sk#9) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Condition : isnotnull(ss_customer_sk#9) (7) BroadcastExchange Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] @@ -194,12 +194,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 6] +(18) CometFilter Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] +Condition : (isnotnull(c_customer_sk#25) AND isnotnull(c_customer_id#26)) -(19) Filter [codegen id : 6] +(19) ColumnarToRow [codegen id : 6] Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] -Condition : (isnotnull(c_customer_sk#25) AND isnotnull(c_customer_id#26)) (20) Scan parquet spark_catalog.default.store_sales Output [6]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_sales_price#35, ss_ext_wholesale_cost#36, ss_ext_list_price#37, ss_sold_date_sk#38] @@ -209,12 +209,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#38), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 4] +(21) CometFilter Input [6]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_sales_price#35, ss_ext_wholesale_cost#36, ss_ext_list_price#37, ss_sold_date_sk#38] +Condition : isnotnull(ss_customer_sk#33) -(22) Filter [codegen id : 4] +(22) ColumnarToRow [codegen id : 4] Input [6]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_sales_price#35, ss_ext_wholesale_cost#36, ss_ext_list_price#37, ss_sold_date_sk#38] -Condition : isnotnull(ss_customer_sk#33) (23) BroadcastExchange Input [6]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_sales_price#35, ss_ext_wholesale_cost#36, ss_ext_list_price#37, ss_sold_date_sk#38] @@ -278,12 +278,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 10] +(35) CometFilter Input [8]: [c_customer_sk#54, c_customer_id#55, c_first_name#56, c_last_name#57, c_preferred_cust_flag#58, c_birth_country#59, c_login#60, c_email_address#61] +Condition : (isnotnull(c_customer_sk#54) AND isnotnull(c_customer_id#55)) -(36) Filter [codegen id : 10] +(36) ColumnarToRow [codegen id : 10] Input [8]: [c_customer_sk#54, c_customer_id#55, c_first_name#56, c_last_name#57, c_preferred_cust_flag#58, c_birth_country#59, c_login#60, c_email_address#61] -Condition : (isnotnull(c_customer_sk#54) AND isnotnull(c_customer_id#55)) (37) Scan parquet spark_catalog.default.catalog_sales Output [6]: [cs_bill_customer_sk#62, cs_ext_discount_amt#63, cs_ext_sales_price#64, cs_ext_wholesale_cost#65, cs_ext_list_price#66, cs_sold_date_sk#67] @@ -293,12 +293,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#67), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 8] +(38) CometFilter Input [6]: [cs_bill_customer_sk#62, cs_ext_discount_amt#63, cs_ext_sales_price#64, cs_ext_wholesale_cost#65, cs_ext_list_price#66, cs_sold_date_sk#67] +Condition : isnotnull(cs_bill_customer_sk#62) -(39) Filter [codegen id : 8] +(39) ColumnarToRow [codegen id : 8] Input [6]: [cs_bill_customer_sk#62, cs_ext_discount_amt#63, cs_ext_sales_price#64, cs_ext_wholesale_cost#65, cs_ext_list_price#66, cs_sold_date_sk#67] -Condition : isnotnull(cs_bill_customer_sk#62) (40) BroadcastExchange Input [6]: [cs_bill_customer_sk#62, cs_ext_discount_amt#63, cs_ext_sales_price#64, cs_ext_wholesale_cost#65, cs_ext_list_price#66, cs_sold_date_sk#67] @@ -370,12 +370,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(54) ColumnarToRow [codegen id : 14] +(54) CometFilter Input [8]: [c_customer_sk#78, c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85] +Condition : (isnotnull(c_customer_sk#78) AND isnotnull(c_customer_id#79)) -(55) Filter [codegen id : 14] +(55) ColumnarToRow [codegen id : 14] Input [8]: [c_customer_sk#78, c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85] -Condition : (isnotnull(c_customer_sk#78) AND isnotnull(c_customer_id#79)) (56) Scan parquet spark_catalog.default.catalog_sales Output [6]: [cs_bill_customer_sk#86, cs_ext_discount_amt#87, cs_ext_sales_price#88, cs_ext_wholesale_cost#89, cs_ext_list_price#90, cs_sold_date_sk#91] @@ -385,12 +385,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#91), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 12] +(57) CometFilter Input [6]: [cs_bill_customer_sk#86, cs_ext_discount_amt#87, cs_ext_sales_price#88, cs_ext_wholesale_cost#89, cs_ext_list_price#90, cs_sold_date_sk#91] +Condition : isnotnull(cs_bill_customer_sk#86) -(58) Filter [codegen id : 12] +(58) ColumnarToRow [codegen id : 12] Input [6]: [cs_bill_customer_sk#86, cs_ext_discount_amt#87, cs_ext_sales_price#88, cs_ext_wholesale_cost#89, cs_ext_list_price#90, cs_sold_date_sk#91] -Condition : isnotnull(cs_bill_customer_sk#86) (59) BroadcastExchange Input [6]: [cs_bill_customer_sk#86, cs_ext_discount_amt#87, cs_ext_sales_price#88, cs_ext_wholesale_cost#89, cs_ext_list_price#90, cs_sold_date_sk#91] @@ -458,12 +458,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(72) ColumnarToRow [codegen id : 18] +(72) CometFilter Input [8]: [c_customer_sk#101, c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108] +Condition : (isnotnull(c_customer_sk#101) AND isnotnull(c_customer_id#102)) -(73) Filter [codegen id : 18] +(73) ColumnarToRow [codegen id : 18] Input [8]: [c_customer_sk#101, c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108] -Condition : (isnotnull(c_customer_sk#101) AND isnotnull(c_customer_id#102)) (74) Scan parquet spark_catalog.default.web_sales Output [6]: [ws_bill_customer_sk#109, ws_ext_discount_amt#110, ws_ext_sales_price#111, ws_ext_wholesale_cost#112, ws_ext_list_price#113, ws_sold_date_sk#114] @@ -473,12 +473,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#114), dynamicpruningexpression(ws_s PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(75) ColumnarToRow [codegen id : 16] +(75) CometFilter Input [6]: [ws_bill_customer_sk#109, ws_ext_discount_amt#110, ws_ext_sales_price#111, ws_ext_wholesale_cost#112, ws_ext_list_price#113, ws_sold_date_sk#114] +Condition : isnotnull(ws_bill_customer_sk#109) -(76) Filter [codegen id : 16] +(76) ColumnarToRow [codegen id : 16] Input [6]: [ws_bill_customer_sk#109, ws_ext_discount_amt#110, ws_ext_sales_price#111, ws_ext_wholesale_cost#112, ws_ext_list_price#113, ws_sold_date_sk#114] -Condition : isnotnull(ws_bill_customer_sk#109) (77) BroadcastExchange Input [6]: [ws_bill_customer_sk#109, ws_ext_discount_amt#110, ws_ext_sales_price#111, ws_ext_wholesale_cost#112, ws_ext_list_price#113, ws_sold_date_sk#114] @@ -550,12 +550,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(91) ColumnarToRow [codegen id : 22] +(91) CometFilter Input [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] +Condition : (isnotnull(c_customer_sk#125) AND isnotnull(c_customer_id#126)) -(92) Filter [codegen id : 22] +(92) ColumnarToRow [codegen id : 22] Input [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] -Condition : (isnotnull(c_customer_sk#125) AND isnotnull(c_customer_id#126)) (93) Scan parquet spark_catalog.default.web_sales Output [6]: [ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] @@ -565,12 +565,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#138), dynamicpruningexpression(ws_s PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(94) ColumnarToRow [codegen id : 20] +(94) CometFilter Input [6]: [ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] +Condition : isnotnull(ws_bill_customer_sk#133) -(95) Filter [codegen id : 20] +(95) ColumnarToRow [codegen id : 20] Input [6]: [ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] -Condition : isnotnull(ws_bill_customer_sk#133) (96) BroadcastExchange Input [6]: [ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] @@ -639,8 +639,8 @@ Arguments: 100, [customer_id#46 ASC NULLS FIRST, customer_first_name#47 ASC NULL Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#14 IN dynamicpruning#15 BroadcastExchange (112) -+- * Filter (111) - +- * ColumnarToRow (110) ++- * ColumnarToRow (111) + +- CometFilter (110) +- CometScan parquet spark_catalog.default.date_dim (109) @@ -651,12 +651,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(110) ColumnarToRow [codegen id : 1] +(110) CometFilter Input [2]: [d_date_sk#16, d_year#17] +Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) -(111) Filter [codegen id : 1] +(111) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#16, d_year#17] -Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) (112) BroadcastExchange Input [2]: [d_date_sk#16, d_year#17] @@ -664,8 +664,8 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint) Subquery:2 Hosting operator id = 20 Hosting Expression = ss_sold_date_sk#38 IN dynamicpruning#39 BroadcastExchange (116) -+- * Filter (115) - +- * ColumnarToRow (114) ++- * ColumnarToRow (115) + +- CometFilter (114) +- CometScan parquet spark_catalog.default.date_dim (113) @@ -676,12 +676,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(114) ColumnarToRow [codegen id : 1] +(114) CometFilter Input [2]: [d_date_sk#40, d_year#41] +Condition : ((isnotnull(d_year#41) AND (d_year#41 = 2002)) AND isnotnull(d_date_sk#40)) -(115) Filter [codegen id : 1] +(115) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#40, d_year#41] -Condition : ((isnotnull(d_year#41) AND (d_year#41 = 2002)) AND isnotnull(d_date_sk#40)) (116) BroadcastExchange Input [2]: [d_date_sk#40, d_year#41] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/simplified.txt index f21b1c557..99e255a0e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/simplified.txt @@ -19,23 +19,23 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_customer_id] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] InputAdapter BroadcastExchange #2 WholeStageCodegen (1) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [d_date_sk,d_year] #3 @@ -51,23 +51,23 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_customer_id] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] InputAdapter BroadcastExchange #6 WholeStageCodegen (4) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #7 WholeStageCodegen (1) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [d_date_sk,d_year] #7 @@ -84,16 +84,16 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] BroadcastHashJoin [c_customer_sk,cs_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_customer_id] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] InputAdapter BroadcastExchange #10 WholeStageCodegen (8) - Filter [cs_bill_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_bill_customer_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -110,16 +110,16 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] BroadcastHashJoin [c_customer_sk,cs_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_customer_id] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] InputAdapter BroadcastExchange #13 WholeStageCodegen (12) - Filter [cs_bill_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_bill_customer_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #2 InputAdapter @@ -137,16 +137,16 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_customer_id] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] InputAdapter BroadcastExchange #16 WholeStageCodegen (16) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_bill_customer_sk] CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -163,16 +163,16 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_customer_id] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] InputAdapter BroadcastExchange #19 WholeStageCodegen (20) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_bill_customer_sk] CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/explain.txt index 67e36b91f..f63b94658 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/explain.txt @@ -13,23 +13,23 @@ TakeOrderedAndProject (33) : : : +- * SortMergeJoin LeftOuter (12) : : : :- * Sort (5) : : : : +- Exchange (4) - : : : : +- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : : +- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : : : +- * Sort (11) : : : +- Exchange (10) - : : : +- * Project (9) - : : : +- * Filter (8) - : : : +- * ColumnarToRow (7) + : : : +- * ColumnarToRow (9) + : : : +- CometProject (8) + : : : +- CometFilter (7) : : : +- CometScan parquet spark_catalog.default.catalog_returns (6) : : +- BroadcastExchange (17) - : : +- * Filter (16) - : : +- * ColumnarToRow (15) + : : +- * ColumnarToRow (16) + : : +- CometFilter (15) : : +- CometScan parquet spark_catalog.default.warehouse (14) : +- BroadcastExchange (24) - : +- * Project (23) - : +- * Filter (22) - : +- * ColumnarToRow (21) + : +- * ColumnarToRow (23) + : +- CometProject (22) + : +- CometFilter (21) : +- CometScan parquet spark_catalog.default.item (20) +- ReusedExchange (27) @@ -42,12 +42,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sol PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] +(2) CometFilter Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] +Condition : (isnotnull(cs_warehouse_sk#1) AND isnotnull(cs_item_sk#2)) -(3) Filter [codegen id : 1] +(3) ColumnarToRow [codegen id : 1] Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Condition : (isnotnull(cs_warehouse_sk#1) AND isnotnull(cs_item_sk#2)) (4) Exchange Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] @@ -64,16 +64,16 @@ Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 3] -Input [4]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9, cr_returned_date_sk#10] - -(8) Filter [codegen id : 3] +(7) CometFilter Input [4]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9, cr_returned_date_sk#10] Condition : (isnotnull(cr_order_number#8) AND isnotnull(cr_item_sk#7)) -(9) Project [codegen id : 3] -Output [3]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] +(8) CometProject Input [4]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9, cr_returned_date_sk#10] +Arguments: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9], [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] + +(9) ColumnarToRow [codegen id : 3] +Input [3]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] (10) Exchange Input [3]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] @@ -100,12 +100,12 @@ Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 5] +(15) CometFilter Input [2]: [w_warehouse_sk#11, w_state#12] +Condition : isnotnull(w_warehouse_sk#11) -(16) Filter [codegen id : 5] +(16) ColumnarToRow [codegen id : 5] Input [2]: [w_warehouse_sk#11, w_state#12] -Condition : isnotnull(w_warehouse_sk#11) (17) BroadcastExchange Input [2]: [w_warehouse_sk#11, w_state#12] @@ -128,16 +128,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] - -(22) Filter [codegen id : 6] +(21) CometFilter Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] Condition : (((isnotnull(i_current_price#15) AND (i_current_price#15 >= 0.99)) AND (i_current_price#15 <= 1.49)) AND isnotnull(i_item_sk#13)) -(23) Project [codegen id : 6] -Output [2]: [i_item_sk#13, i_item_id#14] +(22) CometProject Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] +Arguments: [i_item_sk#13, i_item_id#14], [i_item_sk#13, i_item_id#14] + +(23) ColumnarToRow [codegen id : 6] +Input [2]: [i_item_sk#13, i_item_id#14] (24) BroadcastExchange Input [2]: [i_item_sk#13, i_item_id#14] @@ -192,8 +192,8 @@ Arguments: 100, [w_state#12 ASC NULLS FIRST, i_item_id#14 ASC NULLS FIRST], [w_s Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 BroadcastExchange (37) -+- * Filter (36) - +- * ColumnarToRow (35) ++- * ColumnarToRow (36) + +- CometFilter (35) +- CometScan parquet spark_catalog.default.date_dim (34) @@ -204,12 +204,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 1] +(35) CometFilter Input [2]: [d_date_sk#16, d_date#17] +Condition : (((isnotnull(d_date#17) AND (d_date#17 >= 2000-02-10)) AND (d_date#17 <= 2000-04-10)) AND isnotnull(d_date_sk#16)) -(36) Filter [codegen id : 1] +(36) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#16, d_date#17] -Condition : (((isnotnull(d_date#17) AND (d_date#17 >= 2000-02-10)) AND (d_date#17 <= 2000-04-10)) AND isnotnull(d_date_sk#16)) (37) BroadcastExchange Input [2]: [d_date_sk#16, d_date#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/simplified.txt index 6234f2520..10e0735b4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/simplified.txt @@ -19,16 +19,16 @@ TakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after] InputAdapter Exchange [cs_order_number,cs_item_sk] #2 WholeStageCodegen (1) - Filter [cs_warehouse_sk,cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_warehouse_sk,cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_date,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter WholeStageCodegen (4) @@ -36,25 +36,25 @@ TakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after] InputAdapter Exchange [cr_order_number,cr_item_sk] #4 WholeStageCodegen (3) - Project [cr_item_sk,cr_order_number,cr_refunded_cash] - Filter [cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cr_item_sk,cr_order_number,cr_refunded_cash] + CometFilter [cr_order_number,cr_item_sk] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] InputAdapter BroadcastExchange #5 WholeStageCodegen (5) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [w_warehouse_sk] CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] InputAdapter BroadcastExchange #6 WholeStageCodegen (6) - Project [i_item_sk,i_item_id] - Filter [i_current_price,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_item_sk,i_item_id] + CometFilter [i_current_price,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] InputAdapter ReusedExchange [d_date_sk,d_date] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/explain.txt index 38fe288b5..07196ba8c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/explain.txt @@ -5,19 +5,19 @@ TakeOrderedAndProject (20) +- * HashAggregate (17) +- * Project (16) +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (4) - : +- * Filter (3) - : +- * ColumnarToRow (2) + :- * ColumnarToRow (4) + : +- CometProject (3) + : +- CometFilter (2) : +- CometScan parquet spark_catalog.default.item (1) +- BroadcastExchange (14) +- * Project (13) +- * Filter (12) +- * HashAggregate (11) +- Exchange (10) - +- * HashAggregate (9) - +- * Project (8) - +- * Filter (7) - +- * ColumnarToRow (6) + +- * ColumnarToRow (9) + +- CometHashAggregate (8) + +- CometProject (7) + +- CometFilter (6) +- CometScan parquet spark_catalog.default.item (5) @@ -28,16 +28,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manufact_id), GreaterThanOrEqual(i_manufact_id,738), LessThanOrEqual(i_manufact_id,778), IsNotNull(i_manufact)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] Condition : (((isnotnull(i_manufact_id#1) AND (i_manufact_id#1 >= 738)) AND (i_manufact_id#1 <= 778)) AND isnotnull(i_manufact#2)) -(4) Project [codegen id : 3] -Output [2]: [i_manufact#2, i_product_name#3] +(3) CometProject Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] +Arguments: [i_manufact#2, i_product_name#3], [i_manufact#2, i_product_name#3] + +(4) ColumnarToRow [codegen id : 3] +Input [2]: [i_manufact#2, i_product_name#3] (5) Scan parquet spark_catalog.default.item Output [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] @@ -46,42 +46,40 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [Or(Or(And(EqualTo(i_category,Women ),Or(And(And(Or(EqualTo(i_color,powder ),EqualTo(i_color,khaki )),Or(EqualTo(i_units,Ounce ),EqualTo(i_units,Oz ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large ))),And(And(Or(EqualTo(i_color,brown ),EqualTo(i_color,honeydew )),Or(EqualTo(i_units,Bunch ),EqualTo(i_units,Ton ))),Or(EqualTo(i_size,N/A ),EqualTo(i_size,small ))))),And(EqualTo(i_category,Men ),Or(And(And(Or(EqualTo(i_color,floral ),EqualTo(i_color,deep )),Or(EqualTo(i_units,N/A ),EqualTo(i_units,Dozen ))),Or(EqualTo(i_size,petite ),EqualTo(i_size,large ))),And(And(Or(EqualTo(i_color,light ),EqualTo(i_color,cornflower )),Or(EqualTo(i_units,Box ),EqualTo(i_units,Pound ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large )))))),Or(And(EqualTo(i_category,Women ),Or(And(And(Or(EqualTo(i_color,midnight ),EqualTo(i_color,snow )),Or(EqualTo(i_units,Pallet ),EqualTo(i_units,Gross ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large ))),And(And(Or(EqualTo(i_color,cyan ),EqualTo(i_color,papaya )),Or(EqualTo(i_units,Cup ),EqualTo(i_units,Dram ))),Or(EqualTo(i_size,N/A ),EqualTo(i_size,small ))))),And(EqualTo(i_category,Men ),Or(And(And(Or(EqualTo(i_color,orange ),EqualTo(i_color,frosted )),Or(EqualTo(i_units,Each ),EqualTo(i_units,Tbl ))),Or(EqualTo(i_size,petite ),EqualTo(i_size,large ))),And(And(Or(EqualTo(i_color,forest ),EqualTo(i_color,ghost )),Or(EqualTo(i_units,Lb ),EqualTo(i_units,Bundle ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large ))))))), IsNotNull(i_manufact)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] - -(7) Filter [codegen id : 1] +(6) CometFilter Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] Condition : (((((i_category#4 = Women ) AND (((((i_color#7 = powder ) OR (i_color#7 = khaki )) AND ((i_units#8 = Ounce ) OR (i_units#8 = Oz ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))) OR ((((i_color#7 = brown ) OR (i_color#7 = honeydew )) AND ((i_units#8 = Bunch ) OR (i_units#8 = Ton ))) AND ((i_size#6 = N/A ) OR (i_size#6 = small ))))) OR ((i_category#4 = Men ) AND (((((i_color#7 = floral ) OR (i_color#7 = deep )) AND ((i_units#8 = N/A ) OR (i_units#8 = Dozen ))) AND ((i_size#6 = petite ) OR (i_size#6 = large ))) OR ((((i_color#7 = light ) OR (i_color#7 = cornflower )) AND ((i_units#8 = Box ) OR (i_units#8 = Pound ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large )))))) OR (((i_category#4 = Women ) AND (((((i_color#7 = midnight ) OR (i_color#7 = snow )) AND ((i_units#8 = Pallet ) OR (i_units#8 = Gross ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))) OR ((((i_color#7 = cyan ) OR (i_color#7 = papaya )) AND ((i_units#8 = Cup ) OR (i_units#8 = Dram ))) AND ((i_size#6 = N/A ) OR (i_size#6 = small ))))) OR ((i_category#4 = Men ) AND (((((i_color#7 = orange ) OR (i_color#7 = frosted )) AND ((i_units#8 = Each ) OR (i_units#8 = Tbl ))) AND ((i_size#6 = petite ) OR (i_size#6 = large ))) OR ((((i_color#7 = forest ) OR (i_color#7 = ghost )) AND ((i_units#8 = Lb ) OR (i_units#8 = Bundle ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))))))) AND isnotnull(i_manufact#5)) -(8) Project [codegen id : 1] -Output [1]: [i_manufact#5] +(7) CometProject Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] +Arguments: [i_manufact#5], [i_manufact#5] -(9) HashAggregate [codegen id : 1] +(8) CometHashAggregate Input [1]: [i_manufact#5] Keys [1]: [i_manufact#5] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#9] -Results [2]: [i_manufact#5, count#10] + +(9) ColumnarToRow [codegen id : 1] +Input [2]: [i_manufact#5, count#9] (10) Exchange -Input [2]: [i_manufact#5, count#10] +Input [2]: [i_manufact#5, count#9] Arguments: hashpartitioning(i_manufact#5, 5), ENSURE_REQUIREMENTS, [plan_id=1] (11) HashAggregate [codegen id : 2] -Input [2]: [i_manufact#5, count#10] +Input [2]: [i_manufact#5, count#9] Keys [1]: [i_manufact#5] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#11] -Results [2]: [count(1)#11 AS item_cnt#12, i_manufact#5] +Aggregate Attributes [1]: [count(1)#10] +Results [2]: [count(1)#10 AS item_cnt#11, i_manufact#5] (12) Filter [codegen id : 2] -Input [2]: [item_cnt#12, i_manufact#5] -Condition : (item_cnt#12 > 0) +Input [2]: [item_cnt#11, i_manufact#5] +Condition : (item_cnt#11 > 0) (13) Project [codegen id : 2] Output [1]: [i_manufact#5] -Input [2]: [item_cnt#12, i_manufact#5] +Input [2]: [item_cnt#11, i_manufact#5] (14) BroadcastExchange Input [1]: [i_manufact#5] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/simplified.txt index 3d12af6f4..e31217066 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/simplified.txt @@ -7,10 +7,10 @@ TakeOrderedAndProject [i_product_name] HashAggregate [i_product_name] Project [i_product_name] BroadcastHashJoin [i_manufact,i_manufact] - Project [i_manufact,i_product_name] - Filter [i_manufact_id,i_manufact] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_manufact,i_product_name] + CometFilter [i_manufact_id,i_manufact] CometScan parquet spark_catalog.default.item [i_manufact_id,i_manufact,i_product_name] InputAdapter BroadcastExchange #2 @@ -21,9 +21,9 @@ TakeOrderedAndProject [i_product_name] InputAdapter Exchange [i_manufact] #3 WholeStageCodegen (1) - HashAggregate [i_manufact] [count,count] - Project [i_manufact] - Filter [i_category,i_color,i_units,i_size,i_manufact] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometHashAggregate [i_manufact] + CometProject [i_manufact] + CometFilter [i_category,i_color,i_units,i_size,i_manufact] CometScan parquet spark_catalog.default.item [i_category,i_manufact,i_size,i_color,i_units] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/explain.txt index b458c6b45..d51d63d8a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/explain.txt @@ -7,18 +7,18 @@ TakeOrderedAndProject (21) +- * BroadcastHashJoin Inner BuildRight (16) :- * Project (10) : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) + : :- * ColumnarToRow (4) + : : +- CometProject (3) + : : +- CometFilter (2) : : +- CometScan parquet spark_catalog.default.date_dim (1) : +- BroadcastExchange (8) - : +- * Filter (7) - : +- * ColumnarToRow (6) + : +- * ColumnarToRow (7) + : +- CometFilter (6) : +- CometScan parquet spark_catalog.default.store_sales (5) +- BroadcastExchange (15) - +- * Project (14) - +- * Filter (13) - +- * ColumnarToRow (12) + +- * ColumnarToRow (14) + +- CometProject (13) + +- CometFilter (12) +- CometScan parquet spark_catalog.default.item (11) @@ -29,16 +29,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1)) -(4) Project [codegen id : 3] -Output [2]: [d_date_sk#1, d_year#2] +(3) CometProject Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] + +(4) ColumnarToRow [codegen id : 3] +Input [2]: [d_date_sk#1, d_year#2] (5) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] @@ -48,12 +48,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] +(6) CometFilter Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Condition : isnotnull(ss_item_sk#4) -(7) Filter [codegen id : 1] +(7) ColumnarToRow [codegen id : 1] Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_item_sk#4) (8) BroadcastExchange Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] @@ -76,16 +76,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] - -(13) Filter [codegen id : 2] +(12) CometFilter Input [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 1)) AND isnotnull(i_item_sk#7)) -(14) Project [codegen id : 2] -Output [3]: [i_item_sk#7, i_category_id#8, i_category#9] +(13) CometProject Input [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] +Arguments: [i_item_sk#7, i_category_id#8, i_category#9], [i_item_sk#7, i_category_id#8, i_category#9] + +(14) ColumnarToRow [codegen id : 2] +Input [3]: [i_item_sk#7, i_category_id#8, i_category#9] (15) BroadcastExchange Input [3]: [i_item_sk#7, i_category_id#8, i_category#9] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/simplified.txt index 2b6ed6411..67906b8c7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/simplified.txt @@ -9,23 +9,23 @@ TakeOrderedAndProject [sum(ss_ext_sales_price),d_year,i_category_id,i_category] BroadcastHashJoin [ss_item_sk,i_item_sk] Project [d_year,ss_item_sk,ss_ext_sales_price] BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk,d_year] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_year] + CometFilter [d_moy,d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #2 WholeStageCodegen (1) - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Project [i_item_sk,i_category_id,i_category] - Filter [i_manager_id,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_item_sk,i_category_id,i_category] + CometFilter [i_manager_id,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_category_id,i_category,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/explain.txt index 925d9f3df..e892aa469 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/explain.txt @@ -7,18 +7,18 @@ TakeOrderedAndProject (21) +- * BroadcastHashJoin Inner BuildRight (16) :- * Project (10) : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) + : :- * ColumnarToRow (4) + : : +- CometProject (3) + : : +- CometFilter (2) : : +- CometScan parquet spark_catalog.default.date_dim (1) : +- BroadcastExchange (8) - : +- * Filter (7) - : +- * ColumnarToRow (6) + : +- * ColumnarToRow (7) + : +- CometFilter (6) : +- CometScan parquet spark_catalog.default.store_sales (5) +- BroadcastExchange (15) - +- * Project (14) - +- * Filter (13) - +- * ColumnarToRow (12) + +- * ColumnarToRow (14) + +- CometProject (13) + +- CometFilter (12) +- CometScan parquet spark_catalog.default.store (11) @@ -29,16 +29,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] Condition : ((isnotnull(d_year#2) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1)) -(4) Project [codegen id : 3] -Output [2]: [d_date_sk#1, d_day_name#3] +(3) CometProject Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] +Arguments: [d_date_sk#1, d_day_name#3], [d_date_sk#1, d_day_name#3] + +(4) ColumnarToRow [codegen id : 3] +Input [2]: [d_date_sk#1, d_day_name#3] (5) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] @@ -48,12 +48,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] +(6) CometFilter Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] +Condition : isnotnull(ss_store_sk#4) -(7) Filter [codegen id : 1] +(7) ColumnarToRow [codegen id : 1] Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_store_sk#4) (8) BroadcastExchange Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] @@ -76,16 +76,16 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] - -(13) Filter [codegen id : 2] +(12) CometFilter Input [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] Condition : ((isnotnull(s_gmt_offset#10) AND (s_gmt_offset#10 = -5.00)) AND isnotnull(s_store_sk#7)) -(14) Project [codegen id : 2] -Output [3]: [s_store_sk#7, s_store_id#8, s_store_name#9] +(13) CometProject Input [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] +Arguments: [s_store_sk#7, s_store_id#8, s_store_name#9], [s_store_sk#7, s_store_id#8, s_store_name#9] + +(14) ColumnarToRow [codegen id : 2] +Input [3]: [s_store_sk#7, s_store_id#8, s_store_name#9] (15) BroadcastExchange Input [3]: [s_store_sk#7, s_store_id#8, s_store_name#9] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/simplified.txt index b00665aed..ef2043096 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/simplified.txt @@ -9,23 +9,23 @@ TakeOrderedAndProject [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed BroadcastHashJoin [ss_store_sk,s_store_sk] Project [d_day_name,ss_store_sk,ss_sales_price] BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk,d_day_name] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_day_name] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_day_name] InputAdapter BroadcastExchange #2 WholeStageCodegen (1) - Filter [ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Project [s_store_sk,s_store_id,s_store_name] - Filter [s_gmt_offset,s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [s_store_sk,s_store_id,s_store_name] + CometFilter [s_gmt_offset,s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/explain.txt index ba9e48ec7..812f9f391 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/explain.txt @@ -20,9 +20,9 @@ TakeOrderedAndProject (43) : : : +- * HashAggregate (7) : : : +- Exchange (6) : : : +- * HashAggregate (5) - : : : +- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) + : : : +- * ColumnarToRow (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : +- * Sort (31) : : +- Exchange (30) @@ -38,8 +38,8 @@ TakeOrderedAndProject (43) : : +- * HashAggregate (20) : : +- ReusedExchange (19) : +- BroadcastExchange (37) - : +- * Filter (36) - : +- * ColumnarToRow (35) + : +- * ColumnarToRow (36) + : +- CometFilter (35) : +- CometScan parquet spark_catalog.default.item (34) +- ReusedExchange (40) @@ -51,16 +51,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_store_sk#2) AND (ss_store_sk#2 = 4)) -(4) Project [codegen id : 1] -Output [2]: [ss_item_sk#1, ss_net_profit#3] +(3) CometProject Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] +Arguments: [ss_item_sk#1, ss_net_profit#3], [ss_item_sk#1, ss_net_profit#3] + +(4) ColumnarToRow [codegen id : 1] +Input [2]: [ss_item_sk#1, ss_net_profit#3] (5) HashAggregate [codegen id : 1] Input [2]: [ss_item_sk#1, ss_net_profit#3] @@ -195,12 +195,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 11] +(35) CometFilter Input [2]: [i_item_sk#23, i_product_name#24] +Condition : isnotnull(i_item_sk#23) -(36) Filter [codegen id : 11] +(36) ColumnarToRow [codegen id : 11] Input [2]: [i_item_sk#23, i_product_name#24] -Condition : isnotnull(i_item_sk#23) (37) BroadcastExchange Input [2]: [i_item_sk#23, i_product_name#24] @@ -239,9 +239,9 @@ Subquery:1 Hosting operator id = 8 Hosting Expression = Subquery scalar-subquery * HashAggregate (50) +- Exchange (49) +- * HashAggregate (48) - +- * Project (47) - +- * Filter (46) - +- * ColumnarToRow (45) + +- * ColumnarToRow (47) + +- CometProject (46) + +- CometFilter (45) +- CometScan parquet spark_catalog.default.store_sales (44) @@ -252,16 +252,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4), IsNull(ss_addr_sk)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 1] -Input [4]: [ss_addr_sk#29, ss_store_sk#30, ss_net_profit#31, ss_sold_date_sk#32] - -(46) Filter [codegen id : 1] +(45) CometFilter Input [4]: [ss_addr_sk#29, ss_store_sk#30, ss_net_profit#31, ss_sold_date_sk#32] Condition : ((isnotnull(ss_store_sk#30) AND (ss_store_sk#30 = 4)) AND isnull(ss_addr_sk#29)) -(47) Project [codegen id : 1] -Output [2]: [ss_store_sk#30, ss_net_profit#31] +(46) CometProject Input [4]: [ss_addr_sk#29, ss_store_sk#30, ss_net_profit#31, ss_sold_date_sk#32] +Arguments: [ss_store_sk#30, ss_net_profit#31], [ss_store_sk#30, ss_net_profit#31] + +(47) ColumnarToRow [codegen id : 1] +Input [2]: [ss_store_sk#30, ss_net_profit#31] (48) HashAggregate [codegen id : 1] Input [2]: [ss_store_sk#30, ss_net_profit#31] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/simplified.txt index adcad1417..35a3e9efa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/simplified.txt @@ -32,20 +32,20 @@ TakeOrderedAndProject [rnk,best_performing,worst_performing] Exchange [ss_store_sk] #4 WholeStageCodegen (1) HashAggregate [ss_store_sk,ss_net_profit] [sum,count,sum,count] - Project [ss_store_sk,ss_net_profit] - Filter [ss_store_sk,ss_addr_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_store_sk,ss_net_profit] + CometFilter [ss_store_sk,ss_addr_sk] CometScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] HashAggregate [ss_item_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),item_sk,rank_col,sum,count] InputAdapter Exchange [ss_item_sk] #3 WholeStageCodegen (1) HashAggregate [ss_item_sk,ss_net_profit] [sum,count,sum,count] - Project [ss_item_sk,ss_net_profit] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_item_sk,ss_net_profit] + CometFilter [ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] InputAdapter WholeStageCodegen (10) @@ -73,9 +73,9 @@ TakeOrderedAndProject [rnk,best_performing,worst_performing] InputAdapter BroadcastExchange #7 WholeStageCodegen (11) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_product_name] InputAdapter ReusedExchange [i_item_sk,i_product_name] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/explain.txt index aaf17cf45..d0d74569b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/explain.txt @@ -14,26 +14,26 @@ TakeOrderedAndProject (36) : : : +- * BroadcastHashJoin Inner BuildRight (14) : : : :- * Project (9) : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) + : : : : +- * ColumnarToRow (6) + : : : : +- CometFilter (5) : : : : +- CometScan parquet spark_catalog.default.customer (4) : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) + : : : +- * ColumnarToRow (12) + : : : +- CometFilter (11) : : : +- CometScan parquet spark_catalog.default.customer_address (10) : : +- ReusedExchange (16) : +- BroadcastExchange (22) - : +- * Filter (21) - : +- * ColumnarToRow (20) + : +- * ColumnarToRow (21) + : +- CometFilter (20) : +- CometScan parquet spark_catalog.default.item (19) +- BroadcastExchange (29) - +- * Project (28) - +- * Filter (27) - +- * ColumnarToRow (26) + +- * ColumnarToRow (28) + +- CometProject (27) + +- CometFilter (26) +- CometScan parquet spark_catalog.default.item (25) @@ -45,12 +45,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#5), dynamicpruningexpression(ws_sol PushedFilters: [IsNotNull(ws_bill_customer_sk), IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 6] +(2) CometFilter Input [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] +Condition : (isnotnull(ws_bill_customer_sk#3) AND isnotnull(ws_item_sk#2)) -(3) Filter [codegen id : 6] +(3) ColumnarToRow [codegen id : 6] Input [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] -Condition : (isnotnull(ws_bill_customer_sk#3) AND isnotnull(ws_item_sk#2)) (4) Scan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#7, c_current_addr_sk#8] @@ -59,12 +59,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [2]: [c_customer_sk#7, c_current_addr_sk#8] +Condition : (isnotnull(c_customer_sk#7) AND isnotnull(c_current_addr_sk#8)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [2]: [c_customer_sk#7, c_current_addr_sk#8] -Condition : (isnotnull(c_customer_sk#7) AND isnotnull(c_current_addr_sk#8)) (7) BroadcastExchange Input [2]: [c_customer_sk#7, c_current_addr_sk#8] @@ -87,12 +87,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] +(11) CometFilter Input [3]: [ca_address_sk#9, ca_city#10, ca_zip#11] +Condition : isnotnull(ca_address_sk#9) -(12) Filter [codegen id : 2] +(12) ColumnarToRow [codegen id : 2] Input [3]: [ca_address_sk#9, ca_city#10, ca_zip#11] -Condition : isnotnull(ca_address_sk#9) (13) BroadcastExchange Input [3]: [ca_address_sk#9, ca_city#10, ca_zip#11] @@ -128,12 +128,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 4] +(20) CometFilter Input [2]: [i_item_sk#13, i_item_id#14] +Condition : isnotnull(i_item_sk#13) -(21) Filter [codegen id : 4] +(21) ColumnarToRow [codegen id : 4] Input [2]: [i_item_sk#13, i_item_id#14] -Condition : isnotnull(i_item_sk#13) (22) BroadcastExchange Input [2]: [i_item_sk#13, i_item_id#14] @@ -156,16 +156,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [In(i_item_sk, [11,13,17,19,2,23,29,3,5,7])] ReadSchema: struct -(26) ColumnarToRow [codegen id : 5] -Input [2]: [i_item_sk#15, i_item_id#16] - -(27) Filter [codegen id : 5] +(26) CometFilter Input [2]: [i_item_sk#15, i_item_id#16] Condition : i_item_sk#15 IN (2,3,5,7,11,13,17,19,23,29) -(28) Project [codegen id : 5] -Output [1]: [i_item_id#16] +(27) CometProject Input [2]: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_id#16], [i_item_id#16] + +(28) ColumnarToRow [codegen id : 5] +Input [1]: [i_item_id#16] (29) BroadcastExchange Input [1]: [i_item_id#16] @@ -211,9 +211,9 @@ Arguments: 100, [ca_zip#11 ASC NULLS FIRST, ca_city#10 ASC NULLS FIRST], [ca_zip Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#5 IN dynamicpruning#6 BroadcastExchange (41) -+- * Project (40) - +- * Filter (39) - +- * ColumnarToRow (38) ++- * ColumnarToRow (40) + +- CometProject (39) + +- CometFilter (38) +- CometScan parquet spark_catalog.default.date_dim (37) @@ -224,16 +224,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#12, d_year#21, d_qoy#22] - -(39) Filter [codegen id : 1] +(38) CometFilter Input [3]: [d_date_sk#12, d_year#21, d_qoy#22] Condition : ((((isnotnull(d_qoy#22) AND isnotnull(d_year#21)) AND (d_qoy#22 = 2)) AND (d_year#21 = 2001)) AND isnotnull(d_date_sk#12)) -(40) Project [codegen id : 1] -Output [1]: [d_date_sk#12] +(39) CometProject Input [3]: [d_date_sk#12, d_year#21, d_qoy#22] +Arguments: [d_date_sk#12], [d_date_sk#12] + +(40) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#12] (41) BroadcastExchange Input [1]: [d_date_sk#12] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/simplified.txt index 05579f166..383cbb7e3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/simplified.txt @@ -16,46 +16,46 @@ TakeOrderedAndProject [ca_zip,ca_city,sum(ws_sales_price)] BroadcastHashJoin [c_current_addr_sk,ca_address_sk] Project [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk] BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk] - Filter [ws_bill_customer_sk,ws_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_bill_customer_sk,ws_item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_qoy,d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_current_addr_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city,ca_zip] InputAdapter ReusedExchange [d_date_sk] #2 InputAdapter BroadcastExchange #5 WholeStageCodegen (4) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] InputAdapter BroadcastExchange #6 WholeStageCodegen (5) - Project [i_item_id] - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_item_id] + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/explain.txt index d51633680..e07e2ab24 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/explain.txt @@ -15,27 +15,27 @@ TakeOrderedAndProject (39) : : : : +- * BroadcastHashJoin Inner BuildRight (12) : : : : :- * Project (6) : : : : : +- * BroadcastHashJoin Inner BuildRight (5) - : : : : : :- * Filter (3) - : : : : : : +- * ColumnarToRow (2) + : : : : : :- * ColumnarToRow (3) + : : : : : : +- CometFilter (2) : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : : : +- ReusedExchange (4) : : : : +- BroadcastExchange (11) - : : : : +- * Project (10) - : : : : +- * Filter (9) - : : : : +- * ColumnarToRow (8) + : : : : +- * ColumnarToRow (10) + : : : : +- CometProject (9) + : : : : +- CometFilter (8) : : : : +- CometScan parquet spark_catalog.default.store (7) : : : +- BroadcastExchange (18) - : : : +- * Project (17) - : : : +- * Filter (16) - : : : +- * ColumnarToRow (15) + : : : +- * ColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometFilter (15) : : : +- CometScan parquet spark_catalog.default.household_demographics (14) : : +- BroadcastExchange (24) - : : +- * Filter (23) - : : +- * ColumnarToRow (22) + : : +- * ColumnarToRow (23) + : : +- CometFilter (22) : : +- CometScan parquet spark_catalog.default.customer_address (21) : +- BroadcastExchange (33) - : +- * Filter (32) - : +- * ColumnarToRow (31) + : +- * ColumnarToRow (32) + : +- CometFilter (31) : +- CometScan parquet spark_catalog.default.customer (30) +- ReusedExchange (36) @@ -48,12 +48,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] +(2) CometFilter Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] +Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) -(3) Filter [codegen id : 5] +(3) ColumnarToRow [codegen id : 5] Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) (4) ReusedExchange [Reuses operator id: 44] Output [1]: [d_date_sk#10] @@ -75,16 +75,16 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#11, s_city#12] - -(9) Filter [codegen id : 2] +(8) CometFilter Input [2]: [s_store_sk#11, s_city#12] Condition : (s_city#12 IN (Fairview,Midway) AND isnotnull(s_store_sk#11)) -(10) Project [codegen id : 2] -Output [1]: [s_store_sk#11] +(9) CometProject Input [2]: [s_store_sk#11, s_city#12] +Arguments: [s_store_sk#11], [s_store_sk#11] + +(10) ColumnarToRow [codegen id : 2] +Input [1]: [s_store_sk#11] (11) BroadcastExchange Input [1]: [s_store_sk#11] @@ -107,16 +107,16 @@ Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 3] -Input [3]: [hd_demo_sk#13, hd_dep_count#14, hd_vehicle_count#15] - -(16) Filter [codegen id : 3] +(15) CometFilter Input [3]: [hd_demo_sk#13, hd_dep_count#14, hd_vehicle_count#15] Condition : (((hd_dep_count#14 = 4) OR (hd_vehicle_count#15 = 3)) AND isnotnull(hd_demo_sk#13)) -(17) Project [codegen id : 3] -Output [1]: [hd_demo_sk#13] +(16) CometProject Input [3]: [hd_demo_sk#13, hd_dep_count#14, hd_vehicle_count#15] +Arguments: [hd_demo_sk#13], [hd_demo_sk#13] + +(17) ColumnarToRow [codegen id : 3] +Input [1]: [hd_demo_sk#13] (18) BroadcastExchange Input [1]: [hd_demo_sk#13] @@ -139,12 +139,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] ReadSchema: struct -(22) ColumnarToRow [codegen id : 4] +(22) CometFilter Input [2]: [ca_address_sk#16, ca_city#17] +Condition : (isnotnull(ca_address_sk#16) AND isnotnull(ca_city#17)) -(23) Filter [codegen id : 4] +(23) ColumnarToRow [codegen id : 4] Input [2]: [ca_address_sk#16, ca_city#17] -Condition : (isnotnull(ca_address_sk#16) AND isnotnull(ca_city#17)) (24) BroadcastExchange Input [2]: [ca_address_sk#16, ca_city#17] @@ -185,12 +185,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 6] +(31) CometFilter Input [4]: [c_customer_sk#27, c_current_addr_sk#28, c_first_name#29, c_last_name#30] +Condition : (isnotnull(c_customer_sk#27) AND isnotnull(c_current_addr_sk#28)) -(32) Filter [codegen id : 6] +(32) ColumnarToRow [codegen id : 6] Input [4]: [c_customer_sk#27, c_current_addr_sk#28, c_first_name#29, c_last_name#30] -Condition : (isnotnull(c_customer_sk#27) AND isnotnull(c_current_addr_sk#28)) (33) BroadcastExchange Input [4]: [c_customer_sk#27, c_current_addr_sk#28, c_first_name#29, c_last_name#30] @@ -227,9 +227,9 @@ Arguments: 100, [c_last_name#30 ASC NULLS FIRST, c_first_name#29 ASC NULLS FIRST Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 BroadcastExchange (44) -+- * Project (43) - +- * Filter (42) - +- * ColumnarToRow (41) ++- * ColumnarToRow (43) + +- CometProject (42) + +- CometFilter (41) +- CometScan parquet spark_catalog.default.date_dim (40) @@ -240,16 +240,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_dow, [0,6]), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#10, d_year#33, d_dow#34] - -(42) Filter [codegen id : 1] +(41) CometFilter Input [3]: [d_date_sk#10, d_year#33, d_dow#34] Condition : ((d_dow#34 IN (6,0) AND d_year#33 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) -(43) Project [codegen id : 1] -Output [1]: [d_date_sk#10] +(42) CometProject Input [3]: [d_date_sk#10, d_year#33, d_dow#34] +Arguments: [d_date_sk#10], [d_date_sk#10] + +(43) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#10] (44) BroadcastExchange Input [1]: [d_date_sk#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/simplified.txt index 376fc2f62..04c59a2d3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/simplified.txt @@ -17,49 +17,49 @@ TakeOrderedAndProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_nu BroadcastHashJoin [ss_store_sk,s_store_sk] Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_addr_sk,ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk,ss_hdemo_sk,ss_addr_sk,ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dow,d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_dow,d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] InputAdapter ReusedExchange [d_date_sk] #2 InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Project [s_store_sk] - Filter [s_city,s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [s_store_sk] + CometFilter [s_city,s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_city] InputAdapter BroadcastExchange #4 WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [hd_demo_sk] + CometFilter [hd_dep_count,hd_vehicle_count,hd_demo_sk] CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] InputAdapter BroadcastExchange #5 WholeStageCodegen (4) - Filter [ca_address_sk,ca_city] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ca_address_sk,ca_city] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] InputAdapter BroadcastExchange #6 WholeStageCodegen (6) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_current_addr_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] InputAdapter ReusedExchange [ca_address_sk,ca_city] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/explain.txt index 74b0ed8ff..fca7362ce 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/explain.txt @@ -20,17 +20,17 @@ TakeOrderedAndProject (45) : : : +- * BroadcastHashJoin Inner BuildRight (11) : : : :- * Project (9) : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.item (1) : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) + : : : : +- * ColumnarToRow (6) + : : : : +- CometFilter (5) : : : : +- CometScan parquet spark_catalog.default.store_sales (4) : : : +- ReusedExchange (10) : : +- BroadcastExchange (16) - : : +- * Filter (15) - : : +- * ColumnarToRow (14) + : : +- * ColumnarToRow (15) + : : +- CometFilter (14) : : +- CometScan parquet spark_catalog.default.store (13) : +- BroadcastExchange (35) : +- * Project (34) @@ -53,12 +53,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) -(3) Filter [codegen id : 4] +(3) ColumnarToRow [codegen id : 4] Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) (4) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] @@ -68,12 +68,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Condition : (isnotnull(ss_item_sk#4) AND isnotnull(ss_store_sk#5)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_item_sk#4) AND isnotnull(ss_store_sk#5)) (7) BroadcastExchange Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] @@ -109,12 +109,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] +(14) CometFilter Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] +Condition : ((isnotnull(s_store_sk#12) AND isnotnull(s_store_name#13)) AND isnotnull(s_company_name#14)) -(15) Filter [codegen id : 3] +(15) ColumnarToRow [codegen id : 3] Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Condition : ((isnotnull(s_store_sk#12) AND isnotnull(s_store_name#13)) AND isnotnull(s_company_name#14)) (16) BroadcastExchange Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] @@ -253,8 +253,8 @@ Arguments: 100, [(sum_sales#18 - avg_monthly_sales#21) ASC NULLS FIRST, s_store_ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 BroadcastExchange (49) -+- * Filter (48) - +- * ColumnarToRow (47) ++- * ColumnarToRow (48) + +- CometFilter (47) +- CometScan parquet spark_catalog.default.date_dim (46) @@ -265,12 +265,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct -(47) ColumnarToRow [codegen id : 1] +(47) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_moy#11] +Condition : ((((d_year#10 = 1999) OR ((d_year#10 = 1998) AND (d_moy#11 = 12))) OR ((d_year#10 = 2000) AND (d_moy#11 = 1))) AND isnotnull(d_date_sk#9)) -(48) Filter [codegen id : 1] +(48) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((d_year#10 = 1999) OR ((d_year#10 = 1998) AND (d_moy#11 = 12))) OR ((d_year#10 = 2000) AND (d_moy#11 = 1))) AND isnotnull(d_date_sk#9)) (49) BroadcastExchange Input [3]: [d_date_sk#9, d_year#10, d_moy#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/simplified.txt index 5dba00296..80b8da7b1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/simplified.txt @@ -28,32 +28,32 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk,i_category,i_brand] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk,i_category,i_brand] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk,ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_year,d_moy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter ReusedExchange [d_date_sk,d_year,d_moy] #4 InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - Filter [s_store_sk,s_store_name,s_company_name] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk,s_store_name,s_company_name] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] InputAdapter BroadcastExchange #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/explain.txt index c2b675c34..718f3fb31 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/explain.txt @@ -10,21 +10,21 @@ : : +- * BroadcastHashJoin Inner BuildRight (14) : : :- * Project (9) : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) + : : : +- * ColumnarToRow (6) + : : : +- CometFilter (5) : : : +- CometScan parquet spark_catalog.default.store (4) : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) + : : +- * ColumnarToRow (12) + : : +- CometFilter (11) : : +- CometScan parquet spark_catalog.default.customer_demographics (10) : +- BroadcastExchange (20) - : +- * Project (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) + : +- * ColumnarToRow (19) + : +- CometProject (18) + : +- CometFilter (17) : +- CometScan parquet spark_catalog.default.customer_address (16) +- ReusedExchange (23) @@ -37,12 +37,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_addr_sk), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ss_net_profit,0.00),LessThanOrEqual(ss_net_profit,2000.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,3000.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,25000.00)))] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] +(2) CometFilter Input [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Condition : ((((isnotnull(ss_store_sk#3) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_addr_sk#2)) AND ((((ss_sales_price#5 >= 100.00) AND (ss_sales_price#5 <= 150.00)) OR ((ss_sales_price#5 >= 50.00) AND (ss_sales_price#5 <= 100.00))) OR ((ss_sales_price#5 >= 150.00) AND (ss_sales_price#5 <= 200.00)))) AND ((((ss_net_profit#6 >= 0.00) AND (ss_net_profit#6 <= 2000.00)) OR ((ss_net_profit#6 >= 150.00) AND (ss_net_profit#6 <= 3000.00))) OR ((ss_net_profit#6 >= 50.00) AND (ss_net_profit#6 <= 25000.00)))) -(3) Filter [codegen id : 5] +(3) ColumnarToRow [codegen id : 5] Input [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Condition : ((((isnotnull(ss_store_sk#3) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_addr_sk#2)) AND ((((ss_sales_price#5 >= 100.00) AND (ss_sales_price#5 <= 150.00)) OR ((ss_sales_price#5 >= 50.00) AND (ss_sales_price#5 <= 100.00))) OR ((ss_sales_price#5 >= 150.00) AND (ss_sales_price#5 <= 200.00)))) AND ((((ss_net_profit#6 >= 0.00) AND (ss_net_profit#6 <= 2000.00)) OR ((ss_net_profit#6 >= 150.00) AND (ss_net_profit#6 <= 3000.00))) OR ((ss_net_profit#6 >= 50.00) AND (ss_net_profit#6 <= 25000.00)))) (4) Scan parquet spark_catalog.default.store Output [1]: [s_store_sk#9] @@ -51,12 +51,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [1]: [s_store_sk#9] +Condition : isnotnull(s_store_sk#9) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [1]: [s_store_sk#9] -Condition : isnotnull(s_store_sk#9) (7) BroadcastExchange Input [1]: [s_store_sk#9] @@ -79,12 +79,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,4 yr Degree )),And(EqualTo(cd_marital_status,D),EqualTo(cd_education_status,2 yr Degree ))),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College )))] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] +(11) CometFilter Input [3]: [cd_demo_sk#10, cd_marital_status#11, cd_education_status#12] +Condition : (isnotnull(cd_demo_sk#10) AND ((((cd_marital_status#11 = M) AND (cd_education_status#12 = 4 yr Degree )) OR ((cd_marital_status#11 = D) AND (cd_education_status#12 = 2 yr Degree ))) OR ((cd_marital_status#11 = S) AND (cd_education_status#12 = College )))) -(12) Filter [codegen id : 2] +(12) ColumnarToRow [codegen id : 2] Input [3]: [cd_demo_sk#10, cd_marital_status#11, cd_education_status#12] -Condition : (isnotnull(cd_demo_sk#10) AND ((((cd_marital_status#11 = M) AND (cd_education_status#12 = 4 yr Degree )) OR ((cd_marital_status#11 = D) AND (cd_education_status#12 = 2 yr Degree ))) OR ((cd_marital_status#11 = S) AND (cd_education_status#12 = College )))) (13) BroadcastExchange Input [3]: [cd_demo_sk#10, cd_marital_status#11, cd_education_status#12] @@ -107,16 +107,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [CO,OH,TX]),In(ca_state, [KY,MN,OR])),In(ca_state, [CA,MS,VA]))] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [3]: [ca_address_sk#13, ca_state#14, ca_country#15] - -(18) Filter [codegen id : 3] +(17) CometFilter Input [3]: [ca_address_sk#13, ca_state#14, ca_country#15] Condition : (((isnotnull(ca_country#15) AND (ca_country#15 = United States)) AND isnotnull(ca_address_sk#13)) AND ((ca_state#14 IN (CO,OH,TX) OR ca_state#14 IN (OR,MN,KY)) OR ca_state#14 IN (VA,CA,MS))) -(19) Project [codegen id : 3] -Output [2]: [ca_address_sk#13, ca_state#14] +(18) CometProject Input [3]: [ca_address_sk#13, ca_state#14, ca_country#15] +Arguments: [ca_address_sk#13, ca_state#14], [ca_address_sk#13, ca_state#14] + +(19) ColumnarToRow [codegen id : 3] +Input [2]: [ca_address_sk#13, ca_state#14] (20) BroadcastExchange Input [2]: [ca_address_sk#13, ca_state#14] @@ -167,9 +167,9 @@ Results [1]: [sum(ss_quantity#4)#19 AS sum(ss_quantity)#20] Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 BroadcastExchange (33) -+- * Project (32) - +- * Filter (31) - +- * ColumnarToRow (30) ++- * ColumnarToRow (32) + +- CometProject (31) + +- CometFilter (30) +- CometScan parquet spark_catalog.default.date_dim (29) @@ -180,16 +180,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#16, d_year#21] - -(31) Filter [codegen id : 1] +(30) CometFilter Input [2]: [d_date_sk#16, d_year#21] Condition : ((isnotnull(d_year#21) AND (d_year#21 = 2001)) AND isnotnull(d_date_sk#16)) -(32) Project [codegen id : 1] -Output [1]: [d_date_sk#16] +(31) CometProject Input [2]: [d_date_sk#16, d_year#21] +Arguments: [d_date_sk#16], [d_date_sk#16] + +(32) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#16] (33) BroadcastExchange Input [1]: [d_date_sk#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/simplified.txt index 5cc7040fb..4022da74f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/simplified.txt @@ -12,39 +12,39 @@ WholeStageCodegen (6) BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status,ss_sales_price] Project [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk,ss_cdemo_sk,ss_addr_sk,ss_sales_price,ss_net_profit] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk,ss_cdemo_sk,ss_addr_sk,ss_sales_price,ss_net_profit] CometScan parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Filter [s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk] InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - Filter [cd_demo_sk,cd_marital_status,cd_education_status] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - Project [ca_address_sk,ca_state] - Filter [ca_country,ca_address_sk,ca_state] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ca_address_sk,ca_state] + CometFilter [ca_country,ca_address_sk,ca_state] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] InputAdapter ReusedExchange [d_date_sk] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/explain.txt index 1ad84da7d..bbb550e05 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/explain.txt @@ -1,81 +1,78 @@ == Physical Plan == -TakeOrderedAndProject (77) -+- * HashAggregate (76) - +- Exchange (75) - +- * HashAggregate (74) - +- Union (73) - :- * Project (24) - : +- * Filter (23) - : +- Window (22) - : +- * Sort (21) - : +- Window (20) - : +- * Sort (19) - : +- Exchange (18) - : +- * HashAggregate (17) - : +- Exchange (16) - : +- * HashAggregate (15) - : +- * Project (14) - : +- * BroadcastHashJoin Inner BuildRight (13) - : :- * Project (11) - : : +- * BroadcastHashJoin Inner BuildLeft (10) - : : :- BroadcastExchange (5) - : : : +- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : +- * Project (9) - : : +- * Filter (8) - : : +- * ColumnarToRow (7) - : : +- CometScan parquet spark_catalog.default.web_returns (6) - : +- ReusedExchange (12) - :- * Project (48) - : +- * Filter (47) - : +- Window (46) - : +- * Sort (45) - : +- Window (44) - : +- * Sort (43) - : +- Exchange (42) - : +- * HashAggregate (41) - : +- Exchange (40) - : +- * HashAggregate (39) - : +- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (35) - : : +- * BroadcastHashJoin Inner BuildLeft (34) - : : :- BroadcastExchange (29) - : : : +- * Project (28) - : : : +- * Filter (27) - : : : +- * ColumnarToRow (26) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (25) - : : +- * Project (33) - : : +- * Filter (32) - : : +- * ColumnarToRow (31) - : : +- CometScan parquet spark_catalog.default.catalog_returns (30) - : +- ReusedExchange (36) - +- * Project (72) - +- * Filter (71) - +- Window (70) - +- * Sort (69) - +- Window (68) - +- * Sort (67) - +- Exchange (66) - +- * HashAggregate (65) - +- Exchange (64) - +- * HashAggregate (63) - +- * Project (62) - +- * BroadcastHashJoin Inner BuildRight (61) - :- * Project (59) - : +- * BroadcastHashJoin Inner BuildLeft (58) - : :- BroadcastExchange (53) - : : +- * Project (52) - : : +- * Filter (51) - : : +- * ColumnarToRow (50) - : : +- CometScan parquet spark_catalog.default.store_sales (49) - : +- * Project (57) - : +- * Filter (56) - : +- * ColumnarToRow (55) - : +- CometScan parquet spark_catalog.default.store_returns (54) - +- ReusedExchange (60) +TakeOrderedAndProject (74) ++- * HashAggregate (73) + +- Exchange (72) + +- * HashAggregate (71) + +- Union (70) + :- * Project (23) + : +- * Filter (22) + : +- Window (21) + : +- * Sort (20) + : +- Window (19) + : +- * Sort (18) + : +- Exchange (17) + : +- * HashAggregate (16) + : +- Exchange (15) + : +- * HashAggregate (14) + : +- * Project (13) + : +- * BroadcastHashJoin Inner BuildRight (12) + : :- * ColumnarToRow (10) + : : +- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometBroadcastExchange (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : +- CometProject (7) + : : +- CometFilter (6) + : : +- CometScan parquet spark_catalog.default.web_returns (5) + : +- ReusedExchange (11) + :- * Project (46) + : +- * Filter (45) + : +- Window (44) + : +- * Sort (43) + : +- Window (42) + : +- * Sort (41) + : +- Exchange (40) + : +- * HashAggregate (39) + : +- Exchange (38) + : +- * HashAggregate (37) + : +- * Project (36) + : +- * BroadcastHashJoin Inner BuildRight (35) + : :- * ColumnarToRow (33) + : : +- CometProject (32) + : : +- CometBroadcastHashJoin (31) + : : :- CometBroadcastExchange (27) + : : : +- CometProject (26) + : : : +- CometFilter (25) + : : : +- CometScan parquet spark_catalog.default.catalog_sales (24) + : : +- CometProject (30) + : : +- CometFilter (29) + : : +- CometScan parquet spark_catalog.default.catalog_returns (28) + : +- ReusedExchange (34) + +- * Project (69) + +- * Filter (68) + +- Window (67) + +- * Sort (66) + +- Window (65) + +- * Sort (64) + +- Exchange (63) + +- * HashAggregate (62) + +- Exchange (61) + +- * HashAggregate (60) + +- * Project (59) + +- * BroadcastHashJoin Inner BuildRight (58) + :- * ColumnarToRow (56) + : +- CometProject (55) + : +- CometBroadcastHashJoin (54) + : :- CometBroadcastExchange (50) + : : +- CometProject (49) + : : +- CometFilter (48) + : : +- CometScan parquet spark_catalog.default.store_sales (47) + : +- CometProject (53) + : +- CometFilter (52) + : +- CometScan parquet spark_catalog.default.store_returns (51) + +- ReusedExchange (57) (1) Scan parquet spark_catalog.default.web_sales @@ -86,109 +83,105 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#6), dynamicpruningexpression(ws_sol PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] Condition : (((((((isnotnull(ws_net_profit#5) AND isnotnull(ws_net_paid#4)) AND isnotnull(ws_quantity#3)) AND (ws_net_profit#5 > 1.00)) AND (ws_net_paid#4 > 0.00)) AND (ws_quantity#3 > 0)) AND isnotnull(ws_order_number#2)) AND isnotnull(ws_item_sk#1)) -(4) Project [codegen id : 1] -Output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] +(3) CometProject Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] +Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6], [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -(5) BroadcastExchange +(4) CometBroadcastExchange Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=1] +Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -(6) Scan parquet spark_catalog.default.web_returns +(5) Scan parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct -(7) ColumnarToRow -Input [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] - -(8) Filter +(6) CometFilter Input [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] Condition : (((isnotnull(wr_return_amt#11) AND (wr_return_amt#11 > 10000.00)) AND isnotnull(wr_order_number#9)) AND isnotnull(wr_item_sk#8)) -(9) Project -Output [4]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] +(7) CometProject Input [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] +Arguments: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11], [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] -(10) BroadcastHashJoin [codegen id : 3] -Left keys [2]: [ws_order_number#2, ws_item_sk#1] -Right keys [2]: [wr_order_number#9, wr_item_sk#8] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] +Right output [4]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] +Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#9, wr_item_sk#8], Inner -(11) Project [codegen id : 3] -Output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11] +(9) CometProject Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11] -(12) ReusedExchange [Reuses operator id: 82] +(10) ColumnarToRow [codegen id : 2] +Input [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11] + +(11) ReusedExchange [Reuses operator id: 79] Output [1]: [d_date_sk#13] -(13) BroadcastHashJoin [codegen id : 3] +(12) BroadcastHashJoin [codegen id : 2] Left keys [1]: [ws_sold_date_sk#6] Right keys [1]: [d_date_sk#13] Join type: Inner Join condition: None -(14) Project [codegen id : 3] +(13) Project [codegen id : 2] Output [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, wr_return_amt#11] Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11, d_date_sk#13] -(15) HashAggregate [codegen id : 3] +(14) HashAggregate [codegen id : 2] Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, wr_return_amt#11] Keys [1]: [ws_item_sk#1] Functions [4]: [partial_sum(coalesce(wr_return_quantity#10, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] Aggregate Attributes [6]: [sum#14, sum#15, sum#16, isEmpty#17, sum#18, isEmpty#19] Results [7]: [ws_item_sk#1, sum#20, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -(16) Exchange +(15) Exchange Input [7]: [ws_item_sk#1, sum#20, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(17) HashAggregate [codegen id : 4] +(16) HashAggregate [codegen id : 3] Input [7]: [ws_item_sk#1, sum#20, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] Keys [1]: [ws_item_sk#1] Functions [4]: [sum(coalesce(wr_return_quantity#10, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] Aggregate Attributes [4]: [sum(coalesce(wr_return_quantity#10, 0))#26, sum(coalesce(ws_quantity#3, 0))#27, sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00))#28, sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#29] Results [3]: [ws_item_sk#1 AS item#30, (cast(sum(coalesce(wr_return_quantity#10, 0))#26 as decimal(15,4)) / cast(sum(coalesce(ws_quantity#3, 0))#27 as decimal(15,4))) AS return_ratio#31, (cast(sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00))#28 as decimal(15,4)) / cast(sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#29 as decimal(15,4))) AS currency_ratio#32] -(18) Exchange +(17) Exchange Input [3]: [item#30, return_ratio#31, currency_ratio#32] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] -(19) Sort [codegen id : 5] +(18) Sort [codegen id : 4] Input [3]: [item#30, return_ratio#31, currency_ratio#32] Arguments: [return_ratio#31 ASC NULLS FIRST], false, 0 -(20) Window +(19) Window Input [3]: [item#30, return_ratio#31, currency_ratio#32] Arguments: [rank(return_ratio#31) windowspecdefinition(return_ratio#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#33], [return_ratio#31 ASC NULLS FIRST] -(21) Sort [codegen id : 6] +(20) Sort [codegen id : 5] Input [4]: [item#30, return_ratio#31, currency_ratio#32, return_rank#33] Arguments: [currency_ratio#32 ASC NULLS FIRST], false, 0 -(22) Window +(21) Window Input [4]: [item#30, return_ratio#31, currency_ratio#32, return_rank#33] Arguments: [rank(currency_ratio#32) windowspecdefinition(currency_ratio#32 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#34], [currency_ratio#32 ASC NULLS FIRST] -(23) Filter [codegen id : 7] +(22) Filter [codegen id : 6] Input [5]: [item#30, return_ratio#31, currency_ratio#32, return_rank#33, currency_rank#34] Condition : ((return_rank#33 <= 10) OR (currency_rank#34 <= 10)) -(24) Project [codegen id : 7] +(23) Project [codegen id : 6] Output [5]: [web AS channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34] Input [5]: [item#30, return_ratio#31, currency_ratio#32, return_rank#33, currency_rank#34] -(25) Scan parquet spark_catalog.default.catalog_sales +(24) Scan parquet spark_catalog.default.catalog_sales Output [6]: [cs_item_sk#36, cs_order_number#37, cs_quantity#38, cs_net_paid#39, cs_net_profit#40, cs_sold_date_sk#41] Batched: true Location: InMemoryFileIndex [] @@ -196,109 +189,105 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#41), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 8] -Input [6]: [cs_item_sk#36, cs_order_number#37, cs_quantity#38, cs_net_paid#39, cs_net_profit#40, cs_sold_date_sk#41] - -(27) Filter [codegen id : 8] +(25) CometFilter Input [6]: [cs_item_sk#36, cs_order_number#37, cs_quantity#38, cs_net_paid#39, cs_net_profit#40, cs_sold_date_sk#41] Condition : (((((((isnotnull(cs_net_profit#40) AND isnotnull(cs_net_paid#39)) AND isnotnull(cs_quantity#38)) AND (cs_net_profit#40 > 1.00)) AND (cs_net_paid#39 > 0.00)) AND (cs_quantity#38 > 0)) AND isnotnull(cs_order_number#37)) AND isnotnull(cs_item_sk#36)) -(28) Project [codegen id : 8] -Output [5]: [cs_item_sk#36, cs_order_number#37, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41] +(26) CometProject Input [6]: [cs_item_sk#36, cs_order_number#37, cs_quantity#38, cs_net_paid#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: [cs_item_sk#36, cs_order_number#37, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41], [cs_item_sk#36, cs_order_number#37, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41] -(29) BroadcastExchange +(27) CometBroadcastExchange Input [5]: [cs_item_sk#36, cs_order_number#37, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=4] +Arguments: [cs_item_sk#36, cs_order_number#37, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41] -(30) Scan parquet spark_catalog.default.catalog_returns +(28) Scan parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(31) ColumnarToRow -Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] - -(32) Filter +(29) CometFilter Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] Condition : (((isnotnull(cr_return_amount#46) AND (cr_return_amount#46 > 10000.00)) AND isnotnull(cr_order_number#44)) AND isnotnull(cr_item_sk#43)) -(33) Project -Output [4]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46] +(30) CometProject Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] +Arguments: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46], [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46] -(34) BroadcastHashJoin [codegen id : 10] -Left keys [2]: [cs_order_number#37, cs_item_sk#36] -Right keys [2]: [cr_order_number#44, cr_item_sk#43] -Join type: Inner -Join condition: None +(31) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#36, cs_order_number#37, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41] +Right output [4]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46] +Arguments: [cs_order_number#37, cs_item_sk#36], [cr_order_number#44, cr_item_sk#43], Inner -(35) Project [codegen id : 10] -Output [6]: [cs_item_sk#36, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41, cr_return_quantity#45, cr_return_amount#46] +(32) CometProject Input [9]: [cs_item_sk#36, cs_order_number#37, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41, cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46] +Arguments: [cs_item_sk#36, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41, cr_return_quantity#45, cr_return_amount#46], [cs_item_sk#36, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41, cr_return_quantity#45, cr_return_amount#46] + +(33) ColumnarToRow [codegen id : 8] +Input [6]: [cs_item_sk#36, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41, cr_return_quantity#45, cr_return_amount#46] -(36) ReusedExchange [Reuses operator id: 82] +(34) ReusedExchange [Reuses operator id: 79] Output [1]: [d_date_sk#48] -(37) BroadcastHashJoin [codegen id : 10] +(35) BroadcastHashJoin [codegen id : 8] Left keys [1]: [cs_sold_date_sk#41] Right keys [1]: [d_date_sk#48] Join type: Inner Join condition: None -(38) Project [codegen id : 10] +(36) Project [codegen id : 8] Output [5]: [cs_item_sk#36, cs_quantity#38, cs_net_paid#39, cr_return_quantity#45, cr_return_amount#46] Input [7]: [cs_item_sk#36, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41, cr_return_quantity#45, cr_return_amount#46, d_date_sk#48] -(39) HashAggregate [codegen id : 10] +(37) HashAggregate [codegen id : 8] Input [5]: [cs_item_sk#36, cs_quantity#38, cs_net_paid#39, cr_return_quantity#45, cr_return_amount#46] Keys [1]: [cs_item_sk#36] Functions [4]: [partial_sum(coalesce(cr_return_quantity#45, 0)), partial_sum(coalesce(cs_quantity#38, 0)), partial_sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#39 as decimal(12,2)), 0.00))] Aggregate Attributes [6]: [sum#49, sum#50, sum#51, isEmpty#52, sum#53, isEmpty#54] Results [7]: [cs_item_sk#36, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] -(40) Exchange +(38) Exchange Input [7]: [cs_item_sk#36, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] -Arguments: hashpartitioning(cs_item_sk#36, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(cs_item_sk#36, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(41) HashAggregate [codegen id : 11] +(39) HashAggregate [codegen id : 9] Input [7]: [cs_item_sk#36, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] Keys [1]: [cs_item_sk#36] Functions [4]: [sum(coalesce(cr_return_quantity#45, 0)), sum(coalesce(cs_quantity#38, 0)), sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#39 as decimal(12,2)), 0.00))] Aggregate Attributes [4]: [sum(coalesce(cr_return_quantity#45, 0))#61, sum(coalesce(cs_quantity#38, 0))#62, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#63, sum(coalesce(cast(cs_net_paid#39 as decimal(12,2)), 0.00))#64] Results [3]: [cs_item_sk#36 AS item#65, (cast(sum(coalesce(cr_return_quantity#45, 0))#61 as decimal(15,4)) / cast(sum(coalesce(cs_quantity#38, 0))#62 as decimal(15,4))) AS return_ratio#66, (cast(sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#63 as decimal(15,4)) / cast(sum(coalesce(cast(cs_net_paid#39 as decimal(12,2)), 0.00))#64 as decimal(15,4))) AS currency_ratio#67] -(42) Exchange +(40) Exchange Input [3]: [item#65, return_ratio#66, currency_ratio#67] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] -(43) Sort [codegen id : 12] +(41) Sort [codegen id : 10] Input [3]: [item#65, return_ratio#66, currency_ratio#67] Arguments: [return_ratio#66 ASC NULLS FIRST], false, 0 -(44) Window +(42) Window Input [3]: [item#65, return_ratio#66, currency_ratio#67] Arguments: [rank(return_ratio#66) windowspecdefinition(return_ratio#66 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#68], [return_ratio#66 ASC NULLS FIRST] -(45) Sort [codegen id : 13] +(43) Sort [codegen id : 11] Input [4]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68] Arguments: [currency_ratio#67 ASC NULLS FIRST], false, 0 -(46) Window +(44) Window Input [4]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68] Arguments: [rank(currency_ratio#67) windowspecdefinition(currency_ratio#67 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#69], [currency_ratio#67 ASC NULLS FIRST] -(47) Filter [codegen id : 14] +(45) Filter [codegen id : 12] Input [5]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68, currency_rank#69] Condition : ((return_rank#68 <= 10) OR (currency_rank#69 <= 10)) -(48) Project [codegen id : 14] +(46) Project [codegen id : 12] Output [5]: [catalog AS channel#70, item#65, return_ratio#66, return_rank#68, currency_rank#69] Input [5]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68, currency_rank#69] -(49) Scan parquet spark_catalog.default.store_sales +(47) Scan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] Batched: true Location: InMemoryFileIndex [] @@ -306,166 +295,162 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#76), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)] ReadSchema: struct -(50) ColumnarToRow [codegen id : 15] -Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] - -(51) Filter [codegen id : 15] +(48) CometFilter Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] Condition : (((((((isnotnull(ss_net_profit#75) AND isnotnull(ss_net_paid#74)) AND isnotnull(ss_quantity#73)) AND (ss_net_profit#75 > 1.00)) AND (ss_net_paid#74 > 0.00)) AND (ss_quantity#73 > 0)) AND isnotnull(ss_ticket_number#72)) AND isnotnull(ss_item_sk#71)) -(52) Project [codegen id : 15] -Output [5]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76] +(49) CometProject Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] +Arguments: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76], [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76] -(53) BroadcastExchange +(50) CometBroadcastExchange Input [5]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=7] +Arguments: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76] -(54) Scan parquet spark_catalog.default.store_returns +(51) Scan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#78, sr_ticket_number#79, sr_return_quantity#80, sr_return_amt#81, sr_returned_date_sk#82] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(55) ColumnarToRow -Input [5]: [sr_item_sk#78, sr_ticket_number#79, sr_return_quantity#80, sr_return_amt#81, sr_returned_date_sk#82] - -(56) Filter +(52) CometFilter Input [5]: [sr_item_sk#78, sr_ticket_number#79, sr_return_quantity#80, sr_return_amt#81, sr_returned_date_sk#82] Condition : (((isnotnull(sr_return_amt#81) AND (sr_return_amt#81 > 10000.00)) AND isnotnull(sr_ticket_number#79)) AND isnotnull(sr_item_sk#78)) -(57) Project -Output [4]: [sr_item_sk#78, sr_ticket_number#79, sr_return_quantity#80, sr_return_amt#81] +(53) CometProject Input [5]: [sr_item_sk#78, sr_ticket_number#79, sr_return_quantity#80, sr_return_amt#81, sr_returned_date_sk#82] +Arguments: [sr_item_sk#78, sr_ticket_number#79, sr_return_quantity#80, sr_return_amt#81], [sr_item_sk#78, sr_ticket_number#79, sr_return_quantity#80, sr_return_amt#81] -(58) BroadcastHashJoin [codegen id : 17] -Left keys [2]: [ss_ticket_number#72, ss_item_sk#71] -Right keys [2]: [sr_ticket_number#79, sr_item_sk#78] -Join type: Inner -Join condition: None +(54) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76] +Right output [4]: [sr_item_sk#78, sr_ticket_number#79, sr_return_quantity#80, sr_return_amt#81] +Arguments: [ss_ticket_number#72, ss_item_sk#71], [sr_ticket_number#79, sr_item_sk#78], Inner -(59) Project [codegen id : 17] -Output [6]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_return_quantity#80, sr_return_amt#81] +(55) CometProject Input [9]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_item_sk#78, sr_ticket_number#79, sr_return_quantity#80, sr_return_amt#81] +Arguments: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_return_quantity#80, sr_return_amt#81], [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_return_quantity#80, sr_return_amt#81] + +(56) ColumnarToRow [codegen id : 14] +Input [6]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_return_quantity#80, sr_return_amt#81] -(60) ReusedExchange [Reuses operator id: 82] +(57) ReusedExchange [Reuses operator id: 79] Output [1]: [d_date_sk#83] -(61) BroadcastHashJoin [codegen id : 17] +(58) BroadcastHashJoin [codegen id : 14] Left keys [1]: [ss_sold_date_sk#76] Right keys [1]: [d_date_sk#83] Join type: Inner Join condition: None -(62) Project [codegen id : 17] +(59) Project [codegen id : 14] Output [5]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, sr_return_quantity#80, sr_return_amt#81] Input [7]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_return_quantity#80, sr_return_amt#81, d_date_sk#83] -(63) HashAggregate [codegen id : 17] +(60) HashAggregate [codegen id : 14] Input [5]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, sr_return_quantity#80, sr_return_amt#81] Keys [1]: [ss_item_sk#71] Functions [4]: [partial_sum(coalesce(sr_return_quantity#80, 0)), partial_sum(coalesce(ss_quantity#73, 0)), partial_sum(coalesce(cast(sr_return_amt#81 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))] Aggregate Attributes [6]: [sum#84, sum#85, sum#86, isEmpty#87, sum#88, isEmpty#89] Results [7]: [ss_item_sk#71, sum#90, sum#91, sum#92, isEmpty#93, sum#94, isEmpty#95] -(64) Exchange +(61) Exchange Input [7]: [ss_item_sk#71, sum#90, sum#91, sum#92, isEmpty#93, sum#94, isEmpty#95] -Arguments: hashpartitioning(ss_item_sk#71, 5), ENSURE_REQUIREMENTS, [plan_id=8] +Arguments: hashpartitioning(ss_item_sk#71, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(65) HashAggregate [codegen id : 18] +(62) HashAggregate [codegen id : 15] Input [7]: [ss_item_sk#71, sum#90, sum#91, sum#92, isEmpty#93, sum#94, isEmpty#95] Keys [1]: [ss_item_sk#71] Functions [4]: [sum(coalesce(sr_return_quantity#80, 0)), sum(coalesce(ss_quantity#73, 0)), sum(coalesce(cast(sr_return_amt#81 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))] Aggregate Attributes [4]: [sum(coalesce(sr_return_quantity#80, 0))#96, sum(coalesce(ss_quantity#73, 0))#97, sum(coalesce(cast(sr_return_amt#81 as decimal(12,2)), 0.00))#98, sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))#99] Results [3]: [ss_item_sk#71 AS item#100, (cast(sum(coalesce(sr_return_quantity#80, 0))#96 as decimal(15,4)) / cast(sum(coalesce(ss_quantity#73, 0))#97 as decimal(15,4))) AS return_ratio#101, (cast(sum(coalesce(cast(sr_return_amt#81 as decimal(12,2)), 0.00))#98 as decimal(15,4)) / cast(sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))#99 as decimal(15,4))) AS currency_ratio#102] -(66) Exchange +(63) Exchange Input [3]: [item#100, return_ratio#101, currency_ratio#102] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=9] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] -(67) Sort [codegen id : 19] +(64) Sort [codegen id : 16] Input [3]: [item#100, return_ratio#101, currency_ratio#102] Arguments: [return_ratio#101 ASC NULLS FIRST], false, 0 -(68) Window +(65) Window Input [3]: [item#100, return_ratio#101, currency_ratio#102] Arguments: [rank(return_ratio#101) windowspecdefinition(return_ratio#101 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#103], [return_ratio#101 ASC NULLS FIRST] -(69) Sort [codegen id : 20] +(66) Sort [codegen id : 17] Input [4]: [item#100, return_ratio#101, currency_ratio#102, return_rank#103] Arguments: [currency_ratio#102 ASC NULLS FIRST], false, 0 -(70) Window +(67) Window Input [4]: [item#100, return_ratio#101, currency_ratio#102, return_rank#103] Arguments: [rank(currency_ratio#102) windowspecdefinition(currency_ratio#102 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#104], [currency_ratio#102 ASC NULLS FIRST] -(71) Filter [codegen id : 21] +(68) Filter [codegen id : 18] Input [5]: [item#100, return_ratio#101, currency_ratio#102, return_rank#103, currency_rank#104] Condition : ((return_rank#103 <= 10) OR (currency_rank#104 <= 10)) -(72) Project [codegen id : 21] +(69) Project [codegen id : 18] Output [5]: [store AS channel#105, item#100, return_ratio#101, return_rank#103, currency_rank#104] Input [5]: [item#100, return_ratio#101, currency_ratio#102, return_rank#103, currency_rank#104] -(73) Union +(70) Union -(74) HashAggregate [codegen id : 22] +(71) HashAggregate [codegen id : 19] Input [5]: [channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34] Keys [5]: [channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34] Functions: [] Aggregate Attributes: [] Results [5]: [channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34] -(75) Exchange +(72) Exchange Input [5]: [channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34] -Arguments: hashpartitioning(channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34, 5), ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: hashpartitioning(channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(76) HashAggregate [codegen id : 23] +(73) HashAggregate [codegen id : 20] Input [5]: [channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34] Keys [5]: [channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34] Functions: [] Aggregate Attributes: [] Results [5]: [channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34] -(77) TakeOrderedAndProject +(74) TakeOrderedAndProject Input [5]: [channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34] Arguments: 100, [channel#35 ASC NULLS FIRST, return_rank#33 ASC NULLS FIRST, currency_rank#34 ASC NULLS FIRST], [channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (82) -+- * Project (81) - +- * Filter (80) - +- * ColumnarToRow (79) - +- CometScan parquet spark_catalog.default.date_dim (78) +BroadcastExchange (79) ++- * ColumnarToRow (78) + +- CometProject (77) + +- CometFilter (76) + +- CometScan parquet spark_catalog.default.date_dim (75) -(78) Scan parquet spark_catalog.default.date_dim +(75) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#106, d_moy#107] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] ReadSchema: struct -(79) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#13, d_year#106, d_moy#107] - -(80) Filter [codegen id : 1] +(76) CometFilter Input [3]: [d_date_sk#13, d_year#106, d_moy#107] Condition : ((((isnotnull(d_year#106) AND isnotnull(d_moy#107)) AND (d_year#106 = 2001)) AND (d_moy#107 = 12)) AND isnotnull(d_date_sk#13)) -(81) Project [codegen id : 1] -Output [1]: [d_date_sk#13] +(77) CometProject Input [3]: [d_date_sk#13, d_year#106, d_moy#107] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(78) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#13] -(82) BroadcastExchange +(79) BroadcastExchange Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] -Subquery:2 Hosting operator id = 25 Hosting Expression = cs_sold_date_sk#41 IN dynamicpruning#7 +Subquery:2 Hosting operator id = 24 Hosting Expression = cs_sold_date_sk#41 IN dynamicpruning#7 -Subquery:3 Hosting operator id = 49 Hosting Expression = ss_sold_date_sk#76 IN dynamicpruning#7 +Subquery:3 Hosting operator id = 47 Hosting Expression = ss_sold_date_sk#76 IN dynamicpruning#7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/simplified.txt index b707750da..43ebf34cc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/simplified.txt @@ -1,133 +1,121 @@ TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio] - WholeStageCodegen (23) + WholeStageCodegen (20) HashAggregate [channel,item,return_ratio,return_rank,currency_rank] InputAdapter Exchange [channel,item,return_ratio,return_rank,currency_rank] #1 - WholeStageCodegen (22) + WholeStageCodegen (19) HashAggregate [channel,item,return_ratio,return_rank,currency_rank] InputAdapter Union - WholeStageCodegen (7) + WholeStageCodegen (6) Project [item,return_ratio,return_rank,currency_rank] Filter [return_rank,currency_rank] InputAdapter Window [currency_ratio] - WholeStageCodegen (6) + WholeStageCodegen (5) Sort [currency_ratio] InputAdapter Window [return_ratio] - WholeStageCodegen (5) + WholeStageCodegen (4) Sort [return_ratio] InputAdapter Exchange #2 - WholeStageCodegen (4) + WholeStageCodegen (3) HashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] InputAdapter Exchange [ws_item_sk] #3 - WholeStageCodegen (3) + WholeStageCodegen (2) HashAggregate [ws_item_sk,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] Project [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] - BroadcastHashJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Project [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] - Filter [ws_net_profit,ws_net_paid,ws_quantity,ws_order_number,ws_item_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - Filter [wr_return_amt,wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + CometBroadcastExchange #4 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] + CometFilter [ws_net_profit,ws_net_paid,ws_quantity,ws_order_number,ws_item_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_moy,d_date_sk] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_return_amt,wr_order_number,wr_item_sk] CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] InputAdapter ReusedExchange [d_date_sk] #5 - WholeStageCodegen (14) + WholeStageCodegen (12) Project [item,return_ratio,return_rank,currency_rank] Filter [return_rank,currency_rank] InputAdapter Window [currency_ratio] - WholeStageCodegen (13) + WholeStageCodegen (11) Sort [currency_ratio] InputAdapter Window [return_ratio] - WholeStageCodegen (12) + WholeStageCodegen (10) Sort [return_ratio] InputAdapter Exchange #6 - WholeStageCodegen (11) + WholeStageCodegen (9) HashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] InputAdapter Exchange [cs_item_sk] #7 - WholeStageCodegen (10) + WholeStageCodegen (8) HashAggregate [cs_item_sk,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] Project [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] - BroadcastHashJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (8) - Project [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] - Filter [cs_net_profit,cs_net_paid,cs_quantity,cs_order_number,cs_item_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - Filter [cr_return_amount,cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + CometBroadcastExchange #8 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] + CometFilter [cs_net_profit,cs_net_paid,cs_quantity,cs_order_number,cs_item_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_return_amount,cr_order_number,cr_item_sk] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] InputAdapter ReusedExchange [d_date_sk] #5 - WholeStageCodegen (21) + WholeStageCodegen (18) Project [item,return_ratio,return_rank,currency_rank] Filter [return_rank,currency_rank] InputAdapter Window [currency_ratio] - WholeStageCodegen (20) + WholeStageCodegen (17) Sort [currency_ratio] InputAdapter Window [return_ratio] - WholeStageCodegen (19) + WholeStageCodegen (16) Sort [return_ratio] InputAdapter Exchange #9 - WholeStageCodegen (18) + WholeStageCodegen (15) HashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] InputAdapter Exchange [ss_item_sk] #10 - WholeStageCodegen (17) + WholeStageCodegen (14) HashAggregate [ss_item_sk,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] Project [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] - BroadcastHashJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (15) - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] - Filter [ss_net_profit,ss_net_paid,ss_quantity,ss_ticket_number,ss_item_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - Filter [sr_return_amt,sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + CometBroadcastExchange #11 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] + CometFilter [ss_net_profit,ss_net_paid,ss_quantity,ss_ticket_number,ss_item_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_return_amt,sr_ticket_number,sr_item_sk] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] InputAdapter ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/explain.txt index b62cad834..e3f7538d1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/explain.txt @@ -1,78 +1,74 @@ == Physical Plan == -TakeOrderedAndProject (74) -+- * HashAggregate (73) - +- Exchange (72) - +- * HashAggregate (71) - +- * Expand (70) - +- Union (69) - :- * HashAggregate (21) - : +- Exchange (20) - : +- * HashAggregate (19) - : +- * Project (18) - : +- * BroadcastHashJoin Inner BuildRight (17) - : :- * Project (12) - : : +- * BroadcastHashJoin Inner BuildRight (11) - : : :- Union (9) - : : : :- * Project (4) - : : : : +- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- * Project (8) - : : : +- * Filter (7) - : : : +- * ColumnarToRow (6) - : : : +- CometScan parquet spark_catalog.default.store_returns (5) - : : +- ReusedExchange (10) - : +- BroadcastExchange (16) - : +- * Filter (15) - : +- * ColumnarToRow (14) - : +- CometScan parquet spark_catalog.default.store (13) - :- * HashAggregate (42) - : +- Exchange (41) - : +- * HashAggregate (40) - : +- * Project (39) - : +- * BroadcastHashJoin Inner BuildRight (38) - : :- * Project (33) - : : +- * BroadcastHashJoin Inner BuildRight (32) - : : :- Union (30) - : : : :- * Project (25) - : : : : +- * Filter (24) - : : : : +- * ColumnarToRow (23) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (22) - : : : +- * Project (29) - : : : +- * Filter (28) - : : : +- * ColumnarToRow (27) - : : : +- CometScan parquet spark_catalog.default.catalog_returns (26) - : : +- ReusedExchange (31) - : +- BroadcastExchange (37) - : +- * Filter (36) - : +- * ColumnarToRow (35) - : +- CometScan parquet spark_catalog.default.catalog_page (34) - +- * HashAggregate (68) - +- Exchange (67) - +- * HashAggregate (66) - +- * Project (65) - +- * BroadcastHashJoin Inner BuildRight (64) - :- * Project (59) - : +- * BroadcastHashJoin Inner BuildRight (58) - : :- Union (56) - : : :- * Project (46) - : : : +- * Filter (45) - : : : +- * ColumnarToRow (44) - : : : +- CometScan parquet spark_catalog.default.web_sales (43) - : : +- * Project (55) - : : +- * BroadcastHashJoin Inner BuildLeft (54) - : : :- BroadcastExchange (49) - : : : +- * ColumnarToRow (48) - : : : +- CometScan parquet spark_catalog.default.web_returns (47) - : : +- * Project (53) - : : +- * Filter (52) - : : +- * ColumnarToRow (51) - : : +- CometScan parquet spark_catalog.default.web_sales (50) - : +- ReusedExchange (57) - +- BroadcastExchange (63) - +- * Filter (62) - +- * ColumnarToRow (61) - +- CometScan parquet spark_catalog.default.web_site (60) +TakeOrderedAndProject (70) ++- * HashAggregate (69) + +- Exchange (68) + +- * HashAggregate (67) + +- * Expand (66) + +- Union (65) + :- * HashAggregate (20) + : +- Exchange (19) + : +- * HashAggregate (18) + : +- * Project (17) + : +- * BroadcastHashJoin Inner BuildRight (16) + : :- * Project (11) + : : +- * BroadcastHashJoin Inner BuildRight (10) + : : :- * ColumnarToRow (8) + : : : +- CometUnion (7) + : : : :- CometProject (3) + : : : : +- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometProject (6) + : : : +- CometFilter (5) + : : : +- CometScan parquet spark_catalog.default.store_returns (4) + : : +- ReusedExchange (9) + : +- BroadcastExchange (15) + : +- * ColumnarToRow (14) + : +- CometFilter (13) + : +- CometScan parquet spark_catalog.default.store (12) + :- * HashAggregate (40) + : +- Exchange (39) + : +- * HashAggregate (38) + : +- * Project (37) + : +- * BroadcastHashJoin Inner BuildRight (36) + : :- * Project (31) + : : +- * BroadcastHashJoin Inner BuildRight (30) + : : :- * ColumnarToRow (28) + : : : +- CometUnion (27) + : : : :- CometProject (23) + : : : : +- CometFilter (22) + : : : : +- CometScan parquet spark_catalog.default.catalog_sales (21) + : : : +- CometProject (26) + : : : +- CometFilter (25) + : : : +- CometScan parquet spark_catalog.default.catalog_returns (24) + : : +- ReusedExchange (29) + : +- BroadcastExchange (35) + : +- * ColumnarToRow (34) + : +- CometFilter (33) + : +- CometScan parquet spark_catalog.default.catalog_page (32) + +- * HashAggregate (64) + +- Exchange (63) + +- * HashAggregate (62) + +- * Project (61) + +- * BroadcastHashJoin Inner BuildRight (60) + :- * Project (55) + : +- * BroadcastHashJoin Inner BuildRight (54) + : :- * ColumnarToRow (52) + : : +- CometUnion (51) + : : :- CometProject (43) + : : : +- CometFilter (42) + : : : +- CometScan parquet spark_catalog.default.web_sales (41) + : : +- CometProject (50) + : : +- CometBroadcastHashJoin (49) + : : :- CometBroadcastExchange (45) + : : : +- CometScan parquet spark_catalog.default.web_returns (44) + : : +- CometProject (48) + : : +- CometFilter (47) + : : +- CometScan parquet spark_catalog.default.web_sales (46) + : +- ReusedExchange (53) + +- BroadcastExchange (59) + +- * ColumnarToRow (58) + +- CometFilter (57) + +- CometScan parquet spark_catalog.default.web_site (56) (1) Scan parquet spark_catalog.default.store_sales @@ -83,18 +79,15 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Condition : isnotnull(ss_store_sk#1) -(4) Project [codegen id : 1] -Output [6]: [ss_store_sk#1 AS store_sk#6, ss_sold_date_sk#4 AS date_sk#7, ss_ext_sales_price#2 AS sales_price#8, ss_net_profit#3 AS profit#9, 0.00 AS return_amt#10, 0.00 AS net_loss#11] +(3) CometProject Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Arguments: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11], [ss_store_sk#1 AS store_sk#6, ss_sold_date_sk#4 AS date_sk#7, ss_ext_sales_price#2 AS sales_price#8, ss_net_profit#3 AS profit#9, 0.00 AS return_amt#10, 0.00 AS net_loss#11] -(5) Scan parquet spark_catalog.default.store_returns +(4) Scan parquet spark_catalog.default.store_returns Output [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] Batched: true Location: InMemoryFileIndex [] @@ -102,79 +95,81 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#15), dynamicpruningexpression(s PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 2] -Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] - -(7) Filter [codegen id : 2] +(5) CometFilter Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] Condition : isnotnull(sr_store_sk#12) -(8) Project [codegen id : 2] -Output [6]: [sr_store_sk#12 AS store_sk#16, sr_returned_date_sk#15 AS date_sk#17, 0.00 AS sales_price#18, 0.00 AS profit#19, sr_return_amt#13 AS return_amt#20, sr_net_loss#14 AS net_loss#21] +(6) CometProject Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] +Arguments: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21], [sr_store_sk#12 AS store_sk#16, sr_returned_date_sk#15 AS date_sk#17, 0.00 AS sales_price#18, 0.00 AS profit#19, sr_return_amt#13 AS return_amt#20, sr_net_loss#14 AS net_loss#21] -(9) Union +(7) CometUnion +Child 0 Input [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] +Child 1 Input [6]: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21] -(10) ReusedExchange [Reuses operator id: 79] +(8) ColumnarToRow [codegen id : 3] +Input [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] + +(9) ReusedExchange [Reuses operator id: 75] Output [1]: [d_date_sk#22] -(11) BroadcastHashJoin [codegen id : 5] +(10) BroadcastHashJoin [codegen id : 3] Left keys [1]: [date_sk#7] Right keys [1]: [d_date_sk#22] Join type: Inner Join condition: None -(12) Project [codegen id : 5] +(11) Project [codegen id : 3] Output [5]: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] Input [7]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, d_date_sk#22] -(13) Scan parquet spark_catalog.default.store +(12) Scan parquet spark_catalog.default.store Output [2]: [s_store_sk#23, s_store_id#24] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 4] +(13) CometFilter Input [2]: [s_store_sk#23, s_store_id#24] +Condition : isnotnull(s_store_sk#23) -(15) Filter [codegen id : 4] +(14) ColumnarToRow [codegen id : 2] Input [2]: [s_store_sk#23, s_store_id#24] -Condition : isnotnull(s_store_sk#23) -(16) BroadcastExchange +(15) BroadcastExchange Input [2]: [s_store_sk#23, s_store_id#24] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] -(17) BroadcastHashJoin [codegen id : 5] +(16) BroadcastHashJoin [codegen id : 3] Left keys [1]: [store_sk#6] Right keys [1]: [s_store_sk#23] Join type: Inner Join condition: None -(18) Project [codegen id : 5] +(17) Project [codegen id : 3] Output [5]: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#24] Input [7]: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_sk#23, s_store_id#24] -(19) HashAggregate [codegen id : 5] +(18) HashAggregate [codegen id : 3] Input [5]: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#24] Keys [1]: [s_store_id#24] Functions [4]: [partial_sum(UnscaledValue(sales_price#8)), partial_sum(UnscaledValue(return_amt#10)), partial_sum(UnscaledValue(profit#9)), partial_sum(UnscaledValue(net_loss#11))] Aggregate Attributes [4]: [sum#25, sum#26, sum#27, sum#28] Results [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] -(20) Exchange +(19) Exchange Input [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] Arguments: hashpartitioning(s_store_id#24, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(21) HashAggregate [codegen id : 6] +(20) HashAggregate [codegen id : 4] Input [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] Keys [1]: [s_store_id#24] Functions [4]: [sum(UnscaledValue(sales_price#8)), sum(UnscaledValue(return_amt#10)), sum(UnscaledValue(profit#9)), sum(UnscaledValue(net_loss#11))] Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#8))#33, sum(UnscaledValue(return_amt#10))#34, sum(UnscaledValue(profit#9))#35, sum(UnscaledValue(net_loss#11))#36] Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#8))#33,17,2) AS sales#37, MakeDecimal(sum(UnscaledValue(return_amt#10))#34,17,2) AS returns#38, (MakeDecimal(sum(UnscaledValue(profit#9))#35,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#11))#36,17,2)) AS profit#39, store channel AS channel#40, concat(store, s_store_id#24) AS id#41] -(22) Scan parquet spark_catalog.default.catalog_sales +(21) Scan parquet spark_catalog.default.catalog_sales Output [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] Batched: true Location: InMemoryFileIndex [] @@ -182,18 +177,15 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#45), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_catalog_page_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 7] -Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] - -(24) Filter [codegen id : 7] +(22) CometFilter Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] Condition : isnotnull(cs_catalog_page_sk#42) -(25) Project [codegen id : 7] -Output [6]: [cs_catalog_page_sk#42 AS page_sk#47, cs_sold_date_sk#45 AS date_sk#48, cs_ext_sales_price#43 AS sales_price#49, cs_net_profit#44 AS profit#50, 0.00 AS return_amt#51, 0.00 AS net_loss#52] +(23) CometProject Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +Arguments: [page_sk#47, date_sk#48, sales_price#49, profit#50, return_amt#51, net_loss#52], [cs_catalog_page_sk#42 AS page_sk#47, cs_sold_date_sk#45 AS date_sk#48, cs_ext_sales_price#43 AS sales_price#49, cs_net_profit#44 AS profit#50, 0.00 AS return_amt#51, 0.00 AS net_loss#52] -(26) Scan parquet spark_catalog.default.catalog_returns +(24) Scan parquet spark_catalog.default.catalog_returns Output [4]: [cr_catalog_page_sk#53, cr_return_amount#54, cr_net_loss#55, cr_returned_date_sk#56] Batched: true Location: InMemoryFileIndex [] @@ -201,79 +193,81 @@ PartitionFilters: [isnotnull(cr_returned_date_sk#56), dynamicpruningexpression(c PushedFilters: [IsNotNull(cr_catalog_page_sk)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 8] -Input [4]: [cr_catalog_page_sk#53, cr_return_amount#54, cr_net_loss#55, cr_returned_date_sk#56] - -(28) Filter [codegen id : 8] +(25) CometFilter Input [4]: [cr_catalog_page_sk#53, cr_return_amount#54, cr_net_loss#55, cr_returned_date_sk#56] Condition : isnotnull(cr_catalog_page_sk#53) -(29) Project [codegen id : 8] -Output [6]: [cr_catalog_page_sk#53 AS page_sk#57, cr_returned_date_sk#56 AS date_sk#58, 0.00 AS sales_price#59, 0.00 AS profit#60, cr_return_amount#54 AS return_amt#61, cr_net_loss#55 AS net_loss#62] +(26) CometProject Input [4]: [cr_catalog_page_sk#53, cr_return_amount#54, cr_net_loss#55, cr_returned_date_sk#56] +Arguments: [page_sk#57, date_sk#58, sales_price#59, profit#60, return_amt#61, net_loss#62], [cr_catalog_page_sk#53 AS page_sk#57, cr_returned_date_sk#56 AS date_sk#58, 0.00 AS sales_price#59, 0.00 AS profit#60, cr_return_amount#54 AS return_amt#61, cr_net_loss#55 AS net_loss#62] + +(27) CometUnion +Child 0 Input [6]: [page_sk#47, date_sk#48, sales_price#49, profit#50, return_amt#51, net_loss#52] +Child 1 Input [6]: [page_sk#57, date_sk#58, sales_price#59, profit#60, return_amt#61, net_loss#62] -(30) Union +(28) ColumnarToRow [codegen id : 7] +Input [6]: [page_sk#47, date_sk#48, sales_price#49, profit#50, return_amt#51, net_loss#52] -(31) ReusedExchange [Reuses operator id: 79] +(29) ReusedExchange [Reuses operator id: 75] Output [1]: [d_date_sk#63] -(32) BroadcastHashJoin [codegen id : 11] +(30) BroadcastHashJoin [codegen id : 7] Left keys [1]: [date_sk#48] Right keys [1]: [d_date_sk#63] Join type: Inner Join condition: None -(33) Project [codegen id : 11] +(31) Project [codegen id : 7] Output [5]: [page_sk#47, sales_price#49, profit#50, return_amt#51, net_loss#52] Input [7]: [page_sk#47, date_sk#48, sales_price#49, profit#50, return_amt#51, net_loss#52, d_date_sk#63] -(34) Scan parquet spark_catalog.default.catalog_page +(32) Scan parquet spark_catalog.default.catalog_page Output [2]: [cp_catalog_page_sk#64, cp_catalog_page_id#65] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 10] +(33) CometFilter Input [2]: [cp_catalog_page_sk#64, cp_catalog_page_id#65] +Condition : isnotnull(cp_catalog_page_sk#64) -(36) Filter [codegen id : 10] +(34) ColumnarToRow [codegen id : 6] Input [2]: [cp_catalog_page_sk#64, cp_catalog_page_id#65] -Condition : isnotnull(cp_catalog_page_sk#64) -(37) BroadcastExchange +(35) BroadcastExchange Input [2]: [cp_catalog_page_sk#64, cp_catalog_page_id#65] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] -(38) BroadcastHashJoin [codegen id : 11] +(36) BroadcastHashJoin [codegen id : 7] Left keys [1]: [page_sk#47] Right keys [1]: [cp_catalog_page_sk#64] Join type: Inner Join condition: None -(39) Project [codegen id : 11] +(37) Project [codegen id : 7] Output [5]: [sales_price#49, profit#50, return_amt#51, net_loss#52, cp_catalog_page_id#65] Input [7]: [page_sk#47, sales_price#49, profit#50, return_amt#51, net_loss#52, cp_catalog_page_sk#64, cp_catalog_page_id#65] -(40) HashAggregate [codegen id : 11] +(38) HashAggregate [codegen id : 7] Input [5]: [sales_price#49, profit#50, return_amt#51, net_loss#52, cp_catalog_page_id#65] Keys [1]: [cp_catalog_page_id#65] Functions [4]: [partial_sum(UnscaledValue(sales_price#49)), partial_sum(UnscaledValue(return_amt#51)), partial_sum(UnscaledValue(profit#50)), partial_sum(UnscaledValue(net_loss#52))] Aggregate Attributes [4]: [sum#66, sum#67, sum#68, sum#69] Results [5]: [cp_catalog_page_id#65, sum#70, sum#71, sum#72, sum#73] -(41) Exchange +(39) Exchange Input [5]: [cp_catalog_page_id#65, sum#70, sum#71, sum#72, sum#73] Arguments: hashpartitioning(cp_catalog_page_id#65, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(42) HashAggregate [codegen id : 12] +(40) HashAggregate [codegen id : 8] Input [5]: [cp_catalog_page_id#65, sum#70, sum#71, sum#72, sum#73] Keys [1]: [cp_catalog_page_id#65] Functions [4]: [sum(UnscaledValue(sales_price#49)), sum(UnscaledValue(return_amt#51)), sum(UnscaledValue(profit#50)), sum(UnscaledValue(net_loss#52))] Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#49))#74, sum(UnscaledValue(return_amt#51))#75, sum(UnscaledValue(profit#50))#76, sum(UnscaledValue(net_loss#52))#77] Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#49))#74,17,2) AS sales#78, MakeDecimal(sum(UnscaledValue(return_amt#51))#75,17,2) AS returns#79, (MakeDecimal(sum(UnscaledValue(profit#50))#76,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#52))#77,17,2)) AS profit#80, catalog channel AS channel#81, concat(catalog_page, cp_catalog_page_id#65) AS id#82] -(43) Scan parquet spark_catalog.default.web_sales +(41) Scan parquet spark_catalog.default.web_sales Output [4]: [ws_web_site_sk#83, ws_ext_sales_price#84, ws_net_profit#85, ws_sold_date_sk#86] Batched: true Location: InMemoryFileIndex [] @@ -281,188 +275,183 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#86), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_web_site_sk)] ReadSchema: struct -(44) ColumnarToRow [codegen id : 13] -Input [4]: [ws_web_site_sk#83, ws_ext_sales_price#84, ws_net_profit#85, ws_sold_date_sk#86] - -(45) Filter [codegen id : 13] +(42) CometFilter Input [4]: [ws_web_site_sk#83, ws_ext_sales_price#84, ws_net_profit#85, ws_sold_date_sk#86] Condition : isnotnull(ws_web_site_sk#83) -(46) Project [codegen id : 13] -Output [6]: [ws_web_site_sk#83 AS wsr_web_site_sk#88, ws_sold_date_sk#86 AS date_sk#89, ws_ext_sales_price#84 AS sales_price#90, ws_net_profit#85 AS profit#91, 0.00 AS return_amt#92, 0.00 AS net_loss#93] +(43) CometProject Input [4]: [ws_web_site_sk#83, ws_ext_sales_price#84, ws_net_profit#85, ws_sold_date_sk#86] +Arguments: [wsr_web_site_sk#88, date_sk#89, sales_price#90, profit#91, return_amt#92, net_loss#93], [ws_web_site_sk#83 AS wsr_web_site_sk#88, ws_sold_date_sk#86 AS date_sk#89, ws_ext_sales_price#84 AS sales_price#90, ws_net_profit#85 AS profit#91, 0.00 AS return_amt#92, 0.00 AS net_loss#93] -(47) Scan parquet spark_catalog.default.web_returns +(44) Scan parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(wr_returned_date_sk#98), dynamicpruningexpression(wr_returned_date_sk#98 IN dynamicpruning#87)] ReadSchema: struct -(48) ColumnarToRow [codegen id : 14] -Input [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98] - -(49) BroadcastExchange +(45) CometBroadcastExchange Input [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, true] as bigint), 32) | (cast(input[1, int, true] as bigint) & 4294967295))),false), [plan_id=5] +Arguments: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98] -(50) Scan parquet spark_catalog.default.web_sales +(46) Scan parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#99, ws_web_site_sk#100, ws_order_number#101, ws_sold_date_sk#102] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] ReadSchema: struct -(51) ColumnarToRow -Input [4]: [ws_item_sk#99, ws_web_site_sk#100, ws_order_number#101, ws_sold_date_sk#102] - -(52) Filter +(47) CometFilter Input [4]: [ws_item_sk#99, ws_web_site_sk#100, ws_order_number#101, ws_sold_date_sk#102] Condition : ((isnotnull(ws_item_sk#99) AND isnotnull(ws_order_number#101)) AND isnotnull(ws_web_site_sk#100)) -(53) Project -Output [3]: [ws_item_sk#99, ws_web_site_sk#100, ws_order_number#101] +(48) CometProject Input [4]: [ws_item_sk#99, ws_web_site_sk#100, ws_order_number#101, ws_sold_date_sk#102] +Arguments: [ws_item_sk#99, ws_web_site_sk#100, ws_order_number#101], [ws_item_sk#99, ws_web_site_sk#100, ws_order_number#101] -(54) BroadcastHashJoin [codegen id : 15] -Left keys [2]: [wr_item_sk#94, wr_order_number#95] -Right keys [2]: [ws_item_sk#99, ws_order_number#101] -Join type: Inner -Join condition: None +(49) CometBroadcastHashJoin +Left output [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98] +Right output [3]: [ws_item_sk#99, ws_web_site_sk#100, ws_order_number#101] +Arguments: [wr_item_sk#94, wr_order_number#95], [ws_item_sk#99, ws_order_number#101], Inner -(55) Project [codegen id : 15] -Output [6]: [ws_web_site_sk#100 AS wsr_web_site_sk#103, wr_returned_date_sk#98 AS date_sk#104, 0.00 AS sales_price#105, 0.00 AS profit#106, wr_return_amt#96 AS return_amt#107, wr_net_loss#97 AS net_loss#108] +(50) CometProject Input [8]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98, ws_item_sk#99, ws_web_site_sk#100, ws_order_number#101] +Arguments: [wsr_web_site_sk#103, date_sk#104, sales_price#105, profit#106, return_amt#107, net_loss#108], [ws_web_site_sk#100 AS wsr_web_site_sk#103, wr_returned_date_sk#98 AS date_sk#104, 0.00 AS sales_price#105, 0.00 AS profit#106, wr_return_amt#96 AS return_amt#107, wr_net_loss#97 AS net_loss#108] + +(51) CometUnion +Child 0 Input [6]: [wsr_web_site_sk#88, date_sk#89, sales_price#90, profit#91, return_amt#92, net_loss#93] +Child 1 Input [6]: [wsr_web_site_sk#103, date_sk#104, sales_price#105, profit#106, return_amt#107, net_loss#108] -(56) Union +(52) ColumnarToRow [codegen id : 11] +Input [6]: [wsr_web_site_sk#88, date_sk#89, sales_price#90, profit#91, return_amt#92, net_loss#93] -(57) ReusedExchange [Reuses operator id: 79] +(53) ReusedExchange [Reuses operator id: 75] Output [1]: [d_date_sk#109] -(58) BroadcastHashJoin [codegen id : 18] +(54) BroadcastHashJoin [codegen id : 11] Left keys [1]: [date_sk#89] Right keys [1]: [d_date_sk#109] Join type: Inner Join condition: None -(59) Project [codegen id : 18] +(55) Project [codegen id : 11] Output [5]: [wsr_web_site_sk#88, sales_price#90, profit#91, return_amt#92, net_loss#93] Input [7]: [wsr_web_site_sk#88, date_sk#89, sales_price#90, profit#91, return_amt#92, net_loss#93, d_date_sk#109] -(60) Scan parquet spark_catalog.default.web_site +(56) Scan parquet spark_catalog.default.web_site Output [2]: [web_site_sk#110, web_site_id#111] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(61) ColumnarToRow [codegen id : 17] +(57) CometFilter Input [2]: [web_site_sk#110, web_site_id#111] +Condition : isnotnull(web_site_sk#110) -(62) Filter [codegen id : 17] +(58) ColumnarToRow [codegen id : 10] Input [2]: [web_site_sk#110, web_site_id#111] -Condition : isnotnull(web_site_sk#110) -(63) BroadcastExchange +(59) BroadcastExchange Input [2]: [web_site_sk#110, web_site_id#111] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] -(64) BroadcastHashJoin [codegen id : 18] +(60) BroadcastHashJoin [codegen id : 11] Left keys [1]: [wsr_web_site_sk#88] Right keys [1]: [web_site_sk#110] Join type: Inner Join condition: None -(65) Project [codegen id : 18] +(61) Project [codegen id : 11] Output [5]: [sales_price#90, profit#91, return_amt#92, net_loss#93, web_site_id#111] Input [7]: [wsr_web_site_sk#88, sales_price#90, profit#91, return_amt#92, net_loss#93, web_site_sk#110, web_site_id#111] -(66) HashAggregate [codegen id : 18] +(62) HashAggregate [codegen id : 11] Input [5]: [sales_price#90, profit#91, return_amt#92, net_loss#93, web_site_id#111] Keys [1]: [web_site_id#111] Functions [4]: [partial_sum(UnscaledValue(sales_price#90)), partial_sum(UnscaledValue(return_amt#92)), partial_sum(UnscaledValue(profit#91)), partial_sum(UnscaledValue(net_loss#93))] Aggregate Attributes [4]: [sum#112, sum#113, sum#114, sum#115] Results [5]: [web_site_id#111, sum#116, sum#117, sum#118, sum#119] -(67) Exchange +(63) Exchange Input [5]: [web_site_id#111, sum#116, sum#117, sum#118, sum#119] -Arguments: hashpartitioning(web_site_id#111, 5), ENSURE_REQUIREMENTS, [plan_id=7] +Arguments: hashpartitioning(web_site_id#111, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(68) HashAggregate [codegen id : 19] +(64) HashAggregate [codegen id : 12] Input [5]: [web_site_id#111, sum#116, sum#117, sum#118, sum#119] Keys [1]: [web_site_id#111] Functions [4]: [sum(UnscaledValue(sales_price#90)), sum(UnscaledValue(return_amt#92)), sum(UnscaledValue(profit#91)), sum(UnscaledValue(net_loss#93))] Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#90))#120, sum(UnscaledValue(return_amt#92))#121, sum(UnscaledValue(profit#91))#122, sum(UnscaledValue(net_loss#93))#123] Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#90))#120,17,2) AS sales#124, MakeDecimal(sum(UnscaledValue(return_amt#92))#121,17,2) AS returns#125, (MakeDecimal(sum(UnscaledValue(profit#91))#122,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#93))#123,17,2)) AS profit#126, web channel AS channel#127, concat(web_site, web_site_id#111) AS id#128] -(69) Union +(65) Union -(70) Expand [codegen id : 20] +(66) Expand [codegen id : 13] Input [5]: [sales#37, returns#38, profit#39, channel#40, id#41] Arguments: [[sales#37, returns#38, profit#39, channel#40, id#41, 0], [sales#37, returns#38, profit#39, channel#40, null, 1], [sales#37, returns#38, profit#39, null, null, 3]], [sales#37, returns#38, profit#39, channel#129, id#130, spark_grouping_id#131] -(71) HashAggregate [codegen id : 20] +(67) HashAggregate [codegen id : 13] Input [6]: [sales#37, returns#38, profit#39, channel#129, id#130, spark_grouping_id#131] Keys [3]: [channel#129, id#130, spark_grouping_id#131] Functions [3]: [partial_sum(sales#37), partial_sum(returns#38), partial_sum(profit#39)] Aggregate Attributes [6]: [sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] Results [9]: [channel#129, id#130, spark_grouping_id#131, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] -(72) Exchange +(68) Exchange Input [9]: [channel#129, id#130, spark_grouping_id#131, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] -Arguments: hashpartitioning(channel#129, id#130, spark_grouping_id#131, 5), ENSURE_REQUIREMENTS, [plan_id=8] +Arguments: hashpartitioning(channel#129, id#130, spark_grouping_id#131, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(73) HashAggregate [codegen id : 21] +(69) HashAggregate [codegen id : 14] Input [9]: [channel#129, id#130, spark_grouping_id#131, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] Keys [3]: [channel#129, id#130, spark_grouping_id#131] Functions [3]: [sum(sales#37), sum(returns#38), sum(profit#39)] Aggregate Attributes [3]: [sum(sales#37)#144, sum(returns#38)#145, sum(profit#39)#146] Results [5]: [channel#129, id#130, sum(sales#37)#144 AS sales#147, sum(returns#38)#145 AS returns#148, sum(profit#39)#146 AS profit#149] -(74) TakeOrderedAndProject +(70) TakeOrderedAndProject Input [5]: [channel#129, id#130, sales#147, returns#148, profit#149] Arguments: 100, [channel#129 ASC NULLS FIRST, id#130 ASC NULLS FIRST], [channel#129, id#130, sales#147, returns#148, profit#149] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (79) -+- * Project (78) - +- * Filter (77) - +- * ColumnarToRow (76) - +- CometScan parquet spark_catalog.default.date_dim (75) +BroadcastExchange (75) ++- * ColumnarToRow (74) + +- CometProject (73) + +- CometFilter (72) + +- CometScan parquet spark_catalog.default.date_dim (71) -(75) Scan parquet spark_catalog.default.date_dim +(71) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_date#150] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-06), IsNotNull(d_date_sk)] ReadSchema: struct -(76) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#22, d_date#150] - -(77) Filter [codegen id : 1] +(72) CometFilter Input [2]: [d_date_sk#22, d_date#150] Condition : (((isnotnull(d_date#150) AND (d_date#150 >= 2000-08-23)) AND (d_date#150 <= 2000-09-06)) AND isnotnull(d_date_sk#22)) -(78) Project [codegen id : 1] -Output [1]: [d_date_sk#22] +(73) CometProject Input [2]: [d_date_sk#22, d_date#150] +Arguments: [d_date_sk#22], [d_date_sk#22] + +(74) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#22] -(79) BroadcastExchange +(75) BroadcastExchange Input [1]: [d_date_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] -Subquery:2 Hosting operator id = 5 Hosting Expression = sr_returned_date_sk#15 IN dynamicpruning#5 +Subquery:2 Hosting operator id = 4 Hosting Expression = sr_returned_date_sk#15 IN dynamicpruning#5 -Subquery:3 Hosting operator id = 22 Hosting Expression = cs_sold_date_sk#45 IN dynamicpruning#5 +Subquery:3 Hosting operator id = 21 Hosting Expression = cs_sold_date_sk#45 IN dynamicpruning#5 -Subquery:4 Hosting operator id = 26 Hosting Expression = cr_returned_date_sk#56 IN dynamicpruning#5 +Subquery:4 Hosting operator id = 24 Hosting Expression = cr_returned_date_sk#56 IN dynamicpruning#5 -Subquery:5 Hosting operator id = 43 Hosting Expression = ws_sold_date_sk#86 IN dynamicpruning#5 +Subquery:5 Hosting operator id = 41 Hosting Expression = ws_sold_date_sk#86 IN dynamicpruning#5 -Subquery:6 Hosting operator id = 47 Hosting Expression = wr_returned_date_sk#98 IN dynamicpruning#5 +Subquery:6 Hosting operator id = 44 Hosting Expression = wr_returned_date_sk#98 IN dynamicpruning#5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/simplified.txt index cf9a7db64..3d539d591 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/simplified.txt @@ -1,130 +1,111 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (21) + WholeStageCodegen (14) HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] InputAdapter Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (20) + WholeStageCodegen (13) HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] Expand [sales,returns,profit,channel,id] InputAdapter Union - WholeStageCodegen (6) + WholeStageCodegen (4) HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] InputAdapter Exchange [s_store_id] #2 - WholeStageCodegen (5) + WholeStageCodegen (3) HashAggregate [s_store_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] Project [sales_price,profit,return_amt,net_loss,s_store_id] BroadcastHashJoin [store_sk,s_store_sk] Project [store_sk,sales_price,profit,return_amt,net_loss] BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (1) - Project [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - WholeStageCodegen (2) - Project [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] - Filter [sr_store_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 + ColumnarToRow + InputAdapter + CometUnion + CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ss_store_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date,d_date_sk] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [sr_store_sk] + CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 InputAdapter ReusedExchange [d_date_sk] #3 InputAdapter BroadcastExchange #4 - WholeStageCodegen (4) - Filter [s_store_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (2) + ColumnarToRow + InputAdapter + CometFilter [s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - WholeStageCodegen (12) + WholeStageCodegen (8) HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] InputAdapter Exchange [cp_catalog_page_id] #5 - WholeStageCodegen (11) + WholeStageCodegen (7) HashAggregate [cp_catalog_page_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] Project [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] BroadcastHashJoin [page_sk,cp_catalog_page_sk] Project [page_sk,sales_price,profit,return_amt,net_loss] BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (7) - Project [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] - Filter [cs_catalog_page_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - WholeStageCodegen (8) - Project [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] - Filter [cr_catalog_page_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 + ColumnarToRow + InputAdapter + CometUnion + CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cs_catalog_page_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cr_catalog_page_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 InputAdapter ReusedExchange [d_date_sk] #3 InputAdapter BroadcastExchange #6 - WholeStageCodegen (10) - Filter [cp_catalog_page_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (6) + ColumnarToRow + InputAdapter + CometFilter [cp_catalog_page_sk] CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - WholeStageCodegen (19) + WholeStageCodegen (12) HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] InputAdapter Exchange [web_site_id] #7 - WholeStageCodegen (18) + WholeStageCodegen (11) HashAggregate [web_site_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] Project [sales_price,profit,return_amt,net_loss,web_site_id] BroadcastHashJoin [wsr_web_site_sk,web_site_sk] Project [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (13) - Project [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] - Filter [ws_web_site_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - WholeStageCodegen (15) - Project [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] - BroadcastHashJoin [wr_item_sk,wr_order_number,ws_item_sk,ws_order_number] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (14) - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - Project [ws_item_sk,ws_web_site_sk,ws_order_number] - Filter [ws_item_sk,ws_order_number,ws_web_site_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + ColumnarToRow + InputAdapter + CometUnion + CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ws_web_site_sk] + CometScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wr_item_sk,wr_order_number,ws_item_sk,ws_order_number] + CometBroadcastExchange #8 + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] + CometFilter [ws_item_sk,ws_order_number,ws_web_site_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] InputAdapter ReusedExchange [d_date_sk] #3 InputAdapter BroadcastExchange #9 - WholeStageCodegen (17) - Filter [web_site_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (10) + ColumnarToRow + InputAdapter + CometFilter [web_site_sk] CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/explain.txt index 8422950c2..0182e0ac1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/explain.txt @@ -11,20 +11,20 @@ TakeOrderedAndProject (28) : : +- * BroadcastHashJoin Inner BuildRight (14) : : :- * Project (9) : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) + : : : +- * ColumnarToRow (6) + : : : +- CometFilter (5) : : : +- CometScan parquet spark_catalog.default.store_returns (4) : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) + : : +- * ColumnarToRow (12) + : : +- CometFilter (11) : : +- CometScan parquet spark_catalog.default.store (10) : +- BroadcastExchange (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) + : +- * ColumnarToRow (18) + : +- CometFilter (17) : +- CometScan parquet spark_catalog.default.date_dim (16) +- ReusedExchange (22) @@ -37,12 +37,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] +(2) CometFilter Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_store_sk#3)) -(3) Filter [codegen id : 5] +(3) ColumnarToRow [codegen id : 5] Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_store_sk#3)) (4) Scan parquet spark_catalog.default.store_returns Output [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] @@ -52,12 +52,12 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#9), dynamicpruningexpression(sr PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk), IsNotNull(sr_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Condition : ((isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#6)) AND isnotnull(sr_customer_sk#7)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Condition : ((isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#6)) AND isnotnull(sr_customer_sk#7)) (7) BroadcastExchange Input [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] @@ -80,12 +80,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] +(11) CometFilter Input [11]: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21] +Condition : isnotnull(s_store_sk#11) -(12) Filter [codegen id : 2] +(12) ColumnarToRow [codegen id : 2] Input [11]: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21] -Condition : isnotnull(s_store_sk#11) (13) BroadcastExchange Input [11]: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21] @@ -108,12 +108,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] +(17) CometFilter Input [1]: [d_date_sk#22] +Condition : isnotnull(d_date_sk#22) -(18) Filter [codegen id : 3] +(18) ColumnarToRow [codegen id : 3] Input [1]: [d_date_sk#22] -Condition : isnotnull(d_date_sk#22) (19) BroadcastExchange Input [1]: [d_date_sk#22] @@ -168,9 +168,9 @@ Arguments: 100, [s_store_name#12 ASC NULLS FIRST, s_company_id#13 ASC NULLS FIRS Subquery:1 Hosting operator id = 4 Hosting Expression = sr_returned_date_sk#9 IN dynamicpruning#10 BroadcastExchange (33) -+- * Project (32) - +- * Filter (31) - +- * ColumnarToRow (30) ++- * ColumnarToRow (32) + +- CometProject (31) + +- CometFilter (30) +- CometScan parquet spark_catalog.default.date_dim (29) @@ -181,16 +181,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,8), IsNotNull(d_date_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#23, d_year#44, d_moy#45] - -(31) Filter [codegen id : 1] +(30) CometFilter Input [3]: [d_date_sk#23, d_year#44, d_moy#45] Condition : ((((isnotnull(d_year#44) AND isnotnull(d_moy#45)) AND (d_year#44 = 2001)) AND (d_moy#45 = 8)) AND isnotnull(d_date_sk#23)) -(32) Project [codegen id : 1] -Output [1]: [d_date_sk#23] +(31) CometProject Input [3]: [d_date_sk#23, d_year#44, d_moy#45] +Arguments: [d_date_sk#23], [d_date_sk#23] + +(32) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#23] (33) BroadcastExchange Input [1]: [d_date_sk#23] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/simplified.txt index 67c991f71..dfdcaf497 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/simplified.txt @@ -13,38 +13,38 @@ TakeOrderedAndProject [s_store_name,s_company_id,s_street_number,s_street_name,s BroadcastHashJoin [ss_store_sk,s_store_sk] Project [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk] BroadcastHashJoin [ss_ticket_number,ss_item_sk,ss_customer_sk,sr_ticket_number,sr_item_sk,sr_customer_sk] - Filter [ss_ticket_number,ss_item_sk,ss_customer_sk,ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_ticket_number,ss_item_sk,ss_customer_sk,ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] InputAdapter BroadcastExchange #2 WholeStageCodegen (1) - Filter [sr_ticket_number,sr_item_sk,sr_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [sr_ticket_number,sr_item_sk,sr_customer_sk] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_moy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - Filter [d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk] InputAdapter ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/explain.txt index dd0fdb3f0..2613551f0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/explain.txt @@ -17,8 +17,8 @@ TakeOrderedAndProject (37) : +- * HashAggregate (7) : +- * Project (6) : +- * BroadcastHashJoin Inner BuildRight (5) - : :- * Filter (3) - : : +- * ColumnarToRow (2) + : :- * ColumnarToRow (3) + : : +- CometFilter (2) : : +- CometScan parquet spark_catalog.default.web_sales (1) : +- ReusedExchange (4) +- * Sort (30) @@ -32,8 +32,8 @@ TakeOrderedAndProject (37) +- * HashAggregate (22) +- * Project (21) +- * BroadcastHashJoin Inner BuildRight (20) - :- * Filter (18) - : +- * ColumnarToRow (17) + :- * ColumnarToRow (18) + : +- CometFilter (17) : +- CometScan parquet spark_catalog.default.store_sales (16) +- ReusedExchange (19) @@ -46,12 +46,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sol PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 2] +(2) CometFilter Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] +Condition : isnotnull(ws_item_sk#1) -(3) Filter [codegen id : 2] +(3) ColumnarToRow [codegen id : 2] Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Condition : isnotnull(ws_item_sk#1) (4) ReusedExchange [Reuses operator id: 42] Output [2]: [d_date_sk#5, d_date#6] @@ -116,12 +116,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#15), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 8] +(17) CometFilter Input [3]: [ss_item_sk#13, ss_sales_price#14, ss_sold_date_sk#15] +Condition : isnotnull(ss_item_sk#13) -(18) Filter [codegen id : 8] +(18) ColumnarToRow [codegen id : 8] Input [3]: [ss_item_sk#13, ss_sales_price#14, ss_sold_date_sk#15] -Condition : isnotnull(ss_item_sk#13) (19) ReusedExchange [Reuses operator id: 42] Output [2]: [d_date_sk#17, d_date#18] @@ -212,9 +212,9 @@ Arguments: 100, [item_sk#25 ASC NULLS FIRST, d_date#26 ASC NULLS FIRST], [item_s Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 BroadcastExchange (42) -+- * Project (41) - +- * Filter (40) - +- * ColumnarToRow (39) ++- * ColumnarToRow (41) + +- CometProject (40) + +- CometFilter (39) +- CometScan parquet spark_catalog.default.date_dim (38) @@ -225,16 +225,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(39) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#31] - -(40) Filter [codegen id : 1] +(39) CometFilter Input [3]: [d_date_sk#5, d_date#6, d_month_seq#31] Condition : (((isnotnull(d_month_seq#31) AND (d_month_seq#31 >= 1200)) AND (d_month_seq#31 <= 1211)) AND isnotnull(d_date_sk#5)) -(41) Project [codegen id : 1] -Output [2]: [d_date_sk#5, d_date#6] +(40) CometProject Input [3]: [d_date_sk#5, d_date#6, d_month_seq#31] +Arguments: [d_date_sk#5, d_date#6], [d_date_sk#5, d_date#6] + +(41) ColumnarToRow [codegen id : 1] +Input [2]: [d_date_sk#5, d_date#6] (42) BroadcastExchange Input [2]: [d_date_sk#5, d_date#6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/simplified.txt index a6ce8a445..181cd1b98 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/simplified.txt @@ -31,17 +31,17 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store HashAggregate [ws_item_sk,d_date,ws_sales_price] [sum,sum] Project [ws_item_sk,ws_sales_price,d_date] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #5 WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_date] + CometFilter [d_month_seq,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] InputAdapter ReusedExchange [d_date_sk,d_date] #5 @@ -66,9 +66,9 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store HashAggregate [ss_item_sk,d_date,ss_sales_price] [sum,sum] Project [ss_item_sk,ss_sales_price,d_date] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/explain.txt index f8131fa11..3d5317eb0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/explain.txt @@ -7,18 +7,18 @@ TakeOrderedAndProject (21) +- * BroadcastHashJoin Inner BuildRight (16) :- * Project (10) : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) + : :- * ColumnarToRow (4) + : : +- CometProject (3) + : : +- CometFilter (2) : : +- CometScan parquet spark_catalog.default.date_dim (1) : +- BroadcastExchange (8) - : +- * Filter (7) - : +- * ColumnarToRow (6) + : +- * ColumnarToRow (7) + : +- CometFilter (6) : +- CometScan parquet spark_catalog.default.store_sales (5) +- BroadcastExchange (15) - +- * Project (14) - +- * Filter (13) - +- * ColumnarToRow (12) + +- * ColumnarToRow (14) + +- CometProject (13) + +- CometFilter (12) +- CometScan parquet spark_catalog.default.item (11) @@ -29,16 +29,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1)) -(4) Project [codegen id : 3] -Output [2]: [d_date_sk#1, d_year#2] +(3) CometProject Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] + +(4) ColumnarToRow [codegen id : 3] +Input [2]: [d_date_sk#1, d_year#2] (5) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] @@ -48,12 +48,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] +(6) CometFilter Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Condition : isnotnull(ss_item_sk#4) -(7) Filter [codegen id : 1] +(7) ColumnarToRow [codegen id : 1] Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_item_sk#4) (8) BroadcastExchange Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] @@ -76,16 +76,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] - -(13) Filter [codegen id : 2] +(12) CometFilter Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 1)) AND isnotnull(i_item_sk#7)) -(14) Project [codegen id : 2] -Output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] +(13) CometProject Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9], [i_item_sk#7, i_brand_id#8, i_brand#9] + +(14) ColumnarToRow [codegen id : 2] +Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] (15) BroadcastExchange Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/simplified.txt index 591ba96c2..91fdc2f17 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/simplified.txt @@ -9,23 +9,23 @@ TakeOrderedAndProject [d_year,ext_price,brand_id,brand] BroadcastHashJoin [ss_item_sk,i_item_sk] Project [d_year,ss_item_sk,ss_ext_sales_price] BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk,d_year] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_year] + CometFilter [d_moy,d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #2 WholeStageCodegen (1) - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Project [i_item_sk,i_brand_id,i_brand] - Filter [i_manager_id,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_item_sk,i_brand_id,i_brand] + CometFilter [i_manager_id,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/explain.txt index 741e1e050..335dc7fa2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/explain.txt @@ -14,18 +14,18 @@ TakeOrderedAndProject (28) : +- * BroadcastHashJoin Inner BuildRight (12) : :- * Project (10) : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) + : : :- * ColumnarToRow (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) : : : +- CometScan parquet spark_catalog.default.item (1) : : +- BroadcastExchange (8) - : : +- * Filter (7) - : : +- * ColumnarToRow (6) + : : +- * ColumnarToRow (7) + : : +- CometFilter (6) : : +- CometScan parquet spark_catalog.default.store_sales (5) : +- ReusedExchange (11) +- BroadcastExchange (17) - +- * Filter (16) - +- * ColumnarToRow (15) + +- * ColumnarToRow (16) + +- CometFilter (15) +- CometScan parquet spark_catalog.default.store (14) @@ -36,16 +36,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Condition : ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) -(4) Project [codegen id : 4] -Output [2]: [i_item_sk#1, i_manufact_id#5] +(3) CometProject Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] +Arguments: [i_item_sk#1, i_manufact_id#5], [i_item_sk#1, i_manufact_id#5] + +(4) ColumnarToRow [codegen id : 4] +Input [2]: [i_item_sk#1, i_manufact_id#5] (5) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] @@ -55,12 +55,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] +(6) CometFilter Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Condition : (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) -(7) Filter [codegen id : 1] +(7) ColumnarToRow [codegen id : 1] Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Condition : (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) (8) BroadcastExchange Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] @@ -96,12 +96,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 3] +(15) CometFilter Input [1]: [s_store_sk#17] +Condition : isnotnull(s_store_sk#17) -(16) Filter [codegen id : 3] +(16) ColumnarToRow [codegen id : 3] Input [1]: [s_store_sk#17] -Condition : isnotnull(s_store_sk#17) (17) BroadcastExchange Input [1]: [s_store_sk#17] @@ -163,9 +163,9 @@ Arguments: 100, [avg_quarterly_sales#23 ASC NULLS FIRST, sum_sales#21 ASC NULLS Subquery:1 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 BroadcastExchange (33) -+- * Project (32) - +- * Filter (31) - +- * ColumnarToRow (30) ++- * ColumnarToRow (32) + +- CometProject (31) + +- CometFilter (30) +- CometScan parquet spark_catalog.default.date_dim (29) @@ -176,16 +176,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#15, d_month_seq#24, d_qoy#16] - -(31) Filter [codegen id : 1] +(30) CometFilter Input [3]: [d_date_sk#15, d_month_seq#24, d_qoy#16] Condition : (d_month_seq#24 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#15)) -(32) Project [codegen id : 1] -Output [2]: [d_date_sk#15, d_qoy#16] +(31) CometProject Input [3]: [d_date_sk#15, d_month_seq#24, d_qoy#16] +Arguments: [d_date_sk#15, d_qoy#16], [d_date_sk#15, d_qoy#16] + +(32) ColumnarToRow [codegen id : 1] +Input [2]: [d_date_sk#15, d_qoy#16] (33) BroadcastExchange Input [2]: [d_date_sk#15, d_qoy#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/simplified.txt index 5e254d2ac..adda5c34f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/simplified.txt @@ -20,32 +20,32 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] BroadcastHashJoin [i_item_sk,ss_item_sk] - Project [i_item_sk,i_manufact_id] - Filter [i_category,i_class,i_brand,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_item_sk,i_manufact_id] + CometFilter [i_category,i_class,i_brand,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk,ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) - Project [d_date_sk,d_qoy] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_qoy] + CometFilter [d_month_seq,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] InputAdapter ReusedExchange [d_date_sk,d_qoy] #4 InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - Filter [s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/explain.txt index 7ceab6eaa..ca308b19a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/explain.txt @@ -1,60 +1,59 @@ == Physical Plan == -TakeOrderedAndProject (56) -+- * HashAggregate (55) - +- Exchange (54) - +- * HashAggregate (53) - +- * HashAggregate (52) - +- Exchange (51) - +- * HashAggregate (50) - +- * Project (49) - +- * BroadcastHashJoin Inner BuildRight (48) - :- * Project (46) - : +- * BroadcastHashJoin Inner BuildRight (45) - : :- * Project (40) - : : +- * BroadcastHashJoin Inner BuildRight (39) - : : :- * Project (34) - : : : +- * BroadcastHashJoin Inner BuildRight (33) - : : : :- * HashAggregate (28) - : : : : +- Exchange (27) - : : : : +- * HashAggregate (26) - : : : : +- * Project (25) - : : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : : :- * Project (19) - : : : : : +- * BroadcastHashJoin Inner BuildRight (18) - : : : : : :- * Project (16) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : : : : :- Union (9) - : : : : : : : :- * Project (4) - : : : : : : : : +- * Filter (3) - : : : : : : : : +- * ColumnarToRow (2) - : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : : : : : : : +- * Project (8) - : : : : : : : +- * Filter (7) - : : : : : : : +- * ColumnarToRow (6) - : : : : : : : +- CometScan parquet spark_catalog.default.web_sales (5) - : : : : : : +- BroadcastExchange (14) - : : : : : : +- * Project (13) - : : : : : : +- * Filter (12) - : : : : : : +- * ColumnarToRow (11) - : : : : : : +- CometScan parquet spark_catalog.default.item (10) - : : : : : +- ReusedExchange (17) - : : : : +- BroadcastExchange (23) - : : : : +- * Filter (22) - : : : : +- * ColumnarToRow (21) - : : : : +- CometScan parquet spark_catalog.default.customer (20) - : : : +- BroadcastExchange (32) - : : : +- * Filter (31) - : : : +- * ColumnarToRow (30) - : : : +- CometScan parquet spark_catalog.default.store_sales (29) - : : +- BroadcastExchange (38) - : : +- * Filter (37) - : : +- * ColumnarToRow (36) - : : +- CometScan parquet spark_catalog.default.customer_address (35) - : +- BroadcastExchange (44) - : +- * Filter (43) - : +- * ColumnarToRow (42) - : +- CometScan parquet spark_catalog.default.store (41) - +- ReusedExchange (47) +TakeOrderedAndProject (55) ++- * HashAggregate (54) + +- Exchange (53) + +- * HashAggregate (52) + +- * HashAggregate (51) + +- Exchange (50) + +- * HashAggregate (49) + +- * Project (48) + +- * BroadcastHashJoin Inner BuildRight (47) + :- * Project (45) + : +- * BroadcastHashJoin Inner BuildRight (44) + : :- * Project (39) + : : +- * BroadcastHashJoin Inner BuildRight (38) + : : :- * Project (33) + : : : +- * BroadcastHashJoin Inner BuildRight (32) + : : : :- * HashAggregate (27) + : : : : +- Exchange (26) + : : : : +- * HashAggregate (25) + : : : : +- * Project (24) + : : : : +- * BroadcastHashJoin Inner BuildRight (23) + : : : : :- * Project (18) + : : : : : +- * BroadcastHashJoin Inner BuildRight (17) + : : : : : :- * Project (15) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (14) + : : : : : : :- * ColumnarToRow (8) + : : : : : : : +- CometUnion (7) + : : : : : : : :- CometProject (3) + : : : : : : : : +- CometFilter (2) + : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : : : : +- CometProject (6) + : : : : : : : +- CometFilter (5) + : : : : : : : +- CometScan parquet spark_catalog.default.web_sales (4) + : : : : : : +- BroadcastExchange (13) + : : : : : : +- * ColumnarToRow (12) + : : : : : : +- CometProject (11) + : : : : : : +- CometFilter (10) + : : : : : : +- CometScan parquet spark_catalog.default.item (9) + : : : : : +- ReusedExchange (16) + : : : : +- BroadcastExchange (22) + : : : : +- * ColumnarToRow (21) + : : : : +- CometFilter (20) + : : : : +- CometScan parquet spark_catalog.default.customer (19) + : : : +- BroadcastExchange (31) + : : : +- * ColumnarToRow (30) + : : : +- CometFilter (29) + : : : +- CometScan parquet spark_catalog.default.store_sales (28) + : : +- BroadcastExchange (37) + : : +- * ColumnarToRow (36) + : : +- CometFilter (35) + : : +- CometScan parquet spark_catalog.default.customer_address (34) + : +- BroadcastExchange (43) + : +- * ColumnarToRow (42) + : +- CometFilter (41) + : +- CometScan parquet spark_catalog.default.store (40) + +- ReusedExchange (46) (1) Scan parquet spark_catalog.default.catalog_sales @@ -65,18 +64,15 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sol PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] Condition : (isnotnull(cs_item_sk#2) AND isnotnull(cs_bill_customer_sk#1)) -(4) Project [codegen id : 1] -Output [3]: [cs_sold_date_sk#3 AS sold_date_sk#5, cs_bill_customer_sk#1 AS customer_sk#6, cs_item_sk#2 AS item_sk#7] +(3) CometProject Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] +Arguments: [sold_date_sk#5, customer_sk#6, item_sk#7], [cs_sold_date_sk#3 AS sold_date_sk#5, cs_bill_customer_sk#1 AS customer_sk#6, cs_item_sk#2 AS item_sk#7] -(5) Scan parquet spark_catalog.default.web_sales +(4) Scan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#8, ws_bill_customer_sk#9, ws_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] @@ -84,111 +80,113 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#10), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 2] -Input [3]: [ws_item_sk#8, ws_bill_customer_sk#9, ws_sold_date_sk#10] - -(7) Filter [codegen id : 2] +(5) CometFilter Input [3]: [ws_item_sk#8, ws_bill_customer_sk#9, ws_sold_date_sk#10] Condition : (isnotnull(ws_item_sk#8) AND isnotnull(ws_bill_customer_sk#9)) -(8) Project [codegen id : 2] -Output [3]: [ws_sold_date_sk#10 AS sold_date_sk#11, ws_bill_customer_sk#9 AS customer_sk#12, ws_item_sk#8 AS item_sk#13] +(6) CometProject Input [3]: [ws_item_sk#8, ws_bill_customer_sk#9, ws_sold_date_sk#10] +Arguments: [sold_date_sk#11, customer_sk#12, item_sk#13], [ws_sold_date_sk#10 AS sold_date_sk#11, ws_bill_customer_sk#9 AS customer_sk#12, ws_item_sk#8 AS item_sk#13] + +(7) CometUnion +Child 0 Input [3]: [sold_date_sk#5, customer_sk#6, item_sk#7] +Child 1 Input [3]: [sold_date_sk#11, customer_sk#12, item_sk#13] -(9) Union +(8) ColumnarToRow [codegen id : 4] +Input [3]: [sold_date_sk#5, customer_sk#6, item_sk#7] -(10) Scan parquet spark_catalog.default.item +(9) Scan parquet spark_catalog.default.item Output [3]: [i_item_sk#14, i_class#15, i_category#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_category), IsNotNull(i_class), EqualTo(i_category,Women ), EqualTo(i_class,maternity ), IsNotNull(i_item_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 3] -Input [3]: [i_item_sk#14, i_class#15, i_category#16] - -(12) Filter [codegen id : 3] +(10) CometFilter Input [3]: [i_item_sk#14, i_class#15, i_category#16] Condition : ((((isnotnull(i_category#16) AND isnotnull(i_class#15)) AND (i_category#16 = Women )) AND (i_class#15 = maternity )) AND isnotnull(i_item_sk#14)) -(13) Project [codegen id : 3] -Output [1]: [i_item_sk#14] +(11) CometProject Input [3]: [i_item_sk#14, i_class#15, i_category#16] +Arguments: [i_item_sk#14], [i_item_sk#14] -(14) BroadcastExchange +(12) ColumnarToRow [codegen id : 1] +Input [1]: [i_item_sk#14] + +(13) BroadcastExchange Input [1]: [i_item_sk#14] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] -(15) BroadcastHashJoin [codegen id : 6] +(14) BroadcastHashJoin [codegen id : 4] Left keys [1]: [item_sk#7] Right keys [1]: [i_item_sk#14] Join type: Inner Join condition: None -(16) Project [codegen id : 6] +(15) Project [codegen id : 4] Output [2]: [sold_date_sk#5, customer_sk#6] Input [4]: [sold_date_sk#5, customer_sk#6, item_sk#7, i_item_sk#14] -(17) ReusedExchange [Reuses operator id: 61] +(16) ReusedExchange [Reuses operator id: 60] Output [1]: [d_date_sk#17] -(18) BroadcastHashJoin [codegen id : 6] +(17) BroadcastHashJoin [codegen id : 4] Left keys [1]: [sold_date_sk#5] Right keys [1]: [d_date_sk#17] Join type: Inner Join condition: None -(19) Project [codegen id : 6] +(18) Project [codegen id : 4] Output [1]: [customer_sk#6] Input [3]: [sold_date_sk#5, customer_sk#6, d_date_sk#17] -(20) Scan parquet spark_catalog.default.customer +(19) Scan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#18, c_current_addr_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 5] +(20) CometFilter Input [2]: [c_customer_sk#18, c_current_addr_sk#19] +Condition : (isnotnull(c_customer_sk#18) AND isnotnull(c_current_addr_sk#19)) -(22) Filter [codegen id : 5] +(21) ColumnarToRow [codegen id : 3] Input [2]: [c_customer_sk#18, c_current_addr_sk#19] -Condition : (isnotnull(c_customer_sk#18) AND isnotnull(c_current_addr_sk#19)) -(23) BroadcastExchange +(22) BroadcastExchange Input [2]: [c_customer_sk#18, c_current_addr_sk#19] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] -(24) BroadcastHashJoin [codegen id : 6] +(23) BroadcastHashJoin [codegen id : 4] Left keys [1]: [customer_sk#6] Right keys [1]: [c_customer_sk#18] Join type: Inner Join condition: None -(25) Project [codegen id : 6] +(24) Project [codegen id : 4] Output [2]: [c_customer_sk#18, c_current_addr_sk#19] Input [3]: [customer_sk#6, c_customer_sk#18, c_current_addr_sk#19] -(26) HashAggregate [codegen id : 6] +(25) HashAggregate [codegen id : 4] Input [2]: [c_customer_sk#18, c_current_addr_sk#19] Keys [2]: [c_customer_sk#18, c_current_addr_sk#19] Functions: [] Aggregate Attributes: [] Results [2]: [c_customer_sk#18, c_current_addr_sk#19] -(27) Exchange +(26) Exchange Input [2]: [c_customer_sk#18, c_current_addr_sk#19] Arguments: hashpartitioning(c_customer_sk#18, c_current_addr_sk#19, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(28) HashAggregate [codegen id : 11] +(27) HashAggregate [codegen id : 9] Input [2]: [c_customer_sk#18, c_current_addr_sk#19] Keys [2]: [c_customer_sk#18, c_current_addr_sk#19] Functions: [] Aggregate Attributes: [] Results [2]: [c_customer_sk#18, c_current_addr_sk#19] -(29) Scan parquet spark_catalog.default.store_sales +(28) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#20, ss_ext_sales_price#21, ss_sold_date_sk#22] Batched: true Location: InMemoryFileIndex [] @@ -196,290 +194,286 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#22), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 7] +(29) CometFilter Input [3]: [ss_customer_sk#20, ss_ext_sales_price#21, ss_sold_date_sk#22] +Condition : isnotnull(ss_customer_sk#20) -(31) Filter [codegen id : 7] +(30) ColumnarToRow [codegen id : 5] Input [3]: [ss_customer_sk#20, ss_ext_sales_price#21, ss_sold_date_sk#22] -Condition : isnotnull(ss_customer_sk#20) -(32) BroadcastExchange +(31) BroadcastExchange Input [3]: [ss_customer_sk#20, ss_ext_sales_price#21, ss_sold_date_sk#22] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] -(33) BroadcastHashJoin [codegen id : 11] +(32) BroadcastHashJoin [codegen id : 9] Left keys [1]: [c_customer_sk#18] Right keys [1]: [ss_customer_sk#20] Join type: Inner Join condition: None -(34) Project [codegen id : 11] +(33) Project [codegen id : 9] Output [4]: [c_customer_sk#18, c_current_addr_sk#19, ss_ext_sales_price#21, ss_sold_date_sk#22] Input [5]: [c_customer_sk#18, c_current_addr_sk#19, ss_customer_sk#20, ss_ext_sales_price#21, ss_sold_date_sk#22] -(35) Scan parquet spark_catalog.default.customer_address +(34) Scan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#24, ca_county#25, ca_state#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county), IsNotNull(ca_state)] ReadSchema: struct -(36) ColumnarToRow [codegen id : 8] +(35) CometFilter Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] +Condition : ((isnotnull(ca_address_sk#24) AND isnotnull(ca_county#25)) AND isnotnull(ca_state#26)) -(37) Filter [codegen id : 8] +(36) ColumnarToRow [codegen id : 6] Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] -Condition : ((isnotnull(ca_address_sk#24) AND isnotnull(ca_county#25)) AND isnotnull(ca_state#26)) -(38) BroadcastExchange +(37) BroadcastExchange Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] -(39) BroadcastHashJoin [codegen id : 11] +(38) BroadcastHashJoin [codegen id : 9] Left keys [1]: [c_current_addr_sk#19] Right keys [1]: [ca_address_sk#24] Join type: Inner Join condition: None -(40) Project [codegen id : 11] +(39) Project [codegen id : 9] Output [5]: [c_customer_sk#18, ss_ext_sales_price#21, ss_sold_date_sk#22, ca_county#25, ca_state#26] Input [7]: [c_customer_sk#18, c_current_addr_sk#19, ss_ext_sales_price#21, ss_sold_date_sk#22, ca_address_sk#24, ca_county#25, ca_state#26] -(41) Scan parquet spark_catalog.default.store +(40) Scan parquet spark_catalog.default.store Output [2]: [s_county#27, s_state#28] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_county), IsNotNull(s_state)] ReadSchema: struct -(42) ColumnarToRow [codegen id : 9] +(41) CometFilter Input [2]: [s_county#27, s_state#28] +Condition : (isnotnull(s_county#27) AND isnotnull(s_state#28)) -(43) Filter [codegen id : 9] +(42) ColumnarToRow [codegen id : 7] Input [2]: [s_county#27, s_state#28] -Condition : (isnotnull(s_county#27) AND isnotnull(s_state#28)) -(44) BroadcastExchange +(43) BroadcastExchange Input [2]: [s_county#27, s_state#28] Arguments: HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=6] -(45) BroadcastHashJoin [codegen id : 11] +(44) BroadcastHashJoin [codegen id : 9] Left keys [2]: [ca_county#25, ca_state#26] Right keys [2]: [s_county#27, s_state#28] Join type: Inner Join condition: None -(46) Project [codegen id : 11] +(45) Project [codegen id : 9] Output [3]: [c_customer_sk#18, ss_ext_sales_price#21, ss_sold_date_sk#22] Input [7]: [c_customer_sk#18, ss_ext_sales_price#21, ss_sold_date_sk#22, ca_county#25, ca_state#26, s_county#27, s_state#28] -(47) ReusedExchange [Reuses operator id: 66] +(46) ReusedExchange [Reuses operator id: 65] Output [1]: [d_date_sk#29] -(48) BroadcastHashJoin [codegen id : 11] +(47) BroadcastHashJoin [codegen id : 9] Left keys [1]: [ss_sold_date_sk#22] Right keys [1]: [d_date_sk#29] Join type: Inner Join condition: None -(49) Project [codegen id : 11] +(48) Project [codegen id : 9] Output [2]: [c_customer_sk#18, ss_ext_sales_price#21] Input [4]: [c_customer_sk#18, ss_ext_sales_price#21, ss_sold_date_sk#22, d_date_sk#29] -(50) HashAggregate [codegen id : 11] +(49) HashAggregate [codegen id : 9] Input [2]: [c_customer_sk#18, ss_ext_sales_price#21] Keys [1]: [c_customer_sk#18] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#21))] Aggregate Attributes [1]: [sum#30] Results [2]: [c_customer_sk#18, sum#31] -(51) Exchange +(50) Exchange Input [2]: [c_customer_sk#18, sum#31] Arguments: hashpartitioning(c_customer_sk#18, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(52) HashAggregate [codegen id : 12] +(51) HashAggregate [codegen id : 10] Input [2]: [c_customer_sk#18, sum#31] Keys [1]: [c_customer_sk#18] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#21))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#21))#32] Results [1]: [cast((MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#21))#32,17,2) / 50) as int) AS segment#33] -(53) HashAggregate [codegen id : 12] +(52) HashAggregate [codegen id : 10] Input [1]: [segment#33] Keys [1]: [segment#33] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#34] Results [2]: [segment#33, count#35] -(54) Exchange +(53) Exchange Input [2]: [segment#33, count#35] Arguments: hashpartitioning(segment#33, 5), ENSURE_REQUIREMENTS, [plan_id=8] -(55) HashAggregate [codegen id : 13] +(54) HashAggregate [codegen id : 11] Input [2]: [segment#33, count#35] Keys [1]: [segment#33] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#36] Results [3]: [segment#33, count(1)#36 AS num_customers#37, (segment#33 * 50) AS segment_base#38] -(56) TakeOrderedAndProject +(55) TakeOrderedAndProject Input [3]: [segment#33, num_customers#37, segment_base#38] Arguments: 100, [segment#33 ASC NULLS FIRST, num_customers#37 ASC NULLS FIRST], [segment#33, num_customers#37, segment_base#38] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (61) -+- * Project (60) - +- * Filter (59) - +- * ColumnarToRow (58) - +- CometScan parquet spark_catalog.default.date_dim (57) +BroadcastExchange (60) ++- * ColumnarToRow (59) + +- CometProject (58) + +- CometFilter (57) + +- CometScan parquet spark_catalog.default.date_dim (56) -(57) Scan parquet spark_catalog.default.date_dim +(56) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#17, d_year#39, d_moy#40] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,12), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct -(58) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#17, d_year#39, d_moy#40] - -(59) Filter [codegen id : 1] +(57) CometFilter Input [3]: [d_date_sk#17, d_year#39, d_moy#40] Condition : ((((isnotnull(d_moy#40) AND isnotnull(d_year#39)) AND (d_moy#40 = 12)) AND (d_year#39 = 1998)) AND isnotnull(d_date_sk#17)) -(60) Project [codegen id : 1] -Output [1]: [d_date_sk#17] +(58) CometProject Input [3]: [d_date_sk#17, d_year#39, d_moy#40] +Arguments: [d_date_sk#17], [d_date_sk#17] -(61) BroadcastExchange +(59) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#17] + +(60) BroadcastExchange Input [1]: [d_date_sk#17] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] -Subquery:2 Hosting operator id = 5 Hosting Expression = ws_sold_date_sk#10 IN dynamicpruning#4 +Subquery:2 Hosting operator id = 4 Hosting Expression = ws_sold_date_sk#10 IN dynamicpruning#4 -Subquery:3 Hosting operator id = 29 Hosting Expression = ss_sold_date_sk#22 IN dynamicpruning#23 -BroadcastExchange (66) -+- * Project (65) - +- * Filter (64) - +- * ColumnarToRow (63) - +- CometScan parquet spark_catalog.default.date_dim (62) +Subquery:3 Hosting operator id = 28 Hosting Expression = ss_sold_date_sk#22 IN dynamicpruning#23 +BroadcastExchange (65) ++- * ColumnarToRow (64) + +- CometProject (63) + +- CometFilter (62) + +- CometScan parquet spark_catalog.default.date_dim (61) -(62) Scan parquet spark_catalog.default.date_dim +(61) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#29, d_month_seq#41] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,ScalarSubquery#42), LessThanOrEqual(d_month_seq,ScalarSubquery#43), IsNotNull(d_date_sk)] ReadSchema: struct -(63) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#29, d_month_seq#41] - -(64) Filter [codegen id : 1] +(62) CometFilter Input [2]: [d_date_sk#29, d_month_seq#41] Condition : (((isnotnull(d_month_seq#41) AND (d_month_seq#41 >= ReusedSubquery Subquery scalar-subquery#42, [id=#44])) AND (d_month_seq#41 <= ReusedSubquery Subquery scalar-subquery#43, [id=#45])) AND isnotnull(d_date_sk#29)) -(65) Project [codegen id : 1] -Output [1]: [d_date_sk#29] +(63) CometProject Input [2]: [d_date_sk#29, d_month_seq#41] +Arguments: [d_date_sk#29], [d_date_sk#29] -(66) BroadcastExchange +(64) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#29] + +(65) BroadcastExchange Input [1]: [d_date_sk#29] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] -Subquery:4 Hosting operator id = 64 Hosting Expression = ReusedSubquery Subquery scalar-subquery#42, [id=#44] +Subquery:4 Hosting operator id = 62 Hosting Expression = ReusedSubquery Subquery scalar-subquery#42, [id=#44] -Subquery:5 Hosting operator id = 64 Hosting Expression = ReusedSubquery Subquery scalar-subquery#43, [id=#45] +Subquery:5 Hosting operator id = 62 Hosting Expression = ReusedSubquery Subquery scalar-subquery#43, [id=#45] -Subquery:6 Hosting operator id = 62 Hosting Expression = Subquery scalar-subquery#42, [id=#44] -* HashAggregate (73) -+- Exchange (72) - +- * HashAggregate (71) - +- * Project (70) - +- * Filter (69) - +- * ColumnarToRow (68) - +- CometScan parquet spark_catalog.default.date_dim (67) +Subquery:6 Hosting operator id = 61 Hosting Expression = Subquery scalar-subquery#42, [id=#44] +* HashAggregate (72) ++- Exchange (71) + +- * ColumnarToRow (70) + +- CometHashAggregate (69) + +- CometProject (68) + +- CometFilter (67) + +- CometScan parquet spark_catalog.default.date_dim (66) -(67) Scan parquet spark_catalog.default.date_dim +(66) Scan parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#46, d_year#47, d_moy#48] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)] ReadSchema: struct -(68) ColumnarToRow [codegen id : 1] -Input [3]: [d_month_seq#46, d_year#47, d_moy#48] - -(69) Filter [codegen id : 1] +(67) CometFilter Input [3]: [d_month_seq#46, d_year#47, d_moy#48] Condition : (((isnotnull(d_year#47) AND isnotnull(d_moy#48)) AND (d_year#47 = 1998)) AND (d_moy#48 = 12)) -(70) Project [codegen id : 1] -Output [1]: [(d_month_seq#46 + 1) AS (d_month_seq + 1)#49] +(68) CometProject Input [3]: [d_month_seq#46, d_year#47, d_moy#48] +Arguments: [(d_month_seq + 1)#49], [(d_month_seq#46 + 1) AS (d_month_seq + 1)#49] -(71) HashAggregate [codegen id : 1] +(69) CometHashAggregate Input [1]: [(d_month_seq + 1)#49] Keys [1]: [(d_month_seq + 1)#49] Functions: [] -Aggregate Attributes: [] -Results [1]: [(d_month_seq + 1)#49] -(72) Exchange +(70) ColumnarToRow [codegen id : 1] +Input [1]: [(d_month_seq + 1)#49] + +(71) Exchange Input [1]: [(d_month_seq + 1)#49] Arguments: hashpartitioning((d_month_seq + 1)#49, 5), ENSURE_REQUIREMENTS, [plan_id=11] -(73) HashAggregate [codegen id : 2] +(72) HashAggregate [codegen id : 2] Input [1]: [(d_month_seq + 1)#49] Keys [1]: [(d_month_seq + 1)#49] Functions: [] Aggregate Attributes: [] Results [1]: [(d_month_seq + 1)#49] -Subquery:7 Hosting operator id = 62 Hosting Expression = Subquery scalar-subquery#43, [id=#45] -* HashAggregate (80) -+- Exchange (79) - +- * HashAggregate (78) - +- * Project (77) - +- * Filter (76) - +- * ColumnarToRow (75) - +- CometScan parquet spark_catalog.default.date_dim (74) +Subquery:7 Hosting operator id = 61 Hosting Expression = Subquery scalar-subquery#43, [id=#45] +* HashAggregate (79) ++- Exchange (78) + +- * ColumnarToRow (77) + +- CometHashAggregate (76) + +- CometProject (75) + +- CometFilter (74) + +- CometScan parquet spark_catalog.default.date_dim (73) -(74) Scan parquet spark_catalog.default.date_dim +(73) Scan parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#50, d_year#51, d_moy#52] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)] ReadSchema: struct -(75) ColumnarToRow [codegen id : 1] -Input [3]: [d_month_seq#50, d_year#51, d_moy#52] - -(76) Filter [codegen id : 1] +(74) CometFilter Input [3]: [d_month_seq#50, d_year#51, d_moy#52] Condition : (((isnotnull(d_year#51) AND isnotnull(d_moy#52)) AND (d_year#51 = 1998)) AND (d_moy#52 = 12)) -(77) Project [codegen id : 1] -Output [1]: [(d_month_seq#50 + 3) AS (d_month_seq + 3)#53] +(75) CometProject Input [3]: [d_month_seq#50, d_year#51, d_moy#52] +Arguments: [(d_month_seq + 3)#53], [(d_month_seq#50 + 3) AS (d_month_seq + 3)#53] -(78) HashAggregate [codegen id : 1] +(76) CometHashAggregate Input [1]: [(d_month_seq + 3)#53] Keys [1]: [(d_month_seq + 3)#53] Functions: [] -Aggregate Attributes: [] -Results [1]: [(d_month_seq + 3)#53] -(79) Exchange +(77) ColumnarToRow [codegen id : 1] +Input [1]: [(d_month_seq + 3)#53] + +(78) Exchange Input [1]: [(d_month_seq + 3)#53] Arguments: hashpartitioning((d_month_seq + 3)#53, 5), ENSURE_REQUIREMENTS, [plan_id=12] -(80) HashAggregate [codegen id : 2] +(79) HashAggregate [codegen id : 2] Input [1]: [(d_month_seq + 3)#53] Keys [1]: [(d_month_seq + 3)#53] Functions: [] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/simplified.txt index fa7a009d3..30ba4b743 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/simplified.txt @@ -1,14 +1,14 @@ TakeOrderedAndProject [segment,num_customers,segment_base] - WholeStageCodegen (13) + WholeStageCodegen (11) HashAggregate [segment,count] [count(1),num_customers,segment_base,count] InputAdapter Exchange [segment] #1 - WholeStageCodegen (12) + WholeStageCodegen (10) HashAggregate [segment] [count,count] HashAggregate [c_customer_sk,sum] [sum(UnscaledValue(ss_ext_sales_price)),segment,sum] InputAdapter Exchange [c_customer_sk] #2 - WholeStageCodegen (11) + WholeStageCodegen (9) HashAggregate [c_customer_sk,ss_ext_sales_price] [sum,sum] Project [c_customer_sk,ss_ext_sales_price] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] @@ -21,7 +21,7 @@ TakeOrderedAndProject [segment,num_customers,segment_base] HashAggregate [c_customer_sk,c_current_addr_sk] InputAdapter Exchange [c_customer_sk,c_current_addr_sk] #3 - WholeStageCodegen (6) + WholeStageCodegen (4) HashAggregate [c_customer_sk,c_current_addr_sk] Project [c_customer_sk,c_current_addr_sk] BroadcastHashJoin [customer_sk,c_customer_sk] @@ -29,62 +29,57 @@ TakeOrderedAndProject [segment,num_customers,segment_base] BroadcastHashJoin [sold_date_sk,d_date_sk] Project [sold_date_sk,customer_sk] BroadcastHashJoin [item_sk,i_item_sk] - InputAdapter - Union - WholeStageCodegen (1) - Project [cs_sold_date_sk,cs_bill_customer_sk,cs_item_sk] - Filter [cs_item_sk,cs_bill_customer_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (2) - Project [ws_sold_date_sk,ws_bill_customer_sk,ws_item_sk] - Filter [ws_item_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 + ColumnarToRow + InputAdapter + CometUnion + CometProject [cs_sold_date_sk,cs_bill_customer_sk,cs_item_sk] [sold_date_sk,customer_sk,item_sk] + CometFilter [cs_item_sk,cs_bill_customer_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_moy,d_year,d_date_sk] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometProject [ws_sold_date_sk,ws_bill_customer_sk,ws_item_sk] [sold_date_sk,customer_sk,item_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 InputAdapter BroadcastExchange #5 - WholeStageCodegen (3) - Project [i_item_sk] - Filter [i_category,i_class,i_item_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [i_item_sk] + CometFilter [i_category,i_class,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] InputAdapter ReusedExchange [d_date_sk] #4 InputAdapter BroadcastExchange #6 - WholeStageCodegen (5) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (3) + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_current_addr_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] InputAdapter BroadcastExchange #7 - WholeStageCodegen (7) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (5) + ColumnarToRow + InputAdapter + CometFilter [ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ReusedSubquery [(d_month_seq + 1)] #3 - ReusedSubquery [(d_month_seq + 3)] #4 - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_month_seq,d_date_sk] + ReusedSubquery [(d_month_seq + 1)] #3 + ReusedSubquery [(d_month_seq + 3)] #4 CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] Subquery #3 WholeStageCodegen (2) @@ -92,11 +87,11 @@ TakeOrderedAndProject [segment,num_customers,segment_base] InputAdapter Exchange [(d_month_seq + 1)] #9 WholeStageCodegen (1) - HashAggregate [(d_month_seq + 1)] - Project [d_month_seq] - Filter [d_year,d_moy] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometHashAggregate [(d_month_seq + 1)] + CometProject [d_month_seq] [(d_month_seq + 1)] + CometFilter [d_year,d_moy] CometScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] Subquery #4 WholeStageCodegen (2) @@ -104,25 +99,25 @@ TakeOrderedAndProject [segment,num_customers,segment_base] InputAdapter Exchange [(d_month_seq + 3)] #10 WholeStageCodegen (1) - HashAggregate [(d_month_seq + 3)] - Project [d_month_seq] - Filter [d_year,d_moy] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometHashAggregate [(d_month_seq + 3)] + CometProject [d_month_seq] [(d_month_seq + 3)] + CometFilter [d_year,d_moy] CometScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] InputAdapter BroadcastExchange #11 - WholeStageCodegen (8) - Filter [ca_address_sk,ca_county,ca_state] - ColumnarToRow - InputAdapter + WholeStageCodegen (6) + ColumnarToRow + InputAdapter + CometFilter [ca_address_sk,ca_county,ca_state] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] InputAdapter BroadcastExchange #12 - WholeStageCodegen (9) - Filter [s_county,s_state] - ColumnarToRow - InputAdapter + WholeStageCodegen (7) + ColumnarToRow + InputAdapter + CometFilter [s_county,s_state] CometScan parquet spark_catalog.default.store [s_county,s_state] InputAdapter ReusedExchange [d_date_sk] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/explain.txt index 0deabf94b..3d1d689bc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/explain.txt @@ -7,18 +7,18 @@ TakeOrderedAndProject (21) +- * BroadcastHashJoin Inner BuildRight (16) :- * Project (10) : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) + : :- * ColumnarToRow (4) + : : +- CometProject (3) + : : +- CometFilter (2) : : +- CometScan parquet spark_catalog.default.date_dim (1) : +- BroadcastExchange (8) - : +- * Filter (7) - : +- * ColumnarToRow (6) + : +- * ColumnarToRow (7) + : +- CometFilter (6) : +- CometScan parquet spark_catalog.default.store_sales (5) +- BroadcastExchange (15) - +- * Project (14) - +- * Filter (13) - +- * ColumnarToRow (12) + +- * ColumnarToRow (14) + +- CometProject (13) + +- CometFilter (12) +- CometScan parquet spark_catalog.default.item (11) @@ -29,16 +29,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 1999)) AND isnotnull(d_date_sk#1)) -(4) Project [codegen id : 3] -Output [1]: [d_date_sk#1] +(3) CometProject Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Arguments: [d_date_sk#1], [d_date_sk#1] + +(4) ColumnarToRow [codegen id : 3] +Input [1]: [d_date_sk#1] (5) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] @@ -48,12 +48,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] +(6) CometFilter Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Condition : isnotnull(ss_item_sk#4) -(7) Filter [codegen id : 1] +(7) ColumnarToRow [codegen id : 1] Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_item_sk#4) (8) BroadcastExchange Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] @@ -76,16 +76,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,28), IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] - -(13) Filter [codegen id : 2] +(12) CometFilter Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 28)) AND isnotnull(i_item_sk#7)) -(14) Project [codegen id : 2] -Output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] +(13) CometProject Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9], [i_item_sk#7, i_brand_id#8, i_brand#9] + +(14) ColumnarToRow [codegen id : 2] +Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] (15) BroadcastExchange Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/simplified.txt index 911fe96b3..7a0fe8863 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/simplified.txt @@ -9,23 +9,23 @@ TakeOrderedAndProject [ext_price,brand_id,brand] BroadcastHashJoin [ss_item_sk,i_item_sk] Project [ss_item_sk,ss_ext_sales_price] BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_moy,d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #2 WholeStageCodegen (1) - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Project [i_item_sk,i_brand_id,i_brand] - Filter [i_manager_id,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_item_sk,i_brand_id,i_brand] + CometFilter [i_manager_id,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/explain.txt index fef08c190..bbed7eea6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/explain.txt @@ -13,24 +13,24 @@ TakeOrderedAndProject (63) : : +- * BroadcastHashJoin Inner BuildRight (12) : : :- * Project (6) : : : +- * BroadcastHashJoin Inner BuildRight (5) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- ReusedExchange (4) : : +- BroadcastExchange (11) - : : +- * Project (10) - : : +- * Filter (9) - : : +- * ColumnarToRow (8) + : : +- * ColumnarToRow (10) + : : +- CometProject (9) + : : +- CometFilter (8) : : +- CometScan parquet spark_catalog.default.customer_address (7) : +- BroadcastExchange (23) : +- * BroadcastHashJoin LeftSemi BuildRight (22) - : :- * Filter (16) - : : +- * ColumnarToRow (15) + : :- * ColumnarToRow (16) + : : +- CometFilter (15) : : +- CometScan parquet spark_catalog.default.item (14) : +- BroadcastExchange (21) - : +- * Project (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) + : +- * ColumnarToRow (20) + : +- CometProject (19) + : +- CometFilter (18) : +- CometScan parquet spark_catalog.default.item (17) :- * HashAggregate (43) : +- Exchange (42) @@ -41,8 +41,8 @@ TakeOrderedAndProject (63) : : +- * BroadcastHashJoin Inner BuildRight (36) : : :- * Project (34) : : : +- * BroadcastHashJoin Inner BuildRight (33) - : : : :- * Filter (31) - : : : : +- * ColumnarToRow (30) + : : : :- * ColumnarToRow (31) + : : : : +- CometFilter (30) : : : : +- CometScan parquet spark_catalog.default.catalog_sales (29) : : : +- ReusedExchange (32) : : +- ReusedExchange (35) @@ -56,8 +56,8 @@ TakeOrderedAndProject (63) : +- * BroadcastHashJoin Inner BuildRight (51) : :- * Project (49) : : +- * BroadcastHashJoin Inner BuildRight (48) - : : :- * Filter (46) - : : : +- * ColumnarToRow (45) + : : :- * ColumnarToRow (46) + : : : +- CometFilter (45) : : : +- CometScan parquet spark_catalog.default.web_sales (44) : : +- ReusedExchange (47) : +- ReusedExchange (50) @@ -72,12 +72,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) -(3) Filter [codegen id : 5] +(3) ColumnarToRow [codegen id : 5] Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) (4) ReusedExchange [Reuses operator id: 68] Output [1]: [d_date_sk#6] @@ -99,16 +99,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] -Input [2]: [ca_address_sk#7, ca_gmt_offset#8] - -(9) Filter [codegen id : 2] +(8) CometFilter Input [2]: [ca_address_sk#7, ca_gmt_offset#8] Condition : ((isnotnull(ca_gmt_offset#8) AND (ca_gmt_offset#8 = -5.00)) AND isnotnull(ca_address_sk#7)) -(10) Project [codegen id : 2] -Output [1]: [ca_address_sk#7] +(9) CometProject Input [2]: [ca_address_sk#7, ca_gmt_offset#8] +Arguments: [ca_address_sk#7], [ca_address_sk#7] + +(10) ColumnarToRow [codegen id : 2] +Input [1]: [ca_address_sk#7] (11) BroadcastExchange Input [1]: [ca_address_sk#7] @@ -131,12 +131,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 4] +(15) CometFilter Input [2]: [i_item_sk#9, i_item_id#10] +Condition : isnotnull(i_item_sk#9) -(16) Filter [codegen id : 4] +(16) ColumnarToRow [codegen id : 4] Input [2]: [i_item_sk#9, i_item_id#10] -Condition : isnotnull(i_item_sk#9) (17) Scan parquet spark_catalog.default.item Output [2]: [i_item_id#11, i_color#12] @@ -145,16 +145,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [In(i_color, [blanched ,burnished ,slate ])] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [2]: [i_item_id#11, i_color#12] - -(19) Filter [codegen id : 3] +(18) CometFilter Input [2]: [i_item_id#11, i_color#12] Condition : i_color#12 IN (slate ,blanched ,burnished ) -(20) Project [codegen id : 3] -Output [1]: [i_item_id#11] +(19) CometProject Input [2]: [i_item_id#11, i_color#12] +Arguments: [i_item_id#11], [i_item_id#11] + +(20) ColumnarToRow [codegen id : 3] +Input [1]: [i_item_id#11] (21) BroadcastExchange Input [1]: [i_item_id#11] @@ -206,12 +206,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#20), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 11] +(30) CometFilter Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Condition : (isnotnull(cs_bill_addr_sk#17) AND isnotnull(cs_item_sk#18)) -(31) Filter [codegen id : 11] +(31) ColumnarToRow [codegen id : 11] Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] -Condition : (isnotnull(cs_bill_addr_sk#17) AND isnotnull(cs_item_sk#18)) (32) ReusedExchange [Reuses operator id: 68] Output [1]: [d_date_sk#22] @@ -278,12 +278,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#33), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 17] +(45) CometFilter Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] +Condition : (isnotnull(ws_bill_addr_sk#31) AND isnotnull(ws_item_sk#30)) -(46) Filter [codegen id : 17] +(46) ColumnarToRow [codegen id : 17] Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] -Condition : (isnotnull(ws_bill_addr_sk#31) AND isnotnull(ws_item_sk#30)) (47) ReusedExchange [Reuses operator id: 68] Output [1]: [d_date_sk#35] @@ -370,9 +370,9 @@ Arguments: 100, [total_sales#48 ASC NULLS FIRST], [i_item_id#10, total_sales#48] Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 BroadcastExchange (68) -+- * Project (67) - +- * Filter (66) - +- * ColumnarToRow (65) ++- * ColumnarToRow (67) + +- CometProject (66) + +- CometFilter (65) +- CometScan parquet spark_catalog.default.date_dim (64) @@ -383,16 +383,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(65) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#49, d_moy#50] - -(66) Filter [codegen id : 1] +(65) CometFilter Input [3]: [d_date_sk#6, d_year#49, d_moy#50] Condition : ((((isnotnull(d_year#49) AND isnotnull(d_moy#50)) AND (d_year#49 = 2001)) AND (d_moy#50 = 2)) AND isnotnull(d_date_sk#6)) -(67) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(66) CometProject Input [3]: [d_date_sk#6, d_year#49, d_moy#50] +Arguments: [d_date_sk#6], [d_date_sk#6] + +(67) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#6] (68) BroadcastExchange Input [1]: [d_date_sk#6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/simplified.txt index 42b2cbfd3..f781ed1f7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/simplified.txt @@ -19,43 +19,43 @@ TakeOrderedAndProject [total_sales,i_item_id] BroadcastHashJoin [ss_addr_sk,ca_address_sk] Project [ss_item_sk,ss_addr_sk,ss_ext_sales_price] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk,ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_addr_sk,ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_moy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter ReusedExchange [d_date_sk] #3 InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - Project [ca_address_sk] - Filter [ca_gmt_offset,ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_gmt_offset,ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] InputAdapter BroadcastExchange #5 WholeStageCodegen (4) BroadcastHashJoin [i_item_id,i_item_id] - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] InputAdapter BroadcastExchange #6 WholeStageCodegen (3) - Project [i_item_id] - Filter [i_color] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_item_id] + CometFilter [i_color] CometScan parquet spark_catalog.default.item [i_item_id,i_color] WholeStageCodegen (12) HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] @@ -69,9 +69,9 @@ TakeOrderedAndProject [total_sales,i_item_id] BroadcastHashJoin [cs_bill_addr_sk,ca_address_sk] Project [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_bill_addr_sk,cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_bill_addr_sk,cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -92,9 +92,9 @@ TakeOrderedAndProject [total_sales,i_item_id] BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] Project [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk,ws_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_bill_addr_sk,ws_item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/explain.txt index e7e6e9367..8746c36d6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/explain.txt @@ -20,17 +20,17 @@ TakeOrderedAndProject (45) : : : +- * BroadcastHashJoin Inner BuildRight (11) : : : :- * Project (9) : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.item (1) : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) + : : : : +- * ColumnarToRow (6) + : : : : +- CometFilter (5) : : : : +- CometScan parquet spark_catalog.default.catalog_sales (4) : : : +- ReusedExchange (10) : : +- BroadcastExchange (16) - : : +- * Filter (15) - : : +- * ColumnarToRow (14) + : : +- * ColumnarToRow (15) + : : +- CometFilter (14) : : +- CometScan parquet spark_catalog.default.call_center (13) : +- BroadcastExchange (35) : +- * Project (34) @@ -53,12 +53,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) -(3) Filter [codegen id : 4] +(3) ColumnarToRow [codegen id : 4] Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) (4) Scan parquet spark_catalog.default.catalog_sales Output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] @@ -68,12 +68,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#7), dynamicpruningexpression(cs_sol PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] +Condition : (isnotnull(cs_item_sk#5) AND isnotnull(cs_call_center_sk#4)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Condition : (isnotnull(cs_item_sk#5) AND isnotnull(cs_call_center_sk#4)) (7) BroadcastExchange Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] @@ -109,12 +109,12 @@ Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] +(14) CometFilter Input [2]: [cc_call_center_sk#12, cc_name#13] +Condition : (isnotnull(cc_call_center_sk#12) AND isnotnull(cc_name#13)) -(15) Filter [codegen id : 3] +(15) ColumnarToRow [codegen id : 3] Input [2]: [cc_call_center_sk#12, cc_name#13] -Condition : (isnotnull(cc_call_center_sk#12) AND isnotnull(cc_name#13)) (16) BroadcastExchange Input [2]: [cc_call_center_sk#12, cc_name#13] @@ -253,8 +253,8 @@ Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, cc_name# Subquery:1 Hosting operator id = 4 Hosting Expression = cs_sold_date_sk#7 IN dynamicpruning#8 BroadcastExchange (49) -+- * Filter (48) - +- * ColumnarToRow (47) ++- * ColumnarToRow (48) + +- CometFilter (47) +- CometScan parquet spark_catalog.default.date_dim (46) @@ -265,12 +265,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct -(47) ColumnarToRow [codegen id : 1] +(47) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_moy#11] +Condition : ((((d_year#10 = 1999) OR ((d_year#10 = 1998) AND (d_moy#11 = 12))) OR ((d_year#10 = 2000) AND (d_moy#11 = 1))) AND isnotnull(d_date_sk#9)) -(48) Filter [codegen id : 1] +(48) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((d_year#10 = 1999) OR ((d_year#10 = 1998) AND (d_moy#11 = 12))) OR ((d_year#10 = 2000) AND (d_moy#11 = 1))) AND isnotnull(d_date_sk#9)) (49) BroadcastExchange Input [3]: [d_date_sk#9, d_year#10, d_moy#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/simplified.txt index 84c5c5f83..3bc01343a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/simplified.txt @@ -28,32 +28,32 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] BroadcastHashJoin [i_item_sk,cs_item_sk] - Filter [i_item_sk,i_category,i_brand] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk,i_category,i_brand] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Filter [cs_item_sk,cs_call_center_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_item_sk,cs_call_center_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_year,d_moy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter ReusedExchange [d_date_sk,d_year,d_moy] #4 InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - Filter [cc_call_center_sk,cc_name] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cc_call_center_sk,cc_name] CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/explain.txt index bd7b775d6..35fedb6d3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/explain.txt @@ -12,12 +12,12 @@ TakeOrderedAndProject (49) : : +- * BroadcastHashJoin Inner BuildRight (11) : : :- * Project (9) : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) + : : : +- * ColumnarToRow (6) + : : : +- CometFilter (5) : : : +- CometScan parquet spark_catalog.default.item (4) : : +- ReusedExchange (10) : +- BroadcastExchange (30) @@ -29,8 +29,8 @@ TakeOrderedAndProject (49) : +- * BroadcastHashJoin Inner BuildRight (24) : :- * Project (22) : : +- * BroadcastHashJoin Inner BuildRight (21) - : : :- * Filter (19) - : : : +- * ColumnarToRow (18) + : : :- * ColumnarToRow (19) + : : : +- CometFilter (18) : : : +- CometScan parquet spark_catalog.default.catalog_sales (17) : : +- ReusedExchange (20) : +- ReusedExchange (23) @@ -43,8 +43,8 @@ TakeOrderedAndProject (49) +- * BroadcastHashJoin Inner BuildRight (40) :- * Project (38) : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Filter (35) - : : +- * ColumnarToRow (34) + : :- * ColumnarToRow (35) + : : +- CometFilter (34) : : +- CometScan parquet spark_catalog.default.web_sales (33) : +- ReusedExchange (36) +- ReusedExchange (39) @@ -58,12 +58,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Condition : isnotnull(ss_item_sk#1) -(3) Filter [codegen id : 4] +(3) ColumnarToRow [codegen id : 4] Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_item_sk#1) (4) Scan parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_item_id#6] @@ -72,12 +72,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_item_id)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [2]: [i_item_sk#5, i_item_id#6] +Condition : (isnotnull(i_item_sk#5) AND isnotnull(i_item_id#6)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [2]: [i_item_sk#5, i_item_id#6] -Condition : (isnotnull(i_item_sk#5) AND isnotnull(i_item_id#6)) (7) BroadcastExchange Input [2]: [i_item_sk#5, i_item_id#6] @@ -136,12 +136,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#15), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 8] +(18) CometFilter Input [3]: [cs_item_sk#13, cs_ext_sales_price#14, cs_sold_date_sk#15] +Condition : isnotnull(cs_item_sk#13) -(19) Filter [codegen id : 8] +(19) ColumnarToRow [codegen id : 8] Input [3]: [cs_item_sk#13, cs_ext_sales_price#14, cs_sold_date_sk#15] -Condition : isnotnull(cs_item_sk#13) (20) ReusedExchange [Reuses operator id: 7] Output [2]: [i_item_sk#17, i_item_id#18] @@ -213,12 +213,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#27), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 13] +(34) CometFilter Input [3]: [ws_item_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] +Condition : isnotnull(ws_item_sk#25) -(35) Filter [codegen id : 13] +(35) ColumnarToRow [codegen id : 13] Input [3]: [ws_item_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] -Condition : isnotnull(ws_item_sk#25) (36) ReusedExchange [Reuses operator id: 7] Output [2]: [i_item_sk#29, i_item_id#30] @@ -292,13 +292,13 @@ Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dyn BroadcastExchange (60) +- * Project (59) +- * BroadcastHashJoin LeftSemi BuildRight (58) - :- * Filter (52) - : +- * ColumnarToRow (51) + :- * ColumnarToRow (52) + : +- CometFilter (51) : +- CometScan parquet spark_catalog.default.date_dim (50) +- BroadcastExchange (57) - +- * Project (56) - +- * Filter (55) - +- * ColumnarToRow (54) + +- * ColumnarToRow (56) + +- CometProject (55) + +- CometFilter (54) +- CometScan parquet spark_catalog.default.date_dim (53) @@ -309,12 +309,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(51) ColumnarToRow [codegen id : 2] +(51) CometFilter Input [2]: [d_date_sk#7, d_date#41] +Condition : isnotnull(d_date_sk#7) -(52) Filter [codegen id : 2] +(52) ColumnarToRow [codegen id : 2] Input [2]: [d_date_sk#7, d_date#41] -Condition : isnotnull(d_date_sk#7) (53) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date#42, d_week_seq#43] @@ -323,16 +323,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#44)] ReadSchema: struct -(54) ColumnarToRow [codegen id : 1] -Input [2]: [d_date#42, d_week_seq#43] - -(55) Filter [codegen id : 1] +(54) CometFilter Input [2]: [d_date#42, d_week_seq#43] Condition : (isnotnull(d_week_seq#43) AND (d_week_seq#43 = ReusedSubquery Subquery scalar-subquery#44, [id=#45])) -(56) Project [codegen id : 1] -Output [1]: [d_date#42] +(55) CometProject Input [2]: [d_date#42, d_week_seq#43] +Arguments: [d_date#42], [d_date#42] + +(56) ColumnarToRow [codegen id : 1] +Input [1]: [d_date#42] (57) BroadcastExchange Input [1]: [d_date#42] @@ -352,12 +352,12 @@ Input [2]: [d_date_sk#7, d_date#41] Input [1]: [d_date_sk#7] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] -Subquery:2 Hosting operator id = 55 Hosting Expression = ReusedSubquery Subquery scalar-subquery#44, [id=#45] +Subquery:2 Hosting operator id = 54 Hosting Expression = ReusedSubquery Subquery scalar-subquery#44, [id=#45] Subquery:3 Hosting operator id = 53 Hosting Expression = Subquery scalar-subquery#44, [id=#45] -* Project (64) -+- * Filter (63) - +- * ColumnarToRow (62) +* ColumnarToRow (64) ++- CometProject (63) + +- CometFilter (62) +- CometScan parquet spark_catalog.default.date_dim (61) @@ -368,16 +368,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), EqualTo(d_date,2000-01-03)] ReadSchema: struct -(62) ColumnarToRow [codegen id : 1] -Input [2]: [d_date#46, d_week_seq#47] - -(63) Filter [codegen id : 1] +(62) CometFilter Input [2]: [d_date#46, d_week_seq#47] Condition : (isnotnull(d_date#46) AND (d_date#46 = 2000-01-03)) -(64) Project [codegen id : 1] -Output [1]: [d_week_seq#47] +(63) CometProject Input [2]: [d_date#46, d_week_seq#47] +Arguments: [d_week_seq#47], [d_week_seq#47] + +(64) ColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#47] Subquery:4 Hosting operator id = 17 Hosting Expression = cs_sold_date_sk#15 IN dynamicpruning#4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/simplified.txt index 007ac0887..c7ed479ad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/simplified.txt @@ -14,41 +14,41 @@ TakeOrderedAndProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_ext_sales_price,ss_sold_date_sk,i_item_id] BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (2) Project [d_date_sk] BroadcastHashJoin [d_date,d_date] - Filter [d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date] - Filter [d_week_seq] - ReusedSubquery [d_week_seq] #2 - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date] + CometFilter [d_week_seq] + ReusedSubquery [d_week_seq] #2 CometScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] Subquery #2 WholeStageCodegen (1) - Project [d_week_seq] - Filter [d_date] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_date] CometScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] InputAdapter BroadcastExchange #4 WholeStageCodegen (1) - Filter [i_item_sk,i_item_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk,i_item_id] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] InputAdapter ReusedExchange [d_date_sk] #2 @@ -65,9 +65,9 @@ TakeOrderedAndProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [cs_ext_sales_price,cs_sold_date_sk,i_item_id] BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -87,9 +87,9 @@ TakeOrderedAndProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [ws_ext_sales_price,ws_sold_date_sk,i_item_id] BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/explain.txt index aaea9f38f..023c54bdd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/explain.txt @@ -11,21 +11,21 @@ TakeOrderedAndProject (44) : : : +- * HashAggregate (10) : : : +- * Project (9) : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) + : : : +- * ColumnarToRow (6) + : : : +- CometFilter (5) : : : +- CometScan parquet spark_catalog.default.date_dim (4) : : +- BroadcastExchange (16) - : : +- * Filter (15) - : : +- * ColumnarToRow (14) + : : +- * ColumnarToRow (15) + : : +- CometFilter (14) : : +- CometScan parquet spark_catalog.default.store (13) : +- BroadcastExchange (23) - : +- * Project (22) - : +- * Filter (21) - : +- * ColumnarToRow (20) + : +- * ColumnarToRow (22) + : +- CometProject (21) + : +- CometFilter (20) : +- CometScan parquet spark_catalog.default.date_dim (19) +- BroadcastExchange (41) +- * Project (40) @@ -35,13 +35,13 @@ TakeOrderedAndProject (44) : :- * HashAggregate (27) : : +- ReusedExchange (26) : +- BroadcastExchange (31) - : +- * Filter (30) - : +- * ColumnarToRow (29) + : +- * ColumnarToRow (30) + : +- CometFilter (29) : +- CometScan parquet spark_catalog.default.store (28) +- BroadcastExchange (38) - +- * Project (37) - +- * Filter (36) - +- * ColumnarToRow (35) + +- * ColumnarToRow (37) + +- CometProject (36) + +- CometFilter (35) +- CometScan parquet spark_catalog.default.date_dim (34) @@ -53,12 +53,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 2] +(2) CometFilter Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] +Condition : isnotnull(ss_store_sk#1) -(3) Filter [codegen id : 2] +(3) ColumnarToRow [codegen id : 2] Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_store_sk#1) (4) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] @@ -67,12 +67,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk), IsNotNull(d_week_seq)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] +Condition : (isnotnull(d_date_sk#4) AND isnotnull(d_week_seq#5)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] -Condition : (isnotnull(d_date_sk#4) AND isnotnull(d_week_seq#5)) (7) BroadcastExchange Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] @@ -113,12 +113,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_id)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] +(14) CometFilter Input [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] +Condition : (isnotnull(s_store_sk#35) AND isnotnull(s_store_id#36)) -(15) Filter [codegen id : 3] +(15) ColumnarToRow [codegen id : 3] Input [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] -Condition : (isnotnull(s_store_sk#35) AND isnotnull(s_store_id#36)) (16) BroadcastExchange Input [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] @@ -141,16 +141,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_week_seq)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 4] -Input [2]: [d_month_seq#38, d_week_seq#39] - -(21) Filter [codegen id : 4] +(20) CometFilter Input [2]: [d_month_seq#38, d_week_seq#39] Condition : (((isnotnull(d_month_seq#38) AND (d_month_seq#38 >= 1212)) AND (d_month_seq#38 <= 1223)) AND isnotnull(d_week_seq#39)) -(22) Project [codegen id : 4] -Output [1]: [d_week_seq#39] +(21) CometProject Input [2]: [d_month_seq#38, d_week_seq#39] +Arguments: [d_week_seq#39], [d_week_seq#39] + +(22) ColumnarToRow [codegen id : 4] +Input [1]: [d_week_seq#39] (23) BroadcastExchange Input [1]: [d_week_seq#39] @@ -183,12 +183,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_id)] ReadSchema: struct -(29) ColumnarToRow [codegen id : 7] +(29) CometFilter Input [2]: [s_store_sk#68, s_store_id#69] +Condition : (isnotnull(s_store_sk#68) AND isnotnull(s_store_id#69)) -(30) Filter [codegen id : 7] +(30) ColumnarToRow [codegen id : 7] Input [2]: [s_store_sk#68, s_store_id#69] -Condition : (isnotnull(s_store_sk#68) AND isnotnull(s_store_id#69)) (31) BroadcastExchange Input [2]: [s_store_sk#68, s_store_id#69] @@ -211,16 +211,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1224), LessThanOrEqual(d_month_seq,1235), IsNotNull(d_week_seq)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 8] -Input [2]: [d_month_seq#70, d_week_seq#71] - -(36) Filter [codegen id : 8] +(35) CometFilter Input [2]: [d_month_seq#70, d_week_seq#71] Condition : (((isnotnull(d_month_seq#70) AND (d_month_seq#70 >= 1224)) AND (d_month_seq#70 <= 1235)) AND isnotnull(d_week_seq#71)) -(37) Project [codegen id : 8] -Output [1]: [d_week_seq#71] +(36) CometProject Input [2]: [d_month_seq#70, d_week_seq#71] +Arguments: [d_week_seq#71], [d_week_seq#71] + +(37) ColumnarToRow [codegen id : 8] +Input [1]: [d_week_seq#71] (38) BroadcastExchange Input [1]: [d_week_seq#71] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/simplified.txt index 53d4d0b92..9ad61e946 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/simplified.txt @@ -13,31 +13,31 @@ TakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_s HashAggregate [d_week_seq,ss_store_sk,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] Project [ss_store_sk,ss_sales_price,d_week_seq,d_day_name] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] InputAdapter BroadcastExchange #2 WholeStageCodegen (1) - Filter [d_date_sk,d_week_seq] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_week_seq] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] InputAdapter BroadcastExchange #3 WholeStageCodegen (3) - Filter [s_store_sk,s_store_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk,s_store_id] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] InputAdapter BroadcastExchange #4 WholeStageCodegen (4) - Project [d_week_seq] - Filter [d_month_seq,d_week_seq] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_month_seq,d_week_seq] CometScan parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] InputAdapter BroadcastExchange #5 @@ -52,15 +52,15 @@ TakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_s InputAdapter BroadcastExchange #6 WholeStageCodegen (7) - Filter [s_store_sk,s_store_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk,s_store_id] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] InputAdapter BroadcastExchange #7 WholeStageCodegen (8) - Project [d_week_seq] - Filter [d_month_seq,d_week_seq] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_month_seq,d_week_seq] CometScan parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/explain.txt index 6b4318918..f8ae10ebe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/explain.txt @@ -12,31 +12,31 @@ TakeOrderedAndProject (39) : : +- * BroadcastHashJoin Inner BuildRight (14) : : :- * Project (9) : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.customer_address (1) : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) + : : : +- * ColumnarToRow (6) + : : : +- CometFilter (5) : : : +- CometScan parquet spark_catalog.default.customer (4) : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) + : : +- * ColumnarToRow (12) + : : +- CometFilter (11) : : +- CometScan parquet spark_catalog.default.store_sales (10) : +- ReusedExchange (16) +- BroadcastExchange (32) +- * Project (31) +- * BroadcastHashJoin Inner BuildRight (30) - :- * Filter (21) - : +- * ColumnarToRow (20) + :- * ColumnarToRow (21) + : +- CometFilter (20) : +- CometScan parquet spark_catalog.default.item (19) +- BroadcastExchange (29) +- * Filter (28) +- * HashAggregate (27) +- Exchange (26) +- * HashAggregate (25) - +- * Filter (24) - +- * ColumnarToRow (23) + +- * ColumnarToRow (24) + +- CometFilter (23) +- CometScan parquet spark_catalog.default.item (22) @@ -47,12 +47,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 7] +(2) CometFilter Input [2]: [ca_address_sk#1, ca_state#2] +Condition : isnotnull(ca_address_sk#1) -(3) Filter [codegen id : 7] +(3) ColumnarToRow [codegen id : 7] Input [2]: [ca_address_sk#1, ca_state#2] -Condition : isnotnull(ca_address_sk#1) (4) Scan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#3, c_current_addr_sk#4] @@ -61,12 +61,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [2]: [c_customer_sk#3, c_current_addr_sk#4] +Condition : (isnotnull(c_current_addr_sk#4) AND isnotnull(c_customer_sk#3)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Condition : (isnotnull(c_current_addr_sk#4) AND isnotnull(c_customer_sk#3)) (7) BroadcastExchange Input [2]: [c_customer_sk#3, c_current_addr_sk#4] @@ -90,12 +90,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] +(11) CometFilter Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] +Condition : (isnotnull(ss_customer_sk#6) AND isnotnull(ss_item_sk#5)) -(12) Filter [codegen id : 2] +(12) ColumnarToRow [codegen id : 2] Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_customer_sk#6) AND isnotnull(ss_item_sk#5)) (13) BroadcastExchange Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] @@ -131,12 +131,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_category), IsNotNull(i_item_sk)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 6] +(20) CometFilter Input [3]: [i_item_sk#10, i_current_price#11, i_category#12] +Condition : ((isnotnull(i_current_price#11) AND isnotnull(i_category#12)) AND isnotnull(i_item_sk#10)) -(21) Filter [codegen id : 6] +(21) ColumnarToRow [codegen id : 6] Input [3]: [i_item_sk#10, i_current_price#11, i_category#12] -Condition : ((isnotnull(i_current_price#11) AND isnotnull(i_category#12)) AND isnotnull(i_item_sk#10)) (22) Scan parquet spark_catalog.default.item Output [2]: [i_current_price#13, i_category#14] @@ -145,12 +145,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_category)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 4] +(23) CometFilter Input [2]: [i_current_price#13, i_category#14] +Condition : isnotnull(i_category#14) -(24) Filter [codegen id : 4] +(24) ColumnarToRow [codegen id : 4] Input [2]: [i_current_price#13, i_category#14] -Condition : isnotnull(i_category#14) (25) HashAggregate [codegen id : 4] Input [2]: [i_current_price#13, i_category#14] @@ -232,9 +232,9 @@ Arguments: 100, [cnt#25 ASC NULLS FIRST], [state#24, cnt#25] Subquery:1 Hosting operator id = 10 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 BroadcastExchange (44) -+- * Project (43) - +- * Filter (42) - +- * ColumnarToRow (41) ++- * ColumnarToRow (43) + +- CometProject (42) + +- CometFilter (41) +- CometScan parquet spark_catalog.default.date_dim (40) @@ -245,30 +245,30 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), EqualTo(d_month_seq,ScalarSubquery#27), IsNotNull(d_date_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#9, d_month_seq#26] - -(42) Filter [codegen id : 1] +(41) CometFilter Input [2]: [d_date_sk#9, d_month_seq#26] Condition : ((isnotnull(d_month_seq#26) AND (d_month_seq#26 = ReusedSubquery Subquery scalar-subquery#27, [id=#28])) AND isnotnull(d_date_sk#9)) -(43) Project [codegen id : 1] -Output [1]: [d_date_sk#9] +(42) CometProject Input [2]: [d_date_sk#9, d_month_seq#26] +Arguments: [d_date_sk#9], [d_date_sk#9] + +(43) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#9] (44) BroadcastExchange Input [1]: [d_date_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] -Subquery:2 Hosting operator id = 42 Hosting Expression = ReusedSubquery Subquery scalar-subquery#27, [id=#28] +Subquery:2 Hosting operator id = 41 Hosting Expression = ReusedSubquery Subquery scalar-subquery#27, [id=#28] Subquery:3 Hosting operator id = 40 Hosting Expression = Subquery scalar-subquery#27, [id=#28] * HashAggregate (51) +- Exchange (50) - +- * HashAggregate (49) - +- * Project (48) - +- * Filter (47) - +- * ColumnarToRow (46) + +- * ColumnarToRow (49) + +- CometHashAggregate (48) + +- CometProject (47) + +- CometFilter (46) +- CometScan parquet spark_catalog.default.date_dim (45) @@ -279,23 +279,21 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)] ReadSchema: struct -(46) ColumnarToRow [codegen id : 1] -Input [3]: [d_month_seq#29, d_year#30, d_moy#31] - -(47) Filter [codegen id : 1] +(46) CometFilter Input [3]: [d_month_seq#29, d_year#30, d_moy#31] Condition : (((isnotnull(d_year#30) AND isnotnull(d_moy#31)) AND (d_year#30 = 2000)) AND (d_moy#31 = 1)) -(48) Project [codegen id : 1] -Output [1]: [d_month_seq#29] +(47) CometProject Input [3]: [d_month_seq#29, d_year#30, d_moy#31] +Arguments: [d_month_seq#29], [d_month_seq#29] -(49) HashAggregate [codegen id : 1] +(48) CometHashAggregate Input [1]: [d_month_seq#29] Keys [1]: [d_month_seq#29] Functions: [] -Aggregate Attributes: [] -Results [1]: [d_month_seq#29] + +(49) ColumnarToRow [codegen id : 1] +Input [1]: [d_month_seq#29] (50) Exchange Input [1]: [d_month_seq#29] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/simplified.txt index 7078a77e4..c2d5a6ce8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/simplified.txt @@ -14,32 +14,32 @@ TakeOrderedAndProject [cnt,state] BroadcastHashJoin [c_customer_sk,ss_customer_sk] Project [ca_state,c_customer_sk] BroadcastHashJoin [ca_address_sk,c_current_addr_sk] - Filter [ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #2 WholeStageCodegen (1) - Filter [c_current_addr_sk,c_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_current_addr_sk,c_customer_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Filter [ss_customer_sk,ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_customer_sk,ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ReusedSubquery [d_month_seq] #2 - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_month_seq,d_date_sk] + ReusedSubquery [d_month_seq] #2 CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] Subquery #2 WholeStageCodegen (2) @@ -47,11 +47,11 @@ TakeOrderedAndProject [cnt,state] InputAdapter Exchange [d_month_seq] #5 WholeStageCodegen (1) - HashAggregate [d_month_seq] - Project [d_month_seq] - Filter [d_year,d_moy] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometHashAggregate [d_month_seq] + CometProject [d_month_seq] + CometFilter [d_year,d_moy] CometScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] InputAdapter ReusedExchange [d_date_sk] #4 @@ -60,9 +60,9 @@ TakeOrderedAndProject [cnt,state] WholeStageCodegen (6) Project [i_item_sk] BroadcastHashJoin [i_category,i_category,i_current_price,avg(i_current_price)] - Filter [i_current_price,i_category,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_current_price,i_category,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] InputAdapter BroadcastExchange #7 @@ -73,7 +73,7 @@ TakeOrderedAndProject [cnt,state] Exchange [i_category] #8 WholeStageCodegen (4) HashAggregate [i_category,i_current_price] [sum,count,sum,count] - Filter [i_category] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_category] CometScan parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/explain.txt index 98fb9f2f6..78f4b27ac 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/explain.txt @@ -13,24 +13,24 @@ TakeOrderedAndProject (63) : : +- * BroadcastHashJoin Inner BuildRight (12) : : :- * Project (6) : : : +- * BroadcastHashJoin Inner BuildRight (5) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- ReusedExchange (4) : : +- BroadcastExchange (11) - : : +- * Project (10) - : : +- * Filter (9) - : : +- * ColumnarToRow (8) + : : +- * ColumnarToRow (10) + : : +- CometProject (9) + : : +- CometFilter (8) : : +- CometScan parquet spark_catalog.default.customer_address (7) : +- BroadcastExchange (23) : +- * BroadcastHashJoin LeftSemi BuildRight (22) - : :- * Filter (16) - : : +- * ColumnarToRow (15) + : :- * ColumnarToRow (16) + : : +- CometFilter (15) : : +- CometScan parquet spark_catalog.default.item (14) : +- BroadcastExchange (21) - : +- * Project (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) + : +- * ColumnarToRow (20) + : +- CometProject (19) + : +- CometFilter (18) : +- CometScan parquet spark_catalog.default.item (17) :- * HashAggregate (43) : +- Exchange (42) @@ -41,8 +41,8 @@ TakeOrderedAndProject (63) : : +- * BroadcastHashJoin Inner BuildRight (36) : : :- * Project (34) : : : +- * BroadcastHashJoin Inner BuildRight (33) - : : : :- * Filter (31) - : : : : +- * ColumnarToRow (30) + : : : :- * ColumnarToRow (31) + : : : : +- CometFilter (30) : : : : +- CometScan parquet spark_catalog.default.catalog_sales (29) : : : +- ReusedExchange (32) : : +- ReusedExchange (35) @@ -56,8 +56,8 @@ TakeOrderedAndProject (63) : +- * BroadcastHashJoin Inner BuildRight (51) : :- * Project (49) : : +- * BroadcastHashJoin Inner BuildRight (48) - : : :- * Filter (46) - : : : +- * ColumnarToRow (45) + : : :- * ColumnarToRow (46) + : : : +- CometFilter (45) : : : +- CometScan parquet spark_catalog.default.web_sales (44) : : +- ReusedExchange (47) : +- ReusedExchange (50) @@ -72,12 +72,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) -(3) Filter [codegen id : 5] +(3) ColumnarToRow [codegen id : 5] Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) (4) ReusedExchange [Reuses operator id: 68] Output [1]: [d_date_sk#6] @@ -99,16 +99,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] -Input [2]: [ca_address_sk#7, ca_gmt_offset#8] - -(9) Filter [codegen id : 2] +(8) CometFilter Input [2]: [ca_address_sk#7, ca_gmt_offset#8] Condition : ((isnotnull(ca_gmt_offset#8) AND (ca_gmt_offset#8 = -5.00)) AND isnotnull(ca_address_sk#7)) -(10) Project [codegen id : 2] -Output [1]: [ca_address_sk#7] +(9) CometProject Input [2]: [ca_address_sk#7, ca_gmt_offset#8] +Arguments: [ca_address_sk#7], [ca_address_sk#7] + +(10) ColumnarToRow [codegen id : 2] +Input [1]: [ca_address_sk#7] (11) BroadcastExchange Input [1]: [ca_address_sk#7] @@ -131,12 +131,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 4] +(15) CometFilter Input [2]: [i_item_sk#9, i_item_id#10] +Condition : isnotnull(i_item_sk#9) -(16) Filter [codegen id : 4] +(16) ColumnarToRow [codegen id : 4] Input [2]: [i_item_sk#9, i_item_id#10] -Condition : isnotnull(i_item_sk#9) (17) Scan parquet spark_catalog.default.item Output [2]: [i_item_id#11, i_category#12] @@ -145,16 +145,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Music )] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [2]: [i_item_id#11, i_category#12] - -(19) Filter [codegen id : 3] +(18) CometFilter Input [2]: [i_item_id#11, i_category#12] Condition : (isnotnull(i_category#12) AND (i_category#12 = Music )) -(20) Project [codegen id : 3] -Output [1]: [i_item_id#11] +(19) CometProject Input [2]: [i_item_id#11, i_category#12] +Arguments: [i_item_id#11], [i_item_id#11] + +(20) ColumnarToRow [codegen id : 3] +Input [1]: [i_item_id#11] (21) BroadcastExchange Input [1]: [i_item_id#11] @@ -206,12 +206,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#20), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 11] +(30) CometFilter Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Condition : (isnotnull(cs_bill_addr_sk#17) AND isnotnull(cs_item_sk#18)) -(31) Filter [codegen id : 11] +(31) ColumnarToRow [codegen id : 11] Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] -Condition : (isnotnull(cs_bill_addr_sk#17) AND isnotnull(cs_item_sk#18)) (32) ReusedExchange [Reuses operator id: 68] Output [1]: [d_date_sk#22] @@ -278,12 +278,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#33), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 17] +(45) CometFilter Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] +Condition : (isnotnull(ws_bill_addr_sk#31) AND isnotnull(ws_item_sk#30)) -(46) Filter [codegen id : 17] +(46) ColumnarToRow [codegen id : 17] Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] -Condition : (isnotnull(ws_bill_addr_sk#31) AND isnotnull(ws_item_sk#30)) (47) ReusedExchange [Reuses operator id: 68] Output [1]: [d_date_sk#35] @@ -370,9 +370,9 @@ Arguments: 100, [i_item_id#10 ASC NULLS FIRST, total_sales#48 ASC NULLS FIRST], Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 BroadcastExchange (68) -+- * Project (67) - +- * Filter (66) - +- * ColumnarToRow (65) ++- * ColumnarToRow (67) + +- CometProject (66) + +- CometFilter (65) +- CometScan parquet spark_catalog.default.date_dim (64) @@ -383,16 +383,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,9), IsNotNull(d_date_sk)] ReadSchema: struct -(65) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#49, d_moy#50] - -(66) Filter [codegen id : 1] +(65) CometFilter Input [3]: [d_date_sk#6, d_year#49, d_moy#50] Condition : ((((isnotnull(d_year#49) AND isnotnull(d_moy#50)) AND (d_year#49 = 1998)) AND (d_moy#50 = 9)) AND isnotnull(d_date_sk#6)) -(67) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(66) CometProject Input [3]: [d_date_sk#6, d_year#49, d_moy#50] +Arguments: [d_date_sk#6], [d_date_sk#6] + +(67) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#6] (68) BroadcastExchange Input [1]: [d_date_sk#6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/simplified.txt index 0af0fd412..b010414a8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/simplified.txt @@ -19,43 +19,43 @@ TakeOrderedAndProject [i_item_id,total_sales] BroadcastHashJoin [ss_addr_sk,ca_address_sk] Project [ss_item_sk,ss_addr_sk,ss_ext_sales_price] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk,ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_addr_sk,ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_moy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter ReusedExchange [d_date_sk] #3 InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - Project [ca_address_sk] - Filter [ca_gmt_offset,ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_gmt_offset,ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] InputAdapter BroadcastExchange #5 WholeStageCodegen (4) BroadcastHashJoin [i_item_id,i_item_id] - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] InputAdapter BroadcastExchange #6 WholeStageCodegen (3) - Project [i_item_id] - Filter [i_category] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_item_id] + CometFilter [i_category] CometScan parquet spark_catalog.default.item [i_item_id,i_category] WholeStageCodegen (12) HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] @@ -69,9 +69,9 @@ TakeOrderedAndProject [i_item_id,total_sales] BroadcastHashJoin [cs_bill_addr_sk,ca_address_sk] Project [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_bill_addr_sk,cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_bill_addr_sk,cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -92,9 +92,9 @@ TakeOrderedAndProject [i_item_id,total_sales] BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] Project [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk,ws_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_bill_addr_sk,ws_item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/explain.txt index b70ffc460..766362167 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/explain.txt @@ -16,33 +16,33 @@ : : : : : +- * BroadcastHashJoin Inner BuildRight (16) : : : : : :- * Project (10) : : : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) + : : : : : : :- * ColumnarToRow (3) + : : : : : : : +- CometFilter (2) : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : : : : +- BroadcastExchange (8) - : : : : : : +- * Project (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) + : : : : : : +- * ColumnarToRow (7) + : : : : : : +- CometProject (6) + : : : : : : +- CometFilter (5) : : : : : : +- CometScan parquet spark_catalog.default.store (4) : : : : : +- BroadcastExchange (15) - : : : : : +- * Project (14) - : : : : : +- * Filter (13) - : : : : : +- * ColumnarToRow (12) + : : : : : +- * ColumnarToRow (14) + : : : : : +- CometProject (13) + : : : : : +- CometFilter (12) : : : : : +- CometScan parquet spark_catalog.default.promotion (11) : : : : +- ReusedExchange (18) : : : +- BroadcastExchange (24) - : : : +- * Filter (23) - : : : +- * ColumnarToRow (22) + : : : +- * ColumnarToRow (23) + : : : +- CometFilter (22) : : : +- CometScan parquet spark_catalog.default.customer (21) : : +- BroadcastExchange (31) - : : +- * Project (30) - : : +- * Filter (29) - : : +- * ColumnarToRow (28) + : : +- * ColumnarToRow (30) + : : +- CometProject (29) + : : +- CometFilter (28) : : +- CometScan parquet spark_catalog.default.customer_address (27) : +- BroadcastExchange (38) - : +- * Project (37) - : +- * Filter (36) - : +- * ColumnarToRow (35) + : +- * ColumnarToRow (37) + : +- CometProject (36) + : +- CometFilter (35) : +- CometScan parquet spark_catalog.default.item (34) +- BroadcastExchange (65) +- * HashAggregate (64) @@ -58,8 +58,8 @@ : : : +- * BroadcastHashJoin Inner BuildRight (51) : : : :- * Project (49) : : : : +- * BroadcastHashJoin Inner BuildRight (48) - : : : : :- * Filter (46) - : : : : : +- * ColumnarToRow (45) + : : : : :- * ColumnarToRow (46) + : : : : : +- CometFilter (45) : : : : : +- CometScan parquet spark_catalog.default.store_sales (44) : : : : +- ReusedExchange (47) : : : +- ReusedExchange (50) @@ -76,12 +76,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 7] +(2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Condition : (((isnotnull(ss_store_sk#3) AND isnotnull(ss_promo_sk#4)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_item_sk#1)) -(3) Filter [codegen id : 7] +(3) ColumnarToRow [codegen id : 7] Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_store_sk#3) AND isnotnull(ss_promo_sk#4)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_item_sk#1)) (4) Scan parquet spark_catalog.default.store Output [2]: [s_store_sk#8, s_gmt_offset#9] @@ -90,16 +90,16 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [s_store_sk#8, s_gmt_offset#9] - -(6) Filter [codegen id : 1] +(5) CometFilter Input [2]: [s_store_sk#8, s_gmt_offset#9] Condition : ((isnotnull(s_gmt_offset#9) AND (s_gmt_offset#9 = -5.00)) AND isnotnull(s_store_sk#8)) -(7) Project [codegen id : 1] -Output [1]: [s_store_sk#8] +(6) CometProject Input [2]: [s_store_sk#8, s_gmt_offset#9] +Arguments: [s_store_sk#8], [s_store_sk#8] + +(7) ColumnarToRow [codegen id : 1] +Input [1]: [s_store_sk#8] (8) BroadcastExchange Input [1]: [s_store_sk#8] @@ -122,16 +122,16 @@ Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [Or(Or(EqualTo(p_channel_dmail,Y),EqualTo(p_channel_email,Y)),EqualTo(p_channel_tv,Y)), IsNotNull(p_promo_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [4]: [p_promo_sk#10, p_channel_dmail#11, p_channel_email#12, p_channel_tv#13] - -(13) Filter [codegen id : 2] +(12) CometFilter Input [4]: [p_promo_sk#10, p_channel_dmail#11, p_channel_email#12, p_channel_tv#13] Condition : ((((p_channel_dmail#11 = Y) OR (p_channel_email#12 = Y)) OR (p_channel_tv#13 = Y)) AND isnotnull(p_promo_sk#10)) -(14) Project [codegen id : 2] -Output [1]: [p_promo_sk#10] +(13) CometProject Input [4]: [p_promo_sk#10, p_channel_dmail#11, p_channel_email#12, p_channel_tv#13] +Arguments: [p_promo_sk#10], [p_promo_sk#10] + +(14) ColumnarToRow [codegen id : 2] +Input [1]: [p_promo_sk#10] (15) BroadcastExchange Input [1]: [p_promo_sk#10] @@ -167,12 +167,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(22) ColumnarToRow [codegen id : 4] +(22) CometFilter Input [2]: [c_customer_sk#15, c_current_addr_sk#16] +Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_current_addr_sk#16)) -(23) Filter [codegen id : 4] +(23) ColumnarToRow [codegen id : 4] Input [2]: [c_customer_sk#15, c_current_addr_sk#16] -Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_current_addr_sk#16)) (24) BroadcastExchange Input [2]: [c_customer_sk#15, c_current_addr_sk#16] @@ -195,16 +195,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 5] -Input [2]: [ca_address_sk#17, ca_gmt_offset#18] - -(29) Filter [codegen id : 5] +(28) CometFilter Input [2]: [ca_address_sk#17, ca_gmt_offset#18] Condition : ((isnotnull(ca_gmt_offset#18) AND (ca_gmt_offset#18 = -5.00)) AND isnotnull(ca_address_sk#17)) -(30) Project [codegen id : 5] -Output [1]: [ca_address_sk#17] +(29) CometProject Input [2]: [ca_address_sk#17, ca_gmt_offset#18] +Arguments: [ca_address_sk#17], [ca_address_sk#17] + +(30) ColumnarToRow [codegen id : 5] +Input [1]: [ca_address_sk#17] (31) BroadcastExchange Input [1]: [ca_address_sk#17] @@ -227,16 +227,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Jewelry ), IsNotNull(i_item_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 6] -Input [2]: [i_item_sk#19, i_category#20] - -(36) Filter [codegen id : 6] +(35) CometFilter Input [2]: [i_item_sk#19, i_category#20] Condition : ((isnotnull(i_category#20) AND (i_category#20 = Jewelry )) AND isnotnull(i_item_sk#19)) -(37) Project [codegen id : 6] -Output [1]: [i_item_sk#19] +(36) CometProject Input [2]: [i_item_sk#19, i_category#20] +Arguments: [i_item_sk#19], [i_item_sk#19] + +(37) ColumnarToRow [codegen id : 6] +Input [1]: [i_item_sk#19] (38) BroadcastExchange Input [1]: [i_item_sk#19] @@ -278,12 +278,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#29), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 13] +(45) CometFilter Input [5]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] +Condition : ((isnotnull(ss_store_sk#27) AND isnotnull(ss_customer_sk#26)) AND isnotnull(ss_item_sk#25)) -(46) Filter [codegen id : 13] +(46) ColumnarToRow [codegen id : 13] Input [5]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] -Condition : ((isnotnull(ss_store_sk#27) AND isnotnull(ss_customer_sk#26)) AND isnotnull(ss_item_sk#25)) (47) ReusedExchange [Reuses operator id: 8] Output [1]: [s_store_sk#31] @@ -384,9 +384,9 @@ Input [2]: [promotions#24, total#40] Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 BroadcastExchange (72) -+- * Project (71) - +- * Filter (70) - +- * ColumnarToRow (69) ++- * ColumnarToRow (71) + +- CometProject (70) + +- CometFilter (69) +- CometScan parquet spark_catalog.default.date_dim (68) @@ -397,16 +397,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct -(69) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#14, d_year#42, d_moy#43] - -(70) Filter [codegen id : 1] +(69) CometFilter Input [3]: [d_date_sk#14, d_year#42, d_moy#43] Condition : ((((isnotnull(d_year#42) AND isnotnull(d_moy#43)) AND (d_year#42 = 1998)) AND (d_moy#43 = 11)) AND isnotnull(d_date_sk#14)) -(71) Project [codegen id : 1] -Output [1]: [d_date_sk#14] +(70) CometProject Input [3]: [d_date_sk#14, d_year#42, d_moy#43] +Arguments: [d_date_sk#14], [d_date_sk#14] + +(71) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#14] (72) BroadcastExchange Input [1]: [d_date_sk#14] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/simplified.txt index 2272a80ba..2c3d07ac6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/simplified.txt @@ -18,58 +18,58 @@ WholeStageCodegen (15) BroadcastHashJoin [ss_promo_sk,p_promo_sk] Project [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk,ss_promo_sk,ss_customer_sk,ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk,ss_promo_sk,ss_customer_sk,ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_moy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Project [s_store_sk] - Filter [s_gmt_offset,s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [s_store_sk] + CometFilter [s_gmt_offset,s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_gmt_offset] InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - Project [p_promo_sk] - Filter [p_channel_dmail,p_channel_email,p_channel_tv,p_promo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [p_promo_sk] + CometFilter [p_channel_dmail,p_channel_email,p_channel_tv,p_promo_sk] CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] InputAdapter ReusedExchange [d_date_sk] #2 InputAdapter BroadcastExchange #5 WholeStageCodegen (4) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_current_addr_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] InputAdapter BroadcastExchange #6 WholeStageCodegen (5) - Project [ca_address_sk] - Filter [ca_gmt_offset,ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_gmt_offset,ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] InputAdapter BroadcastExchange #7 WholeStageCodegen (6) - Project [i_item_sk] - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_item_sk] + CometFilter [i_category,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_category] InputAdapter BroadcastExchange #8 @@ -89,9 +89,9 @@ WholeStageCodegen (15) BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk,ss_customer_sk,ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk,ss_customer_sk,ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62/explain.txt index 55423bcd1..0607d8077 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62/explain.txt @@ -11,25 +11,25 @@ TakeOrderedAndProject (32) : : +- * BroadcastHashJoin Inner BuildRight (14) : : :- * Project (9) : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.web_sales (1) : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) + : : : +- * ColumnarToRow (6) + : : : +- CometFilter (5) : : : +- CometScan parquet spark_catalog.default.warehouse (4) : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) + : : +- * ColumnarToRow (12) + : : +- CometFilter (11) : : +- CometScan parquet spark_catalog.default.ship_mode (10) : +- BroadcastExchange (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) + : +- * ColumnarToRow (18) + : +- CometFilter (17) : +- CometScan parquet spark_catalog.default.web_site (16) +- BroadcastExchange (26) - +- * Project (25) - +- * Filter (24) - +- * ColumnarToRow (23) + +- * ColumnarToRow (25) + +- CometProject (24) + +- CometFilter (23) +- CometScan parquet spark_catalog.default.date_dim (22) @@ -40,12 +40,12 @@ Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_warehouse_sk), IsNotNull(ws_ship_mode_sk), IsNotNull(ws_web_site_sk), IsNotNull(ws_ship_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] +(2) CometFilter Input [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] +Condition : (((isnotnull(ws_warehouse_sk#4) AND isnotnull(ws_ship_mode_sk#3)) AND isnotnull(ws_web_site_sk#2)) AND isnotnull(ws_ship_date_sk#1)) -(3) Filter [codegen id : 5] +(3) ColumnarToRow [codegen id : 5] Input [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] -Condition : (((isnotnull(ws_warehouse_sk#4) AND isnotnull(ws_ship_mode_sk#3)) AND isnotnull(ws_web_site_sk#2)) AND isnotnull(ws_ship_date_sk#1)) (4) Scan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] @@ -54,12 +54,12 @@ Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Condition : isnotnull(w_warehouse_sk#6) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Condition : isnotnull(w_warehouse_sk#6) (7) BroadcastExchange Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] @@ -82,12 +82,12 @@ Location [not included in comparison]/{warehouse_dir}/ship_mode] PushedFilters: [IsNotNull(sm_ship_mode_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] +(11) CometFilter Input [2]: [sm_ship_mode_sk#8, sm_type#9] +Condition : isnotnull(sm_ship_mode_sk#8) -(12) Filter [codegen id : 2] +(12) ColumnarToRow [codegen id : 2] Input [2]: [sm_ship_mode_sk#8, sm_type#9] -Condition : isnotnull(sm_ship_mode_sk#8) (13) BroadcastExchange Input [2]: [sm_ship_mode_sk#8, sm_type#9] @@ -110,12 +110,12 @@ Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] +(17) CometFilter Input [2]: [web_site_sk#10, web_name#11] +Condition : isnotnull(web_site_sk#10) -(18) Filter [codegen id : 3] +(18) ColumnarToRow [codegen id : 3] Input [2]: [web_site_sk#10, web_name#11] -Condition : isnotnull(web_site_sk#10) (19) BroadcastExchange Input [2]: [web_site_sk#10, web_name#11] @@ -138,16 +138,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 4] -Input [2]: [d_date_sk#12, d_month_seq#13] - -(24) Filter [codegen id : 4] +(23) CometFilter Input [2]: [d_date_sk#12, d_month_seq#13] Condition : (((isnotnull(d_month_seq#13) AND (d_month_seq#13 >= 1200)) AND (d_month_seq#13 <= 1211)) AND isnotnull(d_date_sk#12)) -(25) Project [codegen id : 4] -Output [1]: [d_date_sk#12] +(24) CometProject Input [2]: [d_date_sk#12, d_month_seq#13] +Arguments: [d_date_sk#12], [d_date_sk#12] + +(25) ColumnarToRow [codegen id : 4] +Input [1]: [d_date_sk#12] (26) BroadcastExchange Input [1]: [d_date_sk#12] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62/simplified.txt index a49361cb6..5ae522ce1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62/simplified.txt @@ -13,36 +13,36 @@ TakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days BroadcastHashJoin [ws_ship_mode_sk,sm_ship_mode_sk] Project [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name] BroadcastHashJoin [ws_warehouse_sk,w_warehouse_sk] - Filter [ws_warehouse_sk,ws_ship_mode_sk,ws_web_site_sk,ws_ship_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_warehouse_sk,ws_ship_mode_sk,ws_web_site_sk,ws_ship_date_sk] CometScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] InputAdapter BroadcastExchange #2 WholeStageCodegen (1) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [w_warehouse_sk] CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Filter [sm_ship_mode_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [sm_ship_mode_sk] CometScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] InputAdapter BroadcastExchange #4 WholeStageCodegen (3) - Filter [web_site_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [web_site_sk] CometScan parquet spark_catalog.default.web_site [web_site_sk,web_name] InputAdapter BroadcastExchange #5 WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_month_seq,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/explain.txt index 78baf2df8..990bc3195 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/explain.txt @@ -14,18 +14,18 @@ TakeOrderedAndProject (28) : +- * BroadcastHashJoin Inner BuildRight (12) : :- * Project (10) : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) + : : :- * ColumnarToRow (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) : : : +- CometScan parquet spark_catalog.default.item (1) : : +- BroadcastExchange (8) - : : +- * Filter (7) - : : +- * ColumnarToRow (6) + : : +- * ColumnarToRow (7) + : : +- CometFilter (6) : : +- CometScan parquet spark_catalog.default.store_sales (5) : +- ReusedExchange (11) +- BroadcastExchange (17) - +- * Filter (16) - +- * ColumnarToRow (15) + +- * ColumnarToRow (16) + +- CometFilter (15) +- CometScan parquet spark_catalog.default.store (14) @@ -36,16 +36,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,refernece ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] Condition : ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,refernece ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) -(4) Project [codegen id : 4] -Output [2]: [i_item_sk#1, i_manager_id#5] +(3) CometProject Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] +Arguments: [i_item_sk#1, i_manager_id#5], [i_item_sk#1, i_manager_id#5] + +(4) ColumnarToRow [codegen id : 4] +Input [2]: [i_item_sk#1, i_manager_id#5] (5) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] @@ -55,12 +55,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] +(6) CometFilter Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Condition : (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) -(7) Filter [codegen id : 1] +(7) ColumnarToRow [codegen id : 1] Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Condition : (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) (8) BroadcastExchange Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] @@ -96,12 +96,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 3] +(15) CometFilter Input [1]: [s_store_sk#17] +Condition : isnotnull(s_store_sk#17) -(16) Filter [codegen id : 3] +(16) ColumnarToRow [codegen id : 3] Input [1]: [s_store_sk#17] -Condition : isnotnull(s_store_sk#17) (17) BroadcastExchange Input [1]: [s_store_sk#17] @@ -163,9 +163,9 @@ Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#23 ASC NULLS Subquery:1 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 BroadcastExchange (33) -+- * Project (32) - +- * Filter (31) - +- * ColumnarToRow (30) ++- * ColumnarToRow (32) + +- CometProject (31) + +- CometFilter (30) +- CometScan parquet spark_catalog.default.date_dim (29) @@ -176,16 +176,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#15, d_month_seq#24, d_moy#16] - -(31) Filter [codegen id : 1] +(30) CometFilter Input [3]: [d_date_sk#15, d_month_seq#24, d_moy#16] Condition : (d_month_seq#24 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#15)) -(32) Project [codegen id : 1] -Output [2]: [d_date_sk#15, d_moy#16] +(31) CometProject Input [3]: [d_date_sk#15, d_month_seq#24, d_moy#16] +Arguments: [d_date_sk#15, d_moy#16], [d_date_sk#15, d_moy#16] + +(32) ColumnarToRow [codegen id : 1] +Input [2]: [d_date_sk#15, d_moy#16] (33) BroadcastExchange Input [2]: [d_date_sk#15, d_moy#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/simplified.txt index 2520efac5..7f6f8c137 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/simplified.txt @@ -20,32 +20,32 @@ TakeOrderedAndProject [i_manager_id,avg_monthly_sales,sum_sales] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] BroadcastHashJoin [i_item_sk,ss_item_sk] - Project [i_item_sk,i_manager_id] - Filter [i_category,i_class,i_brand,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_item_sk,i_manager_id] + CometFilter [i_category,i_class,i_brand,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manager_id] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk,ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) - Project [d_date_sk,d_moy] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_moy] + CometFilter [d_month_seq,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_moy] InputAdapter ReusedExchange [d_date_sk,d_moy] #4 InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - Filter [s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/explain.txt index b4a82aa37..667362aa3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/explain.txt @@ -1,187 +1,185 @@ == Physical Plan == -* Sort (183) -+- Exchange (182) - +- * Project (181) - +- * SortMergeJoin Inner (180) - :- * Sort (111) - : +- Exchange (110) - : +- * HashAggregate (109) - : +- * HashAggregate (108) - : +- * Project (107) - : +- * BroadcastHashJoin Inner BuildRight (106) - : :- * Project (100) - : : +- * BroadcastHashJoin Inner BuildRight (99) - : : :- * Project (97) - : : : +- * BroadcastHashJoin Inner BuildRight (96) - : : : :- * Project (91) - : : : : +- * BroadcastHashJoin Inner BuildRight (90) - : : : : :- * Project (88) - : : : : : +- * BroadcastHashJoin Inner BuildRight (87) - : : : : : :- * Project (82) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (81) - : : : : : : :- * Project (79) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (78) - : : : : : : : :- * Project (73) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (72) - : : : : : : : : :- * Project (67) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (66) - : : : : : : : : : :- * Project (64) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (63) - : : : : : : : : : : :- * Project (58) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (57) - : : : : : : : : : : : :- * Project (55) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (54) - : : : : : : : : : : : : :- * Project (49) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (48) - : : : : : : : : : : : : : :- * Project (43) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (42) - : : : : : : : : : : : : : : :- * Project (37) - : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (36) - : : : : : : : : : : : : : : : :- * Project (34) - : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (33) - : : : : : : : : : : : : : : : : :- * Sort (12) - : : : : : : : : : : : : : : : : : +- Exchange (11) - : : : : : : : : : : : : : : : : : +- * Project (10) - : : : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildLeft (9) - : : : : : : : : : : : : : : : : : :- BroadcastExchange (4) - : : : : : : : : : : : : : : : : : : +- * Filter (3) - : : : : : : : : : : : : : : : : : : +- * ColumnarToRow (2) - : : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : : : : : : : : : : : : : : +- * Project (8) - : : : : : : : : : : : : : : : : : +- * Filter (7) - : : : : : : : : : : : : : : : : : +- * ColumnarToRow (6) - : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_returns (5) - : : : : : : : : : : : : : : : : +- * Sort (32) - : : : : : : : : : : : : : : : : +- * Project (31) - : : : : : : : : : : : : : : : : +- * Filter (30) - : : : : : : : : : : : : : : : : +- * HashAggregate (29) - : : : : : : : : : : : : : : : : +- Exchange (28) - : : : : : : : : : : : : : : : : +- * HashAggregate (27) - : : : : : : : : : : : : : : : : +- * Project (26) - : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (25) - : : : : : : : : : : : : : : : : :- * Sort (18) - : : : : : : : : : : : : : : : : : +- Exchange (17) - : : : : : : : : : : : : : : : : : +- * Project (16) - : : : : : : : : : : : : : : : : : +- * Filter (15) - : : : : : : : : : : : : : : : : : +- * ColumnarToRow (14) - : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (13) - : : : : : : : : : : : : : : : : +- * Sort (24) - : : : : : : : : : : : : : : : : +- Exchange (23) - : : : : : : : : : : : : : : : : +- * Project (22) - : : : : : : : : : : : : : : : : +- * Filter (21) - : : : : : : : : : : : : : : : : +- * ColumnarToRow (20) - : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (19) - : : : : : : : : : : : : : : : +- ReusedExchange (35) - : : : : : : : : : : : : : : +- BroadcastExchange (41) - : : : : : : : : : : : : : : +- * Filter (40) - : : : : : : : : : : : : : : +- * ColumnarToRow (39) - : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store (38) - : : : : : : : : : : : : : +- BroadcastExchange (47) - : : : : : : : : : : : : : +- * Filter (46) - : : : : : : : : : : : : : +- * ColumnarToRow (45) - : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.customer (44) - : : : : : : : : : : : : +- BroadcastExchange (53) - : : : : : : : : : : : : +- * Filter (52) - : : : : : : : : : : : : +- * ColumnarToRow (51) - : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.date_dim (50) - : : : : : : : : : : : +- ReusedExchange (56) - : : : : : : : : : : +- BroadcastExchange (62) - : : : : : : : : : : +- * Filter (61) - : : : : : : : : : : +- * ColumnarToRow (60) - : : : : : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (59) - : : : : : : : : : +- ReusedExchange (65) - : : : : : : : : +- BroadcastExchange (71) - : : : : : : : : +- * Filter (70) - : : : : : : : : +- * ColumnarToRow (69) - : : : : : : : : +- CometScan parquet spark_catalog.default.promotion (68) - : : : : : : : +- BroadcastExchange (77) - : : : : : : : +- * Filter (76) - : : : : : : : +- * ColumnarToRow (75) - : : : : : : : +- CometScan parquet spark_catalog.default.household_demographics (74) - : : : : : : +- ReusedExchange (80) - : : : : : +- BroadcastExchange (86) - : : : : : +- * Filter (85) - : : : : : +- * ColumnarToRow (84) - : : : : : +- CometScan parquet spark_catalog.default.customer_address (83) - : : : : +- ReusedExchange (89) - : : : +- BroadcastExchange (95) - : : : +- * Filter (94) - : : : +- * ColumnarToRow (93) - : : : +- CometScan parquet spark_catalog.default.income_band (92) - : : +- ReusedExchange (98) - : +- BroadcastExchange (105) - : +- * Project (104) - : +- * Filter (103) - : +- * ColumnarToRow (102) - : +- CometScan parquet spark_catalog.default.item (101) - +- * Sort (179) - +- Exchange (178) - +- * HashAggregate (177) - +- * HashAggregate (176) - +- * Project (175) - +- * BroadcastHashJoin Inner BuildRight (174) - :- * Project (172) - : +- * BroadcastHashJoin Inner BuildRight (171) - : :- * Project (169) - : : +- * BroadcastHashJoin Inner BuildRight (168) - : : :- * Project (166) - : : : +- * BroadcastHashJoin Inner BuildRight (165) - : : : :- * Project (163) - : : : : +- * BroadcastHashJoin Inner BuildRight (162) - : : : : :- * Project (160) - : : : : : +- * BroadcastHashJoin Inner BuildRight (159) - : : : : : :- * Project (157) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (156) - : : : : : : :- * Project (154) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (153) - : : : : : : : :- * Project (151) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (150) - : : : : : : : : :- * Project (148) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (147) - : : : : : : : : : :- * Project (145) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (144) - : : : : : : : : : : :- * Project (142) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (141) - : : : : : : : : : : : :- * Project (139) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (138) - : : : : : : : : : : : : :- * Project (136) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (135) - : : : : : : : : : : : : : :- * Project (133) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (132) - : : : : : : : : : : : : : : :- * Project (130) - : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (129) - : : : : : : : : : : : : : : : :- * Sort (123) - : : : : : : : : : : : : : : : : +- Exchange (122) - : : : : : : : : : : : : : : : : +- * Project (121) - : : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildLeft (120) - : : : : : : : : : : : : : : : : :- BroadcastExchange (115) - : : : : : : : : : : : : : : : : : +- * Filter (114) - : : : : : : : : : : : : : : : : : +- * ColumnarToRow (113) - : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (112) - : : : : : : : : : : : : : : : : +- * Project (119) - : : : : : : : : : : : : : : : : +- * Filter (118) - : : : : : : : : : : : : : : : : +- * ColumnarToRow (117) - : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_returns (116) - : : : : : : : : : : : : : : : +- * Sort (128) - : : : : : : : : : : : : : : : +- * Project (127) - : : : : : : : : : : : : : : : +- * Filter (126) - : : : : : : : : : : : : : : : +- * HashAggregate (125) - : : : : : : : : : : : : : : : +- ReusedExchange (124) - : : : : : : : : : : : : : : +- ReusedExchange (131) - : : : : : : : : : : : : : +- ReusedExchange (134) - : : : : : : : : : : : : +- ReusedExchange (137) - : : : : : : : : : : : +- ReusedExchange (140) - : : : : : : : : : : +- ReusedExchange (143) - : : : : : : : : : +- ReusedExchange (146) - : : : : : : : : +- ReusedExchange (149) - : : : : : : : +- ReusedExchange (152) - : : : : : : +- ReusedExchange (155) - : : : : : +- ReusedExchange (158) - : : : : +- ReusedExchange (161) - : : : +- ReusedExchange (164) - : : +- ReusedExchange (167) - : +- ReusedExchange (170) - +- ReusedExchange (173) +* Sort (181) ++- Exchange (180) + +- * Project (179) + +- * SortMergeJoin Inner (178) + :- * Sort (110) + : +- Exchange (109) + : +- * HashAggregate (108) + : +- * HashAggregate (107) + : +- * Project (106) + : +- * BroadcastHashJoin Inner BuildRight (105) + : :- * Project (99) + : : +- * BroadcastHashJoin Inner BuildRight (98) + : : :- * Project (96) + : : : +- * BroadcastHashJoin Inner BuildRight (95) + : : : :- * Project (90) + : : : : +- * BroadcastHashJoin Inner BuildRight (89) + : : : : :- * Project (87) + : : : : : +- * BroadcastHashJoin Inner BuildRight (86) + : : : : : :- * Project (81) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (80) + : : : : : : :- * Project (78) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (77) + : : : : : : : :- * Project (72) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (71) + : : : : : : : : :- * Project (66) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (65) + : : : : : : : : : :- * Project (63) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (62) + : : : : : : : : : : :- * Project (57) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (56) + : : : : : : : : : : : :- * Project (54) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (53) + : : : : : : : : : : : : :- * Project (48) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (47) + : : : : : : : : : : : : : :- * Project (42) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (41) + : : : : : : : : : : : : : : :- * Project (36) + : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (35) + : : : : : : : : : : : : : : : :- * Project (33) + : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (32) + : : : : : : : : : : : : : : : : :- * Sort (11) + : : : : : : : : : : : : : : : : : +- Exchange (10) + : : : : : : : : : : : : : : : : : +- * ColumnarToRow (9) + : : : : : : : : : : : : : : : : : +- CometProject (8) + : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) + : : : : : : : : : : : : : : : : : : +- CometFilter (2) + : : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : : : : : : : : : : : : : +- CometProject (6) + : : : : : : : : : : : : : : : : : +- CometFilter (5) + : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_returns (4) + : : : : : : : : : : : : : : : : +- * Sort (31) + : : : : : : : : : : : : : : : : +- * Project (30) + : : : : : : : : : : : : : : : : +- * Filter (29) + : : : : : : : : : : : : : : : : +- * HashAggregate (28) + : : : : : : : : : : : : : : : : +- Exchange (27) + : : : : : : : : : : : : : : : : +- * HashAggregate (26) + : : : : : : : : : : : : : : : : +- * Project (25) + : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (24) + : : : : : : : : : : : : : : : : :- * Sort (17) + : : : : : : : : : : : : : : : : : +- Exchange (16) + : : : : : : : : : : : : : : : : : +- * ColumnarToRow (15) + : : : : : : : : : : : : : : : : : +- CometProject (14) + : : : : : : : : : : : : : : : : : +- CometFilter (13) + : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (12) + : : : : : : : : : : : : : : : : +- * Sort (23) + : : : : : : : : : : : : : : : : +- Exchange (22) + : : : : : : : : : : : : : : : : +- * ColumnarToRow (21) + : : : : : : : : : : : : : : : : +- CometProject (20) + : : : : : : : : : : : : : : : : +- CometFilter (19) + : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (18) + : : : : : : : : : : : : : : : +- ReusedExchange (34) + : : : : : : : : : : : : : : +- BroadcastExchange (40) + : : : : : : : : : : : : : : +- * ColumnarToRow (39) + : : : : : : : : : : : : : : +- CometFilter (38) + : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store (37) + : : : : : : : : : : : : : +- BroadcastExchange (46) + : : : : : : : : : : : : : +- * ColumnarToRow (45) + : : : : : : : : : : : : : +- CometFilter (44) + : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.customer (43) + : : : : : : : : : : : : +- BroadcastExchange (52) + : : : : : : : : : : : : +- * ColumnarToRow (51) + : : : : : : : : : : : : +- CometFilter (50) + : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.date_dim (49) + : : : : : : : : : : : +- ReusedExchange (55) + : : : : : : : : : : +- BroadcastExchange (61) + : : : : : : : : : : +- * ColumnarToRow (60) + : : : : : : : : : : +- CometFilter (59) + : : : : : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (58) + : : : : : : : : : +- ReusedExchange (64) + : : : : : : : : +- BroadcastExchange (70) + : : : : : : : : +- * ColumnarToRow (69) + : : : : : : : : +- CometFilter (68) + : : : : : : : : +- CometScan parquet spark_catalog.default.promotion (67) + : : : : : : : +- BroadcastExchange (76) + : : : : : : : +- * ColumnarToRow (75) + : : : : : : : +- CometFilter (74) + : : : : : : : +- CometScan parquet spark_catalog.default.household_demographics (73) + : : : : : : +- ReusedExchange (79) + : : : : : +- BroadcastExchange (85) + : : : : : +- * ColumnarToRow (84) + : : : : : +- CometFilter (83) + : : : : : +- CometScan parquet spark_catalog.default.customer_address (82) + : : : : +- ReusedExchange (88) + : : : +- BroadcastExchange (94) + : : : +- * ColumnarToRow (93) + : : : +- CometFilter (92) + : : : +- CometScan parquet spark_catalog.default.income_band (91) + : : +- ReusedExchange (97) + : +- BroadcastExchange (104) + : +- * ColumnarToRow (103) + : +- CometProject (102) + : +- CometFilter (101) + : +- CometScan parquet spark_catalog.default.item (100) + +- * Sort (177) + +- Exchange (176) + +- * HashAggregate (175) + +- * HashAggregate (174) + +- * Project (173) + +- * BroadcastHashJoin Inner BuildRight (172) + :- * Project (170) + : +- * BroadcastHashJoin Inner BuildRight (169) + : :- * Project (167) + : : +- * BroadcastHashJoin Inner BuildRight (166) + : : :- * Project (164) + : : : +- * BroadcastHashJoin Inner BuildRight (163) + : : : :- * Project (161) + : : : : +- * BroadcastHashJoin Inner BuildRight (160) + : : : : :- * Project (158) + : : : : : +- * BroadcastHashJoin Inner BuildRight (157) + : : : : : :- * Project (155) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (154) + : : : : : : :- * Project (152) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (151) + : : : : : : : :- * Project (149) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (148) + : : : : : : : : :- * Project (146) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (145) + : : : : : : : : : :- * Project (143) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (142) + : : : : : : : : : : :- * Project (140) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (139) + : : : : : : : : : : : :- * Project (137) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (136) + : : : : : : : : : : : : :- * Project (134) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (133) + : : : : : : : : : : : : : :- * Project (131) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (130) + : : : : : : : : : : : : : : :- * Project (128) + : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (127) + : : : : : : : : : : : : : : : :- * Sort (121) + : : : : : : : : : : : : : : : : +- Exchange (120) + : : : : : : : : : : : : : : : : +- * ColumnarToRow (119) + : : : : : : : : : : : : : : : : +- CometProject (118) + : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (117) + : : : : : : : : : : : : : : : : :- CometBroadcastExchange (113) + : : : : : : : : : : : : : : : : : +- CometFilter (112) + : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (111) + : : : : : : : : : : : : : : : : +- CometProject (116) + : : : : : : : : : : : : : : : : +- CometFilter (115) + : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_returns (114) + : : : : : : : : : : : : : : : +- * Sort (126) + : : : : : : : : : : : : : : : +- * Project (125) + : : : : : : : : : : : : : : : +- * Filter (124) + : : : : : : : : : : : : : : : +- * HashAggregate (123) + : : : : : : : : : : : : : : : +- ReusedExchange (122) + : : : : : : : : : : : : : : +- ReusedExchange (129) + : : : : : : : : : : : : : +- ReusedExchange (132) + : : : : : : : : : : : : +- ReusedExchange (135) + : : : : : : : : : : : +- ReusedExchange (138) + : : : : : : : : : : +- ReusedExchange (141) + : : : : : : : : : +- ReusedExchange (144) + : : : : : : : : +- ReusedExchange (147) + : : : : : : : +- ReusedExchange (150) + : : : : : : +- ReusedExchange (153) + : : : : : +- ReusedExchange (156) + : : : : +- ReusedExchange (159) + : : : +- ReusedExchange (162) + : : +- ReusedExchange (165) + : +- ReusedExchange (168) + +- ReusedExchange (171) (1) Scan parquet spark_catalog.default.store_sales @@ -192,512 +190,508 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AND isnotnull(ss_store_sk#6)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_cdemo_sk#3)) AND isnotnull(ss_promo_sk#7)) AND isnotnull(ss_hdemo_sk#4)) AND isnotnull(ss_addr_sk#5)) -(4) BroadcastExchange +(3) CometBroadcastExchange Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[7, int, false] as bigint) & 4294967295))),false), [plan_id=1] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(5) Scan parquet spark_catalog.default.store_returns +(4) Scan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(6) ColumnarToRow -Input [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] - -(7) Filter +(5) CometFilter Input [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] Condition : (isnotnull(sr_item_sk#14) AND isnotnull(sr_ticket_number#15)) -(8) Project -Output [2]: [sr_item_sk#14, sr_ticket_number#15] +(6) CometProject Input [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] +Arguments: [sr_item_sk#14, sr_ticket_number#15], [sr_item_sk#14, sr_ticket_number#15] -(9) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [ss_item_sk#1, ss_ticket_number#8] -Right keys [2]: [sr_item_sk#14, sr_ticket_number#15] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Right output [2]: [sr_item_sk#14, sr_ticket_number#15] +Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#14, sr_ticket_number#15], Inner -(10) Project [codegen id : 2] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +(8) CometProject Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#14, sr_ticket_number#15] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] + +(9) ColumnarToRow [codegen id : 1] +Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(11) Exchange +(10) Exchange Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(12) Sort [codegen id : 3] +(11) Sort [codegen id : 2] Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0 -(13) Scan parquet spark_catalog.default.catalog_sales +(12) Scan parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 4] -Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] - -(15) Filter [codegen id : 4] +(13) CometFilter Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] Condition : (isnotnull(cs_item_sk#17) AND isnotnull(cs_order_number#18)) -(16) Project [codegen id : 4] -Output [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] +(14) CometProject Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] +Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] + +(15) ColumnarToRow [codegen id : 3] +Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -(17) Exchange +(16) Exchange Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(18) Sort [codegen id : 5] +(17) Sort [codegen id : 4] Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] Arguments: [cs_item_sk#17 ASC NULLS FIRST, cs_order_number#18 ASC NULLS FIRST], false, 0 -(19) Scan parquet spark_catalog.default.catalog_returns +(18) Scan parquet spark_catalog.default.catalog_returns Output [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 6] -Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] - -(21) Filter [codegen id : 6] +(19) CometFilter Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] Condition : (isnotnull(cr_item_sk#21) AND isnotnull(cr_order_number#22)) -(22) Project [codegen id : 6] -Output [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] +(20) CometProject Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] +Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] + +(21) ColumnarToRow [codegen id : 5] +Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -(23) Exchange +(22) Exchange Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(24) Sort [codegen id : 7] +(23) Sort [codegen id : 6] Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] Arguments: [cr_item_sk#21 ASC NULLS FIRST, cr_order_number#22 ASC NULLS FIRST], false, 0 -(25) SortMergeJoin [codegen id : 8] +(24) SortMergeJoin [codegen id : 7] Left keys [2]: [cs_item_sk#17, cs_order_number#18] Right keys [2]: [cr_item_sk#21, cr_order_number#22] Join type: Inner Join condition: None -(26) Project [codegen id : 8] +(25) Project [codegen id : 7] Output [5]: [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] Input [8]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -(27) HashAggregate [codegen id : 8] +(26) HashAggregate [codegen id : 7] Input [5]: [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] Keys [1]: [cs_item_sk#17] Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#19)), partial_sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))] Aggregate Attributes [3]: [sum#27, sum#28, isEmpty#29] Results [4]: [cs_item_sk#17, sum#30, sum#31, isEmpty#32] -(28) Exchange +(27) Exchange Input [4]: [cs_item_sk#17, sum#30, sum#31, isEmpty#32] -Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(29) HashAggregate [codegen id : 9] +(28) HashAggregate [codegen id : 8] Input [4]: [cs_item_sk#17, sum#30, sum#31, isEmpty#32] Keys [1]: [cs_item_sk#17] Functions [2]: [sum(UnscaledValue(cs_ext_list_price#19)), sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))] Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#19))#33, sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))#34] Results [3]: [cs_item_sk#17, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#19))#33,17,2) AS sale#35, sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))#34 AS refund#36] -(30) Filter [codegen id : 9] +(29) Filter [codegen id : 8] Input [3]: [cs_item_sk#17, sale#35, refund#36] Condition : ((isnotnull(sale#35) AND isnotnull(refund#36)) AND (cast(sale#35 as decimal(21,2)) > (2 * refund#36))) -(31) Project [codegen id : 9] +(30) Project [codegen id : 8] Output [1]: [cs_item_sk#17] Input [3]: [cs_item_sk#17, sale#35, refund#36] -(32) Sort [codegen id : 9] +(31) Sort [codegen id : 8] Input [1]: [cs_item_sk#17] Arguments: [cs_item_sk#17 ASC NULLS FIRST], false, 0 -(33) SortMergeJoin [codegen id : 25] +(32) SortMergeJoin [codegen id : 24] Left keys [1]: [ss_item_sk#1] Right keys [1]: [cs_item_sk#17] Join type: Inner Join condition: None -(34) Project [codegen id : 25] +(33) Project [codegen id : 24] Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#17] -(35) ReusedExchange [Reuses operator id: 187] +(34) ReusedExchange [Reuses operator id: 185] Output [2]: [d_date_sk#37, d_year#38] -(36) BroadcastHashJoin [codegen id : 25] +(35) BroadcastHashJoin [codegen id : 24] Left keys [1]: [ss_sold_date_sk#12] Right keys [1]: [d_date_sk#37] Join type: Inner Join condition: None -(37) Project [codegen id : 25] +(36) Project [codegen id : 24] Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38] Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#37, d_year#38] -(38) Scan parquet spark_catalog.default.store +(37) Scan parquet spark_catalog.default.store Output [3]: [s_store_sk#39, s_store_name#40, s_zip#41] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_zip)] ReadSchema: struct -(39) ColumnarToRow [codegen id : 11] +(38) CometFilter Input [3]: [s_store_sk#39, s_store_name#40, s_zip#41] +Condition : ((isnotnull(s_store_sk#39) AND isnotnull(s_store_name#40)) AND isnotnull(s_zip#41)) -(40) Filter [codegen id : 11] +(39) ColumnarToRow [codegen id : 10] Input [3]: [s_store_sk#39, s_store_name#40, s_zip#41] -Condition : ((isnotnull(s_store_sk#39) AND isnotnull(s_store_name#40)) AND isnotnull(s_zip#41)) -(41) BroadcastExchange +(40) BroadcastExchange Input [3]: [s_store_sk#39, s_store_name#40, s_zip#41] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] -(42) BroadcastHashJoin [codegen id : 25] +(41) BroadcastHashJoin [codegen id : 24] Left keys [1]: [ss_store_sk#6] Right keys [1]: [s_store_sk#39] Join type: Inner Join condition: None -(43) Project [codegen id : 25] +(42) Project [codegen id : 24] Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41] Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_sk#39, s_store_name#40, s_zip#41] -(44) Scan parquet spark_catalog.default.customer +(43) Scan parquet spark_catalog.default.customer Output [6]: [c_customer_sk#42, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 12] +(44) CometFilter Input [6]: [c_customer_sk#42, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] +Condition : (((((isnotnull(c_customer_sk#42) AND isnotnull(c_first_sales_date_sk#47)) AND isnotnull(c_first_shipto_date_sk#46)) AND isnotnull(c_current_cdemo_sk#43)) AND isnotnull(c_current_hdemo_sk#44)) AND isnotnull(c_current_addr_sk#45)) -(46) Filter [codegen id : 12] +(45) ColumnarToRow [codegen id : 11] Input [6]: [c_customer_sk#42, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] -Condition : (((((isnotnull(c_customer_sk#42) AND isnotnull(c_first_sales_date_sk#47)) AND isnotnull(c_first_shipto_date_sk#46)) AND isnotnull(c_current_cdemo_sk#43)) AND isnotnull(c_current_hdemo_sk#44)) AND isnotnull(c_current_addr_sk#45)) -(47) BroadcastExchange +(46) BroadcastExchange Input [6]: [c_customer_sk#42, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] -(48) BroadcastHashJoin [codegen id : 25] +(47) BroadcastHashJoin [codegen id : 24] Left keys [1]: [ss_customer_sk#2] Right keys [1]: [c_customer_sk#42] Join type: Inner Join condition: None -(49) Project [codegen id : 25] +(48) Project [codegen id : 24] Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_customer_sk#42, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] -(50) Scan parquet spark_catalog.default.date_dim +(49) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#48, d_year#49] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(51) ColumnarToRow [codegen id : 13] +(50) CometFilter Input [2]: [d_date_sk#48, d_year#49] +Condition : isnotnull(d_date_sk#48) -(52) Filter [codegen id : 13] +(51) ColumnarToRow [codegen id : 12] Input [2]: [d_date_sk#48, d_year#49] -Condition : isnotnull(d_date_sk#48) -(53) BroadcastExchange +(52) BroadcastExchange Input [2]: [d_date_sk#48, d_year#49] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] -(54) BroadcastHashJoin [codegen id : 25] +(53) BroadcastHashJoin [codegen id : 24] Left keys [1]: [c_first_sales_date_sk#47] Right keys [1]: [d_date_sk#48] Join type: Inner Join condition: None -(55) Project [codegen id : 25] +(54) Project [codegen id : 24] Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, d_year#49] Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47, d_date_sk#48, d_year#49] -(56) ReusedExchange [Reuses operator id: 53] +(55) ReusedExchange [Reuses operator id: 52] Output [2]: [d_date_sk#50, d_year#51] -(57) BroadcastHashJoin [codegen id : 25] +(56) BroadcastHashJoin [codegen id : 24] Left keys [1]: [c_first_shipto_date_sk#46] Right keys [1]: [d_date_sk#50] Join type: Inner Join condition: None -(58) Project [codegen id : 25] +(57) Project [codegen id : 24] Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51] Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, d_year#49, d_date_sk#50, d_year#51] -(59) Scan parquet spark_catalog.default.customer_demographics +(58) Scan parquet spark_catalog.default.customer_demographics Output [2]: [cd_demo_sk#52, cd_marital_status#53] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status)] ReadSchema: struct -(60) ColumnarToRow [codegen id : 15] +(59) CometFilter Input [2]: [cd_demo_sk#52, cd_marital_status#53] +Condition : (isnotnull(cd_demo_sk#52) AND isnotnull(cd_marital_status#53)) -(61) Filter [codegen id : 15] +(60) ColumnarToRow [codegen id : 14] Input [2]: [cd_demo_sk#52, cd_marital_status#53] -Condition : (isnotnull(cd_demo_sk#52) AND isnotnull(cd_marital_status#53)) -(62) BroadcastExchange +(61) BroadcastExchange Input [2]: [cd_demo_sk#52, cd_marital_status#53] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] -(63) BroadcastHashJoin [codegen id : 25] +(62) BroadcastHashJoin [codegen id : 24] Left keys [1]: [ss_cdemo_sk#3] Right keys [1]: [cd_demo_sk#52] Join type: Inner Join condition: None -(64) Project [codegen id : 25] +(63) Project [codegen id : 24] Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, cd_marital_status#53] Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, cd_demo_sk#52, cd_marital_status#53] -(65) ReusedExchange [Reuses operator id: 62] +(64) ReusedExchange [Reuses operator id: 61] Output [2]: [cd_demo_sk#54, cd_marital_status#55] -(66) BroadcastHashJoin [codegen id : 25] +(65) BroadcastHashJoin [codegen id : 24] Left keys [1]: [c_current_cdemo_sk#43] Right keys [1]: [cd_demo_sk#54] Join type: Inner Join condition: NOT (cd_marital_status#53 = cd_marital_status#55) -(67) Project [codegen id : 25] +(66) Project [codegen id : 24] Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51] Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, cd_marital_status#53, cd_demo_sk#54, cd_marital_status#55] -(68) Scan parquet spark_catalog.default.promotion +(67) Scan parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#56] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(69) ColumnarToRow [codegen id : 17] +(68) CometFilter Input [1]: [p_promo_sk#56] +Condition : isnotnull(p_promo_sk#56) -(70) Filter [codegen id : 17] +(69) ColumnarToRow [codegen id : 16] Input [1]: [p_promo_sk#56] -Condition : isnotnull(p_promo_sk#56) -(71) BroadcastExchange +(70) BroadcastExchange Input [1]: [p_promo_sk#56] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] -(72) BroadcastHashJoin [codegen id : 25] +(71) BroadcastHashJoin [codegen id : 24] Left keys [1]: [ss_promo_sk#7] Right keys [1]: [p_promo_sk#56] Join type: Inner Join condition: None -(73) Project [codegen id : 25] +(72) Project [codegen id : 24] Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51] Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, p_promo_sk#56] -(74) Scan parquet spark_catalog.default.household_demographics +(73) Scan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#57, hd_income_band_sk#58] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] ReadSchema: struct -(75) ColumnarToRow [codegen id : 18] +(74) CometFilter Input [2]: [hd_demo_sk#57, hd_income_band_sk#58] +Condition : (isnotnull(hd_demo_sk#57) AND isnotnull(hd_income_band_sk#58)) -(76) Filter [codegen id : 18] +(75) ColumnarToRow [codegen id : 17] Input [2]: [hd_demo_sk#57, hd_income_band_sk#58] -Condition : (isnotnull(hd_demo_sk#57) AND isnotnull(hd_income_band_sk#58)) -(77) BroadcastExchange +(76) BroadcastExchange Input [2]: [hd_demo_sk#57, hd_income_band_sk#58] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] -(78) BroadcastHashJoin [codegen id : 25] +(77) BroadcastHashJoin [codegen id : 24] Left keys [1]: [ss_hdemo_sk#4] Right keys [1]: [hd_demo_sk#57] Join type: Inner Join condition: None -(79) Project [codegen id : 25] +(78) Project [codegen id : 24] Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58] Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, hd_demo_sk#57, hd_income_band_sk#58] -(80) ReusedExchange [Reuses operator id: 77] +(79) ReusedExchange [Reuses operator id: 76] Output [2]: [hd_demo_sk#59, hd_income_band_sk#60] -(81) BroadcastHashJoin [codegen id : 25] +(80) BroadcastHashJoin [codegen id : 24] Left keys [1]: [c_current_hdemo_sk#44] Right keys [1]: [hd_demo_sk#59] Join type: Inner Join condition: None -(82) Project [codegen id : 25] +(81) Project [codegen id : 24] Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60] Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58, hd_demo_sk#59, hd_income_band_sk#60] -(83) Scan parquet spark_catalog.default.customer_address +(82) Scan parquet spark_catalog.default.customer_address Output [5]: [ca_address_sk#61, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(84) ColumnarToRow [codegen id : 20] +(83) CometFilter Input [5]: [ca_address_sk#61, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] +Condition : isnotnull(ca_address_sk#61) -(85) Filter [codegen id : 20] +(84) ColumnarToRow [codegen id : 19] Input [5]: [ca_address_sk#61, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] -Condition : isnotnull(ca_address_sk#61) -(86) BroadcastExchange +(85) BroadcastExchange Input [5]: [ca_address_sk#61, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] -(87) BroadcastHashJoin [codegen id : 25] +(86) BroadcastHashJoin [codegen id : 24] Left keys [1]: [ss_addr_sk#5] Right keys [1]: [ca_address_sk#61] Join type: Inner Join condition: None -(88) Project [codegen id : 25] +(87) Project [codegen id : 24] Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60, ca_address_sk#61, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] -(89) ReusedExchange [Reuses operator id: 86] +(88) ReusedExchange [Reuses operator id: 85] Output [5]: [ca_address_sk#66, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] -(90) BroadcastHashJoin [codegen id : 25] +(89) BroadcastHashJoin [codegen id : 24] Left keys [1]: [c_current_addr_sk#45] Right keys [1]: [ca_address_sk#66] Join type: Inner Join condition: None -(91) Project [codegen id : 25] +(90) Project [codegen id : 24] Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_address_sk#66, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] -(92) Scan parquet spark_catalog.default.income_band +(91) Scan parquet spark_catalog.default.income_band Output [1]: [ib_income_band_sk#71] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] PushedFilters: [IsNotNull(ib_income_band_sk)] ReadSchema: struct -(93) ColumnarToRow [codegen id : 22] +(92) CometFilter Input [1]: [ib_income_band_sk#71] +Condition : isnotnull(ib_income_band_sk#71) -(94) Filter [codegen id : 22] +(93) ColumnarToRow [codegen id : 21] Input [1]: [ib_income_band_sk#71] -Condition : isnotnull(ib_income_band_sk#71) -(95) BroadcastExchange +(94) BroadcastExchange Input [1]: [ib_income_band_sk#71] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] -(96) BroadcastHashJoin [codegen id : 25] +(95) BroadcastHashJoin [codegen id : 24] Left keys [1]: [hd_income_band_sk#58] Right keys [1]: [ib_income_band_sk#71] Join type: Inner Join condition: None -(97) Project [codegen id : 25] +(96) Project [codegen id : 24] Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, ib_income_band_sk#71] -(98) ReusedExchange [Reuses operator id: 95] +(97) ReusedExchange [Reuses operator id: 94] Output [1]: [ib_income_band_sk#72] -(99) BroadcastHashJoin [codegen id : 25] +(98) BroadcastHashJoin [codegen id : 24] Left keys [1]: [hd_income_band_sk#60] Right keys [1]: [ib_income_band_sk#72] Join type: Inner Join condition: None -(100) Project [codegen id : 25] +(99) Project [codegen id : 24] Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, ib_income_band_sk#72] -(101) Scan parquet spark_catalog.default.item +(100) Scan parquet spark_catalog.default.item Output [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), In(i_color, [burlywood ,floral ,indian ,medium ,purple ,spring ]), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] ReadSchema: struct -(102) ColumnarToRow [codegen id : 24] -Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] - -(103) Filter [codegen id : 24] +(101) CometFilter Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] Condition : ((((((isnotnull(i_current_price#74) AND i_color#75 IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#74 >= 64.00)) AND (i_current_price#74 <= 74.00)) AND (i_current_price#74 >= 65.00)) AND (i_current_price#74 <= 79.00)) AND isnotnull(i_item_sk#73)) -(104) Project [codegen id : 24] -Output [2]: [i_item_sk#73, i_product_name#76] +(102) CometProject Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] +Arguments: [i_item_sk#73, i_product_name#76], [i_item_sk#73, i_product_name#76] + +(103) ColumnarToRow [codegen id : 23] +Input [2]: [i_item_sk#73, i_product_name#76] -(105) BroadcastExchange +(104) BroadcastExchange Input [2]: [i_item_sk#73, i_product_name#76] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] -(106) BroadcastHashJoin [codegen id : 25] +(105) BroadcastHashJoin [codegen id : 24] Left keys [1]: [ss_item_sk#1] Right keys [1]: [i_item_sk#73] Join type: Inner Join condition: None -(107) Project [codegen id : 25] +(106) Project [codegen id : 24] Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, d_year#49, d_year#51, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#76] Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#76] -(108) HashAggregate [codegen id : 25] +(107) HashAggregate [codegen id : 24] Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, d_year#49, d_year#51, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#76] Keys [15]: [i_product_name#76, i_item_sk#73, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#38, d_year#49, d_year#51] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] Aggregate Attributes [4]: [count#77, sum#78, sum#79, sum#80] Results [19]: [i_product_name#76, i_item_sk#73, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#38, d_year#49, d_year#51, count#81, sum#82, sum#83, sum#84] -(109) HashAggregate [codegen id : 25] +(108) HashAggregate [codegen id : 24] Input [19]: [i_product_name#76, i_item_sk#73, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#38, d_year#49, d_year#51, count#81, sum#82, sum#83, sum#84] Keys [15]: [i_product_name#76, i_item_sk#73, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#38, d_year#49, d_year#51] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] Aggregate Attributes [4]: [count(1)#85, sum(UnscaledValue(ss_wholesale_cost#9))#86, sum(UnscaledValue(ss_list_price#10))#87, sum(UnscaledValue(ss_coupon_amt#11))#88] Results [17]: [i_product_name#76 AS product_name#89, i_item_sk#73 AS item_sk#90, s_store_name#40 AS store_name#91, s_zip#41 AS store_zip#92, ca_street_number#62 AS b_street_number#93, ca_street_name#63 AS b_streen_name#94, ca_city#64 AS b_city#95, ca_zip#65 AS b_zip#96, ca_street_number#67 AS c_street_number#97, ca_street_name#68 AS c_street_name#98, ca_city#69 AS c_city#99, ca_zip#70 AS c_zip#100, d_year#38 AS syear#101, count(1)#85 AS cnt#102, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#86,17,2) AS s1#103, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#87,17,2) AS s2#104, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#88,17,2) AS s3#105] -(110) Exchange +(109) Exchange Input [17]: [product_name#89, item_sk#90, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105] -Arguments: hashpartitioning(item_sk#90, store_name#91, store_zip#92, 5), ENSURE_REQUIREMENTS, [plan_id=15] +Arguments: hashpartitioning(item_sk#90, store_name#91, store_zip#92, 5), ENSURE_REQUIREMENTS, [plan_id=14] -(111) Sort [codegen id : 26] +(110) Sort [codegen id : 25] Input [17]: [product_name#89, item_sk#90, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105] Arguments: [item_sk#90 ASC NULLS FIRST, store_name#91 ASC NULLS FIRST, store_zip#92 ASC NULLS FIRST], false, 0 -(112) Scan parquet spark_catalog.default.store_sales +(111) Scan parquet spark_catalog.default.store_sales Output [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] Batched: true Location: InMemoryFileIndex [] @@ -705,370 +699,366 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#117), dynamicpruningexpression(ss_s PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] ReadSchema: struct -(113) ColumnarToRow [codegen id : 27] -Input [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] - -(114) Filter [codegen id : 27] +(112) CometFilter Input [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] Condition : (((((((isnotnull(ss_item_sk#106) AND isnotnull(ss_ticket_number#113)) AND isnotnull(ss_store_sk#111)) AND isnotnull(ss_customer_sk#107)) AND isnotnull(ss_cdemo_sk#108)) AND isnotnull(ss_promo_sk#112)) AND isnotnull(ss_hdemo_sk#109)) AND isnotnull(ss_addr_sk#110)) -(115) BroadcastExchange +(113) CometBroadcastExchange Input [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[7, int, false] as bigint) & 4294967295))),false), [plan_id=16] +Arguments: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] -(116) Scan parquet spark_catalog.default.store_returns +(114) Scan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#119, sr_ticket_number#120, sr_returned_date_sk#121] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(117) ColumnarToRow -Input [3]: [sr_item_sk#119, sr_ticket_number#120, sr_returned_date_sk#121] - -(118) Filter +(115) CometFilter Input [3]: [sr_item_sk#119, sr_ticket_number#120, sr_returned_date_sk#121] Condition : (isnotnull(sr_item_sk#119) AND isnotnull(sr_ticket_number#120)) -(119) Project -Output [2]: [sr_item_sk#119, sr_ticket_number#120] +(116) CometProject Input [3]: [sr_item_sk#119, sr_ticket_number#120, sr_returned_date_sk#121] +Arguments: [sr_item_sk#119, sr_ticket_number#120], [sr_item_sk#119, sr_ticket_number#120] -(120) BroadcastHashJoin [codegen id : 28] -Left keys [2]: [ss_item_sk#106, ss_ticket_number#113] -Right keys [2]: [sr_item_sk#119, sr_ticket_number#120] -Join type: Inner -Join condition: None +(117) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] +Right output [2]: [sr_item_sk#119, sr_ticket_number#120] +Arguments: [ss_item_sk#106, ss_ticket_number#113], [sr_item_sk#119, sr_ticket_number#120], Inner -(121) Project [codegen id : 28] -Output [11]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] +(118) CometProject Input [14]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, sr_item_sk#119, sr_ticket_number#120] +Arguments: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117], [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] + +(119) ColumnarToRow [codegen id : 26] +Input [11]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] -(122) Exchange +(120) Exchange Input [11]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] -Arguments: hashpartitioning(ss_item_sk#106, 5), ENSURE_REQUIREMENTS, [plan_id=17] +Arguments: hashpartitioning(ss_item_sk#106, 5), ENSURE_REQUIREMENTS, [plan_id=15] -(123) Sort [codegen id : 29] +(121) Sort [codegen id : 27] Input [11]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] Arguments: [ss_item_sk#106 ASC NULLS FIRST], false, 0 -(124) ReusedExchange [Reuses operator id: 28] +(122) ReusedExchange [Reuses operator id: 27] Output [4]: [cs_item_sk#122, sum#123, sum#124, isEmpty#125] -(125) HashAggregate [codegen id : 35] +(123) HashAggregate [codegen id : 33] Input [4]: [cs_item_sk#122, sum#123, sum#124, isEmpty#125] Keys [1]: [cs_item_sk#122] Functions [2]: [sum(UnscaledValue(cs_ext_list_price#126)), sum(((cr_refunded_cash#127 + cr_reversed_charge#128) + cr_store_credit#129))] Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#126))#33, sum(((cr_refunded_cash#127 + cr_reversed_charge#128) + cr_store_credit#129))#34] Results [3]: [cs_item_sk#122, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#126))#33,17,2) AS sale#130, sum(((cr_refunded_cash#127 + cr_reversed_charge#128) + cr_store_credit#129))#34 AS refund#131] -(126) Filter [codegen id : 35] +(124) Filter [codegen id : 33] Input [3]: [cs_item_sk#122, sale#130, refund#131] Condition : ((isnotnull(sale#130) AND isnotnull(refund#131)) AND (cast(sale#130 as decimal(21,2)) > (2 * refund#131))) -(127) Project [codegen id : 35] +(125) Project [codegen id : 33] Output [1]: [cs_item_sk#122] Input [3]: [cs_item_sk#122, sale#130, refund#131] -(128) Sort [codegen id : 35] +(126) Sort [codegen id : 33] Input [1]: [cs_item_sk#122] Arguments: [cs_item_sk#122 ASC NULLS FIRST], false, 0 -(129) SortMergeJoin [codegen id : 51] +(127) SortMergeJoin [codegen id : 49] Left keys [1]: [ss_item_sk#106] Right keys [1]: [cs_item_sk#122] Join type: Inner Join condition: None -(130) Project [codegen id : 51] +(128) Project [codegen id : 49] Output [11]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] Input [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, cs_item_sk#122] -(131) ReusedExchange [Reuses operator id: 191] +(129) ReusedExchange [Reuses operator id: 189] Output [2]: [d_date_sk#132, d_year#133] -(132) BroadcastHashJoin [codegen id : 51] +(130) BroadcastHashJoin [codegen id : 49] Left keys [1]: [ss_sold_date_sk#117] Right keys [1]: [d_date_sk#132] Join type: Inner Join condition: None -(133) Project [codegen id : 51] +(131) Project [codegen id : 49] Output [11]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133] Input [13]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, d_date_sk#132, d_year#133] -(134) ReusedExchange [Reuses operator id: 41] +(132) ReusedExchange [Reuses operator id: 40] Output [3]: [s_store_sk#134, s_store_name#135, s_zip#136] -(135) BroadcastHashJoin [codegen id : 51] +(133) BroadcastHashJoin [codegen id : 49] Left keys [1]: [ss_store_sk#111] Right keys [1]: [s_store_sk#134] Join type: Inner Join condition: None -(136) Project [codegen id : 51] +(134) Project [codegen id : 49] Output [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136] Input [14]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_sk#134, s_store_name#135, s_zip#136] -(137) ReusedExchange [Reuses operator id: 47] +(135) ReusedExchange [Reuses operator id: 46] Output [6]: [c_customer_sk#137, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, c_first_sales_date_sk#142] -(138) BroadcastHashJoin [codegen id : 51] +(136) BroadcastHashJoin [codegen id : 49] Left keys [1]: [ss_customer_sk#107] Right keys [1]: [c_customer_sk#137] Join type: Inner Join condition: None -(139) Project [codegen id : 51] +(137) Project [codegen id : 49] Output [16]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, c_first_sales_date_sk#142] Input [18]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_customer_sk#137, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, c_first_sales_date_sk#142] -(140) ReusedExchange [Reuses operator id: 53] +(138) ReusedExchange [Reuses operator id: 52] Output [2]: [d_date_sk#143, d_year#144] -(141) BroadcastHashJoin [codegen id : 51] +(139) BroadcastHashJoin [codegen id : 49] Left keys [1]: [c_first_sales_date_sk#142] Right keys [1]: [d_date_sk#143] Join type: Inner Join condition: None -(142) Project [codegen id : 51] +(140) Project [codegen id : 49] Output [16]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, d_year#144] Input [18]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, c_first_sales_date_sk#142, d_date_sk#143, d_year#144] -(143) ReusedExchange [Reuses operator id: 53] +(141) ReusedExchange [Reuses operator id: 52] Output [2]: [d_date_sk#145, d_year#146] -(144) BroadcastHashJoin [codegen id : 51] +(142) BroadcastHashJoin [codegen id : 49] Left keys [1]: [c_first_shipto_date_sk#141] Right keys [1]: [d_date_sk#145] Join type: Inner Join condition: None -(145) Project [codegen id : 51] +(143) Project [codegen id : 49] Output [16]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146] Input [18]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, d_year#144, d_date_sk#145, d_year#146] -(146) ReusedExchange [Reuses operator id: 62] +(144) ReusedExchange [Reuses operator id: 61] Output [2]: [cd_demo_sk#147, cd_marital_status#148] -(147) BroadcastHashJoin [codegen id : 51] +(145) BroadcastHashJoin [codegen id : 49] Left keys [1]: [ss_cdemo_sk#108] Right keys [1]: [cd_demo_sk#147] Join type: Inner Join condition: None -(148) Project [codegen id : 51] +(146) Project [codegen id : 49] Output [16]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, cd_marital_status#148] Input [18]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, cd_demo_sk#147, cd_marital_status#148] -(149) ReusedExchange [Reuses operator id: 62] +(147) ReusedExchange [Reuses operator id: 61] Output [2]: [cd_demo_sk#149, cd_marital_status#150] -(150) BroadcastHashJoin [codegen id : 51] +(148) BroadcastHashJoin [codegen id : 49] Left keys [1]: [c_current_cdemo_sk#138] Right keys [1]: [cd_demo_sk#149] Join type: Inner Join condition: NOT (cd_marital_status#148 = cd_marital_status#150) -(151) Project [codegen id : 51] +(149) Project [codegen id : 49] Output [14]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146] Input [18]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, cd_marital_status#148, cd_demo_sk#149, cd_marital_status#150] -(152) ReusedExchange [Reuses operator id: 71] +(150) ReusedExchange [Reuses operator id: 70] Output [1]: [p_promo_sk#151] -(153) BroadcastHashJoin [codegen id : 51] +(151) BroadcastHashJoin [codegen id : 49] Left keys [1]: [ss_promo_sk#112] Right keys [1]: [p_promo_sk#151] Join type: Inner Join condition: None -(154) Project [codegen id : 51] +(152) Project [codegen id : 49] Output [13]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146] Input [15]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, p_promo_sk#151] -(155) ReusedExchange [Reuses operator id: 77] +(153) ReusedExchange [Reuses operator id: 76] Output [2]: [hd_demo_sk#152, hd_income_band_sk#153] -(156) BroadcastHashJoin [codegen id : 51] +(154) BroadcastHashJoin [codegen id : 49] Left keys [1]: [ss_hdemo_sk#109] Right keys [1]: [hd_demo_sk#152] Join type: Inner Join condition: None -(157) Project [codegen id : 51] +(155) Project [codegen id : 49] Output [13]: [ss_item_sk#106, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153] Input [15]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, hd_demo_sk#152, hd_income_band_sk#153] -(158) ReusedExchange [Reuses operator id: 77] +(156) ReusedExchange [Reuses operator id: 76] Output [2]: [hd_demo_sk#154, hd_income_band_sk#155] -(159) BroadcastHashJoin [codegen id : 51] +(157) BroadcastHashJoin [codegen id : 49] Left keys [1]: [c_current_hdemo_sk#139] Right keys [1]: [hd_demo_sk#154] Join type: Inner Join condition: None -(160) Project [codegen id : 51] +(158) Project [codegen id : 49] Output [13]: [ss_item_sk#106, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155] Input [15]: [ss_item_sk#106, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153, hd_demo_sk#154, hd_income_band_sk#155] -(161) ReusedExchange [Reuses operator id: 86] +(159) ReusedExchange [Reuses operator id: 85] Output [5]: [ca_address_sk#156, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160] -(162) BroadcastHashJoin [codegen id : 51] +(160) BroadcastHashJoin [codegen id : 49] Left keys [1]: [ss_addr_sk#110] Right keys [1]: [ca_address_sk#156] Join type: Inner Join condition: None -(163) Project [codegen id : 51] +(161) Project [codegen id : 49] Output [16]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160] Input [18]: [ss_item_sk#106, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155, ca_address_sk#156, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160] -(164) ReusedExchange [Reuses operator id: 86] +(162) ReusedExchange [Reuses operator id: 85] Output [5]: [ca_address_sk#161, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165] -(165) BroadcastHashJoin [codegen id : 51] +(163) BroadcastHashJoin [codegen id : 49] Left keys [1]: [c_current_addr_sk#140] Right keys [1]: [ca_address_sk#161] Join type: Inner Join condition: None -(166) Project [codegen id : 51] +(164) Project [codegen id : 49] Output [19]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165] Input [21]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_address_sk#161, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165] -(167) ReusedExchange [Reuses operator id: 95] +(165) ReusedExchange [Reuses operator id: 94] Output [1]: [ib_income_band_sk#166] -(168) BroadcastHashJoin [codegen id : 51] +(166) BroadcastHashJoin [codegen id : 49] Left keys [1]: [hd_income_band_sk#153] Right keys [1]: [ib_income_band_sk#166] Join type: Inner Join condition: None -(169) Project [codegen id : 51] +(167) Project [codegen id : 49] Output [18]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165] Input [20]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, ib_income_band_sk#166] -(170) ReusedExchange [Reuses operator id: 95] +(168) ReusedExchange [Reuses operator id: 94] Output [1]: [ib_income_band_sk#167] -(171) BroadcastHashJoin [codegen id : 51] +(169) BroadcastHashJoin [codegen id : 49] Left keys [1]: [hd_income_band_sk#155] Right keys [1]: [ib_income_band_sk#167] Join type: Inner Join condition: None -(172) Project [codegen id : 51] +(170) Project [codegen id : 49] Output [17]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165] Input [19]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, ib_income_band_sk#167] -(173) ReusedExchange [Reuses operator id: 105] +(171) ReusedExchange [Reuses operator id: 104] Output [2]: [i_item_sk#168, i_product_name#169] -(174) BroadcastHashJoin [codegen id : 51] +(172) BroadcastHashJoin [codegen id : 49] Left keys [1]: [ss_item_sk#106] Right keys [1]: [i_item_sk#168] Join type: Inner Join condition: None -(175) Project [codegen id : 51] +(173) Project [codegen id : 49] Output [18]: [ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, d_year#144, d_year#146, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, i_item_sk#168, i_product_name#169] Input [19]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, i_item_sk#168, i_product_name#169] -(176) HashAggregate [codegen id : 51] +(174) HashAggregate [codegen id : 49] Input [18]: [ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, d_year#144, d_year#146, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, i_item_sk#168, i_product_name#169] Keys [15]: [i_product_name#169, i_item_sk#168, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, d_year#133, d_year#144, d_year#146] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#114)), partial_sum(UnscaledValue(ss_list_price#115)), partial_sum(UnscaledValue(ss_coupon_amt#116))] Aggregate Attributes [4]: [count#77, sum#170, sum#171, sum#172] Results [19]: [i_product_name#169, i_item_sk#168, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, d_year#133, d_year#144, d_year#146, count#81, sum#173, sum#174, sum#175] -(177) HashAggregate [codegen id : 51] +(175) HashAggregate [codegen id : 49] Input [19]: [i_product_name#169, i_item_sk#168, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, d_year#133, d_year#144, d_year#146, count#81, sum#173, sum#174, sum#175] Keys [15]: [i_product_name#169, i_item_sk#168, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, d_year#133, d_year#144, d_year#146] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#114)), sum(UnscaledValue(ss_list_price#115)), sum(UnscaledValue(ss_coupon_amt#116))] Aggregate Attributes [4]: [count(1)#85, sum(UnscaledValue(ss_wholesale_cost#114))#86, sum(UnscaledValue(ss_list_price#115))#87, sum(UnscaledValue(ss_coupon_amt#116))#88] Results [8]: [i_item_sk#168 AS item_sk#176, s_store_name#135 AS store_name#177, s_zip#136 AS store_zip#178, d_year#133 AS syear#179, count(1)#85 AS cnt#180, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#114))#86,17,2) AS s1#181, MakeDecimal(sum(UnscaledValue(ss_list_price#115))#87,17,2) AS s2#182, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#116))#88,17,2) AS s3#183] -(178) Exchange +(176) Exchange Input [8]: [item_sk#176, store_name#177, store_zip#178, syear#179, cnt#180, s1#181, s2#182, s3#183] -Arguments: hashpartitioning(item_sk#176, store_name#177, store_zip#178, 5), ENSURE_REQUIREMENTS, [plan_id=18] +Arguments: hashpartitioning(item_sk#176, store_name#177, store_zip#178, 5), ENSURE_REQUIREMENTS, [plan_id=16] -(179) Sort [codegen id : 52] +(177) Sort [codegen id : 50] Input [8]: [item_sk#176, store_name#177, store_zip#178, syear#179, cnt#180, s1#181, s2#182, s3#183] Arguments: [item_sk#176 ASC NULLS FIRST, store_name#177 ASC NULLS FIRST, store_zip#178 ASC NULLS FIRST], false, 0 -(180) SortMergeJoin [codegen id : 53] +(178) SortMergeJoin [codegen id : 51] Left keys [3]: [item_sk#90, store_name#91, store_zip#92] Right keys [3]: [item_sk#176, store_name#177, store_zip#178] Join type: Inner Join condition: (cnt#180 <= cnt#102) -(181) Project [codegen id : 53] +(179) Project [codegen id : 51] Output [21]: [product_name#89, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105, s1#181, s2#182, s3#183, syear#179, cnt#180] Input [25]: [product_name#89, item_sk#90, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105, item_sk#176, store_name#177, store_zip#178, syear#179, cnt#180, s1#181, s2#182, s3#183] -(182) Exchange +(180) Exchange Input [21]: [product_name#89, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105, s1#181, s2#182, s3#183, syear#179, cnt#180] -Arguments: rangepartitioning(product_name#89 ASC NULLS FIRST, store_name#91 ASC NULLS FIRST, cnt#180 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=19] +Arguments: rangepartitioning(product_name#89 ASC NULLS FIRST, store_name#91 ASC NULLS FIRST, cnt#180 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=17] -(183) Sort [codegen id : 54] +(181) Sort [codegen id : 52] Input [21]: [product_name#89, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105, s1#181, s2#182, s3#183, syear#179, cnt#180] Arguments: [product_name#89 ASC NULLS FIRST, store_name#91 ASC NULLS FIRST, cnt#180 ASC NULLS FIRST], true, 0 ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (187) -+- * Filter (186) - +- * ColumnarToRow (185) - +- CometScan parquet spark_catalog.default.date_dim (184) +BroadcastExchange (185) ++- * ColumnarToRow (184) + +- CometFilter (183) + +- CometScan parquet spark_catalog.default.date_dim (182) -(184) Scan parquet spark_catalog.default.date_dim +(182) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#37, d_year#38] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(185) ColumnarToRow [codegen id : 1] +(183) CometFilter Input [2]: [d_date_sk#37, d_year#38] +Condition : ((isnotnull(d_year#38) AND (d_year#38 = 1999)) AND isnotnull(d_date_sk#37)) -(186) Filter [codegen id : 1] +(184) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#37, d_year#38] -Condition : ((isnotnull(d_year#38) AND (d_year#38 = 1999)) AND isnotnull(d_date_sk#37)) -(187) BroadcastExchange +(185) BroadcastExchange Input [2]: [d_date_sk#37, d_year#38] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=20] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=18] -Subquery:2 Hosting operator id = 112 Hosting Expression = ss_sold_date_sk#117 IN dynamicpruning#118 -BroadcastExchange (191) -+- * Filter (190) - +- * ColumnarToRow (189) - +- CometScan parquet spark_catalog.default.date_dim (188) +Subquery:2 Hosting operator id = 111 Hosting Expression = ss_sold_date_sk#117 IN dynamicpruning#118 +BroadcastExchange (189) ++- * ColumnarToRow (188) + +- CometFilter (187) + +- CometScan parquet spark_catalog.default.date_dim (186) -(188) Scan parquet spark_catalog.default.date_dim +(186) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#132, d_year#133] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(189) ColumnarToRow [codegen id : 1] +(187) CometFilter Input [2]: [d_date_sk#132, d_year#133] +Condition : ((isnotnull(d_year#133) AND (d_year#133 = 2000)) AND isnotnull(d_date_sk#132)) -(190) Filter [codegen id : 1] +(188) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#132, d_year#133] -Condition : ((isnotnull(d_year#133) AND (d_year#133 = 2000)) AND isnotnull(d_date_sk#132)) -(191) BroadcastExchange +(189) BroadcastExchange Input [2]: [d_date_sk#132, d_year#133] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=21] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/simplified.txt index 76cac39ea..d972e0082 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/simplified.txt @@ -1,16 +1,16 @@ -WholeStageCodegen (54) +WholeStageCodegen (52) Sort [product_name,store_name,cnt] InputAdapter Exchange [product_name,store_name,cnt] #1 - WholeStageCodegen (53) + WholeStageCodegen (51) Project [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] SortMergeJoin [item_sk,store_name,store_zip,item_sk,store_name,store_zip,cnt,cnt] InputAdapter - WholeStageCodegen (26) + WholeStageCodegen (25) Sort [item_sk,store_name,store_zip] InputAdapter Exchange [item_sk,store_name,store_zip] #2 - WholeStageCodegen (25) + WholeStageCodegen (24) HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] @@ -46,148 +46,144 @@ WholeStageCodegen (54) Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] SortMergeJoin [ss_item_sk,cs_item_sk] InputAdapter - WholeStageCodegen (3) + WholeStageCodegen (2) Sort [ss_item_sk] InputAdapter Exchange [ss_item_sk] #3 - WholeStageCodegen (2) - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_ticket_number,ss_store_sk,ss_customer_sk,ss_cdemo_sk,ss_promo_sk,ss_hdemo_sk,ss_addr_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - Project [sr_item_sk,sr_ticket_number] - Filter [sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] + CometBroadcastExchange #4 + CometFilter [ss_item_sk,ss_ticket_number,ss_store_sk,ss_customer_sk,ss_cdemo_sk,ss_promo_sk,ss_hdemo_sk,ss_addr_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_year,d_date_sk] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] InputAdapter - WholeStageCodegen (9) + WholeStageCodegen (8) Sort [cs_item_sk] Project [cs_item_sk] Filter [sale,refund] HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] InputAdapter Exchange [cs_item_sk] #6 - WholeStageCodegen (8) + WholeStageCodegen (7) HashAggregate [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [sum,sum,isEmpty,sum,sum,isEmpty] Project [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] InputAdapter - WholeStageCodegen (5) + WholeStageCodegen (4) Sort [cs_item_sk,cs_order_number] InputAdapter Exchange [cs_item_sk,cs_order_number] #7 - WholeStageCodegen (4) - Project [cs_item_sk,cs_order_number,cs_ext_list_price] - Filter [cs_item_sk,cs_order_number] - ColumnarToRow - InputAdapter + WholeStageCodegen (3) + ColumnarToRow + InputAdapter + CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] + CometFilter [cs_item_sk,cs_order_number] CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] InputAdapter - WholeStageCodegen (7) + WholeStageCodegen (6) Sort [cr_item_sk,cr_order_number] InputAdapter Exchange [cr_item_sk,cr_order_number] #8 - WholeStageCodegen (6) - Project [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter + WholeStageCodegen (5) + ColumnarToRow + InputAdapter + CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometFilter [cr_item_sk,cr_order_number] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] InputAdapter ReusedExchange [d_date_sk,d_year] #5 InputAdapter BroadcastExchange #9 - WholeStageCodegen (11) - Filter [s_store_sk,s_store_name,s_zip] - ColumnarToRow - InputAdapter + WholeStageCodegen (10) + ColumnarToRow + InputAdapter + CometFilter [s_store_sk,s_store_name,s_zip] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] InputAdapter BroadcastExchange #10 - WholeStageCodegen (12) - Filter [c_customer_sk,c_first_sales_date_sk,c_first_shipto_date_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (11) + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_first_sales_date_sk,c_first_shipto_date_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] InputAdapter BroadcastExchange #11 - WholeStageCodegen (13) - Filter [d_date_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (12) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [d_date_sk,d_year] #11 InputAdapter BroadcastExchange #12 - WholeStageCodegen (15) - Filter [cd_demo_sk,cd_marital_status] - ColumnarToRow - InputAdapter + WholeStageCodegen (14) + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk,cd_marital_status] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] InputAdapter ReusedExchange [cd_demo_sk,cd_marital_status] #12 InputAdapter BroadcastExchange #13 - WholeStageCodegen (17) - Filter [p_promo_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (16) + ColumnarToRow + InputAdapter + CometFilter [p_promo_sk] CometScan parquet spark_catalog.default.promotion [p_promo_sk] InputAdapter BroadcastExchange #14 - WholeStageCodegen (18) - Filter [hd_demo_sk,hd_income_band_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (17) + ColumnarToRow + InputAdapter + CometFilter [hd_demo_sk,hd_income_band_sk] CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] InputAdapter ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 InputAdapter BroadcastExchange #15 - WholeStageCodegen (20) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (19) + ColumnarToRow + InputAdapter + CometFilter [ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] InputAdapter ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 InputAdapter BroadcastExchange #16 - WholeStageCodegen (22) - Filter [ib_income_band_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (21) + ColumnarToRow + InputAdapter + CometFilter [ib_income_band_sk] CometScan parquet spark_catalog.default.income_band [ib_income_band_sk] InputAdapter ReusedExchange [ib_income_band_sk] #16 InputAdapter BroadcastExchange #17 - WholeStageCodegen (24) - Project [i_item_sk,i_product_name] - Filter [i_current_price,i_color,i_item_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (23) + ColumnarToRow + InputAdapter + CometProject [i_item_sk,i_product_name] + CometFilter [i_current_price,i_color,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] InputAdapter - WholeStageCodegen (52) + WholeStageCodegen (50) Sort [item_sk,store_name,store_zip] InputAdapter Exchange [item_sk,store_name,store_zip] #18 - WholeStageCodegen (51) + WholeStageCodegen (49) HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] @@ -223,34 +219,30 @@ WholeStageCodegen (54) Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] SortMergeJoin [ss_item_sk,cs_item_sk] InputAdapter - WholeStageCodegen (29) + WholeStageCodegen (27) Sort [ss_item_sk] InputAdapter Exchange [ss_item_sk] #19 - WholeStageCodegen (28) - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] - InputAdapter - BroadcastExchange #20 - WholeStageCodegen (27) - Filter [ss_item_sk,ss_ticket_number,ss_store_sk,ss_customer_sk,ss_cdemo_sk,ss_promo_sk,ss_hdemo_sk,ss_addr_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #21 - WholeStageCodegen (1) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - Project [sr_item_sk,sr_ticket_number] - Filter [sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter + WholeStageCodegen (26) + ColumnarToRow + InputAdapter + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] + CometBroadcastExchange #20 + CometFilter [ss_item_sk,ss_ticket_number,ss_store_sk,ss_customer_sk,ss_cdemo_sk,ss_promo_sk,ss_hdemo_sk,ss_addr_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #2 + BroadcastExchange #21 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_year,d_date_sk] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] InputAdapter - WholeStageCodegen (35) + WholeStageCodegen (33) Sort [cs_item_sk] Project [cs_item_sk] Filter [sale,refund] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/explain.txt index b9107a072..eda84bb52 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/explain.txt @@ -6,8 +6,8 @@ TakeOrderedAndProject (39) : +- * BroadcastHashJoin Inner BuildRight (21) : :- * Project (16) : : +- * BroadcastHashJoin Inner BuildRight (15) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) + : : :- * ColumnarToRow (3) + : : : +- CometFilter (2) : : : +- CometScan parquet spark_catalog.default.store (1) : : +- BroadcastExchange (14) : : +- * Filter (13) @@ -16,13 +16,13 @@ TakeOrderedAndProject (39) : : +- * HashAggregate (10) : : +- * Project (9) : : +- * BroadcastHashJoin Inner BuildRight (8) - : : :- * Filter (6) - : : : +- * ColumnarToRow (5) + : : :- * ColumnarToRow (6) + : : : +- CometFilter (5) : : : +- CometScan parquet spark_catalog.default.store_sales (4) : : +- ReusedExchange (7) : +- BroadcastExchange (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) + : +- * ColumnarToRow (19) + : +- CometFilter (18) : +- CometScan parquet spark_catalog.default.item (17) +- BroadcastExchange (36) +- * Filter (35) @@ -34,8 +34,8 @@ TakeOrderedAndProject (39) +- * HashAggregate (29) +- * Project (28) +- * BroadcastHashJoin Inner BuildRight (27) - :- * Filter (25) - : +- * ColumnarToRow (24) + :- * ColumnarToRow (25) + : +- CometFilter (24) : +- CometScan parquet spark_catalog.default.store_sales (23) +- ReusedExchange (26) @@ -47,12 +47,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 9] +(2) CometFilter Input [2]: [s_store_sk#1, s_store_name#2] +Condition : isnotnull(s_store_sk#1) -(3) Filter [codegen id : 9] +(3) ColumnarToRow [codegen id : 9] Input [2]: [s_store_sk#1, s_store_name#2] -Condition : isnotnull(s_store_sk#1) (4) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] @@ -62,12 +62,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 2] +(5) CometFilter Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] +Condition : (isnotnull(ss_store_sk#4) AND isnotnull(ss_item_sk#3)) -(6) Filter [codegen id : 2] +(6) ColumnarToRow [codegen id : 2] Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Condition : (isnotnull(ss_store_sk#4) AND isnotnull(ss_item_sk#3)) (7) ReusedExchange [Reuses operator id: 44] Output [1]: [d_date_sk#8] @@ -125,12 +125,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 4] +(18) CometFilter Input [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] +Condition : isnotnull(i_item_sk#13) -(19) Filter [codegen id : 4] +(19) ColumnarToRow [codegen id : 4] Input [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Condition : isnotnull(i_item_sk#13) (20) BroadcastExchange Input [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] @@ -154,12 +154,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#21), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 6] +(24) CometFilter Input [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] +Condition : isnotnull(ss_store_sk#19) -(25) Filter [codegen id : 6] +(25) ColumnarToRow [codegen id : 6] Input [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] -Condition : isnotnull(ss_store_sk#19) (26) ReusedExchange [Reuses operator id: 44] Output [1]: [d_date_sk#23] @@ -236,9 +236,9 @@ Arguments: 100, [s_store_name#2 ASC NULLS FIRST, i_item_desc#14 ASC NULLS FIRST] Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 BroadcastExchange (44) -+- * Project (43) - +- * Filter (42) - +- * ColumnarToRow (41) ++- * ColumnarToRow (43) + +- CometProject (42) + +- CometFilter (41) +- CometScan parquet spark_catalog.default.date_dim (40) @@ -249,16 +249,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1176), LessThanOrEqual(d_month_seq,1187), IsNotNull(d_date_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#8, d_month_seq#34] - -(42) Filter [codegen id : 1] +(41) CometFilter Input [2]: [d_date_sk#8, d_month_seq#34] Condition : (((isnotnull(d_month_seq#34) AND (d_month_seq#34 >= 1176)) AND (d_month_seq#34 <= 1187)) AND isnotnull(d_date_sk#8)) -(43) Project [codegen id : 1] -Output [1]: [d_date_sk#8] +(42) CometProject Input [2]: [d_date_sk#8, d_month_seq#34] +Arguments: [d_date_sk#8], [d_date_sk#8] + +(43) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#8] (44) BroadcastExchange Input [1]: [d_date_sk#8] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/simplified.txt index 6670cbdac..33b695e81 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/simplified.txt @@ -6,9 +6,9 @@ TakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholes BroadcastHashJoin [ss_item_sk,i_item_sk] Project [s_store_name,ss_store_sk,ss_item_sk,revenue] BroadcastHashJoin [s_store_sk,ss_store_sk] - Filter [s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name] InputAdapter BroadcastExchange #1 @@ -21,26 +21,26 @@ TakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholes HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum] Project [ss_item_sk,ss_store_sk,ss_sales_price] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk,ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_month_seq,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter ReusedExchange [d_date_sk] #3 InputAdapter BroadcastExchange #4 WholeStageCodegen (4) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] InputAdapter BroadcastExchange #5 @@ -58,9 +58,9 @@ TakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholes HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum] Project [ss_item_sk,ss_store_sk,ss_sales_price] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/explain.txt index b3e8005c9..a26c457aa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/explain.txt @@ -15,23 +15,23 @@ TakeOrderedAndProject (52) : : : +- * BroadcastHashJoin Inner BuildRight (11) : : : :- * Project (9) : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) + : : : : +- * ColumnarToRow (6) + : : : : +- CometFilter (5) : : : : +- CometScan parquet spark_catalog.default.warehouse (4) : : : +- ReusedExchange (10) : : +- BroadcastExchange (17) - : : +- * Project (16) - : : +- * Filter (15) - : : +- * ColumnarToRow (14) + : : +- * ColumnarToRow (16) + : : +- CometProject (15) + : : +- CometFilter (14) : : +- CometScan parquet spark_catalog.default.time_dim (13) : +- BroadcastExchange (24) - : +- * Project (23) - : +- * Filter (22) - : +- * ColumnarToRow (21) + : +- * ColumnarToRow (23) + : +- CometProject (22) + : +- CometFilter (21) : +- CometScan parquet spark_catalog.default.ship_mode (20) +- * HashAggregate (47) +- Exchange (46) @@ -44,8 +44,8 @@ TakeOrderedAndProject (52) : : +- * BroadcastHashJoin Inner BuildRight (37) : : :- * Project (35) : : : +- * BroadcastHashJoin Inner BuildRight (34) - : : : :- * Filter (32) - : : : : +- * ColumnarToRow (31) + : : : :- * ColumnarToRow (32) + : : : : +- CometFilter (31) : : : : +- CometScan parquet spark_catalog.default.catalog_sales (30) : : : +- ReusedExchange (33) : : +- ReusedExchange (36) @@ -61,12 +61,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#7), dynamicpruningexpression(ws_sol PushedFilters: [IsNotNull(ws_warehouse_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_ship_mode_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] +(2) CometFilter Input [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] +Condition : ((isnotnull(ws_warehouse_sk#3) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_ship_mode_sk#2)) -(3) Filter [codegen id : 5] +(3) ColumnarToRow [codegen id : 5] Input [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] -Condition : ((isnotnull(ws_warehouse_sk#3) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_ship_mode_sk#2)) (4) Scan parquet spark_catalog.default.warehouse Output [7]: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15] @@ -75,12 +75,12 @@ Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [7]: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15] +Condition : isnotnull(w_warehouse_sk#9) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [7]: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15] -Condition : isnotnull(w_warehouse_sk#9) (7) BroadcastExchange Input [7]: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15] @@ -116,16 +116,16 @@ Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_time), GreaterThanOrEqual(t_time,30838), LessThanOrEqual(t_time,59638), IsNotNull(t_time_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] -Input [2]: [t_time_sk#19, t_time#20] - -(15) Filter [codegen id : 3] +(14) CometFilter Input [2]: [t_time_sk#19, t_time#20] Condition : (((isnotnull(t_time#20) AND (t_time#20 >= 30838)) AND (t_time#20 <= 59638)) AND isnotnull(t_time_sk#19)) -(16) Project [codegen id : 3] -Output [1]: [t_time_sk#19] +(15) CometProject Input [2]: [t_time_sk#19, t_time#20] +Arguments: [t_time_sk#19], [t_time_sk#19] + +(16) ColumnarToRow [codegen id : 3] +Input [1]: [t_time_sk#19] (17) BroadcastExchange Input [1]: [t_time_sk#19] @@ -148,16 +148,16 @@ Location [not included in comparison]/{warehouse_dir}/ship_mode] PushedFilters: [In(sm_carrier, [BARIAN ,DHL ]), IsNotNull(sm_ship_mode_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 4] -Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] - -(22) Filter [codegen id : 4] +(21) CometFilter Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] Condition : (sm_carrier#22 IN (DHL ,BARIAN ) AND isnotnull(sm_ship_mode_sk#21)) -(23) Project [codegen id : 4] -Output [1]: [sm_ship_mode_sk#21] +(22) CometProject Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] +Arguments: [sm_ship_mode_sk#21], [sm_ship_mode_sk#21] + +(23) ColumnarToRow [codegen id : 4] +Input [1]: [sm_ship_mode_sk#21] (24) BroadcastExchange Input [1]: [sm_ship_mode_sk#21] @@ -199,12 +199,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#175), dynamicpruningexpression(cs_s PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_sold_time_sk), IsNotNull(cs_ship_mode_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 11] +(31) CometFilter Input [7]: [cs_sold_time_sk#169, cs_ship_mode_sk#170, cs_warehouse_sk#171, cs_quantity#172, cs_sales_price#173, cs_net_paid_inc_tax#174, cs_sold_date_sk#175] +Condition : ((isnotnull(cs_warehouse_sk#171) AND isnotnull(cs_sold_time_sk#169)) AND isnotnull(cs_ship_mode_sk#170)) -(32) Filter [codegen id : 11] +(32) ColumnarToRow [codegen id : 11] Input [7]: [cs_sold_time_sk#169, cs_ship_mode_sk#170, cs_warehouse_sk#171, cs_quantity#172, cs_sales_price#173, cs_net_paid_inc_tax#174, cs_sold_date_sk#175] -Condition : ((isnotnull(cs_warehouse_sk#171) AND isnotnull(cs_sold_time_sk#169)) AND isnotnull(cs_ship_mode_sk#170)) (33) ReusedExchange [Reuses operator id: 7] Output [7]: [w_warehouse_sk#177, w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183] @@ -304,8 +304,8 @@ Arguments: 100, [w_warehouse_name#10 ASC NULLS FIRST], [w_warehouse_name#10, w_w Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#7 IN dynamicpruning#8 BroadcastExchange (56) -+- * Filter (55) - +- * ColumnarToRow (54) ++- * ColumnarToRow (55) + +- CometFilter (54) +- CometScan parquet spark_catalog.default.date_dim (53) @@ -316,12 +316,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(54) ColumnarToRow [codegen id : 1] +(54) CometFilter Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) -(55) Filter [codegen id : 1] +(55) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#16, d_year#17, d_moy#18] -Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) (56) BroadcastExchange Input [3]: [d_date_sk#16, d_year#17, d_moy#18] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/simplified.txt index 4c2daaa3e..8ed74582f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/simplified.txt @@ -21,41 +21,41 @@ TakeOrderedAndProject [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_stat BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] BroadcastHashJoin [ws_warehouse_sk,w_warehouse_sk] - Filter [ws_warehouse_sk,ws_sold_time_sk,ws_ship_mode_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_warehouse_sk,ws_sold_time_sk,ws_ship_mode_sk] CometScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #4 WholeStageCodegen (1) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [w_warehouse_sk] CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] InputAdapter ReusedExchange [d_date_sk,d_year,d_moy] #3 InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - Project [t_time_sk] - Filter [t_time,t_time_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [t_time_sk] + CometFilter [t_time,t_time_sk] CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_time] InputAdapter BroadcastExchange #6 WholeStageCodegen (4) - Project [sm_ship_mode_sk] - Filter [sm_carrier,sm_ship_mode_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [sm_ship_mode_sk] + CometFilter [sm_carrier,sm_ship_mode_sk] CometScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_carrier] WholeStageCodegen (12) HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_moy = 1) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] @@ -71,9 +71,9 @@ TakeOrderedAndProject [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_stat BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] BroadcastHashJoin [cs_warehouse_sk,w_warehouse_sk] - Filter [cs_warehouse_sk,cs_sold_time_sk,cs_ship_mode_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_warehouse_sk,cs_sold_time_sk,cs_ship_mode_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/explain.txt index 192e065e5..d1a6a4f47 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/explain.txt @@ -17,17 +17,17 @@ TakeOrderedAndProject (30) : +- * BroadcastHashJoin Inner BuildRight (11) : :- * Project (6) : : +- * BroadcastHashJoin Inner BuildRight (5) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) + : : :- * ColumnarToRow (3) + : : : +- CometFilter (2) : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : +- ReusedExchange (4) : +- BroadcastExchange (10) - : +- * Filter (9) - : +- * ColumnarToRow (8) + : +- * ColumnarToRow (9) + : +- CometFilter (8) : +- CometScan parquet spark_catalog.default.store (7) +- BroadcastExchange (16) - +- * Filter (15) - +- * ColumnarToRow (14) + +- * ColumnarToRow (15) + +- CometFilter (14) +- CometScan parquet spark_catalog.default.item (13) @@ -39,12 +39,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] +Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) -(3) Filter [codegen id : 4] +(3) ColumnarToRow [codegen id : 4] Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) (4) ReusedExchange [Reuses operator id: 35] Output [4]: [d_date_sk#7, d_year#8, d_moy#9, d_qoy#10] @@ -66,12 +66,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] +(8) CometFilter Input [2]: [s_store_sk#11, s_store_id#12] +Condition : isnotnull(s_store_sk#11) -(9) Filter [codegen id : 2] +(9) ColumnarToRow [codegen id : 2] Input [2]: [s_store_sk#11, s_store_id#12] -Condition : isnotnull(s_store_sk#11) (10) BroadcastExchange Input [2]: [s_store_sk#11, s_store_id#12] @@ -94,12 +94,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] +(14) CometFilter Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] +Condition : isnotnull(i_item_sk#13) -(15) Filter [codegen id : 3] +(15) ColumnarToRow [codegen id : 3] Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Condition : isnotnull(i_item_sk#13) (16) BroadcastExchange Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] @@ -173,9 +173,9 @@ Arguments: 100, [i_category#18 ASC NULLS FIRST, i_class#19 ASC NULLS FIRST, i_br Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 BroadcastExchange (35) -+- * Project (34) - +- * Filter (33) - +- * ColumnarToRow (32) ++- * ColumnarToRow (34) + +- CometProject (33) + +- CometFilter (32) +- CometScan parquet spark_catalog.default.date_dim (31) @@ -186,16 +186,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 1] -Input [5]: [d_date_sk#7, d_month_seq#34, d_year#8, d_moy#9, d_qoy#10] - -(33) Filter [codegen id : 1] +(32) CometFilter Input [5]: [d_date_sk#7, d_month_seq#34, d_year#8, d_moy#9, d_qoy#10] Condition : (((isnotnull(d_month_seq#34) AND (d_month_seq#34 >= 1200)) AND (d_month_seq#34 <= 1211)) AND isnotnull(d_date_sk#7)) -(34) Project [codegen id : 1] -Output [4]: [d_date_sk#7, d_year#8, d_moy#9, d_qoy#10] +(33) CometProject Input [5]: [d_date_sk#7, d_month_seq#34, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#7, d_year#8, d_moy#9, d_qoy#10], [d_date_sk#7, d_year#8, d_moy#9, d_qoy#10] + +(34) ColumnarToRow [codegen id : 1] +Input [4]: [d_date_sk#7, d_year#8, d_moy#9, d_qoy#10] (35) BroadcastExchange Input [4]: [d_date_sk#7, d_year#8, d_moy#9, d_qoy#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/simplified.txt index ecc706b76..e10def397 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/simplified.txt @@ -23,31 +23,31 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_ BroadcastHashJoin [ss_store_sk,s_store_sk] Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk,ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk,d_year,d_moy,d_qoy] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_year,d_moy,d_qoy] + CometFilter [d_month_seq,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] InputAdapter ReusedExchange [d_date_sk,d_year,d_moy,d_qoy] #3 InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/explain.txt index 48b46fab5..734b6c11c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/explain.txt @@ -15,27 +15,27 @@ TakeOrderedAndProject (39) : : : : +- * BroadcastHashJoin Inner BuildRight (12) : : : : :- * Project (6) : : : : : +- * BroadcastHashJoin Inner BuildRight (5) - : : : : : :- * Filter (3) - : : : : : : +- * ColumnarToRow (2) + : : : : : :- * ColumnarToRow (3) + : : : : : : +- CometFilter (2) : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : : : +- ReusedExchange (4) : : : : +- BroadcastExchange (11) - : : : : +- * Project (10) - : : : : +- * Filter (9) - : : : : +- * ColumnarToRow (8) + : : : : +- * ColumnarToRow (10) + : : : : +- CometProject (9) + : : : : +- CometFilter (8) : : : : +- CometScan parquet spark_catalog.default.store (7) : : : +- BroadcastExchange (18) - : : : +- * Project (17) - : : : +- * Filter (16) - : : : +- * ColumnarToRow (15) + : : : +- * ColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometFilter (15) : : : +- CometScan parquet spark_catalog.default.household_demographics (14) : : +- BroadcastExchange (24) - : : +- * Filter (23) - : : +- * ColumnarToRow (22) + : : +- * ColumnarToRow (23) + : : +- CometFilter (22) : : +- CometScan parquet spark_catalog.default.customer_address (21) : +- BroadcastExchange (33) - : +- * Filter (32) - : +- * ColumnarToRow (31) + : +- * ColumnarToRow (32) + : +- CometFilter (31) : +- CometScan parquet spark_catalog.default.customer (30) +- ReusedExchange (36) @@ -48,12 +48,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#9), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] +(2) CometFilter Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] +Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) -(3) Filter [codegen id : 5] +(3) ColumnarToRow [codegen id : 5] Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] -Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) (4) ReusedExchange [Reuses operator id: 44] Output [1]: [d_date_sk#11] @@ -75,16 +75,16 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#12, s_city#13] - -(9) Filter [codegen id : 2] +(8) CometFilter Input [2]: [s_store_sk#12, s_city#13] Condition : (s_city#13 IN (Midway,Fairview) AND isnotnull(s_store_sk#12)) -(10) Project [codegen id : 2] -Output [1]: [s_store_sk#12] +(9) CometProject Input [2]: [s_store_sk#12, s_city#13] +Arguments: [s_store_sk#12], [s_store_sk#12] + +(10) ColumnarToRow [codegen id : 2] +Input [1]: [s_store_sk#12] (11) BroadcastExchange Input [1]: [s_store_sk#12] @@ -107,16 +107,16 @@ Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 3] -Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] - -(16) Filter [codegen id : 3] +(15) CometFilter Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] Condition : (((hd_dep_count#15 = 4) OR (hd_vehicle_count#16 = 3)) AND isnotnull(hd_demo_sk#14)) -(17) Project [codegen id : 3] -Output [1]: [hd_demo_sk#14] +(16) CometProject Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] +Arguments: [hd_demo_sk#14], [hd_demo_sk#14] + +(17) ColumnarToRow [codegen id : 3] +Input [1]: [hd_demo_sk#14] (18) BroadcastExchange Input [1]: [hd_demo_sk#14] @@ -139,12 +139,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] ReadSchema: struct -(22) ColumnarToRow [codegen id : 4] +(22) CometFilter Input [2]: [ca_address_sk#17, ca_city#18] +Condition : (isnotnull(ca_address_sk#17) AND isnotnull(ca_city#18)) -(23) Filter [codegen id : 4] +(23) ColumnarToRow [codegen id : 4] Input [2]: [ca_address_sk#17, ca_city#18] -Condition : (isnotnull(ca_address_sk#17) AND isnotnull(ca_city#18)) (24) BroadcastExchange Input [2]: [ca_address_sk#17, ca_city#18] @@ -185,12 +185,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 6] +(31) CometFilter Input [4]: [c_customer_sk#32, c_current_addr_sk#33, c_first_name#34, c_last_name#35] +Condition : (isnotnull(c_customer_sk#32) AND isnotnull(c_current_addr_sk#33)) -(32) Filter [codegen id : 6] +(32) ColumnarToRow [codegen id : 6] Input [4]: [c_customer_sk#32, c_current_addr_sk#33, c_first_name#34, c_last_name#35] -Condition : (isnotnull(c_customer_sk#32) AND isnotnull(c_current_addr_sk#33)) (33) BroadcastExchange Input [4]: [c_customer_sk#32, c_current_addr_sk#33, c_first_name#34, c_last_name#35] @@ -227,9 +227,9 @@ Arguments: 100, [c_last_name#35 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FI Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 BroadcastExchange (44) -+- * Project (43) - +- * Filter (42) - +- * ColumnarToRow (41) ++- * ColumnarToRow (43) + +- CometProject (42) + +- CometFilter (41) +- CometScan parquet spark_catalog.default.date_dim (40) @@ -240,16 +240,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#11, d_year#38, d_dom#39] - -(42) Filter [codegen id : 1] +(41) CometFilter Input [3]: [d_date_sk#11, d_year#38, d_dom#39] Condition : ((((isnotnull(d_dom#39) AND (d_dom#39 >= 1)) AND (d_dom#39 <= 2)) AND d_year#38 IN (1999,2000,2001)) AND isnotnull(d_date_sk#11)) -(43) Project [codegen id : 1] -Output [1]: [d_date_sk#11] +(42) CometProject Input [3]: [d_date_sk#11, d_year#38, d_dom#39] +Arguments: [d_date_sk#11], [d_date_sk#11] + +(43) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#11] (44) BroadcastExchange Input [1]: [d_date_sk#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/simplified.txt index 20d9432d1..f2680bebb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/simplified.txt @@ -17,49 +17,49 @@ TakeOrderedAndProject [c_last_name,ss_ticket_number,c_first_name,ca_city,bought_ BroadcastHashJoin [ss_store_sk,s_store_sk] Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_addr_sk,ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk,ss_hdemo_sk,ss_addr_sk,ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dom,d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_dom,d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] InputAdapter ReusedExchange [d_date_sk] #2 InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Project [s_store_sk] - Filter [s_city,s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [s_store_sk] + CometFilter [s_city,s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_city] InputAdapter BroadcastExchange #4 WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [hd_demo_sk] + CometFilter [hd_dep_count,hd_vehicle_count,hd_demo_sk] CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] InputAdapter BroadcastExchange #5 WholeStageCodegen (4) - Filter [ca_address_sk,ca_city] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ca_address_sk,ca_city] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] InputAdapter BroadcastExchange #6 WholeStageCodegen (6) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_current_addr_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] InputAdapter ReusedExchange [ca_address_sk,ca_city] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/explain.txt index 522bd93e5..d56b01d73 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/explain.txt @@ -11,8 +11,8 @@ TakeOrderedAndProject (42) : : +- * BroadcastHashJoin LeftAnti BuildRight (24) : : :- * BroadcastHashJoin LeftAnti BuildRight (17) : : : :- * BroadcastHashJoin LeftSemi BuildRight (10) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.customer (1) : : : : +- BroadcastExchange (9) : : : : +- * Project (8) @@ -33,13 +33,13 @@ TakeOrderedAndProject (42) : : : +- CometScan parquet spark_catalog.default.catalog_sales (18) : : +- ReusedExchange (20) : +- BroadcastExchange (30) - : +- * Project (29) - : +- * Filter (28) - : +- * ColumnarToRow (27) + : +- * ColumnarToRow (29) + : +- CometProject (28) + : +- CometFilter (27) : +- CometScan parquet spark_catalog.default.customer_address (26) +- BroadcastExchange (36) - +- * Filter (35) - +- * ColumnarToRow (34) + +- * ColumnarToRow (35) + +- CometFilter (34) +- CometScan parquet spark_catalog.default.customer_demographics (33) @@ -50,12 +50,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 9] +(2) CometFilter Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) -(3) Filter [codegen id : 9] +(3) ColumnarToRow [codegen id : 9] Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) (4) Scan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] @@ -167,16 +167,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [In(ca_state, [GA,KY,NM]), IsNotNull(ca_address_sk)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 7] -Input [2]: [ca_address_sk#16, ca_state#17] - -(28) Filter [codegen id : 7] +(27) CometFilter Input [2]: [ca_address_sk#16, ca_state#17] Condition : (ca_state#17 IN (KY,GA,NM) AND isnotnull(ca_address_sk#16)) -(29) Project [codegen id : 7] -Output [1]: [ca_address_sk#16] +(28) CometProject Input [2]: [ca_address_sk#16, ca_state#17] +Arguments: [ca_address_sk#16], [ca_address_sk#16] + +(29) ColumnarToRow [codegen id : 7] +Input [1]: [ca_address_sk#16] (30) BroadcastExchange Input [1]: [ca_address_sk#16] @@ -199,12 +199,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 8] +(34) CometFilter Input [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_education_status#21, cd_purchase_estimate#22, cd_credit_rating#23] +Condition : isnotnull(cd_demo_sk#18) -(35) Filter [codegen id : 8] +(35) ColumnarToRow [codegen id : 8] Input [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_education_status#21, cd_purchase_estimate#22, cd_credit_rating#23] -Condition : isnotnull(cd_demo_sk#18) (36) BroadcastExchange Input [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_education_status#21, cd_purchase_estimate#22, cd_credit_rating#23] @@ -246,9 +246,9 @@ Arguments: 100, [cd_gender#19 ASC NULLS FIRST, cd_marital_status#20 ASC NULLS FI Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 BroadcastExchange (47) -+- * Project (46) - +- * Filter (45) - +- * ColumnarToRow (44) ++- * ColumnarToRow (46) + +- CometProject (45) + +- CometFilter (44) +- CometScan parquet spark_catalog.default.date_dim (43) @@ -259,16 +259,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,6), IsNotNull(d_date_sk)] ReadSchema: struct -(44) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#7, d_year#30, d_moy#31] - -(45) Filter [codegen id : 1] +(44) CometFilter Input [3]: [d_date_sk#7, d_year#30, d_moy#31] Condition : (((((isnotnull(d_year#30) AND isnotnull(d_moy#31)) AND (d_year#30 = 2001)) AND (d_moy#31 >= 4)) AND (d_moy#31 <= 6)) AND isnotnull(d_date_sk#7)) -(46) Project [codegen id : 1] -Output [1]: [d_date_sk#7] +(45) CometProject Input [3]: [d_date_sk#7, d_year#30, d_moy#31] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(46) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#7] (47) BroadcastExchange Input [1]: [d_date_sk#7] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/simplified.txt index 90f6fc876..f5b4eccfb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/simplified.txt @@ -13,9 +13,9 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_current_addr_sk,c_current_cdemo_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] InputAdapter BroadcastExchange #2 @@ -28,10 +28,10 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_moy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter ReusedExchange [d_date_sk] #3 @@ -60,15 +60,15 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter BroadcastExchange #6 WholeStageCodegen (7) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_state,ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #7 WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/explain.txt index 47e7253dd..790a917e3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/explain.txt @@ -11,23 +11,23 @@ TakeOrderedAndProject (30) : : +- * BroadcastHashJoin Inner BuildRight (12) : : :- * Project (10) : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) + : : : +- * ColumnarToRow (7) + : : : +- CometProject (6) + : : : +- CometFilter (5) : : : +- CometScan parquet spark_catalog.default.customer_demographics (4) : : +- ReusedExchange (11) : +- BroadcastExchange (17) - : +- * Filter (16) - : +- * ColumnarToRow (15) + : +- * ColumnarToRow (16) + : +- CometFilter (15) : +- CometScan parquet spark_catalog.default.item (14) +- BroadcastExchange (24) - +- * Project (23) - +- * Filter (22) - +- * ColumnarToRow (21) + +- * ColumnarToRow (23) + +- CometProject (22) + +- CometFilter (21) +- CometScan parquet spark_catalog.default.promotion (20) @@ -39,12 +39,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] +(2) CometFilter Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) -(3) Filter [codegen id : 5] +(3) ColumnarToRow [codegen id : 5] Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) (4) Scan parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] @@ -53,16 +53,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_marital_status,S), EqualTo(cd_education_status,College ), IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] - -(6) Filter [codegen id : 1] +(5) CometFilter Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (cd_gender#11 = M)) AND (cd_marital_status#12 = S)) AND (cd_education_status#13 = College )) AND isnotnull(cd_demo_sk#10)) -(7) Project [codegen id : 1] -Output [1]: [cd_demo_sk#10] +(6) CometProject Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] +Arguments: [cd_demo_sk#10], [cd_demo_sk#10] + +(7) ColumnarToRow [codegen id : 1] +Input [1]: [cd_demo_sk#10] (8) BroadcastExchange Input [1]: [cd_demo_sk#10] @@ -98,12 +98,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 3] +(15) CometFilter Input [2]: [i_item_sk#15, i_item_id#16] +Condition : isnotnull(i_item_sk#15) -(16) Filter [codegen id : 3] +(16) ColumnarToRow [codegen id : 3] Input [2]: [i_item_sk#15, i_item_id#16] -Condition : isnotnull(i_item_sk#15) (17) BroadcastExchange Input [2]: [i_item_sk#15, i_item_id#16] @@ -126,16 +126,16 @@ Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [Or(EqualTo(p_channel_email,N),EqualTo(p_channel_event,N)), IsNotNull(p_promo_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 4] -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] - -(22) Filter [codegen id : 4] +(21) CometFilter Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] Condition : (((p_channel_email#18 = N) OR (p_channel_event#19 = N)) AND isnotnull(p_promo_sk#17)) -(23) Project [codegen id : 4] -Output [1]: [p_promo_sk#17] +(22) CometProject Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] +Arguments: [p_promo_sk#17], [p_promo_sk#17] + +(23) ColumnarToRow [codegen id : 4] +Input [1]: [p_promo_sk#17] (24) BroadcastExchange Input [1]: [p_promo_sk#17] @@ -177,9 +177,9 @@ Arguments: 100, [i_item_id#16 ASC NULLS FIRST], [i_item_id#16, agg1#40, agg2#41, Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 BroadcastExchange (35) -+- * Project (34) - +- * Filter (33) - +- * ColumnarToRow (32) ++- * ColumnarToRow (34) + +- CometProject (33) + +- CometFilter (32) +- CometScan parquet spark_catalog.default.date_dim (31) @@ -190,16 +190,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#14, d_year#44] - -(33) Filter [codegen id : 1] +(32) CometFilter Input [2]: [d_date_sk#14, d_year#44] Condition : ((isnotnull(d_year#44) AND (d_year#44 = 2000)) AND isnotnull(d_date_sk#14)) -(34) Project [codegen id : 1] -Output [1]: [d_date_sk#14] +(33) CometProject Input [2]: [d_date_sk#14, d_year#44] +Arguments: [d_date_sk#14], [d_date_sk#14] + +(34) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#14] (35) BroadcastExchange Input [1]: [d_date_sk#14] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/simplified.txt index c09f5ad55..2471de20a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/simplified.txt @@ -13,40 +13,40 @@ TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Filter [ss_cdemo_sk,ss_item_sk,ss_promo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_cdemo_sk,ss_item_sk,ss_promo_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Project [cd_demo_sk] - Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cd_demo_sk] + CometFilter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] InputAdapter ReusedExchange [d_date_sk] #2 InputAdapter BroadcastExchange #4 WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] InputAdapter BroadcastExchange #5 WholeStageCodegen (4) - Project [p_promo_sk] - Filter [p_channel_email,p_channel_event,p_promo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [p_promo_sk] + CometFilter [p_channel_email,p_channel_event,p_promo_sk] CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/explain.txt index f96ee1932..c3acf0a8d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/explain.txt @@ -12,14 +12,14 @@ TakeOrderedAndProject (43) +- * BroadcastHashJoin Inner BuildRight (33) :- * Project (6) : +- * BroadcastHashJoin Inner BuildRight (5) - : :- * Filter (3) - : : +- * ColumnarToRow (2) + : :- * ColumnarToRow (3) + : : +- CometFilter (2) : : +- CometScan parquet spark_catalog.default.store_sales (1) : +- ReusedExchange (4) +- BroadcastExchange (32) +- * BroadcastHashJoin LeftSemi BuildRight (31) - :- * Filter (9) - : +- * ColumnarToRow (8) + :- * ColumnarToRow (9) + : +- CometFilter (8) : +- CometScan parquet spark_catalog.default.store (7) +- BroadcastExchange (30) +- * Project (29) @@ -34,12 +34,12 @@ TakeOrderedAndProject (43) +- * BroadcastHashJoin Inner BuildRight (20) :- * Project (18) : +- * BroadcastHashJoin Inner BuildRight (17) - : :- * Filter (12) - : : +- * ColumnarToRow (11) + : :- * ColumnarToRow (12) + : : +- CometFilter (11) : : +- CometScan parquet spark_catalog.default.store_sales (10) : +- BroadcastExchange (16) - : +- * Filter (15) - : +- * ColumnarToRow (14) + : +- * ColumnarToRow (15) + : +- CometFilter (14) : +- CometScan parquet spark_catalog.default.store (13) +- ReusedExchange (19) @@ -52,12 +52,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 8] +(2) CometFilter Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Condition : isnotnull(ss_store_sk#1) -(3) Filter [codegen id : 8] +(3) ColumnarToRow [codegen id : 8] Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_store_sk#1) (4) ReusedExchange [Reuses operator id: 48] Output [1]: [d_date_sk#5] @@ -79,12 +79,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 7] +(8) CometFilter Input [3]: [s_store_sk#6, s_county#7, s_state#8] +Condition : isnotnull(s_store_sk#6) -(9) Filter [codegen id : 7] +(9) ColumnarToRow [codegen id : 7] Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Condition : isnotnull(s_store_sk#6) (10) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] @@ -94,12 +94,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 4] +(11) CometFilter Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Condition : isnotnull(ss_store_sk#9) -(12) Filter [codegen id : 4] +(12) ColumnarToRow [codegen id : 4] Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] -Condition : isnotnull(ss_store_sk#9) (13) Scan parquet spark_catalog.default.store Output [2]: [s_store_sk#13, s_state#14] @@ -108,12 +108,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 2] +(14) CometFilter Input [2]: [s_store_sk#13, s_state#14] +Condition : isnotnull(s_store_sk#13) -(15) Filter [codegen id : 2] +(15) ColumnarToRow [codegen id : 2] Input [2]: [s_store_sk#13, s_state#14] -Condition : isnotnull(s_store_sk#13) (16) BroadcastExchange Input [2]: [s_store_sk#13, s_state#14] @@ -250,9 +250,9 @@ Arguments: 100, [lochierarchy#28 DESC NULLS LAST, CASE WHEN (lochierarchy#28 = 0 Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 BroadcastExchange (48) -+- * Project (47) - +- * Filter (46) - +- * ColumnarToRow (45) ++- * ColumnarToRow (47) + +- CometProject (46) + +- CometFilter (45) +- CometScan parquet spark_catalog.default.date_dim (44) @@ -263,16 +263,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_month_seq#33] - -(46) Filter [codegen id : 1] +(45) CometFilter Input [2]: [d_date_sk#5, d_month_seq#33] Condition : (((isnotnull(d_month_seq#33) AND (d_month_seq#33 >= 1200)) AND (d_month_seq#33 <= 1211)) AND isnotnull(d_date_sk#5)) -(47) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(46) CometProject Input [2]: [d_date_sk#5, d_month_seq#33] +Arguments: [d_date_sk#5], [d_date_sk#5] + +(47) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#5] (48) BroadcastExchange Input [1]: [d_date_sk#5] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/simplified.txt index a15af5f94..268af2e2f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/simplified.txt @@ -18,17 +18,17 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count BroadcastHashJoin [ss_store_sk,s_store_sk] Project [ss_store_sk,ss_net_profit] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_month_seq,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter ReusedExchange [d_date_sk] #3 @@ -36,9 +36,9 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count BroadcastExchange #4 WholeStageCodegen (7) BroadcastHashJoin [s_state,s_state] - Filter [s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] InputAdapter BroadcastExchange #5 @@ -59,17 +59,17 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_net_profit,ss_sold_date_sk,s_state] BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter BroadcastExchange #7 WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_state] InputAdapter ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/explain.txt index 8bf7d4c97..624103b66 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/explain.txt @@ -9,33 +9,33 @@ :- * Project (26) : +- * BroadcastHashJoin Inner BuildLeft (25) : :- BroadcastExchange (5) - : : +- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) + : : +- * ColumnarToRow (4) + : : +- CometProject (3) + : : +- CometFilter (2) : : +- CometScan parquet spark_catalog.default.item (1) : +- Union (24) : :- * Project (11) : : +- * BroadcastHashJoin Inner BuildRight (10) - : : :- * Filter (8) - : : : +- * ColumnarToRow (7) + : : :- * ColumnarToRow (8) + : : : +- CometFilter (7) : : : +- CometScan parquet spark_catalog.default.web_sales (6) : : +- ReusedExchange (9) : :- * Project (17) : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Filter (14) - : : : +- * ColumnarToRow (13) + : : :- * ColumnarToRow (14) + : : : +- CometFilter (13) : : : +- CometScan parquet spark_catalog.default.catalog_sales (12) : : +- ReusedExchange (15) : +- * Project (23) : +- * BroadcastHashJoin Inner BuildRight (22) - : :- * Filter (20) - : : +- * ColumnarToRow (19) + : :- * ColumnarToRow (20) + : : +- CometFilter (19) : : +- CometScan parquet spark_catalog.default.store_sales (18) : +- ReusedExchange (21) +- BroadcastExchange (31) - +- * Project (30) - +- * Filter (29) - +- * ColumnarToRow (28) + +- * ColumnarToRow (30) + +- CometProject (29) + +- CometFilter (28) +- CometScan parquet spark_catalog.default.time_dim (27) @@ -46,16 +46,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] Condition : ((isnotnull(i_manager_id#4) AND (i_manager_id#4 = 1)) AND isnotnull(i_item_sk#1)) -(4) Project [codegen id : 1] -Output [3]: [i_item_sk#1, i_brand_id#2, i_brand#3] +(3) CometProject Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] +Arguments: [i_item_sk#1, i_brand_id#2, i_brand#3], [i_item_sk#1, i_brand_id#2, i_brand#3] + +(4) ColumnarToRow [codegen id : 1] +Input [3]: [i_item_sk#1, i_brand_id#2, i_brand#3] (5) BroadcastExchange Input [3]: [i_item_sk#1, i_brand_id#2, i_brand#3] @@ -69,12 +69,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#8), dynamicpruningexpression(ws_sol PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_sold_time_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 3] +(7) CometFilter Input [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] +Condition : (isnotnull(ws_item_sk#6) AND isnotnull(ws_sold_time_sk#5)) -(8) Filter [codegen id : 3] +(8) ColumnarToRow [codegen id : 3] Input [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] -Condition : (isnotnull(ws_item_sk#6) AND isnotnull(ws_sold_time_sk#5)) (9) ReusedExchange [Reuses operator id: 43] Output [1]: [d_date_sk#10] @@ -97,12 +97,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_sold_time_sk)] ReadSchema: struct -(13) ColumnarToRow [codegen id : 5] +(13) CometFilter Input [4]: [cs_sold_time_sk#14, cs_item_sk#15, cs_ext_sales_price#16, cs_sold_date_sk#17] +Condition : (isnotnull(cs_item_sk#15) AND isnotnull(cs_sold_time_sk#14)) -(14) Filter [codegen id : 5] +(14) ColumnarToRow [codegen id : 5] Input [4]: [cs_sold_time_sk#14, cs_item_sk#15, cs_ext_sales_price#16, cs_sold_date_sk#17] -Condition : (isnotnull(cs_item_sk#15) AND isnotnull(cs_sold_time_sk#14)) (15) ReusedExchange [Reuses operator id: 43] Output [1]: [d_date_sk#19] @@ -125,12 +125,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#26), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_sold_time_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 7] +(19) CometFilter Input [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] +Condition : (isnotnull(ss_item_sk#24) AND isnotnull(ss_sold_time_sk#23)) -(20) Filter [codegen id : 7] +(20) ColumnarToRow [codegen id : 7] Input [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Condition : (isnotnull(ss_item_sk#24) AND isnotnull(ss_sold_time_sk#23)) (21) ReusedExchange [Reuses operator id: 43] Output [1]: [d_date_sk#28] @@ -164,16 +164,16 @@ Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [Or(EqualTo(t_meal_time,breakfast ),EqualTo(t_meal_time,dinner )), IsNotNull(t_time_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 8] -Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] - -(29) Filter [codegen id : 8] +(28) CometFilter Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] Condition : (((t_meal_time#35 = breakfast ) OR (t_meal_time#35 = dinner )) AND isnotnull(t_time_sk#32)) -(30) Project [codegen id : 8] -Output [3]: [t_time_sk#32, t_hour#33, t_minute#34] +(29) CometProject Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] +Arguments: [t_time_sk#32, t_hour#33, t_minute#34], [t_time_sk#32, t_hour#33, t_minute#34] + +(30) ColumnarToRow [codegen id : 8] +Input [3]: [t_time_sk#32, t_hour#33, t_minute#34] (31) BroadcastExchange Input [3]: [t_time_sk#32, t_hour#33, t_minute#34] @@ -219,9 +219,9 @@ Arguments: [ext_price#41 DESC NULLS LAST, brand_id#39 ASC NULLS FIRST], true, 0 Subquery:1 Hosting operator id = 6 Hosting Expression = ws_sold_date_sk#8 IN dynamicpruning#9 BroadcastExchange (43) -+- * Project (42) - +- * Filter (41) - +- * ColumnarToRow (40) ++- * ColumnarToRow (42) + +- CometProject (41) + +- CometFilter (40) +- CometScan parquet spark_catalog.default.date_dim (39) @@ -232,16 +232,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(40) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#10, d_year#42, d_moy#43] - -(41) Filter [codegen id : 1] +(40) CometFilter Input [3]: [d_date_sk#10, d_year#42, d_moy#43] Condition : ((((isnotnull(d_moy#43) AND isnotnull(d_year#42)) AND (d_moy#43 = 11)) AND (d_year#42 = 1999)) AND isnotnull(d_date_sk#10)) -(42) Project [codegen id : 1] -Output [1]: [d_date_sk#10] +(41) CometProject Input [3]: [d_date_sk#10, d_year#42, d_moy#43] +Arguments: [d_date_sk#10], [d_date_sk#10] + +(42) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#10] (43) BroadcastExchange Input [1]: [d_date_sk#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/simplified.txt index c47e3e146..bea5376a0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/simplified.txt @@ -15,36 +15,36 @@ WholeStageCodegen (11) InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Project [i_item_sk,i_brand_id,i_brand] - Filter [i_manager_id,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_item_sk,i_brand_id,i_brand] + CometFilter [i_manager_id,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] InputAdapter Union WholeStageCodegen (3) Project [ws_ext_sales_price,ws_item_sk,ws_sold_time_sk] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk,ws_sold_time_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_item_sk,ws_sold_time_sk] CometScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_moy,d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter ReusedExchange [d_date_sk] #4 WholeStageCodegen (5) Project [cs_ext_sales_price,cs_item_sk,cs_sold_time_sk] BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_item_sk,cs_sold_time_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_item_sk,cs_sold_time_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -52,9 +52,9 @@ WholeStageCodegen (11) WholeStageCodegen (7) Project [ss_ext_sales_price,ss_item_sk,ss_sold_time_sk] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk,ss_sold_time_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk,ss_sold_time_sk] CometScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -62,8 +62,8 @@ WholeStageCodegen (11) InputAdapter BroadcastExchange #5 WholeStageCodegen (8) - Project [t_time_sk,t_hour,t_minute] - Filter [t_meal_time,t_time_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [t_time_sk,t_hour,t_minute] + CometFilter [t_meal_time,t_time_sk] CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute,t_meal_time] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/explain.txt index 35cc300bf..4e5d9e9f6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/explain.txt @@ -25,49 +25,49 @@ TakeOrderedAndProject (70) : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (14) : : : : : : : : :- * Project (9) : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : : : : :- * Filter (3) - : : : : : : : : : : +- * ColumnarToRow (2) + : : : : : : : : : :- * ColumnarToRow (3) + : : : : : : : : : : +- CometFilter (2) : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : : : : : : : : : +- BroadcastExchange (7) - : : : : : : : : : +- * Filter (6) - : : : : : : : : : +- * ColumnarToRow (5) + : : : : : : : : : +- * ColumnarToRow (6) + : : : : : : : : : +- CometFilter (5) : : : : : : : : : +- CometScan parquet spark_catalog.default.inventory (4) : : : : : : : : +- BroadcastExchange (13) - : : : : : : : : +- * Filter (12) - : : : : : : : : +- * ColumnarToRow (11) + : : : : : : : : +- * ColumnarToRow (12) + : : : : : : : : +- CometFilter (11) : : : : : : : : +- CometScan parquet spark_catalog.default.warehouse (10) : : : : : : : +- BroadcastExchange (19) - : : : : : : : +- * Filter (18) - : : : : : : : +- * ColumnarToRow (17) + : : : : : : : +- * ColumnarToRow (18) + : : : : : : : +- CometFilter (17) : : : : : : : +- CometScan parquet spark_catalog.default.item (16) : : : : : : +- BroadcastExchange (26) - : : : : : : +- * Project (25) - : : : : : : +- * Filter (24) - : : : : : : +- * ColumnarToRow (23) + : : : : : : +- * ColumnarToRow (25) + : : : : : : +- CometProject (24) + : : : : : : +- CometFilter (23) : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (22) : : : : : +- BroadcastExchange (33) - : : : : : +- * Project (32) - : : : : : +- * Filter (31) - : : : : : +- * ColumnarToRow (30) + : : : : : +- * ColumnarToRow (32) + : : : : : +- CometProject (31) + : : : : : +- CometFilter (30) : : : : : +- CometScan parquet spark_catalog.default.household_demographics (29) : : : : +- ReusedExchange (36) : : : +- BroadcastExchange (42) - : : : +- * Filter (41) - : : : +- * ColumnarToRow (40) + : : : +- * ColumnarToRow (41) + : : : +- CometFilter (40) : : : +- CometScan parquet spark_catalog.default.date_dim (39) : : +- BroadcastExchange (48) - : : +- * Filter (47) - : : +- * ColumnarToRow (46) + : : +- * ColumnarToRow (47) + : : +- CometFilter (46) : : +- CometScan parquet spark_catalog.default.date_dim (45) : +- BroadcastExchange (54) - : +- * Filter (53) - : +- * ColumnarToRow (52) + : +- * ColumnarToRow (53) + : +- CometFilter (52) : +- CometScan parquet spark_catalog.default.promotion (51) +- * Sort (64) +- Exchange (63) - +- * Project (62) - +- * Filter (61) - +- * ColumnarToRow (60) + +- * ColumnarToRow (62) + +- CometProject (61) + +- CometFilter (60) +- CometScan parquet spark_catalog.default.catalog_returns (59) @@ -79,12 +79,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#8), dynamicpruningexpression(cs_sol PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 10] +(2) CometFilter Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] +Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) -(3) Filter [codegen id : 10] +(3) ColumnarToRow [codegen id : 10] Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) (4) Scan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] @@ -94,12 +94,12 @@ PartitionFilters: [isnotnull(inv_date_sk#13)] PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] +Condition : ((isnotnull(inv_quantity_on_hand#12) AND isnotnull(inv_item_sk#10)) AND isnotnull(inv_warehouse_sk#11)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Condition : ((isnotnull(inv_quantity_on_hand#12) AND isnotnull(inv_item_sk#10)) AND isnotnull(inv_warehouse_sk#11)) (7) BroadcastExchange Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] @@ -122,12 +122,12 @@ Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] +(11) CometFilter Input [2]: [w_warehouse_sk#14, w_warehouse_name#15] +Condition : isnotnull(w_warehouse_sk#14) -(12) Filter [codegen id : 2] +(12) ColumnarToRow [codegen id : 2] Input [2]: [w_warehouse_sk#14, w_warehouse_name#15] -Condition : isnotnull(w_warehouse_sk#14) (13) BroadcastExchange Input [2]: [w_warehouse_sk#14, w_warehouse_name#15] @@ -150,12 +150,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] +(17) CometFilter Input [2]: [i_item_sk#16, i_item_desc#17] +Condition : isnotnull(i_item_sk#16) -(18) Filter [codegen id : 3] +(18) ColumnarToRow [codegen id : 3] Input [2]: [i_item_sk#16, i_item_desc#17] -Condition : isnotnull(i_item_sk#16) (19) BroadcastExchange Input [2]: [i_item_sk#16, i_item_desc#17] @@ -178,16 +178,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_marital_status), EqualTo(cd_marital_status,D), IsNotNull(cd_demo_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 4] -Input [2]: [cd_demo_sk#18, cd_marital_status#19] - -(24) Filter [codegen id : 4] +(23) CometFilter Input [2]: [cd_demo_sk#18, cd_marital_status#19] Condition : ((isnotnull(cd_marital_status#19) AND (cd_marital_status#19 = D)) AND isnotnull(cd_demo_sk#18)) -(25) Project [codegen id : 4] -Output [1]: [cd_demo_sk#18] +(24) CometProject Input [2]: [cd_demo_sk#18, cd_marital_status#19] +Arguments: [cd_demo_sk#18], [cd_demo_sk#18] + +(25) ColumnarToRow [codegen id : 4] +Input [1]: [cd_demo_sk#18] (26) BroadcastExchange Input [1]: [cd_demo_sk#18] @@ -210,16 +210,16 @@ Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_buy_potential), EqualTo(hd_buy_potential,>10000 ), IsNotNull(hd_demo_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 5] -Input [2]: [hd_demo_sk#20, hd_buy_potential#21] - -(31) Filter [codegen id : 5] +(30) CometFilter Input [2]: [hd_demo_sk#20, hd_buy_potential#21] Condition : ((isnotnull(hd_buy_potential#21) AND (hd_buy_potential#21 = >10000 )) AND isnotnull(hd_demo_sk#20)) -(32) Project [codegen id : 5] -Output [1]: [hd_demo_sk#20] +(31) CometProject Input [2]: [hd_demo_sk#20, hd_buy_potential#21] +Arguments: [hd_demo_sk#20], [hd_demo_sk#20] + +(32) ColumnarToRow [codegen id : 5] +Input [1]: [hd_demo_sk#20] (33) BroadcastExchange Input [1]: [hd_demo_sk#20] @@ -255,12 +255,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct -(40) ColumnarToRow [codegen id : 7] +(40) CometFilter Input [2]: [d_date_sk#25, d_week_seq#26] +Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) -(41) Filter [codegen id : 7] +(41) ColumnarToRow [codegen id : 7] Input [2]: [d_date_sk#25, d_week_seq#26] -Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) (42) BroadcastExchange Input [2]: [d_date_sk#25, d_week_seq#26] @@ -283,12 +283,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)] ReadSchema: struct -(46) ColumnarToRow [codegen id : 8] +(46) CometFilter Input [2]: [d_date_sk#27, d_date#28] +Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) -(47) Filter [codegen id : 8] +(47) ColumnarToRow [codegen id : 8] Input [2]: [d_date_sk#27, d_date#28] -Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) (48) BroadcastExchange Input [2]: [d_date_sk#27, d_date#28] @@ -311,12 +311,12 @@ Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(52) ColumnarToRow [codegen id : 9] +(52) CometFilter Input [1]: [p_promo_sk#29] +Condition : isnotnull(p_promo_sk#29) -(53) Filter [codegen id : 9] +(53) ColumnarToRow [codegen id : 9] Input [1]: [p_promo_sk#29] -Condition : isnotnull(p_promo_sk#29) (54) BroadcastExchange Input [1]: [p_promo_sk#29] @@ -347,16 +347,16 @@ Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(60) ColumnarToRow [codegen id : 12] -Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] - -(61) Filter [codegen id : 12] +(60) CometFilter Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Condition : (isnotnull(cr_item_sk#30) AND isnotnull(cr_order_number#31)) -(62) Project [codegen id : 12] -Output [2]: [cr_item_sk#30, cr_order_number#31] +(61) CometProject Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number#31] + +(62) ColumnarToRow [codegen id : 12] +Input [2]: [cr_item_sk#30, cr_order_number#31] (63) Exchange Input [2]: [cr_item_sk#30, cr_order_number#31] @@ -402,9 +402,9 @@ Arguments: 100, [total_cnt#38 DESC NULLS LAST, i_item_desc#17 ASC NULLS FIRST, w Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9 BroadcastExchange (75) -+- * Project (74) - +- * Filter (73) - +- * ColumnarToRow (72) ++- * ColumnarToRow (74) + +- CometProject (73) + +- CometFilter (72) +- CometScan parquet spark_catalog.default.date_dim (71) @@ -415,16 +415,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] ReadSchema: struct -(72) ColumnarToRow [codegen id : 1] -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#39] - -(73) Filter [codegen id : 1] +(72) CometFilter Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#39] Condition : ((((isnotnull(d_year#39) AND (d_year#39 = 1999)) AND isnotnull(d_date_sk#22)) AND isnotnull(d_week_seq#24)) AND isnotnull(d_date#23)) -(74) Project [codegen id : 1] -Output [3]: [d_date_sk#22, d_date#23, d_week_seq#24] +(73) CometProject Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#39] +Arguments: [d_date_sk#22, d_date#23, d_week_seq#24], [d_date_sk#22, d_date#23, d_week_seq#24] + +(74) ColumnarToRow [codegen id : 1] +Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] (75) BroadcastExchange Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/simplified.txt index 59e1279ae..17fc9dee7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/simplified.txt @@ -31,77 +31,77 @@ TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_prom BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] BroadcastHashJoin [cs_item_sk,inv_item_sk,inv_quantity_on_hand,cs_quantity] - Filter [cs_quantity,cs_item_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_ship_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_quantity,cs_item_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_ship_date_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk,d_date,d_week_seq] - Filter [d_year,d_date_sk,d_week_seq,d_date] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_date,d_week_seq] + CometFilter [d_year,d_date_sk,d_week_seq,d_date] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] InputAdapter BroadcastExchange #4 WholeStageCodegen (1) - Filter [inv_quantity_on_hand,inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [inv_quantity_on_hand,inv_item_sk,inv_warehouse_sk] CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] InputAdapter BroadcastExchange #5 WholeStageCodegen (2) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [w_warehouse_sk] CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] InputAdapter BroadcastExchange #6 WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] InputAdapter BroadcastExchange #7 WholeStageCodegen (4) - Project [cd_demo_sk] - Filter [cd_marital_status,cd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cd_demo_sk] + CometFilter [cd_marital_status,cd_demo_sk] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] InputAdapter BroadcastExchange #8 WholeStageCodegen (5) - Project [hd_demo_sk] - Filter [hd_buy_potential,hd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [hd_demo_sk] + CometFilter [hd_buy_potential,hd_demo_sk] CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] InputAdapter ReusedExchange [d_date_sk,d_date,d_week_seq] #3 InputAdapter BroadcastExchange #9 WholeStageCodegen (7) - Filter [d_week_seq,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_week_seq,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] InputAdapter BroadcastExchange #10 WholeStageCodegen (8) - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_date,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter BroadcastExchange #11 WholeStageCodegen (9) - Filter [p_promo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [p_promo_sk] CometScan parquet spark_catalog.default.promotion [p_promo_sk] InputAdapter WholeStageCodegen (13) @@ -109,8 +109,8 @@ TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_prom InputAdapter Exchange [cr_item_sk,cr_order_number] #12 WholeStageCodegen (12) - Project [cr_item_sk,cr_order_number] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/explain.txt index fb1ac906d..ef2c38aea 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/explain.txt @@ -13,23 +13,23 @@ : : +- * BroadcastHashJoin Inner BuildRight (12) : : :- * Project (6) : : : +- * BroadcastHashJoin Inner BuildRight (5) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- ReusedExchange (4) : : +- BroadcastExchange (11) - : : +- * Project (10) - : : +- * Filter (9) - : : +- * ColumnarToRow (8) + : : +- * ColumnarToRow (10) + : : +- CometProject (9) + : : +- CometFilter (8) : : +- CometScan parquet spark_catalog.default.store (7) : +- BroadcastExchange (18) - : +- * Project (17) - : +- * Filter (16) - : +- * ColumnarToRow (15) + : +- * ColumnarToRow (17) + : +- CometProject (16) + : +- CometFilter (15) : +- CometScan parquet spark_catalog.default.household_demographics (14) +- BroadcastExchange (28) - +- * Filter (27) - +- * ColumnarToRow (26) + +- * ColumnarToRow (27) + +- CometFilter (26) +- CometScan parquet spark_catalog.default.customer (25) @@ -41,12 +41,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) -(3) Filter [codegen id : 4] +(3) ColumnarToRow [codegen id : 4] Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) (4) ReusedExchange [Reuses operator id: 37] Output [1]: [d_date_sk#7] @@ -68,16 +68,16 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [In(s_county, [Bronx County,Franklin Parish,Orange County,Williamson County]), IsNotNull(s_store_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#8, s_county#9] - -(9) Filter [codegen id : 2] +(8) CometFilter Input [2]: [s_store_sk#8, s_county#9] Condition : (s_county#9 IN (Williamson County,Franklin Parish,Bronx County,Orange County) AND isnotnull(s_store_sk#8)) -(10) Project [codegen id : 2] -Output [1]: [s_store_sk#8] +(9) CometProject Input [2]: [s_store_sk#8, s_county#9] +Arguments: [s_store_sk#8], [s_store_sk#8] + +(10) ColumnarToRow [codegen id : 2] +Input [1]: [s_store_sk#8] (11) BroadcastExchange Input [1]: [s_store_sk#8] @@ -100,16 +100,16 @@ Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_vehicle_count), Or(EqualTo(hd_buy_potential,>10000 ),EqualTo(hd_buy_potential,unknown )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 3] -Input [4]: [hd_demo_sk#10, hd_buy_potential#11, hd_dep_count#12, hd_vehicle_count#13] - -(16) Filter [codegen id : 3] +(15) CometFilter Input [4]: [hd_demo_sk#10, hd_buy_potential#11, hd_dep_count#12, hd_vehicle_count#13] Condition : ((((isnotnull(hd_vehicle_count#13) AND ((hd_buy_potential#11 = >10000 ) OR (hd_buy_potential#11 = unknown ))) AND (hd_vehicle_count#13 > 0)) AND CASE WHEN (hd_vehicle_count#13 > 0) THEN ((cast(hd_dep_count#12 as double) / cast(hd_vehicle_count#13 as double)) > 1.0) END) AND isnotnull(hd_demo_sk#10)) -(17) Project [codegen id : 3] -Output [1]: [hd_demo_sk#10] +(16) CometProject Input [4]: [hd_demo_sk#10, hd_buy_potential#11, hd_dep_count#12, hd_vehicle_count#13] +Arguments: [hd_demo_sk#10], [hd_demo_sk#10] + +(17) ColumnarToRow [codegen id : 3] +Input [1]: [hd_demo_sk#10] (18) BroadcastExchange Input [1]: [hd_demo_sk#10] @@ -154,12 +154,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 5] +(26) CometFilter Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] +Condition : isnotnull(c_customer_sk#18) -(27) Filter [codegen id : 5] +(27) ColumnarToRow [codegen id : 5] Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] -Condition : isnotnull(c_customer_sk#18) (28) BroadcastExchange Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] @@ -187,9 +187,9 @@ Arguments: [cnt#17 DESC NULLS LAST], true, 0 Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 BroadcastExchange (37) -+- * Project (36) - +- * Filter (35) - +- * ColumnarToRow (34) ++- * ColumnarToRow (36) + +- CometProject (35) + +- CometFilter (34) +- CometScan parquet spark_catalog.default.date_dim (33) @@ -200,16 +200,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#7, d_year#23, d_dom#24] - -(35) Filter [codegen id : 1] +(34) CometFilter Input [3]: [d_date_sk#7, d_year#23, d_dom#24] Condition : ((((isnotnull(d_dom#24) AND (d_dom#24 >= 1)) AND (d_dom#24 <= 2)) AND d_year#23 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7)) -(36) Project [codegen id : 1] -Output [1]: [d_date_sk#7] +(35) CometProject Input [3]: [d_date_sk#7, d_year#23, d_dom#24] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(36) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#7] (37) BroadcastExchange Input [1]: [d_date_sk#7] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/simplified.txt index a2c50bb62..7c5ee1ef5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/simplified.txt @@ -17,40 +17,40 @@ WholeStageCodegen (7) BroadcastHashJoin [ss_store_sk,s_store_sk] Project [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk,ss_hdemo_sk,ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dom,d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_dom,d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] InputAdapter ReusedExchange [d_date_sk] #3 InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - Project [s_store_sk] - Filter [s_county,s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [s_store_sk] + CometFilter [s_county,s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_county] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_vehicle_count,hd_buy_potential,hd_dep_count,hd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [hd_demo_sk] + CometFilter [hd_vehicle_count,hd_buy_potential,hd_dep_count,hd_demo_sk] CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] InputAdapter BroadcastExchange #6 WholeStageCodegen (5) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/explain.txt index 93c71adc4..dad94eb2c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/explain.txt @@ -13,12 +13,12 @@ TakeOrderedAndProject (71) : : : +- * BroadcastHashJoin Inner BuildRight (11) : : : :- * Project (9) : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.customer (1) : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) + : : : : +- * ColumnarToRow (6) + : : : : +- CometFilter (5) : : : : +- CometScan parquet spark_catalog.default.store_sales (4) : : : +- ReusedExchange (10) : : +- BroadcastExchange (32) @@ -29,12 +29,12 @@ TakeOrderedAndProject (71) : : +- * BroadcastHashJoin Inner BuildRight (27) : : :- * Project (25) : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : :- * Filter (19) - : : : : +- * ColumnarToRow (18) + : : : :- * ColumnarToRow (19) + : : : : +- CometFilter (18) : : : : +- CometScan parquet spark_catalog.default.customer (17) : : : +- BroadcastExchange (23) - : : : +- * Filter (22) - : : : +- * ColumnarToRow (21) + : : : +- * ColumnarToRow (22) + : : : +- CometFilter (21) : : : +- CometScan parquet spark_catalog.default.store_sales (20) : : +- ReusedExchange (26) : +- BroadcastExchange (50) @@ -46,12 +46,12 @@ TakeOrderedAndProject (71) : +- * BroadcastHashJoin Inner BuildRight (44) : :- * Project (42) : : +- * BroadcastHashJoin Inner BuildRight (41) - : : :- * Filter (36) - : : : +- * ColumnarToRow (35) + : : :- * ColumnarToRow (36) + : : : +- CometFilter (35) : : : +- CometScan parquet spark_catalog.default.customer (34) : : +- BroadcastExchange (40) - : : +- * Filter (39) - : : +- * ColumnarToRow (38) + : : +- * ColumnarToRow (39) + : : +- CometFilter (38) : : +- CometScan parquet spark_catalog.default.web_sales (37) : +- ReusedExchange (43) +- BroadcastExchange (68) @@ -62,12 +62,12 @@ TakeOrderedAndProject (71) +- * BroadcastHashJoin Inner BuildRight (63) :- * Project (61) : +- * BroadcastHashJoin Inner BuildRight (60) - : :- * Filter (55) - : : +- * ColumnarToRow (54) + : :- * ColumnarToRow (55) + : : +- CometFilter (54) : : +- CometScan parquet spark_catalog.default.customer (53) : +- BroadcastExchange (59) - : +- * Filter (58) - : +- * ColumnarToRow (57) + : +- * ColumnarToRow (58) + : +- CometFilter (57) : +- CometScan parquet spark_catalog.default.web_sales (56) +- ReusedExchange (62) @@ -79,12 +79,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) -(3) Filter [codegen id : 3] +(3) ColumnarToRow [codegen id : 3] Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) (4) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] @@ -94,12 +94,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] +Condition : isnotnull(ss_customer_sk#5) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Condition : isnotnull(ss_customer_sk#5) (7) BroadcastExchange Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] @@ -157,12 +157,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 6] +(18) CometFilter Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Condition : (isnotnull(c_customer_sk#16) AND isnotnull(c_customer_id#17)) -(19) Filter [codegen id : 6] +(19) ColumnarToRow [codegen id : 6] Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] -Condition : (isnotnull(c_customer_sk#16) AND isnotnull(c_customer_id#17)) (20) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] @@ -172,12 +172,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#22), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 4] +(21) CometFilter Input [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Condition : isnotnull(ss_customer_sk#20) -(22) Filter [codegen id : 4] +(22) ColumnarToRow [codegen id : 4] Input [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] -Condition : isnotnull(ss_customer_sk#20) (23) BroadcastExchange Input [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] @@ -241,12 +241,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 10] +(35) CometFilter Input [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] +Condition : (isnotnull(c_customer_sk#32) AND isnotnull(c_customer_id#33)) -(36) Filter [codegen id : 10] +(36) ColumnarToRow [codegen id : 10] Input [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] -Condition : (isnotnull(c_customer_sk#32) AND isnotnull(c_customer_id#33)) (37) Scan parquet spark_catalog.default.web_sales Output [3]: [ws_bill_customer_sk#36, ws_net_paid#37, ws_sold_date_sk#38] @@ -256,12 +256,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#38), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 8] +(38) CometFilter Input [3]: [ws_bill_customer_sk#36, ws_net_paid#37, ws_sold_date_sk#38] +Condition : isnotnull(ws_bill_customer_sk#36) -(39) Filter [codegen id : 8] +(39) ColumnarToRow [codegen id : 8] Input [3]: [ws_bill_customer_sk#36, ws_net_paid#37, ws_sold_date_sk#38] -Condition : isnotnull(ws_bill_customer_sk#36) (40) BroadcastExchange Input [3]: [ws_bill_customer_sk#36, ws_net_paid#37, ws_sold_date_sk#38] @@ -333,12 +333,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(54) ColumnarToRow [codegen id : 14] +(54) CometFilter Input [4]: [c_customer_sk#47, c_customer_id#48, c_first_name#49, c_last_name#50] +Condition : (isnotnull(c_customer_sk#47) AND isnotnull(c_customer_id#48)) -(55) Filter [codegen id : 14] +(55) ColumnarToRow [codegen id : 14] Input [4]: [c_customer_sk#47, c_customer_id#48, c_first_name#49, c_last_name#50] -Condition : (isnotnull(c_customer_sk#47) AND isnotnull(c_customer_id#48)) (56) Scan parquet spark_catalog.default.web_sales Output [3]: [ws_bill_customer_sk#51, ws_net_paid#52, ws_sold_date_sk#53] @@ -348,12 +348,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#53), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 12] +(57) CometFilter Input [3]: [ws_bill_customer_sk#51, ws_net_paid#52, ws_sold_date_sk#53] +Condition : isnotnull(ws_bill_customer_sk#51) -(58) Filter [codegen id : 12] +(58) ColumnarToRow [codegen id : 12] Input [3]: [ws_bill_customer_sk#51, ws_net_paid#52, ws_sold_date_sk#53] -Condition : isnotnull(ws_bill_customer_sk#51) (59) BroadcastExchange Input [3]: [ws_bill_customer_sk#51, ws_net_paid#52, ws_sold_date_sk#53] @@ -422,8 +422,8 @@ Arguments: 100, [customer_id#28 ASC NULLS FIRST, customer_id#28 ASC NULLS FIRST, Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 BroadcastExchange (75) -+- * Filter (74) - +- * ColumnarToRow (73) ++- * ColumnarToRow (74) + +- CometFilter (73) +- CometScan parquet spark_catalog.default.date_dim (72) @@ -434,12 +434,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(73) ColumnarToRow [codegen id : 1] +(73) CometFilter Input [2]: [d_date_sk#9, d_year#10] +Condition : (((isnotnull(d_year#10) AND (d_year#10 = 2001)) AND d_year#10 IN (2001,2002)) AND isnotnull(d_date_sk#9)) -(74) Filter [codegen id : 1] +(74) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#9, d_year#10] -Condition : (((isnotnull(d_year#10) AND (d_year#10 = 2001)) AND d_year#10 IN (2001,2002)) AND isnotnull(d_date_sk#9)) (75) BroadcastExchange Input [2]: [d_date_sk#9, d_year#10] @@ -447,8 +447,8 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint) Subquery:2 Hosting operator id = 20 Hosting Expression = ss_sold_date_sk#22 IN dynamicpruning#23 BroadcastExchange (79) -+- * Filter (78) - +- * ColumnarToRow (77) ++- * ColumnarToRow (78) + +- CometFilter (77) +- CometScan parquet spark_catalog.default.date_dim (76) @@ -459,12 +459,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(77) ColumnarToRow [codegen id : 1] +(77) CometFilter Input [2]: [d_date_sk#24, d_year#25] +Condition : (((isnotnull(d_year#25) AND (d_year#25 = 2002)) AND d_year#25 IN (2001,2002)) AND isnotnull(d_date_sk#24)) -(78) Filter [codegen id : 1] +(78) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#24, d_year#25] -Condition : (((isnotnull(d_year#25) AND (d_year#25 = 2002)) AND d_year#25 IN (2001,2002)) AND isnotnull(d_date_sk#24)) (79) BroadcastExchange Input [2]: [d_date_sk#24, d_year#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/simplified.txt index e66974fba..9d3ae8fbe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/simplified.txt @@ -15,23 +15,23 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_customer_id] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] InputAdapter BroadcastExchange #2 WholeStageCodegen (1) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [d_date_sk,d_year] #3 @@ -47,23 +47,23 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_customer_id] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] InputAdapter BroadcastExchange #6 WholeStageCodegen (4) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #7 WholeStageCodegen (1) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [d_date_sk,d_year] #7 @@ -80,16 +80,16 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_customer_id] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] InputAdapter BroadcastExchange #10 WholeStageCodegen (8) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_bill_customer_sk] CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -106,16 +106,16 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_customer_id] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] InputAdapter BroadcastExchange #13 WholeStageCodegen (12) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_bill_customer_sk] CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/explain.txt index b18c8bbbf..a3d12a1d4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/explain.txt @@ -20,20 +20,20 @@ TakeOrderedAndProject (129) : : : +- * BroadcastHashJoin Inner BuildRight (12) : : : :- * Project (10) : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) + : : : : +- * ColumnarToRow (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) : : : : +- CometScan parquet spark_catalog.default.item (4) : : : +- ReusedExchange (11) : : +- * Sort (21) : : +- Exchange (20) - : : +- * Project (19) - : : +- * Filter (18) - : : +- * ColumnarToRow (17) + : : +- * ColumnarToRow (19) + : : +- CometProject (18) + : : +- CometFilter (17) : : +- CometScan parquet spark_catalog.default.catalog_returns (16) : :- * Project (42) : : +- * SortMergeJoin LeftOuter (41) @@ -43,16 +43,16 @@ TakeOrderedAndProject (129) : : : +- * BroadcastHashJoin Inner BuildRight (31) : : : :- * Project (29) : : : : +- * BroadcastHashJoin Inner BuildRight (28) - : : : : :- * Filter (26) - : : : : : +- * ColumnarToRow (25) + : : : : :- * ColumnarToRow (26) + : : : : : +- CometFilter (25) : : : : : +- CometScan parquet spark_catalog.default.store_sales (24) : : : : +- ReusedExchange (27) : : : +- ReusedExchange (30) : : +- * Sort (40) : : +- Exchange (39) - : : +- * Project (38) - : : +- * Filter (37) - : : +- * ColumnarToRow (36) + : : +- * ColumnarToRow (38) + : : +- CometProject (37) + : : +- CometFilter (36) : : +- CometScan parquet spark_catalog.default.store_returns (35) : +- * Project (61) : +- * SortMergeJoin LeftOuter (60) @@ -62,16 +62,16 @@ TakeOrderedAndProject (129) : : +- * BroadcastHashJoin Inner BuildRight (50) : : :- * Project (48) : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : :- * Filter (45) - : : : : +- * ColumnarToRow (44) + : : : :- * ColumnarToRow (45) + : : : : +- CometFilter (44) : : : : +- CometScan parquet spark_catalog.default.web_sales (43) : : : +- ReusedExchange (46) : : +- ReusedExchange (49) : +- * Sort (59) : +- Exchange (58) - : +- * Project (57) - : +- * Filter (56) - : +- * ColumnarToRow (55) + : +- * ColumnarToRow (57) + : +- CometProject (56) + : +- CometFilter (55) : +- CometScan parquet spark_catalog.default.web_returns (54) +- * Sort (126) +- Exchange (125) @@ -91,8 +91,8 @@ TakeOrderedAndProject (129) : : +- * BroadcastHashJoin Inner BuildRight (79) : : :- * Project (77) : : : +- * BroadcastHashJoin Inner BuildRight (76) - : : : :- * Filter (74) - : : : : +- * ColumnarToRow (73) + : : : :- * ColumnarToRow (74) + : : : : +- CometFilter (73) : : : : +- CometScan parquet spark_catalog.default.catalog_sales (72) : : : +- ReusedExchange (75) : : +- ReusedExchange (78) @@ -106,8 +106,8 @@ TakeOrderedAndProject (129) : : +- * BroadcastHashJoin Inner BuildRight (94) : : :- * Project (92) : : : +- * BroadcastHashJoin Inner BuildRight (91) - : : : :- * Filter (89) - : : : : +- * ColumnarToRow (88) + : : : :- * ColumnarToRow (89) + : : : : +- CometFilter (88) : : : : +- CometScan parquet spark_catalog.default.store_sales (87) : : : +- ReusedExchange (90) : : +- ReusedExchange (93) @@ -121,8 +121,8 @@ TakeOrderedAndProject (129) : +- * BroadcastHashJoin Inner BuildRight (109) : :- * Project (107) : : +- * BroadcastHashJoin Inner BuildRight (106) - : : :- * Filter (104) - : : : +- * ColumnarToRow (103) + : : :- * ColumnarToRow (104) + : : : +- CometFilter (103) : : : +- CometScan parquet spark_catalog.default.web_sales (102) : : +- ReusedExchange (105) : +- ReusedExchange (108) @@ -138,12 +138,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sol PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] +Condition : isnotnull(cs_item_sk#1) -(3) Filter [codegen id : 3] +(3) ColumnarToRow [codegen id : 3] Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Condition : isnotnull(cs_item_sk#1) (4) Scan parquet spark_catalog.default.item Output [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] @@ -152,16 +152,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Books ), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] - -(6) Filter [codegen id : 1] +(5) CometFilter Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] Condition : ((((((isnotnull(i_category#11) AND (i_category#11 = Books )) AND isnotnull(i_item_sk#7)) AND isnotnull(i_brand_id#8)) AND isnotnull(i_class_id#9)) AND isnotnull(i_category_id#10)) AND isnotnull(i_manufact_id#12)) -(7) Project [codegen id : 1] -Output [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +(6) CometProject Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] +Arguments: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12], [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] + +(7) ColumnarToRow [codegen id : 1] +Input [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] (8) BroadcastExchange Input [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] @@ -205,16 +205,16 @@ Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 5] -Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] - -(18) Filter [codegen id : 5] +(17) CometFilter Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] Condition : (isnotnull(cr_order_number#16) AND isnotnull(cr_item_sk#15)) -(19) Project [codegen id : 5] -Output [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] +(18) CometProject Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] +Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18], [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] + +(19) ColumnarToRow [codegen id : 5] +Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] (20) Exchange Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] @@ -242,12 +242,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#26), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 10] +(25) CometFilter Input [5]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26] +Condition : isnotnull(ss_item_sk#22) -(26) Filter [codegen id : 10] +(26) ColumnarToRow [codegen id : 10] Input [5]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Condition : isnotnull(ss_item_sk#22) (27) ReusedExchange [Reuses operator id: 8] Output [5]: [i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] @@ -290,16 +290,16 @@ Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(36) ColumnarToRow [codegen id : 12] -Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] - -(37) Filter [codegen id : 12] +(36) CometFilter Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] Condition : (isnotnull(sr_ticket_number#36) AND isnotnull(sr_item_sk#35)) -(38) Project [codegen id : 12] -Output [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] +(37) CometProject Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] +Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38], [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] + +(38) ColumnarToRow [codegen id : 12] +Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] (39) Exchange Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] @@ -327,12 +327,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#46), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(44) ColumnarToRow [codegen id : 17] +(44) CometFilter Input [5]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46] +Condition : isnotnull(ws_item_sk#42) -(45) Filter [codegen id : 17] +(45) ColumnarToRow [codegen id : 17] Input [5]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46] -Condition : isnotnull(ws_item_sk#42) (46) ReusedExchange [Reuses operator id: 8] Output [5]: [i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] @@ -375,16 +375,16 @@ Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct -(55) ColumnarToRow [codegen id : 19] -Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] - -(56) Filter [codegen id : 19] +(55) CometFilter Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] Condition : (isnotnull(wr_order_number#56) AND isnotnull(wr_item_sk#55)) -(57) Project [codegen id : 19] -Output [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] +(56) CometProject Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] +Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58], [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] + +(57) ColumnarToRow [codegen id : 19] +Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] (58) Exchange Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] @@ -462,12 +462,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#74), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(73) ColumnarToRow [codegen id : 28] +(73) CometFilter Input [5]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74] +Condition : isnotnull(cs_item_sk#70) -(74) Filter [codegen id : 28] +(74) ColumnarToRow [codegen id : 28] Input [5]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74] -Condition : isnotnull(cs_item_sk#70) (75) ReusedExchange [Reuses operator id: 8] Output [5]: [i_item_sk#76, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] @@ -528,12 +528,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#93), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(88) ColumnarToRow [codegen id : 35] +(88) CometFilter Input [5]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, ss_sold_date_sk#93] +Condition : isnotnull(ss_item_sk#89) -(89) Filter [codegen id : 35] +(89) ColumnarToRow [codegen id : 35] Input [5]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, ss_sold_date_sk#93] -Condition : isnotnull(ss_item_sk#89) (90) ReusedExchange [Reuses operator id: 8] Output [5]: [i_item_sk#95, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99] @@ -594,12 +594,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#112), dynamicpruningexpression(ws_s PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(103) ColumnarToRow [codegen id : 42] +(103) CometFilter Input [5]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, ws_sold_date_sk#112] +Condition : isnotnull(ws_item_sk#108) -(104) Filter [codegen id : 42] +(104) ColumnarToRow [codegen id : 42] Input [5]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, ws_sold_date_sk#112] -Condition : isnotnull(ws_item_sk#108) (105) ReusedExchange [Reuses operator id: 8] Output [5]: [i_item_sk#114, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118] @@ -720,8 +720,8 @@ Arguments: 100, [sales_cnt_diff#137 ASC NULLS FIRST], [prev_year#133, year#134, Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 BroadcastExchange (133) -+- * Filter (132) - +- * ColumnarToRow (131) ++- * ColumnarToRow (132) + +- CometFilter (131) +- CometScan parquet spark_catalog.default.date_dim (130) @@ -732,12 +732,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(131) ColumnarToRow [codegen id : 1] +(131) CometFilter Input [2]: [d_date_sk#13, d_year#14] +Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) -(132) Filter [codegen id : 1] +(132) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) (133) BroadcastExchange Input [2]: [d_date_sk#13, d_year#14] @@ -749,8 +749,8 @@ Subquery:3 Hosting operator id = 43 Hosting Expression = ws_sold_date_sk#46 IN d Subquery:4 Hosting operator id = 72 Hosting Expression = cs_sold_date_sk#74 IN dynamicpruning#75 BroadcastExchange (137) -+- * Filter (136) - +- * ColumnarToRow (135) ++- * ColumnarToRow (136) + +- CometFilter (135) +- CometScan parquet spark_catalog.default.date_dim (134) @@ -761,12 +761,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(135) ColumnarToRow [codegen id : 1] +(135) CometFilter Input [2]: [d_date_sk#81, d_year#82] +Condition : ((isnotnull(d_year#82) AND (d_year#82 = 2001)) AND isnotnull(d_date_sk#81)) -(136) Filter [codegen id : 1] +(136) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#81, d_year#82] -Condition : ((isnotnull(d_year#82) AND (d_year#82 = 2001)) AND isnotnull(d_date_sk#81)) (137) BroadcastExchange Input [2]: [d_date_sk#81, d_year#82] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/simplified.txt index 021cddf2b..44bcabcdb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/simplified.txt @@ -34,24 +34,24 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #5 WholeStageCodegen (1) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #6 WholeStageCodegen (1) - Project [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - Filter [i_category,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [i_category,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] InputAdapter ReusedExchange [d_date_sk,d_year] #5 @@ -61,10 +61,10 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat InputAdapter Exchange [cr_order_number,cr_item_sk] #7 WholeStageCodegen (5) - Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - Filter [cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_order_number,cr_item_sk] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] WholeStageCodegen (14) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] @@ -79,9 +79,9 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -94,10 +94,10 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat InputAdapter Exchange [sr_ticket_number,sr_item_sk] #9 WholeStageCodegen (12) - Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_ticket_number,sr_item_sk] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] WholeStageCodegen (21) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] @@ -112,9 +112,9 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -127,10 +127,10 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat InputAdapter Exchange [wr_order_number,wr_item_sk] #11 WholeStageCodegen (19) - Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - Filter [wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_order_number,wr_item_sk] CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] InputAdapter WholeStageCodegen (50) @@ -164,16 +164,16 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #16 WholeStageCodegen (1) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 @@ -197,9 +197,9 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #2 InputAdapter @@ -224,9 +224,9 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76/explain.txt index 69427a8e2..f3b80d8bd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76/explain.txt @@ -8,23 +8,23 @@ TakeOrderedAndProject (38) : +- * BroadcastHashJoin Inner BuildRight (14) : :- * Project (9) : : +- * BroadcastHashJoin Inner BuildRight (8) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) + : : :- * ColumnarToRow (3) + : : : +- CometFilter (2) : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : +- BroadcastExchange (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) + : : +- * ColumnarToRow (6) + : : +- CometFilter (5) : : +- CometScan parquet spark_catalog.default.item (4) : +- BroadcastExchange (13) - : +- * Filter (12) - : +- * ColumnarToRow (11) + : +- * ColumnarToRow (12) + : +- CometFilter (11) : +- CometScan parquet spark_catalog.default.date_dim (10) :- * Project (24) : +- * BroadcastHashJoin Inner BuildRight (23) : :- * Project (21) : : +- * BroadcastHashJoin Inner BuildRight (20) - : : :- * Filter (18) - : : : +- * ColumnarToRow (17) + : : :- * ColumnarToRow (18) + : : : +- CometFilter (17) : : : +- CometScan parquet spark_catalog.default.web_sales (16) : : +- ReusedExchange (19) : +- ReusedExchange (22) @@ -32,8 +32,8 @@ TakeOrderedAndProject (38) +- * BroadcastHashJoin Inner BuildRight (32) :- * Project (30) : +- * BroadcastHashJoin Inner BuildRight (29) - : :- * Filter (27) - : : +- * ColumnarToRow (26) + : :- * ColumnarToRow (27) + : : +- CometFilter (26) : : +- CometScan parquet spark_catalog.default.catalog_sales (25) : +- ReusedExchange (28) +- ReusedExchange (31) @@ -47,12 +47,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Condition : (isnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) -(3) Filter [codegen id : 3] +(3) ColumnarToRow [codegen id : 3] Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Condition : (isnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) (4) Scan parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_category#6] @@ -61,12 +61,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [2]: [i_item_sk#5, i_category#6] +Condition : isnotnull(i_item_sk#5) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [2]: [i_item_sk#5, i_category#6] -Condition : isnotnull(i_item_sk#5) (7) BroadcastExchange Input [2]: [i_item_sk#5, i_category#6] @@ -89,12 +89,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] +(11) CometFilter Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] +Condition : isnotnull(d_date_sk#7) -(12) Filter [codegen id : 2] +(12) ColumnarToRow [codegen id : 2] Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Condition : isnotnull(d_date_sk#7) (13) BroadcastExchange Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] @@ -118,12 +118,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#16)] PushedFilters: [IsNull(ws_ship_customer_sk), IsNotNull(ws_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 6] +(17) CometFilter Input [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] +Condition : (isnull(ws_ship_customer_sk#14) AND isnotnull(ws_item_sk#13)) -(18) Filter [codegen id : 6] +(18) ColumnarToRow [codegen id : 6] Input [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] -Condition : (isnull(ws_ship_customer_sk#14) AND isnotnull(ws_item_sk#13)) (19) ReusedExchange [Reuses operator id: 7] Output [2]: [i_item_sk#17, i_category#18] @@ -159,12 +159,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#28)] PushedFilters: [IsNull(cs_ship_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 9] +(26) CometFilter Input [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] +Condition : (isnull(cs_ship_addr_sk#25) AND isnotnull(cs_item_sk#26)) -(27) Filter [codegen id : 9] +(27) ColumnarToRow [codegen id : 9] Input [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] -Condition : (isnull(cs_ship_addr_sk#25) AND isnotnull(cs_item_sk#26)) (28) ReusedExchange [Reuses operator id: 7] Output [2]: [i_item_sk#29, i_category#30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76/simplified.txt index 074a78dd4..73e6b09af 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76/simplified.txt @@ -12,32 +12,32 @@ TakeOrderedAndProject [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_ BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category] BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk,ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] InputAdapter BroadcastExchange #2 WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_category] InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Filter [d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] WholeStageCodegen (6) Project [ws_ship_customer_sk,d_year,d_qoy,i_category,ws_ext_sales_price] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category] BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_ship_customer_sk,ws_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_ship_customer_sk,ws_item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] InputAdapter ReusedExchange [i_item_sk,i_category] #2 @@ -48,9 +48,9 @@ TakeOrderedAndProject [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_ BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category] BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_ship_addr_sk,cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_ship_addr_sk,cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] InputAdapter ReusedExchange [i_item_sk,i_category] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/explain.txt index a228bce35..bbfa6a4c4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/explain.txt @@ -14,13 +14,13 @@ TakeOrderedAndProject (85) : : +- * BroadcastHashJoin Inner BuildRight (11) : : :- * Project (6) : : : +- * BroadcastHashJoin Inner BuildRight (5) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- ReusedExchange (4) : : +- BroadcastExchange (10) - : : +- * Filter (9) - : : +- * ColumnarToRow (8) + : : +- * ColumnarToRow (9) + : : +- CometFilter (8) : : +- CometScan parquet spark_catalog.default.store (7) : +- BroadcastExchange (28) : +- * HashAggregate (27) @@ -30,8 +30,8 @@ TakeOrderedAndProject (85) : +- * BroadcastHashJoin Inner BuildRight (23) : :- * Project (21) : : +- * BroadcastHashJoin Inner BuildRight (20) - : : :- * Filter (18) - : : : +- * ColumnarToRow (17) + : : :- * ColumnarToRow (18) + : : : +- CometFilter (17) : : : +- CometScan parquet spark_catalog.default.store_returns (16) : : +- ReusedExchange (19) : +- ReusedExchange (22) @@ -63,13 +63,13 @@ TakeOrderedAndProject (85) : +- * BroadcastHashJoin Inner BuildRight (60) : :- * Project (55) : : +- * BroadcastHashJoin Inner BuildRight (54) - : : :- * Filter (52) - : : : +- * ColumnarToRow (51) + : : :- * ColumnarToRow (52) + : : : +- CometFilter (51) : : : +- CometScan parquet spark_catalog.default.web_sales (50) : : +- ReusedExchange (53) : +- BroadcastExchange (59) - : +- * Filter (58) - : +- * ColumnarToRow (57) + : +- * ColumnarToRow (58) + : +- CometFilter (57) : +- CometScan parquet spark_catalog.default.web_page (56) +- BroadcastExchange (77) +- * HashAggregate (76) @@ -79,8 +79,8 @@ TakeOrderedAndProject (85) +- * BroadcastHashJoin Inner BuildRight (72) :- * Project (70) : +- * BroadcastHashJoin Inner BuildRight (69) - : :- * Filter (67) - : : +- * ColumnarToRow (66) + : :- * ColumnarToRow (67) + : : +- CometFilter (66) : : +- CometScan parquet spark_catalog.default.web_returns (65) : +- ReusedExchange (68) +- ReusedExchange (71) @@ -94,12 +94,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Condition : isnotnull(ss_store_sk#1) -(3) Filter [codegen id : 3] +(3) ColumnarToRow [codegen id : 3] Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_store_sk#1) (4) ReusedExchange [Reuses operator id: 90] Output [1]: [d_date_sk#6] @@ -121,12 +121,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] +(8) CometFilter Input [1]: [s_store_sk#7] +Condition : isnotnull(s_store_sk#7) -(9) Filter [codegen id : 2] +(9) ColumnarToRow [codegen id : 2] Input [1]: [s_store_sk#7] -Condition : isnotnull(s_store_sk#7) (10) BroadcastExchange Input [1]: [s_store_sk#7] @@ -168,12 +168,12 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#19), dynamicpruningexpression(s PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 6] +(17) CometFilter Input [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19] +Condition : isnotnull(sr_store_sk#16) -(18) Filter [codegen id : 6] +(18) ColumnarToRow [codegen id : 6] Input [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19] -Condition : isnotnull(sr_store_sk#16) (19) ReusedExchange [Reuses operator id: 90] Output [1]: [d_date_sk#21] @@ -335,12 +335,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#68), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_web_page_sk)] ReadSchema: struct -(51) ColumnarToRow [codegen id : 17] +(51) CometFilter Input [4]: [ws_web_page_sk#65, ws_ext_sales_price#66, ws_net_profit#67, ws_sold_date_sk#68] +Condition : isnotnull(ws_web_page_sk#65) -(52) Filter [codegen id : 17] +(52) ColumnarToRow [codegen id : 17] Input [4]: [ws_web_page_sk#65, ws_ext_sales_price#66, ws_net_profit#67, ws_sold_date_sk#68] -Condition : isnotnull(ws_web_page_sk#65) (53) ReusedExchange [Reuses operator id: 90] Output [1]: [d_date_sk#70] @@ -362,12 +362,12 @@ Location [not included in comparison]/{warehouse_dir}/web_page] PushedFilters: [IsNotNull(wp_web_page_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 16] +(57) CometFilter Input [1]: [wp_web_page_sk#71] +Condition : isnotnull(wp_web_page_sk#71) -(58) Filter [codegen id : 16] +(58) ColumnarToRow [codegen id : 16] Input [1]: [wp_web_page_sk#71] -Condition : isnotnull(wp_web_page_sk#71) (59) BroadcastExchange Input [1]: [wp_web_page_sk#71] @@ -409,12 +409,12 @@ PartitionFilters: [isnotnull(wr_returned_date_sk#83), dynamicpruningexpression(w PushedFilters: [IsNotNull(wr_web_page_sk)] ReadSchema: struct -(66) ColumnarToRow [codegen id : 20] +(66) CometFilter Input [4]: [wr_web_page_sk#80, wr_return_amt#81, wr_net_loss#82, wr_returned_date_sk#83] +Condition : isnotnull(wr_web_page_sk#80) -(67) Filter [codegen id : 20] +(67) ColumnarToRow [codegen id : 20] Input [4]: [wr_web_page_sk#80, wr_return_amt#81, wr_net_loss#82, wr_returned_date_sk#83] -Condition : isnotnull(wr_web_page_sk#80) (68) ReusedExchange [Reuses operator id: 90] Output [1]: [d_date_sk#85] @@ -506,9 +506,9 @@ Arguments: 100, [channel#99 ASC NULLS FIRST, id#100 ASC NULLS FIRST], [channel#9 Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 BroadcastExchange (90) -+- * Project (89) - +- * Filter (88) - +- * ColumnarToRow (87) ++- * ColumnarToRow (89) + +- CometProject (88) + +- CometFilter (87) +- CometScan parquet spark_catalog.default.date_dim (86) @@ -519,16 +519,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-03), LessThanOrEqual(d_date,2000-09-02), IsNotNull(d_date_sk)] ReadSchema: struct -(87) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#6, d_date#120] - -(88) Filter [codegen id : 1] +(87) CometFilter Input [2]: [d_date_sk#6, d_date#120] Condition : (((isnotnull(d_date#120) AND (d_date#120 >= 2000-08-03)) AND (d_date#120 <= 2000-09-02)) AND isnotnull(d_date_sk#6)) -(89) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(88) CometProject Input [2]: [d_date_sk#6, d_date#120] +Arguments: [d_date_sk#6], [d_date_sk#6] + +(89) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#6] (90) BroadcastExchange Input [1]: [d_date_sk#6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/simplified.txt index 36cb2e9f5..d6693067f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/simplified.txt @@ -20,26 +20,26 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] BroadcastHashJoin [ss_store_sk,s_store_sk] Project [ss_store_sk,ss_ext_sales_price,ss_net_profit] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter ReusedExchange [d_date_sk] #3 InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk] InputAdapter BroadcastExchange #5 @@ -53,9 +53,9 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] BroadcastHashJoin [sr_store_sk,s_store_sk] Project [sr_store_sk,sr_return_amt,sr_net_loss] BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Filter [sr_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [sr_store_sk] CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -106,9 +106,9 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] Project [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_web_page_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_web_page_sk] CometScan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -116,9 +116,9 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter BroadcastExchange #11 WholeStageCodegen (16) - Filter [wp_web_page_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [wp_web_page_sk] CometScan parquet spark_catalog.default.web_page [wp_web_page_sk] InputAdapter BroadcastExchange #12 @@ -132,9 +132,9 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] BroadcastHashJoin [wr_web_page_sk,wp_web_page_sk] Project [wr_web_page_sk,wr_return_amt,wr_net_loss] BroadcastHashJoin [wr_returned_date_sk,d_date_sk] - Filter [wr_web_page_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [wr_web_page_sk] CometScan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/explain.txt index e2d0fc3ea..7f2688112 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/explain.txt @@ -15,14 +15,14 @@ TakeOrderedAndProject (70) : : : +- * SortMergeJoin LeftOuter (12) : : : :- * Sort (5) : : : : +- Exchange (4) - : : : : +- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : : +- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- * Sort (11) : : : +- Exchange (10) - : : : +- * Project (9) - : : : +- * Filter (8) - : : : +- * ColumnarToRow (7) + : : : +- * ColumnarToRow (9) + : : : +- CometProject (8) + : : : +- CometFilter (7) : : : +- CometScan parquet spark_catalog.default.store_returns (6) : : +- ReusedExchange (15) : +- * Sort (43) @@ -37,14 +37,14 @@ TakeOrderedAndProject (70) : : +- * SortMergeJoin LeftOuter (33) : : :- * Sort (26) : : : +- Exchange (25) - : : : +- * Filter (24) - : : : +- * ColumnarToRow (23) + : : : +- * ColumnarToRow (24) + : : : +- CometFilter (23) : : : +- CometScan parquet spark_catalog.default.web_sales (22) : : +- * Sort (32) : : +- Exchange (31) - : : +- * Project (30) - : : +- * Filter (29) - : : +- * ColumnarToRow (28) + : : +- * ColumnarToRow (30) + : : +- CometProject (29) + : : +- CometFilter (28) : : +- CometScan parquet spark_catalog.default.web_returns (27) : +- ReusedExchange (36) +- * Sort (67) @@ -59,14 +59,14 @@ TakeOrderedAndProject (70) : +- * SortMergeJoin LeftOuter (57) : :- * Sort (50) : : +- Exchange (49) - : : +- * Filter (48) - : : +- * ColumnarToRow (47) + : : +- * ColumnarToRow (48) + : : +- CometFilter (47) : : +- CometScan parquet spark_catalog.default.catalog_sales (46) : +- * Sort (56) : +- Exchange (55) - : +- * Project (54) - : +- * Filter (53) - : +- * ColumnarToRow (52) + : +- * ColumnarToRow (54) + : +- CometProject (53) + : +- CometFilter (52) : +- CometScan parquet spark_catalog.default.catalog_returns (51) +- ReusedExchange (60) @@ -79,12 +79,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] +(2) CometFilter Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2)) -(3) Filter [codegen id : 1] +(3) ColumnarToRow [codegen id : 1] Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2)) (4) Exchange Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] @@ -101,16 +101,16 @@ Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 3] -Input [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] - -(8) Filter [codegen id : 3] +(7) CometFilter Input [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] Condition : (isnotnull(sr_ticket_number#10) AND isnotnull(sr_item_sk#9)) -(9) Project [codegen id : 3] -Output [2]: [sr_item_sk#9, sr_ticket_number#10] +(8) CometProject Input [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] +Arguments: [sr_item_sk#9, sr_ticket_number#10], [sr_item_sk#9, sr_ticket_number#10] + +(9) ColumnarToRow [codegen id : 3] +Input [2]: [sr_item_sk#9, sr_ticket_number#10] (10) Exchange Input [2]: [sr_item_sk#9, sr_ticket_number#10] @@ -177,12 +177,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#33), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 8] +(23) CometFilter Input [7]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] +Condition : (isnotnull(ws_item_sk#27) AND isnotnull(ws_bill_customer_sk#28)) -(24) Filter [codegen id : 8] +(24) ColumnarToRow [codegen id : 8] Input [7]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] -Condition : (isnotnull(ws_item_sk#27) AND isnotnull(ws_bill_customer_sk#28)) (25) Exchange Input [7]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] @@ -199,16 +199,16 @@ Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 10] -Input [3]: [wr_item_sk#35, wr_order_number#36, wr_returned_date_sk#37] - -(29) Filter [codegen id : 10] +(28) CometFilter Input [3]: [wr_item_sk#35, wr_order_number#36, wr_returned_date_sk#37] Condition : (isnotnull(wr_order_number#36) AND isnotnull(wr_item_sk#35)) -(30) Project [codegen id : 10] -Output [2]: [wr_item_sk#35, wr_order_number#36] +(29) CometProject Input [3]: [wr_item_sk#35, wr_order_number#36, wr_returned_date_sk#37] +Arguments: [wr_item_sk#35, wr_order_number#36], [wr_item_sk#35, wr_order_number#36] + +(30) ColumnarToRow [codegen id : 10] +Input [2]: [wr_item_sk#35, wr_order_number#36] (31) Exchange Input [2]: [wr_item_sk#35, wr_order_number#36] @@ -289,12 +289,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#60), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(47) ColumnarToRow [codegen id : 16] +(47) CometFilter Input [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] +Condition : (isnotnull(cs_item_sk#55) AND isnotnull(cs_bill_customer_sk#54)) -(48) Filter [codegen id : 16] +(48) ColumnarToRow [codegen id : 16] Input [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] -Condition : (isnotnull(cs_item_sk#55) AND isnotnull(cs_bill_customer_sk#54)) (49) Exchange Input [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] @@ -311,16 +311,16 @@ Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(52) ColumnarToRow [codegen id : 18] -Input [3]: [cr_item_sk#62, cr_order_number#63, cr_returned_date_sk#64] - -(53) Filter [codegen id : 18] +(52) CometFilter Input [3]: [cr_item_sk#62, cr_order_number#63, cr_returned_date_sk#64] Condition : (isnotnull(cr_order_number#63) AND isnotnull(cr_item_sk#62)) -(54) Project [codegen id : 18] -Output [2]: [cr_item_sk#62, cr_order_number#63] +(53) CometProject Input [3]: [cr_item_sk#62, cr_order_number#63, cr_returned_date_sk#64] +Arguments: [cr_item_sk#62, cr_order_number#63], [cr_item_sk#62, cr_order_number#63] + +(54) ColumnarToRow [codegen id : 18] +Input [2]: [cr_item_sk#62, cr_order_number#63] (55) Exchange Input [2]: [cr_item_sk#62, cr_order_number#63] @@ -401,8 +401,8 @@ Arguments: 100, [ratio#81 ASC NULLS FIRST, ss_qty#24 DESC NULLS LAST, ss_wc#25 D Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 BroadcastExchange (74) -+- * Filter (73) - +- * ColumnarToRow (72) ++- * ColumnarToRow (73) + +- CometFilter (72) +- CometScan parquet spark_catalog.default.date_dim (71) @@ -413,12 +413,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(72) ColumnarToRow [codegen id : 1] +(72) CometFilter Input [2]: [d_date_sk#12, d_year#13] +Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2000)) AND isnotnull(d_date_sk#12)) -(73) Filter [codegen id : 1] +(73) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#12, d_year#13] -Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2000)) AND isnotnull(d_date_sk#12)) (74) BroadcastExchange Input [2]: [d_date_sk#12, d_year#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/simplified.txt index bb2d54937..280687e30 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/simplified.txt @@ -25,16 +25,16 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes InputAdapter Exchange [ss_ticket_number,ss_item_sk] #2 WholeStageCodegen (1) - Filter [ss_item_sk,ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk,ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter WholeStageCodegen (4) @@ -42,10 +42,10 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes InputAdapter Exchange [sr_ticket_number,sr_item_sk] #4 WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_ticket_number,sr_item_sk] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] InputAdapter ReusedExchange [d_date_sk,d_year] #3 @@ -69,9 +69,9 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes InputAdapter Exchange [ws_order_number,ws_item_sk] #6 WholeStageCodegen (8) - Filter [ws_item_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_item_sk,ws_bill_customer_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -80,10 +80,10 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes InputAdapter Exchange [wr_order_number,wr_item_sk] #7 WholeStageCodegen (10) - Project [wr_item_sk,wr_order_number] - Filter [wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [wr_item_sk,wr_order_number] + CometFilter [wr_order_number,wr_item_sk] CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] InputAdapter ReusedExchange [d_date_sk,d_year] #3 @@ -107,9 +107,9 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes InputAdapter Exchange [cs_order_number,cs_item_sk] #9 WholeStageCodegen (16) - Filter [cs_item_sk,cs_bill_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_item_sk,cs_bill_customer_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -118,10 +118,10 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes InputAdapter Exchange [cr_order_number,cr_item_sk] #10 WholeStageCodegen (18) - Project [cr_item_sk,cr_order_number] - Filter [cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_order_number,cr_item_sk] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] InputAdapter ReusedExchange [d_date_sk,d_year] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/explain.txt index af1d4e56e..c89bad220 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/explain.txt @@ -11,23 +11,23 @@ TakeOrderedAndProject (30) : : +- * BroadcastHashJoin Inner BuildRight (12) : : :- * Project (6) : : : +- * BroadcastHashJoin Inner BuildRight (5) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- ReusedExchange (4) : : +- BroadcastExchange (11) - : : +- * Project (10) - : : +- * Filter (9) - : : +- * ColumnarToRow (8) + : : +- * ColumnarToRow (10) + : : +- CometProject (9) + : : +- CometFilter (8) : : +- CometScan parquet spark_catalog.default.store (7) : +- BroadcastExchange (18) - : +- * Project (17) - : +- * Filter (16) - : +- * ColumnarToRow (15) + : +- * ColumnarToRow (17) + : +- CometProject (16) + : +- CometFilter (15) : +- CometScan parquet spark_catalog.default.household_demographics (14) +- BroadcastExchange (27) - +- * Filter (26) - +- * ColumnarToRow (25) + +- * ColumnarToRow (26) + +- CometFilter (25) +- CometScan parquet spark_catalog.default.customer (24) @@ -39,12 +39,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] +Condition : ((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) -(3) Filter [codegen id : 4] +(3) ColumnarToRow [codegen id : 4] Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) (4) ReusedExchange [Reuses operator id: 35] Output [1]: [d_date_sk#10] @@ -66,16 +66,16 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_number_employees), GreaterThanOrEqual(s_number_employees,200), LessThanOrEqual(s_number_employees,295), IsNotNull(s_store_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] -Input [3]: [s_store_sk#11, s_number_employees#12, s_city#13] - -(9) Filter [codegen id : 2] +(8) CometFilter Input [3]: [s_store_sk#11, s_number_employees#12, s_city#13] Condition : (((isnotnull(s_number_employees#12) AND (s_number_employees#12 >= 200)) AND (s_number_employees#12 <= 295)) AND isnotnull(s_store_sk#11)) -(10) Project [codegen id : 2] -Output [2]: [s_store_sk#11, s_city#13] +(9) CometProject Input [3]: [s_store_sk#11, s_number_employees#12, s_city#13] +Arguments: [s_store_sk#11, s_city#13], [s_store_sk#11, s_city#13] + +(10) ColumnarToRow [codegen id : 2] +Input [2]: [s_store_sk#11, s_city#13] (11) BroadcastExchange Input [2]: [s_store_sk#11, s_city#13] @@ -98,16 +98,16 @@ Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [Or(EqualTo(hd_dep_count,6),GreaterThan(hd_vehicle_count,2)), IsNotNull(hd_demo_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 3] -Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] - -(16) Filter [codegen id : 3] +(15) CometFilter Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] Condition : (((hd_dep_count#15 = 6) OR (hd_vehicle_count#16 > 2)) AND isnotnull(hd_demo_sk#14)) -(17) Project [codegen id : 3] -Output [1]: [hd_demo_sk#14] +(16) CometProject Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] +Arguments: [hd_demo_sk#14], [hd_demo_sk#14] + +(17) ColumnarToRow [codegen id : 3] +Input [1]: [hd_demo_sk#14] (18) BroadcastExchange Input [1]: [hd_demo_sk#14] @@ -148,12 +148,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 5] +(25) CometFilter Input [3]: [c_customer_sk#25, c_first_name#26, c_last_name#27] +Condition : isnotnull(c_customer_sk#25) -(26) Filter [codegen id : 5] +(26) ColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#25, c_first_name#26, c_last_name#27] -Condition : isnotnull(c_customer_sk#25) (27) BroadcastExchange Input [3]: [c_customer_sk#25, c_first_name#26, c_last_name#27] @@ -177,9 +177,9 @@ Arguments: 100, [c_last_name#27 ASC NULLS FIRST, c_first_name#26 ASC NULLS FIRST Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 BroadcastExchange (35) -+- * Project (34) - +- * Filter (33) - +- * ColumnarToRow (32) ++- * ColumnarToRow (34) + +- CometProject (33) + +- CometFilter (32) +- CometScan parquet spark_catalog.default.date_dim (31) @@ -190,16 +190,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_dow), EqualTo(d_dow,1), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#10, d_year#29, d_dow#30] - -(33) Filter [codegen id : 1] +(32) CometFilter Input [3]: [d_date_sk#10, d_year#29, d_dow#30] Condition : (((isnotnull(d_dow#30) AND (d_dow#30 = 1)) AND d_year#29 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) -(34) Project [codegen id : 1] -Output [1]: [d_date_sk#10] +(33) CometProject Input [3]: [d_date_sk#10, d_year#29, d_dow#30] +Arguments: [d_date_sk#10], [d_date_sk#10] + +(34) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#10] (35) BroadcastExchange Input [1]: [d_date_sk#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/simplified.txt index 505434600..4c05c449c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/simplified.txt @@ -13,40 +13,40 @@ TakeOrderedAndProject [c_last_name,c_first_name,s_city,profit,substr(s_city, 1, BroadcastHashJoin [ss_store_sk,s_store_sk] Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk,ss_hdemo_sk,ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dow,d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_dow,d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] InputAdapter ReusedExchange [d_date_sk] #2 InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Project [s_store_sk,s_city] - Filter [s_number_employees,s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [s_store_sk,s_city] + CometFilter [s_number_employees,s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_number_employees,s_city] InputAdapter BroadcastExchange #4 WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [hd_demo_sk] + CometFilter [hd_dep_count,hd_vehicle_count,hd_demo_sk] CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] InputAdapter BroadcastExchange #5 WholeStageCodegen (5) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/explain.txt index 0a4159cd9..7d6d717c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/explain.txt @@ -9,22 +9,22 @@ TakeOrderedAndProject (43) : +- * BroadcastHashJoin Inner BuildRight (11) : :- * Project (6) : : +- * BroadcastHashJoin Inner BuildRight (5) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) + : : :- * ColumnarToRow (3) + : : : +- CometFilter (2) : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : +- ReusedExchange (4) : +- BroadcastExchange (10) - : +- * Filter (9) - : +- * ColumnarToRow (8) + : +- * ColumnarToRow (9) + : +- CometFilter (8) : +- CometScan parquet spark_catalog.default.store (7) +- BroadcastExchange (37) +- * HashAggregate (36) +- Exchange (35) +- * HashAggregate (34) +- * BroadcastHashJoin LeftSemi BuildRight (33) - :- * Project (16) - : +- * Filter (15) - : +- * ColumnarToRow (14) + :- * ColumnarToRow (16) + : +- CometProject (15) + : +- CometFilter (14) : +- CometScan parquet spark_catalog.default.customer_address (13) +- BroadcastExchange (32) +- * Project (31) @@ -34,13 +34,13 @@ TakeOrderedAndProject (43) +- * HashAggregate (27) +- * Project (26) +- * BroadcastHashJoin Inner BuildRight (25) - :- * Filter (19) - : +- * ColumnarToRow (18) + :- * ColumnarToRow (19) + : +- CometFilter (18) : +- CometScan parquet spark_catalog.default.customer_address (17) +- BroadcastExchange (24) - +- * Project (23) - +- * Filter (22) - +- * ColumnarToRow (21) + +- * ColumnarToRow (23) + +- CometProject (22) + +- CometFilter (21) +- CometScan parquet spark_catalog.default.customer (20) @@ -52,12 +52,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 8] +(2) CometFilter Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Condition : isnotnull(ss_store_sk#1) -(3) Filter [codegen id : 8] +(3) ColumnarToRow [codegen id : 8] Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_store_sk#1) (4) ReusedExchange [Reuses operator id: 48] Output [1]: [d_date_sk#5] @@ -79,12 +79,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_zip)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] +(8) CometFilter Input [3]: [s_store_sk#6, s_store_name#7, s_zip#8] +Condition : (isnotnull(s_store_sk#6) AND isnotnull(s_zip#8)) -(9) Filter [codegen id : 2] +(9) ColumnarToRow [codegen id : 2] Input [3]: [s_store_sk#6, s_store_name#7, s_zip#8] -Condition : (isnotnull(s_store_sk#6) AND isnotnull(s_zip#8)) (10) BroadcastExchange Input [3]: [s_store_sk#6, s_store_name#7, s_zip#8] @@ -106,16 +106,16 @@ Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] ReadSchema: struct -(14) ColumnarToRow [codegen id : 6] -Input [1]: [ca_zip#9] - -(15) Filter [codegen id : 6] +(14) CometFilter Input [1]: [ca_zip#9] Condition : (substr(ca_zip#9, 1, 5) INSET 10144, 10336, 10390, 10445, 10516, 10567, 11101, 11356, 11376, 11489, 11634, 11928, 12305, 13354, 13375, 13376, 13394, 13595, 13695, 13955, 14060, 14089, 14171, 14328, 14663, 14867, 14922, 15126, 15146, 15371, 15455, 15559, 15723, 15734, 15765, 15798, 15882, 16021, 16725, 16807, 17043, 17183, 17871, 17879, 17920, 18119, 18270, 18376, 18383, 18426, 18652, 18767, 18799, 18840, 18842, 18845, 18906, 19430, 19505, 19512, 19515, 19736, 19769, 19849, 20004, 20260, 20548, 21076, 21195, 21286, 21309, 21337, 21756, 22152, 22245, 22246, 22351, 22437, 22461, 22685, 22744, 22752, 22927, 23006, 23470, 23932, 23968, 24128, 24206, 24317, 24610, 24671, 24676, 24996, 25003, 25103, 25280, 25486, 25631, 25733, 25782, 25858, 25989, 26065, 26105, 26231, 26233, 26653, 26689, 26859, 27068, 27156, 27385, 27700, 28286, 28488, 28545, 28577, 28587, 28709, 28810, 28898, 28915, 29178, 29741, 29839, 30010, 30122, 30431, 30450, 30469, 30625, 30903, 31016, 31029, 31387, 31671, 31880, 32213, 32754, 33123, 33282, 33515, 33786, 34102, 34322, 34425, 35258, 35458, 35474, 35576, 35850, 35942, 36233, 36420, 36446, 36495, 36634, 37125, 37126, 37930, 38122, 38193, 38415, 38607, 38935, 39127, 39192, 39371, 39516, 39736, 39861, 39972, 40081, 40162, 40558, 40604, 41248, 41367, 41368, 41766, 41918, 42029, 42666, 42961, 43285, 43848, 43933, 44165, 44438, 45200, 45266, 45375, 45549, 45692, 45721, 45748, 46081, 46136, 46820, 47305, 47537, 47770, 48033, 48425, 48583, 49130, 49156, 49448, 50016, 50298, 50308, 50412, 51061, 51103, 51200, 51211, 51622, 51649, 51650, 51798, 51949, 52867, 53179, 53268, 53535, 53672, 54364, 54601, 54917, 55253, 55307, 55565, 56240, 56458, 56529, 56571, 56575, 56616, 56691, 56910, 57047, 57647, 57665, 57834, 57855, 58048, 58058, 58078, 58263, 58470, 58943, 59166, 59402, 60099, 60279, 60576, 61265, 61547, 61810, 61860, 62377, 62496, 62878, 62971, 63089, 63193, 63435, 63792, 63837, 63981, 64034, 64147, 64457, 64528, 64544, 65084, 65164, 66162, 66708, 66864, 67030, 67301, 67467, 67473, 67853, 67875, 67897, 68014, 68100, 68101, 68309, 68341, 68621, 68786, 68806, 68880, 68893, 68908, 69035, 69399, 69913, 69952, 70372, 70466, 70738, 71256, 71286, 71791, 71954, 72013, 72151, 72175, 72305, 72325, 72425, 72550, 72823, 73134, 73171, 73241, 73273, 73520, 73650, 74351, 75691, 76107, 76231, 76232, 76614, 76638, 76698, 77191, 77556, 77610, 77721, 78451, 78567, 78668, 78890, 79077, 79777, 79994, 81019, 81096, 81312, 81426, 82136, 82276, 82636, 83041, 83144, 83444, 83849, 83921, 83926, 83933, 84093, 84935, 85816, 86057, 86198, 86284, 86379, 87343, 87501, 87816, 88086, 88190, 88424, 88885, 89091, 89360, 90225, 90257, 90578, 91068, 91110, 91137, 91393, 92712, 94167, 94627, 94898, 94945, 94983, 96451, 96576, 96765, 96888, 96976, 97189, 97789, 98025, 98235, 98294, 98359, 98569, 99076, 99543 AND isnotnull(substr(ca_zip#9, 1, 5))) -(16) Project [codegen id : 6] -Output [1]: [substr(ca_zip#9, 1, 5) AS ca_zip#10] +(15) CometProject Input [1]: [ca_zip#9] +Arguments: [ca_zip#10], [substr(ca_zip#9, 1, 5) AS ca_zip#10] + +(16) ColumnarToRow [codegen id : 6] +Input [1]: [ca_zip#10] (17) Scan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#11, ca_zip#12] @@ -124,12 +124,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 4] +(18) CometFilter Input [2]: [ca_address_sk#11, ca_zip#12] +Condition : isnotnull(ca_address_sk#11) -(19) Filter [codegen id : 4] +(19) ColumnarToRow [codegen id : 4] Input [2]: [ca_address_sk#11, ca_zip#12] -Condition : isnotnull(ca_address_sk#11) (20) Scan parquet spark_catalog.default.customer Output [2]: [c_current_addr_sk#13, c_preferred_cust_flag#14] @@ -138,16 +138,16 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_preferred_cust_flag), EqualTo(c_preferred_cust_flag,Y), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 3] -Input [2]: [c_current_addr_sk#13, c_preferred_cust_flag#14] - -(22) Filter [codegen id : 3] +(21) CometFilter Input [2]: [c_current_addr_sk#13, c_preferred_cust_flag#14] Condition : ((isnotnull(c_preferred_cust_flag#14) AND (c_preferred_cust_flag#14 = Y)) AND isnotnull(c_current_addr_sk#13)) -(23) Project [codegen id : 3] -Output [1]: [c_current_addr_sk#13] +(22) CometProject Input [2]: [c_current_addr_sk#13, c_preferred_cust_flag#14] +Arguments: [c_current_addr_sk#13], [c_current_addr_sk#13] + +(23) ColumnarToRow [codegen id : 3] +Input [1]: [c_current_addr_sk#13] (24) BroadcastExchange Input [1]: [c_current_addr_sk#13] @@ -257,9 +257,9 @@ Arguments: 100, [s_store_name#7 ASC NULLS FIRST], [s_store_name#7, sum(ss_net_pr Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 BroadcastExchange (48) -+- * Project (47) - +- * Filter (46) - +- * ColumnarToRow (45) ++- * ColumnarToRow (47) + +- CometProject (46) + +- CometFilter (45) +- CometScan parquet spark_catalog.default.date_dim (44) @@ -270,16 +270,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#5, d_year#24, d_qoy#25] - -(46) Filter [codegen id : 1] +(45) CometFilter Input [3]: [d_date_sk#5, d_year#24, d_qoy#25] Condition : ((((isnotnull(d_qoy#25) AND isnotnull(d_year#24)) AND (d_qoy#25 = 2)) AND (d_year#24 = 1998)) AND isnotnull(d_date_sk#5)) -(47) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(46) CometProject Input [3]: [d_date_sk#5, d_year#24, d_qoy#25] +Arguments: [d_date_sk#5], [d_date_sk#5] + +(47) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#5] (48) BroadcastExchange Input [1]: [d_date_sk#5] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/simplified.txt index 0cafe9244..76fa27693 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/simplified.txt @@ -11,26 +11,26 @@ TakeOrderedAndProject [s_store_name,sum(ss_net_profit)] BroadcastHashJoin [ss_store_sk,s_store_sk] Project [ss_store_sk,ss_net_profit] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_qoy,d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter ReusedExchange [d_date_sk] #2 InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Filter [s_store_sk,s_zip] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk,s_zip] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] InputAdapter BroadcastExchange #4 @@ -41,10 +41,10 @@ TakeOrderedAndProject [s_store_name,sum(ss_net_profit)] WholeStageCodegen (6) HashAggregate [ca_zip] BroadcastHashJoin [ca_zip,ca_zip] - Project [ca_zip] - Filter [ca_zip] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ca_zip] [ca_zip] + CometFilter [ca_zip] CometScan parquet spark_catalog.default.customer_address [ca_zip] InputAdapter BroadcastExchange #6 @@ -58,15 +58,15 @@ TakeOrderedAndProject [s_store_name,sum(ss_net_profit)] HashAggregate [ca_zip] [count,count] Project [ca_zip] BroadcastHashJoin [ca_address_sk,c_current_addr_sk] - Filter [ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] InputAdapter BroadcastExchange #8 WholeStageCodegen (3) - Project [c_current_addr_sk] - Filter [c_preferred_cust_flag,c_current_addr_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [c_current_addr_sk] + CometFilter [c_preferred_cust_flag,c_current_addr_sk] CometScan parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/explain.txt index 6bee1df88..db2d015db 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/explain.txt @@ -20,29 +20,29 @@ TakeOrderedAndProject (107) : : : : : +- * SortMergeJoin LeftOuter (12) : : : : : :- * Sort (5) : : : : : : +- Exchange (4) - : : : : : : +- * Filter (3) - : : : : : : +- * ColumnarToRow (2) + : : : : : : +- * ColumnarToRow (3) + : : : : : : +- CometFilter (2) : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : : : +- * Sort (11) : : : : : +- Exchange (10) - : : : : : +- * Project (9) - : : : : : +- * Filter (8) - : : : : : +- * ColumnarToRow (7) + : : : : : +- * ColumnarToRow (9) + : : : : : +- CometProject (8) + : : : : : +- CometFilter (7) : : : : : +- CometScan parquet spark_catalog.default.store_returns (6) : : : : +- ReusedExchange (14) : : : +- BroadcastExchange (20) - : : : +- * Filter (19) - : : : +- * ColumnarToRow (18) + : : : +- * ColumnarToRow (19) + : : : +- CometFilter (18) : : : +- CometScan parquet spark_catalog.default.store (17) : : +- BroadcastExchange (27) - : : +- * Project (26) - : : +- * Filter (25) - : : +- * ColumnarToRow (24) + : : +- * ColumnarToRow (26) + : : +- CometProject (25) + : : +- CometFilter (24) : : +- CometScan parquet spark_catalog.default.item (23) : +- BroadcastExchange (34) - : +- * Project (33) - : +- * Filter (32) - : +- * ColumnarToRow (31) + : +- * ColumnarToRow (33) + : +- CometProject (32) + : +- CometFilter (31) : +- CometScan parquet spark_catalog.default.promotion (30) :- * HashAggregate (70) : +- Exchange (69) @@ -59,19 +59,19 @@ TakeOrderedAndProject (107) : : : : : +- * SortMergeJoin LeftOuter (51) : : : : : :- * Sort (44) : : : : : : +- Exchange (43) - : : : : : : +- * Filter (42) - : : : : : : +- * ColumnarToRow (41) + : : : : : : +- * ColumnarToRow (42) + : : : : : : +- CometFilter (41) : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (40) : : : : : +- * Sort (50) : : : : : +- Exchange (49) - : : : : : +- * Project (48) - : : : : : +- * Filter (47) - : : : : : +- * ColumnarToRow (46) + : : : : : +- * ColumnarToRow (48) + : : : : : +- CometProject (47) + : : : : : +- CometFilter (46) : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (45) : : : : +- ReusedExchange (53) : : : +- BroadcastExchange (59) - : : : +- * Filter (58) - : : : +- * ColumnarToRow (57) + : : : +- * ColumnarToRow (58) + : : : +- CometFilter (57) : : : +- CometScan parquet spark_catalog.default.catalog_page (56) : : +- ReusedExchange (62) : +- ReusedExchange (65) @@ -90,19 +90,19 @@ TakeOrderedAndProject (107) : : : : +- * SortMergeJoin LeftOuter (82) : : : : :- * Sort (75) : : : : : +- Exchange (74) - : : : : : +- * Filter (73) - : : : : : +- * ColumnarToRow (72) + : : : : : +- * ColumnarToRow (73) + : : : : : +- CometFilter (72) : : : : : +- CometScan parquet spark_catalog.default.web_sales (71) : : : : +- * Sort (81) : : : : +- Exchange (80) - : : : : +- * Project (79) - : : : : +- * Filter (78) - : : : : +- * ColumnarToRow (77) + : : : : +- * ColumnarToRow (79) + : : : : +- CometProject (78) + : : : : +- CometFilter (77) : : : : +- CometScan parquet spark_catalog.default.web_returns (76) : : : +- ReusedExchange (84) : : +- BroadcastExchange (90) - : : +- * Filter (89) - : : +- * ColumnarToRow (88) + : : +- * ColumnarToRow (89) + : : +- CometFilter (88) : : +- CometScan parquet spark_catalog.default.web_site (87) : +- ReusedExchange (93) +- ReusedExchange (96) @@ -116,12 +116,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] +(2) CometFilter Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) -(3) Filter [codegen id : 1] +(3) ColumnarToRow [codegen id : 1] Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) (4) Exchange Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] @@ -138,16 +138,16 @@ Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 3] -Input [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] - -(8) Filter [codegen id : 3] +(7) CometFilter Input [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] Condition : (isnotnull(sr_item_sk#9) AND isnotnull(sr_ticket_number#10)) -(9) Project [codegen id : 3] -Output [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] +(8) CometProject Input [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Arguments: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12], [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] + +(9) ColumnarToRow [codegen id : 3] +Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] (10) Exchange Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] @@ -187,12 +187,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 6] +(18) CometFilter Input [2]: [s_store_sk#15, s_store_id#16] +Condition : isnotnull(s_store_sk#15) -(19) Filter [codegen id : 6] +(19) ColumnarToRow [codegen id : 6] Input [2]: [s_store_sk#15, s_store_id#16] -Condition : isnotnull(s_store_sk#15) (20) BroadcastExchange Input [2]: [s_store_sk#15, s_store_id#16] @@ -215,16 +215,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 7] -Input [2]: [i_item_sk#17, i_current_price#18] - -(25) Filter [codegen id : 7] +(24) CometFilter Input [2]: [i_item_sk#17, i_current_price#18] Condition : ((isnotnull(i_current_price#18) AND (i_current_price#18 > 50.00)) AND isnotnull(i_item_sk#17)) -(26) Project [codegen id : 7] -Output [1]: [i_item_sk#17] +(25) CometProject Input [2]: [i_item_sk#17, i_current_price#18] +Arguments: [i_item_sk#17], [i_item_sk#17] + +(26) ColumnarToRow [codegen id : 7] +Input [1]: [i_item_sk#17] (27) BroadcastExchange Input [1]: [i_item_sk#17] @@ -247,16 +247,16 @@ Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_channel_tv), EqualTo(p_channel_tv,N), IsNotNull(p_promo_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 8] -Input [2]: [p_promo_sk#19, p_channel_tv#20] - -(32) Filter [codegen id : 8] +(31) CometFilter Input [2]: [p_promo_sk#19, p_channel_tv#20] Condition : ((isnotnull(p_channel_tv#20) AND (p_channel_tv#20 = N)) AND isnotnull(p_promo_sk#19)) -(33) Project [codegen id : 8] -Output [1]: [p_promo_sk#19] +(32) CometProject Input [2]: [p_promo_sk#19, p_channel_tv#20] +Arguments: [p_promo_sk#19], [p_promo_sk#19] + +(33) ColumnarToRow [codegen id : 8] +Input [1]: [p_promo_sk#19] (34) BroadcastExchange Input [1]: [p_promo_sk#19] @@ -298,12 +298,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#45), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 11] +(41) CometFilter Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +Condition : ((isnotnull(cs_catalog_page_sk#39) AND isnotnull(cs_item_sk#40)) AND isnotnull(cs_promo_sk#41)) -(42) Filter [codegen id : 11] +(42) ColumnarToRow [codegen id : 11] Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Condition : ((isnotnull(cs_catalog_page_sk#39) AND isnotnull(cs_item_sk#40)) AND isnotnull(cs_promo_sk#41)) (43) Exchange Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] @@ -320,16 +320,16 @@ Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(46) ColumnarToRow [codegen id : 13] -Input [5]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50, cr_returned_date_sk#51] - -(47) Filter [codegen id : 13] +(46) CometFilter Input [5]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50, cr_returned_date_sk#51] Condition : (isnotnull(cr_item_sk#47) AND isnotnull(cr_order_number#48)) -(48) Project [codegen id : 13] -Output [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] +(47) CometProject Input [5]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50, cr_returned_date_sk#51] +Arguments: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50], [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] + +(48) ColumnarToRow [codegen id : 13] +Input [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] (49) Exchange Input [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] @@ -369,12 +369,12 @@ Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 16] +(57) CometFilter Input [2]: [cp_catalog_page_sk#53, cp_catalog_page_id#54] +Condition : isnotnull(cp_catalog_page_sk#53) -(58) Filter [codegen id : 16] +(58) ColumnarToRow [codegen id : 16] Input [2]: [cp_catalog_page_sk#53, cp_catalog_page_id#54] -Condition : isnotnull(cp_catalog_page_sk#53) (59) BroadcastExchange Input [2]: [cp_catalog_page_sk#53, cp_catalog_page_id#54] @@ -442,12 +442,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#81), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] ReadSchema: struct -(72) ColumnarToRow [codegen id : 21] +(72) CometFilter Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] +Condition : ((isnotnull(ws_web_site_sk#76) AND isnotnull(ws_item_sk#75)) AND isnotnull(ws_promo_sk#77)) -(73) Filter [codegen id : 21] +(73) ColumnarToRow [codegen id : 21] Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Condition : ((isnotnull(ws_web_site_sk#76) AND isnotnull(ws_item_sk#75)) AND isnotnull(ws_promo_sk#77)) (74) Exchange Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] @@ -464,16 +464,16 @@ Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] ReadSchema: struct -(77) ColumnarToRow [codegen id : 23] -Input [5]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86, wr_returned_date_sk#87] - -(78) Filter [codegen id : 23] +(77) CometFilter Input [5]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86, wr_returned_date_sk#87] Condition : (isnotnull(wr_item_sk#83) AND isnotnull(wr_order_number#84)) -(79) Project [codegen id : 23] -Output [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] +(78) CometProject Input [5]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86, wr_returned_date_sk#87] +Arguments: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86], [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] + +(79) ColumnarToRow [codegen id : 23] +Input [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] (80) Exchange Input [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] @@ -513,12 +513,12 @@ Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(88) ColumnarToRow [codegen id : 26] +(88) CometFilter Input [2]: [web_site_sk#89, web_site_id#90] +Condition : isnotnull(web_site_sk#89) -(89) Filter [codegen id : 26] +(89) ColumnarToRow [codegen id : 26] Input [2]: [web_site_sk#89, web_site_id#90] -Condition : isnotnull(web_site_sk#89) (90) BroadcastExchange Input [2]: [web_site_sk#89, web_site_id#90] @@ -610,9 +610,9 @@ Arguments: 100, [channel#111 ASC NULLS FIRST, id#112 ASC NULLS FIRST], [channel# Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 BroadcastExchange (112) -+- * Project (111) - +- * Filter (110) - +- * ColumnarToRow (109) ++- * ColumnarToRow (111) + +- CometProject (110) + +- CometFilter (109) +- CometScan parquet spark_catalog.default.date_dim (108) @@ -623,16 +623,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-22), IsNotNull(d_date_sk)] ReadSchema: struct -(109) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#14, d_date#132] - -(110) Filter [codegen id : 1] +(109) CometFilter Input [2]: [d_date_sk#14, d_date#132] Condition : (((isnotnull(d_date#132) AND (d_date#132 >= 2000-08-23)) AND (d_date#132 <= 2000-09-22)) AND isnotnull(d_date_sk#14)) -(111) Project [codegen id : 1] -Output [1]: [d_date_sk#14] +(110) CometProject Input [2]: [d_date_sk#14, d_date#132] +Arguments: [d_date_sk#14], [d_date_sk#14] + +(111) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#14] (112) BroadcastExchange Input [1]: [d_date_sk#14] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/simplified.txt index 9b229a658..7e257bdc6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/simplified.txt @@ -30,17 +30,17 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter Exchange [ss_item_sk,ss_ticket_number] #3 WholeStageCodegen (1) - Filter [ss_store_sk,ss_item_sk,ss_promo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk,ss_item_sk,ss_promo_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter WholeStageCodegen (4) @@ -48,35 +48,35 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter Exchange [sr_item_sk,sr_ticket_number] #5 WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - Filter [sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometFilter [sr_item_sk,sr_ticket_number] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] InputAdapter ReusedExchange [d_date_sk] #4 InputAdapter BroadcastExchange #6 WholeStageCodegen (6) - Filter [s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] InputAdapter BroadcastExchange #7 WholeStageCodegen (7) - Project [i_item_sk] - Filter [i_current_price,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_item_sk] + CometFilter [i_current_price,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price] InputAdapter BroadcastExchange #8 WholeStageCodegen (8) - Project [p_promo_sk] - Filter [p_channel_tv,p_promo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [p_promo_sk] + CometFilter [p_channel_tv,p_promo_sk] CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] WholeStageCodegen (20) HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] @@ -100,9 +100,9 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter Exchange [cs_item_sk,cs_order_number] #10 WholeStageCodegen (11) - Filter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -111,19 +111,19 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter Exchange [cr_item_sk,cr_order_number] #11 WholeStageCodegen (13) - Project [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometFilter [cr_item_sk,cr_order_number] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] InputAdapter ReusedExchange [d_date_sk] #4 InputAdapter BroadcastExchange #12 WholeStageCodegen (16) - Filter [cp_catalog_page_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cp_catalog_page_sk] CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] InputAdapter ReusedExchange [i_item_sk] #7 @@ -151,9 +151,9 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter Exchange [ws_item_sk,ws_order_number] #14 WholeStageCodegen (21) - Filter [ws_web_site_sk,ws_item_sk,ws_promo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_web_site_sk,ws_item_sk,ws_promo_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -162,19 +162,19 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter Exchange [wr_item_sk,wr_order_number] #15 WholeStageCodegen (23) - Project [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - Filter [wr_item_sk,wr_order_number] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometFilter [wr_item_sk,wr_order_number] CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] InputAdapter ReusedExchange [d_date_sk] #4 InputAdapter BroadcastExchange #16 WholeStageCodegen (26) - Filter [web_site_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [web_site_sk] CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] InputAdapter ReusedExchange [i_item_sk] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/explain.txt index dba7a3c3d..431d1453d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/explain.txt @@ -14,13 +14,13 @@ TakeOrderedAndProject (48) : : : +- * BroadcastHashJoin Inner BuildRight (11) : : : :- * Project (6) : : : : +- * BroadcastHashJoin Inner BuildRight (5) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (1) : : : : +- ReusedExchange (4) : : : +- BroadcastExchange (10) - : : : +- * Filter (9) - : : : +- * ColumnarToRow (8) + : : : +- * ColumnarToRow (9) + : : : +- CometFilter (8) : : : +- CometScan parquet spark_catalog.default.customer_address (7) : : +- BroadcastExchange (33) : : +- * Filter (32) @@ -34,18 +34,18 @@ TakeOrderedAndProject (48) : : +- * BroadcastHashJoin Inner BuildRight (24) : : :- * Project (22) : : : +- * BroadcastHashJoin Inner BuildRight (21) - : : : :- * Filter (19) - : : : : +- * ColumnarToRow (18) + : : : :- * ColumnarToRow (19) + : : : : +- CometFilter (18) : : : : +- CometScan parquet spark_catalog.default.catalog_returns (17) : : : +- ReusedExchange (20) : : +- ReusedExchange (23) : +- BroadcastExchange (39) - : +- * Filter (38) - : +- * ColumnarToRow (37) + : +- * ColumnarToRow (38) + : +- CometFilter (37) : +- CometScan parquet spark_catalog.default.customer (36) +- BroadcastExchange (45) - +- * Filter (44) - +- * ColumnarToRow (43) + +- * ColumnarToRow (44) + +- CometFilter (43) +- CometScan parquet spark_catalog.default.customer_address (42) @@ -57,12 +57,12 @@ PartitionFilters: [isnotnull(cr_returned_date_sk#4), dynamicpruningexpression(cr PushedFilters: [IsNotNull(cr_returning_addr_sk), IsNotNull(cr_returning_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] +Condition : (isnotnull(cr_returning_addr_sk#2) AND isnotnull(cr_returning_customer_sk#1)) -(3) Filter [codegen id : 3] +(3) ColumnarToRow [codegen id : 3] Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] -Condition : (isnotnull(cr_returning_addr_sk#2) AND isnotnull(cr_returning_customer_sk#1)) (4) ReusedExchange [Reuses operator id: 53] Output [1]: [d_date_sk#6] @@ -84,12 +84,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_state)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] +(8) CometFilter Input [2]: [ca_address_sk#7, ca_state#8] +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_state#8)) -(9) Filter [codegen id : 2] +(9) ColumnarToRow [codegen id : 2] Input [2]: [ca_address_sk#7, ca_state#8] -Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_state#8)) (10) BroadcastExchange Input [2]: [ca_address_sk#7, ca_state#8] @@ -135,12 +135,12 @@ PartitionFilters: [isnotnull(cr_returned_date_sk#18), dynamicpruningexpression(c PushedFilters: [IsNotNull(cr_returning_addr_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 6] +(18) CometFilter Input [4]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18] +Condition : isnotnull(cr_returning_addr_sk#16) -(19) Filter [codegen id : 6] +(19) ColumnarToRow [codegen id : 6] Input [4]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18] -Condition : isnotnull(cr_returning_addr_sk#16) (20) ReusedExchange [Reuses operator id: 53] Output [1]: [d_date_sk#20] @@ -229,12 +229,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 9] +(37) CometFilter Input [6]: [c_customer_sk#33, c_customer_id#34, c_current_addr_sk#35, c_salutation#36, c_first_name#37, c_last_name#38] +Condition : (isnotnull(c_customer_sk#33) AND isnotnull(c_current_addr_sk#35)) -(38) Filter [codegen id : 9] +(38) ColumnarToRow [codegen id : 9] Input [6]: [c_customer_sk#33, c_customer_id#34, c_current_addr_sk#35, c_salutation#36, c_first_name#37, c_last_name#38] -Condition : (isnotnull(c_customer_sk#33) AND isnotnull(c_current_addr_sk#35)) (39) BroadcastExchange Input [6]: [c_customer_sk#33, c_customer_id#34, c_current_addr_sk#35, c_salutation#36, c_first_name#37, c_last_name#38] @@ -257,12 +257,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)] ReadSchema: struct -(43) ColumnarToRow [codegen id : 10] +(43) CometFilter Input [12]: [ca_address_sk#39, ca_street_number#40, ca_street_name#41, ca_street_type#42, ca_suite_number#43, ca_city#44, ca_county#45, ca_state#46, ca_zip#47, ca_country#48, ca_gmt_offset#49, ca_location_type#50] +Condition : ((isnotnull(ca_state#46) AND (ca_state#46 = GA)) AND isnotnull(ca_address_sk#39)) -(44) Filter [codegen id : 10] +(44) ColumnarToRow [codegen id : 10] Input [12]: [ca_address_sk#39, ca_street_number#40, ca_street_name#41, ca_street_type#42, ca_suite_number#43, ca_city#44, ca_county#45, ca_state#46, ca_zip#47, ca_country#48, ca_gmt_offset#49, ca_location_type#50] -Condition : ((isnotnull(ca_state#46) AND (ca_state#46 = GA)) AND isnotnull(ca_address_sk#39)) (45) BroadcastExchange Input [12]: [ca_address_sk#39, ca_street_number#40, ca_street_name#41, ca_street_type#42, ca_suite_number#43, ca_city#44, ca_county#45, ca_state#46, ca_zip#47, ca_country#48, ca_gmt_offset#49, ca_location_type#50] @@ -286,9 +286,9 @@ Arguments: 100, [c_customer_id#34 ASC NULLS FIRST, c_salutation#36 ASC NULLS FIR Subquery:1 Hosting operator id = 1 Hosting Expression = cr_returned_date_sk#4 IN dynamicpruning#5 BroadcastExchange (53) -+- * Project (52) - +- * Filter (51) - +- * ColumnarToRow (50) ++- * ColumnarToRow (52) + +- CometProject (51) + +- CometFilter (50) +- CometScan parquet spark_catalog.default.date_dim (49) @@ -299,16 +299,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(50) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#6, d_year#51] - -(51) Filter [codegen id : 1] +(50) CometFilter Input [2]: [d_date_sk#6, d_year#51] Condition : ((isnotnull(d_year#51) AND (d_year#51 = 2000)) AND isnotnull(d_date_sk#6)) -(52) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(51) CometProject Input [2]: [d_date_sk#6, d_year#51] +Arguments: [d_date_sk#6], [d_date_sk#6] + +(52) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#6] (53) BroadcastExchange Input [1]: [d_date_sk#6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/simplified.txt index 2b63992d3..6f042847c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/simplified.txt @@ -16,26 +16,26 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_st BroadcastHashJoin [cr_returning_addr_sk,ca_address_sk] Project [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] BroadcastHashJoin [cr_returned_date_sk,d_date_sk] - Filter [cr_returning_addr_sk,cr_returning_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cr_returning_addr_sk,cr_returning_customer_sk] CometScan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [d_date_sk] #2 InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Filter [ca_address_sk,ca_state] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ca_address_sk,ca_state] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #4 @@ -55,9 +55,9 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_st BroadcastHashJoin [cr_returning_addr_sk,ca_address_sk] Project [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] BroadcastHashJoin [cr_returned_date_sk,d_date_sk] - Filter [cr_returning_addr_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cr_returning_addr_sk] CometScan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -67,14 +67,14 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_st InputAdapter BroadcastExchange #7 WholeStageCodegen (9) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_current_addr_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] InputAdapter BroadcastExchange #8 WholeStageCodegen (10) - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ca_state,ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/explain.txt index 348f4d58b..7609fa520 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/explain.txt @@ -10,19 +10,19 @@ TakeOrderedAndProject (25) : +- * BroadcastHashJoin Inner BuildRight (13) : :- * Project (11) : : +- * BroadcastHashJoin Inner BuildRight (10) - : : :- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) + : : :- * ColumnarToRow (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) : : : +- CometScan parquet spark_catalog.default.item (1) : : +- BroadcastExchange (9) - : : +- * Project (8) - : : +- * Filter (7) - : : +- * ColumnarToRow (6) + : : +- * ColumnarToRow (8) + : : +- CometProject (7) + : : +- CometFilter (6) : : +- CometScan parquet spark_catalog.default.inventory (5) : +- ReusedExchange (12) - +- * Project (19) - +- * Filter (18) - +- * ColumnarToRow (17) + +- * ColumnarToRow (19) + +- CometProject (18) + +- CometFilter (17) +- CometScan parquet spark_catalog.default.store_sales (16) @@ -33,16 +33,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,62.00), LessThanOrEqual(i_current_price,92.00), In(i_manufact_id, [129,270,423,821]), IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 62.00)) AND (i_current_price#4 <= 92.00)) AND i_manufact_id#5 IN (129,270,821,423)) AND isnotnull(i_item_sk#1)) -(4) Project [codegen id : 3] -Output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +(3) CometProject Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] +Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] + +(4) ColumnarToRow [codegen id : 3] +Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] (5) Scan parquet spark_catalog.default.inventory Output [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] @@ -52,16 +52,16 @@ PartitionFilters: [isnotnull(inv_date_sk#8), dynamicpruningexpression(inv_date_s PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] - -(7) Filter [codegen id : 1] +(6) CometFilter Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] Condition : (((isnotnull(inv_quantity_on_hand#7) AND (inv_quantity_on_hand#7 >= 100)) AND (inv_quantity_on_hand#7 <= 500)) AND isnotnull(inv_item_sk#6)) -(8) Project [codegen id : 1] -Output [2]: [inv_item_sk#6, inv_date_sk#8] +(7) CometProject Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] +Arguments: [inv_item_sk#6, inv_date_sk#8], [inv_item_sk#6, inv_date_sk#8] + +(8) ColumnarToRow [codegen id : 1] +Input [2]: [inv_item_sk#6, inv_date_sk#8] (9) BroadcastExchange Input [2]: [inv_item_sk#6, inv_date_sk#8] @@ -101,16 +101,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(17) ColumnarToRow -Input [2]: [ss_item_sk#11, ss_sold_date_sk#12] - -(18) Filter +(17) CometFilter Input [2]: [ss_item_sk#11, ss_sold_date_sk#12] Condition : isnotnull(ss_item_sk#11) -(19) Project -Output [1]: [ss_item_sk#11] +(18) CometProject Input [2]: [ss_item_sk#11, ss_sold_date_sk#12] +Arguments: [ss_item_sk#11], [ss_item_sk#11] + +(19) ColumnarToRow +Input [1]: [ss_item_sk#11] (20) BroadcastHashJoin [codegen id : 4] Left keys [1]: [i_item_sk#1] @@ -148,9 +148,9 @@ Arguments: 100, [i_item_id#2 ASC NULLS FIRST], [i_item_id#2, i_item_desc#3, i_cu Subquery:1 Hosting operator id = 5 Hosting Expression = inv_date_sk#8 IN dynamicpruning#9 BroadcastExchange (30) -+- * Project (29) - +- * Filter (28) - +- * ColumnarToRow (27) ++- * ColumnarToRow (29) + +- CometProject (28) + +- CometFilter (27) +- CometScan parquet spark_catalog.default.date_dim (26) @@ -161,16 +161,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-05-25), LessThanOrEqual(d_date,2000-07-24), IsNotNull(d_date_sk)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#10, d_date#13] - -(28) Filter [codegen id : 1] +(27) CometFilter Input [2]: [d_date_sk#10, d_date#13] Condition : (((isnotnull(d_date#13) AND (d_date#13 >= 2000-05-25)) AND (d_date#13 <= 2000-07-24)) AND isnotnull(d_date_sk#10)) -(29) Project [codegen id : 1] -Output [1]: [d_date_sk#10] +(28) CometProject Input [2]: [d_date_sk#10, d_date#13] +Arguments: [d_date_sk#10], [d_date_sk#10] + +(29) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#10] (30) BroadcastExchange Input [1]: [d_date_sk#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/simplified.txt index 631237f50..0252eb575 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/simplified.txt @@ -14,31 +14,31 @@ TakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] BroadcastHashJoin [inv_date_sk,d_date_sk] Project [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] BroadcastHashJoin [i_item_sk,inv_item_sk] - Project [i_item_sk,i_item_id,i_item_desc,i_current_price] - Filter [i_current_price,i_manufact_id,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price] + CometFilter [i_current_price,i_manufact_id,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Project [inv_item_sk,inv_date_sk] - Filter [inv_quantity_on_hand,inv_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [inv_item_sk,inv_date_sk] + CometFilter [inv_quantity_on_hand,inv_item_sk] CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter ReusedExchange [d_date_sk] #4 - Project [ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_item_sk] + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/explain.txt index 74f90aab2..5183e0275 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/explain.txt @@ -11,12 +11,12 @@ TakeOrderedAndProject (46) : : +- * BroadcastHashJoin Inner BuildRight (11) : : :- * Project (9) : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_returns (1) : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) + : : : +- * ColumnarToRow (6) + : : : +- CometFilter (5) : : : +- CometScan parquet spark_catalog.default.item (4) : : +- ReusedExchange (10) : +- BroadcastExchange (28) @@ -27,8 +27,8 @@ TakeOrderedAndProject (46) : +- * BroadcastHashJoin Inner BuildRight (23) : :- * Project (21) : : +- * BroadcastHashJoin Inner BuildRight (20) - : : :- * Filter (18) - : : : +- * ColumnarToRow (17) + : : :- * ColumnarToRow (18) + : : : +- CometFilter (17) : : : +- CometScan parquet spark_catalog.default.catalog_returns (16) : : +- ReusedExchange (19) : +- ReusedExchange (22) @@ -40,8 +40,8 @@ TakeOrderedAndProject (46) +- * BroadcastHashJoin Inner BuildRight (38) :- * Project (36) : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Filter (33) - : : +- * ColumnarToRow (32) + : :- * ColumnarToRow (33) + : : +- CometFilter (32) : : +- CometScan parquet spark_catalog.default.web_returns (31) : +- ReusedExchange (34) +- ReusedExchange (37) @@ -55,12 +55,12 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#3), dynamicpruningexpression(sr PushedFilters: [IsNotNull(sr_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] +(2) CometFilter Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] +Condition : isnotnull(sr_item_sk#1) -(3) Filter [codegen id : 5] +(3) ColumnarToRow [codegen id : 5] Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] -Condition : isnotnull(sr_item_sk#1) (4) Scan parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_item_id#6] @@ -69,12 +69,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_item_id)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [2]: [i_item_sk#5, i_item_id#6] +Condition : (isnotnull(i_item_sk#5) AND isnotnull(i_item_id#6)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [2]: [i_item_sk#5, i_item_id#6] -Condition : (isnotnull(i_item_sk#5) AND isnotnull(i_item_id#6)) (7) BroadcastExchange Input [2]: [i_item_sk#5, i_item_id#6] @@ -129,12 +129,12 @@ PartitionFilters: [isnotnull(cr_returned_date_sk#15), dynamicpruningexpression(c PushedFilters: [IsNotNull(cr_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 10] +(17) CometFilter Input [3]: [cr_item_sk#13, cr_return_quantity#14, cr_returned_date_sk#15] +Condition : isnotnull(cr_item_sk#13) -(18) Filter [codegen id : 10] +(18) ColumnarToRow [codegen id : 10] Input [3]: [cr_item_sk#13, cr_return_quantity#14, cr_returned_date_sk#15] -Condition : isnotnull(cr_item_sk#13) (19) ReusedExchange [Reuses operator id: 7] Output [2]: [i_item_sk#17, i_item_id#18] @@ -202,12 +202,12 @@ PartitionFilters: [isnotnull(wr_returned_date_sk#27), dynamicpruningexpression(w PushedFilters: [IsNotNull(wr_item_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 16] +(32) CometFilter Input [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] +Condition : isnotnull(wr_item_sk#25) -(33) Filter [codegen id : 16] +(33) ColumnarToRow [codegen id : 16] Input [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] -Condition : isnotnull(wr_item_sk#25) (34) ReusedExchange [Reuses operator id: 7] Output [2]: [i_item_sk#29, i_item_id#30] @@ -277,8 +277,8 @@ Subquery:1 Hosting operator id = 1 Hosting Expression = sr_returned_date_sk#3 IN BroadcastExchange (62) +- * Project (61) +- * BroadcastHashJoin LeftSemi BuildRight (60) - :- * Filter (49) - : +- * ColumnarToRow (48) + :- * ColumnarToRow (49) + : +- CometFilter (48) : +- CometScan parquet spark_catalog.default.date_dim (47) +- BroadcastExchange (59) +- * Project (58) @@ -286,9 +286,9 @@ BroadcastExchange (62) :- * ColumnarToRow (51) : +- CometScan parquet spark_catalog.default.date_dim (50) +- BroadcastExchange (56) - +- * Project (55) - +- * Filter (54) - +- * ColumnarToRow (53) + +- * ColumnarToRow (55) + +- CometProject (54) + +- CometFilter (53) +- CometScan parquet spark_catalog.default.date_dim (52) @@ -299,12 +299,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(48) ColumnarToRow [codegen id : 3] +(48) CometFilter Input [2]: [d_date_sk#7, d_date#41] +Condition : isnotnull(d_date_sk#7) -(49) Filter [codegen id : 3] +(49) ColumnarToRow [codegen id : 3] Input [2]: [d_date_sk#7, d_date#41] -Condition : isnotnull(d_date_sk#7) (50) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date#42, d_week_seq#43] @@ -322,16 +322,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_date, [2000-06-30,2000-09-27,2000-11-17])] ReadSchema: struct -(53) ColumnarToRow [codegen id : 1] -Input [2]: [d_date#44, d_week_seq#45] - -(54) Filter [codegen id : 1] +(53) CometFilter Input [2]: [d_date#44, d_week_seq#45] Condition : d_date#44 IN (2000-06-30,2000-09-27,2000-11-17) -(55) Project [codegen id : 1] -Output [1]: [d_week_seq#45] +(54) CometProject Input [2]: [d_date#44, d_week_seq#45] +Arguments: [d_week_seq#45], [d_week_seq#45] + +(55) ColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#45] (56) BroadcastExchange Input [1]: [d_week_seq#45] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/simplified.txt index 52f1fe4b5..a8f1ba3f1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/simplified.txt @@ -13,18 +13,18 @@ TakeOrderedAndProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty BroadcastHashJoin [sr_returned_date_sk,d_date_sk] Project [sr_return_quantity,sr_returned_date_sk,i_item_id] BroadcastHashJoin [sr_item_sk,i_item_sk] - Filter [sr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [sr_item_sk] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (3) Project [d_date_sk] BroadcastHashJoin [d_date,d_date] - Filter [d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter BroadcastExchange #3 @@ -37,17 +37,17 @@ TakeOrderedAndProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty InputAdapter BroadcastExchange #4 WholeStageCodegen (1) - Project [d_week_seq] - Filter [d_date] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_date] CometScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) - Filter [i_item_sk,i_item_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk,i_item_id] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] InputAdapter ReusedExchange [d_date_sk] #2 @@ -63,9 +63,9 @@ TakeOrderedAndProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty BroadcastHashJoin [cr_returned_date_sk,d_date_sk] Project [cr_return_quantity,cr_returned_date_sk,i_item_id] BroadcastHashJoin [cr_item_sk,i_item_sk] - Filter [cr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cr_item_sk] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -84,9 +84,9 @@ TakeOrderedAndProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty BroadcastHashJoin [wr_returned_date_sk,d_date_sk] Project [wr_return_quantity,wr_returned_date_sk,i_item_id] BroadcastHashJoin [wr_item_sk,i_item_sk] - Filter [wr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [wr_item_sk] CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84/explain.txt index c7e5238f1..8dc935d1d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84/explain.txt @@ -11,30 +11,30 @@ TakeOrderedAndProject (37) : : : +- * BroadcastHashJoin Inner BuildRight (15) : : : :- * Project (10) : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.customer (1) : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) + : : : : +- * ColumnarToRow (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) : : : : +- CometScan parquet spark_catalog.default.customer_address (4) : : : +- BroadcastExchange (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) + : : : +- * ColumnarToRow (13) + : : : +- CometFilter (12) : : : +- CometScan parquet spark_catalog.default.customer_demographics (11) : : +- BroadcastExchange (20) - : : +- * Filter (19) - : : +- * ColumnarToRow (18) + : : +- * ColumnarToRow (19) + : : +- CometFilter (18) : : +- CometScan parquet spark_catalog.default.household_demographics (17) : +- BroadcastExchange (27) - : +- * Project (26) - : +- * Filter (25) - : +- * ColumnarToRow (24) + : +- * ColumnarToRow (26) + : +- CometProject (25) + : +- CometFilter (24) : +- CometScan parquet spark_catalog.default.income_band (23) - +- * Project (34) - +- * Filter (33) - +- * ColumnarToRow (32) + +- * ColumnarToRow (34) + +- CometProject (33) + +- CometFilter (32) +- CometScan parquet spark_catalog.default.store_returns (31) @@ -45,12 +45,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] +(2) CometFilter Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] +Condition : ((isnotnull(c_current_addr_sk#4) AND isnotnull(c_current_cdemo_sk#2)) AND isnotnull(c_current_hdemo_sk#3)) -(3) Filter [codegen id : 5] +(3) ColumnarToRow [codegen id : 5] Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] -Condition : ((isnotnull(c_current_addr_sk#4) AND isnotnull(c_current_cdemo_sk#2)) AND isnotnull(c_current_hdemo_sk#3)) (4) Scan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#7, ca_city#8] @@ -59,16 +59,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_city), EqualTo(ca_city,Edgewood), IsNotNull(ca_address_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [ca_address_sk#7, ca_city#8] - -(6) Filter [codegen id : 1] +(5) CometFilter Input [2]: [ca_address_sk#7, ca_city#8] Condition : ((isnotnull(ca_city#8) AND (ca_city#8 = Edgewood)) AND isnotnull(ca_address_sk#7)) -(7) Project [codegen id : 1] -Output [1]: [ca_address_sk#7] +(6) CometProject Input [2]: [ca_address_sk#7, ca_city#8] +Arguments: [ca_address_sk#7], [ca_address_sk#7] + +(7) ColumnarToRow [codegen id : 1] +Input [1]: [ca_address_sk#7] (8) BroadcastExchange Input [1]: [ca_address_sk#7] @@ -91,12 +91,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] +(12) CometFilter Input [1]: [cd_demo_sk#9] +Condition : isnotnull(cd_demo_sk#9) -(13) Filter [codegen id : 2] +(13) ColumnarToRow [codegen id : 2] Input [1]: [cd_demo_sk#9] -Condition : isnotnull(cd_demo_sk#9) (14) BroadcastExchange Input [1]: [cd_demo_sk#9] @@ -119,12 +119,12 @@ Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] +(18) CometFilter Input [2]: [hd_demo_sk#10, hd_income_band_sk#11] +Condition : (isnotnull(hd_demo_sk#10) AND isnotnull(hd_income_band_sk#11)) -(19) Filter [codegen id : 3] +(19) ColumnarToRow [codegen id : 3] Input [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Condition : (isnotnull(hd_demo_sk#10) AND isnotnull(hd_income_band_sk#11)) (20) BroadcastExchange Input [2]: [hd_demo_sk#10, hd_income_band_sk#11] @@ -147,16 +147,16 @@ Location [not included in comparison]/{warehouse_dir}/income_band] PushedFilters: [IsNotNull(ib_lower_bound), IsNotNull(ib_upper_bound), GreaterThanOrEqual(ib_lower_bound,38128), LessThanOrEqual(ib_upper_bound,88128), IsNotNull(ib_income_band_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 4] -Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] - -(25) Filter [codegen id : 4] +(24) CometFilter Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] Condition : ((((isnotnull(ib_lower_bound#13) AND isnotnull(ib_upper_bound#14)) AND (ib_lower_bound#13 >= 38128)) AND (ib_upper_bound#14 <= 88128)) AND isnotnull(ib_income_band_sk#12)) -(26) Project [codegen id : 4] -Output [1]: [ib_income_band_sk#12] +(25) CometProject Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] +Arguments: [ib_income_band_sk#12], [ib_income_band_sk#12] + +(26) ColumnarToRow [codegen id : 4] +Input [1]: [ib_income_band_sk#12] (27) BroadcastExchange Input [1]: [ib_income_band_sk#12] @@ -183,16 +183,16 @@ Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_cdemo_sk)] ReadSchema: struct -(32) ColumnarToRow -Input [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] - -(33) Filter +(32) CometFilter Input [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] Condition : isnotnull(sr_cdemo_sk#15) -(34) Project -Output [1]: [sr_cdemo_sk#15] +(33) CometProject Input [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] +Arguments: [sr_cdemo_sk#15], [sr_cdemo_sk#15] + +(34) ColumnarToRow +Input [1]: [sr_cdemo_sk#15] (35) BroadcastHashJoin [codegen id : 6] Left keys [1]: [cd_demo_sk#9] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84/simplified.txt index 157e471bb..be3451d29 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84/simplified.txt @@ -13,42 +13,42 @@ TakeOrderedAndProject [c_customer_id,customer_id,customername] BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] Project [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_first_name,c_last_name] BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk,c_current_hdemo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_current_addr_sk,c_current_cdemo_sk,c_current_hdemo_sk] CometScan parquet spark_catalog.default.customer [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] InputAdapter BroadcastExchange #2 WholeStageCodegen (1) - Project [ca_address_sk] - Filter [ca_city,ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_city,ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk] InputAdapter BroadcastExchange #4 WholeStageCodegen (3) - Filter [hd_demo_sk,hd_income_band_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [hd_demo_sk,hd_income_band_sk] CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] InputAdapter BroadcastExchange #5 WholeStageCodegen (4) - Project [ib_income_band_sk] - Filter [ib_lower_bound,ib_upper_bound,ib_income_band_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ib_income_band_sk] + CometFilter [ib_lower_bound,ib_upper_bound,ib_income_band_sk] CometScan parquet spark_catalog.default.income_band [ib_income_band_sk,ib_lower_bound,ib_upper_bound] - Project [sr_cdemo_sk] - Filter [sr_cdemo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [sr_cdemo_sk] + CometFilter [sr_cdemo_sk] CometScan parquet spark_catalog.default.store_returns [sr_cdemo_sk,sr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/explain.txt index b6a58f11d..ce1cc2262 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/explain.txt @@ -1,52 +1,51 @@ == Physical Plan == -TakeOrderedAndProject (48) -+- * HashAggregate (47) - +- Exchange (46) - +- * HashAggregate (45) - +- * Project (44) - +- * BroadcastHashJoin Inner BuildRight (43) - :- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (35) - : : +- * BroadcastHashJoin Inner BuildRight (34) - : : :- * Project (28) - : : : +- * BroadcastHashJoin Inner BuildRight (27) - : : : :- * Project (22) - : : : : +- * BroadcastHashJoin Inner BuildRight (21) - : : : : :- * Project (16) - : : : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : : : :- * Project (10) - : : : : : : +- * BroadcastHashJoin Inner BuildLeft (9) - : : : : : : :- BroadcastExchange (4) - : : : : : : : +- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : : : : : +- * Project (8) - : : : : : : +- * Filter (7) - : : : : : : +- * ColumnarToRow (6) - : : : : : : +- CometScan parquet spark_catalog.default.web_returns (5) - : : : : : +- BroadcastExchange (14) - : : : : : +- * Filter (13) - : : : : : +- * ColumnarToRow (12) - : : : : : +- CometScan parquet spark_catalog.default.web_page (11) - : : : : +- BroadcastExchange (20) - : : : : +- * Filter (19) - : : : : +- * ColumnarToRow (18) - : : : : +- CometScan parquet spark_catalog.default.customer_demographics (17) - : : : +- BroadcastExchange (26) - : : : +- * Filter (25) - : : : +- * ColumnarToRow (24) - : : : +- CometScan parquet spark_catalog.default.customer_demographics (23) - : : +- BroadcastExchange (33) - : : +- * Project (32) - : : +- * Filter (31) - : : +- * ColumnarToRow (30) - : : +- CometScan parquet spark_catalog.default.customer_address (29) - : +- ReusedExchange (36) - +- BroadcastExchange (42) - +- * Filter (41) - +- * ColumnarToRow (40) - +- CometScan parquet spark_catalog.default.reason (39) +TakeOrderedAndProject (47) ++- * HashAggregate (46) + +- Exchange (45) + +- * HashAggregate (44) + +- * Project (43) + +- * BroadcastHashJoin Inner BuildRight (42) + :- * Project (37) + : +- * BroadcastHashJoin Inner BuildRight (36) + : :- * Project (34) + : : +- * BroadcastHashJoin Inner BuildRight (33) + : : :- * Project (27) + : : : +- * BroadcastHashJoin Inner BuildRight (26) + : : : :- * Project (21) + : : : : +- * BroadcastHashJoin Inner BuildRight (20) + : : : : :- * Project (15) + : : : : : +- * BroadcastHashJoin Inner BuildRight (14) + : : : : : :- * ColumnarToRow (9) + : : : : : : +- CometProject (8) + : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : :- CometBroadcastExchange (3) + : : : : : : : +- CometFilter (2) + : : : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : : : : : +- CometProject (6) + : : : : : : +- CometFilter (5) + : : : : : : +- CometScan parquet spark_catalog.default.web_returns (4) + : : : : : +- BroadcastExchange (13) + : : : : : +- * ColumnarToRow (12) + : : : : : +- CometFilter (11) + : : : : : +- CometScan parquet spark_catalog.default.web_page (10) + : : : : +- BroadcastExchange (19) + : : : : +- * ColumnarToRow (18) + : : : : +- CometFilter (17) + : : : : +- CometScan parquet spark_catalog.default.customer_demographics (16) + : : : +- BroadcastExchange (25) + : : : +- * ColumnarToRow (24) + : : : +- CometFilter (23) + : : : +- CometScan parquet spark_catalog.default.customer_demographics (22) + : : +- BroadcastExchange (32) + : : +- * ColumnarToRow (31) + : : +- CometProject (30) + : : +- CometFilter (29) + : : +- CometScan parquet spark_catalog.default.customer_address (28) + : +- ReusedExchange (35) + +- BroadcastExchange (41) + +- * ColumnarToRow (40) + +- CometFilter (39) + +- CometScan parquet spark_catalog.default.reason (38) (1) Scan parquet spark_catalog.default.web_sales @@ -57,254 +56,250 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#7), dynamicpruningexpression(ws_sol PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_page_sk), Or(Or(And(GreaterThanOrEqual(ws_sales_price,100.00),LessThanOrEqual(ws_sales_price,150.00)),And(GreaterThanOrEqual(ws_sales_price,50.00),LessThanOrEqual(ws_sales_price,100.00))),And(GreaterThanOrEqual(ws_sales_price,150.00),LessThanOrEqual(ws_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ws_net_profit,100.00),LessThanOrEqual(ws_net_profit,200.00)),And(GreaterThanOrEqual(ws_net_profit,150.00),LessThanOrEqual(ws_net_profit,300.00))),And(GreaterThanOrEqual(ws_net_profit,50.00),LessThanOrEqual(ws_net_profit,250.00)))] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] Condition : ((((isnotnull(ws_item_sk#1) AND isnotnull(ws_order_number#3)) AND isnotnull(ws_web_page_sk#2)) AND ((((ws_sales_price#5 >= 100.00) AND (ws_sales_price#5 <= 150.00)) OR ((ws_sales_price#5 >= 50.00) AND (ws_sales_price#5 <= 100.00))) OR ((ws_sales_price#5 >= 150.00) AND (ws_sales_price#5 <= 200.00)))) AND ((((ws_net_profit#6 >= 100.00) AND (ws_net_profit#6 <= 200.00)) OR ((ws_net_profit#6 >= 150.00) AND (ws_net_profit#6 <= 300.00))) OR ((ws_net_profit#6 >= 50.00) AND (ws_net_profit#6 <= 250.00)))) -(4) BroadcastExchange +(3) CometBroadcastExchange Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[2, int, false] as bigint) & 4294967295))),false), [plan_id=1] +Arguments: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -(5) Scan parquet spark_catalog.default.web_returns +(4) Scan parquet spark_catalog.default.web_returns Output [9]: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16, wr_returned_date_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number), IsNotNull(wr_refunded_cdemo_sk), IsNotNull(wr_returning_cdemo_sk), IsNotNull(wr_refunded_addr_sk), IsNotNull(wr_reason_sk)] ReadSchema: struct -(6) ColumnarToRow -Input [9]: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16, wr_returned_date_sk#17] - -(7) Filter +(5) CometFilter Input [9]: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16, wr_returned_date_sk#17] Condition : (((((isnotnull(wr_item_sk#9) AND isnotnull(wr_order_number#14)) AND isnotnull(wr_refunded_cdemo_sk#10)) AND isnotnull(wr_returning_cdemo_sk#12)) AND isnotnull(wr_refunded_addr_sk#11)) AND isnotnull(wr_reason_sk#13)) -(8) Project -Output [8]: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16] +(6) CometProject Input [9]: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16, wr_returned_date_sk#17] +Arguments: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16], [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16] -(9) BroadcastHashJoin [codegen id : 8] -Left keys [2]: [ws_item_sk#1, ws_order_number#3] -Right keys [2]: [wr_item_sk#9, wr_order_number#14] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] +Right output [8]: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16] +Arguments: [ws_item_sk#1, ws_order_number#3], [wr_item_sk#9, wr_order_number#14], Inner -(10) Project [codegen id : 8] -Output [11]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] +(8) CometProject Input [15]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16] +Arguments: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] + +(9) ColumnarToRow [codegen id : 7] +Input [11]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(11) Scan parquet spark_catalog.default.web_page +(10) Scan parquet spark_catalog.default.web_page Output [1]: [wp_web_page_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] PushedFilters: [IsNotNull(wp_web_page_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] +(11) CometFilter Input [1]: [wp_web_page_sk#18] +Condition : isnotnull(wp_web_page_sk#18) -(13) Filter [codegen id : 2] +(12) ColumnarToRow [codegen id : 1] Input [1]: [wp_web_page_sk#18] -Condition : isnotnull(wp_web_page_sk#18) -(14) BroadcastExchange +(13) BroadcastExchange Input [1]: [wp_web_page_sk#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] -(15) BroadcastHashJoin [codegen id : 8] +(14) BroadcastHashJoin [codegen id : 7] Left keys [1]: [ws_web_page_sk#2] Right keys [1]: [wp_web_page_sk#18] Join type: Inner Join condition: None -(16) Project [codegen id : 8] +(15) Project [codegen id : 7] Output [10]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] Input [12]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, wp_web_page_sk#18] -(17) Scan parquet spark_catalog.default.customer_demographics +(16) Scan parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Advanced Degree )),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College ))),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,2 yr Degree )))] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] +(17) CometFilter Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Condition : (((isnotnull(cd_demo_sk#19) AND isnotnull(cd_marital_status#20)) AND isnotnull(cd_education_status#21)) AND ((((cd_marital_status#20 = M) AND (cd_education_status#21 = Advanced Degree )) OR ((cd_marital_status#20 = S) AND (cd_education_status#21 = College ))) OR ((cd_marital_status#20 = W) AND (cd_education_status#21 = 2 yr Degree )))) -(19) Filter [codegen id : 3] +(18) ColumnarToRow [codegen id : 2] Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] -Condition : (((isnotnull(cd_demo_sk#19) AND isnotnull(cd_marital_status#20)) AND isnotnull(cd_education_status#21)) AND ((((cd_marital_status#20 = M) AND (cd_education_status#21 = Advanced Degree )) OR ((cd_marital_status#20 = S) AND (cd_education_status#21 = College ))) OR ((cd_marital_status#20 = W) AND (cd_education_status#21 = 2 yr Degree )))) -(20) BroadcastExchange +(19) BroadcastExchange Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] -(21) BroadcastHashJoin [codegen id : 8] +(20) BroadcastHashJoin [codegen id : 7] Left keys [1]: [wr_refunded_cdemo_sk#10] Right keys [1]: [cd_demo_sk#19] Join type: Inner Join condition: ((((((cd_marital_status#20 = M) AND (cd_education_status#21 = Advanced Degree )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#20 = S) AND (cd_education_status#21 = College )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#20 = W) AND (cd_education_status#21 = 2 yr Degree )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00))) -(22) Project [codegen id : 8] +(21) Project [codegen id : 7] Output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#20, cd_education_status#21] Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] -(23) Scan parquet spark_catalog.default.customer_demographics +(22) Scan parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#22, cd_marital_status#23, cd_education_status#24] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status), IsNotNull(cd_education_status)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 4] +(23) CometFilter Input [3]: [cd_demo_sk#22, cd_marital_status#23, cd_education_status#24] +Condition : ((isnotnull(cd_demo_sk#22) AND isnotnull(cd_marital_status#23)) AND isnotnull(cd_education_status#24)) -(25) Filter [codegen id : 4] +(24) ColumnarToRow [codegen id : 3] Input [3]: [cd_demo_sk#22, cd_marital_status#23, cd_education_status#24] -Condition : ((isnotnull(cd_demo_sk#22) AND isnotnull(cd_marital_status#23)) AND isnotnull(cd_education_status#24)) -(26) BroadcastExchange +(25) BroadcastExchange Input [3]: [cd_demo_sk#22, cd_marital_status#23, cd_education_status#24] -Arguments: HashedRelationBroadcastMode(List(input[0, int, false], input[1, string, false], input[2, string, false]),false), [plan_id=4] +Arguments: HashedRelationBroadcastMode(List(input[0, int, false], input[1, string, false], input[2, string, false]),false), [plan_id=3] -(27) BroadcastHashJoin [codegen id : 8] +(26) BroadcastHashJoin [codegen id : 7] Left keys [3]: [wr_returning_cdemo_sk#12, cd_marital_status#20, cd_education_status#21] Right keys [3]: [cd_demo_sk#22, cd_marital_status#23, cd_education_status#24] Join type: Inner Join condition: None -(28) Project [codegen id : 8] +(27) Project [codegen id : 7] Output [7]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#20, cd_education_status#21, cd_demo_sk#22, cd_marital_status#23, cd_education_status#24] -(29) Scan parquet spark_catalog.default.customer_address +(28) Scan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#25, ca_state#26, ca_country#27] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [IN,NJ,OH]),In(ca_state, [CT,KY,WI])),In(ca_state, [AR,IA,LA]))] ReadSchema: struct -(30) ColumnarToRow [codegen id : 5] -Input [3]: [ca_address_sk#25, ca_state#26, ca_country#27] - -(31) Filter [codegen id : 5] +(29) CometFilter Input [3]: [ca_address_sk#25, ca_state#26, ca_country#27] Condition : (((isnotnull(ca_country#27) AND (ca_country#27 = United States)) AND isnotnull(ca_address_sk#25)) AND ((ca_state#26 IN (IN,OH,NJ) OR ca_state#26 IN (WI,CT,KY)) OR ca_state#26 IN (LA,IA,AR))) -(32) Project [codegen id : 5] -Output [2]: [ca_address_sk#25, ca_state#26] +(30) CometProject Input [3]: [ca_address_sk#25, ca_state#26, ca_country#27] +Arguments: [ca_address_sk#25, ca_state#26], [ca_address_sk#25, ca_state#26] -(33) BroadcastExchange +(31) ColumnarToRow [codegen id : 4] Input [2]: [ca_address_sk#25, ca_state#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] -(34) BroadcastHashJoin [codegen id : 8] +(32) BroadcastExchange +Input [2]: [ca_address_sk#25, ca_state#26] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] + +(33) BroadcastHashJoin [codegen id : 7] Left keys [1]: [wr_refunded_addr_sk#11] Right keys [1]: [ca_address_sk#25] Join type: Inner Join condition: ((((ca_state#26 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#26 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#26 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00))) -(35) Project [codegen id : 8] +(34) Project [codegen id : 7] Output [5]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, ca_address_sk#25, ca_state#26] -(36) ReusedExchange [Reuses operator id: 53] +(35) ReusedExchange [Reuses operator id: 52] Output [1]: [d_date_sk#28] -(37) BroadcastHashJoin [codegen id : 8] +(36) BroadcastHashJoin [codegen id : 7] Left keys [1]: [ws_sold_date_sk#7] Right keys [1]: [d_date_sk#28] Join type: Inner Join condition: None -(38) Project [codegen id : 8] +(37) Project [codegen id : 7] Output [4]: [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, d_date_sk#28] -(39) Scan parquet spark_catalog.default.reason +(38) Scan parquet spark_catalog.default.reason Output [2]: [r_reason_sk#29, r_reason_desc#30] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] PushedFilters: [IsNotNull(r_reason_sk)] ReadSchema: struct -(40) ColumnarToRow [codegen id : 7] +(39) CometFilter Input [2]: [r_reason_sk#29, r_reason_desc#30] +Condition : isnotnull(r_reason_sk#29) -(41) Filter [codegen id : 7] +(40) ColumnarToRow [codegen id : 6] Input [2]: [r_reason_sk#29, r_reason_desc#30] -Condition : isnotnull(r_reason_sk#29) -(42) BroadcastExchange +(41) BroadcastExchange Input [2]: [r_reason_sk#29, r_reason_desc#30] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] -(43) BroadcastHashJoin [codegen id : 8] +(42) BroadcastHashJoin [codegen id : 7] Left keys [1]: [wr_reason_sk#13] Right keys [1]: [r_reason_sk#29] Join type: Inner Join condition: None -(44) Project [codegen id : 8] +(43) Project [codegen id : 7] Output [4]: [ws_quantity#4, wr_fee#15, wr_refunded_cash#16, r_reason_desc#30] Input [6]: [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, r_reason_sk#29, r_reason_desc#30] -(45) HashAggregate [codegen id : 8] +(44) HashAggregate [codegen id : 7] Input [4]: [ws_quantity#4, wr_fee#15, wr_refunded_cash#16, r_reason_desc#30] Keys [1]: [r_reason_desc#30] Functions [3]: [partial_avg(ws_quantity#4), partial_avg(UnscaledValue(wr_refunded_cash#16)), partial_avg(UnscaledValue(wr_fee#15))] Aggregate Attributes [6]: [sum#31, count#32, sum#33, count#34, sum#35, count#36] Results [7]: [r_reason_desc#30, sum#37, count#38, sum#39, count#40, sum#41, count#42] -(46) Exchange +(45) Exchange Input [7]: [r_reason_desc#30, sum#37, count#38, sum#39, count#40, sum#41, count#42] -Arguments: hashpartitioning(r_reason_desc#30, 5), ENSURE_REQUIREMENTS, [plan_id=7] +Arguments: hashpartitioning(r_reason_desc#30, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(47) HashAggregate [codegen id : 9] +(46) HashAggregate [codegen id : 8] Input [7]: [r_reason_desc#30, sum#37, count#38, sum#39, count#40, sum#41, count#42] Keys [1]: [r_reason_desc#30] Functions [3]: [avg(ws_quantity#4), avg(UnscaledValue(wr_refunded_cash#16)), avg(UnscaledValue(wr_fee#15))] Aggregate Attributes [3]: [avg(ws_quantity#4)#43, avg(UnscaledValue(wr_refunded_cash#16))#44, avg(UnscaledValue(wr_fee#15))#45] Results [4]: [substr(r_reason_desc#30, 1, 20) AS substr(r_reason_desc, 1, 20)#46, avg(ws_quantity#4)#43 AS avg(ws_quantity)#47, cast((avg(UnscaledValue(wr_refunded_cash#16))#44 / 100.0) as decimal(11,6)) AS avg(wr_refunded_cash)#48, cast((avg(UnscaledValue(wr_fee#15))#45 / 100.0) as decimal(11,6)) AS avg(wr_fee)#49] -(48) TakeOrderedAndProject +(47) TakeOrderedAndProject Input [4]: [substr(r_reason_desc, 1, 20)#46, avg(ws_quantity)#47, avg(wr_refunded_cash)#48, avg(wr_fee)#49] Arguments: 100, [substr(r_reason_desc, 1, 20)#46 ASC NULLS FIRST, avg(ws_quantity)#47 ASC NULLS FIRST, avg(wr_refunded_cash)#48 ASC NULLS FIRST, avg(wr_fee)#49 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#46, avg(ws_quantity)#47, avg(wr_refunded_cash)#48, avg(wr_fee)#49] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (53) -+- * Project (52) - +- * Filter (51) - +- * ColumnarToRow (50) - +- CometScan parquet spark_catalog.default.date_dim (49) +BroadcastExchange (52) ++- * ColumnarToRow (51) + +- CometProject (50) + +- CometFilter (49) + +- CometScan parquet spark_catalog.default.date_dim (48) -(49) Scan parquet spark_catalog.default.date_dim +(48) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#28, d_year#50] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(50) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#28, d_year#50] - -(51) Filter [codegen id : 1] +(49) CometFilter Input [2]: [d_date_sk#28, d_year#50] Condition : ((isnotnull(d_year#50) AND (d_year#50 = 2000)) AND isnotnull(d_date_sk#28)) -(52) Project [codegen id : 1] -Output [1]: [d_date_sk#28] +(50) CometProject Input [2]: [d_date_sk#28, d_year#50] +Arguments: [d_date_sk#28], [d_date_sk#28] + +(51) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#28] -(53) BroadcastExchange +(52) BroadcastExchange Input [1]: [d_date_sk#28] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/simplified.txt index 99ca4aeaf..7c5ee727b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/simplified.txt @@ -1,9 +1,9 @@ TakeOrderedAndProject [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee)] - WholeStageCodegen (9) + WholeStageCodegen (8) HashAggregate [r_reason_desc,sum,count,sum,count,sum,count] [avg(ws_quantity),avg(UnscaledValue(wr_refunded_cash)),avg(UnscaledValue(wr_fee)),substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee),sum,count,sum,count,sum,count] InputAdapter Exchange [r_reason_desc] #1 - WholeStageCodegen (8) + WholeStageCodegen (7) HashAggregate [r_reason_desc,ws_quantity,wr_refunded_cash,wr_fee] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] Project [ws_quantity,wr_fee,wr_refunded_cash,r_reason_desc] BroadcastHashJoin [wr_reason_sk,r_reason_sk] @@ -17,63 +17,59 @@ TakeOrderedAndProject [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refu BroadcastHashJoin [wr_refunded_cdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status,ws_sales_price] Project [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] - Project [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - BroadcastHashJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ws_item_sk,ws_order_number,ws_web_page_sk,ws_sales_price,ws_net_profit] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - Project [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] - Filter [wr_item_sk,wr_order_number,wr_refunded_cdemo_sk,wr_returning_cdemo_sk,wr_refunded_addr_sk,wr_reason_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number] + CometBroadcastExchange #2 + CometFilter [ws_item_sk,ws_order_number,ws_web_page_sk,ws_sales_price,ws_net_profit] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_date_sk] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometProject [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] + CometFilter [wr_item_sk,wr_order_number,wr_refunded_cdemo_sk,wr_returning_cdemo_sk,wr_refunded_addr_sk,wr_reason_sk] CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] InputAdapter BroadcastExchange #4 - WholeStageCodegen (2) - Filter [wp_web_page_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [wp_web_page_sk] CometScan parquet spark_catalog.default.web_page [wp_web_page_sk] InputAdapter BroadcastExchange #5 - WholeStageCodegen (3) - Filter [cd_demo_sk,cd_marital_status,cd_education_status] - ColumnarToRow - InputAdapter + WholeStageCodegen (2) + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] InputAdapter BroadcastExchange #6 - WholeStageCodegen (4) - Filter [cd_demo_sk,cd_marital_status,cd_education_status] - ColumnarToRow - InputAdapter + WholeStageCodegen (3) + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] InputAdapter BroadcastExchange #7 - WholeStageCodegen (5) - Project [ca_address_sk,ca_state] - Filter [ca_country,ca_address_sk,ca_state] - ColumnarToRow - InputAdapter + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometProject [ca_address_sk,ca_state] + CometFilter [ca_country,ca_address_sk,ca_state] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] InputAdapter ReusedExchange [d_date_sk] #3 InputAdapter BroadcastExchange #8 - WholeStageCodegen (7) - Filter [r_reason_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (6) + ColumnarToRow + InputAdapter + CometFilter [r_reason_sk] CometScan parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/explain.txt index 07e05a0ae..5bfd2925d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/explain.txt @@ -12,13 +12,13 @@ TakeOrderedAndProject (21) +- * BroadcastHashJoin Inner BuildRight (11) :- * Project (6) : +- * BroadcastHashJoin Inner BuildRight (5) - : :- * Filter (3) - : : +- * ColumnarToRow (2) + : :- * ColumnarToRow (3) + : : +- CometFilter (2) : : +- CometScan parquet spark_catalog.default.web_sales (1) : +- ReusedExchange (4) +- BroadcastExchange (10) - +- * Filter (9) - +- * ColumnarToRow (8) + +- * ColumnarToRow (9) + +- CometFilter (8) +- CometScan parquet spark_catalog.default.item (7) @@ -30,12 +30,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sol PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] +Condition : isnotnull(ws_item_sk#1) -(3) Filter [codegen id : 3] +(3) ColumnarToRow [codegen id : 3] Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Condition : isnotnull(ws_item_sk#1) (4) ReusedExchange [Reuses operator id: 26] Output [1]: [d_date_sk#5] @@ -57,12 +57,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] +(8) CometFilter Input [3]: [i_item_sk#6, i_class#7, i_category#8] +Condition : isnotnull(i_item_sk#6) -(9) Filter [codegen id : 2] +(9) ColumnarToRow [codegen id : 2] Input [3]: [i_item_sk#6, i_class#7, i_category#8] -Condition : isnotnull(i_item_sk#6) (10) BroadcastExchange Input [3]: [i_item_sk#6, i_class#7, i_category#8] @@ -124,9 +124,9 @@ Arguments: 100, [lochierarchy#16 DESC NULLS LAST, CASE WHEN (lochierarchy#16 = 0 Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 BroadcastExchange (26) -+- * Project (25) - +- * Filter (24) - +- * ColumnarToRow (23) ++- * ColumnarToRow (25) + +- CometProject (24) + +- CometFilter (23) +- CometScan parquet spark_catalog.default.date_dim (22) @@ -137,16 +137,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_month_seq#21] - -(24) Filter [codegen id : 1] +(23) CometFilter Input [2]: [d_date_sk#5, d_month_seq#21] Condition : (((isnotnull(d_month_seq#21) AND (d_month_seq#21 >= 1200)) AND (d_month_seq#21 <= 1211)) AND isnotnull(d_date_sk#5)) -(25) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(24) CometProject Input [2]: [d_date_sk#5, d_month_seq#21] +Arguments: [d_date_sk#5], [d_date_sk#5] + +(25) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#5] (26) BroadcastExchange Input [1]: [d_date_sk#5] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/simplified.txt index a33617e7f..f9db2ce7a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/simplified.txt @@ -18,24 +18,24 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl BroadcastHashJoin [ws_item_sk,i_item_sk] Project [ws_item_sk,ws_net_paid] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_month_seq,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter ReusedExchange [d_date_sk] #3 InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/explain.txt index bce0869c5..a4c9f13ce 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/explain.txt @@ -12,13 +12,13 @@ : : +- * BroadcastHashJoin Inner BuildRight (11) : : :- * Project (6) : : : +- * BroadcastHashJoin Inner BuildRight (5) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- ReusedExchange (4) : : +- BroadcastExchange (10) - : : +- * Filter (9) - : : +- * ColumnarToRow (8) + : : +- * ColumnarToRow (9) + : : +- CometFilter (8) : : +- CometScan parquet spark_catalog.default.customer (7) : +- BroadcastExchange (28) : +- * HashAggregate (27) @@ -28,8 +28,8 @@ : +- * BroadcastHashJoin Inner BuildRight (23) : :- * Project (21) : : +- * BroadcastHashJoin Inner BuildRight (20) - : : :- * Filter (18) - : : : +- * ColumnarToRow (17) + : : :- * ColumnarToRow (18) + : : : +- CometFilter (17) : : : +- CometScan parquet spark_catalog.default.catalog_sales (16) : : +- ReusedExchange (19) : +- ReusedExchange (22) @@ -41,8 +41,8 @@ +- * BroadcastHashJoin Inner BuildRight (37) :- * Project (35) : +- * BroadcastHashJoin Inner BuildRight (34) - : :- * Filter (32) - : : +- * ColumnarToRow (31) + : :- * ColumnarToRow (32) + : : +- CometFilter (31) : : +- CometScan parquet spark_catalog.default.web_sales (30) : +- ReusedExchange (33) +- ReusedExchange (36) @@ -56,12 +56,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#2), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] +Condition : isnotnull(ss_customer_sk#1) -(3) Filter [codegen id : 3] +(3) ColumnarToRow [codegen id : 3] Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Condition : isnotnull(ss_customer_sk#1) (4) ReusedExchange [Reuses operator id: 52] Output [2]: [d_date_sk#4, d_date#5] @@ -83,12 +83,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] +(8) CometFilter Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Condition : isnotnull(c_customer_sk#6) -(9) Filter [codegen id : 2] +(9) ColumnarToRow [codegen id : 2] Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Condition : isnotnull(c_customer_sk#6) (10) BroadcastExchange Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] @@ -130,12 +130,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#10), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 6] +(17) CometFilter Input [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] +Condition : isnotnull(cs_bill_customer_sk#9) -(18) Filter [codegen id : 6] +(18) ColumnarToRow [codegen id : 6] Input [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] -Condition : isnotnull(cs_bill_customer_sk#9) (19) ReusedExchange [Reuses operator id: 52] Output [2]: [d_date_sk#12, d_date#13] @@ -199,12 +199,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#18), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 10] +(31) CometFilter Input [2]: [ws_bill_customer_sk#17, ws_sold_date_sk#18] +Condition : isnotnull(ws_bill_customer_sk#17) -(32) Filter [codegen id : 10] +(32) ColumnarToRow [codegen id : 10] Input [2]: [ws_bill_customer_sk#17, ws_sold_date_sk#18] -Condition : isnotnull(ws_bill_customer_sk#17) (33) ReusedExchange [Reuses operator id: 52] Output [2]: [d_date_sk#20, d_date#21] @@ -286,9 +286,9 @@ Results [1]: [count(1)#27 AS count(1)#28] Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#2 IN dynamicpruning#3 BroadcastExchange (52) -+- * Project (51) - +- * Filter (50) - +- * ColumnarToRow (49) ++- * ColumnarToRow (51) + +- CometProject (50) + +- CometFilter (49) +- CometScan parquet spark_catalog.default.date_dim (48) @@ -299,16 +299,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(49) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#29] - -(50) Filter [codegen id : 1] +(49) CometFilter Input [3]: [d_date_sk#4, d_date#5, d_month_seq#29] Condition : (((isnotnull(d_month_seq#29) AND (d_month_seq#29 >= 1200)) AND (d_month_seq#29 <= 1211)) AND isnotnull(d_date_sk#4)) -(51) Project [codegen id : 1] -Output [2]: [d_date_sk#4, d_date#5] +(50) CometProject Input [3]: [d_date_sk#4, d_date#5, d_month_seq#29] +Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] + +(51) ColumnarToRow [codegen id : 1] +Input [2]: [d_date_sk#4, d_date#5] (52) BroadcastExchange Input [2]: [d_date_sk#4, d_date#5] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/simplified.txt index 27489c0a9..315afe660 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/simplified.txt @@ -16,26 +16,26 @@ WholeStageCodegen (13) BroadcastHashJoin [ss_customer_sk,c_customer_sk] Project [ss_customer_sk,d_date] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_date] + CometFilter [d_month_seq,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] InputAdapter ReusedExchange [d_date_sk,d_date] #3 InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] InputAdapter BroadcastExchange #5 @@ -49,9 +49,9 @@ WholeStageCodegen (13) BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] Project [cs_bill_customer_sk,d_date] BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_bill_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_bill_customer_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -70,9 +70,9 @@ WholeStageCodegen (13) BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk] Project [ws_bill_customer_sk,d_date] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_bill_customer_sk] CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/explain.txt index ecd6c5498..a5f68e564 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/explain.txt @@ -15,24 +15,24 @@ : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (17) : : : : : : : : :- * Project (11) : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (10) -: : : : : : : : : :- * Project (4) -: : : : : : : : : : +- * Filter (3) -: : : : : : : : : : +- * ColumnarToRow (2) +: : : : : : : : : :- * ColumnarToRow (4) +: : : : : : : : : : +- CometProject (3) +: : : : : : : : : : +- CometFilter (2) : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : : : : : : : +- BroadcastExchange (9) -: : : : : : : : : +- * Project (8) -: : : : : : : : : +- * Filter (7) -: : : : : : : : : +- * ColumnarToRow (6) +: : : : : : : : : +- * ColumnarToRow (8) +: : : : : : : : : +- CometProject (7) +: : : : : : : : : +- CometFilter (6) : : : : : : : : : +- CometScan parquet spark_catalog.default.household_demographics (5) : : : : : : : : +- BroadcastExchange (16) -: : : : : : : : +- * Project (15) -: : : : : : : : +- * Filter (14) -: : : : : : : : +- * ColumnarToRow (13) +: : : : : : : : +- * ColumnarToRow (15) +: : : : : : : : +- CometProject (14) +: : : : : : : : +- CometFilter (13) : : : : : : : : +- CometScan parquet spark_catalog.default.time_dim (12) : : : : : : : +- BroadcastExchange (23) -: : : : : : : +- * Project (22) -: : : : : : : +- * Filter (21) -: : : : : : : +- * ColumnarToRow (20) +: : : : : : : +- * ColumnarToRow (22) +: : : : : : : +- CometProject (21) +: : : : : : : +- CometFilter (20) : : : : : : : +- CometScan parquet spark_catalog.default.store (19) : : : : : : +- BroadcastExchange (49) : : : : : : +- * HashAggregate (48) @@ -44,15 +44,15 @@ : : : : : : : +- * BroadcastHashJoin Inner BuildRight (41) : : : : : : : :- * Project (35) : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (34) -: : : : : : : : :- * Project (32) -: : : : : : : : : +- * Filter (31) -: : : : : : : : : +- * ColumnarToRow (30) +: : : : : : : : :- * ColumnarToRow (32) +: : : : : : : : : +- CometProject (31) +: : : : : : : : : +- CometFilter (30) : : : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (29) : : : : : : : : +- ReusedExchange (33) : : : : : : : +- BroadcastExchange (40) -: : : : : : : +- * Project (39) -: : : : : : : +- * Filter (38) -: : : : : : : +- * ColumnarToRow (37) +: : : : : : : +- * ColumnarToRow (39) +: : : : : : : +- CometProject (38) +: : : : : : : +- CometFilter (37) : : : : : : : +- CometScan parquet spark_catalog.default.time_dim (36) : : : : : : +- ReusedExchange (43) : : : : : +- BroadcastExchange (71) @@ -65,15 +65,15 @@ : : : : : : +- * BroadcastHashJoin Inner BuildRight (63) : : : : : : :- * Project (57) : : : : : : : +- * BroadcastHashJoin Inner BuildRight (56) -: : : : : : : :- * Project (54) -: : : : : : : : +- * Filter (53) -: : : : : : : : +- * ColumnarToRow (52) +: : : : : : : :- * ColumnarToRow (54) +: : : : : : : : +- CometProject (53) +: : : : : : : : +- CometFilter (52) : : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (51) : : : : : : : +- ReusedExchange (55) : : : : : : +- BroadcastExchange (62) -: : : : : : +- * Project (61) -: : : : : : +- * Filter (60) -: : : : : : +- * ColumnarToRow (59) +: : : : : : +- * ColumnarToRow (61) +: : : : : : +- CometProject (60) +: : : : : : +- CometFilter (59) : : : : : : +- CometScan parquet spark_catalog.default.time_dim (58) : : : : : +- ReusedExchange (65) : : : : +- BroadcastExchange (93) @@ -86,15 +86,15 @@ : : : : : +- * BroadcastHashJoin Inner BuildRight (85) : : : : : :- * Project (79) : : : : : : +- * BroadcastHashJoin Inner BuildRight (78) -: : : : : : :- * Project (76) -: : : : : : : +- * Filter (75) -: : : : : : : +- * ColumnarToRow (74) +: : : : : : :- * ColumnarToRow (76) +: : : : : : : +- CometProject (75) +: : : : : : : +- CometFilter (74) : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (73) : : : : : : +- ReusedExchange (77) : : : : : +- BroadcastExchange (84) -: : : : : +- * Project (83) -: : : : : +- * Filter (82) -: : : : : +- * ColumnarToRow (81) +: : : : : +- * ColumnarToRow (83) +: : : : : +- CometProject (82) +: : : : : +- CometFilter (81) : : : : : +- CometScan parquet spark_catalog.default.time_dim (80) : : : : +- ReusedExchange (87) : : : +- BroadcastExchange (115) @@ -107,15 +107,15 @@ : : : : +- * BroadcastHashJoin Inner BuildRight (107) : : : : :- * Project (101) : : : : : +- * BroadcastHashJoin Inner BuildRight (100) -: : : : : :- * Project (98) -: : : : : : +- * Filter (97) -: : : : : : +- * ColumnarToRow (96) +: : : : : :- * ColumnarToRow (98) +: : : : : : +- CometProject (97) +: : : : : : +- CometFilter (96) : : : : : : +- CometScan parquet spark_catalog.default.store_sales (95) : : : : : +- ReusedExchange (99) : : : : +- BroadcastExchange (106) -: : : : +- * Project (105) -: : : : +- * Filter (104) -: : : : +- * ColumnarToRow (103) +: : : : +- * ColumnarToRow (105) +: : : : +- CometProject (104) +: : : : +- CometFilter (103) : : : : +- CometScan parquet spark_catalog.default.time_dim (102) : : : +- ReusedExchange (109) : : +- BroadcastExchange (137) @@ -128,15 +128,15 @@ : : : +- * BroadcastHashJoin Inner BuildRight (129) : : : :- * Project (123) : : : : +- * BroadcastHashJoin Inner BuildRight (122) -: : : : :- * Project (120) -: : : : : +- * Filter (119) -: : : : : +- * ColumnarToRow (118) +: : : : :- * ColumnarToRow (120) +: : : : : +- CometProject (119) +: : : : : +- CometFilter (118) : : : : : +- CometScan parquet spark_catalog.default.store_sales (117) : : : : +- ReusedExchange (121) : : : +- BroadcastExchange (128) -: : : +- * Project (127) -: : : +- * Filter (126) -: : : +- * ColumnarToRow (125) +: : : +- * ColumnarToRow (127) +: : : +- CometProject (126) +: : : +- CometFilter (125) : : : +- CometScan parquet spark_catalog.default.time_dim (124) : : +- ReusedExchange (131) : +- BroadcastExchange (159) @@ -149,15 +149,15 @@ : : +- * BroadcastHashJoin Inner BuildRight (151) : : :- * Project (145) : : : +- * BroadcastHashJoin Inner BuildRight (144) -: : : :- * Project (142) -: : : : +- * Filter (141) -: : : : +- * ColumnarToRow (140) +: : : :- * ColumnarToRow (142) +: : : : +- CometProject (141) +: : : : +- CometFilter (140) : : : : +- CometScan parquet spark_catalog.default.store_sales (139) : : : +- ReusedExchange (143) : : +- BroadcastExchange (150) -: : +- * Project (149) -: : +- * Filter (148) -: : +- * ColumnarToRow (147) +: : +- * ColumnarToRow (149) +: : +- CometProject (148) +: : +- CometFilter (147) : : +- CometScan parquet spark_catalog.default.time_dim (146) : +- ReusedExchange (153) +- BroadcastExchange (181) @@ -170,15 +170,15 @@ : +- * BroadcastHashJoin Inner BuildRight (173) : :- * Project (167) : : +- * BroadcastHashJoin Inner BuildRight (166) - : : :- * Project (164) - : : : +- * Filter (163) - : : : +- * ColumnarToRow (162) + : : :- * ColumnarToRow (164) + : : : +- CometProject (163) + : : : +- CometFilter (162) : : : +- CometScan parquet spark_catalog.default.store_sales (161) : : +- ReusedExchange (165) : +- BroadcastExchange (172) - : +- * Project (171) - : +- * Filter (170) - : +- * ColumnarToRow (169) + : +- * ColumnarToRow (171) + : +- CometProject (170) + : +- CometFilter (169) : +- CometScan parquet spark_catalog.default.time_dim (168) +- ReusedExchange (175) @@ -190,16 +190,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isnotnull(ss_store_sk#3)) -(4) Project [codegen id : 4] -Output [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] +(3) CometProject Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] +Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] + +(4) ColumnarToRow [codegen id : 4] +Input [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] (5) Scan parquet spark_catalog.default.household_demographics Output [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] @@ -208,16 +208,16 @@ Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [Or(Or(And(EqualTo(hd_dep_count,4),LessThanOrEqual(hd_vehicle_count,6)),And(EqualTo(hd_dep_count,2),LessThanOrEqual(hd_vehicle_count,4))),And(EqualTo(hd_dep_count,0),LessThanOrEqual(hd_vehicle_count,2))), IsNotNull(hd_demo_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] - -(7) Filter [codegen id : 1] +(6) CometFilter Input [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] Condition : (((((hd_dep_count#6 = 4) AND (hd_vehicle_count#7 <= 6)) OR ((hd_dep_count#6 = 2) AND (hd_vehicle_count#7 <= 4))) OR ((hd_dep_count#6 = 0) AND (hd_vehicle_count#7 <= 2))) AND isnotnull(hd_demo_sk#5)) -(8) Project [codegen id : 1] -Output [1]: [hd_demo_sk#5] +(7) CometProject Input [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] +Arguments: [hd_demo_sk#5], [hd_demo_sk#5] + +(8) ColumnarToRow [codegen id : 1] +Input [1]: [hd_demo_sk#5] (9) BroadcastExchange Input [1]: [hd_demo_sk#5] @@ -240,16 +240,16 @@ Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,8), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(13) ColumnarToRow [codegen id : 2] -Input [3]: [t_time_sk#8, t_hour#9, t_minute#10] - -(14) Filter [codegen id : 2] +(13) CometFilter Input [3]: [t_time_sk#8, t_hour#9, t_minute#10] Condition : ((((isnotnull(t_hour#9) AND isnotnull(t_minute#10)) AND (t_hour#9 = 8)) AND (t_minute#10 >= 30)) AND isnotnull(t_time_sk#8)) -(15) Project [codegen id : 2] -Output [1]: [t_time_sk#8] +(14) CometProject Input [3]: [t_time_sk#8, t_hour#9, t_minute#10] +Arguments: [t_time_sk#8], [t_time_sk#8] + +(15) ColumnarToRow [codegen id : 2] +Input [1]: [t_time_sk#8] (16) BroadcastExchange Input [1]: [t_time_sk#8] @@ -272,16 +272,16 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_name), EqualTo(s_store_name,ese), IsNotNull(s_store_sk)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 3] -Input [2]: [s_store_sk#11, s_store_name#12] - -(21) Filter [codegen id : 3] +(20) CometFilter Input [2]: [s_store_sk#11, s_store_name#12] Condition : ((isnotnull(s_store_name#12) AND (s_store_name#12 = ese)) AND isnotnull(s_store_sk#11)) -(22) Project [codegen id : 3] -Output [1]: [s_store_sk#11] +(21) CometProject Input [2]: [s_store_sk#11, s_store_name#12] +Arguments: [s_store_sk#11], [s_store_sk#11] + +(22) ColumnarToRow [codegen id : 3] +Input [1]: [s_store_sk#11] (23) BroadcastExchange Input [1]: [s_store_sk#11] @@ -322,16 +322,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 8] -Input [4]: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19, ss_sold_date_sk#20] - -(31) Filter [codegen id : 8] +(30) CometFilter Input [4]: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19, ss_sold_date_sk#20] Condition : ((isnotnull(ss_hdemo_sk#18) AND isnotnull(ss_sold_time_sk#17)) AND isnotnull(ss_store_sk#19)) -(32) Project [codegen id : 8] -Output [3]: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19] +(31) CometProject Input [4]: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19, ss_sold_date_sk#20] +Arguments: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19], [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19] + +(32) ColumnarToRow [codegen id : 8] +Input [3]: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19] (33) ReusedExchange [Reuses operator id: 9] Output [1]: [hd_demo_sk#21] @@ -353,16 +353,16 @@ Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,9), LessThan(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 6] -Input [3]: [t_time_sk#22, t_hour#23, t_minute#24] - -(38) Filter [codegen id : 6] +(37) CometFilter Input [3]: [t_time_sk#22, t_hour#23, t_minute#24] Condition : ((((isnotnull(t_hour#23) AND isnotnull(t_minute#24)) AND (t_hour#23 = 9)) AND (t_minute#24 < 30)) AND isnotnull(t_time_sk#22)) -(39) Project [codegen id : 6] -Output [1]: [t_time_sk#22] +(38) CometProject Input [3]: [t_time_sk#22, t_hour#23, t_minute#24] +Arguments: [t_time_sk#22], [t_time_sk#22] + +(39) ColumnarToRow [codegen id : 6] +Input [1]: [t_time_sk#22] (40) BroadcastExchange Input [1]: [t_time_sk#22] @@ -424,16 +424,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(52) ColumnarToRow [codegen id : 13] -Input [4]: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32, ss_sold_date_sk#33] - -(53) Filter [codegen id : 13] +(52) CometFilter Input [4]: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32, ss_sold_date_sk#33] Condition : ((isnotnull(ss_hdemo_sk#31) AND isnotnull(ss_sold_time_sk#30)) AND isnotnull(ss_store_sk#32)) -(54) Project [codegen id : 13] -Output [3]: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32] +(53) CometProject Input [4]: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32, ss_sold_date_sk#33] +Arguments: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32], [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32] + +(54) ColumnarToRow [codegen id : 13] +Input [3]: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32] (55) ReusedExchange [Reuses operator id: 9] Output [1]: [hd_demo_sk#34] @@ -455,16 +455,16 @@ Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,9), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(59) ColumnarToRow [codegen id : 11] -Input [3]: [t_time_sk#35, t_hour#36, t_minute#37] - -(60) Filter [codegen id : 11] +(59) CometFilter Input [3]: [t_time_sk#35, t_hour#36, t_minute#37] Condition : ((((isnotnull(t_hour#36) AND isnotnull(t_minute#37)) AND (t_hour#36 = 9)) AND (t_minute#37 >= 30)) AND isnotnull(t_time_sk#35)) -(61) Project [codegen id : 11] -Output [1]: [t_time_sk#35] +(60) CometProject Input [3]: [t_time_sk#35, t_hour#36, t_minute#37] +Arguments: [t_time_sk#35], [t_time_sk#35] + +(61) ColumnarToRow [codegen id : 11] +Input [1]: [t_time_sk#35] (62) BroadcastExchange Input [1]: [t_time_sk#35] @@ -526,16 +526,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(74) ColumnarToRow [codegen id : 18] -Input [4]: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45, ss_sold_date_sk#46] - -(75) Filter [codegen id : 18] +(74) CometFilter Input [4]: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45, ss_sold_date_sk#46] Condition : ((isnotnull(ss_hdemo_sk#44) AND isnotnull(ss_sold_time_sk#43)) AND isnotnull(ss_store_sk#45)) -(76) Project [codegen id : 18] -Output [3]: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45] +(75) CometProject Input [4]: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45, ss_sold_date_sk#46] +Arguments: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45], [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45] + +(76) ColumnarToRow [codegen id : 18] +Input [3]: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45] (77) ReusedExchange [Reuses operator id: 9] Output [1]: [hd_demo_sk#47] @@ -557,16 +557,16 @@ Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,10), LessThan(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(81) ColumnarToRow [codegen id : 16] -Input [3]: [t_time_sk#48, t_hour#49, t_minute#50] - -(82) Filter [codegen id : 16] +(81) CometFilter Input [3]: [t_time_sk#48, t_hour#49, t_minute#50] Condition : ((((isnotnull(t_hour#49) AND isnotnull(t_minute#50)) AND (t_hour#49 = 10)) AND (t_minute#50 < 30)) AND isnotnull(t_time_sk#48)) -(83) Project [codegen id : 16] -Output [1]: [t_time_sk#48] +(82) CometProject Input [3]: [t_time_sk#48, t_hour#49, t_minute#50] +Arguments: [t_time_sk#48], [t_time_sk#48] + +(83) ColumnarToRow [codegen id : 16] +Input [1]: [t_time_sk#48] (84) BroadcastExchange Input [1]: [t_time_sk#48] @@ -628,16 +628,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(96) ColumnarToRow [codegen id : 23] -Input [4]: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58, ss_sold_date_sk#59] - -(97) Filter [codegen id : 23] +(96) CometFilter Input [4]: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58, ss_sold_date_sk#59] Condition : ((isnotnull(ss_hdemo_sk#57) AND isnotnull(ss_sold_time_sk#56)) AND isnotnull(ss_store_sk#58)) -(98) Project [codegen id : 23] -Output [3]: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58] +(97) CometProject Input [4]: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58, ss_sold_date_sk#59] +Arguments: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58], [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58] + +(98) ColumnarToRow [codegen id : 23] +Input [3]: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58] (99) ReusedExchange [Reuses operator id: 9] Output [1]: [hd_demo_sk#60] @@ -659,16 +659,16 @@ Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,10), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(103) ColumnarToRow [codegen id : 21] -Input [3]: [t_time_sk#61, t_hour#62, t_minute#63] - -(104) Filter [codegen id : 21] +(103) CometFilter Input [3]: [t_time_sk#61, t_hour#62, t_minute#63] Condition : ((((isnotnull(t_hour#62) AND isnotnull(t_minute#63)) AND (t_hour#62 = 10)) AND (t_minute#63 >= 30)) AND isnotnull(t_time_sk#61)) -(105) Project [codegen id : 21] -Output [1]: [t_time_sk#61] +(104) CometProject Input [3]: [t_time_sk#61, t_hour#62, t_minute#63] +Arguments: [t_time_sk#61], [t_time_sk#61] + +(105) ColumnarToRow [codegen id : 21] +Input [1]: [t_time_sk#61] (106) BroadcastExchange Input [1]: [t_time_sk#61] @@ -730,16 +730,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(118) ColumnarToRow [codegen id : 28] -Input [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, ss_sold_date_sk#72] - -(119) Filter [codegen id : 28] +(118) CometFilter Input [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, ss_sold_date_sk#72] Condition : ((isnotnull(ss_hdemo_sk#70) AND isnotnull(ss_sold_time_sk#69)) AND isnotnull(ss_store_sk#71)) -(120) Project [codegen id : 28] -Output [3]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71] +(119) CometProject Input [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, ss_sold_date_sk#72] +Arguments: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71], [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71] + +(120) ColumnarToRow [codegen id : 28] +Input [3]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71] (121) ReusedExchange [Reuses operator id: 9] Output [1]: [hd_demo_sk#73] @@ -761,16 +761,16 @@ Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,11), LessThan(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(125) ColumnarToRow [codegen id : 26] -Input [3]: [t_time_sk#74, t_hour#75, t_minute#76] - -(126) Filter [codegen id : 26] +(125) CometFilter Input [3]: [t_time_sk#74, t_hour#75, t_minute#76] Condition : ((((isnotnull(t_hour#75) AND isnotnull(t_minute#76)) AND (t_hour#75 = 11)) AND (t_minute#76 < 30)) AND isnotnull(t_time_sk#74)) -(127) Project [codegen id : 26] -Output [1]: [t_time_sk#74] +(126) CometProject Input [3]: [t_time_sk#74, t_hour#75, t_minute#76] +Arguments: [t_time_sk#74], [t_time_sk#74] + +(127) ColumnarToRow [codegen id : 26] +Input [1]: [t_time_sk#74] (128) BroadcastExchange Input [1]: [t_time_sk#74] @@ -832,16 +832,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(140) ColumnarToRow [codegen id : 33] -Input [4]: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84, ss_sold_date_sk#85] - -(141) Filter [codegen id : 33] +(140) CometFilter Input [4]: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84, ss_sold_date_sk#85] Condition : ((isnotnull(ss_hdemo_sk#83) AND isnotnull(ss_sold_time_sk#82)) AND isnotnull(ss_store_sk#84)) -(142) Project [codegen id : 33] -Output [3]: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84] +(141) CometProject Input [4]: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84, ss_sold_date_sk#85] +Arguments: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84], [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84] + +(142) ColumnarToRow [codegen id : 33] +Input [3]: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84] (143) ReusedExchange [Reuses operator id: 9] Output [1]: [hd_demo_sk#86] @@ -863,16 +863,16 @@ Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,11), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(147) ColumnarToRow [codegen id : 31] -Input [3]: [t_time_sk#87, t_hour#88, t_minute#89] - -(148) Filter [codegen id : 31] +(147) CometFilter Input [3]: [t_time_sk#87, t_hour#88, t_minute#89] Condition : ((((isnotnull(t_hour#88) AND isnotnull(t_minute#89)) AND (t_hour#88 = 11)) AND (t_minute#89 >= 30)) AND isnotnull(t_time_sk#87)) -(149) Project [codegen id : 31] -Output [1]: [t_time_sk#87] +(148) CometProject Input [3]: [t_time_sk#87, t_hour#88, t_minute#89] +Arguments: [t_time_sk#87], [t_time_sk#87] + +(149) ColumnarToRow [codegen id : 31] +Input [1]: [t_time_sk#87] (150) BroadcastExchange Input [1]: [t_time_sk#87] @@ -934,16 +934,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(162) ColumnarToRow [codegen id : 38] -Input [4]: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97, ss_sold_date_sk#98] - -(163) Filter [codegen id : 38] +(162) CometFilter Input [4]: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97, ss_sold_date_sk#98] Condition : ((isnotnull(ss_hdemo_sk#96) AND isnotnull(ss_sold_time_sk#95)) AND isnotnull(ss_store_sk#97)) -(164) Project [codegen id : 38] -Output [3]: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97] +(163) CometProject Input [4]: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97, ss_sold_date_sk#98] +Arguments: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97], [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97] + +(164) ColumnarToRow [codegen id : 38] +Input [3]: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97] (165) ReusedExchange [Reuses operator id: 9] Output [1]: [hd_demo_sk#99] @@ -965,16 +965,16 @@ Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,12), LessThan(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(169) ColumnarToRow [codegen id : 36] -Input [3]: [t_time_sk#100, t_hour#101, t_minute#102] - -(170) Filter [codegen id : 36] +(169) CometFilter Input [3]: [t_time_sk#100, t_hour#101, t_minute#102] Condition : ((((isnotnull(t_hour#101) AND isnotnull(t_minute#102)) AND (t_hour#101 = 12)) AND (t_minute#102 < 30)) AND isnotnull(t_time_sk#100)) -(171) Project [codegen id : 36] -Output [1]: [t_time_sk#100] +(170) CometProject Input [3]: [t_time_sk#100, t_hour#101, t_minute#102] +Arguments: [t_time_sk#100], [t_time_sk#100] + +(171) ColumnarToRow [codegen id : 36] +Input [1]: [t_time_sk#100] (172) BroadcastExchange Input [1]: [t_time_sk#100] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/simplified.txt index 3087d3432..b497e0bab 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/simplified.txt @@ -17,34 +17,34 @@ WholeStageCodegen (40) BroadcastHashJoin [ss_sold_time_sk,t_time_sk] Project [ss_sold_time_sk,ss_store_sk] BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] InputAdapter BroadcastExchange #2 WholeStageCodegen (1) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [hd_demo_sk] + CometFilter [hd_dep_count,hd_vehicle_count,hd_demo_sk] CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [t_time_sk] + CometFilter [t_hour,t_minute,t_time_sk] CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] InputAdapter BroadcastExchange #4 WholeStageCodegen (3) - Project [s_store_sk] - Filter [s_store_name,s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [s_store_sk] + CometFilter [s_store_name,s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name] InputAdapter BroadcastExchange #5 @@ -60,20 +60,20 @@ WholeStageCodegen (40) BroadcastHashJoin [ss_sold_time_sk,t_time_sk] Project [ss_sold_time_sk,ss_store_sk] BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] InputAdapter ReusedExchange [hd_demo_sk] #2 InputAdapter BroadcastExchange #7 WholeStageCodegen (6) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [t_time_sk] + CometFilter [t_hour,t_minute,t_time_sk] CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] InputAdapter ReusedExchange [s_store_sk] #4 @@ -91,20 +91,20 @@ WholeStageCodegen (40) BroadcastHashJoin [ss_sold_time_sk,t_time_sk] Project [ss_sold_time_sk,ss_store_sk] BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] InputAdapter ReusedExchange [hd_demo_sk] #2 InputAdapter BroadcastExchange #10 WholeStageCodegen (11) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [t_time_sk] + CometFilter [t_hour,t_minute,t_time_sk] CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] InputAdapter ReusedExchange [s_store_sk] #4 @@ -122,20 +122,20 @@ WholeStageCodegen (40) BroadcastHashJoin [ss_sold_time_sk,t_time_sk] Project [ss_sold_time_sk,ss_store_sk] BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] InputAdapter ReusedExchange [hd_demo_sk] #2 InputAdapter BroadcastExchange #13 WholeStageCodegen (16) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [t_time_sk] + CometFilter [t_hour,t_minute,t_time_sk] CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] InputAdapter ReusedExchange [s_store_sk] #4 @@ -153,20 +153,20 @@ WholeStageCodegen (40) BroadcastHashJoin [ss_sold_time_sk,t_time_sk] Project [ss_sold_time_sk,ss_store_sk] BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] InputAdapter ReusedExchange [hd_demo_sk] #2 InputAdapter BroadcastExchange #16 WholeStageCodegen (21) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [t_time_sk] + CometFilter [t_hour,t_minute,t_time_sk] CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] InputAdapter ReusedExchange [s_store_sk] #4 @@ -184,20 +184,20 @@ WholeStageCodegen (40) BroadcastHashJoin [ss_sold_time_sk,t_time_sk] Project [ss_sold_time_sk,ss_store_sk] BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] InputAdapter ReusedExchange [hd_demo_sk] #2 InputAdapter BroadcastExchange #19 WholeStageCodegen (26) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [t_time_sk] + CometFilter [t_hour,t_minute,t_time_sk] CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] InputAdapter ReusedExchange [s_store_sk] #4 @@ -215,20 +215,20 @@ WholeStageCodegen (40) BroadcastHashJoin [ss_sold_time_sk,t_time_sk] Project [ss_sold_time_sk,ss_store_sk] BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] InputAdapter ReusedExchange [hd_demo_sk] #2 InputAdapter BroadcastExchange #22 WholeStageCodegen (31) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [t_time_sk] + CometFilter [t_hour,t_minute,t_time_sk] CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] InputAdapter ReusedExchange [s_store_sk] #4 @@ -246,20 +246,20 @@ WholeStageCodegen (40) BroadcastHashJoin [ss_sold_time_sk,t_time_sk] Project [ss_sold_time_sk,ss_store_sk] BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] InputAdapter ReusedExchange [hd_demo_sk] #2 InputAdapter BroadcastExchange #25 WholeStageCodegen (36) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [t_time_sk] + CometFilter [t_hour,t_minute,t_time_sk] CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] InputAdapter ReusedExchange [s_store_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/explain.txt index bfa41701d..3657266e2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/explain.txt @@ -14,17 +14,17 @@ TakeOrderedAndProject (27) : +- * BroadcastHashJoin Inner BuildRight (11) : :- * Project (9) : : +- * BroadcastHashJoin Inner BuildRight (8) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) + : : :- * ColumnarToRow (3) + : : : +- CometFilter (2) : : : +- CometScan parquet spark_catalog.default.item (1) : : +- BroadcastExchange (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) + : : +- * ColumnarToRow (6) + : : +- CometFilter (5) : : +- CometScan parquet spark_catalog.default.store_sales (4) : +- ReusedExchange (10) +- BroadcastExchange (16) - +- * Filter (15) - +- * ColumnarToRow (14) + +- * ColumnarToRow (15) + +- CometFilter (14) +- CometScan parquet spark_catalog.default.store (13) @@ -35,12 +35,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [Or(And(In(i_category, [Books ,Electronics ,Sports ]),In(i_class, [computers ,football ,stereo ])),And(In(i_category, [Jewelry ,Men ,Women ]),In(i_class, [birdal ,dresses ,shirts ]))), IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] +Condition : (((i_category#4 IN (Books ,Electronics ,Sports ) AND i_class#3 IN (computers ,stereo ,football )) OR (i_category#4 IN (Men ,Jewelry ,Women ) AND i_class#3 IN (shirts ,birdal ,dresses ))) AND isnotnull(i_item_sk#1)) -(3) Filter [codegen id : 4] +(3) ColumnarToRow [codegen id : 4] Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] -Condition : (((i_category#4 IN (Books ,Electronics ,Sports ) AND i_class#3 IN (computers ,stereo ,football )) OR (i_category#4 IN (Men ,Jewelry ,Women ) AND i_class#3 IN (shirts ,birdal ,dresses ))) AND isnotnull(i_item_sk#1)) (4) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] @@ -50,12 +50,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] +Condition : (isnotnull(ss_item_sk#5) AND isnotnull(ss_store_sk#6)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Condition : (isnotnull(ss_item_sk#5) AND isnotnull(ss_store_sk#6)) (7) BroadcastExchange Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] @@ -91,12 +91,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] +(14) CometFilter Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] +Condition : isnotnull(s_store_sk#12) -(15) Filter [codegen id : 3] +(15) ColumnarToRow [codegen id : 3] Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Condition : isnotnull(s_store_sk#12) (16) BroadcastExchange Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] @@ -158,9 +158,9 @@ Arguments: 100, [(sum_sales#18 - avg_monthly_sales#20) ASC NULLS FIRST, s_store_ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 BroadcastExchange (32) -+- * Project (31) - +- * Filter (30) - +- * ColumnarToRow (29) ++- * ColumnarToRow (31) + +- CometProject (30) + +- CometFilter (29) +- CometScan parquet spark_catalog.default.date_dim (28) @@ -171,16 +171,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(29) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#10, d_year#21, d_moy#11] - -(30) Filter [codegen id : 1] +(29) CometFilter Input [3]: [d_date_sk#10, d_year#21, d_moy#11] Condition : ((isnotnull(d_year#21) AND (d_year#21 = 1999)) AND isnotnull(d_date_sk#10)) -(31) Project [codegen id : 1] -Output [2]: [d_date_sk#10, d_moy#11] +(30) CometProject Input [3]: [d_date_sk#10, d_year#21, d_moy#11] +Arguments: [d_date_sk#10, d_moy#11], [d_date_sk#10, d_moy#11] + +(31) ColumnarToRow [codegen id : 1] +Input [2]: [d_date_sk#10, d_moy#11] (32) BroadcastExchange Input [2]: [d_date_sk#10, d_moy#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/simplified.txt index d7daa4af9..bb9e4e17e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/simplified.txt @@ -20,31 +20,31 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_cla BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_category,i_class,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_category,i_class,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk,ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) - Project [d_date_sk,d_moy] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_moy] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter ReusedExchange [d_date_sk,d_moy] #4 InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - Filter [s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/explain.txt index 1b7aad28b..69c23211b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == * Project (4) -+- * Filter (3) - +- * ColumnarToRow (2) ++- * ColumnarToRow (3) + +- CometFilter (2) +- CometScan parquet spark_catalog.default.reason (1) @@ -12,12 +12,12 @@ Location [not included in comparison]/{warehouse_dir}/reason] PushedFilters: [IsNotNull(r_reason_sk), EqualTo(r_reason_sk,1)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] +(2) CometFilter Input [1]: [r_reason_sk#1] +Condition : (isnotnull(r_reason_sk#1) AND (r_reason_sk#1 = 1)) -(3) Filter [codegen id : 1] +(3) ColumnarToRow [codegen id : 1] Input [1]: [r_reason_sk#1] -Condition : (isnotnull(r_reason_sk#1) AND (r_reason_sk#1 = 1)) (4) Project [codegen id : 1] Output [5]: [CASE WHEN (Subquery scalar-subquery#2, [id=#3].count(1) > 62316685) THEN ReusedSubquery Subquery scalar-subquery#2, [id=#3].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#2, [id=#3].avg(ss_net_paid) END AS bucket1#4, CASE WHEN (Subquery scalar-subquery#5, [id=#6].count(1) > 19045798) THEN ReusedSubquery Subquery scalar-subquery#5, [id=#6].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#5, [id=#6].avg(ss_net_paid) END AS bucket2#7, CASE WHEN (Subquery scalar-subquery#8, [id=#9].count(1) > 365541424) THEN ReusedSubquery Subquery scalar-subquery#8, [id=#9].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#8, [id=#9].avg(ss_net_paid) END AS bucket3#10, CASE WHEN (Subquery scalar-subquery#11, [id=#12].count(1) > 216357808) THEN ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_net_paid) END AS bucket4#13, CASE WHEN (Subquery scalar-subquery#14, [id=#15].count(1) > 184483884) THEN ReusedSubquery Subquery scalar-subquery#14, [id=#15].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#14, [id=#15].avg(ss_net_paid) END AS bucket5#16] @@ -30,9 +30,9 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery +- * HashAggregate (11) +- Exchange (10) +- * HashAggregate (9) - +- * Project (8) - +- * Filter (7) - +- * ColumnarToRow (6) + +- * ColumnarToRow (8) + +- CometProject (7) + +- CometFilter (6) +- CometScan parquet spark_catalog.default.store_sales (5) @@ -43,16 +43,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,1), LessThanOrEqual(ss_quantity,20)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] - -(7) Filter [codegen id : 1] +(6) CometFilter Input [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] Condition : ((isnotnull(ss_quantity#17) AND (ss_quantity#17 >= 1)) AND (ss_quantity#17 <= 20)) -(8) Project [codegen id : 1] -Output [2]: [ss_ext_discount_amt#18, ss_net_paid#19] +(7) CometProject Input [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] +Arguments: [ss_ext_discount_amt#18, ss_net_paid#19], [ss_ext_discount_amt#18, ss_net_paid#19] + +(8) ColumnarToRow [codegen id : 1] +Input [2]: [ss_ext_discount_amt#18, ss_net_paid#19] (9) HashAggregate [codegen id : 1] Input [2]: [ss_ext_discount_amt#18, ss_net_paid#19] @@ -85,9 +85,9 @@ Subquery:4 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery +- * HashAggregate (19) +- Exchange (18) +- * HashAggregate (17) - +- * Project (16) - +- * Filter (15) - +- * ColumnarToRow (14) + +- * ColumnarToRow (16) + +- CometProject (15) + +- CometFilter (14) +- CometScan parquet spark_catalog.default.store_sales (13) @@ -98,16 +98,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,40)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 1] -Input [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] - -(15) Filter [codegen id : 1] +(14) CometFilter Input [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] Condition : ((isnotnull(ss_quantity#38) AND (ss_quantity#38 >= 21)) AND (ss_quantity#38 <= 40)) -(16) Project [codegen id : 1] -Output [2]: [ss_ext_discount_amt#39, ss_net_paid#40] +(15) CometProject Input [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] +Arguments: [ss_ext_discount_amt#39, ss_net_paid#40], [ss_ext_discount_amt#39, ss_net_paid#40] + +(16) ColumnarToRow [codegen id : 1] +Input [2]: [ss_ext_discount_amt#39, ss_net_paid#40] (17) HashAggregate [codegen id : 1] Input [2]: [ss_ext_discount_amt#39, ss_net_paid#40] @@ -140,9 +140,9 @@ Subquery:7 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery +- * HashAggregate (27) +- Exchange (26) +- * HashAggregate (25) - +- * Project (24) - +- * Filter (23) - +- * ColumnarToRow (22) + +- * ColumnarToRow (24) + +- CometProject (23) + +- CometFilter (22) +- CometScan parquet spark_catalog.default.store_sales (21) @@ -153,16 +153,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,41), LessThanOrEqual(ss_quantity,60)] ReadSchema: struct -(22) ColumnarToRow [codegen id : 1] -Input [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] - -(23) Filter [codegen id : 1] +(22) CometFilter Input [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] Condition : ((isnotnull(ss_quantity#59) AND (ss_quantity#59 >= 41)) AND (ss_quantity#59 <= 60)) -(24) Project [codegen id : 1] -Output [2]: [ss_ext_discount_amt#60, ss_net_paid#61] +(23) CometProject Input [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] +Arguments: [ss_ext_discount_amt#60, ss_net_paid#61], [ss_ext_discount_amt#60, ss_net_paid#61] + +(24) ColumnarToRow [codegen id : 1] +Input [2]: [ss_ext_discount_amt#60, ss_net_paid#61] (25) HashAggregate [codegen id : 1] Input [2]: [ss_ext_discount_amt#60, ss_net_paid#61] @@ -195,9 +195,9 @@ Subquery:10 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquer +- * HashAggregate (35) +- Exchange (34) +- * HashAggregate (33) - +- * Project (32) - +- * Filter (31) - +- * ColumnarToRow (30) + +- * ColumnarToRow (32) + +- CometProject (31) + +- CometFilter (30) +- CometScan parquet spark_catalog.default.store_sales (29) @@ -208,16 +208,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,61), LessThanOrEqual(ss_quantity,80)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 1] -Input [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] - -(31) Filter [codegen id : 1] +(30) CometFilter Input [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] Condition : ((isnotnull(ss_quantity#80) AND (ss_quantity#80 >= 61)) AND (ss_quantity#80 <= 80)) -(32) Project [codegen id : 1] -Output [2]: [ss_ext_discount_amt#81, ss_net_paid#82] +(31) CometProject Input [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] +Arguments: [ss_ext_discount_amt#81, ss_net_paid#82], [ss_ext_discount_amt#81, ss_net_paid#82] + +(32) ColumnarToRow [codegen id : 1] +Input [2]: [ss_ext_discount_amt#81, ss_net_paid#82] (33) HashAggregate [codegen id : 1] Input [2]: [ss_ext_discount_amt#81, ss_net_paid#82] @@ -250,9 +250,9 @@ Subquery:13 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquer +- * HashAggregate (43) +- Exchange (42) +- * HashAggregate (41) - +- * Project (40) - +- * Filter (39) - +- * ColumnarToRow (38) + +- * ColumnarToRow (40) + +- CometProject (39) + +- CometFilter (38) +- CometScan parquet spark_catalog.default.store_sales (37) @@ -263,16 +263,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,81), LessThanOrEqual(ss_quantity,100)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 1] -Input [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] - -(39) Filter [codegen id : 1] +(38) CometFilter Input [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] Condition : ((isnotnull(ss_quantity#101) AND (ss_quantity#101 >= 81)) AND (ss_quantity#101 <= 100)) -(40) Project [codegen id : 1] -Output [2]: [ss_ext_discount_amt#102, ss_net_paid#103] +(39) CometProject Input [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] +Arguments: [ss_ext_discount_amt#102, ss_net_paid#103], [ss_ext_discount_amt#102, ss_net_paid#103] + +(40) ColumnarToRow [codegen id : 1] +Input [2]: [ss_ext_discount_amt#102, ss_net_paid#103] (41) HashAggregate [codegen id : 1] Input [2]: [ss_ext_discount_amt#102, ss_net_paid#103] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/simplified.txt index 9e0a4a1ab..fdd3bd293 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/simplified.txt @@ -8,10 +8,10 @@ WholeStageCodegen (1) Exchange #1 WholeStageCodegen (1) HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - Project [ss_ext_discount_amt,ss_net_paid] - Filter [ss_quantity] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity] CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #1 ReusedSubquery [mergedValue] #1 @@ -23,10 +23,10 @@ WholeStageCodegen (1) Exchange #2 WholeStageCodegen (1) HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - Project [ss_ext_discount_amt,ss_net_paid] - Filter [ss_quantity] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity] CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #2 ReusedSubquery [mergedValue] #2 @@ -38,10 +38,10 @@ WholeStageCodegen (1) Exchange #3 WholeStageCodegen (1) HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - Project [ss_ext_discount_amt,ss_net_paid] - Filter [ss_quantity] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity] CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #3 ReusedSubquery [mergedValue] #3 @@ -53,10 +53,10 @@ WholeStageCodegen (1) Exchange #4 WholeStageCodegen (1) HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - Project [ss_ext_discount_amt,ss_net_paid] - Filter [ss_quantity] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity] CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #4 ReusedSubquery [mergedValue] #4 @@ -68,14 +68,14 @@ WholeStageCodegen (1) Exchange #5 WholeStageCodegen (1) HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - Project [ss_ext_discount_amt,ss_net_paid] - Filter [ss_quantity] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity] CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #5 ReusedSubquery [mergedValue] #5 - Filter [r_reason_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [r_reason_sk] CometScan parquet spark_catalog.default.reason [r_reason_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/explain.txt index c6d4a3c0b..fcfbca847 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/explain.txt @@ -10,24 +10,24 @@ : : +- * BroadcastHashJoin Inner BuildRight (17) : : :- * Project (11) : : : +- * BroadcastHashJoin Inner BuildRight (10) - : : : :- * Project (4) - : : : : +- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (4) + : : : : +- CometProject (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.web_sales (1) : : : +- BroadcastExchange (9) - : : : +- * Project (8) - : : : +- * Filter (7) - : : : +- * ColumnarToRow (6) + : : : +- * ColumnarToRow (8) + : : : +- CometProject (7) + : : : +- CometFilter (6) : : : +- CometScan parquet spark_catalog.default.household_demographics (5) : : +- BroadcastExchange (16) - : : +- * Project (15) - : : +- * Filter (14) - : : +- * ColumnarToRow (13) + : : +- * ColumnarToRow (15) + : : +- CometProject (14) + : : +- CometFilter (13) : : +- CometScan parquet spark_catalog.default.time_dim (12) : +- BroadcastExchange (23) - : +- * Project (22) - : +- * Filter (21) - : +- * ColumnarToRow (20) + : +- * ColumnarToRow (22) + : +- CometProject (21) + : +- CometFilter (20) : +- CometScan parquet spark_catalog.default.web_page (19) +- BroadcastExchange (49) +- * HashAggregate (48) @@ -39,15 +39,15 @@ : +- * BroadcastHashJoin Inner BuildRight (41) : :- * Project (35) : : +- * BroadcastHashJoin Inner BuildRight (34) - : : :- * Project (32) - : : : +- * Filter (31) - : : : +- * ColumnarToRow (30) + : : :- * ColumnarToRow (32) + : : : +- CometProject (31) + : : : +- CometFilter (30) : : : +- CometScan parquet spark_catalog.default.web_sales (29) : : +- ReusedExchange (33) : +- BroadcastExchange (40) - : +- * Project (39) - : +- * Filter (38) - : +- * ColumnarToRow (37) + : +- * ColumnarToRow (39) + : +- CometProject (38) + : +- CometFilter (37) : +- CometScan parquet spark_catalog.default.time_dim (36) +- ReusedExchange (43) @@ -59,16 +59,16 @@ Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_ship_hdemo_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_web_page_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] Condition : ((isnotnull(ws_ship_hdemo_sk#2) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_web_page_sk#3)) -(4) Project [codegen id : 4] -Output [3]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] +(3) CometProject Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] +Arguments: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] + +(4) ColumnarToRow [codegen id : 4] +Input [3]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] (5) Scan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#5, hd_dep_count#6] @@ -77,16 +77,16 @@ Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_dep_count), EqualTo(hd_dep_count,6), IsNotNull(hd_demo_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [2]: [hd_demo_sk#5, hd_dep_count#6] - -(7) Filter [codegen id : 1] +(6) CometFilter Input [2]: [hd_demo_sk#5, hd_dep_count#6] Condition : ((isnotnull(hd_dep_count#6) AND (hd_dep_count#6 = 6)) AND isnotnull(hd_demo_sk#5)) -(8) Project [codegen id : 1] -Output [1]: [hd_demo_sk#5] +(7) CometProject Input [2]: [hd_demo_sk#5, hd_dep_count#6] +Arguments: [hd_demo_sk#5], [hd_demo_sk#5] + +(8) ColumnarToRow [codegen id : 1] +Input [1]: [hd_demo_sk#5] (9) BroadcastExchange Input [1]: [hd_demo_sk#5] @@ -109,16 +109,16 @@ Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), GreaterThanOrEqual(t_hour,8), LessThanOrEqual(t_hour,9), IsNotNull(t_time_sk)] ReadSchema: struct -(13) ColumnarToRow [codegen id : 2] -Input [2]: [t_time_sk#7, t_hour#8] - -(14) Filter [codegen id : 2] +(13) CometFilter Input [2]: [t_time_sk#7, t_hour#8] Condition : (((isnotnull(t_hour#8) AND (t_hour#8 >= 8)) AND (t_hour#8 <= 9)) AND isnotnull(t_time_sk#7)) -(15) Project [codegen id : 2] -Output [1]: [t_time_sk#7] +(14) CometProject Input [2]: [t_time_sk#7, t_hour#8] +Arguments: [t_time_sk#7], [t_time_sk#7] + +(15) ColumnarToRow [codegen id : 2] +Input [1]: [t_time_sk#7] (16) BroadcastExchange Input [1]: [t_time_sk#7] @@ -141,16 +141,16 @@ Location [not included in comparison]/{warehouse_dir}/web_page] PushedFilters: [IsNotNull(wp_char_count), GreaterThanOrEqual(wp_char_count,5000), LessThanOrEqual(wp_char_count,5200), IsNotNull(wp_web_page_sk)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 3] -Input [2]: [wp_web_page_sk#9, wp_char_count#10] - -(21) Filter [codegen id : 3] +(20) CometFilter Input [2]: [wp_web_page_sk#9, wp_char_count#10] Condition : (((isnotnull(wp_char_count#10) AND (wp_char_count#10 >= 5000)) AND (wp_char_count#10 <= 5200)) AND isnotnull(wp_web_page_sk#9)) -(22) Project [codegen id : 3] -Output [1]: [wp_web_page_sk#9] +(21) CometProject Input [2]: [wp_web_page_sk#9, wp_char_count#10] +Arguments: [wp_web_page_sk#9], [wp_web_page_sk#9] + +(22) ColumnarToRow [codegen id : 3] +Input [1]: [wp_web_page_sk#9] (23) BroadcastExchange Input [1]: [wp_web_page_sk#9] @@ -191,16 +191,16 @@ Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_ship_hdemo_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_web_page_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 8] -Input [4]: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17, ws_sold_date_sk#18] - -(31) Filter [codegen id : 8] +(30) CometFilter Input [4]: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17, ws_sold_date_sk#18] Condition : ((isnotnull(ws_ship_hdemo_sk#16) AND isnotnull(ws_sold_time_sk#15)) AND isnotnull(ws_web_page_sk#17)) -(32) Project [codegen id : 8] -Output [3]: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17] +(31) CometProject Input [4]: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17, ws_sold_date_sk#18] +Arguments: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17], [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17] + +(32) ColumnarToRow [codegen id : 8] +Input [3]: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17] (33) ReusedExchange [Reuses operator id: 9] Output [1]: [hd_demo_sk#19] @@ -222,16 +222,16 @@ Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), GreaterThanOrEqual(t_hour,19), LessThanOrEqual(t_hour,20), IsNotNull(t_time_sk)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 6] -Input [2]: [t_time_sk#20, t_hour#21] - -(38) Filter [codegen id : 6] +(37) CometFilter Input [2]: [t_time_sk#20, t_hour#21] Condition : (((isnotnull(t_hour#21) AND (t_hour#21 >= 19)) AND (t_hour#21 <= 20)) AND isnotnull(t_time_sk#20)) -(39) Project [codegen id : 6] -Output [1]: [t_time_sk#20] +(38) CometProject Input [2]: [t_time_sk#20, t_hour#21] +Arguments: [t_time_sk#20], [t_time_sk#20] + +(39) ColumnarToRow [codegen id : 6] +Input [1]: [t_time_sk#20] (40) BroadcastExchange Input [1]: [t_time_sk#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/simplified.txt index 3f9582bb0..c4e04b06b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/simplified.txt @@ -12,34 +12,34 @@ WholeStageCodegen (10) BroadcastHashJoin [ws_sold_time_sk,t_time_sk] Project [ws_sold_time_sk,ws_web_page_sk] BroadcastHashJoin [ws_ship_hdemo_sk,hd_demo_sk] - Project [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] - Filter [ws_ship_hdemo_sk,ws_sold_time_sk,ws_web_page_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] + CometFilter [ws_ship_hdemo_sk,ws_sold_time_sk,ws_web_page_sk] CometScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] InputAdapter BroadcastExchange #2 WholeStageCodegen (1) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [hd_demo_sk] + CometFilter [hd_dep_count,hd_demo_sk] CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Project [t_time_sk] - Filter [t_hour,t_time_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [t_time_sk] + CometFilter [t_hour,t_time_sk] CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour] InputAdapter BroadcastExchange #4 WholeStageCodegen (3) - Project [wp_web_page_sk] - Filter [wp_char_count,wp_web_page_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [wp_web_page_sk] + CometFilter [wp_char_count,wp_web_page_sk] CometScan parquet spark_catalog.default.web_page [wp_web_page_sk,wp_char_count] InputAdapter BroadcastExchange #5 @@ -55,20 +55,20 @@ WholeStageCodegen (10) BroadcastHashJoin [ws_sold_time_sk,t_time_sk] Project [ws_sold_time_sk,ws_web_page_sk] BroadcastHashJoin [ws_ship_hdemo_sk,hd_demo_sk] - Project [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] - Filter [ws_ship_hdemo_sk,ws_sold_time_sk,ws_web_page_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] + CometFilter [ws_ship_hdemo_sk,ws_sold_time_sk,ws_web_page_sk] CometScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] InputAdapter ReusedExchange [hd_demo_sk] #2 InputAdapter BroadcastExchange #7 WholeStageCodegen (6) - Project [t_time_sk] - Filter [t_hour,t_time_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [t_time_sk] + CometFilter [t_hour,t_time_sk] CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour] InputAdapter ReusedExchange [wp_web_page_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/explain.txt index 521373bf7..61f35489a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/explain.txt @@ -16,31 +16,31 @@ : : : : +- * BroadcastHashJoin Inner BuildRight (11) : : : : :- * Project (9) : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : :- * Filter (3) - : : : : : : +- * ColumnarToRow (2) + : : : : : :- * ColumnarToRow (3) + : : : : : : +- CometFilter (2) : : : : : : +- CometScan parquet spark_catalog.default.call_center (1) : : : : : +- BroadcastExchange (7) - : : : : : +- * Filter (6) - : : : : : +- * ColumnarToRow (5) + : : : : : +- * ColumnarToRow (6) + : : : : : +- CometFilter (5) : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (4) : : : : +- ReusedExchange (10) : : : +- BroadcastExchange (16) - : : : +- * Filter (15) - : : : +- * ColumnarToRow (14) + : : : +- * ColumnarToRow (15) + : : : +- CometFilter (14) : : : +- CometScan parquet spark_catalog.default.customer (13) : : +- BroadcastExchange (23) - : : +- * Project (22) - : : +- * Filter (21) - : : +- * ColumnarToRow (20) + : : +- * ColumnarToRow (22) + : : +- CometProject (21) + : : +- CometFilter (20) : : +- CometScan parquet spark_catalog.default.customer_address (19) : +- BroadcastExchange (29) - : +- * Filter (28) - : +- * ColumnarToRow (27) + : +- * ColumnarToRow (28) + : +- CometFilter (27) : +- CometScan parquet spark_catalog.default.customer_demographics (26) +- BroadcastExchange (36) - +- * Project (35) - +- * Filter (34) - +- * ColumnarToRow (33) + +- * ColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) +- CometScan parquet spark_catalog.default.household_demographics (32) @@ -51,12 +51,12 @@ Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_call_center_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 7] +(2) CometFilter Input [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] +Condition : isnotnull(cc_call_center_sk#1) -(3) Filter [codegen id : 7] +(3) ColumnarToRow [codegen id : 7] Input [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] -Condition : isnotnull(cc_call_center_sk#1) (4) Scan parquet spark_catalog.default.catalog_returns Output [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] @@ -66,12 +66,12 @@ PartitionFilters: [isnotnull(cr_returned_date_sk#8), dynamicpruningexpression(cr PushedFilters: [IsNotNull(cr_call_center_sk), IsNotNull(cr_returning_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] +Condition : (isnotnull(cr_call_center_sk#6) AND isnotnull(cr_returning_customer_sk#5)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Condition : (isnotnull(cr_call_center_sk#6) AND isnotnull(cr_returning_customer_sk#5)) (7) BroadcastExchange Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] @@ -107,12 +107,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] +(14) CometFilter Input [4]: [c_customer_sk#11, c_current_cdemo_sk#12, c_current_hdemo_sk#13, c_current_addr_sk#14] +Condition : (((isnotnull(c_customer_sk#11) AND isnotnull(c_current_addr_sk#14)) AND isnotnull(c_current_cdemo_sk#12)) AND isnotnull(c_current_hdemo_sk#13)) -(15) Filter [codegen id : 3] +(15) ColumnarToRow [codegen id : 3] Input [4]: [c_customer_sk#11, c_current_cdemo_sk#12, c_current_hdemo_sk#13, c_current_addr_sk#14] -Condition : (((isnotnull(c_customer_sk#11) AND isnotnull(c_current_addr_sk#14)) AND isnotnull(c_current_cdemo_sk#12)) AND isnotnull(c_current_hdemo_sk#13)) (16) BroadcastExchange Input [4]: [c_customer_sk#11, c_current_cdemo_sk#12, c_current_hdemo_sk#13, c_current_addr_sk#14] @@ -135,16 +135,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-7.00), IsNotNull(ca_address_sk)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 4] -Input [2]: [ca_address_sk#15, ca_gmt_offset#16] - -(21) Filter [codegen id : 4] +(20) CometFilter Input [2]: [ca_address_sk#15, ca_gmt_offset#16] Condition : ((isnotnull(ca_gmt_offset#16) AND (ca_gmt_offset#16 = -7.00)) AND isnotnull(ca_address_sk#15)) -(22) Project [codegen id : 4] -Output [1]: [ca_address_sk#15] +(21) CometProject Input [2]: [ca_address_sk#15, ca_gmt_offset#16] +Arguments: [ca_address_sk#15], [ca_address_sk#15] + +(22) ColumnarToRow [codegen id : 4] +Input [1]: [ca_address_sk#15] (23) BroadcastExchange Input [1]: [ca_address_sk#15] @@ -167,12 +167,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Unknown )),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,Advanced Degree ))), IsNotNull(cd_demo_sk)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 5] +(27) CometFilter Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] +Condition : ((((cd_marital_status#18 = M) AND (cd_education_status#19 = Unknown )) OR ((cd_marital_status#18 = W) AND (cd_education_status#19 = Advanced Degree ))) AND isnotnull(cd_demo_sk#17)) -(28) Filter [codegen id : 5] +(28) ColumnarToRow [codegen id : 5] Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Condition : ((((cd_marital_status#18 = M) AND (cd_education_status#19 = Unknown )) OR ((cd_marital_status#18 = W) AND (cd_education_status#19 = Advanced Degree ))) AND isnotnull(cd_demo_sk#17)) (29) BroadcastExchange Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] @@ -195,16 +195,16 @@ Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_buy_potential), StringStartsWith(hd_buy_potential,Unknown), IsNotNull(hd_demo_sk)] ReadSchema: struct -(33) ColumnarToRow [codegen id : 6] -Input [2]: [hd_demo_sk#20, hd_buy_potential#21] - -(34) Filter [codegen id : 6] +(33) CometFilter Input [2]: [hd_demo_sk#20, hd_buy_potential#21] Condition : ((isnotnull(hd_buy_potential#21) AND StartsWith(hd_buy_potential#21, Unknown)) AND isnotnull(hd_demo_sk#20)) -(35) Project [codegen id : 6] -Output [1]: [hd_demo_sk#20] +(34) CometProject Input [2]: [hd_demo_sk#20, hd_buy_potential#21] +Arguments: [hd_demo_sk#20], [hd_demo_sk#20] + +(35) ColumnarToRow [codegen id : 6] +Input [1]: [hd_demo_sk#20] (36) BroadcastExchange Input [1]: [hd_demo_sk#20] @@ -250,9 +250,9 @@ Arguments: [Returns_Loss#28 DESC NULLS LAST], true, 0 Subquery:1 Hosting operator id = 4 Hosting Expression = cr_returned_date_sk#8 IN dynamicpruning#9 BroadcastExchange (48) -+- * Project (47) - +- * Filter (46) - +- * ColumnarToRow (45) ++- * ColumnarToRow (47) + +- CometProject (46) + +- CometFilter (45) +- CometScan parquet spark_catalog.default.date_dim (44) @@ -263,16 +263,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#10, d_year#29, d_moy#30] - -(46) Filter [codegen id : 1] +(45) CometFilter Input [3]: [d_date_sk#10, d_year#29, d_moy#30] Condition : ((((isnotnull(d_year#29) AND isnotnull(d_moy#30)) AND (d_year#29 = 1998)) AND (d_moy#30 = 11)) AND isnotnull(d_date_sk#10)) -(47) Project [codegen id : 1] -Output [1]: [d_date_sk#10] +(46) CometProject Input [3]: [d_date_sk#10, d_year#29, d_moy#30] +Arguments: [d_date_sk#10], [d_date_sk#10] + +(47) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#10] (48) BroadcastExchange Input [1]: [d_date_sk#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/simplified.txt index d40b04b59..e5d62e3c0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/simplified.txt @@ -20,54 +20,54 @@ WholeStageCodegen (9) BroadcastHashJoin [cr_returned_date_sk,d_date_sk] Project [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk] BroadcastHashJoin [cc_call_center_sk,cr_call_center_sk] - Filter [cc_call_center_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cc_call_center_sk] CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Filter [cr_call_center_sk,cr_returning_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cr_call_center_sk,cr_returning_customer_sk] CometScan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_moy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter ReusedExchange [d_date_sk] #4 InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - Filter [c_customer_sk,c_current_addr_sk,c_current_cdemo_sk,c_current_hdemo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_current_addr_sk,c_current_cdemo_sk,c_current_hdemo_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] InputAdapter BroadcastExchange #6 WholeStageCodegen (4) - Project [ca_address_sk] - Filter [ca_gmt_offset,ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_gmt_offset,ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] InputAdapter BroadcastExchange #7 WholeStageCodegen (5) - Filter [cd_marital_status,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cd_marital_status,cd_education_status,cd_demo_sk] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] InputAdapter BroadcastExchange #8 WholeStageCodegen (6) - Project [hd_demo_sk] - Filter [hd_buy_potential,hd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [hd_demo_sk] + CometFilter [hd_buy_potential,hd_demo_sk] CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/explain.txt index 204b23130..5f1f96168 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/explain.txt @@ -8,13 +8,13 @@ : +- * BroadcastHashJoin Inner BuildRight (22) : :- * Project (10) : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) + : : :- * ColumnarToRow (3) + : : : +- CometFilter (2) : : : +- CometScan parquet spark_catalog.default.web_sales (1) : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) + : : +- * ColumnarToRow (7) + : : +- CometProject (6) + : : +- CometFilter (5) : : +- CometScan parquet spark_catalog.default.item (4) : +- BroadcastExchange (21) : +- * Filter (20) @@ -23,8 +23,8 @@ : +- * HashAggregate (17) : +- * Project (16) : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Filter (13) - : : +- * ColumnarToRow (12) + : :- * ColumnarToRow (13) + : : +- CometFilter (12) : : +- CometScan parquet spark_catalog.default.web_sales (11) : +- ReusedExchange (14) +- ReusedExchange (24) @@ -38,12 +38,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sol PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_ext_discount_amt)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 6] +(2) CometFilter Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] +Condition : (isnotnull(ws_item_sk#1) AND isnotnull(ws_ext_discount_amt#2)) -(3) Filter [codegen id : 6] +(3) ColumnarToRow [codegen id : 6] Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] -Condition : (isnotnull(ws_item_sk#1) AND isnotnull(ws_ext_discount_amt#2)) (4) Scan parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_manufact_id#6] @@ -52,16 +52,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,350), IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [i_item_sk#5, i_manufact_id#6] - -(6) Filter [codegen id : 1] +(5) CometFilter Input [2]: [i_item_sk#5, i_manufact_id#6] Condition : ((isnotnull(i_manufact_id#6) AND (i_manufact_id#6 = 350)) AND isnotnull(i_item_sk#5)) -(7) Project [codegen id : 1] -Output [1]: [i_item_sk#5] +(6) CometProject Input [2]: [i_item_sk#5, i_manufact_id#6] +Arguments: [i_item_sk#5], [i_item_sk#5] + +(7) ColumnarToRow [codegen id : 1] +Input [1]: [i_item_sk#5] (8) BroadcastExchange Input [1]: [i_item_sk#5] @@ -85,12 +85,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#9), dynamicpruningexpression(ws_sol PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 3] +(12) CometFilter Input [3]: [ws_item_sk#7, ws_ext_discount_amt#8, ws_sold_date_sk#9] +Condition : isnotnull(ws_item_sk#7) -(13) Filter [codegen id : 3] +(13) ColumnarToRow [codegen id : 3] Input [3]: [ws_item_sk#7, ws_ext_discount_amt#8, ws_sold_date_sk#9] -Condition : isnotnull(ws_item_sk#7) (14) ReusedExchange [Reuses operator id: 34] Output [1]: [d_date_sk#11] @@ -176,9 +176,9 @@ Results [1]: [MakeDecimal(sum(UnscaledValue(ws_ext_discount_amt#2))#21,17,2) AS Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 BroadcastExchange (34) -+- * Project (33) - +- * Filter (32) - +- * ColumnarToRow (31) ++- * ColumnarToRow (33) + +- CometProject (32) + +- CometFilter (31) +- CometScan parquet spark_catalog.default.date_dim (30) @@ -189,16 +189,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#18, d_date#23] - -(32) Filter [codegen id : 1] +(31) CometFilter Input [2]: [d_date_sk#18, d_date#23] Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 2000-01-27)) AND (d_date#23 <= 2000-04-26)) AND isnotnull(d_date_sk#18)) -(33) Project [codegen id : 1] -Output [1]: [d_date_sk#18] +(32) CometProject Input [2]: [d_date_sk#18, d_date#23] +Arguments: [d_date_sk#18], [d_date_sk#18] + +(33) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#18] (34) BroadcastExchange Input [1]: [d_date_sk#18] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/simplified.txt index 33d262f07..a5e724c1f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/simplified.txt @@ -10,25 +10,25 @@ WholeStageCodegen (7) BroadcastHashJoin [i_item_sk,ws_item_sk,ws_ext_discount_amt,(1.3 * avg(ws_ext_discount_amt))] Project [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk,ws_ext_discount_amt] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_item_sk,ws_ext_discount_amt] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Project [i_item_sk] - Filter [i_manufact_id,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_item_sk] + CometFilter [i_manufact_id,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] InputAdapter BroadcastExchange #4 @@ -41,9 +41,9 @@ WholeStageCodegen (7) HashAggregate [ws_item_sk,ws_ext_discount_amt] [sum,count,sum,count] Project [ws_item_sk,ws_ext_discount_amt] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/explain.txt index 121f25701..00ed822f2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/explain.txt @@ -9,19 +9,19 @@ TakeOrderedAndProject (24) : +- * SortMergeJoin Inner (12) : :- * Sort (5) : : +- Exchange (4) - : : +- * Project (3) - : : +- * ColumnarToRow (2) + : : +- * ColumnarToRow (3) + : : +- CometProject (2) : : +- CometScan parquet spark_catalog.default.store_sales (1) : +- * Sort (11) : +- Exchange (10) - : +- * Project (9) - : +- * Filter (8) - : +- * ColumnarToRow (7) + : +- * ColumnarToRow (9) + : +- CometProject (8) + : +- CometFilter (7) : +- CometScan parquet spark_catalog.default.store_returns (6) +- BroadcastExchange (18) - +- * Project (17) - +- * Filter (16) - +- * ColumnarToRow (15) + +- * ColumnarToRow (17) + +- CometProject (16) + +- CometFilter (15) +- CometScan parquet spark_catalog.default.reason (14) @@ -31,12 +31,12 @@ Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] +(2) CometProject Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5], [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] -(3) Project [codegen id : 1] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] +(3) ColumnarToRow [codegen id : 1] +Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] (4) Exchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] @@ -53,16 +53,16 @@ Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number), IsNotNull(sr_reason_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 3] -Input [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] - -(8) Filter [codegen id : 3] +(7) CometFilter Input [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Condition : ((isnotnull(sr_item_sk#7) AND isnotnull(sr_ticket_number#9)) AND isnotnull(sr_reason_sk#8)) -(9) Project [codegen id : 3] -Output [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] +(8) CometProject Input [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10], [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] + +(9) ColumnarToRow [codegen id : 3] +Input [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] (10) Exchange Input [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] @@ -89,16 +89,16 @@ Location [not included in comparison]/{warehouse_dir}/reason] PushedFilters: [IsNotNull(r_reason_desc), EqualTo(r_reason_desc,reason 28 ), IsNotNull(r_reason_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 5] -Input [2]: [r_reason_sk#12, r_reason_desc#13] - -(16) Filter [codegen id : 5] +(15) CometFilter Input [2]: [r_reason_sk#12, r_reason_desc#13] Condition : ((isnotnull(r_reason_desc#13) AND (r_reason_desc#13 = reason 28 )) AND isnotnull(r_reason_sk#12)) -(17) Project [codegen id : 5] -Output [1]: [r_reason_sk#12] +(16) CometProject Input [2]: [r_reason_sk#12, r_reason_desc#13] +Arguments: [r_reason_sk#12], [r_reason_sk#12] + +(17) ColumnarToRow [codegen id : 5] +Input [1]: [r_reason_sk#12] (18) BroadcastExchange Input [1]: [r_reason_sk#12] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/simplified.txt index 775e14725..3ec7ac7b6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/simplified.txt @@ -15,9 +15,9 @@ TakeOrderedAndProject [sumsales,ss_customer_sk] InputAdapter Exchange [ss_item_sk,ss_ticket_number] #2 WholeStageCodegen (1) - Project [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] InputAdapter WholeStageCodegen (4) @@ -25,16 +25,16 @@ TakeOrderedAndProject [sumsales,ss_customer_sk] InputAdapter Exchange [sr_item_sk,sr_ticket_number] #3 WholeStageCodegen (3) - Project [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] - Filter [sr_item_sk,sr_ticket_number,sr_reason_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] + CometFilter [sr_item_sk,sr_ticket_number,sr_reason_sk] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] InputAdapter BroadcastExchange #4 WholeStageCodegen (5) - Project [r_reason_sk] - Filter [r_reason_desc,r_reason_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [r_reason_sk] + CometFilter [r_reason_desc,r_reason_sk] CometScan parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/explain.txt index 180e7e033..d71f96e15 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/explain.txt @@ -15,34 +15,34 @@ : : : : +- * SortMergeJoin LeftSemi (12) : : : : :- * Sort (6) : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : : +- * ColumnarToRow (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) : : : : +- * Sort (11) : : : : +- Exchange (10) - : : : : +- * Project (9) - : : : : +- * ColumnarToRow (8) + : : : : +- * ColumnarToRow (9) + : : : : +- CometProject (8) : : : : +- CometScan parquet spark_catalog.default.web_sales (7) : : : +- * Sort (18) : : : +- Exchange (17) - : : : +- * Project (16) - : : : +- * ColumnarToRow (15) + : : : +- * ColumnarToRow (16) + : : : +- CometProject (15) : : : +- CometScan parquet spark_catalog.default.web_returns (14) : : +- BroadcastExchange (24) - : : +- * Project (23) - : : +- * Filter (22) - : : +- * ColumnarToRow (21) + : : +- * ColumnarToRow (23) + : : +- CometProject (22) + : : +- CometFilter (21) : : +- CometScan parquet spark_catalog.default.date_dim (20) : +- BroadcastExchange (31) - : +- * Project (30) - : +- * Filter (29) - : +- * ColumnarToRow (28) + : +- * ColumnarToRow (30) + : +- CometProject (29) + : +- CometFilter (28) : +- CometScan parquet spark_catalog.default.customer_address (27) +- BroadcastExchange (38) - +- * Project (37) - +- * Filter (36) - +- * ColumnarToRow (35) + +- * ColumnarToRow (37) + +- CometProject (36) + +- CometFilter (35) +- CometScan parquet spark_catalog.default.web_site (34) @@ -53,16 +53,16 @@ Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_ship_date_sk), IsNotNull(ws_ship_addr_sk), IsNotNull(ws_web_site_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] Condition : ((isnotnull(ws_ship_date_sk#1) AND isnotnull(ws_ship_addr_sk#2)) AND isnotnull(ws_web_site_sk#3)) -(4) Project [codegen id : 1] -Output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +(3) CometProject Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] +Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] + +(4) ColumnarToRow [codegen id : 1] +Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] (5) Exchange Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] @@ -78,12 +78,12 @@ Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] ReadSchema: struct -(8) ColumnarToRow [codegen id : 3] +(8) CometProject Input [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] +Arguments: [ws_warehouse_sk#9, ws_order_number#10], [ws_warehouse_sk#9, ws_order_number#10] -(9) Project [codegen id : 3] -Output [2]: [ws_warehouse_sk#9, ws_order_number#10] -Input [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] +(9) ColumnarToRow [codegen id : 3] +Input [2]: [ws_warehouse_sk#9, ws_order_number#10] (10) Exchange Input [2]: [ws_warehouse_sk#9, ws_order_number#10] @@ -109,12 +109,12 @@ Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] ReadSchema: struct -(15) ColumnarToRow [codegen id : 6] +(15) CometProject Input [2]: [wr_order_number#12, wr_returned_date_sk#13] +Arguments: [wr_order_number#12], [wr_order_number#12] -(16) Project [codegen id : 6] -Output [1]: [wr_order_number#12] -Input [2]: [wr_order_number#12, wr_returned_date_sk#13] +(16) ColumnarToRow [codegen id : 6] +Input [1]: [wr_order_number#12] (17) Exchange Input [1]: [wr_order_number#12] @@ -137,16 +137,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-01), LessThanOrEqual(d_date,1999-04-02), IsNotNull(d_date_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 8] -Input [2]: [d_date_sk#14, d_date#15] - -(22) Filter [codegen id : 8] +(21) CometFilter Input [2]: [d_date_sk#14, d_date#15] Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-01)) AND (d_date#15 <= 1999-04-02)) AND isnotnull(d_date_sk#14)) -(23) Project [codegen id : 8] -Output [1]: [d_date_sk#14] +(22) CometProject Input [2]: [d_date_sk#14, d_date#15] +Arguments: [d_date_sk#14], [d_date_sk#14] + +(23) ColumnarToRow [codegen id : 8] +Input [1]: [d_date_sk#14] (24) BroadcastExchange Input [1]: [d_date_sk#14] @@ -169,16 +169,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,IL), IsNotNull(ca_address_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 9] -Input [2]: [ca_address_sk#16, ca_state#17] - -(29) Filter [codegen id : 9] +(28) CometFilter Input [2]: [ca_address_sk#16, ca_state#17] Condition : ((isnotnull(ca_state#17) AND (ca_state#17 = IL)) AND isnotnull(ca_address_sk#16)) -(30) Project [codegen id : 9] -Output [1]: [ca_address_sk#16] +(29) CometProject Input [2]: [ca_address_sk#16, ca_state#17] +Arguments: [ca_address_sk#16], [ca_address_sk#16] + +(30) ColumnarToRow [codegen id : 9] +Input [1]: [ca_address_sk#16] (31) BroadcastExchange Input [1]: [ca_address_sk#16] @@ -201,16 +201,16 @@ Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_company_name), EqualTo(web_company_name,pri ), IsNotNull(web_site_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 10] -Input [2]: [web_site_sk#18, web_company_name#19] - -(36) Filter [codegen id : 10] +(35) CometFilter Input [2]: [web_site_sk#18, web_company_name#19] Condition : ((isnotnull(web_company_name#19) AND (web_company_name#19 = pri )) AND isnotnull(web_site_sk#18)) -(37) Project [codegen id : 10] -Output [1]: [web_site_sk#18] +(36) CometProject Input [2]: [web_site_sk#18, web_company_name#19] +Arguments: [web_site_sk#18], [web_site_sk#18] + +(37) ColumnarToRow [codegen id : 10] +Input [1]: [web_site_sk#18] (38) BroadcastExchange Input [1]: [web_site_sk#18] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/simplified.txt index 1aa3b4dc1..34ddde768 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/simplified.txt @@ -23,10 +23,10 @@ WholeStageCodegen (12) InputAdapter Exchange [ws_order_number] #2 WholeStageCodegen (1) - Project [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - Filter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk] CometScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] InputAdapter WholeStageCodegen (4) @@ -34,9 +34,9 @@ WholeStageCodegen (12) InputAdapter Exchange [ws_order_number] #3 WholeStageCodegen (3) - Project [ws_warehouse_sk,ws_order_number] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ws_warehouse_sk,ws_order_number] CometScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] InputAdapter WholeStageCodegen (7) @@ -44,31 +44,31 @@ WholeStageCodegen (12) InputAdapter Exchange [wr_order_number] #4 WholeStageCodegen (6) - Project [wr_order_number] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [wr_order_number] CometScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] InputAdapter BroadcastExchange #5 WholeStageCodegen (8) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter BroadcastExchange #6 WholeStageCodegen (9) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_state,ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #7 WholeStageCodegen (10) - Project [web_site_sk] - Filter [web_company_name,web_site_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [web_site_sk] + CometFilter [web_company_name,web_site_sk] CometScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/explain.txt index f8ee8ccb8..c8cdce055 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/explain.txt @@ -14,17 +14,17 @@ : : : :- * SortMergeJoin LeftSemi (17) : : : : :- * Sort (6) : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : : +- * ColumnarToRow (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) : : : : +- * Project (16) : : : : +- * SortMergeJoin Inner (15) : : : : :- * Sort (12) : : : : : +- Exchange (11) - : : : : : +- * Project (10) - : : : : : +- * Filter (9) - : : : : : +- * ColumnarToRow (8) + : : : : : +- * ColumnarToRow (10) + : : : : : +- CometProject (9) + : : : : : +- CometFilter (8) : : : : : +- CometScan parquet spark_catalog.default.web_sales (7) : : : : +- * Sort (14) : : : : +- ReusedExchange (13) @@ -32,9 +32,9 @@ : : : +- * SortMergeJoin Inner (30) : : : :- * Sort (23) : : : : +- Exchange (22) - : : : : +- * Project (21) - : : : : +- * Filter (20) - : : : : +- * ColumnarToRow (19) + : : : : +- * ColumnarToRow (21) + : : : : +- CometProject (20) + : : : : +- CometFilter (19) : : : : +- CometScan parquet spark_catalog.default.web_returns (18) : : : +- * Project (29) : : : +- * SortMergeJoin Inner (28) @@ -43,19 +43,19 @@ : : : +- * Sort (27) : : : +- ReusedExchange (26) : : +- BroadcastExchange (37) - : : +- * Project (36) - : : +- * Filter (35) - : : +- * ColumnarToRow (34) + : : +- * ColumnarToRow (36) + : : +- CometProject (35) + : : +- CometFilter (34) : : +- CometScan parquet spark_catalog.default.date_dim (33) : +- BroadcastExchange (44) - : +- * Project (43) - : +- * Filter (42) - : +- * ColumnarToRow (41) + : +- * ColumnarToRow (43) + : +- CometProject (42) + : +- CometFilter (41) : +- CometScan parquet spark_catalog.default.customer_address (40) +- BroadcastExchange (51) - +- * Project (50) - +- * Filter (49) - +- * ColumnarToRow (48) + +- * ColumnarToRow (50) + +- CometProject (49) + +- CometFilter (48) +- CometScan parquet spark_catalog.default.web_site (47) @@ -66,16 +66,16 @@ Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_ship_date_sk), IsNotNull(ws_ship_addr_sk), IsNotNull(ws_web_site_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] Condition : ((isnotnull(ws_ship_date_sk#1) AND isnotnull(ws_ship_addr_sk#2)) AND isnotnull(ws_web_site_sk#3)) -(4) Project [codegen id : 1] -Output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +(3) CometProject Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] +Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] + +(4) ColumnarToRow [codegen id : 1] +Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] (5) Exchange Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] @@ -92,16 +92,16 @@ Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_order_number), IsNotNull(ws_warehouse_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 3] -Input [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] - -(9) Filter [codegen id : 3] +(8) CometFilter Input [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] Condition : (isnotnull(ws_order_number#9) AND isnotnull(ws_warehouse_sk#8)) -(10) Project [codegen id : 3] -Output [2]: [ws_warehouse_sk#8, ws_order_number#9] +(9) CometProject Input [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] +Arguments: [ws_warehouse_sk#8, ws_order_number#9], [ws_warehouse_sk#8, ws_order_number#9] + +(10) ColumnarToRow [codegen id : 3] +Input [2]: [ws_warehouse_sk#8, ws_order_number#9] (11) Exchange Input [2]: [ws_warehouse_sk#8, ws_order_number#9] @@ -141,16 +141,16 @@ Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 9] -Input [2]: [wr_order_number#13, wr_returned_date_sk#14] - -(20) Filter [codegen id : 9] +(19) CometFilter Input [2]: [wr_order_number#13, wr_returned_date_sk#14] Condition : isnotnull(wr_order_number#13) -(21) Project [codegen id : 9] -Output [1]: [wr_order_number#13] +(20) CometProject Input [2]: [wr_order_number#13, wr_returned_date_sk#14] +Arguments: [wr_order_number#13], [wr_order_number#13] + +(21) ColumnarToRow [codegen id : 9] +Input [1]: [wr_order_number#13] (22) Exchange Input [1]: [wr_order_number#13] @@ -207,16 +207,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-01), LessThanOrEqual(d_date,1999-04-02), IsNotNull(d_date_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 17] -Input [2]: [d_date_sk#19, d_date#20] - -(35) Filter [codegen id : 17] +(34) CometFilter Input [2]: [d_date_sk#19, d_date#20] Condition : (((isnotnull(d_date#20) AND (d_date#20 >= 1999-02-01)) AND (d_date#20 <= 1999-04-02)) AND isnotnull(d_date_sk#19)) -(36) Project [codegen id : 17] -Output [1]: [d_date_sk#19] +(35) CometProject Input [2]: [d_date_sk#19, d_date#20] +Arguments: [d_date_sk#19], [d_date_sk#19] + +(36) ColumnarToRow [codegen id : 17] +Input [1]: [d_date_sk#19] (37) BroadcastExchange Input [1]: [d_date_sk#19] @@ -239,16 +239,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,IL), IsNotNull(ca_address_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 18] -Input [2]: [ca_address_sk#21, ca_state#22] - -(42) Filter [codegen id : 18] +(41) CometFilter Input [2]: [ca_address_sk#21, ca_state#22] Condition : ((isnotnull(ca_state#22) AND (ca_state#22 = IL)) AND isnotnull(ca_address_sk#21)) -(43) Project [codegen id : 18] -Output [1]: [ca_address_sk#21] +(42) CometProject Input [2]: [ca_address_sk#21, ca_state#22] +Arguments: [ca_address_sk#21], [ca_address_sk#21] + +(43) ColumnarToRow [codegen id : 18] +Input [1]: [ca_address_sk#21] (44) BroadcastExchange Input [1]: [ca_address_sk#21] @@ -271,16 +271,16 @@ Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_company_name), EqualTo(web_company_name,pri ), IsNotNull(web_site_sk)] ReadSchema: struct -(48) ColumnarToRow [codegen id : 19] -Input [2]: [web_site_sk#23, web_company_name#24] - -(49) Filter [codegen id : 19] +(48) CometFilter Input [2]: [web_site_sk#23, web_company_name#24] Condition : ((isnotnull(web_company_name#24) AND (web_company_name#24 = pri )) AND isnotnull(web_site_sk#23)) -(50) Project [codegen id : 19] -Output [1]: [web_site_sk#23] +(49) CometProject Input [2]: [web_site_sk#23, web_company_name#24] +Arguments: [web_site_sk#23], [web_site_sk#23] + +(50) ColumnarToRow [codegen id : 19] +Input [1]: [web_site_sk#23] (51) BroadcastExchange Input [1]: [web_site_sk#23] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/simplified.txt index 389d74d39..5b699890c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/simplified.txt @@ -22,10 +22,10 @@ WholeStageCodegen (21) InputAdapter Exchange [ws_order_number] #2 WholeStageCodegen (1) - Project [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - Filter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk] CometScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] InputAdapter WholeStageCodegen (7) @@ -37,10 +37,10 @@ WholeStageCodegen (21) InputAdapter Exchange [ws_order_number] #3 WholeStageCodegen (3) - Project [ws_warehouse_sk,ws_order_number] - Filter [ws_order_number,ws_warehouse_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ws_warehouse_sk,ws_order_number] + CometFilter [ws_order_number,ws_warehouse_sk] CometScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] InputAdapter WholeStageCodegen (6) @@ -57,10 +57,10 @@ WholeStageCodegen (21) InputAdapter Exchange [wr_order_number] #4 WholeStageCodegen (9) - Project [wr_order_number] - Filter [wr_order_number] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [wr_order_number] + CometFilter [wr_order_number] CometScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] InputAdapter WholeStageCodegen (15) @@ -79,24 +79,24 @@ WholeStageCodegen (21) InputAdapter BroadcastExchange #5 WholeStageCodegen (17) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter BroadcastExchange #6 WholeStageCodegen (18) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_state,ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #7 WholeStageCodegen (19) - Project [web_site_sk] - Filter [web_company_name,web_site_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [web_site_sk] + CometFilter [web_company_name,web_site_sk] CometScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/explain.txt index e0ce29b43..60c262e9c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/explain.txt @@ -8,24 +8,24 @@ : +- * BroadcastHashJoin Inner BuildRight (17) : :- * Project (11) : : +- * BroadcastHashJoin Inner BuildRight (10) - : : :- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) + : : :- * ColumnarToRow (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : +- BroadcastExchange (9) - : : +- * Project (8) - : : +- * Filter (7) - : : +- * ColumnarToRow (6) + : : +- * ColumnarToRow (8) + : : +- CometProject (7) + : : +- CometFilter (6) : : +- CometScan parquet spark_catalog.default.household_demographics (5) : +- BroadcastExchange (16) - : +- * Project (15) - : +- * Filter (14) - : +- * ColumnarToRow (13) + : +- * ColumnarToRow (15) + : +- CometProject (14) + : +- CometFilter (13) : +- CometScan parquet spark_catalog.default.time_dim (12) +- BroadcastExchange (23) - +- * Project (22) - +- * Filter (21) - +- * ColumnarToRow (20) + +- * ColumnarToRow (22) + +- CometProject (21) + +- CometFilter (20) +- CometScan parquet spark_catalog.default.store (19) @@ -36,16 +36,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isnotnull(ss_store_sk#3)) -(4) Project [codegen id : 4] -Output [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] +(3) CometProject Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] +Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] + +(4) ColumnarToRow [codegen id : 4] +Input [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] (5) Scan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#5, hd_dep_count#6] @@ -54,16 +54,16 @@ Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_dep_count), EqualTo(hd_dep_count,7), IsNotNull(hd_demo_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [2]: [hd_demo_sk#5, hd_dep_count#6] - -(7) Filter [codegen id : 1] +(6) CometFilter Input [2]: [hd_demo_sk#5, hd_dep_count#6] Condition : ((isnotnull(hd_dep_count#6) AND (hd_dep_count#6 = 7)) AND isnotnull(hd_demo_sk#5)) -(8) Project [codegen id : 1] -Output [1]: [hd_demo_sk#5] +(7) CometProject Input [2]: [hd_demo_sk#5, hd_dep_count#6] +Arguments: [hd_demo_sk#5], [hd_demo_sk#5] + +(8) ColumnarToRow [codegen id : 1] +Input [1]: [hd_demo_sk#5] (9) BroadcastExchange Input [1]: [hd_demo_sk#5] @@ -86,16 +86,16 @@ Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,20), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(13) ColumnarToRow [codegen id : 2] -Input [3]: [t_time_sk#7, t_hour#8, t_minute#9] - -(14) Filter [codegen id : 2] +(13) CometFilter Input [3]: [t_time_sk#7, t_hour#8, t_minute#9] Condition : ((((isnotnull(t_hour#8) AND isnotnull(t_minute#9)) AND (t_hour#8 = 20)) AND (t_minute#9 >= 30)) AND isnotnull(t_time_sk#7)) -(15) Project [codegen id : 2] -Output [1]: [t_time_sk#7] +(14) CometProject Input [3]: [t_time_sk#7, t_hour#8, t_minute#9] +Arguments: [t_time_sk#7], [t_time_sk#7] + +(15) ColumnarToRow [codegen id : 2] +Input [1]: [t_time_sk#7] (16) BroadcastExchange Input [1]: [t_time_sk#7] @@ -118,16 +118,16 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_name), EqualTo(s_store_name,ese), IsNotNull(s_store_sk)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 3] -Input [2]: [s_store_sk#10, s_store_name#11] - -(21) Filter [codegen id : 3] +(20) CometFilter Input [2]: [s_store_sk#10, s_store_name#11] Condition : ((isnotnull(s_store_name#11) AND (s_store_name#11 = ese)) AND isnotnull(s_store_sk#10)) -(22) Project [codegen id : 3] -Output [1]: [s_store_sk#10] +(21) CometProject Input [2]: [s_store_sk#10, s_store_name#11] +Arguments: [s_store_sk#10], [s_store_sk#10] + +(22) ColumnarToRow [codegen id : 3] +Input [1]: [s_store_sk#10] (23) BroadcastExchange Input [1]: [s_store_sk#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/simplified.txt index fe9cd8bc8..d1438f48e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/simplified.txt @@ -10,32 +10,32 @@ WholeStageCodegen (5) BroadcastHashJoin [ss_sold_time_sk,t_time_sk] Project [ss_sold_time_sk,ss_store_sk] BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] InputAdapter BroadcastExchange #2 WholeStageCodegen (1) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [hd_demo_sk] + CometFilter [hd_dep_count,hd_demo_sk] CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [t_time_sk] + CometFilter [t_hour,t_minute,t_time_sk] CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] InputAdapter BroadcastExchange #4 WholeStageCodegen (3) - Project [s_store_sk] - Filter [s_store_name,s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [s_store_sk] + CometFilter [s_store_name,s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/explain.txt index 6d54b23d8..66ccf4f22 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/explain.txt @@ -146,9 +146,9 @@ Results [3]: [sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#13) Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 BroadcastExchange (28) -+- * Project (27) - +- * Filter (26) - +- * ColumnarToRow (25) ++- * ColumnarToRow (27) + +- CometProject (26) + +- CometFilter (25) +- CometScan parquet spark_catalog.default.date_dim (24) @@ -159,16 +159,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_month_seq#27] - -(26) Filter [codegen id : 1] +(25) CometFilter Input [2]: [d_date_sk#5, d_month_seq#27] Condition : (((isnotnull(d_month_seq#27) AND (d_month_seq#27 >= 1200)) AND (d_month_seq#27 <= 1211)) AND isnotnull(d_date_sk#5)) -(27) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(26) CometProject Input [2]: [d_date_sk#5, d_month_seq#27] +Arguments: [d_date_sk#5], [d_date_sk#5] + +(27) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#5] (28) BroadcastExchange Input [1]: [d_date_sk#5] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/simplified.txt index ba918e146..be9c20a56 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/simplified.txt @@ -22,10 +22,10 @@ WholeStageCodegen (8) SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_month_seq,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/explain.txt index d4202b01d..3d66a07d0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/explain.txt @@ -13,12 +13,12 @@ +- * BroadcastHashJoin Inner BuildRight (11) :- * Project (9) : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) + : :- * ColumnarToRow (3) + : : +- CometFilter (2) : : +- CometScan parquet spark_catalog.default.store_sales (1) : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) + : +- * ColumnarToRow (6) + : +- CometFilter (5) : +- CometScan parquet spark_catalog.default.item (4) +- ReusedExchange (10) @@ -31,12 +31,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Condition : isnotnull(ss_item_sk#1) -(3) Filter [codegen id : 3] +(3) ColumnarToRow [codegen id : 3] Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_item_sk#1) (4) Scan parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] @@ -45,12 +45,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Condition : (i_category#10 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (i_category#10 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) (7) BroadcastExchange Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] @@ -129,9 +129,9 @@ Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemreve Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 BroadcastExchange (27) -+- * Project (26) - +- * Filter (25) - +- * ColumnarToRow (24) ++- * ColumnarToRow (26) + +- CometProject (25) + +- CometFilter (24) +- CometScan parquet spark_catalog.default.date_dim (23) @@ -142,16 +142,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#11, d_date#19] - -(25) Filter [codegen id : 1] +(24) CometFilter Input [2]: [d_date_sk#11, d_date#19] Condition : (((isnotnull(d_date#19) AND (d_date#19 >= 1999-02-22)) AND (d_date#19 <= 1999-03-24)) AND isnotnull(d_date_sk#11)) -(26) Project [codegen id : 1] -Output [1]: [d_date_sk#11] +(25) CometProject Input [2]: [d_date_sk#11, d_date#19] +Arguments: [d_date_sk#11], [d_date_sk#11] + +(26) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#11] (27) BroadcastExchange Input [1]: [d_date_sk#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/simplified.txt index d3b1f3fe6..9eabb9977 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/simplified.txt @@ -21,24 +21,24 @@ WholeStageCodegen (7) BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_category,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] InputAdapter ReusedExchange [d_date_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99/explain.txt index f8cf6b243..936555026 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99/explain.txt @@ -11,25 +11,25 @@ TakeOrderedAndProject (32) : : +- * BroadcastHashJoin Inner BuildRight (14) : : :- * Project (9) : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) + : : : +- * ColumnarToRow (6) + : : : +- CometFilter (5) : : : +- CometScan parquet spark_catalog.default.warehouse (4) : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) + : : +- * ColumnarToRow (12) + : : +- CometFilter (11) : : +- CometScan parquet spark_catalog.default.ship_mode (10) : +- BroadcastExchange (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) + : +- * ColumnarToRow (18) + : +- CometFilter (17) : +- CometScan parquet spark_catalog.default.call_center (16) +- BroadcastExchange (26) - +- * Project (25) - +- * Filter (24) - +- * ColumnarToRow (23) + +- * ColumnarToRow (25) + +- CometProject (24) + +- CometFilter (23) +- CometScan parquet spark_catalog.default.date_dim (22) @@ -40,12 +40,12 @@ Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_ship_mode_sk), IsNotNull(cs_call_center_sk), IsNotNull(cs_ship_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] +(2) CometFilter Input [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] +Condition : (((isnotnull(cs_warehouse_sk#4) AND isnotnull(cs_ship_mode_sk#3)) AND isnotnull(cs_call_center_sk#2)) AND isnotnull(cs_ship_date_sk#1)) -(3) Filter [codegen id : 5] +(3) ColumnarToRow [codegen id : 5] Input [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] -Condition : (((isnotnull(cs_warehouse_sk#4) AND isnotnull(cs_ship_mode_sk#3)) AND isnotnull(cs_call_center_sk#2)) AND isnotnull(cs_ship_date_sk#1)) (4) Scan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] @@ -54,12 +54,12 @@ Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Condition : isnotnull(w_warehouse_sk#6) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Condition : isnotnull(w_warehouse_sk#6) (7) BroadcastExchange Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] @@ -82,12 +82,12 @@ Location [not included in comparison]/{warehouse_dir}/ship_mode] PushedFilters: [IsNotNull(sm_ship_mode_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] +(11) CometFilter Input [2]: [sm_ship_mode_sk#8, sm_type#9] +Condition : isnotnull(sm_ship_mode_sk#8) -(12) Filter [codegen id : 2] +(12) ColumnarToRow [codegen id : 2] Input [2]: [sm_ship_mode_sk#8, sm_type#9] -Condition : isnotnull(sm_ship_mode_sk#8) (13) BroadcastExchange Input [2]: [sm_ship_mode_sk#8, sm_type#9] @@ -110,12 +110,12 @@ Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_call_center_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] +(17) CometFilter Input [2]: [cc_call_center_sk#10, cc_name#11] +Condition : isnotnull(cc_call_center_sk#10) -(18) Filter [codegen id : 3] +(18) ColumnarToRow [codegen id : 3] Input [2]: [cc_call_center_sk#10, cc_name#11] -Condition : isnotnull(cc_call_center_sk#10) (19) BroadcastExchange Input [2]: [cc_call_center_sk#10, cc_name#11] @@ -138,16 +138,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 4] -Input [2]: [d_date_sk#12, d_month_seq#13] - -(24) Filter [codegen id : 4] +(23) CometFilter Input [2]: [d_date_sk#12, d_month_seq#13] Condition : (((isnotnull(d_month_seq#13) AND (d_month_seq#13 >= 1200)) AND (d_month_seq#13 <= 1211)) AND isnotnull(d_date_sk#12)) -(25) Project [codegen id : 4] -Output [1]: [d_date_sk#12] +(24) CometProject Input [2]: [d_date_sk#12, d_month_seq#13] +Arguments: [d_date_sk#12], [d_date_sk#12] + +(25) ColumnarToRow [codegen id : 4] +Input [1]: [d_date_sk#12] (26) BroadcastExchange Input [1]: [d_date_sk#12] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99/simplified.txt index c165c1fc1..c5f25f079 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99/simplified.txt @@ -13,36 +13,36 @@ TakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days , BroadcastHashJoin [cs_ship_mode_sk,sm_ship_mode_sk] Project [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name] BroadcastHashJoin [cs_warehouse_sk,w_warehouse_sk] - Filter [cs_warehouse_sk,cs_ship_mode_sk,cs_call_center_sk,cs_ship_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_warehouse_sk,cs_ship_mode_sk,cs_call_center_sk,cs_ship_date_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] InputAdapter BroadcastExchange #2 WholeStageCodegen (1) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [w_warehouse_sk] CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Filter [sm_ship_mode_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [sm_ship_mode_sk] CometScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] InputAdapter BroadcastExchange #4 WholeStageCodegen (3) - Filter [cc_call_center_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cc_call_center_sk] CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #5 WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_month_seq,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/explain.txt index a0259dcc2..be0e98db2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/explain.txt @@ -10,8 +10,8 @@ TakeOrderedAndProject (41) : :- * Project (24) : : +- * BroadcastHashJoin LeftSemi BuildRight (23) : : :- * BroadcastHashJoin LeftSemi BuildRight (10) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.customer (1) : : : +- BroadcastExchange (9) : : : +- * Project (8) @@ -32,13 +32,13 @@ TakeOrderedAndProject (41) : : : +- CometScan parquet spark_catalog.default.catalog_sales (16) : : +- ReusedExchange (18) : +- BroadcastExchange (29) - : +- * Project (28) - : +- * Filter (27) - : +- * ColumnarToRow (26) + : +- * ColumnarToRow (28) + : +- CometProject (27) + : +- CometFilter (26) : +- CometScan parquet spark_catalog.default.customer_address (25) +- BroadcastExchange (35) - +- * Filter (34) - +- * ColumnarToRow (33) + +- * ColumnarToRow (34) + +- CometFilter (33) +- CometScan parquet spark_catalog.default.customer_demographics (32) @@ -49,12 +49,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 9] +(2) CometFilter Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) -(3) Filter [codegen id : 9] +(3) ColumnarToRow [codegen id : 9] Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) (4) Scan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] @@ -158,16 +158,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [In(ca_county, [Dona Ana County,Douglas County,Gaines County,Richland County,Walker County]), IsNotNull(ca_address_sk)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 7] -Input [2]: [ca_address_sk#18, ca_county#19] - -(27) Filter [codegen id : 7] +(26) CometFilter Input [2]: [ca_address_sk#18, ca_county#19] Condition : (ca_county#19 IN (Walker County,Richland County,Gaines County,Douglas County,Dona Ana County) AND isnotnull(ca_address_sk#18)) -(28) Project [codegen id : 7] -Output [1]: [ca_address_sk#18] +(27) CometProject Input [2]: [ca_address_sk#18, ca_county#19] +Arguments: [ca_address_sk#18], [ca_address_sk#18] + +(28) ColumnarToRow [codegen id : 7] +Input [1]: [ca_address_sk#18] (29) BroadcastExchange Input [1]: [ca_address_sk#18] @@ -190,12 +190,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(33) ColumnarToRow [codegen id : 8] +(33) CometFilter Input [9]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Condition : isnotnull(cd_demo_sk#20) -(34) Filter [codegen id : 8] +(34) ColumnarToRow [codegen id : 8] Input [9]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Condition : isnotnull(cd_demo_sk#20) (35) BroadcastExchange Input [9]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] @@ -237,9 +237,9 @@ Arguments: 100, [cd_gender#21 ASC NULLS FIRST, cd_marital_status#22 ASC NULLS FI Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 BroadcastExchange (46) -+- * Project (45) - +- * Filter (44) - +- * ColumnarToRow (43) ++- * ColumnarToRow (45) + +- CometProject (44) + +- CometFilter (43) +- CometScan parquet spark_catalog.default.date_dim (42) @@ -250,16 +250,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,7), IsNotNull(d_date_sk)] ReadSchema: struct -(43) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#7, d_year#38, d_moy#39] - -(44) Filter [codegen id : 1] +(43) CometFilter Input [3]: [d_date_sk#7, d_year#38, d_moy#39] Condition : (((((isnotnull(d_year#38) AND isnotnull(d_moy#39)) AND (d_year#38 = 2002)) AND (d_moy#39 >= 4)) AND (d_moy#39 <= 7)) AND isnotnull(d_date_sk#7)) -(45) Project [codegen id : 1] -Output [1]: [d_date_sk#7] +(44) CometProject Input [3]: [d_date_sk#7, d_year#38, d_moy#39] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(45) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#7] (46) BroadcastExchange Input [1]: [d_date_sk#7] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/simplified.txt index 668cec531..3eb2210a6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/simplified.txt @@ -12,9 +12,9 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha Project [c_current_cdemo_sk,c_current_addr_sk] BroadcastHashJoin [c_customer_sk,customer_sk] BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_current_addr_sk,c_current_cdemo_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] InputAdapter BroadcastExchange #2 @@ -27,10 +27,10 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_moy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter ReusedExchange [d_date_sk] #3 @@ -58,15 +58,15 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter BroadcastExchange #5 WholeStageCodegen (7) - Project [ca_address_sk] - Filter [ca_county,ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_county,ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] InputAdapter BroadcastExchange #6 WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/explain.txt index 2c9f35aea..daa1f5243 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/explain.txt @@ -13,12 +13,12 @@ TakeOrderedAndProject (71) : : : +- * BroadcastHashJoin Inner BuildRight (11) : : : :- * Project (9) : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.customer (1) : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) + : : : : +- * ColumnarToRow (6) + : : : : +- CometFilter (5) : : : : +- CometScan parquet spark_catalog.default.store_sales (4) : : : +- ReusedExchange (10) : : +- BroadcastExchange (32) @@ -29,12 +29,12 @@ TakeOrderedAndProject (71) : : +- * BroadcastHashJoin Inner BuildRight (27) : : :- * Project (25) : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : :- * Filter (19) - : : : : +- * ColumnarToRow (18) + : : : :- * ColumnarToRow (19) + : : : : +- CometFilter (18) : : : : +- CometScan parquet spark_catalog.default.customer (17) : : : +- BroadcastExchange (23) - : : : +- * Filter (22) - : : : +- * ColumnarToRow (21) + : : : +- * ColumnarToRow (22) + : : : +- CometFilter (21) : : : +- CometScan parquet spark_catalog.default.store_sales (20) : : +- ReusedExchange (26) : +- BroadcastExchange (50) @@ -46,12 +46,12 @@ TakeOrderedAndProject (71) : +- * BroadcastHashJoin Inner BuildRight (44) : :- * Project (42) : : +- * BroadcastHashJoin Inner BuildRight (41) - : : :- * Filter (36) - : : : +- * ColumnarToRow (35) + : : :- * ColumnarToRow (36) + : : : +- CometFilter (35) : : : +- CometScan parquet spark_catalog.default.customer (34) : : +- BroadcastExchange (40) - : : +- * Filter (39) - : : +- * ColumnarToRow (38) + : : +- * ColumnarToRow (39) + : : +- CometFilter (38) : : +- CometScan parquet spark_catalog.default.web_sales (37) : +- ReusedExchange (43) +- BroadcastExchange (68) @@ -62,12 +62,12 @@ TakeOrderedAndProject (71) +- * BroadcastHashJoin Inner BuildRight (63) :- * Project (61) : +- * BroadcastHashJoin Inner BuildRight (60) - : :- * Filter (55) - : : +- * ColumnarToRow (54) + : :- * ColumnarToRow (55) + : : +- CometFilter (54) : : +- CometScan parquet spark_catalog.default.customer (53) : +- BroadcastExchange (59) - : +- * Filter (58) - : +- * ColumnarToRow (57) + : +- * ColumnarToRow (58) + : +- CometFilter (57) : +- CometScan parquet spark_catalog.default.web_sales (56) +- ReusedExchange (62) @@ -79,12 +79,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) -(3) Filter [codegen id : 3] +(3) ColumnarToRow [codegen id : 3] Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) (4) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] @@ -94,12 +94,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] +Condition : isnotnull(ss_customer_sk#9) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_customer_sk#9) (7) BroadcastExchange Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] @@ -157,12 +157,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 6] +(18) CometFilter Input [8]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] +Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_customer_id#22)) -(19) Filter [codegen id : 6] +(19) ColumnarToRow [codegen id : 6] Input [8]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_customer_id#22)) (20) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] @@ -172,12 +172,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#32), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 4] +(21) CometFilter Input [4]: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] +Condition : isnotnull(ss_customer_sk#29) -(22) Filter [codegen id : 4] +(22) ColumnarToRow [codegen id : 4] Input [4]: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] -Condition : isnotnull(ss_customer_sk#29) (23) BroadcastExchange Input [4]: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] @@ -241,12 +241,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 10] +(35) CometFilter Input [8]: [c_customer_sk#43, c_customer_id#44, c_first_name#45, c_last_name#46, c_preferred_cust_flag#47, c_birth_country#48, c_login#49, c_email_address#50] +Condition : (isnotnull(c_customer_sk#43) AND isnotnull(c_customer_id#44)) -(36) Filter [codegen id : 10] +(36) ColumnarToRow [codegen id : 10] Input [8]: [c_customer_sk#43, c_customer_id#44, c_first_name#45, c_last_name#46, c_preferred_cust_flag#47, c_birth_country#48, c_login#49, c_email_address#50] -Condition : (isnotnull(c_customer_sk#43) AND isnotnull(c_customer_id#44)) (37) Scan parquet spark_catalog.default.web_sales Output [4]: [ws_bill_customer_sk#51, ws_ext_discount_amt#52, ws_ext_list_price#53, ws_sold_date_sk#54] @@ -256,12 +256,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#54), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 8] +(38) CometFilter Input [4]: [ws_bill_customer_sk#51, ws_ext_discount_amt#52, ws_ext_list_price#53, ws_sold_date_sk#54] +Condition : isnotnull(ws_bill_customer_sk#51) -(39) Filter [codegen id : 8] +(39) ColumnarToRow [codegen id : 8] Input [4]: [ws_bill_customer_sk#51, ws_ext_discount_amt#52, ws_ext_list_price#53, ws_sold_date_sk#54] -Condition : isnotnull(ws_bill_customer_sk#51) (40) BroadcastExchange Input [4]: [ws_bill_customer_sk#51, ws_ext_discount_amt#52, ws_ext_list_price#53, ws_sold_date_sk#54] @@ -333,12 +333,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(54) ColumnarToRow [codegen id : 14] +(54) CometFilter Input [8]: [c_customer_sk#63, c_customer_id#64, c_first_name#65, c_last_name#66, c_preferred_cust_flag#67, c_birth_country#68, c_login#69, c_email_address#70] +Condition : (isnotnull(c_customer_sk#63) AND isnotnull(c_customer_id#64)) -(55) Filter [codegen id : 14] +(55) ColumnarToRow [codegen id : 14] Input [8]: [c_customer_sk#63, c_customer_id#64, c_first_name#65, c_last_name#66, c_preferred_cust_flag#67, c_birth_country#68, c_login#69, c_email_address#70] -Condition : (isnotnull(c_customer_sk#63) AND isnotnull(c_customer_id#64)) (56) Scan parquet spark_catalog.default.web_sales Output [4]: [ws_bill_customer_sk#71, ws_ext_discount_amt#72, ws_ext_list_price#73, ws_sold_date_sk#74] @@ -348,12 +348,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#74), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 12] +(57) CometFilter Input [4]: [ws_bill_customer_sk#71, ws_ext_discount_amt#72, ws_ext_list_price#73, ws_sold_date_sk#74] +Condition : isnotnull(ws_bill_customer_sk#71) -(58) Filter [codegen id : 12] +(58) ColumnarToRow [codegen id : 12] Input [4]: [ws_bill_customer_sk#71, ws_ext_discount_amt#72, ws_ext_list_price#73, ws_sold_date_sk#74] -Condition : isnotnull(ws_bill_customer_sk#71) (59) BroadcastExchange Input [4]: [ws_bill_customer_sk#71, ws_ext_discount_amt#72, ws_ext_list_price#73, ws_sold_date_sk#74] @@ -422,8 +422,8 @@ Arguments: 100, [customer_id#38 ASC NULLS FIRST, customer_first_name#39 ASC NULL Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13 BroadcastExchange (75) -+- * Filter (74) - +- * ColumnarToRow (73) ++- * ColumnarToRow (74) + +- CometFilter (73) +- CometScan parquet spark_catalog.default.date_dim (72) @@ -434,12 +434,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(73) ColumnarToRow [codegen id : 1] +(73) CometFilter Input [2]: [d_date_sk#14, d_year#15] +Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2001)) AND isnotnull(d_date_sk#14)) -(74) Filter [codegen id : 1] +(74) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2001)) AND isnotnull(d_date_sk#14)) (75) BroadcastExchange Input [2]: [d_date_sk#14, d_year#15] @@ -447,8 +447,8 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint) Subquery:2 Hosting operator id = 20 Hosting Expression = ss_sold_date_sk#32 IN dynamicpruning#33 BroadcastExchange (79) -+- * Filter (78) - +- * ColumnarToRow (77) ++- * ColumnarToRow (78) + +- CometFilter (77) +- CometScan parquet spark_catalog.default.date_dim (76) @@ -459,12 +459,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(77) ColumnarToRow [codegen id : 1] +(77) CometFilter Input [2]: [d_date_sk#34, d_year#35] +Condition : ((isnotnull(d_year#35) AND (d_year#35 = 2002)) AND isnotnull(d_date_sk#34)) -(78) Filter [codegen id : 1] +(78) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#34, d_year#35] -Condition : ((isnotnull(d_year#35) AND (d_year#35 = 2002)) AND isnotnull(d_date_sk#34)) (79) BroadcastExchange Input [2]: [d_date_sk#34, d_year#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/simplified.txt index 172da54ae..0a30aba05 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/simplified.txt @@ -15,23 +15,23 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_customer_id] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] InputAdapter BroadcastExchange #2 WholeStageCodegen (1) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [d_date_sk,d_year] #3 @@ -47,23 +47,23 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_customer_id] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] InputAdapter BroadcastExchange #6 WholeStageCodegen (4) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #7 WholeStageCodegen (1) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [d_date_sk,d_year] #7 @@ -80,16 +80,16 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_customer_id] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] InputAdapter BroadcastExchange #10 WholeStageCodegen (8) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_bill_customer_sk] CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -106,16 +106,16 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_customer_id] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] InputAdapter BroadcastExchange #13 WholeStageCodegen (12) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_bill_customer_sk] CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/explain.txt index 233ef7cb0..c39a71879 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/explain.txt @@ -11,12 +11,12 @@ TakeOrderedAndProject (20) +- * BroadcastHashJoin Inner BuildRight (11) :- * Project (9) : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) + : :- * ColumnarToRow (3) + : : +- CometFilter (2) : : +- CometScan parquet spark_catalog.default.web_sales (1) : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) + : +- * ColumnarToRow (6) + : +- CometFilter (5) : +- CometScan parquet spark_catalog.default.item (4) +- ReusedExchange (10) @@ -29,12 +29,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sol PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] +Condition : isnotnull(ws_item_sk#1) -(3) Filter [codegen id : 3] +(3) ColumnarToRow [codegen id : 3] Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Condition : isnotnull(ws_item_sk#1) (4) Scan parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] @@ -43,12 +43,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Condition : (i_category#10 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (i_category#10 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) (7) BroadcastExchange Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] @@ -119,9 +119,9 @@ Arguments: 100, [i_category#10 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_ite Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 BroadcastExchange (25) -+- * Project (24) - +- * Filter (23) - +- * ColumnarToRow (22) ++- * ColumnarToRow (24) + +- CometProject (23) + +- CometFilter (22) +- CometScan parquet spark_catalog.default.date_dim (21) @@ -132,16 +132,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(22) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#11, d_date#19] - -(23) Filter [codegen id : 1] +(22) CometFilter Input [2]: [d_date_sk#11, d_date#19] Condition : (((isnotnull(d_date#19) AND (d_date#19 >= 1999-02-22)) AND (d_date#19 <= 1999-03-24)) AND isnotnull(d_date_sk#11)) -(24) Project [codegen id : 1] -Output [1]: [d_date_sk#11] +(23) CometProject Input [2]: [d_date_sk#11, d_date#19] +Arguments: [d_date_sk#11], [d_date_sk#11] + +(24) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#11] (25) BroadcastExchange Input [1]: [d_date_sk#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/simplified.txt index 689e8ae67..1bc2538b4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/simplified.txt @@ -17,24 +17,24 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter BroadcastExchange #4 WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_category,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] InputAdapter ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/explain.txt index 04118281e..ec52cea9f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/explain.txt @@ -10,14 +10,14 @@ TakeOrderedAndProject (84) : :- * Project (59) : : +- * BroadcastHashJoin Inner BuildRight (58) : : :- * BroadcastHashJoin LeftSemi BuildRight (51) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- BroadcastExchange (50) : : : +- * Project (49) : : : +- * BroadcastHashJoin Inner BuildRight (48) - : : : :- * Filter (6) - : : : : +- * ColumnarToRow (5) + : : : :- * ColumnarToRow (6) + : : : : +- CometFilter (5) : : : : +- CometScan parquet spark_catalog.default.item (4) : : : +- BroadcastExchange (47) : : : +- * BroadcastHashJoin LeftSemi BuildRight (46) @@ -28,25 +28,25 @@ TakeOrderedAndProject (84) : : : : +- * BroadcastHashJoin Inner BuildRight (31) : : : : :- * Project (29) : : : : : +- * BroadcastHashJoin Inner BuildRight (28) - : : : : : :- * Filter (9) - : : : : : : +- * ColumnarToRow (8) + : : : : : :- * ColumnarToRow (9) + : : : : : : +- CometFilter (8) : : : : : : +- CometScan parquet spark_catalog.default.store_sales (7) : : : : : +- BroadcastExchange (27) : : : : : +- * BroadcastHashJoin LeftSemi BuildRight (26) - : : : : : :- * Filter (12) - : : : : : : +- * ColumnarToRow (11) + : : : : : :- * ColumnarToRow (12) + : : : : : : +- CometFilter (11) : : : : : : +- CometScan parquet spark_catalog.default.item (10) : : : : : +- BroadcastExchange (25) : : : : : +- * Project (24) : : : : : +- * BroadcastHashJoin Inner BuildRight (23) : : : : : :- * Project (21) : : : : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : : : : :- * Filter (15) - : : : : : : : +- * ColumnarToRow (14) + : : : : : : :- * ColumnarToRow (15) + : : : : : : : +- CometFilter (14) : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (13) : : : : : : +- BroadcastExchange (19) - : : : : : : +- * Filter (18) - : : : : : : +- * ColumnarToRow (17) + : : : : : : +- * ColumnarToRow (18) + : : : : : : +- CometFilter (17) : : : : : : +- CometScan parquet spark_catalog.default.item (16) : : : : : +- ReusedExchange (22) : : : : +- ReusedExchange (30) @@ -55,15 +55,15 @@ TakeOrderedAndProject (84) : : : +- * BroadcastHashJoin Inner BuildRight (43) : : : :- * Project (41) : : : : +- * BroadcastHashJoin Inner BuildRight (40) - : : : : :- * Filter (38) - : : : : : +- * ColumnarToRow (37) + : : : : :- * ColumnarToRow (38) + : : : : : +- CometFilter (37) : : : : : +- CometScan parquet spark_catalog.default.web_sales (36) : : : : +- ReusedExchange (39) : : : +- ReusedExchange (42) : : +- BroadcastExchange (57) : : +- * BroadcastHashJoin LeftSemi BuildRight (56) - : : :- * Filter (54) - : : : +- * ColumnarToRow (53) + : : :- * ColumnarToRow (54) + : : : +- CometFilter (53) : : : +- CometScan parquet spark_catalog.default.item (52) : : +- ReusedExchange (55) : +- ReusedExchange (60) @@ -77,8 +77,8 @@ TakeOrderedAndProject (84) :- * Project (74) : +- * BroadcastHashJoin Inner BuildRight (73) : :- * BroadcastHashJoin LeftSemi BuildRight (71) - : : :- * Filter (69) - : : : +- * ColumnarToRow (68) + : : :- * ColumnarToRow (69) + : : : +- CometFilter (68) : : : +- CometScan parquet spark_catalog.default.store_sales (67) : : +- ReusedExchange (70) : +- ReusedExchange (72) @@ -93,12 +93,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 25] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Condition : isnotnull(ss_item_sk#1) -(3) Filter [codegen id : 25] +(3) ColumnarToRow [codegen id : 25] Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_item_sk#1) (4) Scan parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] @@ -107,12 +107,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 11] +(5) CometFilter Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] +Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) -(6) Filter [codegen id : 11] +(6) ColumnarToRow [codegen id : 11] Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) (7) Scan parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#10, ss_sold_date_sk#11] @@ -122,12 +122,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 6] +(8) CometFilter Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] +Condition : isnotnull(ss_item_sk#10) -(9) Filter [codegen id : 6] +(9) ColumnarToRow [codegen id : 6] Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Condition : isnotnull(ss_item_sk#10) (10) Scan parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] @@ -136,12 +136,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 4] +(11) CometFilter Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16)) -(12) Filter [codegen id : 4] +(12) ColumnarToRow [codegen id : 4] Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16)) (13) Scan parquet spark_catalog.default.catalog_sales Output [2]: [cs_item_sk#17, cs_sold_date_sk#18] @@ -151,12 +151,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] +(14) CometFilter Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] +Condition : isnotnull(cs_item_sk#17) -(15) Filter [codegen id : 3] +(15) ColumnarToRow [codegen id : 3] Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Condition : isnotnull(cs_item_sk#17) (16) Scan parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] @@ -165,12 +165,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 1] +(17) CometFilter Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Condition : isnotnull(i_item_sk#20) -(18) Filter [codegen id : 1] +(18) ColumnarToRow [codegen id : 1] Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Condition : isnotnull(i_item_sk#20) (19) BroadcastExchange Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] @@ -262,12 +262,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#30), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 9] +(37) CometFilter Input [2]: [ws_item_sk#29, ws_sold_date_sk#30] +Condition : isnotnull(ws_item_sk#29) -(38) Filter [codegen id : 9] +(38) ColumnarToRow [codegen id : 9] Input [2]: [ws_item_sk#29, ws_sold_date_sk#30] -Condition : isnotnull(ws_item_sk#29) (39) ReusedExchange [Reuses operator id: 19] Output [4]: [i_item_sk#32, i_brand_id#33, i_class_id#34, i_category_id#35] @@ -336,12 +336,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(53) ColumnarToRow [codegen id : 23] +(53) CometFilter Input [4]: [i_item_sk#38, i_brand_id#39, i_class_id#40, i_category_id#41] +Condition : (((isnotnull(i_item_sk#38) AND isnotnull(i_brand_id#39)) AND isnotnull(i_class_id#40)) AND isnotnull(i_category_id#41)) -(54) Filter [codegen id : 23] +(54) ColumnarToRow [codegen id : 23] Input [4]: [i_item_sk#38, i_brand_id#39, i_class_id#40, i_category_id#41] -Condition : (((isnotnull(i_item_sk#38) AND isnotnull(i_brand_id#39)) AND isnotnull(i_class_id#40)) AND isnotnull(i_category_id#41)) (55) ReusedExchange [Reuses operator id: 50] Output [1]: [ss_item_sk#37] @@ -409,12 +409,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#59), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(68) ColumnarToRow [codegen id : 50] +(68) CometFilter Input [4]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59] +Condition : isnotnull(ss_item_sk#56) -(69) Filter [codegen id : 50] +(69) ColumnarToRow [codegen id : 50] Input [4]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59] -Condition : isnotnull(ss_item_sk#56) (70) ReusedExchange [Reuses operator id: 50] Output [1]: [ss_item_sk#61] @@ -608,9 +608,9 @@ Subquery:4 Hosting operator id = 95 Hosting Expression = ws_sold_date_sk#94 IN d Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 BroadcastExchange (108) -+- * Project (107) - +- * Filter (106) - +- * ColumnarToRow (105) ++- * ColumnarToRow (107) + +- CometProject (106) + +- CometFilter (105) +- CometScan parquet spark_catalog.default.date_dim (104) @@ -621,27 +621,27 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#106), IsNotNull(d_date_sk)] ReadSchema: struct -(105) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#42, d_week_seq#105] - -(106) Filter [codegen id : 1] +(105) CometFilter Input [2]: [d_date_sk#42, d_week_seq#105] Condition : ((isnotnull(d_week_seq#105) AND (d_week_seq#105 = ReusedSubquery Subquery scalar-subquery#106, [id=#107])) AND isnotnull(d_date_sk#42)) -(107) Project [codegen id : 1] -Output [1]: [d_date_sk#42] +(106) CometProject Input [2]: [d_date_sk#42, d_week_seq#105] +Arguments: [d_date_sk#42], [d_date_sk#42] + +(107) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#42] (108) BroadcastExchange Input [1]: [d_date_sk#42] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] -Subquery:6 Hosting operator id = 106 Hosting Expression = ReusedSubquery Subquery scalar-subquery#106, [id=#107] +Subquery:6 Hosting operator id = 105 Hosting Expression = ReusedSubquery Subquery scalar-subquery#106, [id=#107] Subquery:7 Hosting operator id = 104 Hosting Expression = Subquery scalar-subquery#106, [id=#107] -* Project (112) -+- * Filter (111) - +- * ColumnarToRow (110) +* ColumnarToRow (112) ++- CometProject (111) + +- CometFilter (110) +- CometScan parquet spark_catalog.default.date_dim (109) @@ -652,22 +652,22 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,16)] ReadSchema: struct -(110) ColumnarToRow [codegen id : 1] -Input [4]: [d_week_seq#108, d_year#109, d_moy#110, d_dom#111] - -(111) Filter [codegen id : 1] +(110) CometFilter Input [4]: [d_week_seq#108, d_year#109, d_moy#110, d_dom#111] Condition : (((((isnotnull(d_year#109) AND isnotnull(d_moy#110)) AND isnotnull(d_dom#111)) AND (d_year#109 = 1999)) AND (d_moy#110 = 12)) AND (d_dom#111 = 16)) -(112) Project [codegen id : 1] -Output [1]: [d_week_seq#108] +(111) CometProject Input [4]: [d_week_seq#108, d_year#109, d_moy#110, d_dom#111] +Arguments: [d_week_seq#108], [d_week_seq#108] + +(112) ColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#108] Subquery:8 Hosting operator id = 7 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12 BroadcastExchange (117) -+- * Project (116) - +- * Filter (115) - +- * ColumnarToRow (114) ++- * ColumnarToRow (116) + +- CometProject (115) + +- CometFilter (114) +- CometScan parquet spark_catalog.default.date_dim (113) @@ -678,16 +678,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(114) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#25, d_year#112] - -(115) Filter [codegen id : 1] +(114) CometFilter Input [2]: [d_date_sk#25, d_year#112] Condition : (((isnotnull(d_year#112) AND (d_year#112 >= 1998)) AND (d_year#112 <= 2000)) AND isnotnull(d_date_sk#25)) -(116) Project [codegen id : 1] -Output [1]: [d_date_sk#25] +(115) CometProject Input [2]: [d_date_sk#25, d_year#112] +Arguments: [d_date_sk#25], [d_date_sk#25] + +(116) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#25] (117) BroadcastExchange Input [1]: [d_date_sk#25] @@ -701,9 +701,9 @@ Subquery:11 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquer Subquery:12 Hosting operator id = 67 Hosting Expression = ss_sold_date_sk#59 IN dynamicpruning#60 BroadcastExchange (122) -+- * Project (121) - +- * Filter (120) - +- * ColumnarToRow (119) ++- * ColumnarToRow (121) + +- CometProject (120) + +- CometFilter (119) +- CometScan parquet spark_catalog.default.date_dim (118) @@ -714,27 +714,27 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#114), IsNotNull(d_date_sk)] ReadSchema: struct -(119) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#66, d_week_seq#113] - -(120) Filter [codegen id : 1] +(119) CometFilter Input [2]: [d_date_sk#66, d_week_seq#113] Condition : ((isnotnull(d_week_seq#113) AND (d_week_seq#113 = ReusedSubquery Subquery scalar-subquery#114, [id=#115])) AND isnotnull(d_date_sk#66)) -(121) Project [codegen id : 1] -Output [1]: [d_date_sk#66] +(120) CometProject Input [2]: [d_date_sk#66, d_week_seq#113] +Arguments: [d_date_sk#66], [d_date_sk#66] + +(121) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#66] (122) BroadcastExchange Input [1]: [d_date_sk#66] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=15] -Subquery:13 Hosting operator id = 120 Hosting Expression = ReusedSubquery Subquery scalar-subquery#114, [id=#115] +Subquery:13 Hosting operator id = 119 Hosting Expression = ReusedSubquery Subquery scalar-subquery#114, [id=#115] Subquery:14 Hosting operator id = 118 Hosting Expression = Subquery scalar-subquery#114, [id=#115] -* Project (126) -+- * Filter (125) - +- * ColumnarToRow (124) +* ColumnarToRow (126) ++- CometProject (125) + +- CometFilter (124) +- CometScan parquet spark_catalog.default.date_dim (123) @@ -745,15 +745,15 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1998), EqualTo(d_moy,12), EqualTo(d_dom,16)] ReadSchema: struct -(124) ColumnarToRow [codegen id : 1] -Input [4]: [d_week_seq#116, d_year#117, d_moy#118, d_dom#119] - -(125) Filter [codegen id : 1] +(124) CometFilter Input [4]: [d_week_seq#116, d_year#117, d_moy#118, d_dom#119] Condition : (((((isnotnull(d_year#117) AND isnotnull(d_moy#118)) AND isnotnull(d_dom#119)) AND (d_year#117 = 1998)) AND (d_moy#118 = 12)) AND (d_dom#119 = 16)) -(126) Project [codegen id : 1] -Output [1]: [d_week_seq#116] +(125) CometProject Input [4]: [d_week_seq#116, d_year#117, d_moy#118, d_dom#119] +Arguments: [d_week_seq#116], [d_week_seq#116] + +(126) ColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#116] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/simplified.txt index 64646b717..55aa823ab 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/simplified.txt @@ -48,34 +48,34 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] BroadcastHashJoin [ss_item_sk,i_item_sk] BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_week_seq,d_date_sk] - ReusedSubquery [d_week_seq] #2 - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_week_seq,d_date_sk] + ReusedSubquery [d_week_seq] #2 CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] Subquery #2 WholeStageCodegen (1) - Project [d_week_seq] - Filter [d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_year,d_moy,d_dom] CometScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] InputAdapter BroadcastExchange #3 WholeStageCodegen (11) Project [i_item_sk] BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - Filter [i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_brand_id,i_class_id,i_category_id] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] InputAdapter BroadcastExchange #4 @@ -90,25 +90,25 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #3 BroadcastExchange #6 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #7 WholeStageCodegen (4) BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] InputAdapter BroadcastExchange #8 @@ -117,17 +117,17 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #3 InputAdapter BroadcastExchange #9 WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] InputAdapter ReusedExchange [d_date_sk] #6 @@ -140,9 +140,9 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #3 InputAdapter @@ -153,9 +153,9 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ BroadcastExchange #11 WholeStageCodegen (23) BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] InputAdapter ReusedExchange [ss_item_sk] #3 @@ -176,25 +176,25 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] BroadcastHashJoin [ss_item_sk,i_item_sk] BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #5 BroadcastExchange #15 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_week_seq,d_date_sk] - ReusedSubquery [d_week_seq] #6 - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_week_seq,d_date_sk] + ReusedSubquery [d_week_seq] #6 CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] Subquery #6 WholeStageCodegen (1) - Project [d_week_seq] - Filter [d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_year,d_moy,d_dom] CometScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] InputAdapter ReusedExchange [ss_item_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/explain.txt index 4dd70ec51..d76059e01 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/explain.txt @@ -17,14 +17,14 @@ TakeOrderedAndProject (125) : : :- * Project (59) : : : +- * BroadcastHashJoin Inner BuildRight (58) : : : :- * BroadcastHashJoin LeftSemi BuildRight (51) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : : +- BroadcastExchange (50) : : : : +- * Project (49) : : : : +- * BroadcastHashJoin Inner BuildRight (48) - : : : : :- * Filter (6) - : : : : : +- * ColumnarToRow (5) + : : : : :- * ColumnarToRow (6) + : : : : : +- CometFilter (5) : : : : : +- CometScan parquet spark_catalog.default.item (4) : : : : +- BroadcastExchange (47) : : : : +- * BroadcastHashJoin LeftSemi BuildRight (46) @@ -35,25 +35,25 @@ TakeOrderedAndProject (125) : : : : : +- * BroadcastHashJoin Inner BuildRight (31) : : : : : :- * Project (29) : : : : : : +- * BroadcastHashJoin Inner BuildRight (28) - : : : : : : :- * Filter (9) - : : : : : : : +- * ColumnarToRow (8) + : : : : : : :- * ColumnarToRow (9) + : : : : : : : +- CometFilter (8) : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (7) : : : : : : +- BroadcastExchange (27) : : : : : : +- * BroadcastHashJoin LeftSemi BuildRight (26) - : : : : : : :- * Filter (12) - : : : : : : : +- * ColumnarToRow (11) + : : : : : : :- * ColumnarToRow (12) + : : : : : : : +- CometFilter (11) : : : : : : : +- CometScan parquet spark_catalog.default.item (10) : : : : : : +- BroadcastExchange (25) : : : : : : +- * Project (24) : : : : : : +- * BroadcastHashJoin Inner BuildRight (23) : : : : : : :- * Project (21) : : : : : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : : : : : :- * Filter (15) - : : : : : : : : +- * ColumnarToRow (14) + : : : : : : : :- * ColumnarToRow (15) + : : : : : : : : +- CometFilter (14) : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (13) : : : : : : : +- BroadcastExchange (19) - : : : : : : : +- * Filter (18) - : : : : : : : +- * ColumnarToRow (17) + : : : : : : : +- * ColumnarToRow (18) + : : : : : : : +- CometFilter (17) : : : : : : : +- CometScan parquet spark_catalog.default.item (16) : : : : : : +- ReusedExchange (22) : : : : : +- ReusedExchange (30) @@ -62,15 +62,15 @@ TakeOrderedAndProject (125) : : : : +- * BroadcastHashJoin Inner BuildRight (43) : : : : :- * Project (41) : : : : : +- * BroadcastHashJoin Inner BuildRight (40) - : : : : : :- * Filter (38) - : : : : : : +- * ColumnarToRow (37) + : : : : : :- * ColumnarToRow (38) + : : : : : : +- CometFilter (37) : : : : : : +- CometScan parquet spark_catalog.default.web_sales (36) : : : : : +- ReusedExchange (39) : : : : +- ReusedExchange (42) : : : +- BroadcastExchange (57) : : : +- * BroadcastHashJoin LeftSemi BuildRight (56) - : : : :- * Filter (54) - : : : : +- * ColumnarToRow (53) + : : : :- * ColumnarToRow (54) + : : : : +- CometFilter (53) : : : : +- CometScan parquet spark_catalog.default.item (52) : : : +- ReusedExchange (55) : : +- ReusedExchange (60) @@ -83,8 +83,8 @@ TakeOrderedAndProject (125) : : :- * Project (74) : : : +- * BroadcastHashJoin Inner BuildRight (73) : : : :- * BroadcastHashJoin LeftSemi BuildRight (71) - : : : : :- * Filter (69) - : : : : : +- * ColumnarToRow (68) + : : : : :- * ColumnarToRow (69) + : : : : : +- CometFilter (68) : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (67) : : : : +- ReusedExchange (70) : : : +- ReusedExchange (72) @@ -98,8 +98,8 @@ TakeOrderedAndProject (125) : :- * Project (89) : : +- * BroadcastHashJoin Inner BuildRight (88) : : :- * BroadcastHashJoin LeftSemi BuildRight (86) - : : : :- * Filter (84) - : : : : +- * ColumnarToRow (83) + : : : :- * ColumnarToRow (84) + : : : : +- CometFilter (83) : : : : +- CometScan parquet spark_catalog.default.web_sales (82) : : : +- ReusedExchange (85) : : +- ReusedExchange (87) @@ -134,12 +134,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 25] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Condition : isnotnull(ss_item_sk#1) -(3) Filter [codegen id : 25] +(3) ColumnarToRow [codegen id : 25] Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_item_sk#1) (4) Scan parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] @@ -148,12 +148,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 11] +(5) CometFilter Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] +Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) -(6) Filter [codegen id : 11] +(6) ColumnarToRow [codegen id : 11] Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) (7) Scan parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#10, ss_sold_date_sk#11] @@ -163,12 +163,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 6] +(8) CometFilter Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] +Condition : isnotnull(ss_item_sk#10) -(9) Filter [codegen id : 6] +(9) ColumnarToRow [codegen id : 6] Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Condition : isnotnull(ss_item_sk#10) (10) Scan parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] @@ -177,12 +177,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 4] +(11) CometFilter Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16)) -(12) Filter [codegen id : 4] +(12) ColumnarToRow [codegen id : 4] Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16)) (13) Scan parquet spark_catalog.default.catalog_sales Output [2]: [cs_item_sk#17, cs_sold_date_sk#18] @@ -192,12 +192,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] +(14) CometFilter Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] +Condition : isnotnull(cs_item_sk#17) -(15) Filter [codegen id : 3] +(15) ColumnarToRow [codegen id : 3] Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Condition : isnotnull(cs_item_sk#17) (16) Scan parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] @@ -206,12 +206,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 1] +(17) CometFilter Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Condition : isnotnull(i_item_sk#20) -(18) Filter [codegen id : 1] +(18) ColumnarToRow [codegen id : 1] Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Condition : isnotnull(i_item_sk#20) (19) BroadcastExchange Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] @@ -303,12 +303,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#30), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 9] +(37) CometFilter Input [2]: [ws_item_sk#29, ws_sold_date_sk#30] +Condition : isnotnull(ws_item_sk#29) -(38) Filter [codegen id : 9] +(38) ColumnarToRow [codegen id : 9] Input [2]: [ws_item_sk#29, ws_sold_date_sk#30] -Condition : isnotnull(ws_item_sk#29) (39) ReusedExchange [Reuses operator id: 19] Output [4]: [i_item_sk#32, i_brand_id#33, i_class_id#34, i_category_id#35] @@ -377,12 +377,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(53) ColumnarToRow [codegen id : 23] +(53) CometFilter Input [4]: [i_item_sk#38, i_brand_id#39, i_class_id#40, i_category_id#41] +Condition : isnotnull(i_item_sk#38) -(54) Filter [codegen id : 23] +(54) ColumnarToRow [codegen id : 23] Input [4]: [i_item_sk#38, i_brand_id#39, i_class_id#40, i_category_id#41] -Condition : isnotnull(i_item_sk#38) (55) ReusedExchange [Reuses operator id: 50] Output [1]: [ss_item_sk#37] @@ -450,12 +450,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#59), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(68) ColumnarToRow [codegen id : 51] +(68) CometFilter Input [4]: [cs_item_sk#56, cs_quantity#57, cs_list_price#58, cs_sold_date_sk#59] +Condition : isnotnull(cs_item_sk#56) -(69) Filter [codegen id : 51] +(69) ColumnarToRow [codegen id : 51] Input [4]: [cs_item_sk#56, cs_quantity#57, cs_list_price#58, cs_sold_date_sk#59] -Condition : isnotnull(cs_item_sk#56) (70) ReusedExchange [Reuses operator id: 50] Output [1]: [ss_item_sk#61] @@ -522,12 +522,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#81), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(83) ColumnarToRow [codegen id : 77] +(83) CometFilter Input [4]: [ws_item_sk#78, ws_quantity#79, ws_list_price#80, ws_sold_date_sk#81] +Condition : isnotnull(ws_item_sk#78) -(84) Filter [codegen id : 77] +(84) ColumnarToRow [codegen id : 77] Input [4]: [ws_item_sk#78, ws_quantity#79, ws_list_price#80, ws_sold_date_sk#81] -Condition : isnotnull(ws_item_sk#78) (85) ReusedExchange [Reuses operator id: 50] Output [1]: [ss_item_sk#83] @@ -859,9 +859,9 @@ Subquery:2 Hosting operator id = 126 Hosting Expression = ss_sold_date_sk#206 IN Subquery:3 Hosting operator id = 131 Hosting Expression = cs_sold_date_sk#213 IN dynamicpruning#214 BroadcastExchange (149) -+- * Project (148) - +- * Filter (147) - +- * ColumnarToRow (146) ++- * ColumnarToRow (148) + +- CometProject (147) + +- CometFilter (146) +- CometScan parquet spark_catalog.default.date_dim (145) @@ -872,16 +872,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(146) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#215, d_year#231] - -(147) Filter [codegen id : 1] +(146) CometFilter Input [2]: [d_date_sk#215, d_year#231] Condition : (((isnotnull(d_year#231) AND (d_year#231 >= 1998)) AND (d_year#231 <= 2000)) AND isnotnull(d_date_sk#215)) -(148) Project [codegen id : 1] -Output [1]: [d_date_sk#215] +(147) CometProject Input [2]: [d_date_sk#215, d_year#231] +Arguments: [d_date_sk#215], [d_date_sk#215] + +(148) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#215] (149) BroadcastExchange Input [1]: [d_date_sk#215] @@ -891,9 +891,9 @@ Subquery:4 Hosting operator id = 136 Hosting Expression = ws_sold_date_sk#220 IN Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 BroadcastExchange (154) -+- * Project (153) - +- * Filter (152) - +- * ColumnarToRow (151) ++- * ColumnarToRow (153) + +- CometProject (152) + +- CometFilter (151) +- CometScan parquet spark_catalog.default.date_dim (150) @@ -904,16 +904,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct -(151) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#42, d_year#232, d_moy#233] - -(152) Filter [codegen id : 1] +(151) CometFilter Input [3]: [d_date_sk#42, d_year#232, d_moy#233] Condition : ((((isnotnull(d_year#232) AND isnotnull(d_moy#233)) AND (d_year#232 = 2000)) AND (d_moy#233 = 11)) AND isnotnull(d_date_sk#42)) -(153) Project [codegen id : 1] -Output [1]: [d_date_sk#42] +(152) CometProject Input [3]: [d_date_sk#42, d_year#232, d_moy#233] +Arguments: [d_date_sk#42], [d_date_sk#42] + +(153) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#42] (154) BroadcastExchange Input [1]: [d_date_sk#42] @@ -921,9 +921,9 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Subquery:6 Hosting operator id = 7 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12 BroadcastExchange (159) -+- * Project (158) - +- * Filter (157) - +- * ColumnarToRow (156) ++- * ColumnarToRow (158) + +- CometProject (157) + +- CometFilter (156) +- CometScan parquet spark_catalog.default.date_dim (155) @@ -934,16 +934,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(156) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#25, d_year#234] - -(157) Filter [codegen id : 1] +(156) CometFilter Input [2]: [d_date_sk#25, d_year#234] Condition : (((isnotnull(d_year#234) AND (d_year#234 >= 1999)) AND (d_year#234 <= 2001)) AND isnotnull(d_date_sk#25)) -(158) Project [codegen id : 1] -Output [1]: [d_date_sk#25] +(157) CometProject Input [2]: [d_date_sk#25, d_year#234] +Arguments: [d_date_sk#25], [d_date_sk#25] + +(158) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#25] (159) BroadcastExchange Input [1]: [d_date_sk#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/simplified.txt index 7284dab53..a203f9620 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/simplified.txt @@ -44,10 +44,10 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,num SubqueryBroadcast [d_date_sk] #4 BroadcastExchange #15 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [d_date_sk] #15 @@ -70,26 +70,26 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,num Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] BroadcastHashJoin [ss_item_sk,i_item_sk] BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_moy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #5 WholeStageCodegen (11) Project [i_item_sk] BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - Filter [i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_brand_id,i_class_id,i_category_id] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] InputAdapter BroadcastExchange #6 @@ -104,25 +104,25 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,num BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #9 WholeStageCodegen (4) BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] InputAdapter BroadcastExchange #10 @@ -131,17 +131,17 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,num BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #2 InputAdapter BroadcastExchange #11 WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] InputAdapter ReusedExchange [d_date_sk] #8 @@ -154,9 +154,9 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,num BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 InputAdapter @@ -167,9 +167,9 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,num BroadcastExchange #13 WholeStageCodegen (23) BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] InputAdapter ReusedExchange [ss_item_sk] #5 @@ -188,9 +188,9 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,num Project [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] BroadcastHashJoin [cs_item_sk,i_item_sk] BroadcastHashJoin [cs_item_sk,ss_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -212,9 +212,9 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,num Project [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] BroadcastHashJoin [ws_item_sk,i_item_sk] BroadcastHashJoin [ws_item_sk,ss_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/explain.txt index 908089cc5..8bcc0c6ba 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/explain.txt @@ -16,31 +16,31 @@ TakeOrderedAndProject (153) : : : : : +- * BroadcastHashJoin Inner BuildRight (16) : : : : : :- * Project (10) : : : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) + : : : : : : :- * ColumnarToRow (3) + : : : : : : : +- CometFilter (2) : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : : : : : : +- BroadcastExchange (8) - : : : : : : +- * Project (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) + : : : : : : +- * ColumnarToRow (7) + : : : : : : +- CometProject (6) + : : : : : : +- CometFilter (5) : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (4) : : : : : +- BroadcastExchange (15) - : : : : : +- * Project (14) - : : : : : +- * Filter (13) - : : : : : +- * ColumnarToRow (12) + : : : : : +- * ColumnarToRow (14) + : : : : : +- CometProject (13) + : : : : : +- CometFilter (12) : : : : : +- CometScan parquet spark_catalog.default.customer (11) : : : : +- BroadcastExchange (21) - : : : : +- * Filter (20) - : : : : +- * ColumnarToRow (19) + : : : : +- * ColumnarToRow (20) + : : : : +- CometFilter (19) : : : : +- CometScan parquet spark_catalog.default.customer_demographics (18) : : : +- BroadcastExchange (27) - : : : +- * Filter (26) - : : : +- * ColumnarToRow (25) + : : : +- * ColumnarToRow (26) + : : : +- CometFilter (25) : : : +- CometScan parquet spark_catalog.default.customer_address (24) : : +- ReusedExchange (30) : +- BroadcastExchange (36) - : +- * Filter (35) - : +- * ColumnarToRow (34) + : +- * ColumnarToRow (35) + : +- CometFilter (34) : +- CometScan parquet spark_catalog.default.item (33) :- * HashAggregate (68) : +- Exchange (67) @@ -57,15 +57,15 @@ TakeOrderedAndProject (153) : : : : : +- * BroadcastHashJoin Inner BuildRight (49) : : : : : :- * Project (47) : : : : : : +- * BroadcastHashJoin Inner BuildRight (46) - : : : : : : :- * Filter (44) - : : : : : : : +- * ColumnarToRow (43) + : : : : : : :- * ColumnarToRow (44) + : : : : : : : +- CometFilter (43) : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (42) : : : : : : +- ReusedExchange (45) : : : : : +- ReusedExchange (48) : : : : +- ReusedExchange (51) : : : +- BroadcastExchange (57) - : : : +- * Filter (56) - : : : +- * ColumnarToRow (55) + : : : +- * ColumnarToRow (56) + : : : +- CometFilter (55) : : : +- CometScan parquet spark_catalog.default.customer_address (54) : : +- ReusedExchange (60) : +- ReusedExchange (63) @@ -84,16 +84,16 @@ TakeOrderedAndProject (153) : : : : : +- * BroadcastHashJoin Inner BuildRight (76) : : : : : :- * Project (74) : : : : : : +- * BroadcastHashJoin Inner BuildRight (73) - : : : : : : :- * Filter (71) - : : : : : : : +- * ColumnarToRow (70) + : : : : : : :- * ColumnarToRow (71) + : : : : : : : +- CometFilter (70) : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (69) : : : : : : +- ReusedExchange (72) : : : : : +- ReusedExchange (75) : : : : +- ReusedExchange (78) : : : +- BroadcastExchange (85) - : : : +- * Project (84) - : : : +- * Filter (83) - : : : +- * ColumnarToRow (82) + : : : +- * ColumnarToRow (84) + : : : +- CometProject (83) + : : : +- CometFilter (82) : : : +- CometScan parquet spark_catalog.default.customer_address (81) : : +- ReusedExchange (88) : +- ReusedExchange (91) @@ -112,16 +112,16 @@ TakeOrderedAndProject (153) : : : : : +- * BroadcastHashJoin Inner BuildRight (104) : : : : : :- * Project (102) : : : : : : +- * BroadcastHashJoin Inner BuildRight (101) - : : : : : : :- * Filter (99) - : : : : : : : +- * ColumnarToRow (98) + : : : : : : :- * ColumnarToRow (99) + : : : : : : : +- CometFilter (98) : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (97) : : : : : : +- ReusedExchange (100) : : : : : +- ReusedExchange (103) : : : : +- ReusedExchange (106) : : : +- BroadcastExchange (113) - : : : +- * Project (112) - : : : +- * Filter (111) - : : : +- * ColumnarToRow (110) + : : : +- * ColumnarToRow (112) + : : : +- CometProject (111) + : : : +- CometFilter (110) : : : +- CometScan parquet spark_catalog.default.customer_address (109) : : +- ReusedExchange (116) : +- ReusedExchange (119) @@ -140,8 +140,8 @@ TakeOrderedAndProject (153) : : : : +- * BroadcastHashJoin Inner BuildRight (132) : : : : :- * Project (130) : : : : : +- * BroadcastHashJoin Inner BuildRight (129) - : : : : : :- * Filter (127) - : : : : : : +- * ColumnarToRow (126) + : : : : : :- * ColumnarToRow (127) + : : : : : : +- CometFilter (126) : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (125) : : : : : +- ReusedExchange (128) : : : : +- ReusedExchange (131) @@ -149,8 +149,8 @@ TakeOrderedAndProject (153) : : +- ReusedExchange (137) : +- ReusedExchange (140) +- BroadcastExchange (146) - +- * Filter (145) - +- * ColumnarToRow (144) + +- * ColumnarToRow (145) + +- CometFilter (144) +- CometScan parquet spark_catalog.default.item (143) @@ -162,12 +162,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#9), dynamicpruningexpression(cs_sol PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 7] +(2) CometFilter Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) -(3) Filter [codegen id : 7] +(3) ColumnarToRow [codegen id : 7] Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) (4) Scan parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] @@ -176,16 +176,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_education_status,College ), IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] - -(6) Filter [codegen id : 1] +(5) CometFilter Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] Condition : ((((isnotnull(cd_gender#12) AND isnotnull(cd_education_status#13)) AND (cd_gender#12 = M)) AND (cd_education_status#13 = College )) AND isnotnull(cd_demo_sk#11)) -(7) Project [codegen id : 1] -Output [2]: [cd_demo_sk#11, cd_dep_count#14] +(6) CometProject Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] +Arguments: [cd_demo_sk#11, cd_dep_count#14], [cd_demo_sk#11, cd_dep_count#14] + +(7) ColumnarToRow [codegen id : 1] +Input [2]: [cd_demo_sk#11, cd_dep_count#14] (8) BroadcastExchange Input [2]: [cd_demo_sk#11, cd_dep_count#14] @@ -208,16 +208,16 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [In(c_birth_month, [1,10,12,4,5,9]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] - -(13) Filter [codegen id : 2] +(12) CometFilter Input [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] Condition : (((c_birth_month#18 IN (9,5,12,4,1,10) AND isnotnull(c_customer_sk#15)) AND isnotnull(c_current_cdemo_sk#16)) AND isnotnull(c_current_addr_sk#17)) -(14) Project [codegen id : 2] -Output [4]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] +(13) CometProject Input [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] +Arguments: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19], [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] + +(14) ColumnarToRow [codegen id : 2] +Input [4]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] (15) BroadcastExchange Input [4]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] @@ -240,12 +240,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] +(19) CometFilter Input [1]: [cd_demo_sk#20] +Condition : isnotnull(cd_demo_sk#20) -(20) Filter [codegen id : 3] +(20) ColumnarToRow [codegen id : 3] Input [1]: [cd_demo_sk#20] -Condition : isnotnull(cd_demo_sk#20) (21) BroadcastExchange Input [1]: [cd_demo_sk#20] @@ -268,12 +268,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 4] +(25) CometFilter Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] +Condition : (ca_state#23 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#21)) -(26) Filter [codegen id : 4] +(26) ColumnarToRow [codegen id : 4] Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] -Condition : (ca_state#23 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#21)) (27) BroadcastExchange Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] @@ -309,12 +309,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 6] +(34) CometFilter Input [2]: [i_item_sk#26, i_item_id#27] +Condition : isnotnull(i_item_sk#26) -(35) Filter [codegen id : 6] +(35) ColumnarToRow [codegen id : 6] Input [2]: [i_item_sk#26, i_item_id#27] -Condition : isnotnull(i_item_sk#26) (36) BroadcastExchange Input [2]: [i_item_sk#26, i_item_id#27] @@ -356,12 +356,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#85), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(43) ColumnarToRow [codegen id : 15] +(43) CometFilter Input [9]: [cs_bill_customer_sk#77, cs_bill_cdemo_sk#78, cs_item_sk#79, cs_quantity#80, cs_list_price#81, cs_sales_price#82, cs_coupon_amt#83, cs_net_profit#84, cs_sold_date_sk#85] +Condition : ((isnotnull(cs_bill_cdemo_sk#78) AND isnotnull(cs_bill_customer_sk#77)) AND isnotnull(cs_item_sk#79)) -(44) Filter [codegen id : 15] +(44) ColumnarToRow [codegen id : 15] Input [9]: [cs_bill_customer_sk#77, cs_bill_cdemo_sk#78, cs_item_sk#79, cs_quantity#80, cs_list_price#81, cs_sales_price#82, cs_coupon_amt#83, cs_net_profit#84, cs_sold_date_sk#85] -Condition : ((isnotnull(cs_bill_cdemo_sk#78) AND isnotnull(cs_bill_customer_sk#77)) AND isnotnull(cs_item_sk#79)) (45) ReusedExchange [Reuses operator id: 8] Output [2]: [cd_demo_sk#87, cd_dep_count#88] @@ -409,12 +409,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] ReadSchema: struct -(55) ColumnarToRow [codegen id : 12] +(55) CometFilter Input [3]: [ca_address_sk#94, ca_state#95, ca_country#96] +Condition : (ca_state#95 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#94)) -(56) Filter [codegen id : 12] +(56) ColumnarToRow [codegen id : 12] Input [3]: [ca_address_sk#94, ca_state#95, ca_country#96] -Condition : (ca_state#95 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#94)) (57) BroadcastExchange Input [3]: [ca_address_sk#94, ca_state#95, ca_country#96] @@ -482,12 +482,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#158), dynamicpruningexpression(cs_s PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(70) ColumnarToRow [codegen id : 23] +(70) CometFilter Input [9]: [cs_bill_customer_sk#150, cs_bill_cdemo_sk#151, cs_item_sk#152, cs_quantity#153, cs_list_price#154, cs_sales_price#155, cs_coupon_amt#156, cs_net_profit#157, cs_sold_date_sk#158] +Condition : ((isnotnull(cs_bill_cdemo_sk#151) AND isnotnull(cs_bill_customer_sk#150)) AND isnotnull(cs_item_sk#152)) -(71) Filter [codegen id : 23] +(71) ColumnarToRow [codegen id : 23] Input [9]: [cs_bill_customer_sk#150, cs_bill_cdemo_sk#151, cs_item_sk#152, cs_quantity#153, cs_list_price#154, cs_sales_price#155, cs_coupon_amt#156, cs_net_profit#157, cs_sold_date_sk#158] -Condition : ((isnotnull(cs_bill_cdemo_sk#151) AND isnotnull(cs_bill_customer_sk#150)) AND isnotnull(cs_item_sk#152)) (72) ReusedExchange [Reuses operator id: 8] Output [2]: [cd_demo_sk#160, cd_dep_count#161] @@ -535,16 +535,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] ReadSchema: struct -(82) ColumnarToRow [codegen id : 20] -Input [3]: [ca_address_sk#167, ca_state#168, ca_country#169] - -(83) Filter [codegen id : 20] +(82) CometFilter Input [3]: [ca_address_sk#167, ca_state#168, ca_country#169] Condition : (ca_state#168 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#167)) -(84) Project [codegen id : 20] -Output [2]: [ca_address_sk#167, ca_country#169] +(83) CometProject Input [3]: [ca_address_sk#167, ca_state#168, ca_country#169] +Arguments: [ca_address_sk#167, ca_country#169], [ca_address_sk#167, ca_country#169] + +(84) ColumnarToRow [codegen id : 20] +Input [2]: [ca_address_sk#167, ca_country#169] (85) BroadcastExchange Input [2]: [ca_address_sk#167, ca_country#169] @@ -612,12 +612,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#232), dynamicpruningexpression(cs_s PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(98) ColumnarToRow [codegen id : 31] +(98) CometFilter Input [9]: [cs_bill_customer_sk#224, cs_bill_cdemo_sk#225, cs_item_sk#226, cs_quantity#227, cs_list_price#228, cs_sales_price#229, cs_coupon_amt#230, cs_net_profit#231, cs_sold_date_sk#232] +Condition : ((isnotnull(cs_bill_cdemo_sk#225) AND isnotnull(cs_bill_customer_sk#224)) AND isnotnull(cs_item_sk#226)) -(99) Filter [codegen id : 31] +(99) ColumnarToRow [codegen id : 31] Input [9]: [cs_bill_customer_sk#224, cs_bill_cdemo_sk#225, cs_item_sk#226, cs_quantity#227, cs_list_price#228, cs_sales_price#229, cs_coupon_amt#230, cs_net_profit#231, cs_sold_date_sk#232] -Condition : ((isnotnull(cs_bill_cdemo_sk#225) AND isnotnull(cs_bill_customer_sk#224)) AND isnotnull(cs_item_sk#226)) (100) ReusedExchange [Reuses operator id: 8] Output [2]: [cd_demo_sk#234, cd_dep_count#235] @@ -665,16 +665,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] ReadSchema: struct -(110) ColumnarToRow [codegen id : 28] -Input [2]: [ca_address_sk#241, ca_state#242] - -(111) Filter [codegen id : 28] +(110) CometFilter Input [2]: [ca_address_sk#241, ca_state#242] Condition : (ca_state#242 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#241)) -(112) Project [codegen id : 28] -Output [1]: [ca_address_sk#241] +(111) CometProject Input [2]: [ca_address_sk#241, ca_state#242] +Arguments: [ca_address_sk#241], [ca_address_sk#241] + +(112) ColumnarToRow [codegen id : 28] +Input [1]: [ca_address_sk#241] (113) BroadcastExchange Input [1]: [ca_address_sk#241] @@ -742,12 +742,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#306), dynamicpruningexpression(cs_s PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(126) ColumnarToRow [codegen id : 39] +(126) CometFilter Input [9]: [cs_bill_customer_sk#298, cs_bill_cdemo_sk#299, cs_item_sk#300, cs_quantity#301, cs_list_price#302, cs_sales_price#303, cs_coupon_amt#304, cs_net_profit#305, cs_sold_date_sk#306] +Condition : ((isnotnull(cs_bill_cdemo_sk#299) AND isnotnull(cs_bill_customer_sk#298)) AND isnotnull(cs_item_sk#300)) -(127) Filter [codegen id : 39] +(127) ColumnarToRow [codegen id : 39] Input [9]: [cs_bill_customer_sk#298, cs_bill_cdemo_sk#299, cs_item_sk#300, cs_quantity#301, cs_list_price#302, cs_sales_price#303, cs_coupon_amt#304, cs_net_profit#305, cs_sold_date_sk#306] -Condition : ((isnotnull(cs_bill_cdemo_sk#299) AND isnotnull(cs_bill_customer_sk#298)) AND isnotnull(cs_item_sk#300)) (128) ReusedExchange [Reuses operator id: 8] Output [2]: [cd_demo_sk#308, cd_dep_count#309] @@ -821,12 +821,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(144) ColumnarToRow [codegen id : 38] +(144) CometFilter Input [1]: [i_item_sk#317] +Condition : isnotnull(i_item_sk#317) -(145) Filter [codegen id : 38] +(145) ColumnarToRow [codegen id : 38] Input [1]: [i_item_sk#317] -Condition : isnotnull(i_item_sk#317) (146) BroadcastExchange Input [1]: [i_item_sk#317] @@ -870,9 +870,9 @@ Arguments: 100, [ca_country#24 ASC NULLS FIRST, ca_state#23 ASC NULLS FIRST, ca_ Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#9 IN dynamicpruning#10 BroadcastExchange (158) -+- * Project (157) - +- * Filter (156) - +- * ColumnarToRow (155) ++- * ColumnarToRow (157) + +- CometProject (156) + +- CometFilter (155) +- CometScan parquet spark_catalog.default.date_dim (154) @@ -883,16 +883,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(155) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#25, d_year#371] - -(156) Filter [codegen id : 1] +(155) CometFilter Input [2]: [d_date_sk#25, d_year#371] Condition : ((isnotnull(d_year#371) AND (d_year#371 = 2001)) AND isnotnull(d_date_sk#25)) -(157) Project [codegen id : 1] -Output [1]: [d_date_sk#25] +(156) CometProject Input [2]: [d_date_sk#25, d_year#371] +Arguments: [d_date_sk#25], [d_date_sk#25] + +(157) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#25] (158) BroadcastExchange Input [1]: [d_date_sk#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/simplified.txt index 369a6d222..f02809572 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/simplified.txt @@ -18,56 +18,56 @@ TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,ag BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Project [cd_demo_sk,cd_dep_count] - Filter [cd_gender,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cd_demo_sk,cd_dep_count] + CometFilter [cd_gender,cd_education_status,cd_demo_sk] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - Project [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - Filter [c_birth_month,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometFilter [c_birth_month,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk] InputAdapter BroadcastExchange #6 WholeStageCodegen (4) - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ca_state,ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] InputAdapter ReusedExchange [d_date_sk] #2 InputAdapter BroadcastExchange #7 WholeStageCodegen (6) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] WholeStageCodegen (16) HashAggregate [i_item_id,ca_country,ca_state,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] @@ -87,9 +87,9 @@ TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,ag BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -101,9 +101,9 @@ TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,ag InputAdapter BroadcastExchange #9 WholeStageCodegen (12) - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ca_state,ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] InputAdapter ReusedExchange [d_date_sk] #2 @@ -127,9 +127,9 @@ TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,ag BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -141,10 +141,10 @@ TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,ag InputAdapter BroadcastExchange #11 WholeStageCodegen (20) - Project [ca_address_sk,ca_country] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ca_address_sk,ca_country] + CometFilter [ca_state,ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] InputAdapter ReusedExchange [d_date_sk] #2 @@ -168,9 +168,9 @@ TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,ag BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -182,10 +182,10 @@ TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,ag InputAdapter BroadcastExchange #13 WholeStageCodegen (28) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_state,ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter ReusedExchange [d_date_sk] #2 @@ -209,9 +209,9 @@ TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,ag BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -227,7 +227,7 @@ TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,ag InputAdapter BroadcastExchange #15 WholeStageCodegen (38) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/explain.txt index 5c6c60771..1304af2e1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/explain.txt @@ -11,12 +11,12 @@ TakeOrderedAndProject (20) +- * BroadcastHashJoin Inner BuildRight (11) :- * Project (9) : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) + : :- * ColumnarToRow (3) + : : +- CometFilter (2) : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) + : +- * ColumnarToRow (6) + : +- CometFilter (5) : +- CometScan parquet spark_catalog.default.item (4) +- ReusedExchange (10) @@ -29,12 +29,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sol PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] +Condition : isnotnull(cs_item_sk#1) -(3) Filter [codegen id : 3] +(3) ColumnarToRow [codegen id : 3] Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Condition : isnotnull(cs_item_sk#1) (4) Scan parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] @@ -43,12 +43,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Condition : (i_category#10 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (i_category#10 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) (7) BroadcastExchange Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] @@ -119,9 +119,9 @@ Arguments: 100, [i_category#10 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_ite Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 BroadcastExchange (25) -+- * Project (24) - +- * Filter (23) - +- * ColumnarToRow (22) ++- * ColumnarToRow (24) + +- CometProject (23) + +- CometFilter (22) +- CometScan parquet spark_catalog.default.date_dim (21) @@ -132,16 +132,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(22) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#11, d_date#19] - -(23) Filter [codegen id : 1] +(22) CometFilter Input [2]: [d_date_sk#11, d_date#19] Condition : (((isnotnull(d_date#19) AND (d_date#19 >= 1999-02-22)) AND (d_date#19 <= 1999-03-24)) AND isnotnull(d_date_sk#11)) -(24) Project [codegen id : 1] -Output [1]: [d_date_sk#11] +(23) CometProject Input [2]: [d_date_sk#11, d_date#19] +Arguments: [d_date_sk#11], [d_date_sk#11] + +(24) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#11] (25) BroadcastExchange Input [1]: [d_date_sk#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/simplified.txt index b5d7b5cc8..2a2a392cd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/simplified.txt @@ -17,24 +17,24 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter BroadcastExchange #4 WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_category,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] InputAdapter ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/explain.txt index 5085b8ed3..fdebdc8a4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/explain.txt @@ -10,13 +10,13 @@ TakeOrderedAndProject (22) : +- * BroadcastHashJoin Inner BuildRight (11) : :- * Project (6) : : +- * BroadcastHashJoin Inner BuildRight (5) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) + : : :- * ColumnarToRow (3) + : : : +- CometFilter (2) : : : +- CometScan parquet spark_catalog.default.inventory (1) : : +- ReusedExchange (4) : +- BroadcastExchange (10) - : +- * Filter (9) - : +- * ColumnarToRow (8) + : +- * ColumnarToRow (9) + : +- CometFilter (8) : +- CometScan parquet spark_catalog.default.item (7) +- BroadcastExchange (15) +- * ColumnarToRow (14) @@ -31,12 +31,12 @@ PartitionFilters: [isnotnull(inv_date_sk#3), dynamicpruningexpression(inv_date_s PushedFilters: [IsNotNull(inv_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] +Condition : isnotnull(inv_item_sk#1) -(3) Filter [codegen id : 4] +(3) ColumnarToRow [codegen id : 4] Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] -Condition : isnotnull(inv_item_sk#1) (4) ReusedExchange [Reuses operator id: 27] Output [1]: [d_date_sk#5] @@ -58,12 +58,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] +(8) CometFilter Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] +Condition : isnotnull(i_item_sk#6) -(9) Filter [codegen id : 2] +(9) ColumnarToRow [codegen id : 2] Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] -Condition : isnotnull(i_item_sk#6) (10) BroadcastExchange Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] @@ -130,9 +130,9 @@ Arguments: 100, [qoh#21 ASC NULLS FIRST, i_product_name#11 ASC NULLS FIRST, i_br Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#3 IN dynamicpruning#4 BroadcastExchange (27) -+- * Project (26) - +- * Filter (25) - +- * ColumnarToRow (24) ++- * ColumnarToRow (26) + +- CometProject (25) + +- CometFilter (24) +- CometScan parquet spark_catalog.default.date_dim (23) @@ -143,16 +143,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_month_seq#22] - -(25) Filter [codegen id : 1] +(24) CometFilter Input [2]: [d_date_sk#5, d_month_seq#22] Condition : (((isnotnull(d_month_seq#22) AND (d_month_seq#22 >= 1200)) AND (d_month_seq#22 <= 1211)) AND isnotnull(d_date_sk#5)) -(26) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(25) CometProject Input [2]: [d_date_sk#5, d_month_seq#22] +Arguments: [d_date_sk#5], [d_date_sk#5] + +(26) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#5] (27) BroadcastExchange Input [1]: [d_date_sk#5] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/simplified.txt index 853aad173..63a428d4e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/simplified.txt @@ -12,26 +12,26 @@ TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] BroadcastHashJoin [inv_item_sk,i_item_sk] Project [inv_item_sk,inv_quantity_on_hand] BroadcastHashJoin [inv_date_sk,d_date_sk] - Filter [inv_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [inv_item_sk] CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_month_seq,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter ReusedExchange [d_date_sk] #2 InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] InputAdapter BroadcastExchange #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/explain.txt index 4f067d199..1e5f5c2f7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/explain.txt @@ -12,17 +12,17 @@ TakeOrderedAndProject (45) : : +- * BroadcastHashJoin Inner BuildRight (11) : : :- * Project (6) : : : +- * BroadcastHashJoin Inner BuildRight (5) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.inventory (1) : : : +- ReusedExchange (4) : : +- BroadcastExchange (10) - : : +- * Filter (9) - : : +- * ColumnarToRow (8) + : : +- * ColumnarToRow (9) + : : +- CometFilter (8) : : +- CometScan parquet spark_catalog.default.item (7) : +- BroadcastExchange (16) - : +- * Filter (15) - : +- * ColumnarToRow (14) + : +- * ColumnarToRow (15) + : +- CometFilter (14) : +- CometScan parquet spark_catalog.default.warehouse (13) :- * HashAggregate (28) : +- Exchange (27) @@ -54,12 +54,12 @@ PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_s PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) -(3) Filter [codegen id : 4] +(3) ColumnarToRow [codegen id : 4] Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) (4) ReusedExchange [Reuses operator id: 50] Output [1]: [d_date_sk#6] @@ -81,12 +81,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] +(8) CometFilter Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Condition : isnotnull(i_item_sk#7) -(9) Filter [codegen id : 2] +(9) ColumnarToRow [codegen id : 2] Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Condition : isnotnull(i_item_sk#7) (10) BroadcastExchange Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] @@ -109,12 +109,12 @@ Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] +(14) CometFilter Input [1]: [w_warehouse_sk#12] +Condition : isnotnull(w_warehouse_sk#12) -(15) Filter [codegen id : 3] +(15) ColumnarToRow [codegen id : 3] Input [1]: [w_warehouse_sk#12] -Condition : isnotnull(w_warehouse_sk#12) (16) BroadcastExchange Input [1]: [w_warehouse_sk#12] @@ -284,9 +284,9 @@ Arguments: 100, [qoh#28 ASC NULLS FIRST, i_product_name#24 ASC NULLS FIRST, i_br Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 BroadcastExchange (50) -+- * Project (49) - +- * Filter (48) - +- * ColumnarToRow (47) ++- * ColumnarToRow (49) + +- CometProject (48) + +- CometFilter (47) +- CometScan parquet spark_catalog.default.date_dim (46) @@ -297,16 +297,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(47) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#6, d_month_seq#95] - -(48) Filter [codegen id : 1] +(47) CometFilter Input [2]: [d_date_sk#6, d_month_seq#95] Condition : (((isnotnull(d_month_seq#95) AND (d_month_seq#95 >= 1212)) AND (d_month_seq#95 <= 1223)) AND isnotnull(d_date_sk#6)) -(49) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(48) CometProject Input [2]: [d_date_sk#6, d_month_seq#95] +Arguments: [d_date_sk#6], [d_date_sk#6] + +(49) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#6] (50) BroadcastExchange Input [1]: [d_date_sk#6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/simplified.txt index 0819dc055..3aa0745ad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/simplified.txt @@ -14,33 +14,33 @@ TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] BroadcastHashJoin [inv_item_sk,i_item_sk] Project [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] BroadcastHashJoin [inv_date_sk,d_date_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [inv_item_sk,inv_warehouse_sk] CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_month_seq,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter ReusedExchange [d_date_sk] #2 InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] InputAdapter BroadcastExchange #4 WholeStageCodegen (3) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [w_warehouse_sk] CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk] WholeStageCodegen (11) HashAggregate [i_product_name,i_brand,i_class,sum,count] [avg(qoh),i_category,qoh,sum,count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/explain.txt index f2d354a2d..0cba9b059 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/explain.txt @@ -20,32 +20,32 @@ : : : : +- * SortMergeJoin Inner (13) : : : : :- * Sort (6) : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : : +- * ColumnarToRow (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : : +- * Sort (12) : : : : +- Exchange (11) - : : : : +- * Project (10) - : : : : +- * Filter (9) - : : : : +- * ColumnarToRow (8) + : : : : +- * ColumnarToRow (10) + : : : : +- CometProject (9) + : : : : +- CometFilter (8) : : : : +- CometScan parquet spark_catalog.default.store_returns (7) : : : +- BroadcastExchange (19) - : : : +- * Project (18) - : : : +- * Filter (17) - : : : +- * ColumnarToRow (16) + : : : +- * ColumnarToRow (18) + : : : +- CometProject (17) + : : : +- CometFilter (16) : : : +- CometScan parquet spark_catalog.default.store (15) : : +- BroadcastExchange (25) - : : +- * Filter (24) - : : +- * ColumnarToRow (23) + : : +- * ColumnarToRow (24) + : : +- CometFilter (23) : : +- CometScan parquet spark_catalog.default.item (22) : +- BroadcastExchange (31) - : +- * Filter (30) - : +- * ColumnarToRow (29) + : +- * ColumnarToRow (30) + : +- CometFilter (29) : +- CometScan parquet spark_catalog.default.customer (28) +- BroadcastExchange (37) - +- * Filter (36) - +- * ColumnarToRow (35) + +- * ColumnarToRow (36) + +- CometFilter (35) +- CometScan parquet spark_catalog.default.customer_address (34) @@ -56,16 +56,16 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2)) -(4) Project [codegen id : 1] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +(3) CometProject Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] + +(4) ColumnarToRow [codegen id : 1] +Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] (5) Exchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] @@ -82,16 +82,16 @@ Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 3] -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(9) Filter [codegen id : 3] +(8) CometFilter Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) -(10) Project [codegen id : 3] -Output [2]: [sr_item_sk#7, sr_ticket_number#8] +(9) CometProject Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] + +(10) ColumnarToRow [codegen id : 3] +Input [2]: [sr_item_sk#7, sr_ticket_number#8] (11) Exchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] @@ -118,16 +118,16 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 5] -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] - -(17) Filter [codegen id : 5] +(16) CometFilter Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) -(18) Project [codegen id : 5] -Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +(17) CometProject Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14], [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] + +(18) ColumnarToRow [codegen id : 5] +Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] (19) BroadcastExchange Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] @@ -150,12 +150,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_color), EqualTo(i_color,pale ), IsNotNull(i_item_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 6] +(23) CometFilter Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +Condition : ((isnotnull(i_color#18) AND (i_color#18 = pale )) AND isnotnull(i_item_sk#15)) -(24) Filter [codegen id : 6] +(24) ColumnarToRow [codegen id : 6] Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = pale )) AND isnotnull(i_item_sk#15)) (25) BroadcastExchange Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] @@ -178,12 +178,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_birth_country)] ReadSchema: struct -(29) ColumnarToRow [codegen id : 7] +(29) CometFilter Input [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] +Condition : ((isnotnull(c_customer_sk#21) AND isnotnull(c_current_addr_sk#22)) AND isnotnull(c_birth_country#25)) -(30) Filter [codegen id : 7] +(30) ColumnarToRow [codegen id : 7] Input [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Condition : ((isnotnull(c_customer_sk#21) AND isnotnull(c_current_addr_sk#22)) AND isnotnull(c_birth_country#25)) (31) BroadcastExchange Input [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] @@ -206,12 +206,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_country), IsNotNull(ca_zip)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 8] +(35) CometFilter Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] +Condition : ((isnotnull(ca_address_sk#26) AND isnotnull(ca_country#29)) AND isnotnull(ca_zip#28)) -(36) Filter [codegen id : 8] +(36) ColumnarToRow [codegen id : 8] Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] -Condition : ((isnotnull(ca_address_sk#26) AND isnotnull(ca_country#29)) AND isnotnull(ca_zip#28)) (37) BroadcastExchange Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] @@ -300,8 +300,8 @@ Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquer : : : : +- ReusedExchange (51) : : : +- ReusedExchange (55) : : +- BroadcastExchange (61) - : : +- * Filter (60) - : : +- * ColumnarToRow (59) + : : +- * ColumnarToRow (60) + : : +- CometFilter (59) : : +- CometScan parquet spark_catalog.default.item (58) : +- ReusedExchange (64) +- ReusedExchange (67) @@ -351,12 +351,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(59) ColumnarToRow [codegen id : 6] +(59) CometFilter Input [6]: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] +Condition : isnotnull(i_item_sk#53) -(60) Filter [codegen id : 6] +(60) ColumnarToRow [codegen id : 6] Input [6]: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] -Condition : isnotnull(i_item_sk#53) (61) BroadcastExchange Input [6]: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/simplified.txt index 02c50e74d..7024f439f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/simplified.txt @@ -41,9 +41,9 @@ WholeStageCodegen (12) InputAdapter BroadcastExchange #12 WholeStageCodegen (6) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] InputAdapter ReusedExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 @@ -75,10 +75,10 @@ WholeStageCodegen (12) InputAdapter Exchange [ss_ticket_number,ss_item_sk] #4 WholeStageCodegen (1) - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] InputAdapter WholeStageCodegen (4) @@ -86,37 +86,37 @@ WholeStageCodegen (12) InputAdapter Exchange [sr_ticket_number,sr_item_sk] #5 WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_ticket_number,sr_item_sk] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] InputAdapter BroadcastExchange #6 WholeStageCodegen (5) - Project [s_store_sk,s_store_name,s_state,s_zip] - Filter [s_market_id,s_store_sk,s_zip] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_market_id,s_store_sk,s_zip] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] InputAdapter BroadcastExchange #7 WholeStageCodegen (6) - Filter [i_color,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_color,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] InputAdapter BroadcastExchange #8 WholeStageCodegen (7) - Filter [c_customer_sk,c_current_addr_sk,c_birth_country] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_current_addr_sk,c_birth_country] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] InputAdapter BroadcastExchange #9 WholeStageCodegen (8) - Filter [ca_address_sk,ca_country,ca_zip] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ca_address_sk,ca_country,ca_zip] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/explain.txt index 803b7f919..fc43e7271 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/explain.txt @@ -12,22 +12,22 @@ TakeOrderedAndProject (73) : : : +- * BroadcastHashJoin Inner BuildRight (12) : : : :- * Project (10) : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) + : : : : +- * ColumnarToRow (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) : : : : +- CometScan parquet spark_catalog.default.customer_demographics (4) : : : +- ReusedExchange (11) : : +- BroadcastExchange (17) - : : +- * Filter (16) - : : +- * ColumnarToRow (15) + : : +- * ColumnarToRow (16) + : : +- CometFilter (15) : : +- CometScan parquet spark_catalog.default.store (14) : +- BroadcastExchange (23) - : +- * Filter (22) - : +- * ColumnarToRow (21) + : +- * ColumnarToRow (22) + : +- CometFilter (21) : +- CometScan parquet spark_catalog.default.item (20) :- * HashAggregate (50) : +- Exchange (49) @@ -40,15 +40,15 @@ TakeOrderedAndProject (73) : : : +- * BroadcastHashJoin Inner BuildRight (36) : : : :- * Project (34) : : : : +- * BroadcastHashJoin Inner BuildRight (33) - : : : : :- * Filter (31) - : : : : : +- * ColumnarToRow (30) + : : : : :- * ColumnarToRow (31) + : : : : : +- CometFilter (30) : : : : : +- CometScan parquet spark_catalog.default.store_sales (29) : : : : +- ReusedExchange (32) : : : +- ReusedExchange (35) : : +- BroadcastExchange (42) - : : +- * Project (41) - : : +- * Filter (40) - : : +- * ColumnarToRow (39) + : : +- * ColumnarToRow (41) + : : +- CometProject (40) + : : +- CometFilter (39) : : +- CometScan parquet spark_catalog.default.store (38) : +- ReusedExchange (45) +- * HashAggregate (71) @@ -62,15 +62,15 @@ TakeOrderedAndProject (73) : : +- * BroadcastHashJoin Inner BuildRight (58) : : :- * Project (56) : : : +- * BroadcastHashJoin Inner BuildRight (55) - : : : :- * Filter (53) - : : : : +- * ColumnarToRow (52) + : : : :- * ColumnarToRow (53) + : : : : +- CometFilter (52) : : : : +- CometScan parquet spark_catalog.default.store_sales (51) : : : +- ReusedExchange (54) : : +- ReusedExchange (57) : +- ReusedExchange (60) +- BroadcastExchange (66) - +- * Filter (65) - +- * ColumnarToRow (64) + +- * ColumnarToRow (65) + +- CometFilter (64) +- CometScan parquet spark_catalog.default.item (63) @@ -82,12 +82,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] +(2) CometFilter Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) -(3) Filter [codegen id : 5] +(3) ColumnarToRow [codegen id : 5] Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) (4) Scan parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] @@ -96,16 +96,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,F), EqualTo(cd_marital_status,W), EqualTo(cd_education_status,Primary ), IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] - -(6) Filter [codegen id : 1] +(5) CometFilter Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (cd_gender#11 = F)) AND (cd_marital_status#12 = W)) AND (cd_education_status#13 = Primary )) AND isnotnull(cd_demo_sk#10)) -(7) Project [codegen id : 1] -Output [1]: [cd_demo_sk#10] +(6) CometProject Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] +Arguments: [cd_demo_sk#10], [cd_demo_sk#10] + +(7) ColumnarToRow [codegen id : 1] +Input [1]: [cd_demo_sk#10] (8) BroadcastExchange Input [1]: [cd_demo_sk#10] @@ -141,12 +141,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 3] +(15) CometFilter Input [2]: [s_store_sk#15, s_state#16] +Condition : ((isnotnull(s_state#16) AND (s_state#16 = TN)) AND isnotnull(s_store_sk#15)) -(16) Filter [codegen id : 3] +(16) ColumnarToRow [codegen id : 3] Input [2]: [s_store_sk#15, s_state#16] -Condition : ((isnotnull(s_state#16) AND (s_state#16 = TN)) AND isnotnull(s_store_sk#15)) (17) BroadcastExchange Input [2]: [s_store_sk#15, s_state#16] @@ -169,12 +169,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 4] +(21) CometFilter Input [2]: [i_item_sk#17, i_item_id#18] +Condition : isnotnull(i_item_sk#17) -(22) Filter [codegen id : 4] +(22) ColumnarToRow [codegen id : 4] Input [2]: [i_item_sk#17, i_item_id#18] -Condition : isnotnull(i_item_sk#17) (23) BroadcastExchange Input [2]: [i_item_sk#17, i_item_id#18] @@ -216,12 +216,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#55), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 11] +(30) CometFilter Input [8]: [ss_item_sk#48, ss_cdemo_sk#49, ss_store_sk#50, ss_quantity#51, ss_list_price#52, ss_sales_price#53, ss_coupon_amt#54, ss_sold_date_sk#55] +Condition : ((isnotnull(ss_cdemo_sk#49) AND isnotnull(ss_store_sk#50)) AND isnotnull(ss_item_sk#48)) -(31) Filter [codegen id : 11] +(31) ColumnarToRow [codegen id : 11] Input [8]: [ss_item_sk#48, ss_cdemo_sk#49, ss_store_sk#50, ss_quantity#51, ss_list_price#52, ss_sales_price#53, ss_coupon_amt#54, ss_sold_date_sk#55] -Condition : ((isnotnull(ss_cdemo_sk#49) AND isnotnull(ss_store_sk#50)) AND isnotnull(ss_item_sk#48)) (32) ReusedExchange [Reuses operator id: 8] Output [1]: [cd_demo_sk#57] @@ -256,16 +256,16 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] ReadSchema: struct -(39) ColumnarToRow [codegen id : 9] -Input [2]: [s_store_sk#59, s_state#60] - -(40) Filter [codegen id : 9] +(39) CometFilter Input [2]: [s_store_sk#59, s_state#60] Condition : ((isnotnull(s_state#60) AND (s_state#60 = TN)) AND isnotnull(s_store_sk#59)) -(41) Project [codegen id : 9] -Output [1]: [s_store_sk#59] +(40) CometProject Input [2]: [s_store_sk#59, s_state#60] +Arguments: [s_store_sk#59], [s_store_sk#59] + +(41) ColumnarToRow [codegen id : 9] +Input [1]: [s_store_sk#59] (42) BroadcastExchange Input [1]: [s_store_sk#59] @@ -320,12 +320,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#100), dynamicpruningexpression(ss_s PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(52) ColumnarToRow [codegen id : 17] +(52) CometFilter Input [8]: [ss_item_sk#93, ss_cdemo_sk#94, ss_store_sk#95, ss_quantity#96, ss_list_price#97, ss_sales_price#98, ss_coupon_amt#99, ss_sold_date_sk#100] +Condition : ((isnotnull(ss_cdemo_sk#94) AND isnotnull(ss_store_sk#95)) AND isnotnull(ss_item_sk#93)) -(53) Filter [codegen id : 17] +(53) ColumnarToRow [codegen id : 17] Input [8]: [ss_item_sk#93, ss_cdemo_sk#94, ss_store_sk#95, ss_quantity#96, ss_list_price#97, ss_sales_price#98, ss_coupon_amt#99, ss_sold_date_sk#100] -Condition : ((isnotnull(ss_cdemo_sk#94) AND isnotnull(ss_store_sk#95)) AND isnotnull(ss_item_sk#93)) (54) ReusedExchange [Reuses operator id: 8] Output [1]: [cd_demo_sk#102] @@ -373,12 +373,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(64) ColumnarToRow [codegen id : 16] +(64) CometFilter Input [1]: [i_item_sk#105] +Condition : isnotnull(i_item_sk#105) -(65) Filter [codegen id : 16] +(65) ColumnarToRow [codegen id : 16] Input [1]: [i_item_sk#105] -Condition : isnotnull(i_item_sk#105) (66) BroadcastExchange Input [1]: [i_item_sk#105] @@ -422,9 +422,9 @@ Arguments: 100, [i_item_id#18 ASC NULLS FIRST, s_state#16 ASC NULLS FIRST], [i_i Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 BroadcastExchange (78) -+- * Project (77) - +- * Filter (76) - +- * ColumnarToRow (75) ++- * ColumnarToRow (77) + +- CometProject (76) + +- CometFilter (75) +- CometScan parquet spark_catalog.default.date_dim (74) @@ -435,16 +435,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct -(75) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#14, d_year#137] - -(76) Filter [codegen id : 1] +(75) CometFilter Input [2]: [d_date_sk#14, d_year#137] Condition : ((isnotnull(d_year#137) AND (d_year#137 = 1998)) AND isnotnull(d_date_sk#14)) -(77) Project [codegen id : 1] -Output [1]: [d_date_sk#14] +(76) CometProject Input [2]: [d_date_sk#14, d_year#137] +Arguments: [d_date_sk#14], [d_date_sk#14] + +(77) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#14] (78) BroadcastExchange Input [1]: [d_date_sk#14] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/simplified.txt index a05c3e288..32f003798 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/simplified.txt @@ -14,41 +14,41 @@ TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_cdemo_sk,ss_store_sk,ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Project [cd_demo_sk] - Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cd_demo_sk] + CometFilter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] InputAdapter ReusedExchange [d_date_sk] #2 InputAdapter BroadcastExchange #4 WholeStageCodegen (3) - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_state,s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_state] InputAdapter BroadcastExchange #5 WholeStageCodegen (4) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] WholeStageCodegen (12) HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] @@ -64,9 +64,9 @@ TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_cdemo_sk,ss_store_sk,ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -76,10 +76,10 @@ TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] InputAdapter BroadcastExchange #7 WholeStageCodegen (9) - Project [s_store_sk] - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [s_store_sk] + CometFilter [s_state,s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_state] InputAdapter ReusedExchange [i_item_sk,i_item_id] #5 @@ -97,9 +97,9 @@ TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_cdemo_sk,ss_store_sk,ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -111,7 +111,7 @@ TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] InputAdapter BroadcastExchange #9 WholeStageCodegen (16) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/explain.txt index a42f101d7..387fa47f9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/explain.txt @@ -13,23 +13,23 @@ : : +- * BroadcastHashJoin Inner BuildRight (12) : : :- * Project (6) : : : +- * BroadcastHashJoin Inner BuildRight (5) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- ReusedExchange (4) : : +- BroadcastExchange (11) - : : +- * Project (10) - : : +- * Filter (9) - : : +- * ColumnarToRow (8) + : : +- * ColumnarToRow (10) + : : +- CometProject (9) + : : +- CometFilter (8) : : +- CometScan parquet spark_catalog.default.store (7) : +- BroadcastExchange (18) - : +- * Project (17) - : +- * Filter (16) - : +- * ColumnarToRow (15) + : +- * ColumnarToRow (17) + : +- CometProject (16) + : +- CometFilter (15) : +- CometScan parquet spark_catalog.default.household_demographics (14) +- BroadcastExchange (28) - +- * Filter (27) - +- * ColumnarToRow (26) + +- * ColumnarToRow (27) + +- CometFilter (26) +- CometScan parquet spark_catalog.default.customer (25) @@ -41,12 +41,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) -(3) Filter [codegen id : 4] +(3) ColumnarToRow [codegen id : 4] Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) (4) ReusedExchange [Reuses operator id: 37] Output [1]: [d_date_sk#7] @@ -68,16 +68,16 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#8, s_county#9] - -(9) Filter [codegen id : 2] +(8) CometFilter Input [2]: [s_store_sk#8, s_county#9] Condition : ((isnotnull(s_county#9) AND (s_county#9 = Williamson County)) AND isnotnull(s_store_sk#8)) -(10) Project [codegen id : 2] -Output [1]: [s_store_sk#8] +(9) CometProject Input [2]: [s_store_sk#8, s_county#9] +Arguments: [s_store_sk#8], [s_store_sk#8] + +(10) ColumnarToRow [codegen id : 2] +Input [1]: [s_store_sk#8] (11) BroadcastExchange Input [1]: [s_store_sk#8] @@ -100,16 +100,16 @@ Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_vehicle_count), Or(EqualTo(hd_buy_potential,>10000 ),EqualTo(hd_buy_potential,unknown )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 3] -Input [4]: [hd_demo_sk#10, hd_buy_potential#11, hd_dep_count#12, hd_vehicle_count#13] - -(16) Filter [codegen id : 3] +(15) CometFilter Input [4]: [hd_demo_sk#10, hd_buy_potential#11, hd_dep_count#12, hd_vehicle_count#13] Condition : ((((isnotnull(hd_vehicle_count#13) AND ((hd_buy_potential#11 = >10000 ) OR (hd_buy_potential#11 = unknown ))) AND (hd_vehicle_count#13 > 0)) AND CASE WHEN (hd_vehicle_count#13 > 0) THEN ((cast(hd_dep_count#12 as double) / cast(hd_vehicle_count#13 as double)) > 1.2) END) AND isnotnull(hd_demo_sk#10)) -(17) Project [codegen id : 3] -Output [1]: [hd_demo_sk#10] +(16) CometProject Input [4]: [hd_demo_sk#10, hd_buy_potential#11, hd_dep_count#12, hd_vehicle_count#13] +Arguments: [hd_demo_sk#10], [hd_demo_sk#10] + +(17) ColumnarToRow [codegen id : 3] +Input [1]: [hd_demo_sk#10] (18) BroadcastExchange Input [1]: [hd_demo_sk#10] @@ -154,12 +154,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 5] +(26) CometFilter Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] +Condition : isnotnull(c_customer_sk#18) -(27) Filter [codegen id : 5] +(27) ColumnarToRow [codegen id : 5] Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] -Condition : isnotnull(c_customer_sk#18) (28) BroadcastExchange Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] @@ -187,9 +187,9 @@ Arguments: [c_last_name#21 ASC NULLS FIRST, c_first_name#20 ASC NULLS FIRST, c_s Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 BroadcastExchange (37) -+- * Project (36) - +- * Filter (35) - +- * ColumnarToRow (34) ++- * ColumnarToRow (36) + +- CometProject (35) + +- CometFilter (34) +- CometScan parquet spark_catalog.default.date_dim (33) @@ -200,16 +200,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#7, d_year#23, d_dom#24] - -(35) Filter [codegen id : 1] +(34) CometFilter Input [3]: [d_date_sk#7, d_year#23, d_dom#24] Condition : (((((d_dom#24 >= 1) AND (d_dom#24 <= 3)) OR ((d_dom#24 >= 25) AND (d_dom#24 <= 28))) AND d_year#23 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7)) -(36) Project [codegen id : 1] -Output [1]: [d_date_sk#7] +(35) CometProject Input [3]: [d_date_sk#7, d_year#23, d_dom#24] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(36) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#7] (37) BroadcastExchange Input [1]: [d_date_sk#7] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/simplified.txt index cb7ac8dc8..b473e4892 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/simplified.txt @@ -17,40 +17,40 @@ WholeStageCodegen (7) BroadcastHashJoin [ss_store_sk,s_store_sk] Project [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk,ss_hdemo_sk,ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dom,d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_dom,d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] InputAdapter ReusedExchange [d_date_sk] #3 InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - Project [s_store_sk] - Filter [s_county,s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [s_store_sk] + CometFilter [s_county,s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_county] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_vehicle_count,hd_buy_potential,hd_dep_count,hd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [hd_demo_sk] + CometFilter [hd_vehicle_count,hd_buy_potential,hd_dep_count,hd_demo_sk] CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] InputAdapter BroadcastExchange #6 WholeStageCodegen (5) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/explain.txt index 03d608ad6..ce8753277 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/explain.txt @@ -12,8 +12,8 @@ TakeOrderedAndProject (42) : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (24) : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (17) : : : :- * BroadcastHashJoin LeftSemi BuildRight (10) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.customer (1) : : : : +- BroadcastExchange (9) : : : : +- * Project (8) @@ -34,12 +34,12 @@ TakeOrderedAndProject (42) : : : +- CometScan parquet spark_catalog.default.catalog_sales (18) : : +- ReusedExchange (20) : +- BroadcastExchange (30) - : +- * Filter (29) - : +- * ColumnarToRow (28) + : +- * ColumnarToRow (29) + : +- CometFilter (28) : +- CometScan parquet spark_catalog.default.customer_address (27) +- BroadcastExchange (36) - +- * Filter (35) - +- * ColumnarToRow (34) + +- * ColumnarToRow (35) + +- CometFilter (34) +- CometScan parquet spark_catalog.default.customer_demographics (33) @@ -50,12 +50,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 9] +(2) CometFilter Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) -(3) Filter [codegen id : 9] +(3) ColumnarToRow [codegen id : 9] Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) (4) Scan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] @@ -171,12 +171,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 7] +(28) CometFilter Input [2]: [ca_address_sk#18, ca_state#19] +Condition : isnotnull(ca_address_sk#18) -(29) Filter [codegen id : 7] +(29) ColumnarToRow [codegen id : 7] Input [2]: [ca_address_sk#18, ca_state#19] -Condition : isnotnull(ca_address_sk#18) (30) BroadcastExchange Input [2]: [ca_address_sk#18, ca_state#19] @@ -199,12 +199,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 8] +(34) CometFilter Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#20) -(35) Filter [codegen id : 8] +(35) ColumnarToRow [codegen id : 8] Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Condition : isnotnull(cd_demo_sk#20) (36) BroadcastExchange Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] @@ -246,9 +246,9 @@ Arguments: 100, [ca_state#19 ASC NULLS FIRST, cd_gender#21 ASC NULLS FIRST, cd_m Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 BroadcastExchange (47) -+- * Project (46) - +- * Filter (45) - +- * ColumnarToRow (44) ++- * ColumnarToRow (46) + +- CometProject (45) + +- CometFilter (44) +- CometScan parquet spark_catalog.default.date_dim (43) @@ -259,16 +259,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct -(44) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#9, d_year#74, d_qoy#75] - -(45) Filter [codegen id : 1] +(44) CometFilter Input [3]: [d_date_sk#9, d_year#74, d_qoy#75] Condition : ((((isnotnull(d_year#74) AND isnotnull(d_qoy#75)) AND (d_year#74 = 2002)) AND (d_qoy#75 < 4)) AND isnotnull(d_date_sk#9)) -(46) Project [codegen id : 1] -Output [1]: [d_date_sk#9] +(45) CometProject Input [3]: [d_date_sk#9, d_year#74, d_qoy#75] +Arguments: [d_date_sk#9], [d_date_sk#9] + +(46) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#9] (47) BroadcastExchange Input [1]: [d_date_sk#9] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/simplified.txt index 6b81822e4..dc724ca91 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/simplified.txt @@ -14,9 +14,9 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_current_addr_sk,c_current_cdemo_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] InputAdapter BroadcastExchange #2 @@ -29,10 +29,10 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_qoy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_qoy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter ReusedExchange [d_date_sk] #3 @@ -61,14 +61,14 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter BroadcastExchange #6 WholeStageCodegen (7) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #7 WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/explain.txt index 11ca8c0c8..648b19933 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/explain.txt @@ -10,8 +10,8 @@ TakeOrderedAndProject (40) : :- * Project (24) : : +- * BroadcastHashJoin LeftSemi BuildRight (23) : : :- * BroadcastHashJoin LeftSemi BuildRight (10) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.customer (1) : : : +- BroadcastExchange (9) : : : +- * Project (8) @@ -32,12 +32,12 @@ TakeOrderedAndProject (40) : : : +- CometScan parquet spark_catalog.default.catalog_sales (16) : : +- ReusedExchange (18) : +- BroadcastExchange (28) - : +- * Filter (27) - : +- * ColumnarToRow (26) + : +- * ColumnarToRow (27) + : +- CometFilter (26) : +- CometScan parquet spark_catalog.default.customer_address (25) +- BroadcastExchange (34) - +- * Filter (33) - +- * ColumnarToRow (32) + +- * ColumnarToRow (33) + +- CometFilter (32) +- CometScan parquet spark_catalog.default.customer_demographics (31) @@ -48,12 +48,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 9] +(2) CometFilter Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) -(3) Filter [codegen id : 9] +(3) ColumnarToRow [codegen id : 9] Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) (4) Scan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] @@ -157,12 +157,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 7] +(26) CometFilter Input [2]: [ca_address_sk#18, ca_state#19] +Condition : isnotnull(ca_address_sk#18) -(27) Filter [codegen id : 7] +(27) ColumnarToRow [codegen id : 7] Input [2]: [ca_address_sk#18, ca_state#19] -Condition : isnotnull(ca_address_sk#18) (28) BroadcastExchange Input [2]: [ca_address_sk#18, ca_state#19] @@ -185,12 +185,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 8] +(32) CometFilter Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#20) -(33) Filter [codegen id : 8] +(33) ColumnarToRow [codegen id : 8] Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Condition : isnotnull(cd_demo_sk#20) (34) BroadcastExchange Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] @@ -232,9 +232,9 @@ Arguments: 100, [ca_state#19 ASC NULLS FIRST, cd_gender#21 ASC NULLS FIRST, cd_m Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 BroadcastExchange (45) -+- * Project (44) - +- * Filter (43) - +- * ColumnarToRow (42) ++- * ColumnarToRow (44) + +- CometProject (43) + +- CometFilter (42) +- CometScan parquet spark_catalog.default.date_dim (41) @@ -245,16 +245,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,1999), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct -(42) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#7, d_year#74, d_qoy#75] - -(43) Filter [codegen id : 1] +(42) CometFilter Input [3]: [d_date_sk#7, d_year#74, d_qoy#75] Condition : ((((isnotnull(d_year#74) AND isnotnull(d_qoy#75)) AND (d_year#74 = 1999)) AND (d_qoy#75 < 4)) AND isnotnull(d_date_sk#7)) -(44) Project [codegen id : 1] -Output [1]: [d_date_sk#7] +(43) CometProject Input [3]: [d_date_sk#7, d_year#74, d_qoy#75] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(44) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#7] (45) BroadcastExchange Input [1]: [d_date_sk#7] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/simplified.txt index 2cf44995a..e5cb94055 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/simplified.txt @@ -12,9 +12,9 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ Project [c_current_cdemo_sk,c_current_addr_sk] BroadcastHashJoin [c_customer_sk,customsk] BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_current_addr_sk,c_current_cdemo_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] InputAdapter BroadcastExchange #2 @@ -27,10 +27,10 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_qoy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_qoy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter ReusedExchange [d_date_sk] #3 @@ -58,14 +58,14 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter BroadcastExchange #5 WholeStageCodegen (7) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #6 WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/explain.txt index 5a7efec07..2610a698c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/explain.txt @@ -17,18 +17,18 @@ TakeOrderedAndProject (41) : : +- * BroadcastHashJoin Inner BuildRight (11) : : :- * Project (6) : : : +- * BroadcastHashJoin Inner BuildRight (5) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- ReusedExchange (4) : : +- BroadcastExchange (10) - : : +- * Filter (9) - : : +- * ColumnarToRow (8) + : : +- * ColumnarToRow (9) + : : +- CometFilter (8) : : +- CometScan parquet spark_catalog.default.item (7) : +- BroadcastExchange (17) - : +- * Project (16) - : +- * Filter (15) - : +- * ColumnarToRow (14) + : +- * ColumnarToRow (16) + : +- CometProject (15) + : +- CometFilter (14) : +- CometScan parquet spark_catalog.default.store (13) :- * HashAggregate (27) : +- Exchange (26) @@ -50,12 +50,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] +Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) -(3) Filter [codegen id : 4] +(3) ColumnarToRow [codegen id : 4] Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) (4) ReusedExchange [Reuses operator id: 46] Output [1]: [d_date_sk#7] @@ -77,12 +77,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] +(8) CometFilter Input [3]: [i_item_sk#8, i_class#9, i_category#10] +Condition : isnotnull(i_item_sk#8) -(9) Filter [codegen id : 2] +(9) ColumnarToRow [codegen id : 2] Input [3]: [i_item_sk#8, i_class#9, i_category#10] -Condition : isnotnull(i_item_sk#8) (10) BroadcastExchange Input [3]: [i_item_sk#8, i_class#9, i_category#10] @@ -105,16 +105,16 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] -Input [2]: [s_store_sk#11, s_state#12] - -(15) Filter [codegen id : 3] +(14) CometFilter Input [2]: [s_store_sk#11, s_state#12] Condition : ((isnotnull(s_state#12) AND (s_state#12 = TN)) AND isnotnull(s_store_sk#11)) -(16) Project [codegen id : 3] -Output [1]: [s_store_sk#11] +(15) CometProject Input [2]: [s_store_sk#11, s_state#12] +Arguments: [s_store_sk#11], [s_store_sk#11] + +(16) ColumnarToRow [codegen id : 3] +Input [1]: [s_store_sk#11] (17) BroadcastExchange Input [1]: [s_store_sk#11] @@ -248,9 +248,9 @@ Arguments: 100, [lochierarchy#22 DESC NULLS LAST, CASE WHEN (lochierarchy#22 = 0 Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 BroadcastExchange (46) -+- * Project (45) - +- * Filter (44) - +- * ColumnarToRow (43) ++- * ColumnarToRow (45) + +- CometProject (44) + +- CometFilter (43) +- CometScan parquet spark_catalog.default.date_dim (42) @@ -261,16 +261,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(43) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#7, d_year#74] - -(44) Filter [codegen id : 1] +(43) CometFilter Input [2]: [d_date_sk#7, d_year#74] Condition : ((isnotnull(d_year#74) AND (d_year#74 = 2001)) AND isnotnull(d_date_sk#7)) -(45) Project [codegen id : 1] -Output [1]: [d_date_sk#7] +(44) CometProject Input [2]: [d_date_sk#7, d_year#74] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(45) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#7] (46) BroadcastExchange Input [1]: [d_date_sk#7] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/simplified.txt index 9d07011c7..f265d2099 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/simplified.txt @@ -27,34 +27,34 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i BroadcastHashJoin [ss_item_sk,i_item_sk] Project [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk,ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [d_date_sk] #4 InputAdapter BroadcastExchange #5 WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] InputAdapter BroadcastExchange #6 WholeStageCodegen (3) - Project [s_store_sk] - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [s_store_sk] + CometFilter [s_state,s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_state] WholeStageCodegen (11) HashAggregate [i_category,sum,isEmpty,sum,isEmpty] [sum(ss_net_profit),sum(ss_ext_sales_price),gross_margin,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/explain.txt index 95dc7f0cd..9cb7b35cb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/explain.txt @@ -20,17 +20,17 @@ TakeOrderedAndProject (45) : : : +- * BroadcastHashJoin Inner BuildRight (11) : : : :- * Project (9) : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.item (1) : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) + : : : : +- * ColumnarToRow (6) + : : : : +- CometFilter (5) : : : : +- CometScan parquet spark_catalog.default.store_sales (4) : : : +- ReusedExchange (10) : : +- BroadcastExchange (16) - : : +- * Filter (15) - : : +- * ColumnarToRow (14) + : : +- * ColumnarToRow (15) + : : +- CometFilter (14) : : +- CometScan parquet spark_catalog.default.store (13) : +- BroadcastExchange (35) : +- * Project (34) @@ -53,12 +53,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) -(3) Filter [codegen id : 4] +(3) ColumnarToRow [codegen id : 4] Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) (4) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] @@ -68,12 +68,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Condition : (isnotnull(ss_item_sk#4) AND isnotnull(ss_store_sk#5)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_item_sk#4) AND isnotnull(ss_store_sk#5)) (7) BroadcastExchange Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] @@ -109,12 +109,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] +(14) CometFilter Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] +Condition : ((isnotnull(s_store_sk#12) AND isnotnull(s_store_name#13)) AND isnotnull(s_company_name#14)) -(15) Filter [codegen id : 3] +(15) ColumnarToRow [codegen id : 3] Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Condition : ((isnotnull(s_store_sk#12) AND isnotnull(s_store_name#13)) AND isnotnull(s_company_name#14)) (16) BroadcastExchange Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] @@ -253,8 +253,8 @@ Arguments: 100, [(sum_sales#18 - avg_monthly_sales#21) ASC NULLS FIRST, d_moy#11 Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 BroadcastExchange (49) -+- * Filter (48) - +- * ColumnarToRow (47) ++- * ColumnarToRow (48) + +- CometFilter (47) +- CometScan parquet spark_catalog.default.date_dim (46) @@ -265,12 +265,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct -(47) ColumnarToRow [codegen id : 1] +(47) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_moy#11] +Condition : ((((d_year#10 = 1999) OR ((d_year#10 = 1998) AND (d_moy#11 = 12))) OR ((d_year#10 = 2000) AND (d_moy#11 = 1))) AND isnotnull(d_date_sk#9)) -(48) Filter [codegen id : 1] +(48) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((d_year#10 = 1999) OR ((d_year#10 = 1998) AND (d_moy#11 = 12))) OR ((d_year#10 = 2000) AND (d_moy#11 = 1))) AND isnotnull(d_date_sk#9)) (49) BroadcastExchange Input [3]: [d_date_sk#9, d_year#10, d_moy#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/simplified.txt index 1a45368ff..a54895305 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/simplified.txt @@ -28,32 +28,32 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum, BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk,i_category,i_brand] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk,i_category,i_brand] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk,ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_year,d_moy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter ReusedExchange [d_date_sk,d_year,d_moy] #4 InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - Filter [s_store_sk,s_store_name,s_company_name] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk,s_store_name,s_company_name] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] InputAdapter BroadcastExchange #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/explain.txt index 2a8889da9..6591c8b8a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/explain.txt @@ -1,81 +1,78 @@ == Physical Plan == -TakeOrderedAndProject (77) -+- * HashAggregate (76) - +- Exchange (75) - +- * HashAggregate (74) - +- Union (73) - :- * Project (24) - : +- * Filter (23) - : +- Window (22) - : +- * Sort (21) - : +- Window (20) - : +- * Sort (19) - : +- Exchange (18) - : +- * HashAggregate (17) - : +- Exchange (16) - : +- * HashAggregate (15) - : +- * Project (14) - : +- * BroadcastHashJoin Inner BuildRight (13) - : :- * Project (11) - : : +- * BroadcastHashJoin Inner BuildLeft (10) - : : :- BroadcastExchange (5) - : : : +- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : +- * Project (9) - : : +- * Filter (8) - : : +- * ColumnarToRow (7) - : : +- CometScan parquet spark_catalog.default.web_returns (6) - : +- ReusedExchange (12) - :- * Project (48) - : +- * Filter (47) - : +- Window (46) - : +- * Sort (45) - : +- Window (44) - : +- * Sort (43) - : +- Exchange (42) - : +- * HashAggregate (41) - : +- Exchange (40) - : +- * HashAggregate (39) - : +- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (35) - : : +- * BroadcastHashJoin Inner BuildLeft (34) - : : :- BroadcastExchange (29) - : : : +- * Project (28) - : : : +- * Filter (27) - : : : +- * ColumnarToRow (26) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (25) - : : +- * Project (33) - : : +- * Filter (32) - : : +- * ColumnarToRow (31) - : : +- CometScan parquet spark_catalog.default.catalog_returns (30) - : +- ReusedExchange (36) - +- * Project (72) - +- * Filter (71) - +- Window (70) - +- * Sort (69) - +- Window (68) - +- * Sort (67) - +- Exchange (66) - +- * HashAggregate (65) - +- Exchange (64) - +- * HashAggregate (63) - +- * Project (62) - +- * BroadcastHashJoin Inner BuildRight (61) - :- * Project (59) - : +- * BroadcastHashJoin Inner BuildLeft (58) - : :- BroadcastExchange (53) - : : +- * Project (52) - : : +- * Filter (51) - : : +- * ColumnarToRow (50) - : : +- CometScan parquet spark_catalog.default.store_sales (49) - : +- * Project (57) - : +- * Filter (56) - : +- * ColumnarToRow (55) - : +- CometScan parquet spark_catalog.default.store_returns (54) - +- ReusedExchange (60) +TakeOrderedAndProject (74) ++- * HashAggregate (73) + +- Exchange (72) + +- * HashAggregate (71) + +- Union (70) + :- * Project (23) + : +- * Filter (22) + : +- Window (21) + : +- * Sort (20) + : +- Window (19) + : +- * Sort (18) + : +- Exchange (17) + : +- * HashAggregate (16) + : +- Exchange (15) + : +- * HashAggregate (14) + : +- * Project (13) + : +- * BroadcastHashJoin Inner BuildRight (12) + : :- * ColumnarToRow (10) + : : +- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometBroadcastExchange (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : +- CometProject (7) + : : +- CometFilter (6) + : : +- CometScan parquet spark_catalog.default.web_returns (5) + : +- ReusedExchange (11) + :- * Project (46) + : +- * Filter (45) + : +- Window (44) + : +- * Sort (43) + : +- Window (42) + : +- * Sort (41) + : +- Exchange (40) + : +- * HashAggregate (39) + : +- Exchange (38) + : +- * HashAggregate (37) + : +- * Project (36) + : +- * BroadcastHashJoin Inner BuildRight (35) + : :- * ColumnarToRow (33) + : : +- CometProject (32) + : : +- CometBroadcastHashJoin (31) + : : :- CometBroadcastExchange (27) + : : : +- CometProject (26) + : : : +- CometFilter (25) + : : : +- CometScan parquet spark_catalog.default.catalog_sales (24) + : : +- CometProject (30) + : : +- CometFilter (29) + : : +- CometScan parquet spark_catalog.default.catalog_returns (28) + : +- ReusedExchange (34) + +- * Project (69) + +- * Filter (68) + +- Window (67) + +- * Sort (66) + +- Window (65) + +- * Sort (64) + +- Exchange (63) + +- * HashAggregate (62) + +- Exchange (61) + +- * HashAggregate (60) + +- * Project (59) + +- * BroadcastHashJoin Inner BuildRight (58) + :- * ColumnarToRow (56) + : +- CometProject (55) + : +- CometBroadcastHashJoin (54) + : :- CometBroadcastExchange (50) + : : +- CometProject (49) + : : +- CometFilter (48) + : : +- CometScan parquet spark_catalog.default.store_sales (47) + : +- CometProject (53) + : +- CometFilter (52) + : +- CometScan parquet spark_catalog.default.store_returns (51) + +- ReusedExchange (57) (1) Scan parquet spark_catalog.default.web_sales @@ -86,109 +83,105 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#6), dynamicpruningexpression(ws_sol PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] Condition : (((((((isnotnull(ws_net_profit#5) AND isnotnull(ws_net_paid#4)) AND isnotnull(ws_quantity#3)) AND (ws_net_profit#5 > 1.00)) AND (ws_net_paid#4 > 0.00)) AND (ws_quantity#3 > 0)) AND isnotnull(ws_order_number#2)) AND isnotnull(ws_item_sk#1)) -(4) Project [codegen id : 1] -Output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] +(3) CometProject Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] +Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6], [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -(5) BroadcastExchange +(4) CometBroadcastExchange Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=1] +Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -(6) Scan parquet spark_catalog.default.web_returns +(5) Scan parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct -(7) ColumnarToRow -Input [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] - -(8) Filter +(6) CometFilter Input [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] Condition : (((isnotnull(wr_return_amt#11) AND (wr_return_amt#11 > 10000.00)) AND isnotnull(wr_order_number#9)) AND isnotnull(wr_item_sk#8)) -(9) Project -Output [4]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] +(7) CometProject Input [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] +Arguments: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11], [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] -(10) BroadcastHashJoin [codegen id : 3] -Left keys [2]: [ws_order_number#2, ws_item_sk#1] -Right keys [2]: [wr_order_number#9, wr_item_sk#8] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] +Right output [4]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] +Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#9, wr_item_sk#8], Inner -(11) Project [codegen id : 3] -Output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11] +(9) CometProject Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11] -(12) ReusedExchange [Reuses operator id: 82] +(10) ColumnarToRow [codegen id : 2] +Input [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11] + +(11) ReusedExchange [Reuses operator id: 79] Output [1]: [d_date_sk#13] -(13) BroadcastHashJoin [codegen id : 3] +(12) BroadcastHashJoin [codegen id : 2] Left keys [1]: [ws_sold_date_sk#6] Right keys [1]: [d_date_sk#13] Join type: Inner Join condition: None -(14) Project [codegen id : 3] +(13) Project [codegen id : 2] Output [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, wr_return_amt#11] Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11, d_date_sk#13] -(15) HashAggregate [codegen id : 3] +(14) HashAggregate [codegen id : 2] Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, wr_return_amt#11] Keys [1]: [ws_item_sk#1] Functions [4]: [partial_sum(coalesce(wr_return_quantity#10, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] Aggregate Attributes [6]: [sum#14, sum#15, sum#16, isEmpty#17, sum#18, isEmpty#19] Results [7]: [ws_item_sk#1, sum#20, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -(16) Exchange +(15) Exchange Input [7]: [ws_item_sk#1, sum#20, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(17) HashAggregate [codegen id : 4] +(16) HashAggregate [codegen id : 3] Input [7]: [ws_item_sk#1, sum#20, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] Keys [1]: [ws_item_sk#1] Functions [4]: [sum(coalesce(wr_return_quantity#10, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] Aggregate Attributes [4]: [sum(coalesce(wr_return_quantity#10, 0))#26, sum(coalesce(ws_quantity#3, 0))#27, sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00))#28, sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#29] Results [3]: [ws_item_sk#1 AS item#30, (cast(sum(coalesce(wr_return_quantity#10, 0))#26 as decimal(15,4)) / cast(sum(coalesce(ws_quantity#3, 0))#27 as decimal(15,4))) AS return_ratio#31, (cast(sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00))#28 as decimal(15,4)) / cast(sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#29 as decimal(15,4))) AS currency_ratio#32] -(18) Exchange +(17) Exchange Input [3]: [item#30, return_ratio#31, currency_ratio#32] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] -(19) Sort [codegen id : 5] +(18) Sort [codegen id : 4] Input [3]: [item#30, return_ratio#31, currency_ratio#32] Arguments: [return_ratio#31 ASC NULLS FIRST], false, 0 -(20) Window +(19) Window Input [3]: [item#30, return_ratio#31, currency_ratio#32] Arguments: [rank(return_ratio#31) windowspecdefinition(return_ratio#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#33], [return_ratio#31 ASC NULLS FIRST] -(21) Sort [codegen id : 6] +(20) Sort [codegen id : 5] Input [4]: [item#30, return_ratio#31, currency_ratio#32, return_rank#33] Arguments: [currency_ratio#32 ASC NULLS FIRST], false, 0 -(22) Window +(21) Window Input [4]: [item#30, return_ratio#31, currency_ratio#32, return_rank#33] Arguments: [rank(currency_ratio#32) windowspecdefinition(currency_ratio#32 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#34], [currency_ratio#32 ASC NULLS FIRST] -(23) Filter [codegen id : 7] +(22) Filter [codegen id : 6] Input [5]: [item#30, return_ratio#31, currency_ratio#32, return_rank#33, currency_rank#34] Condition : ((return_rank#33 <= 10) OR (currency_rank#34 <= 10)) -(24) Project [codegen id : 7] +(23) Project [codegen id : 6] Output [5]: [web AS channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34] Input [5]: [item#30, return_ratio#31, currency_ratio#32, return_rank#33, currency_rank#34] -(25) Scan parquet spark_catalog.default.catalog_sales +(24) Scan parquet spark_catalog.default.catalog_sales Output [6]: [cs_item_sk#36, cs_order_number#37, cs_quantity#38, cs_net_paid#39, cs_net_profit#40, cs_sold_date_sk#41] Batched: true Location: InMemoryFileIndex [] @@ -196,109 +189,105 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#41), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 8] -Input [6]: [cs_item_sk#36, cs_order_number#37, cs_quantity#38, cs_net_paid#39, cs_net_profit#40, cs_sold_date_sk#41] - -(27) Filter [codegen id : 8] +(25) CometFilter Input [6]: [cs_item_sk#36, cs_order_number#37, cs_quantity#38, cs_net_paid#39, cs_net_profit#40, cs_sold_date_sk#41] Condition : (((((((isnotnull(cs_net_profit#40) AND isnotnull(cs_net_paid#39)) AND isnotnull(cs_quantity#38)) AND (cs_net_profit#40 > 1.00)) AND (cs_net_paid#39 > 0.00)) AND (cs_quantity#38 > 0)) AND isnotnull(cs_order_number#37)) AND isnotnull(cs_item_sk#36)) -(28) Project [codegen id : 8] -Output [5]: [cs_item_sk#36, cs_order_number#37, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41] +(26) CometProject Input [6]: [cs_item_sk#36, cs_order_number#37, cs_quantity#38, cs_net_paid#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: [cs_item_sk#36, cs_order_number#37, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41], [cs_item_sk#36, cs_order_number#37, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41] -(29) BroadcastExchange +(27) CometBroadcastExchange Input [5]: [cs_item_sk#36, cs_order_number#37, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=4] +Arguments: [cs_item_sk#36, cs_order_number#37, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41] -(30) Scan parquet spark_catalog.default.catalog_returns +(28) Scan parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(31) ColumnarToRow -Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] - -(32) Filter +(29) CometFilter Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] Condition : (((isnotnull(cr_return_amount#46) AND (cr_return_amount#46 > 10000.00)) AND isnotnull(cr_order_number#44)) AND isnotnull(cr_item_sk#43)) -(33) Project -Output [4]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46] +(30) CometProject Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] +Arguments: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46], [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46] -(34) BroadcastHashJoin [codegen id : 10] -Left keys [2]: [cs_order_number#37, cs_item_sk#36] -Right keys [2]: [cr_order_number#44, cr_item_sk#43] -Join type: Inner -Join condition: None +(31) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#36, cs_order_number#37, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41] +Right output [4]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46] +Arguments: [cs_order_number#37, cs_item_sk#36], [cr_order_number#44, cr_item_sk#43], Inner -(35) Project [codegen id : 10] -Output [6]: [cs_item_sk#36, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41, cr_return_quantity#45, cr_return_amount#46] +(32) CometProject Input [9]: [cs_item_sk#36, cs_order_number#37, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41, cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46] +Arguments: [cs_item_sk#36, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41, cr_return_quantity#45, cr_return_amount#46], [cs_item_sk#36, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41, cr_return_quantity#45, cr_return_amount#46] + +(33) ColumnarToRow [codegen id : 8] +Input [6]: [cs_item_sk#36, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41, cr_return_quantity#45, cr_return_amount#46] -(36) ReusedExchange [Reuses operator id: 82] +(34) ReusedExchange [Reuses operator id: 79] Output [1]: [d_date_sk#48] -(37) BroadcastHashJoin [codegen id : 10] +(35) BroadcastHashJoin [codegen id : 8] Left keys [1]: [cs_sold_date_sk#41] Right keys [1]: [d_date_sk#48] Join type: Inner Join condition: None -(38) Project [codegen id : 10] +(36) Project [codegen id : 8] Output [5]: [cs_item_sk#36, cs_quantity#38, cs_net_paid#39, cr_return_quantity#45, cr_return_amount#46] Input [7]: [cs_item_sk#36, cs_quantity#38, cs_net_paid#39, cs_sold_date_sk#41, cr_return_quantity#45, cr_return_amount#46, d_date_sk#48] -(39) HashAggregate [codegen id : 10] +(37) HashAggregate [codegen id : 8] Input [5]: [cs_item_sk#36, cs_quantity#38, cs_net_paid#39, cr_return_quantity#45, cr_return_amount#46] Keys [1]: [cs_item_sk#36] Functions [4]: [partial_sum(coalesce(cr_return_quantity#45, 0)), partial_sum(coalesce(cs_quantity#38, 0)), partial_sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#39 as decimal(12,2)), 0.00))] Aggregate Attributes [6]: [sum#49, sum#50, sum#51, isEmpty#52, sum#53, isEmpty#54] Results [7]: [cs_item_sk#36, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] -(40) Exchange +(38) Exchange Input [7]: [cs_item_sk#36, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] -Arguments: hashpartitioning(cs_item_sk#36, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(cs_item_sk#36, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(41) HashAggregate [codegen id : 11] +(39) HashAggregate [codegen id : 9] Input [7]: [cs_item_sk#36, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] Keys [1]: [cs_item_sk#36] Functions [4]: [sum(coalesce(cr_return_quantity#45, 0)), sum(coalesce(cs_quantity#38, 0)), sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#39 as decimal(12,2)), 0.00))] Aggregate Attributes [4]: [sum(coalesce(cr_return_quantity#45, 0))#61, sum(coalesce(cs_quantity#38, 0))#62, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#63, sum(coalesce(cast(cs_net_paid#39 as decimal(12,2)), 0.00))#64] Results [3]: [cs_item_sk#36 AS item#65, (cast(sum(coalesce(cr_return_quantity#45, 0))#61 as decimal(15,4)) / cast(sum(coalesce(cs_quantity#38, 0))#62 as decimal(15,4))) AS return_ratio#66, (cast(sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#63 as decimal(15,4)) / cast(sum(coalesce(cast(cs_net_paid#39 as decimal(12,2)), 0.00))#64 as decimal(15,4))) AS currency_ratio#67] -(42) Exchange +(40) Exchange Input [3]: [item#65, return_ratio#66, currency_ratio#67] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] -(43) Sort [codegen id : 12] +(41) Sort [codegen id : 10] Input [3]: [item#65, return_ratio#66, currency_ratio#67] Arguments: [return_ratio#66 ASC NULLS FIRST], false, 0 -(44) Window +(42) Window Input [3]: [item#65, return_ratio#66, currency_ratio#67] Arguments: [rank(return_ratio#66) windowspecdefinition(return_ratio#66 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#68], [return_ratio#66 ASC NULLS FIRST] -(45) Sort [codegen id : 13] +(43) Sort [codegen id : 11] Input [4]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68] Arguments: [currency_ratio#67 ASC NULLS FIRST], false, 0 -(46) Window +(44) Window Input [4]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68] Arguments: [rank(currency_ratio#67) windowspecdefinition(currency_ratio#67 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#69], [currency_ratio#67 ASC NULLS FIRST] -(47) Filter [codegen id : 14] +(45) Filter [codegen id : 12] Input [5]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68, currency_rank#69] Condition : ((return_rank#68 <= 10) OR (currency_rank#69 <= 10)) -(48) Project [codegen id : 14] +(46) Project [codegen id : 12] Output [5]: [catalog AS channel#70, item#65, return_ratio#66, return_rank#68, currency_rank#69] Input [5]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68, currency_rank#69] -(49) Scan parquet spark_catalog.default.store_sales +(47) Scan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] Batched: true Location: InMemoryFileIndex [] @@ -306,166 +295,162 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#76), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)] ReadSchema: struct -(50) ColumnarToRow [codegen id : 15] -Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] - -(51) Filter [codegen id : 15] +(48) CometFilter Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] Condition : (((((((isnotnull(ss_net_profit#75) AND isnotnull(ss_net_paid#74)) AND isnotnull(ss_quantity#73)) AND (ss_net_profit#75 > 1.00)) AND (ss_net_paid#74 > 0.00)) AND (ss_quantity#73 > 0)) AND isnotnull(ss_ticket_number#72)) AND isnotnull(ss_item_sk#71)) -(52) Project [codegen id : 15] -Output [5]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76] +(49) CometProject Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] +Arguments: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76], [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76] -(53) BroadcastExchange +(50) CometBroadcastExchange Input [5]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=7] +Arguments: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76] -(54) Scan parquet spark_catalog.default.store_returns +(51) Scan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#78, sr_ticket_number#79, sr_return_quantity#80, sr_return_amt#81, sr_returned_date_sk#82] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(55) ColumnarToRow -Input [5]: [sr_item_sk#78, sr_ticket_number#79, sr_return_quantity#80, sr_return_amt#81, sr_returned_date_sk#82] - -(56) Filter +(52) CometFilter Input [5]: [sr_item_sk#78, sr_ticket_number#79, sr_return_quantity#80, sr_return_amt#81, sr_returned_date_sk#82] Condition : (((isnotnull(sr_return_amt#81) AND (sr_return_amt#81 > 10000.00)) AND isnotnull(sr_ticket_number#79)) AND isnotnull(sr_item_sk#78)) -(57) Project -Output [4]: [sr_item_sk#78, sr_ticket_number#79, sr_return_quantity#80, sr_return_amt#81] +(53) CometProject Input [5]: [sr_item_sk#78, sr_ticket_number#79, sr_return_quantity#80, sr_return_amt#81, sr_returned_date_sk#82] +Arguments: [sr_item_sk#78, sr_ticket_number#79, sr_return_quantity#80, sr_return_amt#81], [sr_item_sk#78, sr_ticket_number#79, sr_return_quantity#80, sr_return_amt#81] -(58) BroadcastHashJoin [codegen id : 17] -Left keys [2]: [ss_ticket_number#72, ss_item_sk#71] -Right keys [2]: [sr_ticket_number#79, sr_item_sk#78] -Join type: Inner -Join condition: None +(54) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76] +Right output [4]: [sr_item_sk#78, sr_ticket_number#79, sr_return_quantity#80, sr_return_amt#81] +Arguments: [ss_ticket_number#72, ss_item_sk#71], [sr_ticket_number#79, sr_item_sk#78], Inner -(59) Project [codegen id : 17] -Output [6]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_return_quantity#80, sr_return_amt#81] +(55) CometProject Input [9]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_item_sk#78, sr_ticket_number#79, sr_return_quantity#80, sr_return_amt#81] +Arguments: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_return_quantity#80, sr_return_amt#81], [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_return_quantity#80, sr_return_amt#81] + +(56) ColumnarToRow [codegen id : 14] +Input [6]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_return_quantity#80, sr_return_amt#81] -(60) ReusedExchange [Reuses operator id: 82] +(57) ReusedExchange [Reuses operator id: 79] Output [1]: [d_date_sk#83] -(61) BroadcastHashJoin [codegen id : 17] +(58) BroadcastHashJoin [codegen id : 14] Left keys [1]: [ss_sold_date_sk#76] Right keys [1]: [d_date_sk#83] Join type: Inner Join condition: None -(62) Project [codegen id : 17] +(59) Project [codegen id : 14] Output [5]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, sr_return_quantity#80, sr_return_amt#81] Input [7]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_return_quantity#80, sr_return_amt#81, d_date_sk#83] -(63) HashAggregate [codegen id : 17] +(60) HashAggregate [codegen id : 14] Input [5]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, sr_return_quantity#80, sr_return_amt#81] Keys [1]: [ss_item_sk#71] Functions [4]: [partial_sum(coalesce(sr_return_quantity#80, 0)), partial_sum(coalesce(ss_quantity#73, 0)), partial_sum(coalesce(cast(sr_return_amt#81 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))] Aggregate Attributes [6]: [sum#84, sum#85, sum#86, isEmpty#87, sum#88, isEmpty#89] Results [7]: [ss_item_sk#71, sum#90, sum#91, sum#92, isEmpty#93, sum#94, isEmpty#95] -(64) Exchange +(61) Exchange Input [7]: [ss_item_sk#71, sum#90, sum#91, sum#92, isEmpty#93, sum#94, isEmpty#95] -Arguments: hashpartitioning(ss_item_sk#71, 5), ENSURE_REQUIREMENTS, [plan_id=8] +Arguments: hashpartitioning(ss_item_sk#71, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(65) HashAggregate [codegen id : 18] +(62) HashAggregate [codegen id : 15] Input [7]: [ss_item_sk#71, sum#90, sum#91, sum#92, isEmpty#93, sum#94, isEmpty#95] Keys [1]: [ss_item_sk#71] Functions [4]: [sum(coalesce(sr_return_quantity#80, 0)), sum(coalesce(ss_quantity#73, 0)), sum(coalesce(cast(sr_return_amt#81 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))] Aggregate Attributes [4]: [sum(coalesce(sr_return_quantity#80, 0))#96, sum(coalesce(ss_quantity#73, 0))#97, sum(coalesce(cast(sr_return_amt#81 as decimal(12,2)), 0.00))#98, sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))#99] Results [3]: [ss_item_sk#71 AS item#100, (cast(sum(coalesce(sr_return_quantity#80, 0))#96 as decimal(15,4)) / cast(sum(coalesce(ss_quantity#73, 0))#97 as decimal(15,4))) AS return_ratio#101, (cast(sum(coalesce(cast(sr_return_amt#81 as decimal(12,2)), 0.00))#98 as decimal(15,4)) / cast(sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))#99 as decimal(15,4))) AS currency_ratio#102] -(66) Exchange +(63) Exchange Input [3]: [item#100, return_ratio#101, currency_ratio#102] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=9] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] -(67) Sort [codegen id : 19] +(64) Sort [codegen id : 16] Input [3]: [item#100, return_ratio#101, currency_ratio#102] Arguments: [return_ratio#101 ASC NULLS FIRST], false, 0 -(68) Window +(65) Window Input [3]: [item#100, return_ratio#101, currency_ratio#102] Arguments: [rank(return_ratio#101) windowspecdefinition(return_ratio#101 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#103], [return_ratio#101 ASC NULLS FIRST] -(69) Sort [codegen id : 20] +(66) Sort [codegen id : 17] Input [4]: [item#100, return_ratio#101, currency_ratio#102, return_rank#103] Arguments: [currency_ratio#102 ASC NULLS FIRST], false, 0 -(70) Window +(67) Window Input [4]: [item#100, return_ratio#101, currency_ratio#102, return_rank#103] Arguments: [rank(currency_ratio#102) windowspecdefinition(currency_ratio#102 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#104], [currency_ratio#102 ASC NULLS FIRST] -(71) Filter [codegen id : 21] +(68) Filter [codegen id : 18] Input [5]: [item#100, return_ratio#101, currency_ratio#102, return_rank#103, currency_rank#104] Condition : ((return_rank#103 <= 10) OR (currency_rank#104 <= 10)) -(72) Project [codegen id : 21] +(69) Project [codegen id : 18] Output [5]: [store AS channel#105, item#100, return_ratio#101, return_rank#103, currency_rank#104] Input [5]: [item#100, return_ratio#101, currency_ratio#102, return_rank#103, currency_rank#104] -(73) Union +(70) Union -(74) HashAggregate [codegen id : 22] +(71) HashAggregate [codegen id : 19] Input [5]: [channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34] Keys [5]: [channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34] Functions: [] Aggregate Attributes: [] Results [5]: [channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34] -(75) Exchange +(72) Exchange Input [5]: [channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34] -Arguments: hashpartitioning(channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34, 5), ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: hashpartitioning(channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(76) HashAggregate [codegen id : 23] +(73) HashAggregate [codegen id : 20] Input [5]: [channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34] Keys [5]: [channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34] Functions: [] Aggregate Attributes: [] Results [5]: [channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34] -(77) TakeOrderedAndProject +(74) TakeOrderedAndProject Input [5]: [channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34] Arguments: 100, [channel#35 ASC NULLS FIRST, return_rank#33 ASC NULLS FIRST, currency_rank#34 ASC NULLS FIRST, item#30 ASC NULLS FIRST], [channel#35, item#30, return_ratio#31, return_rank#33, currency_rank#34] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (82) -+- * Project (81) - +- * Filter (80) - +- * ColumnarToRow (79) - +- CometScan parquet spark_catalog.default.date_dim (78) +BroadcastExchange (79) ++- * ColumnarToRow (78) + +- CometProject (77) + +- CometFilter (76) + +- CometScan parquet spark_catalog.default.date_dim (75) -(78) Scan parquet spark_catalog.default.date_dim +(75) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#106, d_moy#107] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] ReadSchema: struct -(79) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#13, d_year#106, d_moy#107] - -(80) Filter [codegen id : 1] +(76) CometFilter Input [3]: [d_date_sk#13, d_year#106, d_moy#107] Condition : ((((isnotnull(d_year#106) AND isnotnull(d_moy#107)) AND (d_year#106 = 2001)) AND (d_moy#107 = 12)) AND isnotnull(d_date_sk#13)) -(81) Project [codegen id : 1] -Output [1]: [d_date_sk#13] +(77) CometProject Input [3]: [d_date_sk#13, d_year#106, d_moy#107] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(78) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#13] -(82) BroadcastExchange +(79) BroadcastExchange Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] -Subquery:2 Hosting operator id = 25 Hosting Expression = cs_sold_date_sk#41 IN dynamicpruning#7 +Subquery:2 Hosting operator id = 24 Hosting Expression = cs_sold_date_sk#41 IN dynamicpruning#7 -Subquery:3 Hosting operator id = 49 Hosting Expression = ss_sold_date_sk#76 IN dynamicpruning#7 +Subquery:3 Hosting operator id = 47 Hosting Expression = ss_sold_date_sk#76 IN dynamicpruning#7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/simplified.txt index b707750da..43ebf34cc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/simplified.txt @@ -1,133 +1,121 @@ TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio] - WholeStageCodegen (23) + WholeStageCodegen (20) HashAggregate [channel,item,return_ratio,return_rank,currency_rank] InputAdapter Exchange [channel,item,return_ratio,return_rank,currency_rank] #1 - WholeStageCodegen (22) + WholeStageCodegen (19) HashAggregate [channel,item,return_ratio,return_rank,currency_rank] InputAdapter Union - WholeStageCodegen (7) + WholeStageCodegen (6) Project [item,return_ratio,return_rank,currency_rank] Filter [return_rank,currency_rank] InputAdapter Window [currency_ratio] - WholeStageCodegen (6) + WholeStageCodegen (5) Sort [currency_ratio] InputAdapter Window [return_ratio] - WholeStageCodegen (5) + WholeStageCodegen (4) Sort [return_ratio] InputAdapter Exchange #2 - WholeStageCodegen (4) + WholeStageCodegen (3) HashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] InputAdapter Exchange [ws_item_sk] #3 - WholeStageCodegen (3) + WholeStageCodegen (2) HashAggregate [ws_item_sk,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] Project [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] - BroadcastHashJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Project [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] - Filter [ws_net_profit,ws_net_paid,ws_quantity,ws_order_number,ws_item_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - Filter [wr_return_amt,wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + CometBroadcastExchange #4 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] + CometFilter [ws_net_profit,ws_net_paid,ws_quantity,ws_order_number,ws_item_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_year,d_moy,d_date_sk] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_return_amt,wr_order_number,wr_item_sk] CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] InputAdapter ReusedExchange [d_date_sk] #5 - WholeStageCodegen (14) + WholeStageCodegen (12) Project [item,return_ratio,return_rank,currency_rank] Filter [return_rank,currency_rank] InputAdapter Window [currency_ratio] - WholeStageCodegen (13) + WholeStageCodegen (11) Sort [currency_ratio] InputAdapter Window [return_ratio] - WholeStageCodegen (12) + WholeStageCodegen (10) Sort [return_ratio] InputAdapter Exchange #6 - WholeStageCodegen (11) + WholeStageCodegen (9) HashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] InputAdapter Exchange [cs_item_sk] #7 - WholeStageCodegen (10) + WholeStageCodegen (8) HashAggregate [cs_item_sk,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] Project [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] - BroadcastHashJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (8) - Project [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] - Filter [cs_net_profit,cs_net_paid,cs_quantity,cs_order_number,cs_item_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - Filter [cr_return_amount,cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + CometBroadcastExchange #8 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] + CometFilter [cs_net_profit,cs_net_paid,cs_quantity,cs_order_number,cs_item_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_return_amount,cr_order_number,cr_item_sk] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] InputAdapter ReusedExchange [d_date_sk] #5 - WholeStageCodegen (21) + WholeStageCodegen (18) Project [item,return_ratio,return_rank,currency_rank] Filter [return_rank,currency_rank] InputAdapter Window [currency_ratio] - WholeStageCodegen (20) + WholeStageCodegen (17) Sort [currency_ratio] InputAdapter Window [return_ratio] - WholeStageCodegen (19) + WholeStageCodegen (16) Sort [return_ratio] InputAdapter Exchange #9 - WholeStageCodegen (18) + WholeStageCodegen (15) HashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] InputAdapter Exchange [ss_item_sk] #10 - WholeStageCodegen (17) + WholeStageCodegen (14) HashAggregate [ss_item_sk,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] Project [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] - BroadcastHashJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (15) - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] - Filter [ss_net_profit,ss_net_paid,ss_quantity,ss_ticket_number,ss_item_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - Filter [sr_return_amt,sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + CometBroadcastExchange #11 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] + CometFilter [ss_net_profit,ss_net_paid,ss_quantity,ss_ticket_number,ss_item_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_return_amt,sr_ticket_number,sr_item_sk] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] InputAdapter ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/explain.txt index cb9e772c9..3892f250a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/explain.txt @@ -27,8 +27,8 @@ TakeOrderedAndProject (67) : : : +- * HashAggregate (7) : : : +- * Project (6) : : : +- * BroadcastHashJoin Inner BuildRight (5) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.web_sales (1) : : : +- ReusedExchange (4) : : +- BroadcastExchange (18) @@ -52,8 +52,8 @@ TakeOrderedAndProject (67) : : +- * HashAggregate (32) : : +- * Project (31) : : +- * BroadcastHashJoin Inner BuildRight (30) - : : :- * Filter (28) - : : : +- * ColumnarToRow (27) + : : :- * ColumnarToRow (28) + : : : +- CometFilter (27) : : : +- CometScan parquet spark_catalog.default.store_sales (26) : : +- ReusedExchange (29) : +- BroadcastExchange (43) @@ -76,12 +76,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sol PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 2] +(2) CometFilter Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] +Condition : isnotnull(ws_item_sk#1) -(3) Filter [codegen id : 2] +(3) ColumnarToRow [codegen id : 2] Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Condition : isnotnull(ws_item_sk#1) (4) ReusedExchange [Reuses operator id: 72] Output [2]: [d_date_sk#5, d_date#6] @@ -193,12 +193,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#26), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 14] +(27) CometFilter Input [3]: [ss_item_sk#24, ss_sales_price#25, ss_sold_date_sk#26] +Condition : isnotnull(ss_item_sk#24) -(28) Filter [codegen id : 14] +(28) ColumnarToRow [codegen id : 14] Input [3]: [ss_item_sk#24, ss_sales_price#25, ss_sold_date_sk#26] -Condition : isnotnull(ss_item_sk#24) (29) ReusedExchange [Reuses operator id: 72] Output [2]: [d_date_sk#28, d_date#29] @@ -383,9 +383,9 @@ Arguments: 100, [item_sk#47 ASC NULLS FIRST, d_date#48 ASC NULLS FIRST], [item_s Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 BroadcastExchange (72) -+- * Project (71) - +- * Filter (70) - +- * ColumnarToRow (69) ++- * ColumnarToRow (71) + +- CometProject (70) + +- CometFilter (69) +- CometScan parquet spark_catalog.default.date_dim (68) @@ -396,16 +396,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(69) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#65] - -(70) Filter [codegen id : 1] +(69) CometFilter Input [3]: [d_date_sk#5, d_date#6, d_month_seq#65] Condition : (((isnotnull(d_month_seq#65) AND (d_month_seq#65 >= 1212)) AND (d_month_seq#65 <= 1223)) AND isnotnull(d_date_sk#5)) -(71) Project [codegen id : 1] -Output [2]: [d_date_sk#5, d_date#6] +(70) CometProject Input [3]: [d_date_sk#5, d_date#6, d_month_seq#65] +Arguments: [d_date_sk#5, d_date#6], [d_date_sk#5, d_date#6] + +(71) ColumnarToRow [codegen id : 1] +Input [2]: [d_date_sk#5, d_date#6] (72) BroadcastExchange Input [2]: [d_date_sk#5, d_date#6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/simplified.txt index 252bc7f69..3109290dc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/simplified.txt @@ -43,17 +43,17 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store HashAggregate [ws_item_sk,d_date,ws_sales_price] [sum,sum] Project [ws_item_sk,ws_sales_price,d_date] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #6 WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_date] + CometFilter [d_month_seq,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] InputAdapter ReusedExchange [d_date_sk,d_date] #6 @@ -95,9 +95,9 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store HashAggregate [ss_item_sk,d_date,ss_sales_price] [sum,sum] Project [ss_item_sk,ss_sales_price,d_date] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/explain.txt index ef343f0e7..dd64cc7bc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/explain.txt @@ -20,17 +20,17 @@ TakeOrderedAndProject (45) : : : +- * BroadcastHashJoin Inner BuildRight (11) : : : :- * Project (9) : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.item (1) : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) + : : : : +- * ColumnarToRow (6) + : : : : +- CometFilter (5) : : : : +- CometScan parquet spark_catalog.default.catalog_sales (4) : : : +- ReusedExchange (10) : : +- BroadcastExchange (16) - : : +- * Filter (15) - : : +- * ColumnarToRow (14) + : : +- * ColumnarToRow (15) + : : +- CometFilter (14) : : +- CometScan parquet spark_catalog.default.call_center (13) : +- BroadcastExchange (35) : +- * Project (34) @@ -53,12 +53,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) -(3) Filter [codegen id : 4] +(3) ColumnarToRow [codegen id : 4] Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) (4) Scan parquet spark_catalog.default.catalog_sales Output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] @@ -68,12 +68,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#7), dynamicpruningexpression(cs_sol PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] +Condition : (isnotnull(cs_item_sk#5) AND isnotnull(cs_call_center_sk#4)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Condition : (isnotnull(cs_item_sk#5) AND isnotnull(cs_call_center_sk#4)) (7) BroadcastExchange Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] @@ -109,12 +109,12 @@ Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] +(14) CometFilter Input [2]: [cc_call_center_sk#12, cc_name#13] +Condition : (isnotnull(cc_call_center_sk#12) AND isnotnull(cc_name#13)) -(15) Filter [codegen id : 3] +(15) ColumnarToRow [codegen id : 3] Input [2]: [cc_call_center_sk#12, cc_name#13] -Condition : (isnotnull(cc_call_center_sk#12) AND isnotnull(cc_name#13)) (16) BroadcastExchange Input [2]: [cc_call_center_sk#12, cc_name#13] @@ -253,8 +253,8 @@ Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, d_year#1 Subquery:1 Hosting operator id = 4 Hosting Expression = cs_sold_date_sk#7 IN dynamicpruning#8 BroadcastExchange (49) -+- * Filter (48) - +- * ColumnarToRow (47) ++- * ColumnarToRow (48) + +- CometFilter (47) +- CometScan parquet spark_catalog.default.date_dim (46) @@ -265,12 +265,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct -(47) ColumnarToRow [codegen id : 1] +(47) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_moy#11] +Condition : ((((d_year#10 = 1999) OR ((d_year#10 = 1998) AND (d_moy#11 = 12))) OR ((d_year#10 = 2000) AND (d_moy#11 = 1))) AND isnotnull(d_date_sk#9)) -(48) Filter [codegen id : 1] +(48) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((d_year#10 = 1999) OR ((d_year#10 = 1998) AND (d_moy#11 = 12))) OR ((d_year#10 = 2000) AND (d_moy#11 = 1))) AND isnotnull(d_date_sk#9)) (49) BroadcastExchange Input [3]: [d_date_sk#9, d_year#10, d_moy#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/simplified.txt index 8e259bf7f..56e33be9e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/simplified.txt @@ -28,32 +28,32 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] BroadcastHashJoin [i_item_sk,cs_item_sk] - Filter [i_item_sk,i_category,i_brand] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk,i_category,i_brand] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - Filter [cs_item_sk,cs_call_center_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_item_sk,cs_call_center_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_year,d_moy,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter ReusedExchange [d_date_sk,d_year,d_moy] #4 InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - Filter [cc_call_center_sk,cc_name] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cc_call_center_sk,cc_name] CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/explain.txt index 8489cf66d..2769e772a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/explain.txt @@ -1,91 +1,87 @@ == Physical Plan == -TakeOrderedAndProject (87) -+- * HashAggregate (86) - +- Exchange (85) - +- * HashAggregate (84) - +- Union (83) - :- * HashAggregate (72) - : +- Exchange (71) - : +- * HashAggregate (70) - : +- Union (69) - : :- * HashAggregate (21) - : : +- Exchange (20) - : : +- * HashAggregate (19) - : : +- * Project (18) - : : +- * BroadcastHashJoin Inner BuildRight (17) - : : :- * Project (12) - : : : +- * BroadcastHashJoin Inner BuildRight (11) - : : : :- Union (9) - : : : : :- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- * Project (8) - : : : : +- * Filter (7) - : : : : +- * ColumnarToRow (6) - : : : : +- CometScan parquet spark_catalog.default.store_returns (5) - : : : +- ReusedExchange (10) - : : +- BroadcastExchange (16) - : : +- * Filter (15) - : : +- * ColumnarToRow (14) - : : +- CometScan parquet spark_catalog.default.store (13) - : :- * HashAggregate (42) - : : +- Exchange (41) - : : +- * HashAggregate (40) - : : +- * Project (39) - : : +- * BroadcastHashJoin Inner BuildRight (38) - : : :- * Project (33) - : : : +- * BroadcastHashJoin Inner BuildRight (32) - : : : :- Union (30) - : : : : :- * Project (25) - : : : : : +- * Filter (24) - : : : : : +- * ColumnarToRow (23) - : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (22) - : : : : +- * Project (29) - : : : : +- * Filter (28) - : : : : +- * ColumnarToRow (27) - : : : : +- CometScan parquet spark_catalog.default.catalog_returns (26) - : : : +- ReusedExchange (31) - : : +- BroadcastExchange (37) - : : +- * Filter (36) - : : +- * ColumnarToRow (35) - : : +- CometScan parquet spark_catalog.default.catalog_page (34) - : +- * HashAggregate (68) - : +- Exchange (67) - : +- * HashAggregate (66) - : +- * Project (65) - : +- * BroadcastHashJoin Inner BuildRight (64) - : :- * Project (59) - : : +- * BroadcastHashJoin Inner BuildRight (58) - : : :- Union (56) - : : : :- * Project (46) - : : : : +- * Filter (45) - : : : : +- * ColumnarToRow (44) - : : : : +- CometScan parquet spark_catalog.default.web_sales (43) - : : : +- * Project (55) - : : : +- * BroadcastHashJoin Inner BuildLeft (54) - : : : :- BroadcastExchange (49) - : : : : +- * ColumnarToRow (48) - : : : : +- CometScan parquet spark_catalog.default.web_returns (47) - : : : +- * Project (53) - : : : +- * Filter (52) - : : : +- * ColumnarToRow (51) - : : : +- CometScan parquet spark_catalog.default.web_sales (50) - : : +- ReusedExchange (57) - : +- BroadcastExchange (63) - : +- * Filter (62) - : +- * ColumnarToRow (61) - : +- CometScan parquet spark_catalog.default.web_site (60) - :- * HashAggregate (77) - : +- Exchange (76) - : +- * HashAggregate (75) - : +- * HashAggregate (74) - : +- ReusedExchange (73) - +- * HashAggregate (82) - +- Exchange (81) - +- * HashAggregate (80) - +- * HashAggregate (79) - +- ReusedExchange (78) +TakeOrderedAndProject (83) ++- * HashAggregate (82) + +- Exchange (81) + +- * HashAggregate (80) + +- Union (79) + :- * HashAggregate (68) + : +- Exchange (67) + : +- * HashAggregate (66) + : +- Union (65) + : :- * HashAggregate (20) + : : +- Exchange (19) + : : +- * HashAggregate (18) + : : +- * Project (17) + : : +- * BroadcastHashJoin Inner BuildRight (16) + : : :- * Project (11) + : : : +- * BroadcastHashJoin Inner BuildRight (10) + : : : :- * ColumnarToRow (8) + : : : : +- CometUnion (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan parquet spark_catalog.default.store_returns (4) + : : : +- ReusedExchange (9) + : : +- BroadcastExchange (15) + : : +- * ColumnarToRow (14) + : : +- CometFilter (13) + : : +- CometScan parquet spark_catalog.default.store (12) + : :- * HashAggregate (40) + : : +- Exchange (39) + : : +- * HashAggregate (38) + : : +- * Project (37) + : : +- * BroadcastHashJoin Inner BuildRight (36) + : : :- * Project (31) + : : : +- * BroadcastHashJoin Inner BuildRight (30) + : : : :- * ColumnarToRow (28) + : : : : +- CometUnion (27) + : : : : :- CometProject (23) + : : : : : +- CometFilter (22) + : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (21) + : : : : +- CometProject (26) + : : : : +- CometFilter (25) + : : : : +- CometScan parquet spark_catalog.default.catalog_returns (24) + : : : +- ReusedExchange (29) + : : +- BroadcastExchange (35) + : : +- * ColumnarToRow (34) + : : +- CometFilter (33) + : : +- CometScan parquet spark_catalog.default.catalog_page (32) + : +- * HashAggregate (64) + : +- Exchange (63) + : +- * HashAggregate (62) + : +- * Project (61) + : +- * BroadcastHashJoin Inner BuildRight (60) + : :- * Project (55) + : : +- * BroadcastHashJoin Inner BuildRight (54) + : : :- * ColumnarToRow (52) + : : : +- CometUnion (51) + : : : :- CometProject (43) + : : : : +- CometFilter (42) + : : : : +- CometScan parquet spark_catalog.default.web_sales (41) + : : : +- CometProject (50) + : : : +- CometBroadcastHashJoin (49) + : : : :- CometBroadcastExchange (45) + : : : : +- CometScan parquet spark_catalog.default.web_returns (44) + : : : +- CometProject (48) + : : : +- CometFilter (47) + : : : +- CometScan parquet spark_catalog.default.web_sales (46) + : : +- ReusedExchange (53) + : +- BroadcastExchange (59) + : +- * ColumnarToRow (58) + : +- CometFilter (57) + : +- CometScan parquet spark_catalog.default.web_site (56) + :- * HashAggregate (73) + : +- Exchange (72) + : +- * HashAggregate (71) + : +- * HashAggregate (70) + : +- ReusedExchange (69) + +- * HashAggregate (78) + +- Exchange (77) + +- * HashAggregate (76) + +- * HashAggregate (75) + +- ReusedExchange (74) (1) Scan parquet spark_catalog.default.store_sales @@ -96,18 +92,15 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Condition : isnotnull(ss_store_sk#1) -(4) Project [codegen id : 1] -Output [6]: [ss_store_sk#1 AS store_sk#6, ss_sold_date_sk#4 AS date_sk#7, ss_ext_sales_price#2 AS sales_price#8, ss_net_profit#3 AS profit#9, 0.00 AS return_amt#10, 0.00 AS net_loss#11] +(3) CometProject Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Arguments: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11], [ss_store_sk#1 AS store_sk#6, ss_sold_date_sk#4 AS date_sk#7, ss_ext_sales_price#2 AS sales_price#8, ss_net_profit#3 AS profit#9, 0.00 AS return_amt#10, 0.00 AS net_loss#11] -(5) Scan parquet spark_catalog.default.store_returns +(4) Scan parquet spark_catalog.default.store_returns Output [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] Batched: true Location: InMemoryFileIndex [] @@ -115,79 +108,81 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#15), dynamicpruningexpression(s PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 2] -Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] - -(7) Filter [codegen id : 2] +(5) CometFilter Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] Condition : isnotnull(sr_store_sk#12) -(8) Project [codegen id : 2] -Output [6]: [sr_store_sk#12 AS store_sk#16, sr_returned_date_sk#15 AS date_sk#17, 0.00 AS sales_price#18, 0.00 AS profit#19, sr_return_amt#13 AS return_amt#20, sr_net_loss#14 AS net_loss#21] +(6) CometProject Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] +Arguments: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21], [sr_store_sk#12 AS store_sk#16, sr_returned_date_sk#15 AS date_sk#17, 0.00 AS sales_price#18, 0.00 AS profit#19, sr_return_amt#13 AS return_amt#20, sr_net_loss#14 AS net_loss#21] + +(7) CometUnion +Child 0 Input [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] +Child 1 Input [6]: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21] -(9) Union +(8) ColumnarToRow [codegen id : 3] +Input [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] -(10) ReusedExchange [Reuses operator id: 92] +(9) ReusedExchange [Reuses operator id: 88] Output [1]: [d_date_sk#22] -(11) BroadcastHashJoin [codegen id : 5] +(10) BroadcastHashJoin [codegen id : 3] Left keys [1]: [date_sk#7] Right keys [1]: [d_date_sk#22] Join type: Inner Join condition: None -(12) Project [codegen id : 5] +(11) Project [codegen id : 3] Output [5]: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] Input [7]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, d_date_sk#22] -(13) Scan parquet spark_catalog.default.store +(12) Scan parquet spark_catalog.default.store Output [2]: [s_store_sk#23, s_store_id#24] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 4] +(13) CometFilter Input [2]: [s_store_sk#23, s_store_id#24] +Condition : isnotnull(s_store_sk#23) -(15) Filter [codegen id : 4] +(14) ColumnarToRow [codegen id : 2] Input [2]: [s_store_sk#23, s_store_id#24] -Condition : isnotnull(s_store_sk#23) -(16) BroadcastExchange +(15) BroadcastExchange Input [2]: [s_store_sk#23, s_store_id#24] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] -(17) BroadcastHashJoin [codegen id : 5] +(16) BroadcastHashJoin [codegen id : 3] Left keys [1]: [store_sk#6] Right keys [1]: [s_store_sk#23] Join type: Inner Join condition: None -(18) Project [codegen id : 5] +(17) Project [codegen id : 3] Output [5]: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#24] Input [7]: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_sk#23, s_store_id#24] -(19) HashAggregate [codegen id : 5] +(18) HashAggregate [codegen id : 3] Input [5]: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#24] Keys [1]: [s_store_id#24] Functions [4]: [partial_sum(UnscaledValue(sales_price#8)), partial_sum(UnscaledValue(return_amt#10)), partial_sum(UnscaledValue(profit#9)), partial_sum(UnscaledValue(net_loss#11))] Aggregate Attributes [4]: [sum#25, sum#26, sum#27, sum#28] Results [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] -(20) Exchange +(19) Exchange Input [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] Arguments: hashpartitioning(s_store_id#24, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(21) HashAggregate [codegen id : 6] +(20) HashAggregate [codegen id : 4] Input [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] Keys [1]: [s_store_id#24] Functions [4]: [sum(UnscaledValue(sales_price#8)), sum(UnscaledValue(return_amt#10)), sum(UnscaledValue(profit#9)), sum(UnscaledValue(net_loss#11))] Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#8))#33, sum(UnscaledValue(return_amt#10))#34, sum(UnscaledValue(profit#9))#35, sum(UnscaledValue(net_loss#11))#36] Results [5]: [store channel AS channel#37, concat(store, s_store_id#24) AS id#38, MakeDecimal(sum(UnscaledValue(sales_price#8))#33,17,2) AS sales#39, MakeDecimal(sum(UnscaledValue(return_amt#10))#34,17,2) AS returns#40, (MakeDecimal(sum(UnscaledValue(profit#9))#35,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#11))#36,17,2)) AS profit#41] -(22) Scan parquet spark_catalog.default.catalog_sales +(21) Scan parquet spark_catalog.default.catalog_sales Output [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] Batched: true Location: InMemoryFileIndex [] @@ -195,18 +190,15 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#45), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_catalog_page_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 7] -Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] - -(24) Filter [codegen id : 7] +(22) CometFilter Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] Condition : isnotnull(cs_catalog_page_sk#42) -(25) Project [codegen id : 7] -Output [6]: [cs_catalog_page_sk#42 AS page_sk#47, cs_sold_date_sk#45 AS date_sk#48, cs_ext_sales_price#43 AS sales_price#49, cs_net_profit#44 AS profit#50, 0.00 AS return_amt#51, 0.00 AS net_loss#52] +(23) CometProject Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +Arguments: [page_sk#47, date_sk#48, sales_price#49, profit#50, return_amt#51, net_loss#52], [cs_catalog_page_sk#42 AS page_sk#47, cs_sold_date_sk#45 AS date_sk#48, cs_ext_sales_price#43 AS sales_price#49, cs_net_profit#44 AS profit#50, 0.00 AS return_amt#51, 0.00 AS net_loss#52] -(26) Scan parquet spark_catalog.default.catalog_returns +(24) Scan parquet spark_catalog.default.catalog_returns Output [4]: [cr_catalog_page_sk#53, cr_return_amount#54, cr_net_loss#55, cr_returned_date_sk#56] Batched: true Location: InMemoryFileIndex [] @@ -214,79 +206,81 @@ PartitionFilters: [isnotnull(cr_returned_date_sk#56), dynamicpruningexpression(c PushedFilters: [IsNotNull(cr_catalog_page_sk)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 8] -Input [4]: [cr_catalog_page_sk#53, cr_return_amount#54, cr_net_loss#55, cr_returned_date_sk#56] - -(28) Filter [codegen id : 8] +(25) CometFilter Input [4]: [cr_catalog_page_sk#53, cr_return_amount#54, cr_net_loss#55, cr_returned_date_sk#56] Condition : isnotnull(cr_catalog_page_sk#53) -(29) Project [codegen id : 8] -Output [6]: [cr_catalog_page_sk#53 AS page_sk#57, cr_returned_date_sk#56 AS date_sk#58, 0.00 AS sales_price#59, 0.00 AS profit#60, cr_return_amount#54 AS return_amt#61, cr_net_loss#55 AS net_loss#62] +(26) CometProject Input [4]: [cr_catalog_page_sk#53, cr_return_amount#54, cr_net_loss#55, cr_returned_date_sk#56] +Arguments: [page_sk#57, date_sk#58, sales_price#59, profit#60, return_amt#61, net_loss#62], [cr_catalog_page_sk#53 AS page_sk#57, cr_returned_date_sk#56 AS date_sk#58, 0.00 AS sales_price#59, 0.00 AS profit#60, cr_return_amount#54 AS return_amt#61, cr_net_loss#55 AS net_loss#62] -(30) Union +(27) CometUnion +Child 0 Input [6]: [page_sk#47, date_sk#48, sales_price#49, profit#50, return_amt#51, net_loss#52] +Child 1 Input [6]: [page_sk#57, date_sk#58, sales_price#59, profit#60, return_amt#61, net_loss#62] -(31) ReusedExchange [Reuses operator id: 92] +(28) ColumnarToRow [codegen id : 7] +Input [6]: [page_sk#47, date_sk#48, sales_price#49, profit#50, return_amt#51, net_loss#52] + +(29) ReusedExchange [Reuses operator id: 88] Output [1]: [d_date_sk#63] -(32) BroadcastHashJoin [codegen id : 11] +(30) BroadcastHashJoin [codegen id : 7] Left keys [1]: [date_sk#48] Right keys [1]: [d_date_sk#63] Join type: Inner Join condition: None -(33) Project [codegen id : 11] +(31) Project [codegen id : 7] Output [5]: [page_sk#47, sales_price#49, profit#50, return_amt#51, net_loss#52] Input [7]: [page_sk#47, date_sk#48, sales_price#49, profit#50, return_amt#51, net_loss#52, d_date_sk#63] -(34) Scan parquet spark_catalog.default.catalog_page +(32) Scan parquet spark_catalog.default.catalog_page Output [2]: [cp_catalog_page_sk#64, cp_catalog_page_id#65] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 10] +(33) CometFilter Input [2]: [cp_catalog_page_sk#64, cp_catalog_page_id#65] +Condition : isnotnull(cp_catalog_page_sk#64) -(36) Filter [codegen id : 10] +(34) ColumnarToRow [codegen id : 6] Input [2]: [cp_catalog_page_sk#64, cp_catalog_page_id#65] -Condition : isnotnull(cp_catalog_page_sk#64) -(37) BroadcastExchange +(35) BroadcastExchange Input [2]: [cp_catalog_page_sk#64, cp_catalog_page_id#65] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] -(38) BroadcastHashJoin [codegen id : 11] +(36) BroadcastHashJoin [codegen id : 7] Left keys [1]: [page_sk#47] Right keys [1]: [cp_catalog_page_sk#64] Join type: Inner Join condition: None -(39) Project [codegen id : 11] +(37) Project [codegen id : 7] Output [5]: [sales_price#49, profit#50, return_amt#51, net_loss#52, cp_catalog_page_id#65] Input [7]: [page_sk#47, sales_price#49, profit#50, return_amt#51, net_loss#52, cp_catalog_page_sk#64, cp_catalog_page_id#65] -(40) HashAggregate [codegen id : 11] +(38) HashAggregate [codegen id : 7] Input [5]: [sales_price#49, profit#50, return_amt#51, net_loss#52, cp_catalog_page_id#65] Keys [1]: [cp_catalog_page_id#65] Functions [4]: [partial_sum(UnscaledValue(sales_price#49)), partial_sum(UnscaledValue(return_amt#51)), partial_sum(UnscaledValue(profit#50)), partial_sum(UnscaledValue(net_loss#52))] Aggregate Attributes [4]: [sum#66, sum#67, sum#68, sum#69] Results [5]: [cp_catalog_page_id#65, sum#70, sum#71, sum#72, sum#73] -(41) Exchange +(39) Exchange Input [5]: [cp_catalog_page_id#65, sum#70, sum#71, sum#72, sum#73] Arguments: hashpartitioning(cp_catalog_page_id#65, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(42) HashAggregate [codegen id : 12] +(40) HashAggregate [codegen id : 8] Input [5]: [cp_catalog_page_id#65, sum#70, sum#71, sum#72, sum#73] Keys [1]: [cp_catalog_page_id#65] Functions [4]: [sum(UnscaledValue(sales_price#49)), sum(UnscaledValue(return_amt#51)), sum(UnscaledValue(profit#50)), sum(UnscaledValue(net_loss#52))] Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#49))#74, sum(UnscaledValue(return_amt#51))#75, sum(UnscaledValue(profit#50))#76, sum(UnscaledValue(net_loss#52))#77] Results [5]: [catalog channel AS channel#78, concat(catalog_page, cp_catalog_page_id#65) AS id#79, MakeDecimal(sum(UnscaledValue(sales_price#49))#74,17,2) AS sales#80, MakeDecimal(sum(UnscaledValue(return_amt#51))#75,17,2) AS returns#81, (MakeDecimal(sum(UnscaledValue(profit#50))#76,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#52))#77,17,2)) AS profit#82] -(43) Scan parquet spark_catalog.default.web_sales +(41) Scan parquet spark_catalog.default.web_sales Output [4]: [ws_web_site_sk#83, ws_ext_sales_price#84, ws_net_profit#85, ws_sold_date_sk#86] Batched: true Location: InMemoryFileIndex [] @@ -294,260 +288,255 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#86), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_web_site_sk)] ReadSchema: struct -(44) ColumnarToRow [codegen id : 13] -Input [4]: [ws_web_site_sk#83, ws_ext_sales_price#84, ws_net_profit#85, ws_sold_date_sk#86] - -(45) Filter [codegen id : 13] +(42) CometFilter Input [4]: [ws_web_site_sk#83, ws_ext_sales_price#84, ws_net_profit#85, ws_sold_date_sk#86] Condition : isnotnull(ws_web_site_sk#83) -(46) Project [codegen id : 13] -Output [6]: [ws_web_site_sk#83 AS wsr_web_site_sk#88, ws_sold_date_sk#86 AS date_sk#89, ws_ext_sales_price#84 AS sales_price#90, ws_net_profit#85 AS profit#91, 0.00 AS return_amt#92, 0.00 AS net_loss#93] +(43) CometProject Input [4]: [ws_web_site_sk#83, ws_ext_sales_price#84, ws_net_profit#85, ws_sold_date_sk#86] +Arguments: [wsr_web_site_sk#88, date_sk#89, sales_price#90, profit#91, return_amt#92, net_loss#93], [ws_web_site_sk#83 AS wsr_web_site_sk#88, ws_sold_date_sk#86 AS date_sk#89, ws_ext_sales_price#84 AS sales_price#90, ws_net_profit#85 AS profit#91, 0.00 AS return_amt#92, 0.00 AS net_loss#93] -(47) Scan parquet spark_catalog.default.web_returns +(44) Scan parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(wr_returned_date_sk#98), dynamicpruningexpression(wr_returned_date_sk#98 IN dynamicpruning#87)] ReadSchema: struct -(48) ColumnarToRow [codegen id : 14] -Input [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98] - -(49) BroadcastExchange +(45) CometBroadcastExchange Input [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, true] as bigint), 32) | (cast(input[1, int, true] as bigint) & 4294967295))),false), [plan_id=5] +Arguments: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98] -(50) Scan parquet spark_catalog.default.web_sales +(46) Scan parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#99, ws_web_site_sk#100, ws_order_number#101, ws_sold_date_sk#102] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] ReadSchema: struct -(51) ColumnarToRow -Input [4]: [ws_item_sk#99, ws_web_site_sk#100, ws_order_number#101, ws_sold_date_sk#102] - -(52) Filter +(47) CometFilter Input [4]: [ws_item_sk#99, ws_web_site_sk#100, ws_order_number#101, ws_sold_date_sk#102] Condition : ((isnotnull(ws_item_sk#99) AND isnotnull(ws_order_number#101)) AND isnotnull(ws_web_site_sk#100)) -(53) Project -Output [3]: [ws_item_sk#99, ws_web_site_sk#100, ws_order_number#101] +(48) CometProject Input [4]: [ws_item_sk#99, ws_web_site_sk#100, ws_order_number#101, ws_sold_date_sk#102] +Arguments: [ws_item_sk#99, ws_web_site_sk#100, ws_order_number#101], [ws_item_sk#99, ws_web_site_sk#100, ws_order_number#101] -(54) BroadcastHashJoin [codegen id : 15] -Left keys [2]: [wr_item_sk#94, wr_order_number#95] -Right keys [2]: [ws_item_sk#99, ws_order_number#101] -Join type: Inner -Join condition: None +(49) CometBroadcastHashJoin +Left output [5]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98] +Right output [3]: [ws_item_sk#99, ws_web_site_sk#100, ws_order_number#101] +Arguments: [wr_item_sk#94, wr_order_number#95], [ws_item_sk#99, ws_order_number#101], Inner -(55) Project [codegen id : 15] -Output [6]: [ws_web_site_sk#100 AS wsr_web_site_sk#103, wr_returned_date_sk#98 AS date_sk#104, 0.00 AS sales_price#105, 0.00 AS profit#106, wr_return_amt#96 AS return_amt#107, wr_net_loss#97 AS net_loss#108] +(50) CometProject Input [8]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97, wr_returned_date_sk#98, ws_item_sk#99, ws_web_site_sk#100, ws_order_number#101] +Arguments: [wsr_web_site_sk#103, date_sk#104, sales_price#105, profit#106, return_amt#107, net_loss#108], [ws_web_site_sk#100 AS wsr_web_site_sk#103, wr_returned_date_sk#98 AS date_sk#104, 0.00 AS sales_price#105, 0.00 AS profit#106, wr_return_amt#96 AS return_amt#107, wr_net_loss#97 AS net_loss#108] -(56) Union +(51) CometUnion +Child 0 Input [6]: [wsr_web_site_sk#88, date_sk#89, sales_price#90, profit#91, return_amt#92, net_loss#93] +Child 1 Input [6]: [wsr_web_site_sk#103, date_sk#104, sales_price#105, profit#106, return_amt#107, net_loss#108] -(57) ReusedExchange [Reuses operator id: 92] +(52) ColumnarToRow [codegen id : 11] +Input [6]: [wsr_web_site_sk#88, date_sk#89, sales_price#90, profit#91, return_amt#92, net_loss#93] + +(53) ReusedExchange [Reuses operator id: 88] Output [1]: [d_date_sk#109] -(58) BroadcastHashJoin [codegen id : 18] +(54) BroadcastHashJoin [codegen id : 11] Left keys [1]: [date_sk#89] Right keys [1]: [d_date_sk#109] Join type: Inner Join condition: None -(59) Project [codegen id : 18] +(55) Project [codegen id : 11] Output [5]: [wsr_web_site_sk#88, sales_price#90, profit#91, return_amt#92, net_loss#93] Input [7]: [wsr_web_site_sk#88, date_sk#89, sales_price#90, profit#91, return_amt#92, net_loss#93, d_date_sk#109] -(60) Scan parquet spark_catalog.default.web_site +(56) Scan parquet spark_catalog.default.web_site Output [2]: [web_site_sk#110, web_site_id#111] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(61) ColumnarToRow [codegen id : 17] +(57) CometFilter Input [2]: [web_site_sk#110, web_site_id#111] +Condition : isnotnull(web_site_sk#110) -(62) Filter [codegen id : 17] +(58) ColumnarToRow [codegen id : 10] Input [2]: [web_site_sk#110, web_site_id#111] -Condition : isnotnull(web_site_sk#110) -(63) BroadcastExchange +(59) BroadcastExchange Input [2]: [web_site_sk#110, web_site_id#111] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] -(64) BroadcastHashJoin [codegen id : 18] +(60) BroadcastHashJoin [codegen id : 11] Left keys [1]: [wsr_web_site_sk#88] Right keys [1]: [web_site_sk#110] Join type: Inner Join condition: None -(65) Project [codegen id : 18] +(61) Project [codegen id : 11] Output [5]: [sales_price#90, profit#91, return_amt#92, net_loss#93, web_site_id#111] Input [7]: [wsr_web_site_sk#88, sales_price#90, profit#91, return_amt#92, net_loss#93, web_site_sk#110, web_site_id#111] -(66) HashAggregate [codegen id : 18] +(62) HashAggregate [codegen id : 11] Input [5]: [sales_price#90, profit#91, return_amt#92, net_loss#93, web_site_id#111] Keys [1]: [web_site_id#111] Functions [4]: [partial_sum(UnscaledValue(sales_price#90)), partial_sum(UnscaledValue(return_amt#92)), partial_sum(UnscaledValue(profit#91)), partial_sum(UnscaledValue(net_loss#93))] Aggregate Attributes [4]: [sum#112, sum#113, sum#114, sum#115] Results [5]: [web_site_id#111, sum#116, sum#117, sum#118, sum#119] -(67) Exchange +(63) Exchange Input [5]: [web_site_id#111, sum#116, sum#117, sum#118, sum#119] -Arguments: hashpartitioning(web_site_id#111, 5), ENSURE_REQUIREMENTS, [plan_id=7] +Arguments: hashpartitioning(web_site_id#111, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(68) HashAggregate [codegen id : 19] +(64) HashAggregate [codegen id : 12] Input [5]: [web_site_id#111, sum#116, sum#117, sum#118, sum#119] Keys [1]: [web_site_id#111] Functions [4]: [sum(UnscaledValue(sales_price#90)), sum(UnscaledValue(return_amt#92)), sum(UnscaledValue(profit#91)), sum(UnscaledValue(net_loss#93))] Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#90))#120, sum(UnscaledValue(return_amt#92))#121, sum(UnscaledValue(profit#91))#122, sum(UnscaledValue(net_loss#93))#123] Results [5]: [web channel AS channel#124, concat(web_site, web_site_id#111) AS id#125, MakeDecimal(sum(UnscaledValue(sales_price#90))#120,17,2) AS sales#126, MakeDecimal(sum(UnscaledValue(return_amt#92))#121,17,2) AS returns#127, (MakeDecimal(sum(UnscaledValue(profit#91))#122,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#93))#123,17,2)) AS profit#128] -(69) Union +(65) Union -(70) HashAggregate [codegen id : 20] +(66) HashAggregate [codegen id : 13] Input [5]: [channel#37, id#38, sales#39, returns#40, profit#41] Keys [2]: [channel#37, id#38] Functions [3]: [partial_sum(sales#39), partial_sum(returns#40), partial_sum(profit#41)] Aggregate Attributes [6]: [sum#129, isEmpty#130, sum#131, isEmpty#132, sum#133, isEmpty#134] Results [8]: [channel#37, id#38, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] -(71) Exchange +(67) Exchange Input [8]: [channel#37, id#38, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] -Arguments: hashpartitioning(channel#37, id#38, 5), ENSURE_REQUIREMENTS, [plan_id=8] +Arguments: hashpartitioning(channel#37, id#38, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(72) HashAggregate [codegen id : 21] +(68) HashAggregate [codegen id : 14] Input [8]: [channel#37, id#38, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] Keys [2]: [channel#37, id#38] Functions [3]: [sum(sales#39), sum(returns#40), sum(profit#41)] Aggregate Attributes [3]: [sum(sales#39)#141, sum(returns#40)#142, sum(profit#41)#143] Results [5]: [channel#37, id#38, cast(sum(sales#39)#141 as decimal(37,2)) AS sales#144, cast(sum(returns#40)#142 as decimal(37,2)) AS returns#145, cast(sum(profit#41)#143 as decimal(38,2)) AS profit#146] -(73) ReusedExchange [Reuses operator id: 71] +(69) ReusedExchange [Reuses operator id: 67] Output [8]: [channel#147, id#148, sum#149, isEmpty#150, sum#151, isEmpty#152, sum#153, isEmpty#154] -(74) HashAggregate [codegen id : 42] +(70) HashAggregate [codegen id : 28] Input [8]: [channel#147, id#148, sum#149, isEmpty#150, sum#151, isEmpty#152, sum#153, isEmpty#154] Keys [2]: [channel#147, id#148] Functions [3]: [sum(sales#155), sum(returns#156), sum(profit#157)] Aggregate Attributes [3]: [sum(sales#155)#141, sum(returns#156)#142, sum(profit#157)#143] Results [4]: [channel#147, sum(sales#155)#141 AS sales#158, sum(returns#156)#142 AS returns#159, sum(profit#157)#143 AS profit#160] -(75) HashAggregate [codegen id : 42] +(71) HashAggregate [codegen id : 28] Input [4]: [channel#147, sales#158, returns#159, profit#160] Keys [1]: [channel#147] Functions [3]: [partial_sum(sales#158), partial_sum(returns#159), partial_sum(profit#160)] Aggregate Attributes [6]: [sum#161, isEmpty#162, sum#163, isEmpty#164, sum#165, isEmpty#166] Results [7]: [channel#147, sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172] -(76) Exchange +(72) Exchange Input [7]: [channel#147, sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172] -Arguments: hashpartitioning(channel#147, 5), ENSURE_REQUIREMENTS, [plan_id=9] +Arguments: hashpartitioning(channel#147, 5), ENSURE_REQUIREMENTS, [plan_id=8] -(77) HashAggregate [codegen id : 43] +(73) HashAggregate [codegen id : 29] Input [7]: [channel#147, sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172] Keys [1]: [channel#147] Functions [3]: [sum(sales#158), sum(returns#159), sum(profit#160)] Aggregate Attributes [3]: [sum(sales#158)#173, sum(returns#159)#174, sum(profit#160)#175] Results [5]: [channel#147, null AS id#176, sum(sales#158)#173 AS sum(sales)#177, sum(returns#159)#174 AS sum(returns)#178, sum(profit#160)#175 AS sum(profit)#179] -(78) ReusedExchange [Reuses operator id: 71] +(74) ReusedExchange [Reuses operator id: 67] Output [8]: [channel#180, id#181, sum#182, isEmpty#183, sum#184, isEmpty#185, sum#186, isEmpty#187] -(79) HashAggregate [codegen id : 64] +(75) HashAggregate [codegen id : 43] Input [8]: [channel#180, id#181, sum#182, isEmpty#183, sum#184, isEmpty#185, sum#186, isEmpty#187] Keys [2]: [channel#180, id#181] Functions [3]: [sum(sales#188), sum(returns#189), sum(profit#190)] Aggregate Attributes [3]: [sum(sales#188)#141, sum(returns#189)#142, sum(profit#190)#143] Results [3]: [sum(sales#188)#141 AS sales#191, sum(returns#189)#142 AS returns#192, sum(profit#190)#143 AS profit#193] -(80) HashAggregate [codegen id : 64] +(76) HashAggregate [codegen id : 43] Input [3]: [sales#191, returns#192, profit#193] Keys: [] Functions [3]: [partial_sum(sales#191), partial_sum(returns#192), partial_sum(profit#193)] Aggregate Attributes [6]: [sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199] Results [6]: [sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205] -(81) Exchange +(77) Exchange Input [6]: [sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=9] -(82) HashAggregate [codegen id : 65] +(78) HashAggregate [codegen id : 44] Input [6]: [sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205] Keys: [] Functions [3]: [sum(sales#191), sum(returns#192), sum(profit#193)] Aggregate Attributes [3]: [sum(sales#191)#206, sum(returns#192)#207, sum(profit#193)#208] Results [5]: [null AS channel#209, null AS id#210, sum(sales#191)#206 AS sum(sales)#211, sum(returns#192)#207 AS sum(returns)#212, sum(profit#193)#208 AS sum(profit)#213] -(83) Union +(79) Union -(84) HashAggregate [codegen id : 66] +(80) HashAggregate [codegen id : 45] Input [5]: [channel#37, id#38, sales#144, returns#145, profit#146] Keys [5]: [channel#37, id#38, sales#144, returns#145, profit#146] Functions: [] Aggregate Attributes: [] Results [5]: [channel#37, id#38, sales#144, returns#145, profit#146] -(85) Exchange +(81) Exchange Input [5]: [channel#37, id#38, sales#144, returns#145, profit#146] -Arguments: hashpartitioning(channel#37, id#38, sales#144, returns#145, profit#146, 5), ENSURE_REQUIREMENTS, [plan_id=11] +Arguments: hashpartitioning(channel#37, id#38, sales#144, returns#145, profit#146, 5), ENSURE_REQUIREMENTS, [plan_id=10] -(86) HashAggregate [codegen id : 67] +(82) HashAggregate [codegen id : 46] Input [5]: [channel#37, id#38, sales#144, returns#145, profit#146] Keys [5]: [channel#37, id#38, sales#144, returns#145, profit#146] Functions: [] Aggregate Attributes: [] Results [5]: [channel#37, id#38, sales#144, returns#145, profit#146] -(87) TakeOrderedAndProject +(83) TakeOrderedAndProject Input [5]: [channel#37, id#38, sales#144, returns#145, profit#146] Arguments: 100, [channel#37 ASC NULLS FIRST, id#38 ASC NULLS FIRST], [channel#37, id#38, sales#144, returns#145, profit#146] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (92) -+- * Project (91) - +- * Filter (90) - +- * ColumnarToRow (89) - +- CometScan parquet spark_catalog.default.date_dim (88) +BroadcastExchange (88) ++- * ColumnarToRow (87) + +- CometProject (86) + +- CometFilter (85) + +- CometScan parquet spark_catalog.default.date_dim (84) -(88) Scan parquet spark_catalog.default.date_dim +(84) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_date#214] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-08-18), IsNotNull(d_date_sk)] ReadSchema: struct -(89) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#22, d_date#214] - -(90) Filter [codegen id : 1] +(85) CometFilter Input [2]: [d_date_sk#22, d_date#214] Condition : (((isnotnull(d_date#214) AND (d_date#214 >= 1998-08-04)) AND (d_date#214 <= 1998-08-18)) AND isnotnull(d_date_sk#22)) -(91) Project [codegen id : 1] -Output [1]: [d_date_sk#22] +(86) CometProject Input [2]: [d_date_sk#22, d_date#214] +Arguments: [d_date_sk#22], [d_date_sk#22] + +(87) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#22] -(92) BroadcastExchange +(88) BroadcastExchange Input [1]: [d_date_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] -Subquery:2 Hosting operator id = 5 Hosting Expression = sr_returned_date_sk#15 IN dynamicpruning#5 +Subquery:2 Hosting operator id = 4 Hosting Expression = sr_returned_date_sk#15 IN dynamicpruning#5 -Subquery:3 Hosting operator id = 22 Hosting Expression = cs_sold_date_sk#45 IN dynamicpruning#5 +Subquery:3 Hosting operator id = 21 Hosting Expression = cs_sold_date_sk#45 IN dynamicpruning#5 -Subquery:4 Hosting operator id = 26 Hosting Expression = cr_returned_date_sk#56 IN dynamicpruning#5 +Subquery:4 Hosting operator id = 24 Hosting Expression = cr_returned_date_sk#56 IN dynamicpruning#5 -Subquery:5 Hosting operator id = 43 Hosting Expression = ws_sold_date_sk#86 IN dynamicpruning#5 +Subquery:5 Hosting operator id = 41 Hosting Expression = ws_sold_date_sk#86 IN dynamicpruning#5 -Subquery:6 Hosting operator id = 47 Hosting Expression = wr_returned_date_sk#98 IN dynamicpruning#5 +Subquery:6 Hosting operator id = 44 Hosting Expression = wr_returned_date_sk#98 IN dynamicpruning#5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/simplified.txt index f132fec3b..aaec304fd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/simplified.txt @@ -1,154 +1,135 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (67) + WholeStageCodegen (46) HashAggregate [channel,id,sales,returns,profit] InputAdapter Exchange [channel,id,sales,returns,profit] #1 - WholeStageCodegen (66) + WholeStageCodegen (45) HashAggregate [channel,id,sales,returns,profit] InputAdapter Union - WholeStageCodegen (21) + WholeStageCodegen (14) HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] InputAdapter Exchange [channel,id] #2 - WholeStageCodegen (20) + WholeStageCodegen (13) HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] InputAdapter Union - WholeStageCodegen (6) + WholeStageCodegen (4) HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] InputAdapter Exchange [s_store_id] #3 - WholeStageCodegen (5) + WholeStageCodegen (3) HashAggregate [s_store_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] Project [sales_price,profit,return_amt,net_loss,s_store_id] BroadcastHashJoin [store_sk,s_store_sk] Project [store_sk,sales_price,profit,return_amt,net_loss] BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (1) - Project [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - WholeStageCodegen (2) - Project [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] - Filter [sr_store_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 + ColumnarToRow + InputAdapter + CometUnion + CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ss_store_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date,d_date_sk] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [sr_store_sk] + CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 InputAdapter ReusedExchange [d_date_sk] #4 InputAdapter BroadcastExchange #5 - WholeStageCodegen (4) - Filter [s_store_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (2) + ColumnarToRow + InputAdapter + CometFilter [s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - WholeStageCodegen (12) + WholeStageCodegen (8) HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] InputAdapter Exchange [cp_catalog_page_id] #6 - WholeStageCodegen (11) + WholeStageCodegen (7) HashAggregate [cp_catalog_page_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] Project [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] BroadcastHashJoin [page_sk,cp_catalog_page_sk] Project [page_sk,sales_price,profit,return_amt,net_loss] BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (7) - Project [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] - Filter [cs_catalog_page_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - WholeStageCodegen (8) - Project [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] - Filter [cr_catalog_page_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 + ColumnarToRow + InputAdapter + CometUnion + CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cs_catalog_page_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cr_catalog_page_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 InputAdapter ReusedExchange [d_date_sk] #4 InputAdapter BroadcastExchange #7 - WholeStageCodegen (10) - Filter [cp_catalog_page_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (6) + ColumnarToRow + InputAdapter + CometFilter [cp_catalog_page_sk] CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - WholeStageCodegen (19) + WholeStageCodegen (12) HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] InputAdapter Exchange [web_site_id] #8 - WholeStageCodegen (18) + WholeStageCodegen (11) HashAggregate [web_site_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] Project [sales_price,profit,return_amt,net_loss,web_site_id] BroadcastHashJoin [wsr_web_site_sk,web_site_sk] Project [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (13) - Project [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] - Filter [ws_web_site_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - WholeStageCodegen (15) - Project [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] - BroadcastHashJoin [wr_item_sk,wr_order_number,ws_item_sk,ws_order_number] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (14) - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - Project [ws_item_sk,ws_web_site_sk,ws_order_number] - Filter [ws_item_sk,ws_order_number,ws_web_site_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + ColumnarToRow + InputAdapter + CometUnion + CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ws_web_site_sk] + CometScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wr_item_sk,wr_order_number,ws_item_sk,ws_order_number] + CometBroadcastExchange #9 + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] + CometFilter [ws_item_sk,ws_order_number,ws_web_site_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] InputAdapter ReusedExchange [d_date_sk] #4 InputAdapter BroadcastExchange #10 - WholeStageCodegen (17) - Filter [web_site_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (10) + ColumnarToRow + InputAdapter + CometFilter [web_site_sk] CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - WholeStageCodegen (43) + WholeStageCodegen (29) HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] InputAdapter Exchange [channel] #11 - WholeStageCodegen (42) + WholeStageCodegen (28) HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] InputAdapter ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (65) + WholeStageCodegen (44) HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] InputAdapter Exchange #12 - WholeStageCodegen (64) + WholeStageCodegen (43) HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/explain.txt index 7e0bfdaf7..a71a3a875 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/explain.txt @@ -12,31 +12,31 @@ TakeOrderedAndProject (39) : : +- * BroadcastHashJoin Inner BuildRight (14) : : :- * Project (9) : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.customer_address (1) : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) + : : : +- * ColumnarToRow (6) + : : : +- CometFilter (5) : : : +- CometScan parquet spark_catalog.default.customer (4) : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) + : : +- * ColumnarToRow (12) + : : +- CometFilter (11) : : +- CometScan parquet spark_catalog.default.store_sales (10) : +- ReusedExchange (16) +- BroadcastExchange (32) +- * Project (31) +- * BroadcastHashJoin Inner BuildRight (30) - :- * Filter (21) - : +- * ColumnarToRow (20) + :- * ColumnarToRow (21) + : +- CometFilter (20) : +- CometScan parquet spark_catalog.default.item (19) +- BroadcastExchange (29) +- * Filter (28) +- * HashAggregate (27) +- Exchange (26) +- * HashAggregate (25) - +- * Filter (24) - +- * ColumnarToRow (23) + +- * ColumnarToRow (24) + +- CometFilter (23) +- CometScan parquet spark_catalog.default.item (22) @@ -47,12 +47,12 @@ Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 7] +(2) CometFilter Input [2]: [ca_address_sk#1, ca_state#2] +Condition : isnotnull(ca_address_sk#1) -(3) Filter [codegen id : 7] +(3) ColumnarToRow [codegen id : 7] Input [2]: [ca_address_sk#1, ca_state#2] -Condition : isnotnull(ca_address_sk#1) (4) Scan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#3, c_current_addr_sk#4] @@ -61,12 +61,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [2]: [c_customer_sk#3, c_current_addr_sk#4] +Condition : (isnotnull(c_current_addr_sk#4) AND isnotnull(c_customer_sk#3)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Condition : (isnotnull(c_current_addr_sk#4) AND isnotnull(c_customer_sk#3)) (7) BroadcastExchange Input [2]: [c_customer_sk#3, c_current_addr_sk#4] @@ -90,12 +90,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] +(11) CometFilter Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] +Condition : (isnotnull(ss_customer_sk#6) AND isnotnull(ss_item_sk#5)) -(12) Filter [codegen id : 2] +(12) ColumnarToRow [codegen id : 2] Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_customer_sk#6) AND isnotnull(ss_item_sk#5)) (13) BroadcastExchange Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] @@ -131,12 +131,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_category), IsNotNull(i_item_sk)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 6] +(20) CometFilter Input [3]: [i_item_sk#10, i_current_price#11, i_category#12] +Condition : ((isnotnull(i_current_price#11) AND isnotnull(i_category#12)) AND isnotnull(i_item_sk#10)) -(21) Filter [codegen id : 6] +(21) ColumnarToRow [codegen id : 6] Input [3]: [i_item_sk#10, i_current_price#11, i_category#12] -Condition : ((isnotnull(i_current_price#11) AND isnotnull(i_category#12)) AND isnotnull(i_item_sk#10)) (22) Scan parquet spark_catalog.default.item Output [2]: [i_current_price#13, i_category#14] @@ -145,12 +145,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_category)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 4] +(23) CometFilter Input [2]: [i_current_price#13, i_category#14] +Condition : isnotnull(i_category#14) -(24) Filter [codegen id : 4] +(24) ColumnarToRow [codegen id : 4] Input [2]: [i_current_price#13, i_category#14] -Condition : isnotnull(i_category#14) (25) HashAggregate [codegen id : 4] Input [2]: [i_current_price#13, i_category#14] @@ -232,9 +232,9 @@ Arguments: 100, [cnt#25 ASC NULLS FIRST, state#24 ASC NULLS FIRST], [state#24, c Subquery:1 Hosting operator id = 10 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 BroadcastExchange (44) -+- * Project (43) - +- * Filter (42) - +- * ColumnarToRow (41) ++- * ColumnarToRow (43) + +- CometProject (42) + +- CometFilter (41) +- CometScan parquet spark_catalog.default.date_dim (40) @@ -245,30 +245,30 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), EqualTo(d_month_seq,ScalarSubquery#27), IsNotNull(d_date_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#9, d_month_seq#26] - -(42) Filter [codegen id : 1] +(41) CometFilter Input [2]: [d_date_sk#9, d_month_seq#26] Condition : ((isnotnull(d_month_seq#26) AND (d_month_seq#26 = ReusedSubquery Subquery scalar-subquery#27, [id=#28])) AND isnotnull(d_date_sk#9)) -(43) Project [codegen id : 1] -Output [1]: [d_date_sk#9] +(42) CometProject Input [2]: [d_date_sk#9, d_month_seq#26] +Arguments: [d_date_sk#9], [d_date_sk#9] + +(43) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#9] (44) BroadcastExchange Input [1]: [d_date_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] -Subquery:2 Hosting operator id = 42 Hosting Expression = ReusedSubquery Subquery scalar-subquery#27, [id=#28] +Subquery:2 Hosting operator id = 41 Hosting Expression = ReusedSubquery Subquery scalar-subquery#27, [id=#28] Subquery:3 Hosting operator id = 40 Hosting Expression = Subquery scalar-subquery#27, [id=#28] * HashAggregate (51) +- Exchange (50) - +- * HashAggregate (49) - +- * Project (48) - +- * Filter (47) - +- * ColumnarToRow (46) + +- * ColumnarToRow (49) + +- CometHashAggregate (48) + +- CometProject (47) + +- CometFilter (46) +- CometScan parquet spark_catalog.default.date_dim (45) @@ -279,23 +279,21 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)] ReadSchema: struct -(46) ColumnarToRow [codegen id : 1] -Input [3]: [d_month_seq#29, d_year#30, d_moy#31] - -(47) Filter [codegen id : 1] +(46) CometFilter Input [3]: [d_month_seq#29, d_year#30, d_moy#31] Condition : (((isnotnull(d_year#30) AND isnotnull(d_moy#31)) AND (d_year#30 = 2000)) AND (d_moy#31 = 1)) -(48) Project [codegen id : 1] -Output [1]: [d_month_seq#29] +(47) CometProject Input [3]: [d_month_seq#29, d_year#30, d_moy#31] +Arguments: [d_month_seq#29], [d_month_seq#29] -(49) HashAggregate [codegen id : 1] +(48) CometHashAggregate Input [1]: [d_month_seq#29] Keys [1]: [d_month_seq#29] Functions: [] -Aggregate Attributes: [] -Results [1]: [d_month_seq#29] + +(49) ColumnarToRow [codegen id : 1] +Input [1]: [d_month_seq#29] (50) Exchange Input [1]: [d_month_seq#29] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/simplified.txt index 7078a77e4..c2d5a6ce8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/simplified.txt @@ -14,32 +14,32 @@ TakeOrderedAndProject [cnt,state] BroadcastHashJoin [c_customer_sk,ss_customer_sk] Project [ca_state,c_customer_sk] BroadcastHashJoin [ca_address_sk,c_current_addr_sk] - Filter [ca_address_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #2 WholeStageCodegen (1) - Filter [c_current_addr_sk,c_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_current_addr_sk,c_customer_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] InputAdapter BroadcastExchange #3 WholeStageCodegen (2) - Filter [ss_customer_sk,ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_customer_sk,ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ReusedSubquery [d_month_seq] #2 - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_month_seq,d_date_sk] + ReusedSubquery [d_month_seq] #2 CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] Subquery #2 WholeStageCodegen (2) @@ -47,11 +47,11 @@ TakeOrderedAndProject [cnt,state] InputAdapter Exchange [d_month_seq] #5 WholeStageCodegen (1) - HashAggregate [d_month_seq] - Project [d_month_seq] - Filter [d_year,d_moy] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometHashAggregate [d_month_seq] + CometProject [d_month_seq] + CometFilter [d_year,d_moy] CometScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] InputAdapter ReusedExchange [d_date_sk] #4 @@ -60,9 +60,9 @@ TakeOrderedAndProject [cnt,state] WholeStageCodegen (6) Project [i_item_sk] BroadcastHashJoin [i_category,i_category,i_current_price,avg(i_current_price)] - Filter [i_current_price,i_category,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_current_price,i_category,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] InputAdapter BroadcastExchange #7 @@ -73,7 +73,7 @@ TakeOrderedAndProject [cnt,state] Exchange [i_category] #8 WholeStageCodegen (4) HashAggregate [i_category,i_current_price] [sum,count,sum,count] - Filter [i_category] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_category] CometScan parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/explain.txt index d2d493a6d..e50a522b8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/explain.txt @@ -1,187 +1,185 @@ == Physical Plan == -* Sort (183) -+- Exchange (182) - +- * Project (181) - +- * SortMergeJoin Inner (180) - :- * Sort (111) - : +- Exchange (110) - : +- * HashAggregate (109) - : +- * HashAggregate (108) - : +- * Project (107) - : +- * BroadcastHashJoin Inner BuildRight (106) - : :- * Project (100) - : : +- * BroadcastHashJoin Inner BuildRight (99) - : : :- * Project (97) - : : : +- * BroadcastHashJoin Inner BuildRight (96) - : : : :- * Project (91) - : : : : +- * BroadcastHashJoin Inner BuildRight (90) - : : : : :- * Project (88) - : : : : : +- * BroadcastHashJoin Inner BuildRight (87) - : : : : : :- * Project (82) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (81) - : : : : : : :- * Project (79) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (78) - : : : : : : : :- * Project (73) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (72) - : : : : : : : : :- * Project (67) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (66) - : : : : : : : : : :- * Project (64) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (63) - : : : : : : : : : : :- * Project (58) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (57) - : : : : : : : : : : : :- * Project (55) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (54) - : : : : : : : : : : : : :- * Project (49) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (48) - : : : : : : : : : : : : : :- * Project (43) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (42) - : : : : : : : : : : : : : : :- * Project (37) - : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (36) - : : : : : : : : : : : : : : : :- * Project (34) - : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (33) - : : : : : : : : : : : : : : : : :- * Sort (12) - : : : : : : : : : : : : : : : : : +- Exchange (11) - : : : : : : : : : : : : : : : : : +- * Project (10) - : : : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildLeft (9) - : : : : : : : : : : : : : : : : : :- BroadcastExchange (4) - : : : : : : : : : : : : : : : : : : +- * Filter (3) - : : : : : : : : : : : : : : : : : : +- * ColumnarToRow (2) - : : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : : : : : : : : : : : : : : +- * Project (8) - : : : : : : : : : : : : : : : : : +- * Filter (7) - : : : : : : : : : : : : : : : : : +- * ColumnarToRow (6) - : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_returns (5) - : : : : : : : : : : : : : : : : +- * Sort (32) - : : : : : : : : : : : : : : : : +- * Project (31) - : : : : : : : : : : : : : : : : +- * Filter (30) - : : : : : : : : : : : : : : : : +- * HashAggregate (29) - : : : : : : : : : : : : : : : : +- Exchange (28) - : : : : : : : : : : : : : : : : +- * HashAggregate (27) - : : : : : : : : : : : : : : : : +- * Project (26) - : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (25) - : : : : : : : : : : : : : : : : :- * Sort (18) - : : : : : : : : : : : : : : : : : +- Exchange (17) - : : : : : : : : : : : : : : : : : +- * Project (16) - : : : : : : : : : : : : : : : : : +- * Filter (15) - : : : : : : : : : : : : : : : : : +- * ColumnarToRow (14) - : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (13) - : : : : : : : : : : : : : : : : +- * Sort (24) - : : : : : : : : : : : : : : : : +- Exchange (23) - : : : : : : : : : : : : : : : : +- * Project (22) - : : : : : : : : : : : : : : : : +- * Filter (21) - : : : : : : : : : : : : : : : : +- * ColumnarToRow (20) - : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (19) - : : : : : : : : : : : : : : : +- ReusedExchange (35) - : : : : : : : : : : : : : : +- BroadcastExchange (41) - : : : : : : : : : : : : : : +- * Filter (40) - : : : : : : : : : : : : : : +- * ColumnarToRow (39) - : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store (38) - : : : : : : : : : : : : : +- BroadcastExchange (47) - : : : : : : : : : : : : : +- * Filter (46) - : : : : : : : : : : : : : +- * ColumnarToRow (45) - : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.customer (44) - : : : : : : : : : : : : +- BroadcastExchange (53) - : : : : : : : : : : : : +- * Filter (52) - : : : : : : : : : : : : +- * ColumnarToRow (51) - : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.date_dim (50) - : : : : : : : : : : : +- ReusedExchange (56) - : : : : : : : : : : +- BroadcastExchange (62) - : : : : : : : : : : +- * Filter (61) - : : : : : : : : : : +- * ColumnarToRow (60) - : : : : : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (59) - : : : : : : : : : +- ReusedExchange (65) - : : : : : : : : +- BroadcastExchange (71) - : : : : : : : : +- * Filter (70) - : : : : : : : : +- * ColumnarToRow (69) - : : : : : : : : +- CometScan parquet spark_catalog.default.promotion (68) - : : : : : : : +- BroadcastExchange (77) - : : : : : : : +- * Filter (76) - : : : : : : : +- * ColumnarToRow (75) - : : : : : : : +- CometScan parquet spark_catalog.default.household_demographics (74) - : : : : : : +- ReusedExchange (80) - : : : : : +- BroadcastExchange (86) - : : : : : +- * Filter (85) - : : : : : +- * ColumnarToRow (84) - : : : : : +- CometScan parquet spark_catalog.default.customer_address (83) - : : : : +- ReusedExchange (89) - : : : +- BroadcastExchange (95) - : : : +- * Filter (94) - : : : +- * ColumnarToRow (93) - : : : +- CometScan parquet spark_catalog.default.income_band (92) - : : +- ReusedExchange (98) - : +- BroadcastExchange (105) - : +- * Project (104) - : +- * Filter (103) - : +- * ColumnarToRow (102) - : +- CometScan parquet spark_catalog.default.item (101) - +- * Sort (179) - +- Exchange (178) - +- * HashAggregate (177) - +- * HashAggregate (176) - +- * Project (175) - +- * BroadcastHashJoin Inner BuildRight (174) - :- * Project (172) - : +- * BroadcastHashJoin Inner BuildRight (171) - : :- * Project (169) - : : +- * BroadcastHashJoin Inner BuildRight (168) - : : :- * Project (166) - : : : +- * BroadcastHashJoin Inner BuildRight (165) - : : : :- * Project (163) - : : : : +- * BroadcastHashJoin Inner BuildRight (162) - : : : : :- * Project (160) - : : : : : +- * BroadcastHashJoin Inner BuildRight (159) - : : : : : :- * Project (157) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (156) - : : : : : : :- * Project (154) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (153) - : : : : : : : :- * Project (151) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (150) - : : : : : : : : :- * Project (148) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (147) - : : : : : : : : : :- * Project (145) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (144) - : : : : : : : : : : :- * Project (142) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (141) - : : : : : : : : : : : :- * Project (139) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (138) - : : : : : : : : : : : : :- * Project (136) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (135) - : : : : : : : : : : : : : :- * Project (133) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (132) - : : : : : : : : : : : : : : :- * Project (130) - : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (129) - : : : : : : : : : : : : : : : :- * Sort (123) - : : : : : : : : : : : : : : : : +- Exchange (122) - : : : : : : : : : : : : : : : : +- * Project (121) - : : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildLeft (120) - : : : : : : : : : : : : : : : : :- BroadcastExchange (115) - : : : : : : : : : : : : : : : : : +- * Filter (114) - : : : : : : : : : : : : : : : : : +- * ColumnarToRow (113) - : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (112) - : : : : : : : : : : : : : : : : +- * Project (119) - : : : : : : : : : : : : : : : : +- * Filter (118) - : : : : : : : : : : : : : : : : +- * ColumnarToRow (117) - : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_returns (116) - : : : : : : : : : : : : : : : +- * Sort (128) - : : : : : : : : : : : : : : : +- * Project (127) - : : : : : : : : : : : : : : : +- * Filter (126) - : : : : : : : : : : : : : : : +- * HashAggregate (125) - : : : : : : : : : : : : : : : +- ReusedExchange (124) - : : : : : : : : : : : : : : +- ReusedExchange (131) - : : : : : : : : : : : : : +- ReusedExchange (134) - : : : : : : : : : : : : +- ReusedExchange (137) - : : : : : : : : : : : +- ReusedExchange (140) - : : : : : : : : : : +- ReusedExchange (143) - : : : : : : : : : +- ReusedExchange (146) - : : : : : : : : +- ReusedExchange (149) - : : : : : : : +- ReusedExchange (152) - : : : : : : +- ReusedExchange (155) - : : : : : +- ReusedExchange (158) - : : : : +- ReusedExchange (161) - : : : +- ReusedExchange (164) - : : +- ReusedExchange (167) - : +- ReusedExchange (170) - +- ReusedExchange (173) +* Sort (181) ++- Exchange (180) + +- * Project (179) + +- * SortMergeJoin Inner (178) + :- * Sort (110) + : +- Exchange (109) + : +- * HashAggregate (108) + : +- * HashAggregate (107) + : +- * Project (106) + : +- * BroadcastHashJoin Inner BuildRight (105) + : :- * Project (99) + : : +- * BroadcastHashJoin Inner BuildRight (98) + : : :- * Project (96) + : : : +- * BroadcastHashJoin Inner BuildRight (95) + : : : :- * Project (90) + : : : : +- * BroadcastHashJoin Inner BuildRight (89) + : : : : :- * Project (87) + : : : : : +- * BroadcastHashJoin Inner BuildRight (86) + : : : : : :- * Project (81) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (80) + : : : : : : :- * Project (78) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (77) + : : : : : : : :- * Project (72) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (71) + : : : : : : : : :- * Project (66) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (65) + : : : : : : : : : :- * Project (63) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (62) + : : : : : : : : : : :- * Project (57) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (56) + : : : : : : : : : : : :- * Project (54) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (53) + : : : : : : : : : : : : :- * Project (48) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (47) + : : : : : : : : : : : : : :- * Project (42) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (41) + : : : : : : : : : : : : : : :- * Project (36) + : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (35) + : : : : : : : : : : : : : : : :- * Project (33) + : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (32) + : : : : : : : : : : : : : : : : :- * Sort (11) + : : : : : : : : : : : : : : : : : +- Exchange (10) + : : : : : : : : : : : : : : : : : +- * ColumnarToRow (9) + : : : : : : : : : : : : : : : : : +- CometProject (8) + : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) + : : : : : : : : : : : : : : : : : : +- CometFilter (2) + : : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : : : : : : : : : : : : : +- CometProject (6) + : : : : : : : : : : : : : : : : : +- CometFilter (5) + : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_returns (4) + : : : : : : : : : : : : : : : : +- * Sort (31) + : : : : : : : : : : : : : : : : +- * Project (30) + : : : : : : : : : : : : : : : : +- * Filter (29) + : : : : : : : : : : : : : : : : +- * HashAggregate (28) + : : : : : : : : : : : : : : : : +- Exchange (27) + : : : : : : : : : : : : : : : : +- * HashAggregate (26) + : : : : : : : : : : : : : : : : +- * Project (25) + : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (24) + : : : : : : : : : : : : : : : : :- * Sort (17) + : : : : : : : : : : : : : : : : : +- Exchange (16) + : : : : : : : : : : : : : : : : : +- * ColumnarToRow (15) + : : : : : : : : : : : : : : : : : +- CometProject (14) + : : : : : : : : : : : : : : : : : +- CometFilter (13) + : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (12) + : : : : : : : : : : : : : : : : +- * Sort (23) + : : : : : : : : : : : : : : : : +- Exchange (22) + : : : : : : : : : : : : : : : : +- * ColumnarToRow (21) + : : : : : : : : : : : : : : : : +- CometProject (20) + : : : : : : : : : : : : : : : : +- CometFilter (19) + : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (18) + : : : : : : : : : : : : : : : +- ReusedExchange (34) + : : : : : : : : : : : : : : +- BroadcastExchange (40) + : : : : : : : : : : : : : : +- * ColumnarToRow (39) + : : : : : : : : : : : : : : +- CometFilter (38) + : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store (37) + : : : : : : : : : : : : : +- BroadcastExchange (46) + : : : : : : : : : : : : : +- * ColumnarToRow (45) + : : : : : : : : : : : : : +- CometFilter (44) + : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.customer (43) + : : : : : : : : : : : : +- BroadcastExchange (52) + : : : : : : : : : : : : +- * ColumnarToRow (51) + : : : : : : : : : : : : +- CometFilter (50) + : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.date_dim (49) + : : : : : : : : : : : +- ReusedExchange (55) + : : : : : : : : : : +- BroadcastExchange (61) + : : : : : : : : : : +- * ColumnarToRow (60) + : : : : : : : : : : +- CometFilter (59) + : : : : : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (58) + : : : : : : : : : +- ReusedExchange (64) + : : : : : : : : +- BroadcastExchange (70) + : : : : : : : : +- * ColumnarToRow (69) + : : : : : : : : +- CometFilter (68) + : : : : : : : : +- CometScan parquet spark_catalog.default.promotion (67) + : : : : : : : +- BroadcastExchange (76) + : : : : : : : +- * ColumnarToRow (75) + : : : : : : : +- CometFilter (74) + : : : : : : : +- CometScan parquet spark_catalog.default.household_demographics (73) + : : : : : : +- ReusedExchange (79) + : : : : : +- BroadcastExchange (85) + : : : : : +- * ColumnarToRow (84) + : : : : : +- CometFilter (83) + : : : : : +- CometScan parquet spark_catalog.default.customer_address (82) + : : : : +- ReusedExchange (88) + : : : +- BroadcastExchange (94) + : : : +- * ColumnarToRow (93) + : : : +- CometFilter (92) + : : : +- CometScan parquet spark_catalog.default.income_band (91) + : : +- ReusedExchange (97) + : +- BroadcastExchange (104) + : +- * ColumnarToRow (103) + : +- CometProject (102) + : +- CometFilter (101) + : +- CometScan parquet spark_catalog.default.item (100) + +- * Sort (177) + +- Exchange (176) + +- * HashAggregate (175) + +- * HashAggregate (174) + +- * Project (173) + +- * BroadcastHashJoin Inner BuildRight (172) + :- * Project (170) + : +- * BroadcastHashJoin Inner BuildRight (169) + : :- * Project (167) + : : +- * BroadcastHashJoin Inner BuildRight (166) + : : :- * Project (164) + : : : +- * BroadcastHashJoin Inner BuildRight (163) + : : : :- * Project (161) + : : : : +- * BroadcastHashJoin Inner BuildRight (160) + : : : : :- * Project (158) + : : : : : +- * BroadcastHashJoin Inner BuildRight (157) + : : : : : :- * Project (155) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (154) + : : : : : : :- * Project (152) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (151) + : : : : : : : :- * Project (149) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (148) + : : : : : : : : :- * Project (146) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (145) + : : : : : : : : : :- * Project (143) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (142) + : : : : : : : : : : :- * Project (140) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (139) + : : : : : : : : : : : :- * Project (137) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (136) + : : : : : : : : : : : : :- * Project (134) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (133) + : : : : : : : : : : : : : :- * Project (131) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (130) + : : : : : : : : : : : : : : :- * Project (128) + : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (127) + : : : : : : : : : : : : : : : :- * Sort (121) + : : : : : : : : : : : : : : : : +- Exchange (120) + : : : : : : : : : : : : : : : : +- * ColumnarToRow (119) + : : : : : : : : : : : : : : : : +- CometProject (118) + : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (117) + : : : : : : : : : : : : : : : : :- CometBroadcastExchange (113) + : : : : : : : : : : : : : : : : : +- CometFilter (112) + : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (111) + : : : : : : : : : : : : : : : : +- CometProject (116) + : : : : : : : : : : : : : : : : +- CometFilter (115) + : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_returns (114) + : : : : : : : : : : : : : : : +- * Sort (126) + : : : : : : : : : : : : : : : +- * Project (125) + : : : : : : : : : : : : : : : +- * Filter (124) + : : : : : : : : : : : : : : : +- * HashAggregate (123) + : : : : : : : : : : : : : : : +- ReusedExchange (122) + : : : : : : : : : : : : : : +- ReusedExchange (129) + : : : : : : : : : : : : : +- ReusedExchange (132) + : : : : : : : : : : : : +- ReusedExchange (135) + : : : : : : : : : : : +- ReusedExchange (138) + : : : : : : : : : : +- ReusedExchange (141) + : : : : : : : : : +- ReusedExchange (144) + : : : : : : : : +- ReusedExchange (147) + : : : : : : : +- ReusedExchange (150) + : : : : : : +- ReusedExchange (153) + : : : : : +- ReusedExchange (156) + : : : : +- ReusedExchange (159) + : : : +- ReusedExchange (162) + : : +- ReusedExchange (165) + : +- ReusedExchange (168) + +- ReusedExchange (171) (1) Scan parquet spark_catalog.default.store_sales @@ -192,512 +190,508 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AND isnotnull(ss_store_sk#6)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_cdemo_sk#3)) AND isnotnull(ss_promo_sk#7)) AND isnotnull(ss_hdemo_sk#4)) AND isnotnull(ss_addr_sk#5)) -(4) BroadcastExchange +(3) CometBroadcastExchange Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[7, int, false] as bigint) & 4294967295))),false), [plan_id=1] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(5) Scan parquet spark_catalog.default.store_returns +(4) Scan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(6) ColumnarToRow -Input [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] - -(7) Filter +(5) CometFilter Input [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] Condition : (isnotnull(sr_item_sk#14) AND isnotnull(sr_ticket_number#15)) -(8) Project -Output [2]: [sr_item_sk#14, sr_ticket_number#15] +(6) CometProject Input [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] +Arguments: [sr_item_sk#14, sr_ticket_number#15], [sr_item_sk#14, sr_ticket_number#15] -(9) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [ss_item_sk#1, ss_ticket_number#8] -Right keys [2]: [sr_item_sk#14, sr_ticket_number#15] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Right output [2]: [sr_item_sk#14, sr_ticket_number#15] +Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#14, sr_ticket_number#15], Inner -(10) Project [codegen id : 2] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +(8) CometProject Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#14, sr_ticket_number#15] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] + +(9) ColumnarToRow [codegen id : 1] +Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(11) Exchange +(10) Exchange Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(12) Sort [codegen id : 3] +(11) Sort [codegen id : 2] Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0 -(13) Scan parquet spark_catalog.default.catalog_sales +(12) Scan parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 4] -Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] - -(15) Filter [codegen id : 4] +(13) CometFilter Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] Condition : (isnotnull(cs_item_sk#17) AND isnotnull(cs_order_number#18)) -(16) Project [codegen id : 4] -Output [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] +(14) CometProject Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] +Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] + +(15) ColumnarToRow [codegen id : 3] +Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -(17) Exchange +(16) Exchange Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(18) Sort [codegen id : 5] +(17) Sort [codegen id : 4] Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] Arguments: [cs_item_sk#17 ASC NULLS FIRST, cs_order_number#18 ASC NULLS FIRST], false, 0 -(19) Scan parquet spark_catalog.default.catalog_returns +(18) Scan parquet spark_catalog.default.catalog_returns Output [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 6] -Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] - -(21) Filter [codegen id : 6] +(19) CometFilter Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] Condition : (isnotnull(cr_item_sk#21) AND isnotnull(cr_order_number#22)) -(22) Project [codegen id : 6] -Output [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] +(20) CometProject Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] +Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] + +(21) ColumnarToRow [codegen id : 5] +Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -(23) Exchange +(22) Exchange Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(24) Sort [codegen id : 7] +(23) Sort [codegen id : 6] Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] Arguments: [cr_item_sk#21 ASC NULLS FIRST, cr_order_number#22 ASC NULLS FIRST], false, 0 -(25) SortMergeJoin [codegen id : 8] +(24) SortMergeJoin [codegen id : 7] Left keys [2]: [cs_item_sk#17, cs_order_number#18] Right keys [2]: [cr_item_sk#21, cr_order_number#22] Join type: Inner Join condition: None -(26) Project [codegen id : 8] +(25) Project [codegen id : 7] Output [5]: [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] Input [8]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -(27) HashAggregate [codegen id : 8] +(26) HashAggregate [codegen id : 7] Input [5]: [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] Keys [1]: [cs_item_sk#17] Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#19)), partial_sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))] Aggregate Attributes [3]: [sum#27, sum#28, isEmpty#29] Results [4]: [cs_item_sk#17, sum#30, sum#31, isEmpty#32] -(28) Exchange +(27) Exchange Input [4]: [cs_item_sk#17, sum#30, sum#31, isEmpty#32] -Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(29) HashAggregate [codegen id : 9] +(28) HashAggregate [codegen id : 8] Input [4]: [cs_item_sk#17, sum#30, sum#31, isEmpty#32] Keys [1]: [cs_item_sk#17] Functions [2]: [sum(UnscaledValue(cs_ext_list_price#19)), sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))] Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#19))#33, sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))#34] Results [3]: [cs_item_sk#17, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#19))#33,17,2) AS sale#35, sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))#34 AS refund#36] -(30) Filter [codegen id : 9] +(29) Filter [codegen id : 8] Input [3]: [cs_item_sk#17, sale#35, refund#36] Condition : ((isnotnull(sale#35) AND isnotnull(refund#36)) AND (cast(sale#35 as decimal(21,2)) > (2 * refund#36))) -(31) Project [codegen id : 9] +(30) Project [codegen id : 8] Output [1]: [cs_item_sk#17] Input [3]: [cs_item_sk#17, sale#35, refund#36] -(32) Sort [codegen id : 9] +(31) Sort [codegen id : 8] Input [1]: [cs_item_sk#17] Arguments: [cs_item_sk#17 ASC NULLS FIRST], false, 0 -(33) SortMergeJoin [codegen id : 25] +(32) SortMergeJoin [codegen id : 24] Left keys [1]: [ss_item_sk#1] Right keys [1]: [cs_item_sk#17] Join type: Inner Join condition: None -(34) Project [codegen id : 25] +(33) Project [codegen id : 24] Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#17] -(35) ReusedExchange [Reuses operator id: 187] +(34) ReusedExchange [Reuses operator id: 185] Output [2]: [d_date_sk#37, d_year#38] -(36) BroadcastHashJoin [codegen id : 25] +(35) BroadcastHashJoin [codegen id : 24] Left keys [1]: [ss_sold_date_sk#12] Right keys [1]: [d_date_sk#37] Join type: Inner Join condition: None -(37) Project [codegen id : 25] +(36) Project [codegen id : 24] Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38] Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#37, d_year#38] -(38) Scan parquet spark_catalog.default.store +(37) Scan parquet spark_catalog.default.store Output [3]: [s_store_sk#39, s_store_name#40, s_zip#41] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_zip)] ReadSchema: struct -(39) ColumnarToRow [codegen id : 11] +(38) CometFilter Input [3]: [s_store_sk#39, s_store_name#40, s_zip#41] +Condition : ((isnotnull(s_store_sk#39) AND isnotnull(s_store_name#40)) AND isnotnull(s_zip#41)) -(40) Filter [codegen id : 11] +(39) ColumnarToRow [codegen id : 10] Input [3]: [s_store_sk#39, s_store_name#40, s_zip#41] -Condition : ((isnotnull(s_store_sk#39) AND isnotnull(s_store_name#40)) AND isnotnull(s_zip#41)) -(41) BroadcastExchange +(40) BroadcastExchange Input [3]: [s_store_sk#39, s_store_name#40, s_zip#41] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] -(42) BroadcastHashJoin [codegen id : 25] +(41) BroadcastHashJoin [codegen id : 24] Left keys [1]: [ss_store_sk#6] Right keys [1]: [s_store_sk#39] Join type: Inner Join condition: None -(43) Project [codegen id : 25] +(42) Project [codegen id : 24] Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41] Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_sk#39, s_store_name#40, s_zip#41] -(44) Scan parquet spark_catalog.default.customer +(43) Scan parquet spark_catalog.default.customer Output [6]: [c_customer_sk#42, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 12] +(44) CometFilter Input [6]: [c_customer_sk#42, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] +Condition : (((((isnotnull(c_customer_sk#42) AND isnotnull(c_first_sales_date_sk#47)) AND isnotnull(c_first_shipto_date_sk#46)) AND isnotnull(c_current_cdemo_sk#43)) AND isnotnull(c_current_hdemo_sk#44)) AND isnotnull(c_current_addr_sk#45)) -(46) Filter [codegen id : 12] +(45) ColumnarToRow [codegen id : 11] Input [6]: [c_customer_sk#42, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] -Condition : (((((isnotnull(c_customer_sk#42) AND isnotnull(c_first_sales_date_sk#47)) AND isnotnull(c_first_shipto_date_sk#46)) AND isnotnull(c_current_cdemo_sk#43)) AND isnotnull(c_current_hdemo_sk#44)) AND isnotnull(c_current_addr_sk#45)) -(47) BroadcastExchange +(46) BroadcastExchange Input [6]: [c_customer_sk#42, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] -(48) BroadcastHashJoin [codegen id : 25] +(47) BroadcastHashJoin [codegen id : 24] Left keys [1]: [ss_customer_sk#2] Right keys [1]: [c_customer_sk#42] Join type: Inner Join condition: None -(49) Project [codegen id : 25] +(48) Project [codegen id : 24] Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_customer_sk#42, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] -(50) Scan parquet spark_catalog.default.date_dim +(49) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#48, d_year#49] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(51) ColumnarToRow [codegen id : 13] +(50) CometFilter Input [2]: [d_date_sk#48, d_year#49] +Condition : isnotnull(d_date_sk#48) -(52) Filter [codegen id : 13] +(51) ColumnarToRow [codegen id : 12] Input [2]: [d_date_sk#48, d_year#49] -Condition : isnotnull(d_date_sk#48) -(53) BroadcastExchange +(52) BroadcastExchange Input [2]: [d_date_sk#48, d_year#49] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] -(54) BroadcastHashJoin [codegen id : 25] +(53) BroadcastHashJoin [codegen id : 24] Left keys [1]: [c_first_sales_date_sk#47] Right keys [1]: [d_date_sk#48] Join type: Inner Join condition: None -(55) Project [codegen id : 25] +(54) Project [codegen id : 24] Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, d_year#49] Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47, d_date_sk#48, d_year#49] -(56) ReusedExchange [Reuses operator id: 53] +(55) ReusedExchange [Reuses operator id: 52] Output [2]: [d_date_sk#50, d_year#51] -(57) BroadcastHashJoin [codegen id : 25] +(56) BroadcastHashJoin [codegen id : 24] Left keys [1]: [c_first_shipto_date_sk#46] Right keys [1]: [d_date_sk#50] Join type: Inner Join condition: None -(58) Project [codegen id : 25] +(57) Project [codegen id : 24] Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51] Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, d_year#49, d_date_sk#50, d_year#51] -(59) Scan parquet spark_catalog.default.customer_demographics +(58) Scan parquet spark_catalog.default.customer_demographics Output [2]: [cd_demo_sk#52, cd_marital_status#53] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status)] ReadSchema: struct -(60) ColumnarToRow [codegen id : 15] +(59) CometFilter Input [2]: [cd_demo_sk#52, cd_marital_status#53] +Condition : (isnotnull(cd_demo_sk#52) AND isnotnull(cd_marital_status#53)) -(61) Filter [codegen id : 15] +(60) ColumnarToRow [codegen id : 14] Input [2]: [cd_demo_sk#52, cd_marital_status#53] -Condition : (isnotnull(cd_demo_sk#52) AND isnotnull(cd_marital_status#53)) -(62) BroadcastExchange +(61) BroadcastExchange Input [2]: [cd_demo_sk#52, cd_marital_status#53] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] -(63) BroadcastHashJoin [codegen id : 25] +(62) BroadcastHashJoin [codegen id : 24] Left keys [1]: [ss_cdemo_sk#3] Right keys [1]: [cd_demo_sk#52] Join type: Inner Join condition: None -(64) Project [codegen id : 25] +(63) Project [codegen id : 24] Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, cd_marital_status#53] Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, cd_demo_sk#52, cd_marital_status#53] -(65) ReusedExchange [Reuses operator id: 62] +(64) ReusedExchange [Reuses operator id: 61] Output [2]: [cd_demo_sk#54, cd_marital_status#55] -(66) BroadcastHashJoin [codegen id : 25] +(65) BroadcastHashJoin [codegen id : 24] Left keys [1]: [c_current_cdemo_sk#43] Right keys [1]: [cd_demo_sk#54] Join type: Inner Join condition: NOT (cd_marital_status#53 = cd_marital_status#55) -(67) Project [codegen id : 25] +(66) Project [codegen id : 24] Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51] Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, cd_marital_status#53, cd_demo_sk#54, cd_marital_status#55] -(68) Scan parquet spark_catalog.default.promotion +(67) Scan parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#56] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(69) ColumnarToRow [codegen id : 17] +(68) CometFilter Input [1]: [p_promo_sk#56] +Condition : isnotnull(p_promo_sk#56) -(70) Filter [codegen id : 17] +(69) ColumnarToRow [codegen id : 16] Input [1]: [p_promo_sk#56] -Condition : isnotnull(p_promo_sk#56) -(71) BroadcastExchange +(70) BroadcastExchange Input [1]: [p_promo_sk#56] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] -(72) BroadcastHashJoin [codegen id : 25] +(71) BroadcastHashJoin [codegen id : 24] Left keys [1]: [ss_promo_sk#7] Right keys [1]: [p_promo_sk#56] Join type: Inner Join condition: None -(73) Project [codegen id : 25] +(72) Project [codegen id : 24] Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51] Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, p_promo_sk#56] -(74) Scan parquet spark_catalog.default.household_demographics +(73) Scan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#57, hd_income_band_sk#58] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] ReadSchema: struct -(75) ColumnarToRow [codegen id : 18] +(74) CometFilter Input [2]: [hd_demo_sk#57, hd_income_band_sk#58] +Condition : (isnotnull(hd_demo_sk#57) AND isnotnull(hd_income_band_sk#58)) -(76) Filter [codegen id : 18] +(75) ColumnarToRow [codegen id : 17] Input [2]: [hd_demo_sk#57, hd_income_band_sk#58] -Condition : (isnotnull(hd_demo_sk#57) AND isnotnull(hd_income_band_sk#58)) -(77) BroadcastExchange +(76) BroadcastExchange Input [2]: [hd_demo_sk#57, hd_income_band_sk#58] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] -(78) BroadcastHashJoin [codegen id : 25] +(77) BroadcastHashJoin [codegen id : 24] Left keys [1]: [ss_hdemo_sk#4] Right keys [1]: [hd_demo_sk#57] Join type: Inner Join condition: None -(79) Project [codegen id : 25] +(78) Project [codegen id : 24] Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58] Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, hd_demo_sk#57, hd_income_band_sk#58] -(80) ReusedExchange [Reuses operator id: 77] +(79) ReusedExchange [Reuses operator id: 76] Output [2]: [hd_demo_sk#59, hd_income_band_sk#60] -(81) BroadcastHashJoin [codegen id : 25] +(80) BroadcastHashJoin [codegen id : 24] Left keys [1]: [c_current_hdemo_sk#44] Right keys [1]: [hd_demo_sk#59] Join type: Inner Join condition: None -(82) Project [codegen id : 25] +(81) Project [codegen id : 24] Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60] Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58, hd_demo_sk#59, hd_income_band_sk#60] -(83) Scan parquet spark_catalog.default.customer_address +(82) Scan parquet spark_catalog.default.customer_address Output [5]: [ca_address_sk#61, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(84) ColumnarToRow [codegen id : 20] +(83) CometFilter Input [5]: [ca_address_sk#61, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] +Condition : isnotnull(ca_address_sk#61) -(85) Filter [codegen id : 20] +(84) ColumnarToRow [codegen id : 19] Input [5]: [ca_address_sk#61, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] -Condition : isnotnull(ca_address_sk#61) -(86) BroadcastExchange +(85) BroadcastExchange Input [5]: [ca_address_sk#61, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] -(87) BroadcastHashJoin [codegen id : 25] +(86) BroadcastHashJoin [codegen id : 24] Left keys [1]: [ss_addr_sk#5] Right keys [1]: [ca_address_sk#61] Join type: Inner Join condition: None -(88) Project [codegen id : 25] +(87) Project [codegen id : 24] Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60, ca_address_sk#61, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] -(89) ReusedExchange [Reuses operator id: 86] +(88) ReusedExchange [Reuses operator id: 85] Output [5]: [ca_address_sk#66, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] -(90) BroadcastHashJoin [codegen id : 25] +(89) BroadcastHashJoin [codegen id : 24] Left keys [1]: [c_current_addr_sk#45] Right keys [1]: [ca_address_sk#66] Join type: Inner Join condition: None -(91) Project [codegen id : 25] +(90) Project [codegen id : 24] Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_address_sk#66, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] -(92) Scan parquet spark_catalog.default.income_band +(91) Scan parquet spark_catalog.default.income_band Output [1]: [ib_income_band_sk#71] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] PushedFilters: [IsNotNull(ib_income_band_sk)] ReadSchema: struct -(93) ColumnarToRow [codegen id : 22] +(92) CometFilter Input [1]: [ib_income_band_sk#71] +Condition : isnotnull(ib_income_band_sk#71) -(94) Filter [codegen id : 22] +(93) ColumnarToRow [codegen id : 21] Input [1]: [ib_income_band_sk#71] -Condition : isnotnull(ib_income_band_sk#71) -(95) BroadcastExchange +(94) BroadcastExchange Input [1]: [ib_income_band_sk#71] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] -(96) BroadcastHashJoin [codegen id : 25] +(95) BroadcastHashJoin [codegen id : 24] Left keys [1]: [hd_income_band_sk#58] Right keys [1]: [ib_income_band_sk#71] Join type: Inner Join condition: None -(97) Project [codegen id : 25] +(96) Project [codegen id : 24] Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, ib_income_band_sk#71] -(98) ReusedExchange [Reuses operator id: 95] +(97) ReusedExchange [Reuses operator id: 94] Output [1]: [ib_income_band_sk#72] -(99) BroadcastHashJoin [codegen id : 25] +(98) BroadcastHashJoin [codegen id : 24] Left keys [1]: [hd_income_band_sk#60] Right keys [1]: [ib_income_band_sk#72] Join type: Inner Join condition: None -(100) Project [codegen id : 25] +(99) Project [codegen id : 24] Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, ib_income_band_sk#72] -(101) Scan parquet spark_catalog.default.item +(100) Scan parquet spark_catalog.default.item Output [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), In(i_color, [burlywood ,floral ,indian ,medium ,purple ,spring ]), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] ReadSchema: struct -(102) ColumnarToRow [codegen id : 24] -Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] - -(103) Filter [codegen id : 24] +(101) CometFilter Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] Condition : ((((((isnotnull(i_current_price#74) AND i_color#75 IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#74 >= 64.00)) AND (i_current_price#74 <= 74.00)) AND (i_current_price#74 >= 65.00)) AND (i_current_price#74 <= 79.00)) AND isnotnull(i_item_sk#73)) -(104) Project [codegen id : 24] -Output [2]: [i_item_sk#73, i_product_name#76] +(102) CometProject Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] +Arguments: [i_item_sk#73, i_product_name#76], [i_item_sk#73, i_product_name#76] + +(103) ColumnarToRow [codegen id : 23] +Input [2]: [i_item_sk#73, i_product_name#76] -(105) BroadcastExchange +(104) BroadcastExchange Input [2]: [i_item_sk#73, i_product_name#76] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] -(106) BroadcastHashJoin [codegen id : 25] +(105) BroadcastHashJoin [codegen id : 24] Left keys [1]: [ss_item_sk#1] Right keys [1]: [i_item_sk#73] Join type: Inner Join condition: None -(107) Project [codegen id : 25] +(106) Project [codegen id : 24] Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, d_year#49, d_year#51, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#76] Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#76] -(108) HashAggregate [codegen id : 25] +(107) HashAggregate [codegen id : 24] Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, d_year#49, d_year#51, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#76] Keys [15]: [i_product_name#76, i_item_sk#73, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#38, d_year#49, d_year#51] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] Aggregate Attributes [4]: [count#77, sum#78, sum#79, sum#80] Results [19]: [i_product_name#76, i_item_sk#73, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#38, d_year#49, d_year#51, count#81, sum#82, sum#83, sum#84] -(109) HashAggregate [codegen id : 25] +(108) HashAggregate [codegen id : 24] Input [19]: [i_product_name#76, i_item_sk#73, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#38, d_year#49, d_year#51, count#81, sum#82, sum#83, sum#84] Keys [15]: [i_product_name#76, i_item_sk#73, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#38, d_year#49, d_year#51] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] Aggregate Attributes [4]: [count(1)#85, sum(UnscaledValue(ss_wholesale_cost#9))#86, sum(UnscaledValue(ss_list_price#10))#87, sum(UnscaledValue(ss_coupon_amt#11))#88] Results [17]: [i_product_name#76 AS product_name#89, i_item_sk#73 AS item_sk#90, s_store_name#40 AS store_name#91, s_zip#41 AS store_zip#92, ca_street_number#62 AS b_street_number#93, ca_street_name#63 AS b_streen_name#94, ca_city#64 AS b_city#95, ca_zip#65 AS b_zip#96, ca_street_number#67 AS c_street_number#97, ca_street_name#68 AS c_street_name#98, ca_city#69 AS c_city#99, ca_zip#70 AS c_zip#100, d_year#38 AS syear#101, count(1)#85 AS cnt#102, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#86,17,2) AS s1#103, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#87,17,2) AS s2#104, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#88,17,2) AS s3#105] -(110) Exchange +(109) Exchange Input [17]: [product_name#89, item_sk#90, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105] -Arguments: hashpartitioning(item_sk#90, store_name#91, store_zip#92, 5), ENSURE_REQUIREMENTS, [plan_id=15] +Arguments: hashpartitioning(item_sk#90, store_name#91, store_zip#92, 5), ENSURE_REQUIREMENTS, [plan_id=14] -(111) Sort [codegen id : 26] +(110) Sort [codegen id : 25] Input [17]: [product_name#89, item_sk#90, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105] Arguments: [item_sk#90 ASC NULLS FIRST, store_name#91 ASC NULLS FIRST, store_zip#92 ASC NULLS FIRST], false, 0 -(112) Scan parquet spark_catalog.default.store_sales +(111) Scan parquet spark_catalog.default.store_sales Output [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] Batched: true Location: InMemoryFileIndex [] @@ -705,370 +699,366 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#117), dynamicpruningexpression(ss_s PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] ReadSchema: struct -(113) ColumnarToRow [codegen id : 27] -Input [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] - -(114) Filter [codegen id : 27] +(112) CometFilter Input [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] Condition : (((((((isnotnull(ss_item_sk#106) AND isnotnull(ss_ticket_number#113)) AND isnotnull(ss_store_sk#111)) AND isnotnull(ss_customer_sk#107)) AND isnotnull(ss_cdemo_sk#108)) AND isnotnull(ss_promo_sk#112)) AND isnotnull(ss_hdemo_sk#109)) AND isnotnull(ss_addr_sk#110)) -(115) BroadcastExchange +(113) CometBroadcastExchange Input [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[7, int, false] as bigint) & 4294967295))),false), [plan_id=16] +Arguments: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] -(116) Scan parquet spark_catalog.default.store_returns +(114) Scan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#119, sr_ticket_number#120, sr_returned_date_sk#121] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(117) ColumnarToRow -Input [3]: [sr_item_sk#119, sr_ticket_number#120, sr_returned_date_sk#121] - -(118) Filter +(115) CometFilter Input [3]: [sr_item_sk#119, sr_ticket_number#120, sr_returned_date_sk#121] Condition : (isnotnull(sr_item_sk#119) AND isnotnull(sr_ticket_number#120)) -(119) Project -Output [2]: [sr_item_sk#119, sr_ticket_number#120] +(116) CometProject Input [3]: [sr_item_sk#119, sr_ticket_number#120, sr_returned_date_sk#121] +Arguments: [sr_item_sk#119, sr_ticket_number#120], [sr_item_sk#119, sr_ticket_number#120] -(120) BroadcastHashJoin [codegen id : 28] -Left keys [2]: [ss_item_sk#106, ss_ticket_number#113] -Right keys [2]: [sr_item_sk#119, sr_ticket_number#120] -Join type: Inner -Join condition: None +(117) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] +Right output [2]: [sr_item_sk#119, sr_ticket_number#120] +Arguments: [ss_item_sk#106, ss_ticket_number#113], [sr_item_sk#119, sr_ticket_number#120], Inner -(121) Project [codegen id : 28] -Output [11]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] +(118) CometProject Input [14]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, sr_item_sk#119, sr_ticket_number#120] +Arguments: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117], [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] + +(119) ColumnarToRow [codegen id : 26] +Input [11]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] -(122) Exchange +(120) Exchange Input [11]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] -Arguments: hashpartitioning(ss_item_sk#106, 5), ENSURE_REQUIREMENTS, [plan_id=17] +Arguments: hashpartitioning(ss_item_sk#106, 5), ENSURE_REQUIREMENTS, [plan_id=15] -(123) Sort [codegen id : 29] +(121) Sort [codegen id : 27] Input [11]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] Arguments: [ss_item_sk#106 ASC NULLS FIRST], false, 0 -(124) ReusedExchange [Reuses operator id: 28] +(122) ReusedExchange [Reuses operator id: 27] Output [4]: [cs_item_sk#122, sum#123, sum#124, isEmpty#125] -(125) HashAggregate [codegen id : 35] +(123) HashAggregate [codegen id : 33] Input [4]: [cs_item_sk#122, sum#123, sum#124, isEmpty#125] Keys [1]: [cs_item_sk#122] Functions [2]: [sum(UnscaledValue(cs_ext_list_price#126)), sum(((cr_refunded_cash#127 + cr_reversed_charge#128) + cr_store_credit#129))] Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#126))#33, sum(((cr_refunded_cash#127 + cr_reversed_charge#128) + cr_store_credit#129))#34] Results [3]: [cs_item_sk#122, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#126))#33,17,2) AS sale#130, sum(((cr_refunded_cash#127 + cr_reversed_charge#128) + cr_store_credit#129))#34 AS refund#131] -(126) Filter [codegen id : 35] +(124) Filter [codegen id : 33] Input [3]: [cs_item_sk#122, sale#130, refund#131] Condition : ((isnotnull(sale#130) AND isnotnull(refund#131)) AND (cast(sale#130 as decimal(21,2)) > (2 * refund#131))) -(127) Project [codegen id : 35] +(125) Project [codegen id : 33] Output [1]: [cs_item_sk#122] Input [3]: [cs_item_sk#122, sale#130, refund#131] -(128) Sort [codegen id : 35] +(126) Sort [codegen id : 33] Input [1]: [cs_item_sk#122] Arguments: [cs_item_sk#122 ASC NULLS FIRST], false, 0 -(129) SortMergeJoin [codegen id : 51] +(127) SortMergeJoin [codegen id : 49] Left keys [1]: [ss_item_sk#106] Right keys [1]: [cs_item_sk#122] Join type: Inner Join condition: None -(130) Project [codegen id : 51] +(128) Project [codegen id : 49] Output [11]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] Input [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, cs_item_sk#122] -(131) ReusedExchange [Reuses operator id: 191] +(129) ReusedExchange [Reuses operator id: 189] Output [2]: [d_date_sk#132, d_year#133] -(132) BroadcastHashJoin [codegen id : 51] +(130) BroadcastHashJoin [codegen id : 49] Left keys [1]: [ss_sold_date_sk#117] Right keys [1]: [d_date_sk#132] Join type: Inner Join condition: None -(133) Project [codegen id : 51] +(131) Project [codegen id : 49] Output [11]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133] Input [13]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, d_date_sk#132, d_year#133] -(134) ReusedExchange [Reuses operator id: 41] +(132) ReusedExchange [Reuses operator id: 40] Output [3]: [s_store_sk#134, s_store_name#135, s_zip#136] -(135) BroadcastHashJoin [codegen id : 51] +(133) BroadcastHashJoin [codegen id : 49] Left keys [1]: [ss_store_sk#111] Right keys [1]: [s_store_sk#134] Join type: Inner Join condition: None -(136) Project [codegen id : 51] +(134) Project [codegen id : 49] Output [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136] Input [14]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_sk#134, s_store_name#135, s_zip#136] -(137) ReusedExchange [Reuses operator id: 47] +(135) ReusedExchange [Reuses operator id: 46] Output [6]: [c_customer_sk#137, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, c_first_sales_date_sk#142] -(138) BroadcastHashJoin [codegen id : 51] +(136) BroadcastHashJoin [codegen id : 49] Left keys [1]: [ss_customer_sk#107] Right keys [1]: [c_customer_sk#137] Join type: Inner Join condition: None -(139) Project [codegen id : 51] +(137) Project [codegen id : 49] Output [16]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, c_first_sales_date_sk#142] Input [18]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_customer_sk#137, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, c_first_sales_date_sk#142] -(140) ReusedExchange [Reuses operator id: 53] +(138) ReusedExchange [Reuses operator id: 52] Output [2]: [d_date_sk#143, d_year#144] -(141) BroadcastHashJoin [codegen id : 51] +(139) BroadcastHashJoin [codegen id : 49] Left keys [1]: [c_first_sales_date_sk#142] Right keys [1]: [d_date_sk#143] Join type: Inner Join condition: None -(142) Project [codegen id : 51] +(140) Project [codegen id : 49] Output [16]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, d_year#144] Input [18]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, c_first_sales_date_sk#142, d_date_sk#143, d_year#144] -(143) ReusedExchange [Reuses operator id: 53] +(141) ReusedExchange [Reuses operator id: 52] Output [2]: [d_date_sk#145, d_year#146] -(144) BroadcastHashJoin [codegen id : 51] +(142) BroadcastHashJoin [codegen id : 49] Left keys [1]: [c_first_shipto_date_sk#141] Right keys [1]: [d_date_sk#145] Join type: Inner Join condition: None -(145) Project [codegen id : 51] +(143) Project [codegen id : 49] Output [16]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146] Input [18]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, d_year#144, d_date_sk#145, d_year#146] -(146) ReusedExchange [Reuses operator id: 62] +(144) ReusedExchange [Reuses operator id: 61] Output [2]: [cd_demo_sk#147, cd_marital_status#148] -(147) BroadcastHashJoin [codegen id : 51] +(145) BroadcastHashJoin [codegen id : 49] Left keys [1]: [ss_cdemo_sk#108] Right keys [1]: [cd_demo_sk#147] Join type: Inner Join condition: None -(148) Project [codegen id : 51] +(146) Project [codegen id : 49] Output [16]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, cd_marital_status#148] Input [18]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, cd_demo_sk#147, cd_marital_status#148] -(149) ReusedExchange [Reuses operator id: 62] +(147) ReusedExchange [Reuses operator id: 61] Output [2]: [cd_demo_sk#149, cd_marital_status#150] -(150) BroadcastHashJoin [codegen id : 51] +(148) BroadcastHashJoin [codegen id : 49] Left keys [1]: [c_current_cdemo_sk#138] Right keys [1]: [cd_demo_sk#149] Join type: Inner Join condition: NOT (cd_marital_status#148 = cd_marital_status#150) -(151) Project [codegen id : 51] +(149) Project [codegen id : 49] Output [14]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146] Input [18]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, cd_marital_status#148, cd_demo_sk#149, cd_marital_status#150] -(152) ReusedExchange [Reuses operator id: 71] +(150) ReusedExchange [Reuses operator id: 70] Output [1]: [p_promo_sk#151] -(153) BroadcastHashJoin [codegen id : 51] +(151) BroadcastHashJoin [codegen id : 49] Left keys [1]: [ss_promo_sk#112] Right keys [1]: [p_promo_sk#151] Join type: Inner Join condition: None -(154) Project [codegen id : 51] +(152) Project [codegen id : 49] Output [13]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146] Input [15]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, p_promo_sk#151] -(155) ReusedExchange [Reuses operator id: 77] +(153) ReusedExchange [Reuses operator id: 76] Output [2]: [hd_demo_sk#152, hd_income_band_sk#153] -(156) BroadcastHashJoin [codegen id : 51] +(154) BroadcastHashJoin [codegen id : 49] Left keys [1]: [ss_hdemo_sk#109] Right keys [1]: [hd_demo_sk#152] Join type: Inner Join condition: None -(157) Project [codegen id : 51] +(155) Project [codegen id : 49] Output [13]: [ss_item_sk#106, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153] Input [15]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, hd_demo_sk#152, hd_income_band_sk#153] -(158) ReusedExchange [Reuses operator id: 77] +(156) ReusedExchange [Reuses operator id: 76] Output [2]: [hd_demo_sk#154, hd_income_band_sk#155] -(159) BroadcastHashJoin [codegen id : 51] +(157) BroadcastHashJoin [codegen id : 49] Left keys [1]: [c_current_hdemo_sk#139] Right keys [1]: [hd_demo_sk#154] Join type: Inner Join condition: None -(160) Project [codegen id : 51] +(158) Project [codegen id : 49] Output [13]: [ss_item_sk#106, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155] Input [15]: [ss_item_sk#106, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153, hd_demo_sk#154, hd_income_band_sk#155] -(161) ReusedExchange [Reuses operator id: 86] +(159) ReusedExchange [Reuses operator id: 85] Output [5]: [ca_address_sk#156, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160] -(162) BroadcastHashJoin [codegen id : 51] +(160) BroadcastHashJoin [codegen id : 49] Left keys [1]: [ss_addr_sk#110] Right keys [1]: [ca_address_sk#156] Join type: Inner Join condition: None -(163) Project [codegen id : 51] +(161) Project [codegen id : 49] Output [16]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160] Input [18]: [ss_item_sk#106, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155, ca_address_sk#156, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160] -(164) ReusedExchange [Reuses operator id: 86] +(162) ReusedExchange [Reuses operator id: 85] Output [5]: [ca_address_sk#161, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165] -(165) BroadcastHashJoin [codegen id : 51] +(163) BroadcastHashJoin [codegen id : 49] Left keys [1]: [c_current_addr_sk#140] Right keys [1]: [ca_address_sk#161] Join type: Inner Join condition: None -(166) Project [codegen id : 51] +(164) Project [codegen id : 49] Output [19]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165] Input [21]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_address_sk#161, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165] -(167) ReusedExchange [Reuses operator id: 95] +(165) ReusedExchange [Reuses operator id: 94] Output [1]: [ib_income_band_sk#166] -(168) BroadcastHashJoin [codegen id : 51] +(166) BroadcastHashJoin [codegen id : 49] Left keys [1]: [hd_income_band_sk#153] Right keys [1]: [ib_income_band_sk#166] Join type: Inner Join condition: None -(169) Project [codegen id : 51] +(167) Project [codegen id : 49] Output [18]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165] Input [20]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, ib_income_band_sk#166] -(170) ReusedExchange [Reuses operator id: 95] +(168) ReusedExchange [Reuses operator id: 94] Output [1]: [ib_income_band_sk#167] -(171) BroadcastHashJoin [codegen id : 51] +(169) BroadcastHashJoin [codegen id : 49] Left keys [1]: [hd_income_band_sk#155] Right keys [1]: [ib_income_band_sk#167] Join type: Inner Join condition: None -(172) Project [codegen id : 51] +(170) Project [codegen id : 49] Output [17]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165] Input [19]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, ib_income_band_sk#167] -(173) ReusedExchange [Reuses operator id: 105] +(171) ReusedExchange [Reuses operator id: 104] Output [2]: [i_item_sk#168, i_product_name#169] -(174) BroadcastHashJoin [codegen id : 51] +(172) BroadcastHashJoin [codegen id : 49] Left keys [1]: [ss_item_sk#106] Right keys [1]: [i_item_sk#168] Join type: Inner Join condition: None -(175) Project [codegen id : 51] +(173) Project [codegen id : 49] Output [18]: [ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, d_year#144, d_year#146, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, i_item_sk#168, i_product_name#169] Input [19]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, i_item_sk#168, i_product_name#169] -(176) HashAggregate [codegen id : 51] +(174) HashAggregate [codegen id : 49] Input [18]: [ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, d_year#144, d_year#146, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, i_item_sk#168, i_product_name#169] Keys [15]: [i_product_name#169, i_item_sk#168, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, d_year#133, d_year#144, d_year#146] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#114)), partial_sum(UnscaledValue(ss_list_price#115)), partial_sum(UnscaledValue(ss_coupon_amt#116))] Aggregate Attributes [4]: [count#77, sum#170, sum#171, sum#172] Results [19]: [i_product_name#169, i_item_sk#168, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, d_year#133, d_year#144, d_year#146, count#81, sum#173, sum#174, sum#175] -(177) HashAggregate [codegen id : 51] +(175) HashAggregate [codegen id : 49] Input [19]: [i_product_name#169, i_item_sk#168, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, d_year#133, d_year#144, d_year#146, count#81, sum#173, sum#174, sum#175] Keys [15]: [i_product_name#169, i_item_sk#168, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, d_year#133, d_year#144, d_year#146] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#114)), sum(UnscaledValue(ss_list_price#115)), sum(UnscaledValue(ss_coupon_amt#116))] Aggregate Attributes [4]: [count(1)#85, sum(UnscaledValue(ss_wholesale_cost#114))#86, sum(UnscaledValue(ss_list_price#115))#87, sum(UnscaledValue(ss_coupon_amt#116))#88] Results [8]: [i_item_sk#168 AS item_sk#176, s_store_name#135 AS store_name#177, s_zip#136 AS store_zip#178, d_year#133 AS syear#179, count(1)#85 AS cnt#180, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#114))#86,17,2) AS s1#181, MakeDecimal(sum(UnscaledValue(ss_list_price#115))#87,17,2) AS s2#182, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#116))#88,17,2) AS s3#183] -(178) Exchange +(176) Exchange Input [8]: [item_sk#176, store_name#177, store_zip#178, syear#179, cnt#180, s1#181, s2#182, s3#183] -Arguments: hashpartitioning(item_sk#176, store_name#177, store_zip#178, 5), ENSURE_REQUIREMENTS, [plan_id=18] +Arguments: hashpartitioning(item_sk#176, store_name#177, store_zip#178, 5), ENSURE_REQUIREMENTS, [plan_id=16] -(179) Sort [codegen id : 52] +(177) Sort [codegen id : 50] Input [8]: [item_sk#176, store_name#177, store_zip#178, syear#179, cnt#180, s1#181, s2#182, s3#183] Arguments: [item_sk#176 ASC NULLS FIRST, store_name#177 ASC NULLS FIRST, store_zip#178 ASC NULLS FIRST], false, 0 -(180) SortMergeJoin [codegen id : 53] +(178) SortMergeJoin [codegen id : 51] Left keys [3]: [item_sk#90, store_name#91, store_zip#92] Right keys [3]: [item_sk#176, store_name#177, store_zip#178] Join type: Inner Join condition: (cnt#180 <= cnt#102) -(181) Project [codegen id : 53] +(179) Project [codegen id : 51] Output [21]: [product_name#89, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105, s1#181, s2#182, s3#183, syear#179, cnt#180] Input [25]: [product_name#89, item_sk#90, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105, item_sk#176, store_name#177, store_zip#178, syear#179, cnt#180, s1#181, s2#182, s3#183] -(182) Exchange +(180) Exchange Input [21]: [product_name#89, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105, s1#181, s2#182, s3#183, syear#179, cnt#180] -Arguments: rangepartitioning(product_name#89 ASC NULLS FIRST, store_name#91 ASC NULLS FIRST, cnt#180 ASC NULLS FIRST, s1#103 ASC NULLS FIRST, s1#181 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=19] +Arguments: rangepartitioning(product_name#89 ASC NULLS FIRST, store_name#91 ASC NULLS FIRST, cnt#180 ASC NULLS FIRST, s1#103 ASC NULLS FIRST, s1#181 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=17] -(183) Sort [codegen id : 54] +(181) Sort [codegen id : 52] Input [21]: [product_name#89, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105, s1#181, s2#182, s3#183, syear#179, cnt#180] Arguments: [product_name#89 ASC NULLS FIRST, store_name#91 ASC NULLS FIRST, cnt#180 ASC NULLS FIRST, s1#103 ASC NULLS FIRST, s1#181 ASC NULLS FIRST], true, 0 ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (187) -+- * Filter (186) - +- * ColumnarToRow (185) - +- CometScan parquet spark_catalog.default.date_dim (184) +BroadcastExchange (185) ++- * ColumnarToRow (184) + +- CometFilter (183) + +- CometScan parquet spark_catalog.default.date_dim (182) -(184) Scan parquet spark_catalog.default.date_dim +(182) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#37, d_year#38] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(185) ColumnarToRow [codegen id : 1] +(183) CometFilter Input [2]: [d_date_sk#37, d_year#38] +Condition : ((isnotnull(d_year#38) AND (d_year#38 = 1999)) AND isnotnull(d_date_sk#37)) -(186) Filter [codegen id : 1] +(184) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#37, d_year#38] -Condition : ((isnotnull(d_year#38) AND (d_year#38 = 1999)) AND isnotnull(d_date_sk#37)) -(187) BroadcastExchange +(185) BroadcastExchange Input [2]: [d_date_sk#37, d_year#38] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=20] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=18] -Subquery:2 Hosting operator id = 112 Hosting Expression = ss_sold_date_sk#117 IN dynamicpruning#118 -BroadcastExchange (191) -+- * Filter (190) - +- * ColumnarToRow (189) - +- CometScan parquet spark_catalog.default.date_dim (188) +Subquery:2 Hosting operator id = 111 Hosting Expression = ss_sold_date_sk#117 IN dynamicpruning#118 +BroadcastExchange (189) ++- * ColumnarToRow (188) + +- CometFilter (187) + +- CometScan parquet spark_catalog.default.date_dim (186) -(188) Scan parquet spark_catalog.default.date_dim +(186) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#132, d_year#133] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(189) ColumnarToRow [codegen id : 1] +(187) CometFilter Input [2]: [d_date_sk#132, d_year#133] +Condition : ((isnotnull(d_year#133) AND (d_year#133 = 2000)) AND isnotnull(d_date_sk#132)) -(190) Filter [codegen id : 1] +(188) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#132, d_year#133] -Condition : ((isnotnull(d_year#133) AND (d_year#133 = 2000)) AND isnotnull(d_date_sk#132)) -(191) BroadcastExchange +(189) BroadcastExchange Input [2]: [d_date_sk#132, d_year#133] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=21] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/simplified.txt index 410352945..2a0bc5bce 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/simplified.txt @@ -1,16 +1,16 @@ -WholeStageCodegen (54) +WholeStageCodegen (52) Sort [product_name,store_name,cnt,s1,s1] InputAdapter Exchange [product_name,store_name,cnt,s1,s1] #1 - WholeStageCodegen (53) + WholeStageCodegen (51) Project [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] SortMergeJoin [item_sk,store_name,store_zip,item_sk,store_name,store_zip,cnt,cnt] InputAdapter - WholeStageCodegen (26) + WholeStageCodegen (25) Sort [item_sk,store_name,store_zip] InputAdapter Exchange [item_sk,store_name,store_zip] #2 - WholeStageCodegen (25) + WholeStageCodegen (24) HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] @@ -46,148 +46,144 @@ WholeStageCodegen (54) Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] SortMergeJoin [ss_item_sk,cs_item_sk] InputAdapter - WholeStageCodegen (3) + WholeStageCodegen (2) Sort [ss_item_sk] InputAdapter Exchange [ss_item_sk] #3 - WholeStageCodegen (2) - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_ticket_number,ss_store_sk,ss_customer_sk,ss_cdemo_sk,ss_promo_sk,ss_hdemo_sk,ss_addr_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - Project [sr_item_sk,sr_ticket_number] - Filter [sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] + CometBroadcastExchange #4 + CometFilter [ss_item_sk,ss_ticket_number,ss_store_sk,ss_customer_sk,ss_cdemo_sk,ss_promo_sk,ss_hdemo_sk,ss_addr_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_year,d_date_sk] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] InputAdapter - WholeStageCodegen (9) + WholeStageCodegen (8) Sort [cs_item_sk] Project [cs_item_sk] Filter [sale,refund] HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] InputAdapter Exchange [cs_item_sk] #6 - WholeStageCodegen (8) + WholeStageCodegen (7) HashAggregate [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [sum,sum,isEmpty,sum,sum,isEmpty] Project [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] InputAdapter - WholeStageCodegen (5) + WholeStageCodegen (4) Sort [cs_item_sk,cs_order_number] InputAdapter Exchange [cs_item_sk,cs_order_number] #7 - WholeStageCodegen (4) - Project [cs_item_sk,cs_order_number,cs_ext_list_price] - Filter [cs_item_sk,cs_order_number] - ColumnarToRow - InputAdapter + WholeStageCodegen (3) + ColumnarToRow + InputAdapter + CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] + CometFilter [cs_item_sk,cs_order_number] CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] InputAdapter - WholeStageCodegen (7) + WholeStageCodegen (6) Sort [cr_item_sk,cr_order_number] InputAdapter Exchange [cr_item_sk,cr_order_number] #8 - WholeStageCodegen (6) - Project [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter + WholeStageCodegen (5) + ColumnarToRow + InputAdapter + CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometFilter [cr_item_sk,cr_order_number] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] InputAdapter ReusedExchange [d_date_sk,d_year] #5 InputAdapter BroadcastExchange #9 - WholeStageCodegen (11) - Filter [s_store_sk,s_store_name,s_zip] - ColumnarToRow - InputAdapter + WholeStageCodegen (10) + ColumnarToRow + InputAdapter + CometFilter [s_store_sk,s_store_name,s_zip] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] InputAdapter BroadcastExchange #10 - WholeStageCodegen (12) - Filter [c_customer_sk,c_first_sales_date_sk,c_first_shipto_date_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (11) + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_first_sales_date_sk,c_first_shipto_date_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] InputAdapter BroadcastExchange #11 - WholeStageCodegen (13) - Filter [d_date_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (12) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [d_date_sk,d_year] #11 InputAdapter BroadcastExchange #12 - WholeStageCodegen (15) - Filter [cd_demo_sk,cd_marital_status] - ColumnarToRow - InputAdapter + WholeStageCodegen (14) + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk,cd_marital_status] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] InputAdapter ReusedExchange [cd_demo_sk,cd_marital_status] #12 InputAdapter BroadcastExchange #13 - WholeStageCodegen (17) - Filter [p_promo_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (16) + ColumnarToRow + InputAdapter + CometFilter [p_promo_sk] CometScan parquet spark_catalog.default.promotion [p_promo_sk] InputAdapter BroadcastExchange #14 - WholeStageCodegen (18) - Filter [hd_demo_sk,hd_income_band_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (17) + ColumnarToRow + InputAdapter + CometFilter [hd_demo_sk,hd_income_band_sk] CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] InputAdapter ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 InputAdapter BroadcastExchange #15 - WholeStageCodegen (20) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (19) + ColumnarToRow + InputAdapter + CometFilter [ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] InputAdapter ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 InputAdapter BroadcastExchange #16 - WholeStageCodegen (22) - Filter [ib_income_band_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (21) + ColumnarToRow + InputAdapter + CometFilter [ib_income_band_sk] CometScan parquet spark_catalog.default.income_band [ib_income_band_sk] InputAdapter ReusedExchange [ib_income_band_sk] #16 InputAdapter BroadcastExchange #17 - WholeStageCodegen (24) - Project [i_item_sk,i_product_name] - Filter [i_current_price,i_color,i_item_sk] - ColumnarToRow - InputAdapter + WholeStageCodegen (23) + ColumnarToRow + InputAdapter + CometProject [i_item_sk,i_product_name] + CometFilter [i_current_price,i_color,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] InputAdapter - WholeStageCodegen (52) + WholeStageCodegen (50) Sort [item_sk,store_name,store_zip] InputAdapter Exchange [item_sk,store_name,store_zip] #18 - WholeStageCodegen (51) + WholeStageCodegen (49) HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] @@ -223,34 +219,30 @@ WholeStageCodegen (54) Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] SortMergeJoin [ss_item_sk,cs_item_sk] InputAdapter - WholeStageCodegen (29) + WholeStageCodegen (27) Sort [ss_item_sk] InputAdapter Exchange [ss_item_sk] #19 - WholeStageCodegen (28) - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] - InputAdapter - BroadcastExchange #20 - WholeStageCodegen (27) - Filter [ss_item_sk,ss_ticket_number,ss_store_sk,ss_customer_sk,ss_cdemo_sk,ss_promo_sk,ss_hdemo_sk,ss_addr_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #21 - WholeStageCodegen (1) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - Project [sr_item_sk,sr_ticket_number] - Filter [sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter + WholeStageCodegen (26) + ColumnarToRow + InputAdapter + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] + CometBroadcastExchange #20 + CometFilter [ss_item_sk,ss_ticket_number,ss_store_sk,ss_customer_sk,ss_cdemo_sk,ss_promo_sk,ss_hdemo_sk,ss_addr_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #2 + BroadcastExchange #21 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_year,d_date_sk] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] InputAdapter - WholeStageCodegen (35) + WholeStageCodegen (33) Sort [cs_item_sk] Project [cs_item_sk] Filter [sale,refund] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/explain.txt index 5c6c416ce..6ca27323a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/explain.txt @@ -17,17 +17,17 @@ TakeOrderedAndProject (70) : : +- * BroadcastHashJoin Inner BuildRight (11) : : :- * Project (6) : : : +- * BroadcastHashJoin Inner BuildRight (5) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : :- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- ReusedExchange (4) : : +- BroadcastExchange (10) - : : +- * Filter (9) - : : +- * ColumnarToRow (8) + : : +- * ColumnarToRow (9) + : : +- CometFilter (8) : : +- CometScan parquet spark_catalog.default.store (7) : +- BroadcastExchange (16) - : +- * Filter (15) - : +- * ColumnarToRow (14) + : +- * ColumnarToRow (15) + : +- CometFilter (14) : +- CometScan parquet spark_catalog.default.item (13) :- * HashAggregate (26) : +- Exchange (25) @@ -79,12 +79,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] +Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) -(3) Filter [codegen id : 4] +(3) ColumnarToRow [codegen id : 4] Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) (4) ReusedExchange [Reuses operator id: 75] Output [4]: [d_date_sk#7, d_year#8, d_moy#9, d_qoy#10] @@ -106,12 +106,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] +(8) CometFilter Input [2]: [s_store_sk#11, s_store_id#12] +Condition : isnotnull(s_store_sk#11) -(9) Filter [codegen id : 2] +(9) ColumnarToRow [codegen id : 2] Input [2]: [s_store_sk#11, s_store_id#12] -Condition : isnotnull(s_store_sk#11) (10) BroadcastExchange Input [2]: [s_store_sk#11, s_store_id#12] @@ -134,12 +134,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] +(14) CometFilter Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] +Condition : isnotnull(i_item_sk#13) -(15) Filter [codegen id : 3] +(15) ColumnarToRow [codegen id : 3] Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Condition : isnotnull(i_item_sk#13) (16) BroadcastExchange Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] @@ -435,9 +435,9 @@ Arguments: 100, [i_category#23 ASC NULLS FIRST, i_class#24 ASC NULLS FIRST, i_br Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 BroadcastExchange (75) -+- * Project (74) - +- * Filter (73) - +- * ColumnarToRow (72) ++- * ColumnarToRow (74) + +- CometProject (73) + +- CometFilter (72) +- CometScan parquet spark_catalog.default.date_dim (71) @@ -448,16 +448,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(72) ColumnarToRow [codegen id : 1] -Input [5]: [d_date_sk#7, d_month_seq#221, d_year#8, d_moy#9, d_qoy#10] - -(73) Filter [codegen id : 1] +(72) CometFilter Input [5]: [d_date_sk#7, d_month_seq#221, d_year#8, d_moy#9, d_qoy#10] Condition : (((isnotnull(d_month_seq#221) AND (d_month_seq#221 >= 1212)) AND (d_month_seq#221 <= 1223)) AND isnotnull(d_date_sk#7)) -(74) Project [codegen id : 1] -Output [4]: [d_date_sk#7, d_year#8, d_moy#9, d_qoy#10] +(73) CometProject Input [5]: [d_date_sk#7, d_month_seq#221, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#7, d_year#8, d_moy#9, d_qoy#10], [d_date_sk#7, d_year#8, d_moy#9, d_qoy#10] + +(74) ColumnarToRow [codegen id : 1] +Input [4]: [d_date_sk#7, d_year#8, d_moy#9, d_qoy#10] (75) BroadcastExchange Input [4]: [d_date_sk#7, d_year#8, d_moy#9, d_qoy#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/simplified.txt index aed12f8ec..4d2d3972d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/simplified.txt @@ -25,33 +25,33 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_ BroadcastHashJoin [ss_store_sk,s_store_sk] Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk,ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk,d_year,d_moy,d_qoy] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_year,d_moy,d_qoy] + CometFilter [d_month_seq,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] InputAdapter ReusedExchange [d_date_sk,d_year,d_moy,d_qoy] #3 InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] WholeStageCodegen (11) HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sum,isEmpty] [sum(sumsales),s_store_id,sumsales,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/explain.txt index 189cbfd42..6a81a2ec1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/explain.txt @@ -15,14 +15,14 @@ TakeOrderedAndProject (56) : +- * BroadcastHashJoin Inner BuildRight (33) : :- * Project (6) : : +- * BroadcastHashJoin Inner BuildRight (5) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) + : : :- * ColumnarToRow (3) + : : : +- CometFilter (2) : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : +- ReusedExchange (4) : +- BroadcastExchange (32) : +- * BroadcastHashJoin LeftSemi BuildRight (31) - : :- * Filter (9) - : : +- * ColumnarToRow (8) + : :- * ColumnarToRow (9) + : : +- CometFilter (8) : : +- CometScan parquet spark_catalog.default.store (7) : +- BroadcastExchange (30) : +- * Project (29) @@ -37,12 +37,12 @@ TakeOrderedAndProject (56) : +- * BroadcastHashJoin Inner BuildRight (20) : :- * Project (18) : : +- * BroadcastHashJoin Inner BuildRight (17) - : : :- * Filter (12) - : : : +- * ColumnarToRow (11) + : : :- * ColumnarToRow (12) + : : : +- CometFilter (11) : : : +- CometScan parquet spark_catalog.default.store_sales (10) : : +- BroadcastExchange (16) - : : +- * Filter (15) - : : +- * ColumnarToRow (14) + : : +- * ColumnarToRow (15) + : : +- CometFilter (14) : : +- CometScan parquet spark_catalog.default.store (13) : +- ReusedExchange (19) :- * HashAggregate (42) @@ -65,12 +65,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 8] +(2) CometFilter Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Condition : isnotnull(ss_store_sk#1) -(3) Filter [codegen id : 8] +(3) ColumnarToRow [codegen id : 8] Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_store_sk#1) (4) ReusedExchange [Reuses operator id: 61] Output [1]: [d_date_sk#5] @@ -92,12 +92,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 7] +(8) CometFilter Input [3]: [s_store_sk#6, s_county#7, s_state#8] +Condition : isnotnull(s_store_sk#6) -(9) Filter [codegen id : 7] +(9) ColumnarToRow [codegen id : 7] Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Condition : isnotnull(s_store_sk#6) (10) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] @@ -107,12 +107,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 4] +(11) CometFilter Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Condition : isnotnull(ss_store_sk#9) -(12) Filter [codegen id : 4] +(12) ColumnarToRow [codegen id : 4] Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] -Condition : isnotnull(ss_store_sk#9) (13) Scan parquet spark_catalog.default.store Output [2]: [s_store_sk#13, s_state#14] @@ -121,12 +121,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 2] +(14) CometFilter Input [2]: [s_store_sk#13, s_state#14] +Condition : isnotnull(s_store_sk#13) -(15) Filter [codegen id : 2] +(15) ColumnarToRow [codegen id : 2] Input [2]: [s_store_sk#13, s_state#14] -Condition : isnotnull(s_store_sk#13) (16) BroadcastExchange Input [2]: [s_store_sk#13, s_state#14] @@ -335,9 +335,9 @@ Arguments: 100, [lochierarchy#27 DESC NULLS LAST, CASE WHEN (lochierarchy#27 = 0 Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 BroadcastExchange (61) -+- * Project (60) - +- * Filter (59) - +- * ColumnarToRow (58) ++- * ColumnarToRow (60) + +- CometProject (59) + +- CometFilter (58) +- CometScan parquet spark_catalog.default.date_dim (57) @@ -348,16 +348,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(58) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_month_seq#61] - -(59) Filter [codegen id : 1] +(58) CometFilter Input [2]: [d_date_sk#5, d_month_seq#61] Condition : (((isnotnull(d_month_seq#61) AND (d_month_seq#61 >= 1212)) AND (d_month_seq#61 <= 1223)) AND isnotnull(d_date_sk#5)) -(60) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(59) CometProject Input [2]: [d_date_sk#5, d_month_seq#61] +Arguments: [d_date_sk#5], [d_date_sk#5] + +(60) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#5] (61) BroadcastExchange Input [1]: [d_date_sk#5] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/simplified.txt index 4155cfa24..2fd04badf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/simplified.txt @@ -25,17 +25,17 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count BroadcastHashJoin [ss_store_sk,s_store_sk] Project [ss_store_sk,ss_net_profit] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_month_seq,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter ReusedExchange [d_date_sk] #4 @@ -43,9 +43,9 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count BroadcastExchange #5 WholeStageCodegen (7) BroadcastHashJoin [s_state,s_state] - Filter [s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] InputAdapter BroadcastExchange #6 @@ -66,17 +66,17 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_net_profit,ss_sold_date_sk,s_state] BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter BroadcastExchange #8 WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_state] InputAdapter ReusedExchange [d_date_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/explain.txt index 68229396a..c88573838 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/explain.txt @@ -25,49 +25,49 @@ TakeOrderedAndProject (70) : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (14) : : : : : : : : :- * Project (9) : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : : : : :- * Filter (3) - : : : : : : : : : : +- * ColumnarToRow (2) + : : : : : : : : : :- * ColumnarToRow (3) + : : : : : : : : : : +- CometFilter (2) : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : : : : : : : : : +- BroadcastExchange (7) - : : : : : : : : : +- * Filter (6) - : : : : : : : : : +- * ColumnarToRow (5) + : : : : : : : : : +- * ColumnarToRow (6) + : : : : : : : : : +- CometFilter (5) : : : : : : : : : +- CometScan parquet spark_catalog.default.inventory (4) : : : : : : : : +- BroadcastExchange (13) - : : : : : : : : +- * Filter (12) - : : : : : : : : +- * ColumnarToRow (11) + : : : : : : : : +- * ColumnarToRow (12) + : : : : : : : : +- CometFilter (11) : : : : : : : : +- CometScan parquet spark_catalog.default.warehouse (10) : : : : : : : +- BroadcastExchange (19) - : : : : : : : +- * Filter (18) - : : : : : : : +- * ColumnarToRow (17) + : : : : : : : +- * ColumnarToRow (18) + : : : : : : : +- CometFilter (17) : : : : : : : +- CometScan parquet spark_catalog.default.item (16) : : : : : : +- BroadcastExchange (26) - : : : : : : +- * Project (25) - : : : : : : +- * Filter (24) - : : : : : : +- * ColumnarToRow (23) + : : : : : : +- * ColumnarToRow (25) + : : : : : : +- CometProject (24) + : : : : : : +- CometFilter (23) : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (22) : : : : : +- BroadcastExchange (33) - : : : : : +- * Project (32) - : : : : : +- * Filter (31) - : : : : : +- * ColumnarToRow (30) + : : : : : +- * ColumnarToRow (32) + : : : : : +- CometProject (31) + : : : : : +- CometFilter (30) : : : : : +- CometScan parquet spark_catalog.default.household_demographics (29) : : : : +- ReusedExchange (36) : : : +- BroadcastExchange (42) - : : : +- * Filter (41) - : : : +- * ColumnarToRow (40) + : : : +- * ColumnarToRow (41) + : : : +- CometFilter (40) : : : +- CometScan parquet spark_catalog.default.date_dim (39) : : +- BroadcastExchange (48) - : : +- * Filter (47) - : : +- * ColumnarToRow (46) + : : +- * ColumnarToRow (47) + : : +- CometFilter (46) : : +- CometScan parquet spark_catalog.default.date_dim (45) : +- BroadcastExchange (54) - : +- * Filter (53) - : +- * ColumnarToRow (52) + : +- * ColumnarToRow (53) + : +- CometFilter (52) : +- CometScan parquet spark_catalog.default.promotion (51) +- * Sort (64) +- Exchange (63) - +- * Project (62) - +- * Filter (61) - +- * ColumnarToRow (60) + +- * ColumnarToRow (62) + +- CometProject (61) + +- CometFilter (60) +- CometScan parquet spark_catalog.default.catalog_returns (59) @@ -79,12 +79,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#8), dynamicpruningexpression(cs_sol PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 10] +(2) CometFilter Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] +Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) -(3) Filter [codegen id : 10] +(3) ColumnarToRow [codegen id : 10] Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) (4) Scan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] @@ -94,12 +94,12 @@ PartitionFilters: [isnotnull(inv_date_sk#13)] PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] +Condition : ((isnotnull(inv_quantity_on_hand#12) AND isnotnull(inv_item_sk#10)) AND isnotnull(inv_warehouse_sk#11)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Condition : ((isnotnull(inv_quantity_on_hand#12) AND isnotnull(inv_item_sk#10)) AND isnotnull(inv_warehouse_sk#11)) (7) BroadcastExchange Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] @@ -122,12 +122,12 @@ Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] +(11) CometFilter Input [2]: [w_warehouse_sk#14, w_warehouse_name#15] +Condition : isnotnull(w_warehouse_sk#14) -(12) Filter [codegen id : 2] +(12) ColumnarToRow [codegen id : 2] Input [2]: [w_warehouse_sk#14, w_warehouse_name#15] -Condition : isnotnull(w_warehouse_sk#14) (13) BroadcastExchange Input [2]: [w_warehouse_sk#14, w_warehouse_name#15] @@ -150,12 +150,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] +(17) CometFilter Input [2]: [i_item_sk#16, i_item_desc#17] +Condition : isnotnull(i_item_sk#16) -(18) Filter [codegen id : 3] +(18) ColumnarToRow [codegen id : 3] Input [2]: [i_item_sk#16, i_item_desc#17] -Condition : isnotnull(i_item_sk#16) (19) BroadcastExchange Input [2]: [i_item_sk#16, i_item_desc#17] @@ -178,16 +178,16 @@ Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_marital_status), EqualTo(cd_marital_status,M), IsNotNull(cd_demo_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 4] -Input [2]: [cd_demo_sk#18, cd_marital_status#19] - -(24) Filter [codegen id : 4] +(23) CometFilter Input [2]: [cd_demo_sk#18, cd_marital_status#19] Condition : ((isnotnull(cd_marital_status#19) AND (cd_marital_status#19 = M)) AND isnotnull(cd_demo_sk#18)) -(25) Project [codegen id : 4] -Output [1]: [cd_demo_sk#18] +(24) CometProject Input [2]: [cd_demo_sk#18, cd_marital_status#19] +Arguments: [cd_demo_sk#18], [cd_demo_sk#18] + +(25) ColumnarToRow [codegen id : 4] +Input [1]: [cd_demo_sk#18] (26) BroadcastExchange Input [1]: [cd_demo_sk#18] @@ -210,16 +210,16 @@ Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_buy_potential), EqualTo(hd_buy_potential,1001-5000 ), IsNotNull(hd_demo_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 5] -Input [2]: [hd_demo_sk#20, hd_buy_potential#21] - -(31) Filter [codegen id : 5] +(30) CometFilter Input [2]: [hd_demo_sk#20, hd_buy_potential#21] Condition : ((isnotnull(hd_buy_potential#21) AND (hd_buy_potential#21 = 1001-5000 )) AND isnotnull(hd_demo_sk#20)) -(32) Project [codegen id : 5] -Output [1]: [hd_demo_sk#20] +(31) CometProject Input [2]: [hd_demo_sk#20, hd_buy_potential#21] +Arguments: [hd_demo_sk#20], [hd_demo_sk#20] + +(32) ColumnarToRow [codegen id : 5] +Input [1]: [hd_demo_sk#20] (33) BroadcastExchange Input [1]: [hd_demo_sk#20] @@ -255,12 +255,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct -(40) ColumnarToRow [codegen id : 7] +(40) CometFilter Input [2]: [d_date_sk#25, d_week_seq#26] +Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) -(41) Filter [codegen id : 7] +(41) ColumnarToRow [codegen id : 7] Input [2]: [d_date_sk#25, d_week_seq#26] -Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) (42) BroadcastExchange Input [2]: [d_date_sk#25, d_week_seq#26] @@ -283,12 +283,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)] ReadSchema: struct -(46) ColumnarToRow [codegen id : 8] +(46) CometFilter Input [2]: [d_date_sk#27, d_date#28] +Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) -(47) Filter [codegen id : 8] +(47) ColumnarToRow [codegen id : 8] Input [2]: [d_date_sk#27, d_date#28] -Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) (48) BroadcastExchange Input [2]: [d_date_sk#27, d_date#28] @@ -311,12 +311,12 @@ Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(52) ColumnarToRow [codegen id : 9] +(52) CometFilter Input [1]: [p_promo_sk#29] +Condition : isnotnull(p_promo_sk#29) -(53) Filter [codegen id : 9] +(53) ColumnarToRow [codegen id : 9] Input [1]: [p_promo_sk#29] -Condition : isnotnull(p_promo_sk#29) (54) BroadcastExchange Input [1]: [p_promo_sk#29] @@ -347,16 +347,16 @@ Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(60) ColumnarToRow [codegen id : 12] -Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] - -(61) Filter [codegen id : 12] +(60) CometFilter Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Condition : (isnotnull(cr_item_sk#30) AND isnotnull(cr_order_number#31)) -(62) Project [codegen id : 12] -Output [2]: [cr_item_sk#30, cr_order_number#31] +(61) CometProject Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number#31] + +(62) ColumnarToRow [codegen id : 12] +Input [2]: [cr_item_sk#30, cr_order_number#31] (63) Exchange Input [2]: [cr_item_sk#30, cr_order_number#31] @@ -402,9 +402,9 @@ Arguments: 100, [total_cnt#38 DESC NULLS LAST, i_item_desc#17 ASC NULLS FIRST, w Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9 BroadcastExchange (75) -+- * Project (74) - +- * Filter (73) - +- * ColumnarToRow (72) ++- * ColumnarToRow (74) + +- CometProject (73) + +- CometFilter (72) +- CometScan parquet spark_catalog.default.date_dim (71) @@ -415,16 +415,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] ReadSchema: struct -(72) ColumnarToRow [codegen id : 1] -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#39] - -(73) Filter [codegen id : 1] +(72) CometFilter Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#39] Condition : ((((isnotnull(d_year#39) AND (d_year#39 = 2001)) AND isnotnull(d_date_sk#22)) AND isnotnull(d_week_seq#24)) AND isnotnull(d_date#23)) -(74) Project [codegen id : 1] -Output [3]: [d_date_sk#22, d_date#23, d_week_seq#24] +(73) CometProject Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#39] +Arguments: [d_date_sk#22, d_date#23, d_week_seq#24], [d_date_sk#22, d_date#23, d_week_seq#24] + +(74) ColumnarToRow [codegen id : 1] +Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] (75) BroadcastExchange Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/simplified.txt index 59e1279ae..17fc9dee7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/simplified.txt @@ -31,77 +31,77 @@ TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_prom BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] BroadcastHashJoin [cs_item_sk,inv_item_sk,inv_quantity_on_hand,cs_quantity] - Filter [cs_quantity,cs_item_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_ship_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_quantity,cs_item_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_ship_date_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Project [d_date_sk,d_date,d_week_seq] - Filter [d_year,d_date_sk,d_week_seq,d_date] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_date,d_week_seq] + CometFilter [d_year,d_date_sk,d_week_seq,d_date] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] InputAdapter BroadcastExchange #4 WholeStageCodegen (1) - Filter [inv_quantity_on_hand,inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [inv_quantity_on_hand,inv_item_sk,inv_warehouse_sk] CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] InputAdapter BroadcastExchange #5 WholeStageCodegen (2) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [w_warehouse_sk] CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] InputAdapter BroadcastExchange #6 WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] InputAdapter BroadcastExchange #7 WholeStageCodegen (4) - Project [cd_demo_sk] - Filter [cd_marital_status,cd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cd_demo_sk] + CometFilter [cd_marital_status,cd_demo_sk] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] InputAdapter BroadcastExchange #8 WholeStageCodegen (5) - Project [hd_demo_sk] - Filter [hd_buy_potential,hd_demo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [hd_demo_sk] + CometFilter [hd_buy_potential,hd_demo_sk] CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] InputAdapter ReusedExchange [d_date_sk,d_date,d_week_seq] #3 InputAdapter BroadcastExchange #9 WholeStageCodegen (7) - Filter [d_week_seq,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_week_seq,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] InputAdapter BroadcastExchange #10 WholeStageCodegen (8) - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_date,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter BroadcastExchange #11 WholeStageCodegen (9) - Filter [p_promo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [p_promo_sk] CometScan parquet spark_catalog.default.promotion [p_promo_sk] InputAdapter WholeStageCodegen (13) @@ -109,8 +109,8 @@ TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_prom InputAdapter Exchange [cr_item_sk,cr_order_number] #12 WholeStageCodegen (12) - Project [cr_item_sk,cr_order_number] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/explain.txt index a2f00f500..ad8c33ca1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/explain.txt @@ -13,12 +13,12 @@ TakeOrderedAndProject (71) : : : +- * BroadcastHashJoin Inner BuildRight (11) : : : :- * Project (9) : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.customer (1) : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) + : : : : +- * ColumnarToRow (6) + : : : : +- CometFilter (5) : : : : +- CometScan parquet spark_catalog.default.store_sales (4) : : : +- ReusedExchange (10) : : +- BroadcastExchange (32) @@ -29,12 +29,12 @@ TakeOrderedAndProject (71) : : +- * BroadcastHashJoin Inner BuildRight (27) : : :- * Project (25) : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : :- * Filter (19) - : : : : +- * ColumnarToRow (18) + : : : :- * ColumnarToRow (19) + : : : : +- CometFilter (18) : : : : +- CometScan parquet spark_catalog.default.customer (17) : : : +- BroadcastExchange (23) - : : : +- * Filter (22) - : : : +- * ColumnarToRow (21) + : : : +- * ColumnarToRow (22) + : : : +- CometFilter (21) : : : +- CometScan parquet spark_catalog.default.store_sales (20) : : +- ReusedExchange (26) : +- BroadcastExchange (50) @@ -46,12 +46,12 @@ TakeOrderedAndProject (71) : +- * BroadcastHashJoin Inner BuildRight (44) : :- * Project (42) : : +- * BroadcastHashJoin Inner BuildRight (41) - : : :- * Filter (36) - : : : +- * ColumnarToRow (35) + : : :- * ColumnarToRow (36) + : : : +- CometFilter (35) : : : +- CometScan parquet spark_catalog.default.customer (34) : : +- BroadcastExchange (40) - : : +- * Filter (39) - : : +- * ColumnarToRow (38) + : : +- * ColumnarToRow (39) + : : +- CometFilter (38) : : +- CometScan parquet spark_catalog.default.web_sales (37) : +- ReusedExchange (43) +- BroadcastExchange (68) @@ -62,12 +62,12 @@ TakeOrderedAndProject (71) +- * BroadcastHashJoin Inner BuildRight (63) :- * Project (61) : +- * BroadcastHashJoin Inner BuildRight (60) - : :- * Filter (55) - : : +- * ColumnarToRow (54) + : :- * ColumnarToRow (55) + : : +- CometFilter (54) : : +- CometScan parquet spark_catalog.default.customer (53) : +- BroadcastExchange (59) - : +- * Filter (58) - : +- * ColumnarToRow (57) + : +- * ColumnarToRow (58) + : +- CometFilter (57) : +- CometScan parquet spark_catalog.default.web_sales (56) +- ReusedExchange (62) @@ -79,12 +79,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) -(3) Filter [codegen id : 3] +(3) ColumnarToRow [codegen id : 3] Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) (4) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] @@ -94,12 +94,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] +Condition : isnotnull(ss_customer_sk#5) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Condition : isnotnull(ss_customer_sk#5) (7) BroadcastExchange Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] @@ -157,12 +157,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 6] +(18) CometFilter Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Condition : (isnotnull(c_customer_sk#16) AND isnotnull(c_customer_id#17)) -(19) Filter [codegen id : 6] +(19) ColumnarToRow [codegen id : 6] Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] -Condition : (isnotnull(c_customer_sk#16) AND isnotnull(c_customer_id#17)) (20) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] @@ -172,12 +172,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#22), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 4] +(21) CometFilter Input [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Condition : isnotnull(ss_customer_sk#20) -(22) Filter [codegen id : 4] +(22) ColumnarToRow [codegen id : 4] Input [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] -Condition : isnotnull(ss_customer_sk#20) (23) BroadcastExchange Input [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] @@ -241,12 +241,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 10] +(35) CometFilter Input [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] +Condition : (isnotnull(c_customer_sk#32) AND isnotnull(c_customer_id#33)) -(36) Filter [codegen id : 10] +(36) ColumnarToRow [codegen id : 10] Input [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] -Condition : (isnotnull(c_customer_sk#32) AND isnotnull(c_customer_id#33)) (37) Scan parquet spark_catalog.default.web_sales Output [3]: [ws_bill_customer_sk#36, ws_net_paid#37, ws_sold_date_sk#38] @@ -256,12 +256,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#38), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 8] +(38) CometFilter Input [3]: [ws_bill_customer_sk#36, ws_net_paid#37, ws_sold_date_sk#38] +Condition : isnotnull(ws_bill_customer_sk#36) -(39) Filter [codegen id : 8] +(39) ColumnarToRow [codegen id : 8] Input [3]: [ws_bill_customer_sk#36, ws_net_paid#37, ws_sold_date_sk#38] -Condition : isnotnull(ws_bill_customer_sk#36) (40) BroadcastExchange Input [3]: [ws_bill_customer_sk#36, ws_net_paid#37, ws_sold_date_sk#38] @@ -333,12 +333,12 @@ Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(54) ColumnarToRow [codegen id : 14] +(54) CometFilter Input [4]: [c_customer_sk#47, c_customer_id#48, c_first_name#49, c_last_name#50] +Condition : (isnotnull(c_customer_sk#47) AND isnotnull(c_customer_id#48)) -(55) Filter [codegen id : 14] +(55) ColumnarToRow [codegen id : 14] Input [4]: [c_customer_sk#47, c_customer_id#48, c_first_name#49, c_last_name#50] -Condition : (isnotnull(c_customer_sk#47) AND isnotnull(c_customer_id#48)) (56) Scan parquet spark_catalog.default.web_sales Output [3]: [ws_bill_customer_sk#51, ws_net_paid#52, ws_sold_date_sk#53] @@ -348,12 +348,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#53), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 12] +(57) CometFilter Input [3]: [ws_bill_customer_sk#51, ws_net_paid#52, ws_sold_date_sk#53] +Condition : isnotnull(ws_bill_customer_sk#51) -(58) Filter [codegen id : 12] +(58) ColumnarToRow [codegen id : 12] Input [3]: [ws_bill_customer_sk#51, ws_net_paid#52, ws_sold_date_sk#53] -Condition : isnotnull(ws_bill_customer_sk#51) (59) BroadcastExchange Input [3]: [ws_bill_customer_sk#51, ws_net_paid#52, ws_sold_date_sk#53] @@ -422,8 +422,8 @@ Arguments: 100, [customer_first_name#29 ASC NULLS FIRST, customer_id#28 ASC NULL Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 BroadcastExchange (75) -+- * Filter (74) - +- * ColumnarToRow (73) ++- * ColumnarToRow (74) + +- CometFilter (73) +- CometScan parquet spark_catalog.default.date_dim (72) @@ -434,12 +434,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(73) ColumnarToRow [codegen id : 1] +(73) CometFilter Input [2]: [d_date_sk#9, d_year#10] +Condition : (((isnotnull(d_year#10) AND (d_year#10 = 2001)) AND d_year#10 IN (2001,2002)) AND isnotnull(d_date_sk#9)) -(74) Filter [codegen id : 1] +(74) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#9, d_year#10] -Condition : (((isnotnull(d_year#10) AND (d_year#10 = 2001)) AND d_year#10 IN (2001,2002)) AND isnotnull(d_date_sk#9)) (75) BroadcastExchange Input [2]: [d_date_sk#9, d_year#10] @@ -447,8 +447,8 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint) Subquery:2 Hosting operator id = 20 Hosting Expression = ss_sold_date_sk#22 IN dynamicpruning#23 BroadcastExchange (79) -+- * Filter (78) - +- * ColumnarToRow (77) ++- * ColumnarToRow (78) + +- CometFilter (77) +- CometScan parquet spark_catalog.default.date_dim (76) @@ -459,12 +459,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(77) ColumnarToRow [codegen id : 1] +(77) CometFilter Input [2]: [d_date_sk#24, d_year#25] +Condition : (((isnotnull(d_year#25) AND (d_year#25 = 2002)) AND d_year#25 IN (2001,2002)) AND isnotnull(d_date_sk#24)) -(78) Filter [codegen id : 1] +(78) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#24, d_year#25] -Condition : (((isnotnull(d_year#25) AND (d_year#25 = 2002)) AND d_year#25 IN (2001,2002)) AND isnotnull(d_date_sk#24)) (79) BroadcastExchange Input [2]: [d_date_sk#24, d_year#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/simplified.txt index 9d6161f56..26989b0c0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/simplified.txt @@ -15,23 +15,23 @@ TakeOrderedAndProject [customer_first_name,customer_id,customer_last_name] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_customer_id] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] InputAdapter BroadcastExchange #2 WholeStageCodegen (1) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [d_date_sk,d_year] #3 @@ -47,23 +47,23 @@ TakeOrderedAndProject [customer_first_name,customer_id,customer_last_name] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_customer_id] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] InputAdapter BroadcastExchange #6 WholeStageCodegen (4) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #7 WholeStageCodegen (1) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [d_date_sk,d_year] #7 @@ -80,16 +80,16 @@ TakeOrderedAndProject [customer_first_name,customer_id,customer_last_name] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_customer_id] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] InputAdapter BroadcastExchange #10 WholeStageCodegen (8) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_bill_customer_sk] CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -106,16 +106,16 @@ TakeOrderedAndProject [customer_first_name,customer_id,customer_last_name] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_customer_id] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] InputAdapter BroadcastExchange #13 WholeStageCodegen (12) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_bill_customer_sk] CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/explain.txt index e52462afa..5c8480a96 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/explain.txt @@ -20,20 +20,20 @@ TakeOrderedAndProject (129) : : : +- * BroadcastHashJoin Inner BuildRight (12) : : : :- * Project (10) : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) + : : : : +- * ColumnarToRow (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) : : : : +- CometScan parquet spark_catalog.default.item (4) : : : +- ReusedExchange (11) : : +- * Sort (21) : : +- Exchange (20) - : : +- * Project (19) - : : +- * Filter (18) - : : +- * ColumnarToRow (17) + : : +- * ColumnarToRow (19) + : : +- CometProject (18) + : : +- CometFilter (17) : : +- CometScan parquet spark_catalog.default.catalog_returns (16) : :- * Project (42) : : +- * SortMergeJoin LeftOuter (41) @@ -43,16 +43,16 @@ TakeOrderedAndProject (129) : : : +- * BroadcastHashJoin Inner BuildRight (31) : : : :- * Project (29) : : : : +- * BroadcastHashJoin Inner BuildRight (28) - : : : : :- * Filter (26) - : : : : : +- * ColumnarToRow (25) + : : : : :- * ColumnarToRow (26) + : : : : : +- CometFilter (25) : : : : : +- CometScan parquet spark_catalog.default.store_sales (24) : : : : +- ReusedExchange (27) : : : +- ReusedExchange (30) : : +- * Sort (40) : : +- Exchange (39) - : : +- * Project (38) - : : +- * Filter (37) - : : +- * ColumnarToRow (36) + : : +- * ColumnarToRow (38) + : : +- CometProject (37) + : : +- CometFilter (36) : : +- CometScan parquet spark_catalog.default.store_returns (35) : +- * Project (61) : +- * SortMergeJoin LeftOuter (60) @@ -62,16 +62,16 @@ TakeOrderedAndProject (129) : : +- * BroadcastHashJoin Inner BuildRight (50) : : :- * Project (48) : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : :- * Filter (45) - : : : : +- * ColumnarToRow (44) + : : : :- * ColumnarToRow (45) + : : : : +- CometFilter (44) : : : : +- CometScan parquet spark_catalog.default.web_sales (43) : : : +- ReusedExchange (46) : : +- ReusedExchange (49) : +- * Sort (59) : +- Exchange (58) - : +- * Project (57) - : +- * Filter (56) - : +- * ColumnarToRow (55) + : +- * ColumnarToRow (57) + : +- CometProject (56) + : +- CometFilter (55) : +- CometScan parquet spark_catalog.default.web_returns (54) +- * Sort (126) +- Exchange (125) @@ -91,8 +91,8 @@ TakeOrderedAndProject (129) : : +- * BroadcastHashJoin Inner BuildRight (79) : : :- * Project (77) : : : +- * BroadcastHashJoin Inner BuildRight (76) - : : : :- * Filter (74) - : : : : +- * ColumnarToRow (73) + : : : :- * ColumnarToRow (74) + : : : : +- CometFilter (73) : : : : +- CometScan parquet spark_catalog.default.catalog_sales (72) : : : +- ReusedExchange (75) : : +- ReusedExchange (78) @@ -106,8 +106,8 @@ TakeOrderedAndProject (129) : : +- * BroadcastHashJoin Inner BuildRight (94) : : :- * Project (92) : : : +- * BroadcastHashJoin Inner BuildRight (91) - : : : :- * Filter (89) - : : : : +- * ColumnarToRow (88) + : : : :- * ColumnarToRow (89) + : : : : +- CometFilter (88) : : : : +- CometScan parquet spark_catalog.default.store_sales (87) : : : +- ReusedExchange (90) : : +- ReusedExchange (93) @@ -121,8 +121,8 @@ TakeOrderedAndProject (129) : +- * BroadcastHashJoin Inner BuildRight (109) : :- * Project (107) : : +- * BroadcastHashJoin Inner BuildRight (106) - : : :- * Filter (104) - : : : +- * ColumnarToRow (103) + : : :- * ColumnarToRow (104) + : : : +- CometFilter (103) : : : +- CometScan parquet spark_catalog.default.web_sales (102) : : +- ReusedExchange (105) : +- ReusedExchange (108) @@ -138,12 +138,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sol PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] +Condition : isnotnull(cs_item_sk#1) -(3) Filter [codegen id : 3] +(3) ColumnarToRow [codegen id : 3] Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Condition : isnotnull(cs_item_sk#1) (4) Scan parquet spark_catalog.default.item Output [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] @@ -152,16 +152,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Books ), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] - -(6) Filter [codegen id : 1] +(5) CometFilter Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] Condition : ((((((isnotnull(i_category#11) AND (i_category#11 = Books )) AND isnotnull(i_item_sk#7)) AND isnotnull(i_brand_id#8)) AND isnotnull(i_class_id#9)) AND isnotnull(i_category_id#10)) AND isnotnull(i_manufact_id#12)) -(7) Project [codegen id : 1] -Output [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +(6) CometProject Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] +Arguments: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12], [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] + +(7) ColumnarToRow [codegen id : 1] +Input [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] (8) BroadcastExchange Input [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] @@ -205,16 +205,16 @@ Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 5] -Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] - -(18) Filter [codegen id : 5] +(17) CometFilter Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] Condition : (isnotnull(cr_order_number#16) AND isnotnull(cr_item_sk#15)) -(19) Project [codegen id : 5] -Output [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] +(18) CometProject Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] +Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18], [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] + +(19) ColumnarToRow [codegen id : 5] +Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] (20) Exchange Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] @@ -242,12 +242,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#26), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 10] +(25) CometFilter Input [5]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26] +Condition : isnotnull(ss_item_sk#22) -(26) Filter [codegen id : 10] +(26) ColumnarToRow [codegen id : 10] Input [5]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Condition : isnotnull(ss_item_sk#22) (27) ReusedExchange [Reuses operator id: 8] Output [5]: [i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] @@ -290,16 +290,16 @@ Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(36) ColumnarToRow [codegen id : 12] -Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] - -(37) Filter [codegen id : 12] +(36) CometFilter Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] Condition : (isnotnull(sr_ticket_number#36) AND isnotnull(sr_item_sk#35)) -(38) Project [codegen id : 12] -Output [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] +(37) CometProject Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] +Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38], [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] + +(38) ColumnarToRow [codegen id : 12] +Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] (39) Exchange Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] @@ -327,12 +327,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#46), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(44) ColumnarToRow [codegen id : 17] +(44) CometFilter Input [5]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46] +Condition : isnotnull(ws_item_sk#42) -(45) Filter [codegen id : 17] +(45) ColumnarToRow [codegen id : 17] Input [5]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46] -Condition : isnotnull(ws_item_sk#42) (46) ReusedExchange [Reuses operator id: 8] Output [5]: [i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] @@ -375,16 +375,16 @@ Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct -(55) ColumnarToRow [codegen id : 19] -Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] - -(56) Filter [codegen id : 19] +(55) CometFilter Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] Condition : (isnotnull(wr_order_number#56) AND isnotnull(wr_item_sk#55)) -(57) Project [codegen id : 19] -Output [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] +(56) CometProject Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] +Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58], [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] + +(57) ColumnarToRow [codegen id : 19] +Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] (58) Exchange Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] @@ -462,12 +462,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#74), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(73) ColumnarToRow [codegen id : 28] +(73) CometFilter Input [5]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74] +Condition : isnotnull(cs_item_sk#70) -(74) Filter [codegen id : 28] +(74) ColumnarToRow [codegen id : 28] Input [5]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74] -Condition : isnotnull(cs_item_sk#70) (75) ReusedExchange [Reuses operator id: 8] Output [5]: [i_item_sk#76, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] @@ -528,12 +528,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#93), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(88) ColumnarToRow [codegen id : 35] +(88) CometFilter Input [5]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, ss_sold_date_sk#93] +Condition : isnotnull(ss_item_sk#89) -(89) Filter [codegen id : 35] +(89) ColumnarToRow [codegen id : 35] Input [5]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, ss_sold_date_sk#93] -Condition : isnotnull(ss_item_sk#89) (90) ReusedExchange [Reuses operator id: 8] Output [5]: [i_item_sk#95, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99] @@ -594,12 +594,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#112), dynamicpruningexpression(ws_s PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(103) ColumnarToRow [codegen id : 42] +(103) CometFilter Input [5]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, ws_sold_date_sk#112] +Condition : isnotnull(ws_item_sk#108) -(104) Filter [codegen id : 42] +(104) ColumnarToRow [codegen id : 42] Input [5]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, ws_sold_date_sk#112] -Condition : isnotnull(ws_item_sk#108) (105) ReusedExchange [Reuses operator id: 8] Output [5]: [i_item_sk#114, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118] @@ -720,8 +720,8 @@ Arguments: 100, [sales_cnt_diff#137 ASC NULLS FIRST, sales_amt_diff#138 ASC NULL Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 BroadcastExchange (133) -+- * Filter (132) - +- * ColumnarToRow (131) ++- * ColumnarToRow (132) + +- CometFilter (131) +- CometScan parquet spark_catalog.default.date_dim (130) @@ -732,12 +732,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(131) ColumnarToRow [codegen id : 1] +(131) CometFilter Input [2]: [d_date_sk#13, d_year#14] +Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) -(132) Filter [codegen id : 1] +(132) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) (133) BroadcastExchange Input [2]: [d_date_sk#13, d_year#14] @@ -749,8 +749,8 @@ Subquery:3 Hosting operator id = 43 Hosting Expression = ws_sold_date_sk#46 IN d Subquery:4 Hosting operator id = 72 Hosting Expression = cs_sold_date_sk#74 IN dynamicpruning#75 BroadcastExchange (137) -+- * Filter (136) - +- * ColumnarToRow (135) ++- * ColumnarToRow (136) + +- CometFilter (135) +- CometScan parquet spark_catalog.default.date_dim (134) @@ -761,12 +761,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(135) ColumnarToRow [codegen id : 1] +(135) CometFilter Input [2]: [d_date_sk#81, d_year#82] +Condition : ((isnotnull(d_year#82) AND (d_year#82 = 2001)) AND isnotnull(d_date_sk#81)) -(136) Filter [codegen id : 1] +(136) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#81, d_year#82] -Condition : ((isnotnull(d_year#82) AND (d_year#82 = 2001)) AND isnotnull(d_date_sk#81)) (137) BroadcastExchange Input [2]: [d_date_sk#81, d_year#82] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/simplified.txt index 3a7ec65a0..9939b2fe2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/simplified.txt @@ -34,24 +34,24 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #5 WholeStageCodegen (1) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #6 WholeStageCodegen (1) - Project [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - Filter [i_category,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [i_category,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] InputAdapter ReusedExchange [d_date_sk,d_year] #5 @@ -61,10 +61,10 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i InputAdapter Exchange [cr_order_number,cr_item_sk] #7 WholeStageCodegen (5) - Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - Filter [cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_order_number,cr_item_sk] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] WholeStageCodegen (14) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] @@ -79,9 +79,9 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -94,10 +94,10 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i InputAdapter Exchange [sr_ticket_number,sr_item_sk] #9 WholeStageCodegen (12) - Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_ticket_number,sr_item_sk] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] WholeStageCodegen (21) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] @@ -112,9 +112,9 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -127,10 +127,10 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i InputAdapter Exchange [wr_order_number,wr_item_sk] #11 WholeStageCodegen (19) - Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - Filter [wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_order_number,wr_item_sk] CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] InputAdapter WholeStageCodegen (50) @@ -164,16 +164,16 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #16 WholeStageCodegen (1) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 @@ -197,9 +197,9 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #2 InputAdapter @@ -224,9 +224,9 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/explain.txt index ec9c0ac47..5c1865267 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/explain.txt @@ -17,13 +17,13 @@ TakeOrderedAndProject (98) : : : +- * BroadcastHashJoin Inner BuildRight (11) : : : :- * Project (6) : : : : +- * BroadcastHashJoin Inner BuildRight (5) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) + : : : : :- * ColumnarToRow (3) + : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : : +- ReusedExchange (4) : : : +- BroadcastExchange (10) - : : : +- * Filter (9) - : : : +- * ColumnarToRow (8) + : : : +- * ColumnarToRow (9) + : : : +- CometFilter (8) : : : +- CometScan parquet spark_catalog.default.store (7) : : +- BroadcastExchange (28) : : +- * HashAggregate (27) @@ -33,8 +33,8 @@ TakeOrderedAndProject (98) : : +- * BroadcastHashJoin Inner BuildRight (23) : : :- * Project (21) : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : :- * Filter (18) - : : : : +- * ColumnarToRow (17) + : : : :- * ColumnarToRow (18) + : : : : +- CometFilter (17) : : : : +- CometScan parquet spark_catalog.default.store_returns (16) : : : +- ReusedExchange (19) : : +- ReusedExchange (22) @@ -66,13 +66,13 @@ TakeOrderedAndProject (98) : : +- * BroadcastHashJoin Inner BuildRight (60) : : :- * Project (55) : : : +- * BroadcastHashJoin Inner BuildRight (54) - : : : :- * Filter (52) - : : : : +- * ColumnarToRow (51) + : : : :- * ColumnarToRow (52) + : : : : +- CometFilter (51) : : : : +- CometScan parquet spark_catalog.default.web_sales (50) : : : +- ReusedExchange (53) : : +- BroadcastExchange (59) - : : +- * Filter (58) - : : +- * ColumnarToRow (57) + : : +- * ColumnarToRow (58) + : : +- CometFilter (57) : : +- CometScan parquet spark_catalog.default.web_page (56) : +- BroadcastExchange (77) : +- * HashAggregate (76) @@ -82,8 +82,8 @@ TakeOrderedAndProject (98) : +- * BroadcastHashJoin Inner BuildRight (72) : :- * Project (70) : : +- * BroadcastHashJoin Inner BuildRight (69) - : : :- * Filter (67) - : : : +- * ColumnarToRow (66) + : : :- * ColumnarToRow (67) + : : : +- CometFilter (66) : : : +- CometScan parquet spark_catalog.default.web_returns (65) : : +- ReusedExchange (68) : +- ReusedExchange (71) @@ -107,12 +107,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Condition : isnotnull(ss_store_sk#1) -(3) Filter [codegen id : 3] +(3) ColumnarToRow [codegen id : 3] Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_store_sk#1) (4) ReusedExchange [Reuses operator id: 103] Output [1]: [d_date_sk#6] @@ -134,12 +134,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] +(8) CometFilter Input [1]: [s_store_sk#7] +Condition : isnotnull(s_store_sk#7) -(9) Filter [codegen id : 2] +(9) ColumnarToRow [codegen id : 2] Input [1]: [s_store_sk#7] -Condition : isnotnull(s_store_sk#7) (10) BroadcastExchange Input [1]: [s_store_sk#7] @@ -181,12 +181,12 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#19), dynamicpruningexpression(s PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 6] +(17) CometFilter Input [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19] +Condition : isnotnull(sr_store_sk#16) -(18) Filter [codegen id : 6] +(18) ColumnarToRow [codegen id : 6] Input [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19] -Condition : isnotnull(sr_store_sk#16) (19) ReusedExchange [Reuses operator id: 103] Output [1]: [d_date_sk#21] @@ -348,12 +348,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#68), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_web_page_sk)] ReadSchema: struct -(51) ColumnarToRow [codegen id : 17] +(51) CometFilter Input [4]: [ws_web_page_sk#65, ws_ext_sales_price#66, ws_net_profit#67, ws_sold_date_sk#68] +Condition : isnotnull(ws_web_page_sk#65) -(52) Filter [codegen id : 17] +(52) ColumnarToRow [codegen id : 17] Input [4]: [ws_web_page_sk#65, ws_ext_sales_price#66, ws_net_profit#67, ws_sold_date_sk#68] -Condition : isnotnull(ws_web_page_sk#65) (53) ReusedExchange [Reuses operator id: 103] Output [1]: [d_date_sk#70] @@ -375,12 +375,12 @@ Location [not included in comparison]/{warehouse_dir}/web_page] PushedFilters: [IsNotNull(wp_web_page_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 16] +(57) CometFilter Input [1]: [wp_web_page_sk#71] +Condition : isnotnull(wp_web_page_sk#71) -(58) Filter [codegen id : 16] +(58) ColumnarToRow [codegen id : 16] Input [1]: [wp_web_page_sk#71] -Condition : isnotnull(wp_web_page_sk#71) (59) BroadcastExchange Input [1]: [wp_web_page_sk#71] @@ -422,12 +422,12 @@ PartitionFilters: [isnotnull(wr_returned_date_sk#83), dynamicpruningexpression(w PushedFilters: [IsNotNull(wr_web_page_sk)] ReadSchema: struct -(66) ColumnarToRow [codegen id : 20] +(66) CometFilter Input [4]: [wr_web_page_sk#80, wr_return_amt#81, wr_net_loss#82, wr_returned_date_sk#83] +Condition : isnotnull(wr_web_page_sk#80) -(67) Filter [codegen id : 20] +(67) ColumnarToRow [codegen id : 20] Input [4]: [wr_web_page_sk#80, wr_return_amt#81, wr_net_loss#82, wr_returned_date_sk#83] -Condition : isnotnull(wr_web_page_sk#80) (68) ReusedExchange [Reuses operator id: 103] Output [1]: [d_date_sk#85] @@ -591,9 +591,9 @@ Arguments: 100, [channel#31 ASC NULLS FIRST, id#32 ASC NULLS FIRST], [channel#31 Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 BroadcastExchange (103) -+- * Project (102) - +- * Filter (101) - +- * ColumnarToRow (100) ++- * ColumnarToRow (102) + +- CometProject (101) + +- CometFilter (100) +- CometScan parquet spark_catalog.default.date_dim (99) @@ -604,16 +604,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] ReadSchema: struct -(100) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#6, d_date#184] - -(101) Filter [codegen id : 1] +(100) CometFilter Input [2]: [d_date_sk#6, d_date#184] Condition : (((isnotnull(d_date#184) AND (d_date#184 >= 1998-08-04)) AND (d_date#184 <= 1998-09-03)) AND isnotnull(d_date_sk#6)) -(102) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(101) CometProject Input [2]: [d_date_sk#6, d_date#184] +Arguments: [d_date_sk#6], [d_date_sk#6] + +(102) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#6] (103) BroadcastExchange Input [1]: [d_date_sk#6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/simplified.txt index 9b7a2b9ef..670a7e6c3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/simplified.txt @@ -27,26 +27,26 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] BroadcastHashJoin [ss_store_sk,s_store_sk] Project [ss_store_sk,ss_ext_sales_price,ss_net_profit] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk] CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter ReusedExchange [d_date_sk] #4 InputAdapter BroadcastExchange #5 WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk] InputAdapter BroadcastExchange #6 @@ -60,9 +60,9 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] BroadcastHashJoin [sr_store_sk,s_store_sk] Project [sr_store_sk,sr_return_amt,sr_net_loss] BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Filter [sr_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [sr_store_sk] CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -113,9 +113,9 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] Project [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_web_page_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_web_page_sk] CometScan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -123,9 +123,9 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter BroadcastExchange #12 WholeStageCodegen (16) - Filter [wp_web_page_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [wp_web_page_sk] CometScan parquet spark_catalog.default.web_page [wp_web_page_sk] InputAdapter BroadcastExchange #13 @@ -139,9 +139,9 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] BroadcastHashJoin [wr_web_page_sk,wp_web_page_sk] Project [wr_web_page_sk,wr_return_amt,wr_net_loss] BroadcastHashJoin [wr_returned_date_sk,d_date_sk] - Filter [wr_web_page_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [wr_web_page_sk] CometScan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/explain.txt index 76c968e91..c7ee5b1c9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/explain.txt @@ -15,14 +15,14 @@ TakeOrderedAndProject (70) : : : +- * SortMergeJoin LeftOuter (12) : : : :- * Sort (5) : : : : +- Exchange (4) - : : : : +- * Filter (3) - : : : : +- * ColumnarToRow (2) + : : : : +- * ColumnarToRow (3) + : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- * Sort (11) : : : +- Exchange (10) - : : : +- * Project (9) - : : : +- * Filter (8) - : : : +- * ColumnarToRow (7) + : : : +- * ColumnarToRow (9) + : : : +- CometProject (8) + : : : +- CometFilter (7) : : : +- CometScan parquet spark_catalog.default.store_returns (6) : : +- ReusedExchange (15) : +- * Sort (43) @@ -37,14 +37,14 @@ TakeOrderedAndProject (70) : : +- * SortMergeJoin LeftOuter (33) : : :- * Sort (26) : : : +- Exchange (25) - : : : +- * Filter (24) - : : : +- * ColumnarToRow (23) + : : : +- * ColumnarToRow (24) + : : : +- CometFilter (23) : : : +- CometScan parquet spark_catalog.default.web_sales (22) : : +- * Sort (32) : : +- Exchange (31) - : : +- * Project (30) - : : +- * Filter (29) - : : +- * ColumnarToRow (28) + : : +- * ColumnarToRow (30) + : : +- CometProject (29) + : : +- CometFilter (28) : : +- CometScan parquet spark_catalog.default.web_returns (27) : +- ReusedExchange (36) +- * Sort (67) @@ -59,14 +59,14 @@ TakeOrderedAndProject (70) : +- * SortMergeJoin LeftOuter (57) : :- * Sort (50) : : +- Exchange (49) - : : +- * Filter (48) - : : +- * ColumnarToRow (47) + : : +- * ColumnarToRow (48) + : : +- CometFilter (47) : : +- CometScan parquet spark_catalog.default.catalog_sales (46) : +- * Sort (56) : +- Exchange (55) - : +- * Project (54) - : +- * Filter (53) - : +- * ColumnarToRow (52) + : +- * ColumnarToRow (54) + : +- CometProject (53) + : +- CometFilter (52) : +- CometScan parquet spark_catalog.default.catalog_returns (51) +- ReusedExchange (60) @@ -79,12 +79,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] +(2) CometFilter Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2)) -(3) Filter [codegen id : 1] +(3) ColumnarToRow [codegen id : 1] Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2)) (4) Exchange Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] @@ -101,16 +101,16 @@ Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 3] -Input [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] - -(8) Filter [codegen id : 3] +(7) CometFilter Input [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] Condition : (isnotnull(sr_ticket_number#10) AND isnotnull(sr_item_sk#9)) -(9) Project [codegen id : 3] -Output [2]: [sr_item_sk#9, sr_ticket_number#10] +(8) CometProject Input [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] +Arguments: [sr_item_sk#9, sr_ticket_number#10], [sr_item_sk#9, sr_ticket_number#10] + +(9) ColumnarToRow [codegen id : 3] +Input [2]: [sr_item_sk#9, sr_ticket_number#10] (10) Exchange Input [2]: [sr_item_sk#9, sr_ticket_number#10] @@ -177,12 +177,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#33), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 8] +(23) CometFilter Input [7]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] +Condition : (isnotnull(ws_item_sk#27) AND isnotnull(ws_bill_customer_sk#28)) -(24) Filter [codegen id : 8] +(24) ColumnarToRow [codegen id : 8] Input [7]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] -Condition : (isnotnull(ws_item_sk#27) AND isnotnull(ws_bill_customer_sk#28)) (25) Exchange Input [7]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] @@ -199,16 +199,16 @@ Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 10] -Input [3]: [wr_item_sk#35, wr_order_number#36, wr_returned_date_sk#37] - -(29) Filter [codegen id : 10] +(28) CometFilter Input [3]: [wr_item_sk#35, wr_order_number#36, wr_returned_date_sk#37] Condition : (isnotnull(wr_order_number#36) AND isnotnull(wr_item_sk#35)) -(30) Project [codegen id : 10] -Output [2]: [wr_item_sk#35, wr_order_number#36] +(29) CometProject Input [3]: [wr_item_sk#35, wr_order_number#36, wr_returned_date_sk#37] +Arguments: [wr_item_sk#35, wr_order_number#36], [wr_item_sk#35, wr_order_number#36] + +(30) ColumnarToRow [codegen id : 10] +Input [2]: [wr_item_sk#35, wr_order_number#36] (31) Exchange Input [2]: [wr_item_sk#35, wr_order_number#36] @@ -289,12 +289,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#60), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(47) ColumnarToRow [codegen id : 16] +(47) CometFilter Input [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] +Condition : (isnotnull(cs_item_sk#55) AND isnotnull(cs_bill_customer_sk#54)) -(48) Filter [codegen id : 16] +(48) ColumnarToRow [codegen id : 16] Input [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] -Condition : (isnotnull(cs_item_sk#55) AND isnotnull(cs_bill_customer_sk#54)) (49) Exchange Input [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] @@ -311,16 +311,16 @@ Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(52) ColumnarToRow [codegen id : 18] -Input [3]: [cr_item_sk#62, cr_order_number#63, cr_returned_date_sk#64] - -(53) Filter [codegen id : 18] +(52) CometFilter Input [3]: [cr_item_sk#62, cr_order_number#63, cr_returned_date_sk#64] Condition : (isnotnull(cr_order_number#63) AND isnotnull(cr_item_sk#62)) -(54) Project [codegen id : 18] -Output [2]: [cr_item_sk#62, cr_order_number#63] +(53) CometProject Input [3]: [cr_item_sk#62, cr_order_number#63, cr_returned_date_sk#64] +Arguments: [cr_item_sk#62, cr_order_number#63], [cr_item_sk#62, cr_order_number#63] + +(54) ColumnarToRow [codegen id : 18] +Input [2]: [cr_item_sk#62, cr_order_number#63] (55) Exchange Input [2]: [cr_item_sk#62, cr_order_number#63] @@ -401,8 +401,8 @@ Arguments: 100, [ss_sold_year#23 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 BroadcastExchange (74) -+- * Filter (73) - +- * ColumnarToRow (72) ++- * ColumnarToRow (73) + +- CometFilter (72) +- CometScan parquet spark_catalog.default.date_dim (71) @@ -413,12 +413,12 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(72) ColumnarToRow [codegen id : 1] +(72) CometFilter Input [2]: [d_date_sk#12, d_year#13] +Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2000)) AND isnotnull(d_date_sk#12)) -(73) Filter [codegen id : 1] +(73) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#12, d_year#13] -Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2000)) AND isnotnull(d_date_sk#12)) (74) BroadcastExchange Input [2]: [d_date_sk#12, d_year#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/simplified.txt index f962e8fb8..49bd173f6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/simplified.txt @@ -25,16 +25,16 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp InputAdapter Exchange [ss_ticket_number,ss_item_sk] #2 WholeStageCodegen (1) - Filter [ss_item_sk,ss_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk,ss_customer_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [d_year,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter WholeStageCodegen (4) @@ -42,10 +42,10 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp InputAdapter Exchange [sr_ticket_number,sr_item_sk] #4 WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_ticket_number,sr_item_sk] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] InputAdapter ReusedExchange [d_date_sk,d_year] #3 @@ -69,9 +69,9 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp InputAdapter Exchange [ws_order_number,ws_item_sk] #6 WholeStageCodegen (8) - Filter [ws_item_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_item_sk,ws_bill_customer_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -80,10 +80,10 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp InputAdapter Exchange [wr_order_number,wr_item_sk] #7 WholeStageCodegen (10) - Project [wr_item_sk,wr_order_number] - Filter [wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [wr_item_sk,wr_order_number] + CometFilter [wr_order_number,wr_item_sk] CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] InputAdapter ReusedExchange [d_date_sk,d_year] #3 @@ -107,9 +107,9 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp InputAdapter Exchange [cs_order_number,cs_item_sk] #9 WholeStageCodegen (16) - Filter [cs_item_sk,cs_bill_customer_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_item_sk,cs_bill_customer_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -118,10 +118,10 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp InputAdapter Exchange [cr_order_number,cr_item_sk] #10 WholeStageCodegen (18) - Project [cr_item_sk,cr_order_number] - Filter [cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_order_number,cr_item_sk] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] InputAdapter ReusedExchange [d_date_sk,d_year] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/explain.txt index d01fc085a..8f47f4f0e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/explain.txt @@ -23,29 +23,29 @@ TakeOrderedAndProject (120) : : : : : : +- * SortMergeJoin LeftOuter (12) : : : : : : :- * Sort (5) : : : : : : : +- Exchange (4) - : : : : : : : +- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) + : : : : : : : +- * ColumnarToRow (3) + : : : : : : : +- CometFilter (2) : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : : : : +- * Sort (11) : : : : : : +- Exchange (10) - : : : : : : +- * Project (9) - : : : : : : +- * Filter (8) - : : : : : : +- * ColumnarToRow (7) + : : : : : : +- * ColumnarToRow (9) + : : : : : : +- CometProject (8) + : : : : : : +- CometFilter (7) : : : : : : +- CometScan parquet spark_catalog.default.store_returns (6) : : : : : +- ReusedExchange (14) : : : : +- BroadcastExchange (20) - : : : : +- * Filter (19) - : : : : +- * ColumnarToRow (18) + : : : : +- * ColumnarToRow (19) + : : : : +- CometFilter (18) : : : : +- CometScan parquet spark_catalog.default.store (17) : : : +- BroadcastExchange (27) - : : : +- * Project (26) - : : : +- * Filter (25) - : : : +- * ColumnarToRow (24) + : : : +- * ColumnarToRow (26) + : : : +- CometProject (25) + : : : +- CometFilter (24) : : : +- CometScan parquet spark_catalog.default.item (23) : : +- BroadcastExchange (34) - : : +- * Project (33) - : : +- * Filter (32) - : : +- * ColumnarToRow (31) + : : +- * ColumnarToRow (33) + : : +- CometProject (32) + : : +- CometFilter (31) : : +- CometScan parquet spark_catalog.default.promotion (30) : :- * HashAggregate (70) : : +- Exchange (69) @@ -62,19 +62,19 @@ TakeOrderedAndProject (120) : : : : : : +- * SortMergeJoin LeftOuter (51) : : : : : : :- * Sort (44) : : : : : : : +- Exchange (43) - : : : : : : : +- * Filter (42) - : : : : : : : +- * ColumnarToRow (41) + : : : : : : : +- * ColumnarToRow (42) + : : : : : : : +- CometFilter (41) : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (40) : : : : : : +- * Sort (50) : : : : : : +- Exchange (49) - : : : : : : +- * Project (48) - : : : : : : +- * Filter (47) - : : : : : : +- * ColumnarToRow (46) + : : : : : : +- * ColumnarToRow (48) + : : : : : : +- CometProject (47) + : : : : : : +- CometFilter (46) : : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (45) : : : : : +- ReusedExchange (53) : : : : +- BroadcastExchange (59) - : : : : +- * Filter (58) - : : : : +- * ColumnarToRow (57) + : : : : +- * ColumnarToRow (58) + : : : : +- CometFilter (57) : : : : +- CometScan parquet spark_catalog.default.catalog_page (56) : : : +- ReusedExchange (62) : : +- ReusedExchange (65) @@ -93,19 +93,19 @@ TakeOrderedAndProject (120) : : : : : +- * SortMergeJoin LeftOuter (82) : : : : : :- * Sort (75) : : : : : : +- Exchange (74) - : : : : : : +- * Filter (73) - : : : : : : +- * ColumnarToRow (72) + : : : : : : +- * ColumnarToRow (73) + : : : : : : +- CometFilter (72) : : : : : : +- CometScan parquet spark_catalog.default.web_sales (71) : : : : : +- * Sort (81) : : : : : +- Exchange (80) - : : : : : +- * Project (79) - : : : : : +- * Filter (78) - : : : : : +- * ColumnarToRow (77) + : : : : : +- * ColumnarToRow (79) + : : : : : +- CometProject (78) + : : : : : +- CometFilter (77) : : : : : +- CometScan parquet spark_catalog.default.web_returns (76) : : : : +- ReusedExchange (84) : : : +- BroadcastExchange (90) - : : : +- * Filter (89) - : : : +- * ColumnarToRow (88) + : : : +- * ColumnarToRow (89) + : : : +- CometFilter (88) : : : +- CometScan parquet spark_catalog.default.web_site (87) : : +- ReusedExchange (93) : +- ReusedExchange (96) @@ -129,12 +129,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] +(2) CometFilter Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) -(3) Filter [codegen id : 1] +(3) ColumnarToRow [codegen id : 1] Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) (4) Exchange Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] @@ -151,16 +151,16 @@ Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 3] -Input [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] - -(8) Filter [codegen id : 3] +(7) CometFilter Input [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] Condition : (isnotnull(sr_item_sk#9) AND isnotnull(sr_ticket_number#10)) -(9) Project [codegen id : 3] -Output [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] +(8) CometProject Input [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Arguments: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12], [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] + +(9) ColumnarToRow [codegen id : 3] +Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] (10) Exchange Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] @@ -200,12 +200,12 @@ Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 6] +(18) CometFilter Input [2]: [s_store_sk#15, s_store_id#16] +Condition : isnotnull(s_store_sk#15) -(19) Filter [codegen id : 6] +(19) ColumnarToRow [codegen id : 6] Input [2]: [s_store_sk#15, s_store_id#16] -Condition : isnotnull(s_store_sk#15) (20) BroadcastExchange Input [2]: [s_store_sk#15, s_store_id#16] @@ -228,16 +228,16 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 7] -Input [2]: [i_item_sk#17, i_current_price#18] - -(25) Filter [codegen id : 7] +(24) CometFilter Input [2]: [i_item_sk#17, i_current_price#18] Condition : ((isnotnull(i_current_price#18) AND (i_current_price#18 > 50.00)) AND isnotnull(i_item_sk#17)) -(26) Project [codegen id : 7] -Output [1]: [i_item_sk#17] +(25) CometProject Input [2]: [i_item_sk#17, i_current_price#18] +Arguments: [i_item_sk#17], [i_item_sk#17] + +(26) ColumnarToRow [codegen id : 7] +Input [1]: [i_item_sk#17] (27) BroadcastExchange Input [1]: [i_item_sk#17] @@ -260,16 +260,16 @@ Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_channel_tv), EqualTo(p_channel_tv,N), IsNotNull(p_promo_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 8] -Input [2]: [p_promo_sk#19, p_channel_tv#20] - -(32) Filter [codegen id : 8] +(31) CometFilter Input [2]: [p_promo_sk#19, p_channel_tv#20] Condition : ((isnotnull(p_channel_tv#20) AND (p_channel_tv#20 = N)) AND isnotnull(p_promo_sk#19)) -(33) Project [codegen id : 8] -Output [1]: [p_promo_sk#19] +(32) CometProject Input [2]: [p_promo_sk#19, p_channel_tv#20] +Arguments: [p_promo_sk#19], [p_promo_sk#19] + +(33) ColumnarToRow [codegen id : 8] +Input [1]: [p_promo_sk#19] (34) BroadcastExchange Input [1]: [p_promo_sk#19] @@ -311,12 +311,12 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#45), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 11] +(41) CometFilter Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +Condition : ((isnotnull(cs_catalog_page_sk#39) AND isnotnull(cs_item_sk#40)) AND isnotnull(cs_promo_sk#41)) -(42) Filter [codegen id : 11] +(42) ColumnarToRow [codegen id : 11] Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Condition : ((isnotnull(cs_catalog_page_sk#39) AND isnotnull(cs_item_sk#40)) AND isnotnull(cs_promo_sk#41)) (43) Exchange Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] @@ -333,16 +333,16 @@ Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(46) ColumnarToRow [codegen id : 13] -Input [5]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50, cr_returned_date_sk#51] - -(47) Filter [codegen id : 13] +(46) CometFilter Input [5]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50, cr_returned_date_sk#51] Condition : (isnotnull(cr_item_sk#47) AND isnotnull(cr_order_number#48)) -(48) Project [codegen id : 13] -Output [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] +(47) CometProject Input [5]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50, cr_returned_date_sk#51] +Arguments: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50], [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] + +(48) ColumnarToRow [codegen id : 13] +Input [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] (49) Exchange Input [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] @@ -382,12 +382,12 @@ Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 16] +(57) CometFilter Input [2]: [cp_catalog_page_sk#53, cp_catalog_page_id#54] +Condition : isnotnull(cp_catalog_page_sk#53) -(58) Filter [codegen id : 16] +(58) ColumnarToRow [codegen id : 16] Input [2]: [cp_catalog_page_sk#53, cp_catalog_page_id#54] -Condition : isnotnull(cp_catalog_page_sk#53) (59) BroadcastExchange Input [2]: [cp_catalog_page_sk#53, cp_catalog_page_id#54] @@ -455,12 +455,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#81), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] ReadSchema: struct -(72) ColumnarToRow [codegen id : 21] +(72) CometFilter Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] +Condition : ((isnotnull(ws_web_site_sk#76) AND isnotnull(ws_item_sk#75)) AND isnotnull(ws_promo_sk#77)) -(73) Filter [codegen id : 21] +(73) ColumnarToRow [codegen id : 21] Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Condition : ((isnotnull(ws_web_site_sk#76) AND isnotnull(ws_item_sk#75)) AND isnotnull(ws_promo_sk#77)) (74) Exchange Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] @@ -477,16 +477,16 @@ Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] ReadSchema: struct -(77) ColumnarToRow [codegen id : 23] -Input [5]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86, wr_returned_date_sk#87] - -(78) Filter [codegen id : 23] +(77) CometFilter Input [5]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86, wr_returned_date_sk#87] Condition : (isnotnull(wr_item_sk#83) AND isnotnull(wr_order_number#84)) -(79) Project [codegen id : 23] -Output [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] +(78) CometProject Input [5]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86, wr_returned_date_sk#87] +Arguments: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86], [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] + +(79) ColumnarToRow [codegen id : 23] +Input [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] (80) Exchange Input [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] @@ -526,12 +526,12 @@ Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(88) ColumnarToRow [codegen id : 26] +(88) CometFilter Input [2]: [web_site_sk#89, web_site_id#90] +Condition : isnotnull(web_site_sk#89) -(89) Filter [codegen id : 26] +(89) ColumnarToRow [codegen id : 26] Input [2]: [web_site_sk#89, web_site_id#90] -Condition : isnotnull(web_site_sk#89) (90) BroadcastExchange Input [2]: [web_site_sk#89, web_site_id#90] @@ -695,9 +695,9 @@ Arguments: 100, [channel#34 ASC NULLS FIRST, id#35 ASC NULLS FIRST], [channel#34 Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 BroadcastExchange (125) -+- * Project (124) - +- * Filter (123) - +- * ColumnarToRow (122) ++- * ColumnarToRow (124) + +- CometProject (123) + +- CometFilter (122) +- CometScan parquet spark_catalog.default.date_dim (121) @@ -708,16 +708,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] ReadSchema: struct -(122) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#14, d_date#196] - -(123) Filter [codegen id : 1] +(122) CometFilter Input [2]: [d_date_sk#14, d_date#196] Condition : (((isnotnull(d_date#196) AND (d_date#196 >= 1998-08-04)) AND (d_date#196 <= 1998-09-03)) AND isnotnull(d_date_sk#14)) -(124) Project [codegen id : 1] -Output [1]: [d_date_sk#14] +(123) CometProject Input [2]: [d_date_sk#14, d_date#196] +Arguments: [d_date_sk#14], [d_date_sk#14] + +(124) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#14] (125) BroadcastExchange Input [1]: [d_date_sk#14] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/simplified.txt index 047ac07a6..34e47dcba 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/simplified.txt @@ -37,17 +37,17 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter Exchange [ss_item_sk,ss_ticket_number] #4 WholeStageCodegen (1) - Filter [ss_store_sk,ss_item_sk,ss_promo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_store_sk,ss_item_sk,ss_promo_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #5 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter WholeStageCodegen (4) @@ -55,35 +55,35 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter Exchange [sr_item_sk,sr_ticket_number] #6 WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - Filter [sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometFilter [sr_item_sk,sr_ticket_number] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] InputAdapter ReusedExchange [d_date_sk] #5 InputAdapter BroadcastExchange #7 WholeStageCodegen (6) - Filter [s_store_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [s_store_sk] CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] InputAdapter BroadcastExchange #8 WholeStageCodegen (7) - Project [i_item_sk] - Filter [i_current_price,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [i_item_sk] + CometFilter [i_current_price,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price] InputAdapter BroadcastExchange #9 WholeStageCodegen (8) - Project [p_promo_sk] - Filter [p_channel_tv,p_promo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [p_promo_sk] + CometFilter [p_channel_tv,p_promo_sk] CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] WholeStageCodegen (20) HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] @@ -107,9 +107,9 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter Exchange [cs_item_sk,cs_order_number] #11 WholeStageCodegen (11) - Filter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -118,19 +118,19 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter Exchange [cr_item_sk,cr_order_number] #12 WholeStageCodegen (13) - Project [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometFilter [cr_item_sk,cr_order_number] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] InputAdapter ReusedExchange [d_date_sk] #5 InputAdapter BroadcastExchange #13 WholeStageCodegen (16) - Filter [cp_catalog_page_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [cp_catalog_page_sk] CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] InputAdapter ReusedExchange [i_item_sk] #8 @@ -158,9 +158,9 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter Exchange [ws_item_sk,ws_order_number] #15 WholeStageCodegen (21) - Filter [ws_web_site_sk,ws_item_sk,ws_promo_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_web_site_sk,ws_item_sk,ws_promo_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 InputAdapter @@ -169,19 +169,19 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter Exchange [wr_item_sk,wr_order_number] #16 WholeStageCodegen (23) - Project [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - Filter [wr_item_sk,wr_order_number] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometFilter [wr_item_sk,wr_order_number] CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] InputAdapter ReusedExchange [d_date_sk] #5 InputAdapter BroadcastExchange #17 WholeStageCodegen (26) - Filter [web_site_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [web_site_sk] CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] InputAdapter ReusedExchange [i_item_sk] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/explain.txt index 7aac30e5f..58e7f5825 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/explain.txt @@ -15,13 +15,13 @@ TakeOrderedAndProject (34) : +- * BroadcastHashJoin Inner BuildRight (11) : :- * Project (6) : : +- * BroadcastHashJoin Inner BuildRight (5) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) + : : :- * ColumnarToRow (3) + : : : +- CometFilter (2) : : : +- CometScan parquet spark_catalog.default.web_sales (1) : : +- ReusedExchange (4) : +- BroadcastExchange (10) - : +- * Filter (9) - : +- * ColumnarToRow (8) + : +- * ColumnarToRow (9) + : +- CometFilter (8) : +- CometScan parquet spark_catalog.default.item (7) :- * HashAggregate (20) : +- Exchange (19) @@ -43,12 +43,12 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sol PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] +Condition : isnotnull(ws_item_sk#1) -(3) Filter [codegen id : 3] +(3) ColumnarToRow [codegen id : 3] Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Condition : isnotnull(ws_item_sk#1) (4) ReusedExchange [Reuses operator id: 39] Output [1]: [d_date_sk#5] @@ -70,12 +70,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 2] +(8) CometFilter Input [3]: [i_item_sk#6, i_class#7, i_category#8] +Condition : isnotnull(i_item_sk#6) -(9) Filter [codegen id : 2] +(9) ColumnarToRow [codegen id : 2] Input [3]: [i_item_sk#6, i_class#7, i_category#8] -Condition : isnotnull(i_item_sk#6) (10) BroadcastExchange Input [3]: [i_item_sk#6, i_class#7, i_category#8] @@ -209,9 +209,9 @@ Arguments: 100, [lochierarchy#15 DESC NULLS LAST, CASE WHEN (lochierarchy#15 = 0 Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 BroadcastExchange (39) -+- * Project (38) - +- * Filter (37) - +- * ColumnarToRow (36) ++- * ColumnarToRow (38) + +- CometProject (37) + +- CometFilter (36) +- CometScan parquet spark_catalog.default.date_dim (35) @@ -222,16 +222,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(36) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_month_seq#49] - -(37) Filter [codegen id : 1] +(36) CometFilter Input [2]: [d_date_sk#5, d_month_seq#49] Condition : (((isnotnull(d_month_seq#49) AND (d_month_seq#49 >= 1212)) AND (d_month_seq#49 <= 1223)) AND isnotnull(d_date_sk#5)) -(38) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(37) CometProject Input [2]: [d_date_sk#5, d_month_seq#49] +Arguments: [d_date_sk#5], [d_date_sk#5] + +(38) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#5] (39) BroadcastExchange Input [1]: [d_date_sk#5] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/simplified.txt index 5146ee0d2..5c5e08885 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/simplified.txt @@ -25,26 +25,26 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl BroadcastHashJoin [ws_item_sk,i_item_sk] Project [ws_item_sk,ws_net_paid] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ws_item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_month_seq,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter ReusedExchange [d_date_sk] #4 InputAdapter BroadcastExchange #5 WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] WholeStageCodegen (9) HashAggregate [i_category,sum,isEmpty] [sum(total_sum),total_sum,i_class,g_category,g_class,lochierarchy,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/explain.txt index a2f3cffb6..f77b7ec93 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/explain.txt @@ -12,12 +12,12 @@ +- * BroadcastHashJoin Inner BuildRight (11) :- * Project (9) : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) + : :- * ColumnarToRow (3) + : : +- CometFilter (2) : : +- CometScan parquet spark_catalog.default.store_sales (1) : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) + : +- * ColumnarToRow (6) + : +- CometFilter (5) : +- CometScan parquet spark_catalog.default.item (4) +- ReusedExchange (10) @@ -30,12 +30,12 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sol PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Condition : isnotnull(ss_item_sk#1) -(3) Filter [codegen id : 3] +(3) ColumnarToRow [codegen id : 3] Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_item_sk#1) (4) Scan parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] @@ -44,12 +44,12 @@ Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(5) CometFilter Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Condition : (i_category#10 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) -(6) Filter [codegen id : 1] +(6) ColumnarToRow [codegen id : 1] Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (i_category#10 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) (7) BroadcastExchange Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] @@ -124,9 +124,9 @@ Arguments: [i_category#10 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_item_id# Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 BroadcastExchange (26) -+- * Project (25) - +- * Filter (24) - +- * ColumnarToRow (23) ++- * ColumnarToRow (25) + +- CometProject (24) + +- CometFilter (23) +- CometScan parquet spark_catalog.default.date_dim (22) @@ -137,16 +137,16 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#11, d_date#19] - -(24) Filter [codegen id : 1] +(23) CometFilter Input [2]: [d_date_sk#11, d_date#19] Condition : (((isnotnull(d_date#19) AND (d_date#19 >= 1999-02-22)) AND (d_date#19 <= 1999-03-24)) AND isnotnull(d_date_sk#11)) -(25) Project [codegen id : 1] -Output [1]: [d_date_sk#11] +(24) CometProject Input [2]: [d_date_sk#11, d_date#19] +Arguments: [d_date_sk#11], [d_date_sk#11] + +(25) ColumnarToRow [codegen id : 1] +Input [1]: [d_date_sk#11] (26) BroadcastExchange Input [1]: [d_date_sk#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/simplified.txt index 5aa7c17f1..b7489a0af 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/simplified.txt @@ -20,24 +20,24 @@ WholeStageCodegen (7) BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [ss_item_sk] CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter + ColumnarToRow + InputAdapter + CometFilter [i_category,i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] InputAdapter ReusedExchange [d_date_sk] #4 diff --git a/spark/src/test/scala/org/apache/comet/CometCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometCastSuite.scala index 4d73596a7..fd2218965 100644 --- a/spark/src/test/scala/org/apache/comet/CometCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometCastSuite.scala @@ -990,12 +990,9 @@ class CometCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { if (CometSparkSessionExtensions.isSpark40Plus) { // for Spark 4 we expect to sparkException carries the message assert( - cometMessage == sparkException.getMessage - .split("\n") - .head + sparkException.getMessage .replace(".WITH_SUGGESTION] ", "]") - .replace(" SQLSTATE: 22003", "") - .replace(" SQLSTATE: 22018", "")) + .startsWith(cometMessage)) } else if (CometSparkSessionExtensions.isSpark34Plus) { // for Spark 3.4 we expect to reproduce the error message exactly assert(cometMessage == sparkMessage) diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala index 22d016c66..691d2cd63 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala @@ -298,7 +298,6 @@ trait CometPlanStabilitySuite extends DisableAdaptiveExecutionSuite with TPCDSBa } class CometTPCDSV1_4_PlanStabilitySuite extends CometPlanStabilitySuite { - // TODO: remove the Spark 4.0 golden file once ANSI is fully supported private val planName = if (isSpark40Plus) "approved-plans-v1_4-spark4_0" else "approved-plans-v1_4" override val goldenFilePath: String = @@ -312,7 +311,6 @@ class CometTPCDSV1_4_PlanStabilitySuite extends CometPlanStabilitySuite { } class CometTPCDSV2_7_PlanStabilitySuite extends CometPlanStabilitySuite { - // TODO: remove the Spark 4.0 golden file once ANSI is fully supported private val planName = if (isSpark40Plus) "approved-plans-v2_7-spark4_0" else "approved-plans-v2_7" override val goldenFilePath: String =