diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/explain.txt index 762f3a4f6..8e581548b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/explain.txt @@ -31,9 +31,9 @@ TakeOrderedAndProject (40) : : : +- CometScan parquet spark_catalog.default.store_returns (11) : : +- ReusedExchange (14) : +- BroadcastExchange (31) - : +- * ColumnarToRow (30) - : +- CometProject (29) - : +- CometFilter (28) + : +- * Project (30) + : +- * Filter (29) + : +- * ColumnarToRow (28) : +- CometScan parquet spark_catalog.default.store (27) +- BroadcastExchange (37) +- * ColumnarToRow (36) @@ -177,19 +177,19 @@ Input [5]: [ctr_customer_sk#10, ctr_store_sk#11, ctr_total_return#12, (avg(ctr_t Output [2]: [s_store_sk#29, s_state#30] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] ReadSchema: struct -(28) CometFilter +(28) ColumnarToRow [codegen id : 7] Input [2]: [s_store_sk#29, s_state#30] -Condition : ((isnotnull(s_state#30) AND (s_state#30 = TN)) AND isnotnull(s_store_sk#29)) -(29) CometProject +(29) Filter [codegen id : 7] Input [2]: [s_store_sk#29, s_state#30] -Arguments: [s_store_sk#29], [s_store_sk#29] +Condition : ((isnotnull(s_state#30) AND (rpad(s_state#30, 2, ) = TN)) AND isnotnull(s_store_sk#29)) -(30) ColumnarToRow [codegen id : 7] -Input [1]: [s_store_sk#29] +(30) Project [codegen id : 7] +Output [1]: [s_store_sk#29] +Input [2]: [s_store_sk#29, s_state#30] (31) BroadcastExchange Input [1]: [s_store_sk#29] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/simplified.txt index 688fb69a8..feb667dcc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/simplified.txt @@ -54,10 +54,10 @@ TakeOrderedAndProject [c_customer_id] InputAdapter BroadcastExchange #6 WholeStageCodegen (7) - ColumnarToRow - InputAdapter - CometProject [s_store_sk] - CometFilter [s_state,s_store_sk] + Project [s_store_sk] + Filter [s_state,s_store_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.store [s_store_sk,s_state] InputAdapter BroadcastExchange #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/explain.txt index 6cf7f4b08..52f2c660a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/explain.txt @@ -15,8 +15,8 @@ TakeOrderedAndProject (20) : : +- CometFilter (2) : : +- CometScan parquet spark_catalog.default.web_sales (1) : +- BroadcastExchange (7) - : +- * ColumnarToRow (6) - : +- CometFilter (5) + : +- * Filter (6) + : +- * ColumnarToRow (5) : +- CometScan parquet spark_catalog.default.item (4) +- ReusedExchange (10) @@ -40,15 +40,15 @@ Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) CometFilter +(5) ColumnarToRow [codegen id : 1] Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (i_category#10 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) -(6) ColumnarToRow [codegen id : 1] +(6) Filter [codegen id : 1] Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Condition : (rpad(i_category#10, 50, ) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) (7) BroadcastExchange Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/simplified.txt index fae1c6dba..b9753f91b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/simplified.txt @@ -32,9 +32,9 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter BroadcastExchange #4 WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [i_category,i_item_sk] + Filter [i_category,i_item_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] InputAdapter ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/explain.txt index a647b1f05..26a45e60b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/explain.txt @@ -20,14 +20,14 @@ : : : : +- CometFilter (5) : : : : +- CometScan parquet spark_catalog.default.store (4) : : : +- BroadcastExchange (14) - : : : +- * ColumnarToRow (13) - : : : +- CometProject (12) - : : : +- CometFilter (11) + : : : +- * Project (13) + : : : +- * Filter (12) + : : : +- * ColumnarToRow (11) : : : +- CometScan parquet spark_catalog.default.customer_address (10) : : +- ReusedExchange (17) : +- BroadcastExchange (23) - : +- * ColumnarToRow (22) - : +- CometFilter (21) + : +- * Filter (22) + : +- * ColumnarToRow (21) : +- CometScan parquet spark_catalog.default.customer_demographics (20) +- BroadcastExchange (29) +- * ColumnarToRow (28) @@ -82,19 +82,19 @@ Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quant Output [3]: [ca_address_sk#13, ca_state#14, ca_country#15] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [OH,TX]),In(ca_state, [KY,NM,OR])),In(ca_state, [MS,TX,VA]))] +PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] ReadSchema: struct -(11) CometFilter +(11) ColumnarToRow [codegen id : 2] Input [3]: [ca_address_sk#13, ca_state#14, ca_country#15] -Condition : (((isnotnull(ca_country#15) AND (ca_country#15 = United States)) AND isnotnull(ca_address_sk#13)) AND ((ca_state#14 IN (TX,OH) OR ca_state#14 IN (OR,NM,KY)) OR ca_state#14 IN (VA,TX,MS))) -(12) CometProject +(12) Filter [codegen id : 2] Input [3]: [ca_address_sk#13, ca_state#14, ca_country#15] -Arguments: [ca_address_sk#13, ca_state#14], [ca_address_sk#13, ca_state#14] +Condition : (((isnotnull(ca_country#15) AND (ca_country#15 = United States)) AND isnotnull(ca_address_sk#13)) AND ((rpad(ca_state#14, 2, ) IN (TX,OH) OR rpad(ca_state#14, 2, ) IN (OR,NM,KY)) OR rpad(ca_state#14, 2, ) IN (VA,TX,MS))) -(13) ColumnarToRow [codegen id : 2] -Input [2]: [ca_address_sk#13, ca_state#14] +(13) Project [codegen id : 2] +Output [2]: [ca_address_sk#13, ca_state#14] +Input [3]: [ca_address_sk#13, ca_state#14, ca_country#15] (14) BroadcastExchange Input [2]: [ca_address_sk#13, ca_state#14] @@ -104,7 +104,7 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Left keys [1]: [ss_addr_sk#3] Right keys [1]: [ca_address_sk#13] Join type: Inner -Join condition: ((((ca_state#14 IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((ca_state#14 IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((ca_state#14 IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00))) +Join condition: ((((rpad(ca_state#14, 2, ) IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((rpad(ca_state#14, 2, ) IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((rpad(ca_state#14, 2, ) IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00))) (16) Project [codegen id : 6] Output [7]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] @@ -127,15 +127,15 @@ Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ex Output [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Advanced Degree )),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College ))),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,2 yr Degree )))] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(21) CometFilter +(21) ColumnarToRow [codegen id : 4] Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Condition : (isnotnull(cd_demo_sk#17) AND ((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) OR ((cd_marital_status#18 = S) AND (cd_education_status#19 = College ))) OR ((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )))) -(22) ColumnarToRow [codegen id : 4] +(22) Filter [codegen id : 4] Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] +Condition : (isnotnull(cd_demo_sk#17) AND ((((rpad(cd_marital_status#18, 1, ) = M) AND (rpad(cd_education_status#19, 20, ) = Advanced Degree )) OR ((rpad(cd_marital_status#18, 1, ) = S) AND (rpad(cd_education_status#19, 20, ) = College ))) OR ((rpad(cd_marital_status#18, 1, ) = W) AND (rpad(cd_education_status#19, 20, ) = 2 yr Degree )))) (23) BroadcastExchange Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] @@ -145,7 +145,7 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint) Left keys [1]: [ss_cdemo_sk#1] Right keys [1]: [cd_demo_sk#17] Join type: Inner -Join condition: ((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#18 = S) AND (cd_education_status#19 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))) +Join condition: ((((((rpad(cd_marital_status#18, 1, ) = M) AND (rpad(cd_education_status#19, 20, ) = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((rpad(cd_marital_status#18, 1, ) = S) AND (rpad(cd_education_status#19, 20, ) = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((rpad(cd_marital_status#18, 1, ) = W) AND (rpad(cd_education_status#19, 20, ) = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))) (25) Project [codegen id : 6] Output [7]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19] @@ -173,7 +173,7 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint) Left keys [1]: [ss_hdemo_sk#2] Right keys [1]: [hd_demo_sk#20] Join type: Inner -Join condition: (((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#21 = 3)) OR (((((cd_marital_status#18 = S) AND (cd_education_status#19 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#21 = 1))) OR (((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#21 = 1))) +Join condition: (((((((rpad(cd_marital_status#18, 1, ) = M) AND (rpad(cd_education_status#19, 20, ) = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#21 = 3)) OR (((((rpad(cd_marital_status#18, 1, ) = S) AND (rpad(cd_education_status#19, 20, ) = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#21 = 1))) OR (((((rpad(cd_marital_status#18, 1, ) = W) AND (rpad(cd_education_status#19, 20, ) = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#21 = 1))) (31) Project [codegen id : 6] Output [3]: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/simplified.txt index 5e5fc41f8..c72a3fc10 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/simplified.txt @@ -36,19 +36,19 @@ WholeStageCodegen (7) InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometProject [ca_address_sk,ca_state] - CometFilter [ca_country,ca_address_sk,ca_state] + Project [ca_address_sk,ca_state] + Filter [ca_country,ca_address_sk,ca_state] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] InputAdapter ReusedExchange [d_date_sk] #2 InputAdapter BroadcastExchange #5 WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + Filter [cd_demo_sk,cd_marital_status,cd_education_status] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] InputAdapter BroadcastExchange #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/explain.txt index 13a621f77..9fb6a4114 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/explain.txt @@ -88,7 +88,7 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint) Left keys [1]: [c_current_addr_sk#6] Right keys [1]: [ca_address_sk#7] Join type: Inner -Join condition: ((substr(ca_zip#9, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#8 IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00)) +Join condition: ((substr(ca_zip#9, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR rpad(ca_state#8, 2, ) IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00)) (15) Project [codegen id : 4] Output [3]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#9] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/explain.txt index ccec341ad..6ab6e3d95 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/explain.txt @@ -35,9 +35,9 @@ : : +- CometFilter (21) : : +- CometScan parquet spark_catalog.default.date_dim (20) : +- BroadcastExchange (31) - : +- * ColumnarToRow (30) - : +- CometProject (29) - : +- CometFilter (28) + : +- * Project (30) + : +- * Filter (29) + : +- * ColumnarToRow (28) : +- CometScan parquet spark_catalog.default.customer_address (27) +- BroadcastExchange (38) +- * ColumnarToRow (37) @@ -166,19 +166,19 @@ Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_ Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_state), IsNotNull(ca_address_sk)] ReadSchema: struct -(28) CometFilter +(28) ColumnarToRow [codegen id : 9] Input [2]: [ca_address_sk#16, ca_state#17] -Condition : ((isnotnull(ca_state#17) AND (ca_state#17 = GA)) AND isnotnull(ca_address_sk#16)) -(29) CometProject +(29) Filter [codegen id : 9] Input [2]: [ca_address_sk#16, ca_state#17] -Arguments: [ca_address_sk#16], [ca_address_sk#16] +Condition : ((isnotnull(ca_state#17) AND (rpad(ca_state#17, 2, ) = GA)) AND isnotnull(ca_address_sk#16)) -(30) ColumnarToRow [codegen id : 9] -Input [1]: [ca_address_sk#16] +(30) Project [codegen id : 9] +Output [1]: [ca_address_sk#16] +Input [2]: [ca_address_sk#16, ca_state#17] (31) BroadcastExchange Input [1]: [ca_address_sk#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/simplified.txt index a55c182be..246cdacc5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/simplified.txt @@ -59,10 +59,10 @@ WholeStageCodegen (12) InputAdapter BroadcastExchange #6 WholeStageCodegen (9) - ColumnarToRow - InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_state,ca_address_sk] + Project [ca_address_sk] + Filter [ca_state,ca_address_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/explain.txt index 0de98cfb0..1868b9f2d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/explain.txt @@ -235,9 +235,9 @@ Arguments: 100, [i_item_id#25 ASC NULLS FIRST, i_item_desc#26 ASC NULLS FIRST, s Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 BroadcastExchange (45) -+- * ColumnarToRow (44) - +- CometProject (43) - +- CometFilter (42) ++- * Project (44) + +- * Filter (43) + +- * ColumnarToRow (42) +- CometScan parquet spark_catalog.default.date_dim (41) @@ -245,19 +245,19 @@ BroadcastExchange (45) Output [2]: [d_date_sk#19, d_quarter_name#84] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_quarter_name), EqualTo(d_quarter_name,2001Q1), IsNotNull(d_date_sk)] +PushedFilters: [IsNotNull(d_quarter_name), IsNotNull(d_date_sk)] ReadSchema: struct -(42) CometFilter +(42) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#19, d_quarter_name#84] -Condition : ((isnotnull(d_quarter_name#84) AND (d_quarter_name#84 = 2001Q1)) AND isnotnull(d_date_sk#19)) -(43) CometProject +(43) Filter [codegen id : 1] Input [2]: [d_date_sk#19, d_quarter_name#84] -Arguments: [d_date_sk#19], [d_date_sk#19] +Condition : ((isnotnull(d_quarter_name#84) AND (rpad(d_quarter_name#84, 6, ) = 2001Q1)) AND isnotnull(d_date_sk#19)) -(44) ColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#19] +(44) Project [codegen id : 1] +Output [1]: [d_date_sk#19] +Input [2]: [d_date_sk#19, d_quarter_name#84] (45) BroadcastExchange Input [1]: [d_date_sk#19] @@ -265,9 +265,9 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Subquery:2 Hosting operator id = 4 Hosting Expression = sr_returned_date_sk#12 IN dynamicpruning#13 BroadcastExchange (50) -+- * ColumnarToRow (49) - +- CometProject (48) - +- CometFilter (47) ++- * Project (49) + +- * Filter (48) + +- * ColumnarToRow (47) +- CometScan parquet spark_catalog.default.date_dim (46) @@ -275,19 +275,19 @@ BroadcastExchange (50) Output [2]: [d_date_sk#20, d_quarter_name#85] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_quarter_name, [2001Q1,2001Q2,2001Q3]), IsNotNull(d_date_sk)] +PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(47) CometFilter +(47) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#20, d_quarter_name#85] -Condition : (d_quarter_name#85 IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#20)) -(48) CometProject +(48) Filter [codegen id : 1] Input [2]: [d_date_sk#20, d_quarter_name#85] -Arguments: [d_date_sk#20], [d_date_sk#20] +Condition : (rpad(d_quarter_name#85, 6, ) IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#20)) -(49) ColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#20] +(49) Project [codegen id : 1] +Output [1]: [d_date_sk#20] +Input [2]: [d_date_sk#20, d_quarter_name#85] (50) BroadcastExchange Input [1]: [d_date_sk#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/simplified.txt index 9f4d67dec..cb0fb51ba 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/simplified.txt @@ -26,10 +26,10 @@ TakeOrderedAndProject [i_item_id,i_item_desc,s_state,store_sales_quantitycount,s SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_quarter_name,d_date_sk] + Project [d_date_sk] + Filter [d_quarter_name,d_date_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] InputAdapter BroadcastExchange #3 @@ -41,10 +41,10 @@ TakeOrderedAndProject [i_item_id,i_item_desc,s_state,store_sales_quantitycount,s SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #4 WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_quarter_name,d_date_sk] + Project [d_date_sk] + Filter [d_quarter_name,d_date_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] InputAdapter BroadcastExchange #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/explain.txt index 1e9c660c5..6bd9b0e0a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/explain.txt @@ -20,9 +20,9 @@ TakeOrderedAndProject (43) : : : : : : +- CometFilter (2) : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : : : : : +- BroadcastExchange (8) - : : : : : +- * ColumnarToRow (7) - : : : : : +- CometProject (6) - : : : : : +- CometFilter (5) + : : : : : +- * Project (7) + : : : : : +- * Filter (6) + : : : : : +- * ColumnarToRow (5) : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (4) : : : : +- BroadcastExchange (15) : : : : +- * ColumnarToRow (14) @@ -34,8 +34,8 @@ TakeOrderedAndProject (43) : : : +- CometFilter (19) : : : +- CometScan parquet spark_catalog.default.customer_demographics (18) : : +- BroadcastExchange (27) - : : +- * ColumnarToRow (26) - : : +- CometFilter (25) + : : +- * Filter (26) + : : +- * ColumnarToRow (25) : : +- CometScan parquet spark_catalog.default.customer_address (24) : +- ReusedExchange (30) +- BroadcastExchange (36) @@ -63,19 +63,19 @@ Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity Output [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_education_status), EqualTo(cd_gender,F), EqualTo(cd_education_status,Unknown ), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) CometFilter +(5) ColumnarToRow [codegen id : 1] Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] -Condition : ((((isnotnull(cd_gender#12) AND isnotnull(cd_education_status#13)) AND (cd_gender#12 = F)) AND (cd_education_status#13 = Unknown )) AND isnotnull(cd_demo_sk#11)) -(6) CometProject +(6) Filter [codegen id : 1] Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] -Arguments: [cd_demo_sk#11, cd_dep_count#14], [cd_demo_sk#11, cd_dep_count#14] +Condition : ((((isnotnull(cd_gender#12) AND isnotnull(cd_education_status#13)) AND (rpad(cd_gender#12, 1, ) = F)) AND (rpad(cd_education_status#13, 20, ) = Unknown )) AND isnotnull(cd_demo_sk#11)) -(7) ColumnarToRow [codegen id : 1] -Input [2]: [cd_demo_sk#11, cd_dep_count#14] +(7) Project [codegen id : 1] +Output [2]: [cd_demo_sk#11, cd_dep_count#14] +Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] (8) BroadcastExchange Input [2]: [cd_demo_sk#11, cd_dep_count#14] @@ -155,15 +155,15 @@ Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_ Output [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [IN,MS,ND,NM,OK,VA]), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(25) CometFilter +(25) ColumnarToRow [codegen id : 4] Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] -Condition : (ca_state#23 IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#21)) -(26) ColumnarToRow [codegen id : 4] +(26) Filter [codegen id : 4] Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] +Condition : (rpad(ca_state#23, 2, ) IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#21)) (27) BroadcastExchange Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/simplified.txt index 47911b9ba..a8c4f9a5b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/simplified.txt @@ -33,10 +33,10 @@ TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,ag InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [cd_demo_sk,cd_dep_count] - CometFilter [cd_gender,cd_education_status,cd_demo_sk] + Project [cd_demo_sk,cd_dep_count] + Filter [cd_gender,cd_education_status,cd_demo_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] InputAdapter BroadcastExchange #4 @@ -56,9 +56,9 @@ TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,ag InputAdapter BroadcastExchange #6 WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometFilter [ca_state,ca_address_sk] + Filter [ca_state,ca_address_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] InputAdapter ReusedExchange [d_date_sk] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/explain.txt index 1215adf3f..77e4e88db 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/explain.txt @@ -97,7 +97,7 @@ Input [5]: [sold_date_sk#3, sales_price#4, d_date_sk#9, d_week_seq#10, d_day_nam (13) HashAggregate [codegen id : 2] Input [3]: [sales_price#4, d_week_seq#10, d_day_name#11] Keys [1]: [d_week_seq#10] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] +Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Sunday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Monday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Tuesday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Wednesday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Thursday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Friday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Saturday ) THEN sales_price#4 END))] Aggregate Attributes [7]: [sum#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18] Results [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] @@ -108,9 +108,9 @@ Arguments: hashpartitioning(d_week_seq#10, 5), ENSURE_REQUIREMENTS, [plan_id=2] (15) HashAggregate [codegen id : 8] Input [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] Keys [1]: [d_week_seq#10] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))#32] -Results [8]: [d_week_seq#10, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END))#26,17,2) AS sun_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END))#27,17,2) AS mon_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END))#28,17,2) AS tue_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#29,17,2) AS wed_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END))#30,17,2) AS thu_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END))#31,17,2) AS fri_sales#38, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))#32,17,2) AS sat_sales#39] +Functions [7]: [sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Saturday ) THEN sales_price#4 END))] +Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Sunday ) THEN sales_price#4 END))#26, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Monday ) THEN sales_price#4 END))#27, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Tuesday ) THEN sales_price#4 END))#28, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Wednesday) THEN sales_price#4 END))#29, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Thursday ) THEN sales_price#4 END))#30, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Friday ) THEN sales_price#4 END))#31, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Saturday ) THEN sales_price#4 END))#32] +Results [8]: [d_week_seq#10, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Sunday ) THEN sales_price#4 END))#26,17,2) AS sun_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Monday ) THEN sales_price#4 END))#27,17,2) AS mon_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Tuesday ) THEN sales_price#4 END))#28,17,2) AS tue_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Wednesday) THEN sales_price#4 END))#29,17,2) AS wed_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Thursday ) THEN sales_price#4 END))#30,17,2) AS thu_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Friday ) THEN sales_price#4 END))#31,17,2) AS fri_sales#38, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#11, 9, ) = Saturday ) THEN sales_price#4 END))#32,17,2) AS sat_sales#39] (16) Scan parquet spark_catalog.default.date_dim Output [2]: [d_week_seq#40, d_year#41] @@ -150,9 +150,9 @@ Output [8]: [d_week_seq#50, sum#51, sum#52, sum#53, sum#54, sum#55, sum#56, sum# (24) HashAggregate [codegen id : 7] Input [8]: [d_week_seq#50, sum#51, sum#52, sum#53, sum#54, sum#55, sum#56, sum#57] Keys [1]: [d_week_seq#50] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#58 = Sunday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Monday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Tuesday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Wednesday) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Thursday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Friday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Saturday ) THEN sales_price#59 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#58 = Sunday ) THEN sales_price#59 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Monday ) THEN sales_price#59 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Tuesday ) THEN sales_price#59 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Wednesday) THEN sales_price#59 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Thursday ) THEN sales_price#59 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Friday ) THEN sales_price#59 END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Saturday ) THEN sales_price#59 END))#32] -Results [8]: [d_week_seq#50, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Sunday ) THEN sales_price#59 END))#26,17,2) AS sun_sales#60, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Monday ) THEN sales_price#59 END))#27,17,2) AS mon_sales#61, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Tuesday ) THEN sales_price#59 END))#28,17,2) AS tue_sales#62, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Wednesday) THEN sales_price#59 END))#29,17,2) AS wed_sales#63, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Thursday ) THEN sales_price#59 END))#30,17,2) AS thu_sales#64, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Friday ) THEN sales_price#59 END))#31,17,2) AS fri_sales#65, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Saturday ) THEN sales_price#59 END))#32,17,2) AS sat_sales#66] +Functions [7]: [sum(UnscaledValue(CASE WHEN (rpad(d_day_name#58, 9, ) = Sunday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#58, 9, ) = Monday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#58, 9, ) = Tuesday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#58, 9, ) = Wednesday) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#58, 9, ) = Thursday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#58, 9, ) = Friday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#58, 9, ) = Saturday ) THEN sales_price#59 END))] +Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (rpad(d_day_name#58, 9, ) = Sunday ) THEN sales_price#59 END))#26, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#58, 9, ) = Monday ) THEN sales_price#59 END))#27, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#58, 9, ) = Tuesday ) THEN sales_price#59 END))#28, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#58, 9, ) = Wednesday) THEN sales_price#59 END))#29, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#58, 9, ) = Thursday ) THEN sales_price#59 END))#30, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#58, 9, ) = Friday ) THEN sales_price#59 END))#31, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#58, 9, ) = Saturday ) THEN sales_price#59 END))#32] +Results [8]: [d_week_seq#50, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#58, 9, ) = Sunday ) THEN sales_price#59 END))#26,17,2) AS sun_sales#60, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#58, 9, ) = Monday ) THEN sales_price#59 END))#27,17,2) AS mon_sales#61, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#58, 9, ) = Tuesday ) THEN sales_price#59 END))#28,17,2) AS tue_sales#62, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#58, 9, ) = Wednesday) THEN sales_price#59 END))#29,17,2) AS wed_sales#63, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#58, 9, ) = Thursday ) THEN sales_price#59 END))#30,17,2) AS thu_sales#64, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#58, 9, ) = Friday ) THEN sales_price#59 END))#31,17,2) AS fri_sales#65, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#58, 9, ) = Saturday ) THEN sales_price#59 END))#32,17,2) AS sat_sales#66] (25) Scan parquet spark_catalog.default.date_dim Output [2]: [d_week_seq#67, d_year#68] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/simplified.txt index 8856ce80d..8b4c7bb91 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/simplified.txt @@ -7,7 +7,7 @@ WholeStageCodegen (9) BroadcastHashJoin [d_week_seq1,d_week_seq2] Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] BroadcastHashJoin [d_week_seq,d_week_seq] - HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] + HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Saturday ) THEN sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] InputAdapter Exchange [d_week_seq] #2 WholeStageCodegen (2) @@ -41,7 +41,7 @@ WholeStageCodegen (9) WholeStageCodegen (7) Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] BroadcastHashJoin [d_week_seq,d_week_seq] - HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] + HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Saturday ) THEN sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] InputAdapter ReusedExchange [d_week_seq,sum,sum,sum,sum,sum,sum,sum] #2 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/explain.txt index 333ef218c..3be96601c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/explain.txt @@ -15,8 +15,8 @@ TakeOrderedAndProject (20) : : +- CometFilter (2) : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : +- BroadcastExchange (7) - : +- * ColumnarToRow (6) - : +- CometFilter (5) + : +- * Filter (6) + : +- * ColumnarToRow (5) : +- CometScan parquet spark_catalog.default.item (4) +- ReusedExchange (10) @@ -40,15 +40,15 @@ Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) CometFilter +(5) ColumnarToRow [codegen id : 1] Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (i_category#10 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) -(6) ColumnarToRow [codegen id : 1] +(6) Filter [codegen id : 1] Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Condition : (rpad(i_category#10, 50, ) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) (7) BroadcastExchange Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/simplified.txt index 52c42bdf2..415b80754 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/simplified.txt @@ -32,9 +32,9 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter BroadcastExchange #4 WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [i_category,i_item_sk] + Filter [i_category,i_item_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] InputAdapter ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/explain.txt index 7241b5ea0..3dc458b13 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/explain.txt @@ -34,8 +34,8 @@ : : : +- CometFilter (16) : : : +- CometScan parquet spark_catalog.default.store (15) : : +- BroadcastExchange (25) - : : +- * ColumnarToRow (24) - : : +- CometFilter (23) + : : +- * Filter (24) + : : +- * ColumnarToRow (23) : : +- CometScan parquet spark_catalog.default.item (22) : +- BroadcastExchange (31) : +- * ColumnarToRow (30) @@ -145,15 +145,15 @@ Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_stor Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_color), EqualTo(i_color,pale ), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_color), IsNotNull(i_item_sk)] ReadSchema: struct -(23) CometFilter +(23) ColumnarToRow [codegen id : 6] Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = pale )) AND isnotnull(i_item_sk#15)) -(24) ColumnarToRow [codegen id : 6] +(24) Filter [codegen id : 6] Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +Condition : ((isnotnull(i_color#18) AND (rpad(i_color#18, 20, ) = pale )) AND isnotnull(i_item_sk#15)) (25) BroadcastExchange Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/simplified.txt index 8ebd45fd1..cec6bb641 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/simplified.txt @@ -98,9 +98,9 @@ WholeStageCodegen (11) InputAdapter BroadcastExchange #6 WholeStageCodegen (6) - ColumnarToRow - InputAdapter - CometFilter [i_color,i_item_sk] + Filter [i_color,i_item_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] InputAdapter BroadcastExchange #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/explain.txt index 0ac5639b7..6a51b1f25 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/explain.txt @@ -34,8 +34,8 @@ : : : +- CometFilter (16) : : : +- CometScan parquet spark_catalog.default.store (15) : : +- BroadcastExchange (25) - : : +- * ColumnarToRow (24) - : : +- CometFilter (23) + : : +- * Filter (24) + : : +- * ColumnarToRow (23) : : +- CometScan parquet spark_catalog.default.item (22) : +- BroadcastExchange (31) : +- * ColumnarToRow (30) @@ -145,15 +145,15 @@ Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_stor Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_color), EqualTo(i_color,chiffon ), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_color), IsNotNull(i_item_sk)] ReadSchema: struct -(23) CometFilter +(23) ColumnarToRow [codegen id : 6] Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = chiffon )) AND isnotnull(i_item_sk#15)) -(24) ColumnarToRow [codegen id : 6] +(24) Filter [codegen id : 6] Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +Condition : ((isnotnull(i_color#18) AND (rpad(i_color#18, 20, ) = chiffon )) AND isnotnull(i_item_sk#15)) (25) BroadcastExchange Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/simplified.txt index 8ebd45fd1..cec6bb641 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/simplified.txt @@ -98,9 +98,9 @@ WholeStageCodegen (11) InputAdapter BroadcastExchange #6 WholeStageCodegen (6) - ColumnarToRow - InputAdapter - CometFilter [i_color,i_item_sk] + Filter [i_color,i_item_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] InputAdapter BroadcastExchange #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/explain.txt index c90dcd024..25439ca45 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/explain.txt @@ -15,9 +15,9 @@ TakeOrderedAndProject (30) : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : : : +- BroadcastExchange (8) - : : : +- * ColumnarToRow (7) - : : : +- CometProject (6) - : : : +- CometFilter (5) + : : : +- * Project (7) + : : : +- * Filter (6) + : : : +- * ColumnarToRow (5) : : : +- CometScan parquet spark_catalog.default.customer_demographics (4) : : +- ReusedExchange (11) : +- BroadcastExchange (17) @@ -25,9 +25,9 @@ TakeOrderedAndProject (30) : +- CometFilter (15) : +- CometScan parquet spark_catalog.default.item (14) +- BroadcastExchange (24) - +- * ColumnarToRow (23) - +- CometProject (22) - +- CometFilter (21) + +- * Project (23) + +- * Filter (22) + +- * ColumnarToRow (21) +- CometScan parquet spark_catalog.default.promotion (20) @@ -50,19 +50,19 @@ Input [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_l Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_marital_status,S), EqualTo(cd_education_status,College ), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) CometFilter +(5) ColumnarToRow [codegen id : 1] Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (cd_gender#11 = M)) AND (cd_marital_status#12 = S)) AND (cd_education_status#13 = College )) AND isnotnull(cd_demo_sk#10)) -(6) CometProject +(6) Filter [codegen id : 1] Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Arguments: [cd_demo_sk#10], [cd_demo_sk#10] +Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (rpad(cd_gender#11, 1, ) = M)) AND (rpad(cd_marital_status#12, 1, ) = S)) AND (rpad(cd_education_status#13, 20, ) = College )) AND isnotnull(cd_demo_sk#10)) -(7) ColumnarToRow [codegen id : 1] -Input [1]: [cd_demo_sk#10] +(7) Project [codegen id : 1] +Output [1]: [cd_demo_sk#10] +Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] (8) BroadcastExchange Input [1]: [cd_demo_sk#10] @@ -123,19 +123,19 @@ Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sale Output [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [Or(EqualTo(p_channel_email,N),EqualTo(p_channel_event,N)), IsNotNull(p_promo_sk)] +PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(21) CometFilter +(21) ColumnarToRow [codegen id : 4] Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Condition : (((p_channel_email#18 = N) OR (p_channel_event#19 = N)) AND isnotnull(p_promo_sk#17)) -(22) CometProject +(22) Filter [codegen id : 4] Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Arguments: [p_promo_sk#17], [p_promo_sk#17] +Condition : (((rpad(p_channel_email#18, 1, ) = N) OR (rpad(p_channel_event#19, 1, ) = N)) AND isnotnull(p_promo_sk#17)) -(23) ColumnarToRow [codegen id : 4] -Input [1]: [p_promo_sk#17] +(23) Project [codegen id : 4] +Output [1]: [p_promo_sk#17] +Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] (24) BroadcastExchange Input [1]: [p_promo_sk#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/simplified.txt index 7d3893624..db0687495 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/simplified.txt @@ -28,10 +28,10 @@ TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [cd_demo_sk] - CometFilter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] + Project [cd_demo_sk] + Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] InputAdapter ReusedExchange [d_date_sk] #2 @@ -45,8 +45,8 @@ TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] InputAdapter BroadcastExchange #5 WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometProject [p_promo_sk] - CometFilter [p_channel_email,p_channel_event,p_promo_sk] + Project [p_promo_sk] + Filter [p_channel_email,p_channel_event,p_promo_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/explain.txt index e41077ed3..2794e92c0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/explain.txt @@ -16,14 +16,14 @@ TakeOrderedAndProject (30) : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- BroadcastExchange (8) - : : : +- * ColumnarToRow (7) - : : : +- CometProject (6) - : : : +- CometFilter (5) + : : : +- * Project (7) + : : : +- * Filter (6) + : : : +- * ColumnarToRow (5) : : : +- CometScan parquet spark_catalog.default.customer_demographics (4) : : +- ReusedExchange (11) : +- BroadcastExchange (17) - : +- * ColumnarToRow (16) - : +- CometFilter (15) + : +- * Filter (16) + : +- * ColumnarToRow (15) : +- CometScan parquet spark_catalog.default.store (14) +- BroadcastExchange (23) +- * ColumnarToRow (22) @@ -50,19 +50,19 @@ Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_p Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_marital_status,S), EqualTo(cd_education_status,College ), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) CometFilter +(5) ColumnarToRow [codegen id : 1] Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (cd_gender#11 = M)) AND (cd_marital_status#12 = S)) AND (cd_education_status#13 = College )) AND isnotnull(cd_demo_sk#10)) -(6) CometProject +(6) Filter [codegen id : 1] Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Arguments: [cd_demo_sk#10], [cd_demo_sk#10] +Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (rpad(cd_gender#11, 1, ) = M)) AND (rpad(cd_marital_status#12, 1, ) = S)) AND (rpad(cd_education_status#13, 20, ) = College )) AND isnotnull(cd_demo_sk#10)) -(7) ColumnarToRow [codegen id : 1] -Input [1]: [cd_demo_sk#10] +(7) Project [codegen id : 1] +Output [1]: [cd_demo_sk#10] +Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] (8) BroadcastExchange Input [1]: [cd_demo_sk#10] @@ -95,15 +95,15 @@ Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sale Output [2]: [s_store_sk#15, s_state#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] ReadSchema: struct -(15) CometFilter +(15) ColumnarToRow [codegen id : 3] Input [2]: [s_store_sk#15, s_state#16] -Condition : ((isnotnull(s_state#16) AND (s_state#16 = TN)) AND isnotnull(s_store_sk#15)) -(16) ColumnarToRow [codegen id : 3] +(16) Filter [codegen id : 3] Input [2]: [s_store_sk#15, s_state#16] +Condition : ((isnotnull(s_state#16) AND (rpad(s_state#16, 2, ) = TN)) AND isnotnull(s_store_sk#15)) (17) BroadcastExchange Input [2]: [s_store_sk#15, s_state#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/simplified.txt index 9d073ff67..b86e4494f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/simplified.txt @@ -29,19 +29,19 @@ TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [cd_demo_sk] - CometFilter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] + Project [cd_demo_sk] + Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] InputAdapter ReusedExchange [d_date_sk] #2 InputAdapter BroadcastExchange #4 WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometFilter [s_state,s_store_sk] + Filter [s_state,s_store_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.store [s_store_sk,s_state] InputAdapter BroadcastExchange #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/explain.txt index 098d00824..e13c5d2f7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/explain.txt @@ -44,9 +44,9 @@ TakeOrderedAndProject (49) : +- CometFilter (37) : +- CometScan parquet spark_catalog.default.customer (36) +- BroadcastExchange (46) - +- * ColumnarToRow (45) - +- CometProject (44) - +- CometFilter (43) + +- * Project (45) + +- * Filter (44) + +- * ColumnarToRow (43) +- CometScan parquet spark_catalog.default.customer_address (42) @@ -255,19 +255,19 @@ Input [16]: [ctr_customer_sk#12, ctr_total_return#14, c_customer_sk#33, c_custom Output [2]: [ca_address_sk#47, ca_state#48] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_state), IsNotNull(ca_address_sk)] ReadSchema: struct -(43) CometFilter +(43) ColumnarToRow [codegen id : 10] Input [2]: [ca_address_sk#47, ca_state#48] -Condition : ((isnotnull(ca_state#48) AND (ca_state#48 = GA)) AND isnotnull(ca_address_sk#47)) -(44) CometProject +(44) Filter [codegen id : 10] Input [2]: [ca_address_sk#47, ca_state#48] -Arguments: [ca_address_sk#47], [ca_address_sk#47] +Condition : ((isnotnull(ca_state#48) AND (rpad(ca_state#48, 2, ) = GA)) AND isnotnull(ca_address_sk#47)) -(45) ColumnarToRow [codegen id : 10] -Input [1]: [ca_address_sk#47] +(45) Project [codegen id : 10] +Output [1]: [ca_address_sk#47] +Input [2]: [ca_address_sk#47, ca_state#48] (46) BroadcastExchange Input [1]: [ca_address_sk#47] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/simplified.txt index 365f7f973..127eae18a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/simplified.txt @@ -74,8 +74,8 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_pre InputAdapter BroadcastExchange #8 WholeStageCodegen (10) - ColumnarToRow - InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_state,ca_address_sk] + Project [ca_address_sk] + Filter [ca_state,ca_address_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/explain.txt index f9541481c..89a27002b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/explain.txt @@ -28,9 +28,9 @@ TakeOrderedAndProject (63) : : +- CometFilter (15) : : +- CometScan parquet spark_catalog.default.item (14) : +- BroadcastExchange (21) - : +- * ColumnarToRow (20) - : +- CometProject (19) - : +- CometFilter (18) + : +- * Project (20) + : +- * Filter (19) + : +- * ColumnarToRow (18) : +- CometScan parquet spark_catalog.default.item (17) :- * HashAggregate (43) : +- Exchange (42) @@ -142,19 +142,19 @@ Input [2]: [i_item_sk#9, i_manufact_id#10] Output [2]: [i_category#11, i_manufact_id#12] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Electronics )] +PushedFilters: [IsNotNull(i_category)] ReadSchema: struct -(18) CometFilter +(18) ColumnarToRow [codegen id : 3] Input [2]: [i_category#11, i_manufact_id#12] -Condition : (isnotnull(i_category#11) AND (i_category#11 = Electronics )) -(19) CometProject +(19) Filter [codegen id : 3] Input [2]: [i_category#11, i_manufact_id#12] -Arguments: [i_manufact_id#12], [i_manufact_id#12] +Condition : (isnotnull(i_category#11) AND (rpad(i_category#11, 50, ) = Electronics )) -(20) ColumnarToRow [codegen id : 3] -Input [1]: [i_manufact_id#12] +(20) Project [codegen id : 3] +Output [1]: [i_manufact_id#12] +Input [2]: [i_category#11, i_manufact_id#12] (21) BroadcastExchange Input [1]: [i_manufact_id#12] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/simplified.txt index 4ab82379f..827c606a8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/simplified.txt @@ -52,10 +52,10 @@ TakeOrderedAndProject [total_sales,i_manufact_id] InputAdapter BroadcastExchange #6 WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [i_manufact_id] - CometFilter [i_category] + Project [i_manufact_id] + Filter [i_category] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.item [i_category,i_manufact_id] WholeStageCodegen (12) HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/explain.txt index d0f166fe9..f9528d536 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/explain.txt @@ -23,9 +23,9 @@ : : +- CometFilter (8) : : +- CometScan parquet spark_catalog.default.store (7) : +- BroadcastExchange (18) - : +- * ColumnarToRow (17) - : +- CometProject (16) - : +- CometFilter (15) + : +- * Project (17) + : +- * Filter (16) + : +- * ColumnarToRow (15) : +- CometScan parquet spark_catalog.default.household_demographics (14) +- BroadcastExchange (28) +- * ColumnarToRow (27) @@ -97,19 +97,19 @@ Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, Output [4]: [hd_demo_sk#10, hd_buy_potential#11, hd_dep_count#12, hd_vehicle_count#13] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_vehicle_count), Or(EqualTo(hd_buy_potential,>10000 ),EqualTo(hd_buy_potential,unknown )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] +PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct -(15) CometFilter +(15) ColumnarToRow [codegen id : 3] Input [4]: [hd_demo_sk#10, hd_buy_potential#11, hd_dep_count#12, hd_vehicle_count#13] -Condition : ((((isnotnull(hd_vehicle_count#13) AND ((hd_buy_potential#11 = >10000 ) OR (hd_buy_potential#11 = unknown ))) AND (hd_vehicle_count#13 > 0)) AND CASE WHEN (hd_vehicle_count#13 > 0) THEN ((cast(hd_dep_count#12 as double) / cast(hd_vehicle_count#13 as double)) > 1.2) END) AND isnotnull(hd_demo_sk#10)) -(16) CometProject +(16) Filter [codegen id : 3] Input [4]: [hd_demo_sk#10, hd_buy_potential#11, hd_dep_count#12, hd_vehicle_count#13] -Arguments: [hd_demo_sk#10], [hd_demo_sk#10] +Condition : ((((isnotnull(hd_vehicle_count#13) AND ((rpad(hd_buy_potential#11, 15, ) = >10000 ) OR (rpad(hd_buy_potential#11, 15, ) = unknown ))) AND (hd_vehicle_count#13 > 0)) AND CASE WHEN (hd_vehicle_count#13 > 0) THEN ((cast(hd_dep_count#12 as double) / cast(hd_vehicle_count#13 as double)) > 1.2) END) AND isnotnull(hd_demo_sk#10)) -(17) ColumnarToRow [codegen id : 3] -Input [1]: [hd_demo_sk#10] +(17) Project [codegen id : 3] +Output [1]: [hd_demo_sk#10] +Input [4]: [hd_demo_sk#10, hd_buy_potential#11, hd_dep_count#12, hd_vehicle_count#13] (18) BroadcastExchange Input [1]: [hd_demo_sk#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/simplified.txt index 80405a784..4ec662a9e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/simplified.txt @@ -42,10 +42,10 @@ WholeStageCodegen (7) InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [hd_demo_sk] - CometFilter [hd_vehicle_count,hd_buy_potential,hd_dep_count,hd_demo_sk] + Project [hd_demo_sk] + Filter [hd_vehicle_count,hd_buy_potential,hd_dep_count,hd_demo_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] InputAdapter BroadcastExchange #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/explain.txt index 39b838157..e2b90d8c7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/explain.txt @@ -23,9 +23,9 @@ TakeOrderedAndProject (28) : +- CometFilter (8) : +- CometScan parquet spark_catalog.default.item (7) +- BroadcastExchange (17) - +- * ColumnarToRow (16) - +- CometProject (15) - +- CometFilter (14) + +- * Project (16) + +- * Filter (15) + +- * ColumnarToRow (14) +- CometScan parquet spark_catalog.default.store (13) @@ -89,19 +89,19 @@ Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, Output [2]: [s_store_sk#11, s_state#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] ReadSchema: struct -(14) CometFilter +(14) ColumnarToRow [codegen id : 3] Input [2]: [s_store_sk#11, s_state#12] -Condition : ((isnotnull(s_state#12) AND (s_state#12 = TN)) AND isnotnull(s_store_sk#11)) -(15) CometProject +(15) Filter [codegen id : 3] Input [2]: [s_store_sk#11, s_state#12] -Arguments: [s_store_sk#11], [s_store_sk#11] +Condition : ((isnotnull(s_state#12) AND (rpad(s_state#12, 2, ) = TN)) AND isnotnull(s_store_sk#11)) -(16) ColumnarToRow [codegen id : 3] -Input [1]: [s_store_sk#11] +(16) Project [codegen id : 3] +Output [1]: [s_store_sk#11] +Input [2]: [s_store_sk#11, s_state#12] (17) BroadcastExchange Input [1]: [s_store_sk#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/simplified.txt index 7eeb607c3..444766b0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/simplified.txt @@ -44,8 +44,8 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [s_store_sk] - CometFilter [s_state,s_store_sk] + Project [s_store_sk] + Filter [s_state,s_store_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.store [s_store_sk,s_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/explain.txt index 07196ba8c..de41f3eab 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/explain.txt @@ -14,10 +14,10 @@ TakeOrderedAndProject (20) +- * Filter (12) +- * HashAggregate (11) +- Exchange (10) - +- * ColumnarToRow (9) - +- CometHashAggregate (8) - +- CometProject (7) - +- CometFilter (6) + +- * HashAggregate (9) + +- * Project (8) + +- * Filter (7) + +- * ColumnarToRow (6) +- CometScan parquet spark_catalog.default.item (5) @@ -43,43 +43,45 @@ Input [2]: [i_manufact#2, i_product_name#3] Output [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [Or(Or(And(EqualTo(i_category,Women ),Or(And(And(Or(EqualTo(i_color,powder ),EqualTo(i_color,khaki )),Or(EqualTo(i_units,Ounce ),EqualTo(i_units,Oz ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large ))),And(And(Or(EqualTo(i_color,brown ),EqualTo(i_color,honeydew )),Or(EqualTo(i_units,Bunch ),EqualTo(i_units,Ton ))),Or(EqualTo(i_size,N/A ),EqualTo(i_size,small ))))),And(EqualTo(i_category,Men ),Or(And(And(Or(EqualTo(i_color,floral ),EqualTo(i_color,deep )),Or(EqualTo(i_units,N/A ),EqualTo(i_units,Dozen ))),Or(EqualTo(i_size,petite ),EqualTo(i_size,large ))),And(And(Or(EqualTo(i_color,light ),EqualTo(i_color,cornflower )),Or(EqualTo(i_units,Box ),EqualTo(i_units,Pound ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large )))))),Or(And(EqualTo(i_category,Women ),Or(And(And(Or(EqualTo(i_color,midnight ),EqualTo(i_color,snow )),Or(EqualTo(i_units,Pallet ),EqualTo(i_units,Gross ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large ))),And(And(Or(EqualTo(i_color,cyan ),EqualTo(i_color,papaya )),Or(EqualTo(i_units,Cup ),EqualTo(i_units,Dram ))),Or(EqualTo(i_size,N/A ),EqualTo(i_size,small ))))),And(EqualTo(i_category,Men ),Or(And(And(Or(EqualTo(i_color,orange ),EqualTo(i_color,frosted )),Or(EqualTo(i_units,Each ),EqualTo(i_units,Tbl ))),Or(EqualTo(i_size,petite ),EqualTo(i_size,large ))),And(And(Or(EqualTo(i_color,forest ),EqualTo(i_color,ghost )),Or(EqualTo(i_units,Lb ),EqualTo(i_units,Bundle ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large ))))))), IsNotNull(i_manufact)] +PushedFilters: [IsNotNull(i_manufact)] ReadSchema: struct -(6) CometFilter +(6) ColumnarToRow [codegen id : 1] Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] -Condition : (((((i_category#4 = Women ) AND (((((i_color#7 = powder ) OR (i_color#7 = khaki )) AND ((i_units#8 = Ounce ) OR (i_units#8 = Oz ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))) OR ((((i_color#7 = brown ) OR (i_color#7 = honeydew )) AND ((i_units#8 = Bunch ) OR (i_units#8 = Ton ))) AND ((i_size#6 = N/A ) OR (i_size#6 = small ))))) OR ((i_category#4 = Men ) AND (((((i_color#7 = floral ) OR (i_color#7 = deep )) AND ((i_units#8 = N/A ) OR (i_units#8 = Dozen ))) AND ((i_size#6 = petite ) OR (i_size#6 = large ))) OR ((((i_color#7 = light ) OR (i_color#7 = cornflower )) AND ((i_units#8 = Box ) OR (i_units#8 = Pound ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large )))))) OR (((i_category#4 = Women ) AND (((((i_color#7 = midnight ) OR (i_color#7 = snow )) AND ((i_units#8 = Pallet ) OR (i_units#8 = Gross ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))) OR ((((i_color#7 = cyan ) OR (i_color#7 = papaya )) AND ((i_units#8 = Cup ) OR (i_units#8 = Dram ))) AND ((i_size#6 = N/A ) OR (i_size#6 = small ))))) OR ((i_category#4 = Men ) AND (((((i_color#7 = orange ) OR (i_color#7 = frosted )) AND ((i_units#8 = Each ) OR (i_units#8 = Tbl ))) AND ((i_size#6 = petite ) OR (i_size#6 = large ))) OR ((((i_color#7 = forest ) OR (i_color#7 = ghost )) AND ((i_units#8 = Lb ) OR (i_units#8 = Bundle ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))))))) AND isnotnull(i_manufact#5)) -(7) CometProject +(7) Filter [codegen id : 1] Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] -Arguments: [i_manufact#5], [i_manufact#5] +Condition : (((((rpad(i_category#4, 50, ) = Women ) AND (((((rpad(i_color#7, 20, ) = powder ) OR (rpad(i_color#7, 20, ) = khaki )) AND ((rpad(i_units#8, 10, ) = Ounce ) OR (rpad(i_units#8, 10, ) = Oz ))) AND ((rpad(i_size#6, 20, ) = medium ) OR (rpad(i_size#6, 20, ) = extra large ))) OR ((((rpad(i_color#7, 20, ) = brown ) OR (rpad(i_color#7, 20, ) = honeydew )) AND ((rpad(i_units#8, 10, ) = Bunch ) OR (rpad(i_units#8, 10, ) = Ton ))) AND ((rpad(i_size#6, 20, ) = N/A ) OR (rpad(i_size#6, 20, ) = small ))))) OR ((rpad(i_category#4, 50, ) = Men ) AND (((((rpad(i_color#7, 20, ) = floral ) OR (rpad(i_color#7, 20, ) = deep )) AND ((rpad(i_units#8, 10, ) = N/A ) OR (rpad(i_units#8, 10, ) = Dozen ))) AND ((rpad(i_size#6, 20, ) = petite ) OR (rpad(i_size#6, 20, ) = large ))) OR ((((rpad(i_color#7, 20, ) = light ) OR (rpad(i_color#7, 20, ) = cornflower )) AND ((rpad(i_units#8, 10, ) = Box ) OR (rpad(i_units#8, 10, ) = Pound ))) AND ((rpad(i_size#6, 20, ) = medium ) OR (rpad(i_size#6, 20, ) = extra large )))))) OR (((rpad(i_category#4, 50, ) = Women ) AND (((((rpad(i_color#7, 20, ) = midnight ) OR (rpad(i_color#7, 20, ) = snow )) AND ((rpad(i_units#8, 10, ) = Pallet ) OR (rpad(i_units#8, 10, ) = Gross ))) AND ((rpad(i_size#6, 20, ) = medium ) OR (rpad(i_size#6, 20, ) = extra large ))) OR ((((rpad(i_color#7, 20, ) = cyan ) OR (rpad(i_color#7, 20, ) = papaya )) AND ((rpad(i_units#8, 10, ) = Cup ) OR (rpad(i_units#8, 10, ) = Dram ))) AND ((rpad(i_size#6, 20, ) = N/A ) OR (rpad(i_size#6, 20, ) = small ))))) OR ((rpad(i_category#4, 50, ) = Men ) AND (((((rpad(i_color#7, 20, ) = orange ) OR (rpad(i_color#7, 20, ) = frosted )) AND ((rpad(i_units#8, 10, ) = Each ) OR (rpad(i_units#8, 10, ) = Tbl ))) AND ((rpad(i_size#6, 20, ) = petite ) OR (rpad(i_size#6, 20, ) = large ))) OR ((((rpad(i_color#7, 20, ) = forest ) OR (rpad(i_color#7, 20, ) = ghost )) AND ((rpad(i_units#8, 10, ) = Lb ) OR (rpad(i_units#8, 10, ) = Bundle ))) AND ((rpad(i_size#6, 20, ) = medium ) OR (rpad(i_size#6, 20, ) = extra large ))))))) AND isnotnull(i_manufact#5)) -(8) CometHashAggregate +(8) Project [codegen id : 1] +Output [1]: [i_manufact#5] +Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] + +(9) HashAggregate [codegen id : 1] Input [1]: [i_manufact#5] Keys [1]: [i_manufact#5] Functions [1]: [partial_count(1)] - -(9) ColumnarToRow [codegen id : 1] -Input [2]: [i_manufact#5, count#9] +Aggregate Attributes [1]: [count#9] +Results [2]: [i_manufact#5, count#10] (10) Exchange -Input [2]: [i_manufact#5, count#9] +Input [2]: [i_manufact#5, count#10] Arguments: hashpartitioning(i_manufact#5, 5), ENSURE_REQUIREMENTS, [plan_id=1] (11) HashAggregate [codegen id : 2] -Input [2]: [i_manufact#5, count#9] +Input [2]: [i_manufact#5, count#10] Keys [1]: [i_manufact#5] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#10] -Results [2]: [count(1)#10 AS item_cnt#11, i_manufact#5] +Aggregate Attributes [1]: [count(1)#11] +Results [2]: [count(1)#11 AS item_cnt#12, i_manufact#5] (12) Filter [codegen id : 2] -Input [2]: [item_cnt#11, i_manufact#5] -Condition : (item_cnt#11 > 0) +Input [2]: [item_cnt#12, i_manufact#5] +Condition : (item_cnt#12 > 0) (13) Project [codegen id : 2] Output [1]: [i_manufact#5] -Input [2]: [item_cnt#11, i_manufact#5] +Input [2]: [item_cnt#12, i_manufact#5] (14) BroadcastExchange Input [1]: [i_manufact#5] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/simplified.txt index e31217066..313f6b0ae 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/simplified.txt @@ -21,9 +21,9 @@ TakeOrderedAndProject [i_product_name] InputAdapter Exchange [i_manufact] #3 WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [i_manufact] - CometProject [i_manufact] - CometFilter [i_category,i_color,i_units,i_size,i_manufact] + HashAggregate [i_manufact] [count,count] + Project [i_manufact] + Filter [i_category,i_color,i_units,i_size,i_manufact] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.item [i_category,i_manufact,i_size,i_color,i_units] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/explain.txt index e892aa469..4e6ef5bb4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/explain.txt @@ -104,7 +104,7 @@ Input [6]: [d_day_name#3, ss_store_sk#4, ss_sales_price#5, s_store_sk#7, s_store (18) HashAggregate [codegen id : 3] Input [4]: [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9] Keys [2]: [s_store_name#9, s_store_id#8] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))] +Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Sunday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Monday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Tuesday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Wednesday) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Thursday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Friday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Saturday ) THEN ss_sales_price#5 END))] Aggregate Attributes [7]: [sum#11, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17] Results [9]: [s_store_name#9, s_store_id#8, sum#18, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24] @@ -115,9 +115,9 @@ Arguments: hashpartitioning(s_store_name#9, s_store_id#8, 5), ENSURE_REQUIREMENT (20) HashAggregate [codegen id : 4] Input [9]: [s_store_name#9, s_store_id#8, sum#18, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24] Keys [2]: [s_store_name#9, s_store_id#8] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))#31] -Results [9]: [s_store_name#9, s_store_id#8, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END))#25,17,2) AS sun_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END))#26,17,2) AS mon_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END))#27,17,2) AS tue_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END))#28,17,2) AS wed_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END))#29,17,2) AS thu_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END))#30,17,2) AS fri_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))#31,17,2) AS sat_sales#38] +Functions [7]: [sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Sunday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Monday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Tuesday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Wednesday) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Thursday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Friday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Saturday ) THEN ss_sales_price#5 END))] +Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Sunday ) THEN ss_sales_price#5 END))#25, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Monday ) THEN ss_sales_price#5 END))#26, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Tuesday ) THEN ss_sales_price#5 END))#27, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Wednesday) THEN ss_sales_price#5 END))#28, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Thursday ) THEN ss_sales_price#5 END))#29, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Friday ) THEN ss_sales_price#5 END))#30, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Saturday ) THEN ss_sales_price#5 END))#31] +Results [9]: [s_store_name#9, s_store_id#8, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Sunday ) THEN ss_sales_price#5 END))#25,17,2) AS sun_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Monday ) THEN ss_sales_price#5 END))#26,17,2) AS mon_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Tuesday ) THEN ss_sales_price#5 END))#27,17,2) AS tue_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Wednesday) THEN ss_sales_price#5 END))#28,17,2) AS wed_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Thursday ) THEN ss_sales_price#5 END))#29,17,2) AS thu_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Friday ) THEN ss_sales_price#5 END))#30,17,2) AS fri_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#3, 9, ) = Saturday ) THEN ss_sales_price#5 END))#31,17,2) AS sat_sales#38] (21) TakeOrderedAndProject Input [9]: [s_store_name#9, s_store_id#8, sun_sales#32, mon_sales#33, tue_sales#34, wed_sales#35, thu_sales#36, fri_sales#37, sat_sales#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/simplified.txt index ef2043096..55ddc1a5b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/simplified.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] WholeStageCodegen (4) - HashAggregate [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] + HashAggregate [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] InputAdapter Exchange [s_store_name,s_store_id] #1 WholeStageCodegen (3) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/explain.txt index 718f3fb31..8ee66e976 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/explain.txt @@ -18,13 +18,13 @@ : : : +- CometFilter (5) : : : +- CometScan parquet spark_catalog.default.store (4) : : +- BroadcastExchange (13) - : : +- * ColumnarToRow (12) - : : +- CometFilter (11) + : : +- * Filter (12) + : : +- * ColumnarToRow (11) : : +- CometScan parquet spark_catalog.default.customer_demographics (10) : +- BroadcastExchange (20) - : +- * ColumnarToRow (19) - : +- CometProject (18) - : +- CometFilter (17) + : +- * Project (19) + : +- * Filter (18) + : +- * ColumnarToRow (17) : +- CometScan parquet spark_catalog.default.customer_address (16) +- ReusedExchange (23) @@ -76,15 +76,15 @@ Input [8]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_ Output [3]: [cd_demo_sk#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,4 yr Degree )),And(EqualTo(cd_marital_status,D),EqualTo(cd_education_status,2 yr Degree ))),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College )))] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(11) CometFilter +(11) ColumnarToRow [codegen id : 2] Input [3]: [cd_demo_sk#10, cd_marital_status#11, cd_education_status#12] -Condition : (isnotnull(cd_demo_sk#10) AND ((((cd_marital_status#11 = M) AND (cd_education_status#12 = 4 yr Degree )) OR ((cd_marital_status#11 = D) AND (cd_education_status#12 = 2 yr Degree ))) OR ((cd_marital_status#11 = S) AND (cd_education_status#12 = College )))) -(12) ColumnarToRow [codegen id : 2] +(12) Filter [codegen id : 2] Input [3]: [cd_demo_sk#10, cd_marital_status#11, cd_education_status#12] +Condition : (isnotnull(cd_demo_sk#10) AND ((((rpad(cd_marital_status#11, 1, ) = M) AND (rpad(cd_education_status#12, 20, ) = 4 yr Degree )) OR ((rpad(cd_marital_status#11, 1, ) = D) AND (rpad(cd_education_status#12, 20, ) = 2 yr Degree ))) OR ((rpad(cd_marital_status#11, 1, ) = S) AND (rpad(cd_education_status#12, 20, ) = College )))) (13) BroadcastExchange Input [3]: [cd_demo_sk#10, cd_marital_status#11, cd_education_status#12] @@ -94,7 +94,7 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint) Left keys [1]: [ss_cdemo_sk#1] Right keys [1]: [cd_demo_sk#10] Join type: Inner -Join condition: ((((((cd_marital_status#11 = M) AND (cd_education_status#12 = 4 yr Degree )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((cd_marital_status#11 = D) AND (cd_education_status#12 = 2 yr Degree )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((cd_marital_status#11 = S) AND (cd_education_status#12 = College )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00))) +Join condition: ((((((rpad(cd_marital_status#11, 1, ) = M) AND (rpad(cd_education_status#12, 20, ) = 4 yr Degree )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((rpad(cd_marital_status#11, 1, ) = D) AND (rpad(cd_education_status#12, 20, ) = 2 yr Degree )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((rpad(cd_marital_status#11, 1, ) = S) AND (rpad(cd_education_status#12, 20, ) = College )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00))) (15) Project [codegen id : 5] Output [4]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] @@ -104,19 +104,19 @@ Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net Output [3]: [ca_address_sk#13, ca_state#14, ca_country#15] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [CO,OH,TX]),In(ca_state, [KY,MN,OR])),In(ca_state, [CA,MS,VA]))] +PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] ReadSchema: struct -(17) CometFilter +(17) ColumnarToRow [codegen id : 3] Input [3]: [ca_address_sk#13, ca_state#14, ca_country#15] -Condition : (((isnotnull(ca_country#15) AND (ca_country#15 = United States)) AND isnotnull(ca_address_sk#13)) AND ((ca_state#14 IN (CO,OH,TX) OR ca_state#14 IN (OR,MN,KY)) OR ca_state#14 IN (VA,CA,MS))) -(18) CometProject +(18) Filter [codegen id : 3] Input [3]: [ca_address_sk#13, ca_state#14, ca_country#15] -Arguments: [ca_address_sk#13, ca_state#14], [ca_address_sk#13, ca_state#14] +Condition : (((isnotnull(ca_country#15) AND (ca_country#15 = United States)) AND isnotnull(ca_address_sk#13)) AND ((rpad(ca_state#14, 2, ) IN (CO,OH,TX) OR rpad(ca_state#14, 2, ) IN (OR,MN,KY)) OR rpad(ca_state#14, 2, ) IN (VA,CA,MS))) -(19) ColumnarToRow [codegen id : 3] -Input [2]: [ca_address_sk#13, ca_state#14] +(19) Project [codegen id : 3] +Output [2]: [ca_address_sk#13, ca_state#14] +Input [3]: [ca_address_sk#13, ca_state#14, ca_country#15] (20) BroadcastExchange Input [2]: [ca_address_sk#13, ca_state#14] @@ -126,7 +126,7 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Left keys [1]: [ss_addr_sk#2] Right keys [1]: [ca_address_sk#13] Join type: Inner -Join condition: ((((ca_state#14 IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((ca_state#14 IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((ca_state#14 IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00))) +Join condition: ((((rpad(ca_state#14, 2, ) IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((rpad(ca_state#14, 2, ) IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((rpad(ca_state#14, 2, ) IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00))) (22) Project [codegen id : 5] Output [2]: [ss_quantity#4, ss_sold_date_sk#7] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/simplified.txt index 4022da74f..f1e3a5f4c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/simplified.txt @@ -34,17 +34,17 @@ WholeStageCodegen (6) InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + Filter [cd_demo_sk,cd_marital_status,cd_education_status] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [ca_address_sk,ca_state] - CometFilter [ca_country,ca_address_sk,ca_state] + Project [ca_address_sk,ca_state] + Filter [ca_country,ca_address_sk,ca_state] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] InputAdapter ReusedExchange [d_date_sk] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/explain.txt index 335dc7fa2..3ea06fa5a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/explain.txt @@ -14,9 +14,9 @@ TakeOrderedAndProject (28) : +- * BroadcastHashJoin Inner BuildRight (12) : :- * Project (10) : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * ColumnarToRow (4) - : : : +- CometProject (3) - : : : +- CometFilter (2) + : : :- * Project (4) + : : : +- * Filter (3) + : : : +- * ColumnarToRow (2) : : : +- CometScan parquet spark_catalog.default.item (1) : : +- BroadcastExchange (8) : : +- * ColumnarToRow (7) @@ -33,19 +33,19 @@ TakeOrderedAndProject (28) Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(2) CometFilter +(2) ColumnarToRow [codegen id : 4] Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Condition : ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) -(3) CometProject +(3) Filter [codegen id : 4] Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_manufact_id#5], [i_item_sk#1, i_manufact_id#5] +Condition : ((((rpad(i_category#4, 50, ) IN (Books ,Children ,Electronics ) AND rpad(i_class#3, 50, ) IN (personal ,portable ,reference ,self-help )) AND rpad(i_brand#2, 50, ) IN (scholaramalgamalg #6 ,scholaramalgamalg #7 ,exportiunivamalg #8 ,scholaramalgamalg #8 )) OR ((rpad(i_category#4, 50, ) IN (Women ,Music ,Men ) AND rpad(i_class#3, 50, ) IN (accessories ,classical ,fragrances ,pants )) AND rpad(i_brand#2, 50, ) IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) -(4) ColumnarToRow [codegen id : 4] -Input [2]: [i_item_sk#1, i_manufact_id#5] +(4) Project [codegen id : 4] +Output [2]: [i_item_sk#1, i_manufact_id#5] +Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] (5) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/simplified.txt index adda5c34f..51dfac414 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/simplified.txt @@ -20,10 +20,10 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] BroadcastHashJoin [i_item_sk,ss_item_sk] - ColumnarToRow - InputAdapter - CometProject [i_item_sk,i_manufact_id] - CometFilter [i_category,i_class,i_brand,i_item_sk] + Project [i_item_sk,i_manufact_id] + Filter [i_category,i_class,i_brand,i_item_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] InputAdapter BroadcastExchange #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/explain.txt index ca308b19a..0f3f4fe34 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/explain.txt @@ -32,9 +32,9 @@ TakeOrderedAndProject (55) : : : : : : : +- CometFilter (5) : : : : : : : +- CometScan parquet spark_catalog.default.web_sales (4) : : : : : : +- BroadcastExchange (13) - : : : : : : +- * ColumnarToRow (12) - : : : : : : +- CometProject (11) - : : : : : : +- CometFilter (10) + : : : : : : +- * Project (12) + : : : : : : +- * Filter (11) + : : : : : : +- * ColumnarToRow (10) : : : : : : +- CometScan parquet spark_catalog.default.item (9) : : : : : +- ReusedExchange (16) : : : : +- BroadcastExchange (22) @@ -99,19 +99,19 @@ Input [3]: [sold_date_sk#5, customer_sk#6, item_sk#7] Output [3]: [i_item_sk#14, i_class#15, i_category#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), IsNotNull(i_class), EqualTo(i_category,Women ), EqualTo(i_class,maternity ), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_category), IsNotNull(i_class), IsNotNull(i_item_sk)] ReadSchema: struct -(10) CometFilter +(10) ColumnarToRow [codegen id : 1] Input [3]: [i_item_sk#14, i_class#15, i_category#16] -Condition : ((((isnotnull(i_category#16) AND isnotnull(i_class#15)) AND (i_category#16 = Women )) AND (i_class#15 = maternity )) AND isnotnull(i_item_sk#14)) -(11) CometProject +(11) Filter [codegen id : 1] Input [3]: [i_item_sk#14, i_class#15, i_category#16] -Arguments: [i_item_sk#14], [i_item_sk#14] +Condition : ((((isnotnull(i_category#16) AND isnotnull(i_class#15)) AND (rpad(i_category#16, 50, ) = Women )) AND (rpad(i_class#15, 50, ) = maternity )) AND isnotnull(i_item_sk#14)) -(12) ColumnarToRow [codegen id : 1] -Input [1]: [i_item_sk#14] +(12) Project [codegen id : 1] +Output [1]: [i_item_sk#14] +Input [3]: [i_item_sk#14, i_class#15, i_category#16] (13) BroadcastExchange Input [1]: [i_item_sk#14] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/simplified.txt index 30ba4b743..582594dec 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/simplified.txt @@ -50,10 +50,10 @@ TakeOrderedAndProject [segment,num_customers,segment_base] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [i_item_sk] - CometFilter [i_category,i_class,i_item_sk] + Project [i_item_sk] + Filter [i_category,i_class,i_item_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] InputAdapter ReusedExchange [d_date_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/explain.txt index bbed7eea6..0033edd4d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/explain.txt @@ -28,9 +28,9 @@ TakeOrderedAndProject (63) : : +- CometFilter (15) : : +- CometScan parquet spark_catalog.default.item (14) : +- BroadcastExchange (21) - : +- * ColumnarToRow (20) - : +- CometProject (19) - : +- CometFilter (18) + : +- * Project (20) + : +- * Filter (19) + : +- * ColumnarToRow (18) : +- CometScan parquet spark_catalog.default.item (17) :- * HashAggregate (43) : +- Exchange (42) @@ -142,19 +142,18 @@ Input [2]: [i_item_sk#9, i_item_id#10] Output [2]: [i_item_id#11, i_color#12] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_color, [blanched ,burnished ,slate ])] ReadSchema: struct -(18) CometFilter +(18) ColumnarToRow [codegen id : 3] Input [2]: [i_item_id#11, i_color#12] -Condition : i_color#12 IN (slate ,blanched ,burnished ) -(19) CometProject +(19) Filter [codegen id : 3] Input [2]: [i_item_id#11, i_color#12] -Arguments: [i_item_id#11], [i_item_id#11] +Condition : rpad(i_color#12, 20, ) IN (slate ,blanched ,burnished ) -(20) ColumnarToRow [codegen id : 3] -Input [1]: [i_item_id#11] +(20) Project [codegen id : 3] +Output [1]: [i_item_id#11] +Input [2]: [i_item_id#11, i_color#12] (21) BroadcastExchange Input [1]: [i_item_id#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/simplified.txt index f781ed1f7..68d680583 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/simplified.txt @@ -52,10 +52,10 @@ TakeOrderedAndProject [total_sales,i_item_id] InputAdapter BroadcastExchange #6 WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [i_item_id] - CometFilter [i_color] + Project [i_item_id] + Filter [i_color] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.item [i_item_id,i_color] WholeStageCodegen (12) HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/explain.txt index 023c54bdd..00a418ac6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/explain.txt @@ -91,7 +91,7 @@ Input [6]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_w (10) HashAggregate [codegen id : 2] Input [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6] Keys [2]: [d_week_seq#5, ss_store_sk#1] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] +Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Sunday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Monday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Tuesday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Wednesday) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Thursday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Friday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Saturday ) THEN ss_sales_price#2 END))] Aggregate Attributes [7]: [sum#7, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13] Results [9]: [d_week_seq#5, ss_store_sk#1, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20] @@ -102,9 +102,9 @@ Arguments: hashpartitioning(d_week_seq#5, ss_store_sk#1, 5), ENSURE_REQUIREMENTS (12) HashAggregate [codegen id : 10] Input [9]: [d_week_seq#5, ss_store_sk#1, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20] Keys [2]: [d_week_seq#5, ss_store_sk#1] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END))#21, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END))#22, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))#27] -Results [9]: [d_week_seq#5, ss_store_sk#1, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END))#21,17,2) AS sun_sales#28, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END))#22,17,2) AS mon_sales#29, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END))#23,17,2) AS tue_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END))#24,17,2) AS wed_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END))#25,17,2) AS thu_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END))#26,17,2) AS fri_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))#27,17,2) AS sat_sales#34] +Functions [7]: [sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Saturday ) THEN ss_sales_price#2 END))] +Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Sunday ) THEN ss_sales_price#2 END))#21, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Monday ) THEN ss_sales_price#2 END))#22, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Tuesday ) THEN ss_sales_price#2 END))#23, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Wednesday) THEN ss_sales_price#2 END))#24, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Thursday ) THEN ss_sales_price#2 END))#25, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Friday ) THEN ss_sales_price#2 END))#26, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Saturday ) THEN ss_sales_price#2 END))#27] +Results [9]: [d_week_seq#5, ss_store_sk#1, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Sunday ) THEN ss_sales_price#2 END))#21,17,2) AS sun_sales#28, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Monday ) THEN ss_sales_price#2 END))#22,17,2) AS mon_sales#29, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Tuesday ) THEN ss_sales_price#2 END))#23,17,2) AS tue_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Wednesday) THEN ss_sales_price#2 END))#24,17,2) AS wed_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Thursday ) THEN ss_sales_price#2 END))#25,17,2) AS thu_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Friday ) THEN ss_sales_price#2 END))#26,17,2) AS fri_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#6, 9, ) = Saturday ) THEN ss_sales_price#2 END))#27,17,2) AS sat_sales#34] (13) Scan parquet spark_catalog.default.store Output [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] @@ -172,9 +172,9 @@ Output [9]: [d_week_seq#50, ss_store_sk#51, sum#52, sum#53, sum#54, sum#55, sum# (27) HashAggregate [codegen id : 9] Input [9]: [d_week_seq#50, ss_store_sk#51, sum#52, sum#53, sum#54, sum#55, sum#56, sum#57, sum#58] Keys [2]: [d_week_seq#50, ss_store_sk#51] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#59 = Sunday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Monday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Tuesday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Wednesday) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Thursday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Friday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Saturday ) THEN ss_sales_price#60 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#59 = Sunday ) THEN ss_sales_price#60 END))#21, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Monday ) THEN ss_sales_price#60 END))#22, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Tuesday ) THEN ss_sales_price#60 END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Wednesday) THEN ss_sales_price#60 END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Thursday ) THEN ss_sales_price#60 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Friday ) THEN ss_sales_price#60 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Saturday ) THEN ss_sales_price#60 END))#27] -Results [9]: [d_week_seq#50, ss_store_sk#51, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Sunday ) THEN ss_sales_price#60 END))#21,17,2) AS sun_sales#61, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Monday ) THEN ss_sales_price#60 END))#22,17,2) AS mon_sales#62, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Tuesday ) THEN ss_sales_price#60 END))#23,17,2) AS tue_sales#63, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Wednesday) THEN ss_sales_price#60 END))#24,17,2) AS wed_sales#64, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Thursday ) THEN ss_sales_price#60 END))#25,17,2) AS thu_sales#65, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Friday ) THEN ss_sales_price#60 END))#26,17,2) AS fri_sales#66, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Saturday ) THEN ss_sales_price#60 END))#27,17,2) AS sat_sales#67] +Functions [7]: [sum(UnscaledValue(CASE WHEN (rpad(d_day_name#59, 9, ) = Sunday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#59, 9, ) = Monday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#59, 9, ) = Tuesday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#59, 9, ) = Wednesday) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#59, 9, ) = Thursday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#59, 9, ) = Friday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (rpad(d_day_name#59, 9, ) = Saturday ) THEN ss_sales_price#60 END))] +Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (rpad(d_day_name#59, 9, ) = Sunday ) THEN ss_sales_price#60 END))#21, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#59, 9, ) = Monday ) THEN ss_sales_price#60 END))#22, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#59, 9, ) = Tuesday ) THEN ss_sales_price#60 END))#23, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#59, 9, ) = Wednesday) THEN ss_sales_price#60 END))#24, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#59, 9, ) = Thursday ) THEN ss_sales_price#60 END))#25, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#59, 9, ) = Friday ) THEN ss_sales_price#60 END))#26, sum(UnscaledValue(CASE WHEN (rpad(d_day_name#59, 9, ) = Saturday ) THEN ss_sales_price#60 END))#27] +Results [9]: [d_week_seq#50, ss_store_sk#51, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#59, 9, ) = Sunday ) THEN ss_sales_price#60 END))#21,17,2) AS sun_sales#61, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#59, 9, ) = Monday ) THEN ss_sales_price#60 END))#22,17,2) AS mon_sales#62, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#59, 9, ) = Tuesday ) THEN ss_sales_price#60 END))#23,17,2) AS tue_sales#63, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#59, 9, ) = Wednesday) THEN ss_sales_price#60 END))#24,17,2) AS wed_sales#64, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#59, 9, ) = Thursday ) THEN ss_sales_price#60 END))#25,17,2) AS thu_sales#65, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#59, 9, ) = Friday ) THEN ss_sales_price#60 END))#26,17,2) AS fri_sales#66, MakeDecimal(sum(UnscaledValue(CASE WHEN (rpad(d_day_name#59, 9, ) = Saturday ) THEN ss_sales_price#60 END))#27,17,2) AS sat_sales#67] (28) Scan parquet spark_catalog.default.store Output [2]: [s_store_sk#68, s_store_id#69] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/simplified.txt index 9ad61e946..b4df56c1e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/simplified.txt @@ -6,7 +6,7 @@ TakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_s BroadcastHashJoin [d_week_seq,d_week_seq] Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name] BroadcastHashJoin [ss_store_sk,s_store_sk] - HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] + HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] InputAdapter Exchange [d_week_seq,ss_store_sk] #1 WholeStageCodegen (2) @@ -46,7 +46,7 @@ TakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_s BroadcastHashJoin [d_week_seq,d_week_seq] Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id] BroadcastHashJoin [ss_store_sk,s_store_sk] - HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] + HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (rpad(d_day_name, 9, ) = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] InputAdapter ReusedExchange [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] #1 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/explain.txt index 78f4b27ac..f87fc9a95 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/explain.txt @@ -28,9 +28,9 @@ TakeOrderedAndProject (63) : : +- CometFilter (15) : : +- CometScan parquet spark_catalog.default.item (14) : +- BroadcastExchange (21) - : +- * ColumnarToRow (20) - : +- CometProject (19) - : +- CometFilter (18) + : +- * Project (20) + : +- * Filter (19) + : +- * ColumnarToRow (18) : +- CometScan parquet spark_catalog.default.item (17) :- * HashAggregate (43) : +- Exchange (42) @@ -142,19 +142,19 @@ Input [2]: [i_item_sk#9, i_item_id#10] Output [2]: [i_item_id#11, i_category#12] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Music )] +PushedFilters: [IsNotNull(i_category)] ReadSchema: struct -(18) CometFilter +(18) ColumnarToRow [codegen id : 3] Input [2]: [i_item_id#11, i_category#12] -Condition : (isnotnull(i_category#12) AND (i_category#12 = Music )) -(19) CometProject +(19) Filter [codegen id : 3] Input [2]: [i_item_id#11, i_category#12] -Arguments: [i_item_id#11], [i_item_id#11] +Condition : (isnotnull(i_category#12) AND (rpad(i_category#12, 50, ) = Music )) -(20) ColumnarToRow [codegen id : 3] -Input [1]: [i_item_id#11] +(20) Project [codegen id : 3] +Output [1]: [i_item_id#11] +Input [2]: [i_item_id#11, i_category#12] (21) BroadcastExchange Input [1]: [i_item_id#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/simplified.txt index b010414a8..9a49aa581 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/simplified.txt @@ -52,10 +52,10 @@ TakeOrderedAndProject [i_item_id,total_sales] InputAdapter BroadcastExchange #6 WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [i_item_id] - CometFilter [i_category] + Project [i_item_id] + Filter [i_category] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.item [i_item_id,i_category] WholeStageCodegen (12) HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/explain.txt index 766362167..0abed473b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/explain.txt @@ -25,9 +25,9 @@ : : : : : : +- CometFilter (5) : : : : : : +- CometScan parquet spark_catalog.default.store (4) : : : : : +- BroadcastExchange (15) - : : : : : +- * ColumnarToRow (14) - : : : : : +- CometProject (13) - : : : : : +- CometFilter (12) + : : : : : +- * Project (14) + : : : : : +- * Filter (13) + : : : : : +- * ColumnarToRow (12) : : : : : +- CometScan parquet spark_catalog.default.promotion (11) : : : : +- ReusedExchange (18) : : : +- BroadcastExchange (24) @@ -40,9 +40,9 @@ : : +- CometFilter (28) : : +- CometScan parquet spark_catalog.default.customer_address (27) : +- BroadcastExchange (38) - : +- * ColumnarToRow (37) - : +- CometProject (36) - : +- CometFilter (35) + : +- * Project (37) + : +- * Filter (36) + : +- * ColumnarToRow (35) : +- CometScan parquet spark_catalog.default.item (34) +- BroadcastExchange (65) +- * HashAggregate (64) @@ -119,19 +119,19 @@ Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext Output [4]: [p_promo_sk#10, p_channel_dmail#11, p_channel_email#12, p_channel_tv#13] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [Or(Or(EqualTo(p_channel_dmail,Y),EqualTo(p_channel_email,Y)),EqualTo(p_channel_tv,Y)), IsNotNull(p_promo_sk)] +PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(12) CometFilter +(12) ColumnarToRow [codegen id : 2] Input [4]: [p_promo_sk#10, p_channel_dmail#11, p_channel_email#12, p_channel_tv#13] -Condition : ((((p_channel_dmail#11 = Y) OR (p_channel_email#12 = Y)) OR (p_channel_tv#13 = Y)) AND isnotnull(p_promo_sk#10)) -(13) CometProject +(13) Filter [codegen id : 2] Input [4]: [p_promo_sk#10, p_channel_dmail#11, p_channel_email#12, p_channel_tv#13] -Arguments: [p_promo_sk#10], [p_promo_sk#10] +Condition : ((((rpad(p_channel_dmail#11, 1, ) = Y) OR (rpad(p_channel_email#12, 1, ) = Y)) OR (rpad(p_channel_tv#13, 1, ) = Y)) AND isnotnull(p_promo_sk#10)) -(14) ColumnarToRow [codegen id : 2] -Input [1]: [p_promo_sk#10] +(14) Project [codegen id : 2] +Output [1]: [p_promo_sk#10] +Input [4]: [p_promo_sk#10, p_channel_dmail#11, p_channel_email#12, p_channel_tv#13] (15) BroadcastExchange Input [1]: [p_promo_sk#10] @@ -224,19 +224,19 @@ Input [4]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#16, ca_address Output [2]: [i_item_sk#19, i_category#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Jewelry ), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_category), IsNotNull(i_item_sk)] ReadSchema: struct -(35) CometFilter +(35) ColumnarToRow [codegen id : 6] Input [2]: [i_item_sk#19, i_category#20] -Condition : ((isnotnull(i_category#20) AND (i_category#20 = Jewelry )) AND isnotnull(i_item_sk#19)) -(36) CometProject +(36) Filter [codegen id : 6] Input [2]: [i_item_sk#19, i_category#20] -Arguments: [i_item_sk#19], [i_item_sk#19] +Condition : ((isnotnull(i_category#20) AND (rpad(i_category#20, 50, ) = Jewelry )) AND isnotnull(i_item_sk#19)) -(37) ColumnarToRow [codegen id : 6] -Input [1]: [i_item_sk#19] +(37) Project [codegen id : 6] +Output [1]: [i_item_sk#19] +Input [2]: [i_item_sk#19, i_category#20] (38) BroadcastExchange Input [1]: [i_item_sk#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/simplified.txt index 2c3d07ac6..60e13638e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/simplified.txt @@ -41,10 +41,10 @@ WholeStageCodegen (15) InputAdapter BroadcastExchange #4 WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometProject [p_promo_sk] - CometFilter [p_channel_dmail,p_channel_email,p_channel_tv,p_promo_sk] + Project [p_promo_sk] + Filter [p_channel_dmail,p_channel_email,p_channel_tv,p_promo_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] InputAdapter ReusedExchange [d_date_sk] #2 @@ -66,10 +66,10 @@ WholeStageCodegen (15) InputAdapter BroadcastExchange #7 WholeStageCodegen (6) - ColumnarToRow - InputAdapter - CometProject [i_item_sk] - CometFilter [i_category,i_item_sk] + Project [i_item_sk] + Filter [i_category,i_item_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.item [i_item_sk,i_category] InputAdapter BroadcastExchange #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/explain.txt index 990bc3195..bf43aaeed 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/explain.txt @@ -14,9 +14,9 @@ TakeOrderedAndProject (28) : +- * BroadcastHashJoin Inner BuildRight (12) : :- * Project (10) : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * ColumnarToRow (4) - : : : +- CometProject (3) - : : : +- CometFilter (2) + : : :- * Project (4) + : : : +- * Filter (3) + : : : +- * ColumnarToRow (2) : : : +- CometScan parquet spark_catalog.default.item (1) : : +- BroadcastExchange (8) : : +- * ColumnarToRow (7) @@ -33,19 +33,19 @@ TakeOrderedAndProject (28) Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,refernece ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(2) CometFilter +(2) ColumnarToRow [codegen id : 4] Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Condition : ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,refernece ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) -(3) CometProject +(3) Filter [codegen id : 4] Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Arguments: [i_item_sk#1, i_manager_id#5], [i_item_sk#1, i_manager_id#5] +Condition : ((((rpad(i_category#4, 50, ) IN (Books ,Children ,Electronics ) AND rpad(i_class#3, 50, ) IN (personal ,portable ,refernece ,self-help )) AND rpad(i_brand#2, 50, ) IN (scholaramalgamalg #6 ,scholaramalgamalg #7 ,exportiunivamalg #8 ,scholaramalgamalg #8 )) OR ((rpad(i_category#4, 50, ) IN (Women ,Music ,Men ) AND rpad(i_class#3, 50, ) IN (accessories ,classical ,fragrances ,pants )) AND rpad(i_brand#2, 50, ) IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) -(4) ColumnarToRow [codegen id : 4] -Input [2]: [i_item_sk#1, i_manager_id#5] +(4) Project [codegen id : 4] +Output [2]: [i_item_sk#1, i_manager_id#5] +Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] (5) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/simplified.txt index 7f6f8c137..bb3ddc59a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/simplified.txt @@ -20,10 +20,10 @@ TakeOrderedAndProject [i_manager_id,avg_monthly_sales,sum_sales] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] BroadcastHashJoin [i_item_sk,ss_item_sk] - ColumnarToRow - InputAdapter - CometProject [i_item_sk,i_manager_id] - CometFilter [i_category,i_class,i_brand,i_item_sk] + Project [i_item_sk,i_manager_id] + Filter [i_category,i_class,i_brand,i_item_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manager_id] InputAdapter BroadcastExchange #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/explain.txt index 667362aa3..b2609a4a1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/explain.txt @@ -109,9 +109,9 @@ : : : +- CometScan parquet spark_catalog.default.income_band (91) : : +- ReusedExchange (97) : +- BroadcastExchange (104) - : +- * ColumnarToRow (103) - : +- CometProject (102) - : +- CometFilter (101) + : +- * Project (103) + : +- * Filter (102) + : +- * ColumnarToRow (101) : +- CometScan parquet spark_catalog.default.item (100) +- * Sort (177) +- Exchange (176) @@ -641,19 +641,19 @@ Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt# Output [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), In(i_color, [burlywood ,floral ,indian ,medium ,purple ,spring ]), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] ReadSchema: struct -(101) CometFilter +(101) ColumnarToRow [codegen id : 23] Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] -Condition : ((((((isnotnull(i_current_price#74) AND i_color#75 IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#74 >= 64.00)) AND (i_current_price#74 <= 74.00)) AND (i_current_price#74 >= 65.00)) AND (i_current_price#74 <= 79.00)) AND isnotnull(i_item_sk#73)) -(102) CometProject +(102) Filter [codegen id : 23] Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] -Arguments: [i_item_sk#73, i_product_name#76], [i_item_sk#73, i_product_name#76] +Condition : ((((((isnotnull(i_current_price#74) AND rpad(i_color#75, 20, ) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#74 >= 64.00)) AND (i_current_price#74 <= 74.00)) AND (i_current_price#74 >= 65.00)) AND (i_current_price#74 <= 79.00)) AND isnotnull(i_item_sk#73)) -(103) ColumnarToRow [codegen id : 23] -Input [2]: [i_item_sk#73, i_product_name#76] +(103) Project [codegen id : 23] +Output [2]: [i_item_sk#73, i_product_name#76] +Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] (104) BroadcastExchange Input [2]: [i_item_sk#73, i_product_name#76] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/simplified.txt index d972e0082..94a52fe85 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/simplified.txt @@ -173,10 +173,10 @@ WholeStageCodegen (52) InputAdapter BroadcastExchange #17 WholeStageCodegen (23) - ColumnarToRow - InputAdapter - CometProject [i_item_sk,i_product_name] - CometFilter [i_current_price,i_color,i_item_sk] + Project [i_item_sk,i_product_name] + Filter [i_current_price,i_color,i_item_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] InputAdapter WholeStageCodegen (50) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/explain.txt index a26c457aa..b74b6c554 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/explain.txt @@ -29,9 +29,9 @@ TakeOrderedAndProject (52) : : +- CometFilter (14) : : +- CometScan parquet spark_catalog.default.time_dim (13) : +- BroadcastExchange (24) - : +- * ColumnarToRow (23) - : +- CometProject (22) - : +- CometFilter (21) + : +- * Project (23) + : +- * Filter (22) + : +- * ColumnarToRow (21) : +- CometScan parquet spark_catalog.default.ship_mode (20) +- * HashAggregate (47) +- Exchange (46) @@ -145,19 +145,19 @@ Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_p Output [2]: [sm_ship_mode_sk#21, sm_carrier#22] Batched: true Location [not included in comparison]/{warehouse_dir}/ship_mode] -PushedFilters: [In(sm_carrier, [BARIAN ,DHL ]), IsNotNull(sm_ship_mode_sk)] +PushedFilters: [IsNotNull(sm_ship_mode_sk)] ReadSchema: struct -(21) CometFilter +(21) ColumnarToRow [codegen id : 4] Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] -Condition : (sm_carrier#22 IN (DHL ,BARIAN ) AND isnotnull(sm_ship_mode_sk#21)) -(22) CometProject +(22) Filter [codegen id : 4] Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] -Arguments: [sm_ship_mode_sk#21], [sm_ship_mode_sk#21] +Condition : (rpad(sm_carrier#22, 20, ) IN (DHL ,BARIAN ) AND isnotnull(sm_ship_mode_sk#21)) -(23) ColumnarToRow [codegen id : 4] -Input [1]: [sm_ship_mode_sk#21] +(23) Project [codegen id : 4] +Output [1]: [sm_ship_mode_sk#21] +Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] (24) BroadcastExchange Input [1]: [sm_ship_mode_sk#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/simplified.txt index 8ed74582f..e82c80d99 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/simplified.txt @@ -52,10 +52,10 @@ TakeOrderedAndProject [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_stat InputAdapter BroadcastExchange #6 WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometProject [sm_ship_mode_sk] - CometFilter [sm_carrier,sm_ship_mode_sk] + Project [sm_ship_mode_sk] + Filter [sm_carrier,sm_ship_mode_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_carrier] WholeStageCodegen (12) HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_moy = 1) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/explain.txt index d56b01d73..7553f934b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/explain.txt @@ -33,9 +33,9 @@ TakeOrderedAndProject (42) : : : +- CometScan parquet spark_catalog.default.catalog_sales (18) : : +- ReusedExchange (20) : +- BroadcastExchange (30) - : +- * ColumnarToRow (29) - : +- CometProject (28) - : +- CometFilter (27) + : +- * Project (29) + : +- * Filter (28) + : +- * ColumnarToRow (27) : +- CometScan parquet spark_catalog.default.customer_address (26) +- BroadcastExchange (36) +- * ColumnarToRow (35) @@ -164,19 +164,19 @@ Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [GA,KY,NM]), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(27) CometFilter +(27) ColumnarToRow [codegen id : 7] Input [2]: [ca_address_sk#16, ca_state#17] -Condition : (ca_state#17 IN (KY,GA,NM) AND isnotnull(ca_address_sk#16)) -(28) CometProject +(28) Filter [codegen id : 7] Input [2]: [ca_address_sk#16, ca_state#17] -Arguments: [ca_address_sk#16], [ca_address_sk#16] +Condition : (rpad(ca_state#17, 2, ) IN (KY,GA,NM) AND isnotnull(ca_address_sk#16)) -(29) ColumnarToRow [codegen id : 7] -Input [1]: [ca_address_sk#16] +(29) Project [codegen id : 7] +Output [1]: [ca_address_sk#16] +Input [2]: [ca_address_sk#16, ca_state#17] (30) BroadcastExchange Input [1]: [ca_address_sk#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/simplified.txt index f5b4eccfb..d3b383a2a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/simplified.txt @@ -60,10 +60,10 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter BroadcastExchange #6 WholeStageCodegen (7) - ColumnarToRow - InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_state,ca_address_sk] + Project [ca_address_sk] + Filter [ca_state,ca_address_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/explain.txt index 790a917e3..9b1c53632 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/explain.txt @@ -15,9 +15,9 @@ TakeOrderedAndProject (30) : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- BroadcastExchange (8) - : : : +- * ColumnarToRow (7) - : : : +- CometProject (6) - : : : +- CometFilter (5) + : : : +- * Project (7) + : : : +- * Filter (6) + : : : +- * ColumnarToRow (5) : : : +- CometScan parquet spark_catalog.default.customer_demographics (4) : : +- ReusedExchange (11) : +- BroadcastExchange (17) @@ -25,9 +25,9 @@ TakeOrderedAndProject (30) : +- CometFilter (15) : +- CometScan parquet spark_catalog.default.item (14) +- BroadcastExchange (24) - +- * ColumnarToRow (23) - +- CometProject (22) - +- CometFilter (21) + +- * Project (23) + +- * Filter (22) + +- * ColumnarToRow (21) +- CometScan parquet spark_catalog.default.promotion (20) @@ -50,19 +50,19 @@ Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_p Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_marital_status,S), EqualTo(cd_education_status,College ), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) CometFilter +(5) ColumnarToRow [codegen id : 1] Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (cd_gender#11 = M)) AND (cd_marital_status#12 = S)) AND (cd_education_status#13 = College )) AND isnotnull(cd_demo_sk#10)) -(6) CometProject +(6) Filter [codegen id : 1] Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Arguments: [cd_demo_sk#10], [cd_demo_sk#10] +Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (rpad(cd_gender#11, 1, ) = M)) AND (rpad(cd_marital_status#12, 1, ) = S)) AND (rpad(cd_education_status#13, 20, ) = College )) AND isnotnull(cd_demo_sk#10)) -(7) ColumnarToRow [codegen id : 1] -Input [1]: [cd_demo_sk#10] +(7) Project [codegen id : 1] +Output [1]: [cd_demo_sk#10] +Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] (8) BroadcastExchange Input [1]: [cd_demo_sk#10] @@ -123,19 +123,19 @@ Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sale Output [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [Or(EqualTo(p_channel_email,N),EqualTo(p_channel_event,N)), IsNotNull(p_promo_sk)] +PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(21) CometFilter +(21) ColumnarToRow [codegen id : 4] Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Condition : (((p_channel_email#18 = N) OR (p_channel_event#19 = N)) AND isnotnull(p_promo_sk#17)) -(22) CometProject +(22) Filter [codegen id : 4] Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Arguments: [p_promo_sk#17], [p_promo_sk#17] +Condition : (((rpad(p_channel_email#18, 1, ) = N) OR (rpad(p_channel_event#19, 1, ) = N)) AND isnotnull(p_promo_sk#17)) -(23) ColumnarToRow [codegen id : 4] -Input [1]: [p_promo_sk#17] +(23) Project [codegen id : 4] +Output [1]: [p_promo_sk#17] +Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] (24) BroadcastExchange Input [1]: [p_promo_sk#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/simplified.txt index 2471de20a..80ecf4478 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/simplified.txt @@ -28,10 +28,10 @@ TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [cd_demo_sk] - CometFilter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] + Project [cd_demo_sk] + Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] InputAdapter ReusedExchange [d_date_sk] #2 @@ -45,8 +45,8 @@ TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] InputAdapter BroadcastExchange #5 WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometProject [p_promo_sk] - CometFilter [p_channel_email,p_channel_event,p_promo_sk] + Project [p_promo_sk] + Filter [p_channel_email,p_channel_event,p_promo_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/explain.txt index 624103b66..3facf3c08 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/explain.txt @@ -33,9 +33,9 @@ : : +- CometScan parquet spark_catalog.default.store_sales (18) : +- ReusedExchange (21) +- BroadcastExchange (31) - +- * ColumnarToRow (30) - +- CometProject (29) - +- CometFilter (28) + +- * Project (30) + +- * Filter (29) + +- * ColumnarToRow (28) +- CometScan parquet spark_catalog.default.time_dim (27) @@ -161,19 +161,19 @@ Input [6]: [i_item_sk#1, i_brand_id#2, i_brand#3, ext_price#11, sold_item_sk#12, Output [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] -PushedFilters: [Or(EqualTo(t_meal_time,breakfast ),EqualTo(t_meal_time,dinner )), IsNotNull(t_time_sk)] +PushedFilters: [IsNotNull(t_time_sk)] ReadSchema: struct -(28) CometFilter +(28) ColumnarToRow [codegen id : 8] Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] -Condition : (((t_meal_time#35 = breakfast ) OR (t_meal_time#35 = dinner )) AND isnotnull(t_time_sk#32)) -(29) CometProject +(29) Filter [codegen id : 8] Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] -Arguments: [t_time_sk#32, t_hour#33, t_minute#34], [t_time_sk#32, t_hour#33, t_minute#34] +Condition : (((rpad(t_meal_time#35, 20, ) = breakfast ) OR (rpad(t_meal_time#35, 20, ) = dinner )) AND isnotnull(t_time_sk#32)) -(30) ColumnarToRow [codegen id : 8] -Input [3]: [t_time_sk#32, t_hour#33, t_minute#34] +(30) Project [codegen id : 8] +Output [3]: [t_time_sk#32, t_hour#33, t_minute#34] +Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] (31) BroadcastExchange Input [3]: [t_time_sk#32, t_hour#33, t_minute#34] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/simplified.txt index bea5376a0..eaf36c135 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/simplified.txt @@ -62,8 +62,8 @@ WholeStageCodegen (11) InputAdapter BroadcastExchange #5 WholeStageCodegen (8) - ColumnarToRow - InputAdapter - CometProject [t_time_sk,t_hour,t_minute] - CometFilter [t_meal_time,t_time_sk] + Project [t_time_sk,t_hour,t_minute] + Filter [t_meal_time,t_time_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute,t_meal_time] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/explain.txt index 4e5d9e9f6..6889421e3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/explain.txt @@ -41,14 +41,14 @@ TakeOrderedAndProject (70) : : : : : : : +- CometFilter (17) : : : : : : : +- CometScan parquet spark_catalog.default.item (16) : : : : : : +- BroadcastExchange (26) - : : : : : : +- * ColumnarToRow (25) - : : : : : : +- CometProject (24) - : : : : : : +- CometFilter (23) + : : : : : : +- * Project (25) + : : : : : : +- * Filter (24) + : : : : : : +- * ColumnarToRow (23) : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (22) : : : : : +- BroadcastExchange (33) - : : : : : +- * ColumnarToRow (32) - : : : : : +- CometProject (31) - : : : : : +- CometFilter (30) + : : : : : +- * Project (32) + : : : : : +- * Filter (31) + : : : : : +- * ColumnarToRow (30) : : : : : +- CometScan parquet spark_catalog.default.household_demographics (29) : : : : +- ReusedExchange (36) : : : +- BroadcastExchange (42) @@ -175,19 +175,19 @@ Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_ Output [2]: [cd_demo_sk#18, cd_marital_status#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_marital_status), EqualTo(cd_marital_status,D), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_marital_status), IsNotNull(cd_demo_sk)] ReadSchema: struct -(23) CometFilter +(23) ColumnarToRow [codegen id : 4] Input [2]: [cd_demo_sk#18, cd_marital_status#19] -Condition : ((isnotnull(cd_marital_status#19) AND (cd_marital_status#19 = D)) AND isnotnull(cd_demo_sk#18)) -(24) CometProject +(24) Filter [codegen id : 4] Input [2]: [cd_demo_sk#18, cd_marital_status#19] -Arguments: [cd_demo_sk#18], [cd_demo_sk#18] +Condition : ((isnotnull(cd_marital_status#19) AND (rpad(cd_marital_status#19, 1, ) = D)) AND isnotnull(cd_demo_sk#18)) -(25) ColumnarToRow [codegen id : 4] -Input [1]: [cd_demo_sk#18] +(25) Project [codegen id : 4] +Output [1]: [cd_demo_sk#18] +Input [2]: [cd_demo_sk#18, cd_marital_status#19] (26) BroadcastExchange Input [1]: [cd_demo_sk#18] @@ -207,19 +207,19 @@ Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_ Output [2]: [hd_demo_sk#20, hd_buy_potential#21] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_buy_potential), EqualTo(hd_buy_potential,>10000 ), IsNotNull(hd_demo_sk)] +PushedFilters: [IsNotNull(hd_buy_potential), IsNotNull(hd_demo_sk)] ReadSchema: struct -(30) CometFilter +(30) ColumnarToRow [codegen id : 5] Input [2]: [hd_demo_sk#20, hd_buy_potential#21] -Condition : ((isnotnull(hd_buy_potential#21) AND (hd_buy_potential#21 = >10000 )) AND isnotnull(hd_demo_sk#20)) -(31) CometProject +(31) Filter [codegen id : 5] Input [2]: [hd_demo_sk#20, hd_buy_potential#21] -Arguments: [hd_demo_sk#20], [hd_demo_sk#20] +Condition : ((isnotnull(hd_buy_potential#21) AND (rpad(hd_buy_potential#21, 15, ) = >10000 )) AND isnotnull(hd_demo_sk#20)) -(32) ColumnarToRow [codegen id : 5] -Input [1]: [hd_demo_sk#20] +(32) Project [codegen id : 5] +Output [1]: [hd_demo_sk#20] +Input [2]: [hd_demo_sk#20, hd_buy_potential#21] (33) BroadcastExchange Input [1]: [hd_demo_sk#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/simplified.txt index 17fc9dee7..e41fe84f5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/simplified.txt @@ -67,18 +67,18 @@ TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_prom InputAdapter BroadcastExchange #7 WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometProject [cd_demo_sk] - CometFilter [cd_marital_status,cd_demo_sk] + Project [cd_demo_sk] + Filter [cd_marital_status,cd_demo_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] InputAdapter BroadcastExchange #8 WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometProject [hd_demo_sk] - CometFilter [hd_buy_potential,hd_demo_sk] + Project [hd_demo_sk] + Filter [hd_buy_potential,hd_demo_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] InputAdapter ReusedExchange [d_date_sk,d_date,d_week_seq] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/explain.txt index ef2c38aea..e904b83ef 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/explain.txt @@ -23,9 +23,9 @@ : : +- CometFilter (8) : : +- CometScan parquet spark_catalog.default.store (7) : +- BroadcastExchange (18) - : +- * ColumnarToRow (17) - : +- CometProject (16) - : +- CometFilter (15) + : +- * Project (17) + : +- * Filter (16) + : +- * ColumnarToRow (15) : +- CometScan parquet spark_catalog.default.household_demographics (14) +- BroadcastExchange (28) +- * ColumnarToRow (27) @@ -97,19 +97,19 @@ Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, Output [4]: [hd_demo_sk#10, hd_buy_potential#11, hd_dep_count#12, hd_vehicle_count#13] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_vehicle_count), Or(EqualTo(hd_buy_potential,>10000 ),EqualTo(hd_buy_potential,unknown )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] +PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct -(15) CometFilter +(15) ColumnarToRow [codegen id : 3] Input [4]: [hd_demo_sk#10, hd_buy_potential#11, hd_dep_count#12, hd_vehicle_count#13] -Condition : ((((isnotnull(hd_vehicle_count#13) AND ((hd_buy_potential#11 = >10000 ) OR (hd_buy_potential#11 = unknown ))) AND (hd_vehicle_count#13 > 0)) AND CASE WHEN (hd_vehicle_count#13 > 0) THEN ((cast(hd_dep_count#12 as double) / cast(hd_vehicle_count#13 as double)) > 1.0) END) AND isnotnull(hd_demo_sk#10)) -(16) CometProject +(16) Filter [codegen id : 3] Input [4]: [hd_demo_sk#10, hd_buy_potential#11, hd_dep_count#12, hd_vehicle_count#13] -Arguments: [hd_demo_sk#10], [hd_demo_sk#10] +Condition : ((((isnotnull(hd_vehicle_count#13) AND ((rpad(hd_buy_potential#11, 15, ) = >10000 ) OR (rpad(hd_buy_potential#11, 15, ) = unknown ))) AND (hd_vehicle_count#13 > 0)) AND CASE WHEN (hd_vehicle_count#13 > 0) THEN ((cast(hd_dep_count#12 as double) / cast(hd_vehicle_count#13 as double)) > 1.0) END) AND isnotnull(hd_demo_sk#10)) -(17) ColumnarToRow [codegen id : 3] -Input [1]: [hd_demo_sk#10] +(17) Project [codegen id : 3] +Output [1]: [hd_demo_sk#10] +Input [4]: [hd_demo_sk#10, hd_buy_potential#11, hd_dep_count#12, hd_vehicle_count#13] (18) BroadcastExchange Input [1]: [hd_demo_sk#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/simplified.txt index 7c5ee1ef5..6e347d640 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/simplified.txt @@ -42,10 +42,10 @@ WholeStageCodegen (7) InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [hd_demo_sk] - CometFilter [hd_vehicle_count,hd_buy_potential,hd_dep_count,hd_demo_sk] + Project [hd_demo_sk] + Filter [hd_vehicle_count,hd_buy_potential,hd_dep_count,hd_demo_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] InputAdapter BroadcastExchange #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/explain.txt index a3d12a1d4..180048192 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/explain.txt @@ -24,9 +24,9 @@ TakeOrderedAndProject (129) : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : : : : +- BroadcastExchange (8) - : : : : +- * ColumnarToRow (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) + : : : : +- * Project (7) + : : : : +- * Filter (6) + : : : : +- * ColumnarToRow (5) : : : : +- CometScan parquet spark_catalog.default.item (4) : : : +- ReusedExchange (11) : : +- * Sort (21) @@ -149,19 +149,19 @@ Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4 Output [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Books ), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] +PushedFilters: [IsNotNull(i_category), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] ReadSchema: struct -(5) CometFilter +(5) ColumnarToRow [codegen id : 1] Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] -Condition : ((((((isnotnull(i_category#11) AND (i_category#11 = Books )) AND isnotnull(i_item_sk#7)) AND isnotnull(i_brand_id#8)) AND isnotnull(i_class_id#9)) AND isnotnull(i_category_id#10)) AND isnotnull(i_manufact_id#12)) -(6) CometProject +(6) Filter [codegen id : 1] Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] -Arguments: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12], [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +Condition : ((((((isnotnull(i_category#11) AND (rpad(i_category#11, 50, ) = Books )) AND isnotnull(i_item_sk#7)) AND isnotnull(i_brand_id#8)) AND isnotnull(i_class_id#9)) AND isnotnull(i_category_id#10)) AND isnotnull(i_manufact_id#12)) -(7) ColumnarToRow [codegen id : 1] -Input [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +(7) Project [codegen id : 1] +Output [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] (8) BroadcastExchange Input [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/simplified.txt index 44bcabcdb..b36c0d7b2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/simplified.txt @@ -48,10 +48,10 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat InputAdapter BroadcastExchange #6 WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [i_category,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + Project [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + Filter [i_category,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] InputAdapter ReusedExchange [d_date_sk,d_year] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/explain.txt index 7d6d717c1..1c4943fe7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/explain.txt @@ -38,9 +38,9 @@ TakeOrderedAndProject (43) : +- CometFilter (18) : +- CometScan parquet spark_catalog.default.customer_address (17) +- BroadcastExchange (24) - +- * ColumnarToRow (23) - +- CometProject (22) - +- CometFilter (21) + +- * Project (23) + +- * Filter (22) + +- * ColumnarToRow (21) +- CometScan parquet spark_catalog.default.customer (20) @@ -135,19 +135,19 @@ Input [2]: [ca_address_sk#11, ca_zip#12] Output [2]: [c_current_addr_sk#13, c_preferred_cust_flag#14] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_preferred_cust_flag), EqualTo(c_preferred_cust_flag,Y), IsNotNull(c_current_addr_sk)] +PushedFilters: [IsNotNull(c_preferred_cust_flag), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(21) CometFilter +(21) ColumnarToRow [codegen id : 3] Input [2]: [c_current_addr_sk#13, c_preferred_cust_flag#14] -Condition : ((isnotnull(c_preferred_cust_flag#14) AND (c_preferred_cust_flag#14 = Y)) AND isnotnull(c_current_addr_sk#13)) -(22) CometProject +(22) Filter [codegen id : 3] Input [2]: [c_current_addr_sk#13, c_preferred_cust_flag#14] -Arguments: [c_current_addr_sk#13], [c_current_addr_sk#13] +Condition : ((isnotnull(c_preferred_cust_flag#14) AND (rpad(c_preferred_cust_flag#14, 1, ) = Y)) AND isnotnull(c_current_addr_sk#13)) -(23) ColumnarToRow [codegen id : 3] -Input [1]: [c_current_addr_sk#13] +(23) Project [codegen id : 3] +Output [1]: [c_current_addr_sk#13] +Input [2]: [c_current_addr_sk#13, c_preferred_cust_flag#14] (24) BroadcastExchange Input [1]: [c_current_addr_sk#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/simplified.txt index 76fa27693..ec57b40fb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/simplified.txt @@ -65,8 +65,8 @@ TakeOrderedAndProject [s_store_name,sum(ss_net_profit)] InputAdapter BroadcastExchange #8 WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [c_current_addr_sk] - CometFilter [c_preferred_cust_flag,c_current_addr_sk] + Project [c_current_addr_sk] + Filter [c_preferred_cust_flag,c_current_addr_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/explain.txt index db2d015db..48b9c4607 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/explain.txt @@ -40,9 +40,9 @@ TakeOrderedAndProject (107) : : +- CometFilter (24) : : +- CometScan parquet spark_catalog.default.item (23) : +- BroadcastExchange (34) - : +- * ColumnarToRow (33) - : +- CometProject (32) - : +- CometFilter (31) + : +- * Project (33) + : +- * Filter (32) + : +- * ColumnarToRow (31) : +- CometScan parquet spark_catalog.default.promotion (30) :- * HashAggregate (70) : +- Exchange (69) @@ -244,19 +244,19 @@ Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, Output [2]: [p_promo_sk#19, p_channel_tv#20] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [IsNotNull(p_channel_tv), EqualTo(p_channel_tv,N), IsNotNull(p_promo_sk)] +PushedFilters: [IsNotNull(p_channel_tv), IsNotNull(p_promo_sk)] ReadSchema: struct -(31) CometFilter +(31) ColumnarToRow [codegen id : 8] Input [2]: [p_promo_sk#19, p_channel_tv#20] -Condition : ((isnotnull(p_channel_tv#20) AND (p_channel_tv#20 = N)) AND isnotnull(p_promo_sk#19)) -(32) CometProject +(32) Filter [codegen id : 8] Input [2]: [p_promo_sk#19, p_channel_tv#20] -Arguments: [p_promo_sk#19], [p_promo_sk#19] +Condition : ((isnotnull(p_channel_tv#20) AND (rpad(p_channel_tv#20, 1, ) = N)) AND isnotnull(p_promo_sk#19)) -(33) ColumnarToRow [codegen id : 8] -Input [1]: [p_promo_sk#19] +(33) Project [codegen id : 8] +Output [1]: [p_promo_sk#19] +Input [2]: [p_promo_sk#19, p_channel_tv#20] (34) BroadcastExchange Input [1]: [p_promo_sk#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/simplified.txt index 7e257bdc6..dabe9b01a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/simplified.txt @@ -73,10 +73,10 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter BroadcastExchange #8 WholeStageCodegen (8) - ColumnarToRow - InputAdapter - CometProject [p_promo_sk] - CometFilter [p_channel_tv,p_promo_sk] + Project [p_promo_sk] + Filter [p_channel_tv,p_promo_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] WholeStageCodegen (20) HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/explain.txt index 431d1453d..eca327cf8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/explain.txt @@ -44,8 +44,8 @@ TakeOrderedAndProject (48) : +- CometFilter (37) : +- CometScan parquet spark_catalog.default.customer (36) +- BroadcastExchange (45) - +- * ColumnarToRow (44) - +- CometFilter (43) + +- * Filter (44) + +- * ColumnarToRow (43) +- CometScan parquet spark_catalog.default.customer_address (42) @@ -254,15 +254,15 @@ Input [8]: [ctr_customer_sk#12, ctr_total_return#14, c_customer_sk#33, c_custome Output [12]: [ca_address_sk#39, ca_street_number#40, ca_street_name#41, ca_street_type#42, ca_suite_number#43, ca_city#44, ca_county#45, ca_state#46, ca_zip#47, ca_country#48, ca_gmt_offset#49, ca_location_type#50] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_state), IsNotNull(ca_address_sk)] ReadSchema: struct -(43) CometFilter +(43) ColumnarToRow [codegen id : 10] Input [12]: [ca_address_sk#39, ca_street_number#40, ca_street_name#41, ca_street_type#42, ca_suite_number#43, ca_city#44, ca_county#45, ca_state#46, ca_zip#47, ca_country#48, ca_gmt_offset#49, ca_location_type#50] -Condition : ((isnotnull(ca_state#46) AND (ca_state#46 = GA)) AND isnotnull(ca_address_sk#39)) -(44) ColumnarToRow [codegen id : 10] +(44) Filter [codegen id : 10] Input [12]: [ca_address_sk#39, ca_street_number#40, ca_street_name#41, ca_street_type#42, ca_suite_number#43, ca_city#44, ca_county#45, ca_state#46, ca_zip#47, ca_country#48, ca_gmt_offset#49, ca_location_type#50] +Condition : ((isnotnull(ca_state#46) AND (rpad(ca_state#46, 2, ) = GA)) AND isnotnull(ca_address_sk#39)) (45) BroadcastExchange Input [12]: [ca_address_sk#39, ca_street_number#40, ca_street_name#41, ca_street_type#42, ca_suite_number#43, ca_city#44, ca_county#45, ca_state#46, ca_zip#47, ca_country#48, ca_gmt_offset#49, ca_location_type#50] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/simplified.txt index 6f042847c..1585ae3e1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/simplified.txt @@ -74,7 +74,7 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_st InputAdapter BroadcastExchange #8 WholeStageCodegen (10) - ColumnarToRow - InputAdapter - CometFilter [ca_state,ca_address_sk] + Filter [ca_state,ca_address_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/explain.txt index ce1cc2262..1bf8d338a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/explain.txt @@ -29,17 +29,17 @@ TakeOrderedAndProject (47) : : : : : +- CometFilter (11) : : : : : +- CometScan parquet spark_catalog.default.web_page (10) : : : : +- BroadcastExchange (19) - : : : : +- * ColumnarToRow (18) - : : : : +- CometFilter (17) + : : : : +- * Filter (18) + : : : : +- * ColumnarToRow (17) : : : : +- CometScan parquet spark_catalog.default.customer_demographics (16) : : : +- BroadcastExchange (25) : : : +- * ColumnarToRow (24) : : : +- CometFilter (23) : : : +- CometScan parquet spark_catalog.default.customer_demographics (22) : : +- BroadcastExchange (32) - : : +- * ColumnarToRow (31) - : : +- CometProject (30) - : : +- CometFilter (29) + : : +- * Project (31) + : : +- * Filter (30) + : : +- * ColumnarToRow (29) : : +- CometScan parquet spark_catalog.default.customer_address (28) : +- ReusedExchange (35) +- BroadcastExchange (41) @@ -123,15 +123,15 @@ Input [12]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Advanced Degree )),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College ))),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,2 yr Degree )))] +PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status), IsNotNull(cd_education_status)] ReadSchema: struct -(17) CometFilter +(17) ColumnarToRow [codegen id : 2] Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] -Condition : (((isnotnull(cd_demo_sk#19) AND isnotnull(cd_marital_status#20)) AND isnotnull(cd_education_status#21)) AND ((((cd_marital_status#20 = M) AND (cd_education_status#21 = Advanced Degree )) OR ((cd_marital_status#20 = S) AND (cd_education_status#21 = College ))) OR ((cd_marital_status#20 = W) AND (cd_education_status#21 = 2 yr Degree )))) -(18) ColumnarToRow [codegen id : 2] +(18) Filter [codegen id : 2] Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Condition : (((isnotnull(cd_demo_sk#19) AND isnotnull(cd_marital_status#20)) AND isnotnull(cd_education_status#21)) AND ((((rpad(cd_marital_status#20, 1, ) = M) AND (rpad(cd_education_status#21, 20, ) = Advanced Degree )) OR ((rpad(cd_marital_status#20, 1, ) = S) AND (rpad(cd_education_status#21, 20, ) = College ))) OR ((rpad(cd_marital_status#20, 1, ) = W) AND (rpad(cd_education_status#21, 20, ) = 2 yr Degree )))) (19) BroadcastExchange Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] @@ -141,7 +141,7 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint) Left keys [1]: [wr_refunded_cdemo_sk#10] Right keys [1]: [cd_demo_sk#19] Join type: Inner -Join condition: ((((((cd_marital_status#20 = M) AND (cd_education_status#21 = Advanced Degree )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#20 = S) AND (cd_education_status#21 = College )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#20 = W) AND (cd_education_status#21 = 2 yr Degree )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00))) +Join condition: ((((((rpad(cd_marital_status#20, 1, ) = M) AND (rpad(cd_education_status#21, 20, ) = Advanced Degree )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((rpad(cd_marital_status#20, 1, ) = S) AND (rpad(cd_education_status#21, 20, ) = College )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((rpad(cd_marital_status#20, 1, ) = W) AND (rpad(cd_education_status#21, 20, ) = 2 yr Degree )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00))) (21) Project [codegen id : 7] Output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#20, cd_education_status#21] @@ -179,19 +179,19 @@ Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr Output [3]: [ca_address_sk#25, ca_state#26, ca_country#27] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [IN,NJ,OH]),In(ca_state, [CT,KY,WI])),In(ca_state, [AR,IA,LA]))] +PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] ReadSchema: struct -(29) CometFilter +(29) ColumnarToRow [codegen id : 4] Input [3]: [ca_address_sk#25, ca_state#26, ca_country#27] -Condition : (((isnotnull(ca_country#27) AND (ca_country#27 = United States)) AND isnotnull(ca_address_sk#25)) AND ((ca_state#26 IN (IN,OH,NJ) OR ca_state#26 IN (WI,CT,KY)) OR ca_state#26 IN (LA,IA,AR))) -(30) CometProject +(30) Filter [codegen id : 4] Input [3]: [ca_address_sk#25, ca_state#26, ca_country#27] -Arguments: [ca_address_sk#25, ca_state#26], [ca_address_sk#25, ca_state#26] +Condition : (((isnotnull(ca_country#27) AND (ca_country#27 = United States)) AND isnotnull(ca_address_sk#25)) AND ((rpad(ca_state#26, 2, ) IN (IN,OH,NJ) OR rpad(ca_state#26, 2, ) IN (WI,CT,KY)) OR rpad(ca_state#26, 2, ) IN (LA,IA,AR))) -(31) ColumnarToRow [codegen id : 4] -Input [2]: [ca_address_sk#25, ca_state#26] +(31) Project [codegen id : 4] +Output [2]: [ca_address_sk#25, ca_state#26] +Input [3]: [ca_address_sk#25, ca_state#26, ca_country#27] (32) BroadcastExchange Input [2]: [ca_address_sk#25, ca_state#26] @@ -201,7 +201,7 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Left keys [1]: [wr_refunded_addr_sk#11] Right keys [1]: [ca_address_sk#25] Join type: Inner -Join condition: ((((ca_state#26 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#26 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#26 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00))) +Join condition: ((((rpad(ca_state#26, 2, ) IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((rpad(ca_state#26, 2, ) IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((rpad(ca_state#26, 2, ) IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00))) (34) Project [codegen id : 7] Output [5]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/simplified.txt index 7c5ee727b..7b6e44fee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/simplified.txt @@ -45,9 +45,9 @@ TakeOrderedAndProject [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refu InputAdapter BroadcastExchange #5 WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + Filter [cd_demo_sk,cd_marital_status,cd_education_status] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] InputAdapter BroadcastExchange #6 @@ -59,10 +59,10 @@ TakeOrderedAndProject [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refu InputAdapter BroadcastExchange #7 WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometProject [ca_address_sk,ca_state] - CometFilter [ca_country,ca_address_sk,ca_state] + Project [ca_address_sk,ca_state] + Filter [ca_country,ca_address_sk,ca_state] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] InputAdapter ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/explain.txt index 3657266e2..37112b309 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/explain.txt @@ -14,8 +14,8 @@ TakeOrderedAndProject (27) : +- * BroadcastHashJoin Inner BuildRight (11) : :- * Project (9) : : +- * BroadcastHashJoin Inner BuildRight (8) - : : :- * ColumnarToRow (3) - : : : +- CometFilter (2) + : : :- * Filter (3) + : : : +- * ColumnarToRow (2) : : : +- CometScan parquet spark_catalog.default.item (1) : : +- BroadcastExchange (7) : : +- * ColumnarToRow (6) @@ -32,15 +32,15 @@ TakeOrderedAndProject (27) Output [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [Or(And(In(i_category, [Books ,Electronics ,Sports ]),In(i_class, [computers ,football ,stereo ])),And(In(i_category, [Jewelry ,Men ,Women ]),In(i_class, [birdal ,dresses ,shirts ]))), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(2) CometFilter +(2) ColumnarToRow [codegen id : 4] Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] -Condition : (((i_category#4 IN (Books ,Electronics ,Sports ) AND i_class#3 IN (computers ,stereo ,football )) OR (i_category#4 IN (Men ,Jewelry ,Women ) AND i_class#3 IN (shirts ,birdal ,dresses ))) AND isnotnull(i_item_sk#1)) -(3) ColumnarToRow [codegen id : 4] +(3) Filter [codegen id : 4] Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] +Condition : (((rpad(i_category#4, 50, ) IN (Books ,Electronics ,Sports ) AND rpad(i_class#3, 50, ) IN (computers ,stereo ,football )) OR (rpad(i_category#4, 50, ) IN (Men ,Jewelry ,Women ) AND rpad(i_class#3, 50, ) IN (shirts ,birdal ,dresses ))) AND isnotnull(i_item_sk#1)) (4) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/simplified.txt index bb9e4e17e..cc3be75c2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/simplified.txt @@ -20,9 +20,9 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_cla BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] BroadcastHashJoin [i_item_sk,ss_item_sk] - ColumnarToRow - InputAdapter - CometFilter [i_category,i_class,i_item_sk] + Filter [i_category,i_class,i_item_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] InputAdapter BroadcastExchange #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/explain.txt index 61f35489a..68755ad44 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/explain.txt @@ -34,8 +34,8 @@ : : +- CometFilter (20) : : +- CometScan parquet spark_catalog.default.customer_address (19) : +- BroadcastExchange (29) - : +- * ColumnarToRow (28) - : +- CometFilter (27) + : +- * Filter (28) + : +- * ColumnarToRow (27) : +- CometScan parquet spark_catalog.default.customer_demographics (26) +- BroadcastExchange (36) +- * ColumnarToRow (35) @@ -164,15 +164,15 @@ Input [8]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_curre Output [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Unknown )),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,Advanced Degree ))), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(27) CometFilter +(27) ColumnarToRow [codegen id : 5] Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Condition : ((((cd_marital_status#18 = M) AND (cd_education_status#19 = Unknown )) OR ((cd_marital_status#18 = W) AND (cd_education_status#19 = Advanced Degree ))) AND isnotnull(cd_demo_sk#17)) -(28) ColumnarToRow [codegen id : 5] +(28) Filter [codegen id : 5] Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] +Condition : ((((rpad(cd_marital_status#18, 1, ) = M) AND (rpad(cd_education_status#19, 20, ) = Unknown )) OR ((rpad(cd_marital_status#18, 1, ) = W) AND (rpad(cd_education_status#19, 20, ) = Advanced Degree ))) AND isnotnull(cd_demo_sk#17)) (29) BroadcastExchange Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/simplified.txt index e5d62e3c0..89e030fdc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/simplified.txt @@ -59,9 +59,9 @@ WholeStageCodegen (9) InputAdapter BroadcastExchange #7 WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometFilter [cd_marital_status,cd_education_status,cd_demo_sk] + Filter [cd_marital_status,cd_education_status,cd_demo_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] InputAdapter BroadcastExchange #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/explain.txt index 00ed822f2..a68d6ca1a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/explain.txt @@ -19,9 +19,9 @@ TakeOrderedAndProject (24) : +- CometFilter (7) : +- CometScan parquet spark_catalog.default.store_returns (6) +- BroadcastExchange (18) - +- * ColumnarToRow (17) - +- CometProject (16) - +- CometFilter (15) + +- * Project (17) + +- * Filter (16) + +- * ColumnarToRow (15) +- CometScan parquet spark_catalog.default.reason (14) @@ -86,19 +86,19 @@ Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, s Output [2]: [r_reason_sk#12, r_reason_desc#13] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] -PushedFilters: [IsNotNull(r_reason_desc), EqualTo(r_reason_desc,reason 28 ), IsNotNull(r_reason_sk)] +PushedFilters: [IsNotNull(r_reason_desc), IsNotNull(r_reason_sk)] ReadSchema: struct -(15) CometFilter +(15) ColumnarToRow [codegen id : 5] Input [2]: [r_reason_sk#12, r_reason_desc#13] -Condition : ((isnotnull(r_reason_desc#13) AND (r_reason_desc#13 = reason 28 )) AND isnotnull(r_reason_sk#12)) -(16) CometProject +(16) Filter [codegen id : 5] Input [2]: [r_reason_sk#12, r_reason_desc#13] -Arguments: [r_reason_sk#12], [r_reason_sk#12] +Condition : ((isnotnull(r_reason_desc#13) AND (rpad(r_reason_desc#13, 100, ) = reason 28 )) AND isnotnull(r_reason_sk#12)) -(17) ColumnarToRow [codegen id : 5] -Input [1]: [r_reason_sk#12] +(17) Project [codegen id : 5] +Output [1]: [r_reason_sk#12] +Input [2]: [r_reason_sk#12, r_reason_desc#13] (18) BroadcastExchange Input [1]: [r_reason_sk#12] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/simplified.txt index 3ec7ac7b6..a212e9107 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/simplified.txt @@ -33,8 +33,8 @@ TakeOrderedAndProject [sumsales,ss_customer_sk] InputAdapter BroadcastExchange #4 WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometProject [r_reason_sk] - CometFilter [r_reason_desc,r_reason_sk] + Project [r_reason_sk] + Filter [r_reason_desc,r_reason_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/explain.txt index d71f96e15..ad1eddae5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/explain.txt @@ -35,14 +35,14 @@ : : +- CometFilter (21) : : +- CometScan parquet spark_catalog.default.date_dim (20) : +- BroadcastExchange (31) - : +- * ColumnarToRow (30) - : +- CometProject (29) - : +- CometFilter (28) + : +- * Project (30) + : +- * Filter (29) + : +- * ColumnarToRow (28) : +- CometScan parquet spark_catalog.default.customer_address (27) +- BroadcastExchange (38) - +- * ColumnarToRow (37) - +- CometProject (36) - +- CometFilter (35) + +- * Project (37) + +- * Filter (36) + +- * ColumnarToRow (35) +- CometScan parquet spark_catalog.default.web_site (34) @@ -166,19 +166,19 @@ Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_num Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,IL), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_state), IsNotNull(ca_address_sk)] ReadSchema: struct -(28) CometFilter +(28) ColumnarToRow [codegen id : 9] Input [2]: [ca_address_sk#16, ca_state#17] -Condition : ((isnotnull(ca_state#17) AND (ca_state#17 = IL)) AND isnotnull(ca_address_sk#16)) -(29) CometProject +(29) Filter [codegen id : 9] Input [2]: [ca_address_sk#16, ca_state#17] -Arguments: [ca_address_sk#16], [ca_address_sk#16] +Condition : ((isnotnull(ca_state#17) AND (rpad(ca_state#17, 2, ) = IL)) AND isnotnull(ca_address_sk#16)) -(30) ColumnarToRow [codegen id : 9] -Input [1]: [ca_address_sk#16] +(30) Project [codegen id : 9] +Output [1]: [ca_address_sk#16] +Input [2]: [ca_address_sk#16, ca_state#17] (31) BroadcastExchange Input [1]: [ca_address_sk#16] @@ -198,19 +198,19 @@ Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_ Output [2]: [web_site_sk#18, web_company_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] -PushedFilters: [IsNotNull(web_company_name), EqualTo(web_company_name,pri ), IsNotNull(web_site_sk)] +PushedFilters: [IsNotNull(web_company_name), IsNotNull(web_site_sk)] ReadSchema: struct -(35) CometFilter +(35) ColumnarToRow [codegen id : 10] Input [2]: [web_site_sk#18, web_company_name#19] -Condition : ((isnotnull(web_company_name#19) AND (web_company_name#19 = pri )) AND isnotnull(web_site_sk#18)) -(36) CometProject +(36) Filter [codegen id : 10] Input [2]: [web_site_sk#18, web_company_name#19] -Arguments: [web_site_sk#18], [web_site_sk#18] +Condition : ((isnotnull(web_company_name#19) AND (rpad(web_company_name#19, 50, ) = pri )) AND isnotnull(web_site_sk#18)) -(37) ColumnarToRow [codegen id : 10] -Input [1]: [web_site_sk#18] +(37) Project [codegen id : 10] +Output [1]: [web_site_sk#18] +Input [2]: [web_site_sk#18, web_company_name#19] (38) BroadcastExchange Input [1]: [web_site_sk#18] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/simplified.txt index 34ddde768..21f68f004 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/simplified.txt @@ -59,16 +59,16 @@ WholeStageCodegen (12) InputAdapter BroadcastExchange #6 WholeStageCodegen (9) - ColumnarToRow - InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_state,ca_address_sk] + Project [ca_address_sk] + Filter [ca_state,ca_address_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #7 WholeStageCodegen (10) - ColumnarToRow - InputAdapter - CometProject [web_site_sk] - CometFilter [web_company_name,web_site_sk] + Project [web_site_sk] + Filter [web_company_name,web_site_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/explain.txt index c8cdce055..8376c2cd0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/explain.txt @@ -48,14 +48,14 @@ : : +- CometFilter (34) : : +- CometScan parquet spark_catalog.default.date_dim (33) : +- BroadcastExchange (44) - : +- * ColumnarToRow (43) - : +- CometProject (42) - : +- CometFilter (41) + : +- * Project (43) + : +- * Filter (42) + : +- * ColumnarToRow (41) : +- CometScan parquet spark_catalog.default.customer_address (40) +- BroadcastExchange (51) - +- * ColumnarToRow (50) - +- CometProject (49) - +- CometFilter (48) + +- * Project (50) + +- * Filter (49) + +- * ColumnarToRow (48) +- CometScan parquet spark_catalog.default.web_site (47) @@ -236,19 +236,19 @@ Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_num Output [2]: [ca_address_sk#21, ca_state#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,IL), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_state), IsNotNull(ca_address_sk)] ReadSchema: struct -(41) CometFilter +(41) ColumnarToRow [codegen id : 18] Input [2]: [ca_address_sk#21, ca_state#22] -Condition : ((isnotnull(ca_state#22) AND (ca_state#22 = IL)) AND isnotnull(ca_address_sk#21)) -(42) CometProject +(42) Filter [codegen id : 18] Input [2]: [ca_address_sk#21, ca_state#22] -Arguments: [ca_address_sk#21], [ca_address_sk#21] +Condition : ((isnotnull(ca_state#22) AND (rpad(ca_state#22, 2, ) = IL)) AND isnotnull(ca_address_sk#21)) -(43) ColumnarToRow [codegen id : 18] -Input [1]: [ca_address_sk#21] +(43) Project [codegen id : 18] +Output [1]: [ca_address_sk#21] +Input [2]: [ca_address_sk#21, ca_state#22] (44) BroadcastExchange Input [1]: [ca_address_sk#21] @@ -268,19 +268,19 @@ Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_ Output [2]: [web_site_sk#23, web_company_name#24] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] -PushedFilters: [IsNotNull(web_company_name), EqualTo(web_company_name,pri ), IsNotNull(web_site_sk)] +PushedFilters: [IsNotNull(web_company_name), IsNotNull(web_site_sk)] ReadSchema: struct -(48) CometFilter +(48) ColumnarToRow [codegen id : 19] Input [2]: [web_site_sk#23, web_company_name#24] -Condition : ((isnotnull(web_company_name#24) AND (web_company_name#24 = pri )) AND isnotnull(web_site_sk#23)) -(49) CometProject +(49) Filter [codegen id : 19] Input [2]: [web_site_sk#23, web_company_name#24] -Arguments: [web_site_sk#23], [web_site_sk#23] +Condition : ((isnotnull(web_company_name#24) AND (rpad(web_company_name#24, 50, ) = pri )) AND isnotnull(web_site_sk#23)) -(50) ColumnarToRow [codegen id : 19] -Input [1]: [web_site_sk#23] +(50) Project [codegen id : 19] +Output [1]: [web_site_sk#23] +Input [2]: [web_site_sk#23, web_company_name#24] (51) BroadcastExchange Input [1]: [web_site_sk#23] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/simplified.txt index 5b699890c..c6e416c28 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/simplified.txt @@ -87,16 +87,16 @@ WholeStageCodegen (21) InputAdapter BroadcastExchange #6 WholeStageCodegen (18) - ColumnarToRow - InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_state,ca_address_sk] + Project [ca_address_sk] + Filter [ca_state,ca_address_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #7 WholeStageCodegen (19) - ColumnarToRow - InputAdapter - CometProject [web_site_sk] - CometFilter [web_company_name,web_site_sk] + Project [web_site_sk] + Filter [web_company_name,web_site_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/explain.txt index 3d66a07d0..a3f56fa79 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/explain.txt @@ -17,8 +17,8 @@ : : +- CometFilter (2) : : +- CometScan parquet spark_catalog.default.store_sales (1) : +- BroadcastExchange (7) - : +- * ColumnarToRow (6) - : +- CometFilter (5) + : +- * Filter (6) + : +- * ColumnarToRow (5) : +- CometScan parquet spark_catalog.default.item (4) +- ReusedExchange (10) @@ -42,15 +42,15 @@ Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) CometFilter +(5) ColumnarToRow [codegen id : 1] Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (i_category#10 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) -(6) ColumnarToRow [codegen id : 1] +(6) Filter [codegen id : 1] Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Condition : (rpad(i_category#10, 50, ) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) (7) BroadcastExchange Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/simplified.txt index 9eabb9977..7dff38b11 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/simplified.txt @@ -36,9 +36,9 @@ WholeStageCodegen (7) InputAdapter BroadcastExchange #5 WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [i_category,i_item_sk] + Filter [i_category,i_item_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] InputAdapter ReusedExchange [d_date_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/explain.txt index c39a71879..47477775b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/explain.txt @@ -15,8 +15,8 @@ TakeOrderedAndProject (20) : : +- CometFilter (2) : : +- CometScan parquet spark_catalog.default.web_sales (1) : +- BroadcastExchange (7) - : +- * ColumnarToRow (6) - : +- CometFilter (5) + : +- * Filter (6) + : +- * ColumnarToRow (5) : +- CometScan parquet spark_catalog.default.item (4) +- ReusedExchange (10) @@ -40,15 +40,15 @@ Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) CometFilter +(5) ColumnarToRow [codegen id : 1] Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (i_category#10 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) -(6) ColumnarToRow [codegen id : 1] +(6) Filter [codegen id : 1] Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Condition : (rpad(i_category#10, 50, ) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) (7) BroadcastExchange Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/simplified.txt index 1bc2538b4..14c8825ab 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/simplified.txt @@ -32,9 +32,9 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter BroadcastExchange #4 WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [i_category,i_item_sk] + Filter [i_category,i_item_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] InputAdapter ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/explain.txt index 8bcc0c6ba..7e3f5d500 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/explain.txt @@ -20,9 +20,9 @@ TakeOrderedAndProject (153) : : : : : : : +- CometFilter (2) : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : : : : : : +- BroadcastExchange (8) - : : : : : : +- * ColumnarToRow (7) - : : : : : : +- CometProject (6) - : : : : : : +- CometFilter (5) + : : : : : : +- * Project (7) + : : : : : : +- * Filter (6) + : : : : : : +- * ColumnarToRow (5) : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (4) : : : : : +- BroadcastExchange (15) : : : : : +- * ColumnarToRow (14) @@ -34,8 +34,8 @@ TakeOrderedAndProject (153) : : : : +- CometFilter (19) : : : : +- CometScan parquet spark_catalog.default.customer_demographics (18) : : : +- BroadcastExchange (27) - : : : +- * ColumnarToRow (26) - : : : +- CometFilter (25) + : : : +- * Filter (26) + : : : +- * ColumnarToRow (25) : : : +- CometScan parquet spark_catalog.default.customer_address (24) : : +- ReusedExchange (30) : +- BroadcastExchange (36) @@ -64,8 +64,8 @@ TakeOrderedAndProject (153) : : : : : +- ReusedExchange (48) : : : : +- ReusedExchange (51) : : : +- BroadcastExchange (57) - : : : +- * ColumnarToRow (56) - : : : +- CometFilter (55) + : : : +- * Filter (56) + : : : +- * ColumnarToRow (55) : : : +- CometScan parquet spark_catalog.default.customer_address (54) : : +- ReusedExchange (60) : +- ReusedExchange (63) @@ -91,9 +91,9 @@ TakeOrderedAndProject (153) : : : : : +- ReusedExchange (75) : : : : +- ReusedExchange (78) : : : +- BroadcastExchange (85) - : : : +- * ColumnarToRow (84) - : : : +- CometProject (83) - : : : +- CometFilter (82) + : : : +- * Project (84) + : : : +- * Filter (83) + : : : +- * ColumnarToRow (82) : : : +- CometScan parquet spark_catalog.default.customer_address (81) : : +- ReusedExchange (88) : +- ReusedExchange (91) @@ -119,9 +119,9 @@ TakeOrderedAndProject (153) : : : : : +- ReusedExchange (103) : : : : +- ReusedExchange (106) : : : +- BroadcastExchange (113) - : : : +- * ColumnarToRow (112) - : : : +- CometProject (111) - : : : +- CometFilter (110) + : : : +- * Project (112) + : : : +- * Filter (111) + : : : +- * ColumnarToRow (110) : : : +- CometScan parquet spark_catalog.default.customer_address (109) : : +- ReusedExchange (116) : +- ReusedExchange (119) @@ -173,19 +173,19 @@ Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity Output [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_education_status,College ), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) CometFilter +(5) ColumnarToRow [codegen id : 1] Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] -Condition : ((((isnotnull(cd_gender#12) AND isnotnull(cd_education_status#13)) AND (cd_gender#12 = M)) AND (cd_education_status#13 = College )) AND isnotnull(cd_demo_sk#11)) -(6) CometProject +(6) Filter [codegen id : 1] Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] -Arguments: [cd_demo_sk#11, cd_dep_count#14], [cd_demo_sk#11, cd_dep_count#14] +Condition : ((((isnotnull(cd_gender#12) AND isnotnull(cd_education_status#13)) AND (rpad(cd_gender#12, 1, ) = M)) AND (rpad(cd_education_status#13, 20, ) = College )) AND isnotnull(cd_demo_sk#11)) -(7) ColumnarToRow [codegen id : 1] -Input [2]: [cd_demo_sk#11, cd_dep_count#14] +(7) Project [codegen id : 1] +Output [2]: [cd_demo_sk#11, cd_dep_count#14] +Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] (8) BroadcastExchange Input [2]: [cd_demo_sk#11, cd_dep_count#14] @@ -265,15 +265,15 @@ Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_ Output [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(25) CometFilter +(25) ColumnarToRow [codegen id : 4] Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] -Condition : (ca_state#23 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#21)) -(26) ColumnarToRow [codegen id : 4] +(26) Filter [codegen id : 4] Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] +Condition : (rpad(ca_state#23, 2, ) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#21)) (27) BroadcastExchange Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] @@ -406,15 +406,15 @@ Input [12]: [cs_item_sk#79, cs_quantity#80, cs_list_price#81, cs_sales_price#82, Output [3]: [ca_address_sk#94, ca_state#95, ca_country#96] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(55) CometFilter +(55) ColumnarToRow [codegen id : 12] Input [3]: [ca_address_sk#94, ca_state#95, ca_country#96] -Condition : (ca_state#95 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#94)) -(56) ColumnarToRow [codegen id : 12] +(56) Filter [codegen id : 12] Input [3]: [ca_address_sk#94, ca_state#95, ca_country#96] +Condition : (rpad(ca_state#95, 2, ) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#94)) (57) BroadcastExchange Input [3]: [ca_address_sk#94, ca_state#95, ca_country#96] @@ -532,19 +532,19 @@ Input [12]: [cs_item_sk#152, cs_quantity#153, cs_list_price#154, cs_sales_price# Output [3]: [ca_address_sk#167, ca_state#168, ca_country#169] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(82) CometFilter +(82) ColumnarToRow [codegen id : 20] Input [3]: [ca_address_sk#167, ca_state#168, ca_country#169] -Condition : (ca_state#168 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#167)) -(83) CometProject +(83) Filter [codegen id : 20] Input [3]: [ca_address_sk#167, ca_state#168, ca_country#169] -Arguments: [ca_address_sk#167, ca_country#169], [ca_address_sk#167, ca_country#169] +Condition : (rpad(ca_state#168, 2, ) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#167)) -(84) ColumnarToRow [codegen id : 20] -Input [2]: [ca_address_sk#167, ca_country#169] +(84) Project [codegen id : 20] +Output [2]: [ca_address_sk#167, ca_country#169] +Input [3]: [ca_address_sk#167, ca_state#168, ca_country#169] (85) BroadcastExchange Input [2]: [ca_address_sk#167, ca_country#169] @@ -662,19 +662,19 @@ Input [12]: [cs_item_sk#226, cs_quantity#227, cs_list_price#228, cs_sales_price# Output [2]: [ca_address_sk#241, ca_state#242] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(110) CometFilter +(110) ColumnarToRow [codegen id : 28] Input [2]: [ca_address_sk#241, ca_state#242] -Condition : (ca_state#242 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#241)) -(111) CometProject +(111) Filter [codegen id : 28] Input [2]: [ca_address_sk#241, ca_state#242] -Arguments: [ca_address_sk#241], [ca_address_sk#241] +Condition : (rpad(ca_state#242, 2, ) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#241)) -(112) ColumnarToRow [codegen id : 28] -Input [1]: [ca_address_sk#241] +(112) Project [codegen id : 28] +Output [1]: [ca_address_sk#241] +Input [2]: [ca_address_sk#241, ca_state#242] (113) BroadcastExchange Input [1]: [ca_address_sk#241] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/simplified.txt index f02809572..4b6b6b432 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/simplified.txt @@ -33,10 +33,10 @@ TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,ag InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [cd_demo_sk,cd_dep_count] - CometFilter [cd_gender,cd_education_status,cd_demo_sk] + Project [cd_demo_sk,cd_dep_count] + Filter [cd_gender,cd_education_status,cd_demo_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] InputAdapter BroadcastExchange #4 @@ -56,9 +56,9 @@ TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,ag InputAdapter BroadcastExchange #6 WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometFilter [ca_state,ca_address_sk] + Filter [ca_state,ca_address_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] InputAdapter ReusedExchange [d_date_sk] #2 @@ -101,9 +101,9 @@ TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,ag InputAdapter BroadcastExchange #9 WholeStageCodegen (12) - ColumnarToRow - InputAdapter - CometFilter [ca_state,ca_address_sk] + Filter [ca_state,ca_address_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] InputAdapter ReusedExchange [d_date_sk] #2 @@ -141,10 +141,10 @@ TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,ag InputAdapter BroadcastExchange #11 WholeStageCodegen (20) - ColumnarToRow - InputAdapter - CometProject [ca_address_sk,ca_country] - CometFilter [ca_state,ca_address_sk] + Project [ca_address_sk,ca_country] + Filter [ca_state,ca_address_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] InputAdapter ReusedExchange [d_date_sk] #2 @@ -182,10 +182,10 @@ TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,ag InputAdapter BroadcastExchange #13 WholeStageCodegen (28) - ColumnarToRow - InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_state,ca_address_sk] + Project [ca_address_sk] + Filter [ca_state,ca_address_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter ReusedExchange [d_date_sk] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/explain.txt index 1304af2e1..71d813c15 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/explain.txt @@ -15,8 +15,8 @@ TakeOrderedAndProject (20) : : +- CometFilter (2) : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : +- BroadcastExchange (7) - : +- * ColumnarToRow (6) - : +- CometFilter (5) + : +- * Filter (6) + : +- * ColumnarToRow (5) : +- CometScan parquet spark_catalog.default.item (4) +- ReusedExchange (10) @@ -40,15 +40,15 @@ Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) CometFilter +(5) ColumnarToRow [codegen id : 1] Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (i_category#10 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) -(6) ColumnarToRow [codegen id : 1] +(6) Filter [codegen id : 1] Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Condition : (rpad(i_category#10, 50, ) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) (7) BroadcastExchange Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/simplified.txt index 2a2a392cd..f0051a18f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/simplified.txt @@ -32,9 +32,9 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter BroadcastExchange #4 WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [i_category,i_item_sk] + Filter [i_category,i_item_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] InputAdapter ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/explain.txt index 0cba9b059..49f49291b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/explain.txt @@ -36,8 +36,8 @@ : : : +- CometFilter (16) : : : +- CometScan parquet spark_catalog.default.store (15) : : +- BroadcastExchange (25) - : : +- * ColumnarToRow (24) - : : +- CometFilter (23) + : : +- * Filter (24) + : : +- * ColumnarToRow (23) : : +- CometScan parquet spark_catalog.default.item (22) : +- BroadcastExchange (31) : +- * ColumnarToRow (30) @@ -147,15 +147,15 @@ Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_stor Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_color), EqualTo(i_color,pale ), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_color), IsNotNull(i_item_sk)] ReadSchema: struct -(23) CometFilter +(23) ColumnarToRow [codegen id : 6] Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = pale )) AND isnotnull(i_item_sk#15)) -(24) ColumnarToRow [codegen id : 6] +(24) Filter [codegen id : 6] Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +Condition : ((isnotnull(i_color#18) AND (rpad(i_color#18, 20, ) = pale )) AND isnotnull(i_item_sk#15)) (25) BroadcastExchange Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/simplified.txt index 7024f439f..9abde9681 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/simplified.txt @@ -102,9 +102,9 @@ WholeStageCodegen (12) InputAdapter BroadcastExchange #7 WholeStageCodegen (6) - ColumnarToRow - InputAdapter - CometFilter [i_color,i_item_sk] + Filter [i_color,i_item_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] InputAdapter BroadcastExchange #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/explain.txt index fc43e7271..a1363ced6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/explain.txt @@ -16,14 +16,14 @@ TakeOrderedAndProject (73) : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : : +- BroadcastExchange (8) - : : : : +- * ColumnarToRow (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) + : : : : +- * Project (7) + : : : : +- * Filter (6) + : : : : +- * ColumnarToRow (5) : : : : +- CometScan parquet spark_catalog.default.customer_demographics (4) : : : +- ReusedExchange (11) : : +- BroadcastExchange (17) - : : +- * ColumnarToRow (16) - : : +- CometFilter (15) + : : +- * Filter (16) + : : +- * ColumnarToRow (15) : : +- CometScan parquet spark_catalog.default.store (14) : +- BroadcastExchange (23) : +- * ColumnarToRow (22) @@ -46,9 +46,9 @@ TakeOrderedAndProject (73) : : : : +- ReusedExchange (32) : : : +- ReusedExchange (35) : : +- BroadcastExchange (42) - : : +- * ColumnarToRow (41) - : : +- CometProject (40) - : : +- CometFilter (39) + : : +- * Project (41) + : : +- * Filter (40) + : : +- * ColumnarToRow (39) : : +- CometScan parquet spark_catalog.default.store (38) : +- ReusedExchange (45) +- * HashAggregate (71) @@ -93,19 +93,19 @@ Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_p Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,F), EqualTo(cd_marital_status,W), EqualTo(cd_education_status,Primary ), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) CometFilter +(5) ColumnarToRow [codegen id : 1] Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (cd_gender#11 = F)) AND (cd_marital_status#12 = W)) AND (cd_education_status#13 = Primary )) AND isnotnull(cd_demo_sk#10)) -(6) CometProject +(6) Filter [codegen id : 1] Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Arguments: [cd_demo_sk#10], [cd_demo_sk#10] +Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (rpad(cd_gender#11, 1, ) = F)) AND (rpad(cd_marital_status#12, 1, ) = W)) AND (rpad(cd_education_status#13, 20, ) = Primary )) AND isnotnull(cd_demo_sk#10)) -(7) ColumnarToRow [codegen id : 1] -Input [1]: [cd_demo_sk#10] +(7) Project [codegen id : 1] +Output [1]: [cd_demo_sk#10] +Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] (8) BroadcastExchange Input [1]: [cd_demo_sk#10] @@ -138,15 +138,15 @@ Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sale Output [2]: [s_store_sk#15, s_state#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] ReadSchema: struct -(15) CometFilter +(15) ColumnarToRow [codegen id : 3] Input [2]: [s_store_sk#15, s_state#16] -Condition : ((isnotnull(s_state#16) AND (s_state#16 = TN)) AND isnotnull(s_store_sk#15)) -(16) ColumnarToRow [codegen id : 3] +(16) Filter [codegen id : 3] Input [2]: [s_store_sk#15, s_state#16] +Condition : ((isnotnull(s_state#16) AND (rpad(s_state#16, 2, ) = TN)) AND isnotnull(s_store_sk#15)) (17) BroadcastExchange Input [2]: [s_store_sk#15, s_state#16] @@ -253,19 +253,19 @@ Input [8]: [ss_item_sk#48, ss_store_sk#50, ss_quantity#51, ss_list_price#52, ss_ Output [2]: [s_store_sk#59, s_state#60] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] ReadSchema: struct -(39) CometFilter +(39) ColumnarToRow [codegen id : 9] Input [2]: [s_store_sk#59, s_state#60] -Condition : ((isnotnull(s_state#60) AND (s_state#60 = TN)) AND isnotnull(s_store_sk#59)) -(40) CometProject +(40) Filter [codegen id : 9] Input [2]: [s_store_sk#59, s_state#60] -Arguments: [s_store_sk#59], [s_store_sk#59] +Condition : ((isnotnull(s_state#60) AND (rpad(s_state#60, 2, ) = TN)) AND isnotnull(s_store_sk#59)) -(41) ColumnarToRow [codegen id : 9] -Input [1]: [s_store_sk#59] +(41) Project [codegen id : 9] +Output [1]: [s_store_sk#59] +Input [2]: [s_store_sk#59, s_state#60] (42) BroadcastExchange Input [1]: [s_store_sk#59] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/simplified.txt index 32f003798..95af794bb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/simplified.txt @@ -29,19 +29,19 @@ TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] InputAdapter BroadcastExchange #3 WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [cd_demo_sk] - CometFilter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] + Project [cd_demo_sk] + Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] InputAdapter ReusedExchange [d_date_sk] #2 InputAdapter BroadcastExchange #4 WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometFilter [s_state,s_store_sk] + Filter [s_state,s_store_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.store [s_store_sk,s_state] InputAdapter BroadcastExchange #5 @@ -76,10 +76,10 @@ TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] InputAdapter BroadcastExchange #7 WholeStageCodegen (9) - ColumnarToRow - InputAdapter - CometProject [s_store_sk] - CometFilter [s_state,s_store_sk] + Project [s_store_sk] + Filter [s_state,s_store_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.store [s_store_sk,s_state] InputAdapter ReusedExchange [i_item_sk,i_item_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/explain.txt index 387fa47f9..dfa91cb1a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/explain.txt @@ -23,9 +23,9 @@ : : +- CometFilter (8) : : +- CometScan parquet spark_catalog.default.store (7) : +- BroadcastExchange (18) - : +- * ColumnarToRow (17) - : +- CometProject (16) - : +- CometFilter (15) + : +- * Project (17) + : +- * Filter (16) + : +- * ColumnarToRow (15) : +- CometScan parquet spark_catalog.default.household_demographics (14) +- BroadcastExchange (28) +- * ColumnarToRow (27) @@ -97,19 +97,19 @@ Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, Output [4]: [hd_demo_sk#10, hd_buy_potential#11, hd_dep_count#12, hd_vehicle_count#13] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_vehicle_count), Or(EqualTo(hd_buy_potential,>10000 ),EqualTo(hd_buy_potential,unknown )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] +PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct -(15) CometFilter +(15) ColumnarToRow [codegen id : 3] Input [4]: [hd_demo_sk#10, hd_buy_potential#11, hd_dep_count#12, hd_vehicle_count#13] -Condition : ((((isnotnull(hd_vehicle_count#13) AND ((hd_buy_potential#11 = >10000 ) OR (hd_buy_potential#11 = unknown ))) AND (hd_vehicle_count#13 > 0)) AND CASE WHEN (hd_vehicle_count#13 > 0) THEN ((cast(hd_dep_count#12 as double) / cast(hd_vehicle_count#13 as double)) > 1.2) END) AND isnotnull(hd_demo_sk#10)) -(16) CometProject +(16) Filter [codegen id : 3] Input [4]: [hd_demo_sk#10, hd_buy_potential#11, hd_dep_count#12, hd_vehicle_count#13] -Arguments: [hd_demo_sk#10], [hd_demo_sk#10] +Condition : ((((isnotnull(hd_vehicle_count#13) AND ((rpad(hd_buy_potential#11, 15, ) = >10000 ) OR (rpad(hd_buy_potential#11, 15, ) = unknown ))) AND (hd_vehicle_count#13 > 0)) AND CASE WHEN (hd_vehicle_count#13 > 0) THEN ((cast(hd_dep_count#12 as double) / cast(hd_vehicle_count#13 as double)) > 1.2) END) AND isnotnull(hd_demo_sk#10)) -(17) ColumnarToRow [codegen id : 3] -Input [1]: [hd_demo_sk#10] +(17) Project [codegen id : 3] +Output [1]: [hd_demo_sk#10] +Input [4]: [hd_demo_sk#10, hd_buy_potential#11, hd_dep_count#12, hd_vehicle_count#13] (18) BroadcastExchange Input [1]: [hd_demo_sk#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/simplified.txt index b473e4892..4c8edb980 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/simplified.txt @@ -42,10 +42,10 @@ WholeStageCodegen (7) InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [hd_demo_sk] - CometFilter [hd_vehicle_count,hd_buy_potential,hd_dep_count,hd_demo_sk] + Project [hd_demo_sk] + Filter [hd_vehicle_count,hd_buy_potential,hd_dep_count,hd_demo_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] InputAdapter BroadcastExchange #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/explain.txt index 2610a698c..ba4da1839 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/explain.txt @@ -26,9 +26,9 @@ TakeOrderedAndProject (41) : : +- CometFilter (8) : : +- CometScan parquet spark_catalog.default.item (7) : +- BroadcastExchange (17) - : +- * ColumnarToRow (16) - : +- CometProject (15) - : +- CometFilter (14) + : +- * Project (16) + : +- * Filter (15) + : +- * ColumnarToRow (14) : +- CometScan parquet spark_catalog.default.store (13) :- * HashAggregate (27) : +- Exchange (26) @@ -102,19 +102,19 @@ Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, Output [2]: [s_store_sk#11, s_state#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] ReadSchema: struct -(14) CometFilter +(14) ColumnarToRow [codegen id : 3] Input [2]: [s_store_sk#11, s_state#12] -Condition : ((isnotnull(s_state#12) AND (s_state#12 = TN)) AND isnotnull(s_store_sk#11)) -(15) CometProject +(15) Filter [codegen id : 3] Input [2]: [s_store_sk#11, s_state#12] -Arguments: [s_store_sk#11], [s_store_sk#11] +Condition : ((isnotnull(s_state#12) AND (rpad(s_state#12, 2, ) = TN)) AND isnotnull(s_store_sk#11)) -(16) ColumnarToRow [codegen id : 3] -Input [1]: [s_store_sk#11] +(16) Project [codegen id : 3] +Output [1]: [s_store_sk#11] +Input [2]: [s_store_sk#11, s_state#12] (17) BroadcastExchange Input [1]: [s_store_sk#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/simplified.txt index f265d2099..50c467198 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/simplified.txt @@ -51,10 +51,10 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i InputAdapter BroadcastExchange #6 WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [s_store_sk] - CometFilter [s_state,s_store_sk] + Project [s_store_sk] + Filter [s_state,s_store_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.store [s_store_sk,s_state] WholeStageCodegen (11) HashAggregate [i_category,sum,isEmpty,sum,isEmpty] [sum(ss_net_profit),sum(ss_ext_sales_price),gross_margin,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/explain.txt index e50a522b8..b4b884b3b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/explain.txt @@ -109,9 +109,9 @@ : : : +- CometScan parquet spark_catalog.default.income_band (91) : : +- ReusedExchange (97) : +- BroadcastExchange (104) - : +- * ColumnarToRow (103) - : +- CometProject (102) - : +- CometFilter (101) + : +- * Project (103) + : +- * Filter (102) + : +- * ColumnarToRow (101) : +- CometScan parquet spark_catalog.default.item (100) +- * Sort (177) +- Exchange (176) @@ -641,19 +641,19 @@ Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt# Output [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), In(i_color, [burlywood ,floral ,indian ,medium ,purple ,spring ]), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] ReadSchema: struct -(101) CometFilter +(101) ColumnarToRow [codegen id : 23] Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] -Condition : ((((((isnotnull(i_current_price#74) AND i_color#75 IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#74 >= 64.00)) AND (i_current_price#74 <= 74.00)) AND (i_current_price#74 >= 65.00)) AND (i_current_price#74 <= 79.00)) AND isnotnull(i_item_sk#73)) -(102) CometProject +(102) Filter [codegen id : 23] Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] -Arguments: [i_item_sk#73, i_product_name#76], [i_item_sk#73, i_product_name#76] +Condition : ((((((isnotnull(i_current_price#74) AND rpad(i_color#75, 20, ) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#74 >= 64.00)) AND (i_current_price#74 <= 74.00)) AND (i_current_price#74 >= 65.00)) AND (i_current_price#74 <= 79.00)) AND isnotnull(i_item_sk#73)) -(103) ColumnarToRow [codegen id : 23] -Input [2]: [i_item_sk#73, i_product_name#76] +(103) Project [codegen id : 23] +Output [2]: [i_item_sk#73, i_product_name#76] +Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] (104) BroadcastExchange Input [2]: [i_item_sk#73, i_product_name#76] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/simplified.txt index 2a0bc5bce..d24d4b4fa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/simplified.txt @@ -173,10 +173,10 @@ WholeStageCodegen (52) InputAdapter BroadcastExchange #17 WholeStageCodegen (23) - ColumnarToRow - InputAdapter - CometProject [i_item_sk,i_product_name] - CometFilter [i_current_price,i_color,i_item_sk] + Project [i_item_sk,i_product_name] + Filter [i_current_price,i_color,i_item_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] InputAdapter WholeStageCodegen (50) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/explain.txt index c88573838..e1b53ef95 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/explain.txt @@ -41,14 +41,14 @@ TakeOrderedAndProject (70) : : : : : : : +- CometFilter (17) : : : : : : : +- CometScan parquet spark_catalog.default.item (16) : : : : : : +- BroadcastExchange (26) - : : : : : : +- * ColumnarToRow (25) - : : : : : : +- CometProject (24) - : : : : : : +- CometFilter (23) + : : : : : : +- * Project (25) + : : : : : : +- * Filter (24) + : : : : : : +- * ColumnarToRow (23) : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (22) : : : : : +- BroadcastExchange (33) - : : : : : +- * ColumnarToRow (32) - : : : : : +- CometProject (31) - : : : : : +- CometFilter (30) + : : : : : +- * Project (32) + : : : : : +- * Filter (31) + : : : : : +- * ColumnarToRow (30) : : : : : +- CometScan parquet spark_catalog.default.household_demographics (29) : : : : +- ReusedExchange (36) : : : +- BroadcastExchange (42) @@ -175,19 +175,19 @@ Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_ Output [2]: [cd_demo_sk#18, cd_marital_status#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_marital_status), EqualTo(cd_marital_status,M), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_marital_status), IsNotNull(cd_demo_sk)] ReadSchema: struct -(23) CometFilter +(23) ColumnarToRow [codegen id : 4] Input [2]: [cd_demo_sk#18, cd_marital_status#19] -Condition : ((isnotnull(cd_marital_status#19) AND (cd_marital_status#19 = M)) AND isnotnull(cd_demo_sk#18)) -(24) CometProject +(24) Filter [codegen id : 4] Input [2]: [cd_demo_sk#18, cd_marital_status#19] -Arguments: [cd_demo_sk#18], [cd_demo_sk#18] +Condition : ((isnotnull(cd_marital_status#19) AND (rpad(cd_marital_status#19, 1, ) = M)) AND isnotnull(cd_demo_sk#18)) -(25) ColumnarToRow [codegen id : 4] -Input [1]: [cd_demo_sk#18] +(25) Project [codegen id : 4] +Output [1]: [cd_demo_sk#18] +Input [2]: [cd_demo_sk#18, cd_marital_status#19] (26) BroadcastExchange Input [1]: [cd_demo_sk#18] @@ -207,19 +207,19 @@ Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_ Output [2]: [hd_demo_sk#20, hd_buy_potential#21] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_buy_potential), EqualTo(hd_buy_potential,1001-5000 ), IsNotNull(hd_demo_sk)] +PushedFilters: [IsNotNull(hd_buy_potential), IsNotNull(hd_demo_sk)] ReadSchema: struct -(30) CometFilter +(30) ColumnarToRow [codegen id : 5] Input [2]: [hd_demo_sk#20, hd_buy_potential#21] -Condition : ((isnotnull(hd_buy_potential#21) AND (hd_buy_potential#21 = 1001-5000 )) AND isnotnull(hd_demo_sk#20)) -(31) CometProject +(31) Filter [codegen id : 5] Input [2]: [hd_demo_sk#20, hd_buy_potential#21] -Arguments: [hd_demo_sk#20], [hd_demo_sk#20] +Condition : ((isnotnull(hd_buy_potential#21) AND (rpad(hd_buy_potential#21, 15, ) = 1001-5000 )) AND isnotnull(hd_demo_sk#20)) -(32) ColumnarToRow [codegen id : 5] -Input [1]: [hd_demo_sk#20] +(32) Project [codegen id : 5] +Output [1]: [hd_demo_sk#20] +Input [2]: [hd_demo_sk#20, hd_buy_potential#21] (33) BroadcastExchange Input [1]: [hd_demo_sk#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/simplified.txt index 17fc9dee7..e41fe84f5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/simplified.txt @@ -67,18 +67,18 @@ TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_prom InputAdapter BroadcastExchange #7 WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometProject [cd_demo_sk] - CometFilter [cd_marital_status,cd_demo_sk] + Project [cd_demo_sk] + Filter [cd_marital_status,cd_demo_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] InputAdapter BroadcastExchange #8 WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometProject [hd_demo_sk] - CometFilter [hd_buy_potential,hd_demo_sk] + Project [hd_demo_sk] + Filter [hd_buy_potential,hd_demo_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] InputAdapter ReusedExchange [d_date_sk,d_date,d_week_seq] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/explain.txt index 5c8480a96..4780e687d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/explain.txt @@ -24,9 +24,9 @@ TakeOrderedAndProject (129) : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : : : : +- BroadcastExchange (8) - : : : : +- * ColumnarToRow (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) + : : : : +- * Project (7) + : : : : +- * Filter (6) + : : : : +- * ColumnarToRow (5) : : : : +- CometScan parquet spark_catalog.default.item (4) : : : +- ReusedExchange (11) : : +- * Sort (21) @@ -149,19 +149,19 @@ Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4 Output [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Books ), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] +PushedFilters: [IsNotNull(i_category), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] ReadSchema: struct -(5) CometFilter +(5) ColumnarToRow [codegen id : 1] Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] -Condition : ((((((isnotnull(i_category#11) AND (i_category#11 = Books )) AND isnotnull(i_item_sk#7)) AND isnotnull(i_brand_id#8)) AND isnotnull(i_class_id#9)) AND isnotnull(i_category_id#10)) AND isnotnull(i_manufact_id#12)) -(6) CometProject +(6) Filter [codegen id : 1] Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] -Arguments: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12], [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +Condition : ((((((isnotnull(i_category#11) AND (rpad(i_category#11, 50, ) = Books )) AND isnotnull(i_item_sk#7)) AND isnotnull(i_brand_id#8)) AND isnotnull(i_class_id#9)) AND isnotnull(i_category_id#10)) AND isnotnull(i_manufact_id#12)) -(7) ColumnarToRow [codegen id : 1] -Input [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +(7) Project [codegen id : 1] +Output [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] (8) BroadcastExchange Input [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/simplified.txt index 9939b2fe2..67d2d8f5a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/simplified.txt @@ -48,10 +48,10 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i InputAdapter BroadcastExchange #6 WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [i_category,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + Project [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + Filter [i_category,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] InputAdapter ReusedExchange [d_date_sk,d_year] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/explain.txt index 8f47f4f0e..f073fbf33 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/explain.txt @@ -43,9 +43,9 @@ TakeOrderedAndProject (120) : : : +- CometFilter (24) : : : +- CometScan parquet spark_catalog.default.item (23) : : +- BroadcastExchange (34) - : : +- * ColumnarToRow (33) - : : +- CometProject (32) - : : +- CometFilter (31) + : : +- * Project (33) + : : +- * Filter (32) + : : +- * ColumnarToRow (31) : : +- CometScan parquet spark_catalog.default.promotion (30) : :- * HashAggregate (70) : : +- Exchange (69) @@ -257,19 +257,19 @@ Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, Output [2]: [p_promo_sk#19, p_channel_tv#20] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [IsNotNull(p_channel_tv), EqualTo(p_channel_tv,N), IsNotNull(p_promo_sk)] +PushedFilters: [IsNotNull(p_channel_tv), IsNotNull(p_promo_sk)] ReadSchema: struct -(31) CometFilter +(31) ColumnarToRow [codegen id : 8] Input [2]: [p_promo_sk#19, p_channel_tv#20] -Condition : ((isnotnull(p_channel_tv#20) AND (p_channel_tv#20 = N)) AND isnotnull(p_promo_sk#19)) -(32) CometProject +(32) Filter [codegen id : 8] Input [2]: [p_promo_sk#19, p_channel_tv#20] -Arguments: [p_promo_sk#19], [p_promo_sk#19] +Condition : ((isnotnull(p_channel_tv#20) AND (rpad(p_channel_tv#20, 1, ) = N)) AND isnotnull(p_promo_sk#19)) -(33) ColumnarToRow [codegen id : 8] -Input [1]: [p_promo_sk#19] +(33) Project [codegen id : 8] +Output [1]: [p_promo_sk#19] +Input [2]: [p_promo_sk#19, p_channel_tv#20] (34) BroadcastExchange Input [1]: [p_promo_sk#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/simplified.txt index 34e47dcba..a07ef128c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/simplified.txt @@ -80,10 +80,10 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter BroadcastExchange #9 WholeStageCodegen (8) - ColumnarToRow - InputAdapter - CometProject [p_promo_sk] - CometFilter [p_channel_tv,p_promo_sk] + Project [p_promo_sk] + Filter [p_channel_tv,p_promo_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] WholeStageCodegen (20) HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/explain.txt index f77b7ec93..a1908b18b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/explain.txt @@ -16,8 +16,8 @@ : : +- CometFilter (2) : : +- CometScan parquet spark_catalog.default.store_sales (1) : +- BroadcastExchange (7) - : +- * ColumnarToRow (6) - : +- CometFilter (5) + : +- * Filter (6) + : +- * ColumnarToRow (5) : +- CometScan parquet spark_catalog.default.item (4) +- ReusedExchange (10) @@ -41,15 +41,15 @@ Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) CometFilter +(5) ColumnarToRow [codegen id : 1] Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (i_category#10 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) -(6) ColumnarToRow [codegen id : 1] +(6) Filter [codegen id : 1] Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Condition : (rpad(i_category#10, 50, ) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) (7) BroadcastExchange Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/simplified.txt index b7489a0af..a685378c7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/simplified.txt @@ -35,9 +35,9 @@ WholeStageCodegen (7) InputAdapter BroadcastExchange #5 WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [i_category,i_item_sk] + Filter [i_category,i_item_sk] + ColumnarToRow + InputAdapter CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] InputAdapter ReusedExchange [d_date_sk] #4