From f36184612e9550f94cf9f715a4bfef2a5f4f05e8 Mon Sep 17 00:00:00 2001 From: Parth Chandra Date: Thu, 18 Apr 2024 19:04:13 -0700 Subject: [PATCH] Address review comments --- .../CometTPCDSQueriesList-results.txt | 840 +++++++++++------- .../CometTPCHQueriesList-results.txt | 131 +-- .../comet/CometSparkSessionExtensions.scala | 135 +-- .../apache/comet/ExtendedExplainInfo.scala | 10 +- .../apache/comet/serde/QueryPlanSerde.scala | 10 +- .../apache/comet/CometExpressionSuite.scala | 2 +- .../spark/sql/CometTPCQueryListBase.scala | 13 +- 7 files changed, 701 insertions(+), 440 deletions(-) diff --git a/spark/inspections/CometTPCDSQueriesList-results.txt b/spark/inspections/CometTPCDSQueriesList-results.txt index 9ca4eda36c..13f99a1ac9 100644 --- a/spark/inspections/CometTPCDSQueriesList-results.txt +++ b/spark/inspections/CometTPCDSQueriesList-results.txt @@ -1,622 +1,838 @@ -Query: q1. Comet Exec: Enabled (CometProject, CometFilter) +Query: q1. Comet Exec: Enabled (CometFilter, CometProject) Query: q1: ExplainInfo: -BroadcastHashJoin is not supported +ObjectHashAggregate is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +xxhash64 is not supported +SortMergeJoin disabled because not all child plans are native -Query: q2. Comet Exec: Enabled (CometUnion, CometProject, CometFilter) +Query: q2. Comet Exec: Enabled (CometFilter, CometProject, CometUnion) Query: q2: ExplainInfo: -BroadcastHashJoin is not supported +ObjectHashAggregate is not supported +xxhash64 is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q3. Comet Exec: Enabled (CometProject, CometFilter) +Query: q3. Comet Exec: Enabled (CometFilter, CometProject) Query: q3: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q4. Comet Exec: Enabled (CometProject, CometFilter) +Query: q4. Comet Exec: Enabled (CometFilter, CometProject) Query: q4: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q5. Comet Exec: Enabled (CometUnion, CometProject, CometFilter) +Query: q5. Comet Exec: Enabled (CometFilter, CometProject, CometUnion) Query: q5: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +Union disabled because not all child plans are native -Query: q6. Comet Exec: Enabled (CometHashAggregate, CometProject, CometFilter) +Query: q6. Comet Exec: Enabled (CometHashAggregate, CometFilter, CometProject) Query: q6: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q7. Comet Exec: Enabled (CometFilter, CometProject) Query: q7: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q8. Comet Exec: Enabled (CometProject, CometFilter) +Query: q8. Comet Exec: Enabled (CometFilter, CometProject, CometSort) Query: q8: ExplainInfo: -BroadcastHashJoin is not supported +ObjectHashAggregate is not supported +getstructfield is not supported +xxhash64 is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native Query: q9. Comet Exec: Enabled (CometFilter) Query: q9: ExplainInfo: named_struct is not supported getstructfield is not supported -Query: q10. Comet Exec: Enabled (CometProject, CometFilter) +Query: q10. Comet Exec: Enabled (CometFilter, CometProject) Query: q10: ExplainInfo: -BroadcastHashJoin is not supported +ObjectHashAggregate is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +xxhash64 is not supported +SortMergeJoin disabled because not all child plans are native -Query: q11. Comet Exec: Enabled (CometProject, CometFilter) +Query: q11. Comet Exec: Enabled (CometFilter, CometProject) Query: q11: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q12. Comet Exec: Enabled (CometProject, CometFilter) +Query: q12. Comet Exec: Enabled (CometFilter, CometProject) Query: q12: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Window is not supported -Query: q13. Comet Exec: Enabled (CometProject, CometFilter) +Query: q13. Comet Exec: Enabled (CometFilter, CometProject) Query: q13: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q14a. Comet Exec: Enabled (CometProject, CometFilter) +Query: q14a. Comet Exec: Enabled (CometFilter, CometProject, CometSort) Query: q14a: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native - -Query: q14b. Comet Exec: Enabled (CometProject, CometFilter) +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native +ObjectHashAggregate is not supported +xxhash64 is not supported +Union disabled because not all child plans are native + +Query: q14b. Comet Exec: Enabled (CometFilter, CometProject, CometSort) Query: q14b: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native - -Query: q15. Comet Exec: Enabled (CometProject, CometFilter) +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native +ObjectHashAggregate is not supported +xxhash64 is not supported +Union disabled because not all child plans are native + +Query: q15. Comet Exec: Enabled (CometFilter, CometProject) Query: q15: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q16. Comet Exec: Enabled (CometProject, CometFilter) +Query: q16. Comet Exec: Enabled (CometFilter, CometProject) Query: q16: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q17. Comet Exec: Enabled (CometProject, CometFilter) +Query: q17. Comet Exec: Enabled (CometFilter, CometProject) Query: q17: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q18. Comet Exec: Enabled (CometProject, CometFilter) +Query: q18. Comet Exec: Enabled (CometFilter, CometProject) Query: q18: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q19. Comet Exec: Enabled (CometProject, CometFilter) +Query: q19. Comet Exec: Enabled (CometFilter, CometProject) Query: q19: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q20. Comet Exec: Enabled (CometProject, CometFilter) +Query: q20. Comet Exec: Enabled (CometFilter, CometProject) Query: q20: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Window is not supported -Query: q21. Comet Exec: Enabled (CometProject, CometFilter) +Query: q21. Comet Exec: Enabled (CometFilter, CometProject) Query: q21: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q22. Comet Exec: Enabled (CometProject, CometFilter) +Query: q22. Comet Exec: Enabled (CometFilter, CometProject) Query: q22: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q23a. Comet Exec: Enabled (CometProject, CometFilter) +Query: q23a. Comet Exec: Enabled (CometFilter, CometProject) Query: q23a: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native - -Query: q23b. Comet Exec: Enabled (CometProject, CometFilter) +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +ObjectHashAggregate is not supported +xxhash64 is not supported +SortMergeJoin disabled because not all child plans are native +Union disabled because not all child plans are native + +Query: q23b. Comet Exec: Enabled (CometFilter, CometProject, CometSort) Query: q23b: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native - -Query: q24a. Comet Exec: Enabled (CometProject, CometFilter) +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +ObjectHashAggregate is not supported +xxhash64 is not supported +SortMergeJoin disabled because not all child plans are native +Union disabled because not all child plans are native + +Query: q24a. Comet Exec: Enabled (CometFilter, CometProject) Query: q24a: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q24b. Comet Exec: Enabled (CometProject, CometFilter) +Query: q24b. Comet Exec: Enabled (CometFilter, CometProject) Query: q24b: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q25. Comet Exec: Enabled (CometProject, CometFilter) +Query: q25. Comet Exec: Enabled (CometFilter, CometProject) Query: q25: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q26. Comet Exec: Enabled (CometProject, CometFilter) +Query: q26. Comet Exec: Enabled (CometFilter, CometProject) Query: q26: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q27. Comet Exec: Enabled (CometProject, CometFilter) +Query: q27. Comet Exec: Enabled (CometFilter, CometProject) Query: q27: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q28. Comet Exec: Enabled (CometHashAggregate, CometProject, CometFilter) +Query: q28. Comet Exec: Enabled (CometHashAggregate, CometFilter, CometProject) Query: q28: ExplainInfo: Unsupported aggregation mode PartialMerge -BroadcastNestedLoopJoin is not supported +BroadcastExchange is not supported -Query: q29. Comet Exec: Enabled (CometProject, CometFilter) +Query: q29. Comet Exec: Enabled (CometFilter, CometProject) Query: q29: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q30. Comet Exec: Enabled (CometProject, CometFilter) +Query: q30. Comet Exec: Enabled (CometFilter, CometProject) Query: q30: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +ObjectHashAggregate is not supported +xxhash64 is not supported +SortMergeJoin disabled because not all child plans are native Query: q31. Comet Exec: Enabled (CometFilter) Query: q31: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q32. Comet Exec: Enabled (CometProject, CometFilter) +Query: q32. Comet Exec: Enabled (CometFilter, CometProject) Query: q32: ExplainInfo: -BroadcastHashJoin is not supported +ObjectHashAggregate is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +xxhash64 is not supported +SortMergeJoin disabled because not all child plans are native -Query: q33. Comet Exec: Enabled (CometProject, CometFilter) +Query: q33. Comet Exec: Enabled (CometFilter, CometProject) Query: q33: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +Union disabled because not all child plans are native -Query: q34. Comet Exec: Enabled (CometProject, CometFilter) +Query: q34. Comet Exec: Enabled (CometFilter, CometProject) Query: q34: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q35. Comet Exec: Enabled (CometProject, CometFilter) +Query: q35. Comet Exec: Enabled (CometFilter, CometProject, CometSort) Query: q35: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q36. Comet Exec: Enabled (CometProject, CometFilter) +Query: q36. Comet Exec: Enabled (CometFilter, CometProject) Query: q36: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Window is not supported -Query: q37. Comet Exec: Enabled (CometProject, CometFilter) +Query: q37. Comet Exec: Enabled (CometFilter, CometProject) Query: q37: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q38. Comet Exec: Enabled (CometProject, CometFilter) +Query: q38. Comet Exec: Enabled (CometFilter, CometProject) Query: q38: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q39a. Comet Exec: Enabled (CometProject, CometFilter) +Query: q39a. Comet Exec: Enabled (CometFilter, CometProject) Query: q39a: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q39b. Comet Exec: Enabled (CometProject, CometFilter) +Query: q39b. Comet Exec: Enabled (CometFilter, CometProject) Query: q39b: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q40. Comet Exec: Enabled (CometProject, CometFilter) +Query: q40. Comet Exec: Enabled (CometFilter, CometProject) Query: q40: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q41. Comet Exec: Enabled (CometHashAggregate, CometProject, CometFilter) +Query: q41. Comet Exec: Enabled (CometFilter, CometProject) Query: q41: ExplainInfo: -BroadcastHashJoin is not supported +ObjectHashAggregate is not supported +xxhash64 is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q42. Comet Exec: Enabled (CometProject, CometFilter) +Query: q42. Comet Exec: Enabled (CometFilter, CometProject) Query: q42: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q43. Comet Exec: Enabled (CometProject, CometFilter) +Query: q43. Comet Exec: Enabled (CometFilter, CometProject) Query: q43: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q44. Comet Exec: Enabled (CometHashAggregate, CometSort, CometProject, CometFilter) +Query: q44. Comet Exec: Enabled (CometHashAggregate, CometFilter, CometProject, CometSort) Query: q44: ExplainInfo: Window is not supported -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q45. Comet Exec: Enabled (CometProject, CometFilter) +Query: q45. Comet Exec: Enabled (CometFilter, CometProject) Query: q45: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q46. Comet Exec: Enabled (CometProject, CometFilter) +Query: q46. Comet Exec: Enabled (CometFilter, CometProject) Query: q46: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q47. Comet Exec: Enabled (CometProject, CometFilter) +Query: q47. Comet Exec: Enabled (CometFilter, CometProject) Query: q47: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Window is not supported +SortMergeJoin disabled because not all child plans are native -Query: q48. Comet Exec: Enabled (CometProject, CometFilter) +Query: q48. Comet Exec: Enabled (CometFilter, CometProject) Query: q48: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q49. Comet Exec: Enabled (CometProject, CometFilter) +Query: q49. Comet Exec: Enabled (CometFilter, CometProject) Query: q49: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Window is not supported -Not all subqueries for union are native +Union disabled because not all child plans are native -Query: q50. Comet Exec: Enabled (CometProject, CometFilter) +Query: q50. Comet Exec: Enabled (CometFilter, CometProject) Query: q50: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q51. Comet Exec: Enabled (CometProject, CometFilter) +Query: q51. Comet Exec: Enabled (CometFilter, CometProject) Query: q51: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Window is not supported -SortMergeJoin is not supported +SortMergeJoin disabled because not all child plans are native -Query: q52. Comet Exec: Enabled (CometProject, CometFilter) +Query: q52. Comet Exec: Enabled (CometFilter, CometProject) Query: q52: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q53. Comet Exec: Enabled (CometProject, CometFilter) +Query: q53. Comet Exec: Enabled (CometFilter, CometProject) Query: q53: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Window is not supported -Query: q54. Comet Exec: Enabled (CometUnion, CometProject, CometFilter) +Query: q54. Comet Exec: Enabled (CometFilter, CometProject, CometUnion) Query: q54: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +ObjectHashAggregate is not supported +xxhash64 is not supported +SortMergeJoin disabled because not all child plans are native -Query: q55. Comet Exec: Enabled (CometProject, CometFilter) +Query: q55. Comet Exec: Enabled (CometFilter, CometProject) Query: q55: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q56. Comet Exec: Enabled (CometProject, CometFilter) +Query: q56. Comet Exec: Enabled (CometFilter, CometProject) Query: q56: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +Union disabled because not all child plans are native -Query: q57. Comet Exec: Enabled (CometProject, CometFilter) +Query: q57. Comet Exec: Enabled (CometFilter, CometProject) Query: q57: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Window is not supported +SortMergeJoin disabled because not all child plans are native -Query: q58. Comet Exec: Enabled (CometProject, CometFilter) +Query: q58. Comet Exec: Enabled (CometFilter, CometProject) Query: q58: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q59. Comet Exec: Enabled (CometProject, CometFilter) +Query: q59. Comet Exec: Enabled (CometFilter, CometProject) Query: q59: ExplainInfo: -BroadcastHashJoin is not supported +ObjectHashAggregate is not supported +xxhash64 is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q60. Comet Exec: Enabled (CometProject, CometFilter) +Query: q60. Comet Exec: Enabled (CometFilter, CometProject) Query: q60: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +Union disabled because not all child plans are native -Query: q61. Comet Exec: Enabled (CometProject, CometFilter) +Query: q61. Comet Exec: Enabled (CometFilter, CometProject) Query: q61: ExplainInfo: -BroadcastHashJoin is not supported -BroadcastNestedLoopJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q62. Comet Exec: Enabled (CometProject, CometFilter) +Query: q62. Comet Exec: Enabled (CometFilter, CometProject) Query: q62: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q63. Comet Exec: Enabled (CometProject, CometFilter) +Query: q63. Comet Exec: Enabled (CometFilter, CometProject) Query: q63: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Window is not supported -Query: q64. Comet Exec: Enabled (CometProject, CometFilter) +Query: q64. Comet Exec: Enabled (CometFilter, CometProject) Query: q64: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +ObjectHashAggregate is not supported +BroadcastHashJoin disabled because not all child plans are native +xxhash64 is not supported +SortMergeJoin disabled because not all child plans are native Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q65. Comet Exec: Enabled (CometProject, CometFilter) +Query: q65. Comet Exec: Enabled (CometFilter, CometProject) Query: q65: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q66. Comet Exec: Enabled (CometProject, CometFilter) +Query: q66. Comet Exec: Enabled (CometFilter, CometProject) Query: q66: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +Union disabled because not all child plans are native -Query: q67. Comet Exec: Enabled (CometProject, CometFilter) +Query: q67. Comet Exec: Enabled (CometFilter, CometProject) Query: q67: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Window is not supported -Query: q68. Comet Exec: Enabled (CometProject, CometFilter) +Query: q68. Comet Exec: Enabled (CometFilter, CometProject) Query: q68: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q69. Comet Exec: Enabled (CometProject, CometFilter) +Query: q69. Comet Exec: Enabled (CometFilter, CometProject, CometSort) Query: q69: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q70. Comet Exec: Enabled (CometProject, CometFilter) +Query: q70. Comet Exec: Enabled (CometFilter, CometProject, CometSort) Query: q70: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Window is not supported +SortMergeJoin disabled because not all child plans are native -Query: q71. Comet Exec: Enabled (CometProject, CometFilter) +Query: q71. Comet Exec: Enabled (CometFilter, CometProject) Query: q71: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +Union disabled because not all child plans are native Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q72. Comet Exec: Enabled (CometProject, CometFilter) +Query: q72. Comet Exec: Enabled (CometFilter, CometProject) Query: q72: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q73. Comet Exec: Enabled (CometProject, CometFilter) +Query: q73. Comet Exec: Enabled (CometFilter, CometProject) Query: q73: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q74. Comet Exec: Enabled (CometProject, CometFilter) +Query: q74. Comet Exec: Enabled (CometFilter, CometProject) Query: q74: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q75. Comet Exec: Enabled (CometProject, CometFilter) +Query: q75. Comet Exec: Enabled (CometFilter, CometProject) Query: q75: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +Union disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q76. Comet Exec: Enabled (CometProject, CometFilter) +Query: q76. Comet Exec: Enabled (CometFilter, CometProject) Query: q76: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +Union disabled because not all child plans are native -Query: q77. Comet Exec: Enabled (CometProject, CometFilter) +Query: q77. Comet Exec: Enabled (CometFilter, CometProject) Query: q77: ExplainInfo: -BroadcastHashJoin is not supported -BroadcastNestedLoopJoin is not supported -Not all subqueries for union are native +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native +Union disabled because not all child plans are native Query: q78. Comet Exec: Enabled (CometFilter) Query: q78: ExplainInfo: -BroadcastHashJoin is not supported -Comet does not support Spark's BigDecimal rounding +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q79. Comet Exec: Enabled (CometProject, CometFilter) +Query: q79. Comet Exec: Enabled (CometFilter, CometProject) Query: q79: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q80. Comet Exec: Enabled (CometProject, CometFilter) +Query: q80. Comet Exec: Enabled (CometFilter, CometProject) Query: q80: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +Union disabled because not all child plans are native -Query: q81. Comet Exec: Enabled (CometProject, CometFilter) +Query: q81. Comet Exec: Enabled (CometFilter, CometProject) Query: q81: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +ObjectHashAggregate is not supported +xxhash64 is not supported +SortMergeJoin disabled because not all child plans are native -Query: q82. Comet Exec: Enabled (CometProject, CometFilter) +Query: q82. Comet Exec: Enabled (CometFilter, CometProject) Query: q82: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q83. Comet Exec: Enabled (CometProject, CometFilter) +Query: q83. Comet Exec: Enabled (CometFilter, CometProject) Query: q83: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q84. Comet Exec: Enabled (CometProject, CometFilter) +Query: q84. Comet Exec: Enabled (CometFilter, CometProject) Query: q84: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q85. Comet Exec: Enabled (CometProject, CometFilter) +Query: q85. Comet Exec: Enabled (CometFilter, CometProject) Query: q85: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q86. Comet Exec: Enabled (CometProject, CometFilter) +Query: q86. Comet Exec: Enabled (CometFilter, CometProject) Query: q86: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Window is not supported -Query: q87. Comet Exec: Enabled (CometProject, CometFilter) +Query: q87. Comet Exec: Enabled (CometFilter, CometProject) Query: q87: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q88. Comet Exec: Enabled (CometProject, CometFilter) +Query: q88. Comet Exec: Enabled (CometFilter, CometProject) Query: q88: ExplainInfo: -BroadcastHashJoin is not supported -BroadcastNestedLoopJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q89. Comet Exec: Enabled (CometProject, CometFilter) +Query: q89. Comet Exec: Enabled (CometFilter, CometProject) Query: q89: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Window is not supported -Query: q90. Comet Exec: Enabled (CometProject, CometFilter) +Query: q90. Comet Exec: Enabled (CometFilter, CometProject) Query: q90: ExplainInfo: -BroadcastHashJoin is not supported -BroadcastNestedLoopJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q91. Comet Exec: Enabled (CometProject, CometFilter) +Query: q91. Comet Exec: Enabled (CometFilter, CometProject) Query: q91: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q92. Comet Exec: Enabled (CometProject, CometFilter) +Query: q92. Comet Exec: Enabled (CometFilter, CometProject) Query: q92: ExplainInfo: -BroadcastHashJoin is not supported +ObjectHashAggregate is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +xxhash64 is not supported +SortMergeJoin disabled because not all child plans are native -Query: q93. Comet Exec: Enabled (CometProject, CometFilter) +Query: q93. Comet Exec: Enabled (CometFilter, CometProject) Query: q93: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q94. Comet Exec: Enabled (CometProject, CometFilter) +Query: q94. Comet Exec: Enabled (CometFilter, CometProject) Query: q94: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q95. Comet Exec: Enabled (CometProject, CometFilter) +Query: q95. Comet Exec: Enabled (CometFilter, CometProject, CometSort) Query: q95: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q96. Comet Exec: Enabled (CometProject, CometFilter) +Query: q96. Comet Exec: Enabled (CometFilter, CometProject) Query: q96: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q97. Comet Exec: Enabled (CometProject, CometFilter) +Query: q97. Comet Exec: Enabled (CometFilter, CometProject) Query: q97: ExplainInfo: -BroadcastHashJoin is not supported -SortMergeJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q98. Comet Exec: Enabled (CometProject, CometFilter) +Query: q98. Comet Exec: Enabled (CometFilter, CometProject) Query: q98: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Window is not supported Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q99. Comet Exec: Enabled (CometProject, CometFilter) +Query: q99. Comet Exec: Enabled (CometFilter, CometProject) Query: q99: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q5a-v2.7. Comet Exec: Enabled (CometUnion, CometProject, CometFilter) +Query: q5a-v2.7. Comet Exec: Enabled (CometFilter, CometProject, CometUnion) Query: q5a-v2.7: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +Union disabled because not all child plans are native -Query: q6-v2.7. Comet Exec: Enabled (CometHashAggregate, CometProject, CometFilter) +Query: q6-v2.7. Comet Exec: Enabled (CometHashAggregate, CometFilter, CometProject) Query: q6-v2.7: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q10a-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q10a-v2.7. Comet Exec: Enabled (CometFilter, CometProject, CometSort) Query: q10a-v2.7: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +Union disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q11-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q11-v2.7. Comet Exec: Enabled (CometFilter, CometProject) Query: q11-v2.7: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q12-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q12-v2.7. Comet Exec: Enabled (CometFilter, CometProject) Query: q12-v2.7: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Window is not supported -Query: q14-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q14-v2.7. Comet Exec: Enabled (CometFilter, CometProject, CometSort) Query: q14-v2.7: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native - -Query: q14a-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native +ObjectHashAggregate is not supported +xxhash64 is not supported +Union disabled because not all child plans are native + +Query: q14a-v2.7. Comet Exec: Enabled (CometFilter, CometProject, CometSort) Query: q14a-v2.7: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native - -Query: q18a-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native +ObjectHashAggregate is not supported +xxhash64 is not supported +Union disabled because not all child plans are native + +Query: q18a-v2.7. Comet Exec: Enabled (CometFilter, CometProject) Query: q18a-v2.7: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +Union disabled because not all child plans are native -Query: q20-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q20-v2.7. Comet Exec: Enabled (CometFilter, CometProject) Query: q20-v2.7: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Window is not supported -Query: q22-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q22-v2.7. Comet Exec: Enabled (CometFilter, CometProject) Query: q22-v2.7: ExplainInfo: -BroadcastHashJoin is not supported -BroadcastNestedLoopJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +Native Broadcast is not enabled -Query: q22a-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q22a-v2.7. Comet Exec: Enabled (CometFilter, CometProject) Query: q22a-v2.7: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +Union disabled because not all child plans are native -Query: q24-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q24-v2.7. Comet Exec: Enabled (CometFilter, CometProject) Query: q24-v2.7: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q27a-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q27a-v2.7. Comet Exec: Enabled (CometFilter, CometProject) Query: q27a-v2.7: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +Union disabled because not all child plans are native -Query: q34-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q34-v2.7. Comet Exec: Enabled (CometFilter, CometProject) Query: q34-v2.7: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q35-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q35-v2.7. Comet Exec: Enabled (CometFilter, CometProject, CometSort) Query: q35-v2.7: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q35a-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q35a-v2.7. Comet Exec: Enabled (CometFilter, CometProject, CometSort) Query: q35a-v2.7: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +Union disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q36a-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q36a-v2.7. Comet Exec: Enabled (CometFilter, CometProject) Query: q36a-v2.7: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +Union disabled because not all child plans are native Window is not supported -Query: q47-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q47-v2.7. Comet Exec: Enabled (CometFilter, CometProject) Query: q47-v2.7: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Window is not supported +SortMergeJoin disabled because not all child plans are native -Query: q49-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q49-v2.7. Comet Exec: Enabled (CometFilter, CometProject) Query: q49-v2.7: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Window is not supported -Not all subqueries for union are native +Union disabled because not all child plans are native -Query: q51a-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q51a-v2.7. Comet Exec: Enabled (CometFilter, CometProject) Query: q51a-v2.7: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Window is not supported -SortMergeJoin is not supported +SortMergeJoin disabled because not all child plans are native -Query: q57-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q57-v2.7. Comet Exec: Enabled (CometFilter, CometProject) Query: q57-v2.7: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Window is not supported +SortMergeJoin disabled because not all child plans are native -Query: q64-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q64-v2.7. Comet Exec: Enabled (CometFilter, CometProject) Query: q64-v2.7: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +ObjectHashAggregate is not supported +BroadcastHashJoin disabled because not all child plans are native +xxhash64 is not supported +SortMergeJoin disabled because not all child plans are native Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q67a-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q67a-v2.7. Comet Exec: Enabled (CometFilter, CometProject) Query: q67a-v2.7: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +Union disabled because not all child plans are native Window is not supported -Query: q70a-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q70a-v2.7. Comet Exec: Enabled (CometFilter, CometProject, CometSort) Query: q70a-v2.7: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Window is not supported -Not all subqueries for union are native +SortMergeJoin disabled because not all child plans are native +Union disabled because not all child plans are native -Query: q72-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q72-v2.7. Comet Exec: Enabled (CometFilter, CometProject) Query: q72-v2.7: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q74-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q74-v2.7. Comet Exec: Enabled (CometFilter, CometProject) Query: q74-v2.7: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q75-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q75-v2.7. Comet Exec: Enabled (CometFilter, CometProject) Query: q75-v2.7: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +Union disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q77a-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q77a-v2.7. Comet Exec: Enabled (CometFilter, CometProject) Query: q77a-v2.7: ExplainInfo: -BroadcastHashJoin is not supported -BroadcastNestedLoopJoin is not supported -Not all subqueries for union are native +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native +Union disabled because not all child plans are native Query: q78-v2.7. Comet Exec: Enabled (CometFilter) Query: q78-v2.7: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q80a-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q80a-v2.7. Comet Exec: Enabled (CometFilter, CometProject) Query: q80a-v2.7: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +Union disabled because not all child plans are native -Query: q86a-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q86a-v2.7. Comet Exec: Enabled (CometFilter, CometProject) Query: q86a-v2.7: ExplainInfo: -BroadcastHashJoin is not supported -Not all subqueries for union are native +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +Union disabled because not all child plans are native Window is not supported -Query: q98-v2.7. Comet Exec: Enabled (CometProject, CometFilter) +Query: q98-v2.7. Comet Exec: Enabled (CometFilter, CometProject) Query: q98-v2.7: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Window is not supported Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning + diff --git a/spark/inspections/CometTPCHQueriesList-results.txt b/spark/inspections/CometTPCHQueriesList-results.txt index c9bc6ce6de..b51286d807 100644 --- a/spark/inspections/CometTPCHQueriesList-results.txt +++ b/spark/inspections/CometTPCHQueriesList-results.txt @@ -1,121 +1,142 @@ -Query: q1 TPCH Snappy. Comet Exec: Enabled (CometHashAggregate, CometProject) +Query: q1 TPCH Snappy. Comet Exec: Enabled (CometHashAggregate, CometFilter, CometProject) Query: q1 TPCH Snappy: ExplainInfo: Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q2 TPCH Snappy. Comet Exec: Enabled (CometSort, CometSortMergeJoin, CometProject, CometFilter) +Query: q2 TPCH Snappy. Comet Exec: Enabled (CometFilter, CometProject) Query: q2 TPCH Snappy: ExplainInfo: +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native ObjectHashAggregate is not supported -might_contain is not supported -BroadcastHashJoin is not supported -SortMergeJoin is not supported +SortMergeJoin disabled because not all child plans are native +xxhash64 is not supported -Query: q3 TPCH Snappy. Comet Exec: Enabled (CometHashAggregate, CometSort, CometTakeOrderedAndProject, CometSortMergeJoin, CometProject, CometFilter) +Query: q3 TPCH Snappy. Comet Exec: Enabled (CometFilter, CometProject, CometSort) Query: q3 TPCH Snappy: ExplainInfo: +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native - -Query: q4 TPCH Snappy. Comet Exec: Enabled (CometHashAggregate, CometSort, CometSortMergeJoin, CometProject, CometFilter) +Query: q4 TPCH Snappy. Comet Exec: Enabled (CometFilter, CometProject) Query: q4 TPCH Snappy: ExplainInfo: +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q5 TPCH Snappy. Comet Exec: Enabled (CometSort, CometSortMergeJoin, CometProject, CometFilter) +Query: q5 TPCH Snappy. Comet Exec: Enabled (CometFilter, CometProject, CometSort) Query: q5 TPCH Snappy: ExplainInfo: +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native ObjectHashAggregate is not supported -might_contain is not supported -BroadcastHashJoin is not supported +SortMergeJoin disabled because not all child plans are native +xxhash64 is not supported Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q6 TPCH Snappy. Comet Exec: Enabled (CometHashAggregate, CometProject, CometFilter) +Query: q6 TPCH Snappy. Comet Exec: Enabled (CometHashAggregate, CometFilter, CometProject) Query: q6 TPCH Snappy: ExplainInfo: -Query: q7 TPCH Snappy. Comet Exec: Enabled (CometSort, CometProject, CometFilter) +Query: q7 TPCH Snappy. Comet Exec: Enabled (CometFilter) Query: q7 TPCH Snappy: ExplainInfo: -ObjectHashAggregate is not supported -might_contain is not supported -SortMergeJoin is not supported -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q8 TPCH Snappy. Comet Exec: Enabled (CometSort, CometSortMergeJoin, CometProject, CometFilter) +Query: q8 TPCH Snappy. Comet Exec: Enabled (CometFilter, CometProject) Query: q8 TPCH Snappy: ExplainInfo: -ObjectHashAggregate is not supported -might_contain is not supported -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q9 TPCH Snappy. Comet Exec: Enabled (CometSort, CometSortMergeJoin, CometProject, CometFilter) +Query: q9 TPCH Snappy. Comet Exec: Enabled (CometFilter, CometProject, CometSort) Query: q9 TPCH Snappy: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q10 TPCH Snappy. Comet Exec: Enabled (CometSort, CometSortMergeJoin, CometProject, CometFilter) +Query: q10 TPCH Snappy. Comet Exec: Enabled (CometFilter, CometProject, CometSort) Query: q10 TPCH Snappy: ExplainInfo: -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q11 TPCH Snappy. Comet Exec: Enabled (CometSort, CometProject, CometFilter) +Query: q11 TPCH Snappy. Comet Exec: Enabled (CometFilter, CometProject) Query: q11 TPCH Snappy: ExplainInfo: -ObjectHashAggregate is not supported -might_contain is not supported -SortMergeJoin is not supported -BroadcastHashJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q12 TPCH Snappy. Comet Exec: Enabled (CometHashAggregate, CometSort, CometSortMergeJoin, CometProject, CometFilter) +Query: q12 TPCH Snappy. Comet Exec: Enabled (CometFilter, CometProject) Query: q12 TPCH Snappy: ExplainInfo: +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q13 TPCH Snappy. Comet Exec: Enabled (CometHashAggregate, CometSort, CometSortMergeJoin, CometProject, CometFilter) +Query: q13 TPCH Snappy. Comet Exec: Enabled (CometFilter, CometProject) Query: q13 TPCH Snappy: ExplainInfo: +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q14 TPCH Snappy. Comet Exec: Enabled (CometHashAggregate, CometSort, CometSortMergeJoin, CometProject, CometFilter) +Query: q14 TPCH Snappy. Comet Exec: Enabled (CometFilter, CometProject) Query: q14 TPCH Snappy: ExplainInfo: +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native - -Query: q15 TPCH Snappy. Comet Exec: Enabled (CometHashAggregate, CometSort, CometSortMergeJoin, CometProject, CometFilter) +Query: q15 TPCH Snappy. Comet Exec: Enabled (CometHashAggregate, CometFilter, CometProject) Query: q15 TPCH Snappy: ExplainInfo: +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q16 TPCH Snappy. Comet Exec: Enabled (CometSort, CometProject, CometFilter) +Query: q16 TPCH Snappy. Comet Exec: Enabled (CometFilter, CometProject) Query: q16 TPCH Snappy: ExplainInfo: -BroadcastHashJoin is not supported -SortMergeJoin is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q17 TPCH Snappy. Comet Exec: Enabled (CometHashAggregate, CometSort, CometSortMergeJoin, CometProject, CometFilter) +Query: q17 TPCH Snappy. Comet Exec: Enabled (CometHashAggregate, CometFilter, CometProject, CometSort) Query: q17 TPCH Snappy: ExplainInfo: -Sort merge join with a join condition is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native -Query: q18 TPCH Snappy. Comet Exec: Enabled (CometHashAggregate, CometSort, CometTakeOrderedAndProject, CometSortMergeJoin, CometProject, CometFilter) +Query: q18 TPCH Snappy. Comet Exec: Enabled (CometHashAggregate, CometFilter, CometProject) Query: q18 TPCH Snappy: ExplainInfo: +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native +SortMergeJoin disabled because not all child plans are native - -Query: q19 TPCH Snappy. Comet Exec: Enabled (CometSort, CometProject, CometFilter) +Query: q19 TPCH Snappy. Comet Exec: Enabled (CometFilter, CometProject) Query: q19 TPCH Snappy: ExplainInfo: -Sort merge join with a join condition is not supported +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q20 TPCH Snappy. Comet Exec: Enabled (CometHashAggregate, CometSort, CometSortMergeJoin, CometProject, CometFilter) +Query: q20 TPCH Snappy. Comet Exec: Enabled (CometFilter, CometProject) Query: q20 TPCH Snappy: ExplainInfo: +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native ObjectHashAggregate is not supported -Sort merge join with a join condition is not supported -might_contain is not supported -SortMergeJoin is not supported -BroadcastHashJoin is not supported +SortMergeJoin disabled because not all child plans are native +xxhash64 is not supported Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q21 TPCH Snappy. Comet Exec: Enabled (CometSort, CometProject, CometFilter) +Query: q21 TPCH Snappy. Comet Exec: Enabled (CometFilter, CometProject, CometSort) Query: q21 TPCH Snappy: ExplainInfo: ObjectHashAggregate is not supported Sort merge join with a join condition is not supported -SortMergeJoin is not supported -might_contain is not supported -BroadcastHashJoin is not supported +xxhash64 is not supported +SortMergeJoin disabled because not all child plans are native +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native -Query: q22 TPCH Snappy. Comet Exec: Enabled (CometHashAggregate, CometSort, CometSortMergeJoin, CometProject, CometFilter) +Query: q22 TPCH Snappy. Comet Exec: Enabled (CometFilter) Query: q22 TPCH Snappy: ExplainInfo: +BroadcastExchange is not supported +BroadcastHashJoin disabled because not all child plans are native Shuffle: unsupported Spark partitioning: org.apache.spark.sql.catalyst.plans.physical.RangePartitioning -Query: q1 TPCH Extended Snappy. Comet Exec: Enabled (CometHashAggregate, CometProject, CometFilter) +Query: q1 TPCH Extended Snappy. Comet Exec: Enabled (CometHashAggregate, CometFilter, CometProject) Query: q1 TPCH Extended Snappy: ExplainInfo: diff --git a/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala b/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala index d5abd2b609..3fe830f047 100644 --- a/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala +++ b/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala @@ -46,7 +46,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.comet.CometConf._ -import org.apache.comet.CometSparkSessionExtensions.{isANSIEnabled, isCometBroadCastForceEnabled, isCometColumnarShuffleEnabled, isCometEnabled, isCometExecEnabled, isCometOperatorEnabled, isCometScan, isCometScanEnabled, isCometShuffleEnabled, isSchemaSupported, shouldApplyRowToColumnar, withInfo} +import org.apache.comet.CometSparkSessionExtensions.{createMessage, isANSIEnabled, isCometBroadCastForceEnabled, isCometColumnarShuffleEnabled, isCometEnabled, isCometExecEnabled, isCometOperatorEnabled, isCometScan, isCometScanEnabled, isCometShuffleEnabled, isSchemaSupported, shouldApplyRowToColumnar, withInfo} import org.apache.comet.parquet.{CometParquetScan, SupportsComet} import org.apache.comet.serde.OperatorOuterClass.Operator import org.apache.comet.serde.QueryPlanSerde @@ -101,20 +101,17 @@ class CometSparkSessionExtensions // unsupported parquet data source V2 case scanExec: BatchScanExec if scanExec.scan.isInstanceOf[ParquetScan] => val requiredSchema = scanExec.scan.asInstanceOf[ParquetScan].readDataSchema - var info1: Option[String] = None - if (isSchemaSupported(requiredSchema)) { - info1 = Some(s"Schema $requiredSchema is not supported") - } + val info1 = createMessage( + !isSchemaSupported(requiredSchema), + s"Schema $requiredSchema is not supported") val readPartitionSchema = scanExec.scan.asInstanceOf[ParquetScan].readPartitionSchema - var info2: Option[String] = None - if (isSchemaSupported(readPartitionSchema)) { - info2 = Some(s"Schema $readPartitionSchema is not supported") - } + val info2 = createMessage( + !isSchemaSupported(readPartitionSchema), + s"Schema $readPartitionSchema is not supported") // Comet does not support pushedAggregate - var info3: Option[String] = None - if (!getPushedAggregate(scanExec.scan.asInstanceOf[ParquetScan]).isEmpty) { - info3 = Some("Comet does not support pushed aggregate") - } + val info3 = createMessage( + getPushedAggregate(scanExec.scan.asInstanceOf[ParquetScan]).isDefined, + "Comet does not support pushed aggregate") withInfo(scanExec, Seq(info1, info2, info3).flatten.mkString("\n")) scanExec @@ -173,14 +170,12 @@ class CometSparkSessionExtensions _, _, _) => - var info1: Option[String] = None - if (!isSchemaSupported(requiredSchema)) { - info1 = Some(s"Schema $requiredSchema is not supported") - } - var info2: Option[String] = None - if (!isSchemaSupported(partitionSchema)) { - info2 = Some(s"Schema $partitionSchema is not supported") - } + val info1 = createMessage( + !isSchemaSupported(requiredSchema), + s"Schema $requiredSchema is not supported") + val info2 = createMessage( + !isSchemaSupported(partitionSchema), + s"Schema $partitionSchema is not supported") withInfo(scanExec, Seq(info1, info2).flatten.mkString(",")) scanExec } @@ -434,6 +429,14 @@ class CometSparkSessionExtensions op } + case op: ShuffledHashJoinExec if !isCometOperatorEnabled(conf, "hash_join") => + withInfo(op, "ShuffleHashJoin is not enabled") + op + + case op: ShuffledHashJoinExec if !op.children.forall(isCometNative(_)) => + withInfo(op, "ShuffleHashJoin disabled because not all child plans are native") + op + case op: BroadcastHashJoinExec if isCometOperatorEnabled(conf, "broadcast_hash_join") && op.children.forall(isCometNative(_)) => @@ -455,6 +458,14 @@ class CometSparkSessionExtensions op } + case op: BroadcastHashJoinExec if !isCometOperatorEnabled(conf, "broadcast_hash_join") => + withInfo(op, "BroadcastHashJoin is not enabled") + op + + case op: BroadcastHashJoinExec if !op.children.forall(isCometNative(_)) => + withInfo(op, "BroadcastHashJoin disabled because not all child plans are native") + op + case op: SortMergeJoinExec if isCometOperatorEnabled(conf, "sort_merge_join") && op.children.forall(isCometNative(_)) => @@ -474,6 +485,13 @@ class CometSparkSessionExtensions case None => op } + case op: SortMergeJoinExec if !isCometOperatorEnabled(conf, "sort_merge_join") => + withInfo(op, "SortMergeJoin is not enabled") + op + + case op: SortMergeJoinExec if !op.children.forall(isCometNative(_)) => + withInfo(op, "SortMergeJoin disabled because not all child plans are native") + op case c @ CoalesceExec(numPartitions, child) if isCometOperatorEnabled(conf, "coalesce") @@ -486,6 +504,14 @@ class CometSparkSessionExtensions c } + case c @ CoalesceExec(_, _) if !isCometOperatorEnabled(conf, "coalesce") => + withInfo(c, "Coalesce is not enabled") + c + + case op: CoalesceExec if !op.children.forall(isCometNative(_)) => + withInfo(op, "Coalesce disabled because not all child plans are native") + op + case s: TakeOrderedAndProjectExec if isCometNative(s.child) && isCometOperatorEnabled(conf, "takeOrderedAndProjectExec") && isCometShuffleEnabled(conf) && @@ -500,14 +526,12 @@ class CometSparkSessionExtensions } case s: TakeOrderedAndProjectExec => - var info1: Option[String] = None - if (!isCometOperatorEnabled(conf, "takeOrderedAndProjectExec")) { - info1 = Some("TakeOrderedAndProject is not enabled") - } - var info2: Option[String] = None - if (!isCometShuffleEnabled(conf)) { - info2 = Some("TakeOrderedAndProject requires shuffle to be enabled") - } + val info1 = createMessage( + !isCometOperatorEnabled(conf, "takeOrderedAndProjectExec"), + "TakeOrderedAndProject is not enabled") + val info2 = createMessage( + !isCometShuffleEnabled(conf), + "TakeOrderedAndProject requires shuffle to be enabled") withInfo(s, Seq(info1, info2).flatten.mkString(",")) s @@ -522,18 +546,14 @@ class CometSparkSessionExtensions u } - case u: UnionExec => - var info1: Option[String] = None - if (!isCometOperatorEnabled(conf, "union")) { - info1 = Some("Union is not enabled") - } - var info2: Option[String] = None - if (!u.children.forall(isCometNative)) { - info2 = Some("Not all subqueries for union are native") - } - withInfo(u, Seq(info1, info2).flatten.mkString(",")) + case u: UnionExec if !isCometOperatorEnabled(conf, "union") => + withInfo(u, "Union is not enabled") u + case op: UnionExec if !op.children.forall(isCometNative(_)) => + withInfo(op, "Union disabled because not all child plans are native") + op + // For AQE broadcast stage on a Comet broadcast exchange case s @ BroadcastQueryStageExec(_, _: CometBroadcastExchangeExec, _) => val newOp = transform1(s) @@ -645,27 +665,19 @@ class CometSparkSessionExtensions case s: ShuffleExchangeExec => val isShuffleEnabled = isCometShuffleEnabled(conf) - var msg1: Option[String] = None - if (!isShuffleEnabled) { - msg1 = Some("Native shuffle is not enabled") - } + val msg1 = createMessage(!isShuffleEnabled, "Native shuffle is not enabled") val columnarShuffleEnabled = isCometColumnarShuffleEnabled(conf) - var msg2: Option[String] = None - if (isShuffleEnabled && !columnarShuffleEnabled && !QueryPlanSerde + val msg2 = createMessage( + isShuffleEnabled && !columnarShuffleEnabled && !QueryPlanSerde .supportPartitioning(s.child.output, s.outputPartitioning) - ._1) { - msg2 = Some( - "Shuffle: " + - s"${QueryPlanSerde.supportPartitioning(s.child.output, s.outputPartitioning)._2}") - } - var msg3: Option[String] = None - if (isShuffleEnabled && columnarShuffleEnabled && !QueryPlanSerde + ._1, + "Shuffle: " + + s"${QueryPlanSerde.supportPartitioning(s.child.output, s.outputPartitioning)._2}") + val msg3 = createMessage( + isShuffleEnabled && columnarShuffleEnabled && !QueryPlanSerde .supportPartitioningTypes(s.child.output) - ._1) { - val info = - QueryPlanSerde.supportPartitioningTypes(s.child.output)._2 - msg3 = Some(s"Columnar shuffle: $info") - } + ._1, + s"Columnar shuffle: ${QueryPlanSerde.supportPartitioningTypes(s.child.output)._2}") withInfo(s, Seq(msg1, msg2, msg3).flatten.mkString(",")) s @@ -981,4 +993,13 @@ object CometSparkSessionExtensions extends Logging { } node } + + // Helper to reduce boilerplate + def createMessage(condition: Boolean, message: => String): Option[String] = { + if (condition) { + Some(message) + } else { + None + } + } } diff --git a/spark/src/main/scala/org/apache/comet/ExtendedExplainInfo.scala b/spark/src/main/scala/org/apache/comet/ExtendedExplainInfo.scala index aebbfdda9a..cbf3e76dbd 100644 --- a/spark/src/main/scala/org/apache/comet/ExtendedExplainInfo.scala +++ b/spark/src/main/scala/org/apache/comet/ExtendedExplainInfo.scala @@ -32,7 +32,7 @@ class ExtendedExplainInfo extends ExtendedExplainGenerator { override def generateExtendedInfo(plan: SparkPlan): String = { val info = extensionInfo(plan) - info.distinct.mkString("\n") + info.distinct.mkString("\n").trim } private def getActualPlan(node: TreeNode[_]): TreeNode[_] = { @@ -49,13 +49,13 @@ class ExtendedExplainInfo extends ExtendedExplainGenerator { var info = mutable.Seq[String]() val sorted = sortup(node) sorted.foreach { p => - val s = - getActualPlan(p).getTagValue(CometExplainInfo.EXTENSION_INFO).getOrElse("") - if (s.nonEmpty) { + val all: Array[String] = + getActualPlan(p).getTagValue(CometExplainInfo.EXTENSION_INFO).getOrElse("").split("\n") + for (s <- all) { info = info :+ s } } - info + info.filter(!_.contentEquals("\n")) } // get all plan nodes, breadth first, leaf nodes first diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 1e85637364..ab5c7f97bb 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -2001,7 +2001,7 @@ object QueryPlanSerde extends Logging with ShimQueryPlanSerde { val filterBuilder = OperatorOuterClass.Filter.newBuilder().setPredicate(cond.get) Some(result.setFilter(filterBuilder).build()) } else { - withInfo(op, null, child) + withInfo(op, null, condition, child) None } @@ -2107,7 +2107,7 @@ object QueryPlanSerde extends Logging with ShimQueryPlanSerde { if (resultExprs.exists(_.isEmpty)) { val msg = s"Unsupported result expressions found in: ${resultExpressions}" emitWarning(msg) - withInfo(op, msg) + withInfo(op, msg, resultExpressions: _*) return None } hashAggBuilder.addAllResultExprs(resultExprs.map(_.get).asJava) @@ -2151,7 +2151,7 @@ object QueryPlanSerde extends Logging with ShimQueryPlanSerde { if (resultExprs.exists(_.isEmpty)) { val msg = s"Unsupported result expressions found in: ${resultExpressions}" emitWarning(msg) - withInfo(op, msg) + withInfo(op, msg, resultExpressions: _*) return None } hashAggBuilder.addAllResultExprs(resultExprs.map(_.get).asJava) @@ -2287,6 +2287,10 @@ object QueryPlanSerde extends Logging with ShimQueryPlanSerde { None } + case join: SortMergeJoinExec if !isCometOperatorEnabled(op.conf, "sort_merge_join") => + withInfo(join, "SortMergeJoin is not enabled") + None + case op if isCometSink(op) => // These operators are source of Comet native execution chain val scanBuilder = OperatorOuterClass.Scan.newBuilder() diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 87c2126af6..422cecc5fb 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -1422,7 +1422,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { "AQEShuffleRead is not supported\n" + "make_interval is not supported\n" + "BroadcastExchange is not supported\n" + - "BroadcastHashJoin is not supported")) + "BroadcastHashJoin disabled because not all child plans are native")) .foreach(test => { val qry = test._1 val expected = test._2 diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryListBase.scala b/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryListBase.scala index e0d28c647b..a008dbd58e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryListBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryListBase.scala @@ -33,7 +33,6 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.comet.{CometConf, ExtendedExplainInfo} import org.apache.comet.shims.ShimCometSparkSessionExtensions -import org.apache.comet.shims.ShimCometSparkSessionExtensions.supportsExtendedExplainInfo trait CometTPCQueryListBase extends CometTPCQueryBase @@ -88,7 +87,9 @@ trait CometTPCQueryListBase CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", - CometConf.COMET_EXEC_ALL_OPERATOR_ENABLED.key -> "true") { + CometConf.COMET_EXEC_ALL_OPERATOR_ENABLED.key -> "true", + "spark.sql.optimizer.runtime.bloomFilter.creationSideThreshold" -> "1MB", + "spark.sql.optimizer.runtime.bloomFilter.applicationSideScanSizeThreshold" -> "1MB") { val df = cometSpark.sql(queryString) val cometPlans = mutable.HashSet.empty[String] @@ -105,11 +106,9 @@ trait CometTPCQueryListBase } else { out.println(s"Query: $name$nameSuffix. Comet Exec: Disabled") } - if (supportsExtendedExplainInfo(df.queryExecution)) { - out.println( - s"Query: $name$nameSuffix: ExplainInfo:\n" + - s"${new ExtendedExplainInfo().generateExtendedInfo(executedPlan)}\n") - } + out.println( + s"Query: $name$nameSuffix: ExplainInfo:\n" + + s"${new ExtendedExplainInfo().generateExtendedInfo(executedPlan)}\n") } } }