diff --git a/Parser/ParserNode.cpp b/Parser/ParserNode.cpp index 7f066bc27f..5699889f87 100644 --- a/Parser/ParserNode.cpp +++ b/Parser/ParserNode.cpp @@ -2643,6 +2643,10 @@ std::shared_ptr getResultSet(QueryStateProxy query_state_proxy, g_pending_query_interrupt_freq, ExecutorType::Native, outer_fragment_indices}; + // we can skip to check rowwise query hint since we assume false by default in this eo + const auto columnar_output_enabled = + query_hints.isHintRegistered(QueryHint::kColumnarOutput); + eo.output_columnar_hint = columnar_output_enabled; ExecutionResult result{std::make_shared(std::vector{}, ExecutorDeviceType::CPU, QueryMemoryDescriptor(), @@ -2699,6 +2703,10 @@ size_t LocalConnector::getOuterFragmentCount(QueryStateProxy query_state_proxy, false, 0.9, false}; + // we can skip to check rowwise query hint since we assume false by default in this eo + const auto columnar_output_enabled = + query_hints.isHintRegistered(QueryHint::kColumnarOutput); + eo.output_columnar_hint = columnar_output_enabled; return ra_executor.getOuterFragmentCount(co, eo); } diff --git a/QueryEngine/QueryHint.h b/QueryEngine/QueryHint.h index 398e58518f..8bd44ae75c 100644 --- a/QueryEngine/QueryHint.h +++ b/QueryEngine/QueryHint.h @@ -27,6 +27,8 @@ // and let remaining enum value to be auto-incremented enum QueryHint { kCpuMode = 0, + kColumnarOutput, + kRowwiseOutput, kOverlapsBucketThreshold, kOverlapsMaxSize, kOverlapsAllowGpuBuild, @@ -39,6 +41,8 @@ enum QueryHint { static const std::unordered_map SupportedQueryHints = { {"cpu_mode", QueryHint::kCpuMode}, + {"columnar_output", QueryHint::kColumnarOutput}, + {"rowwise_output", QueryHint::kRowwiseOutput}, {"overlaps_bucket_threshold", QueryHint::kOverlapsBucketThreshold}, {"overlaps_max_size", QueryHint::kOverlapsMaxSize}, {"overlaps_allow_gpu_build", QueryHint::kOverlapsAllowGpuBuild}, @@ -132,6 +136,8 @@ struct RegisteredQueryHint { // registered and its detailed info such as the hint's parameter values given by user RegisteredQueryHint() : cpu_mode(false) + , columnar_output(g_enable_columnar_output) + , rowwise_output(!g_enable_columnar_output) , overlaps_bucket_threshold(std::numeric_limits::max()) , overlaps_max_size(g_overlaps_max_table_size_bytes) , overlaps_allow_gpu_build(true) @@ -141,6 +147,8 @@ struct RegisteredQueryHint { RegisteredQueryHint& operator=(const RegisteredQueryHint& other) { cpu_mode = other.cpu_mode; + columnar_output = other.columnar_output; + rowwise_output = other.rowwise_output; overlaps_bucket_threshold = other.overlaps_bucket_threshold; overlaps_max_size = other.overlaps_max_size; overlaps_allow_gpu_build = other.overlaps_allow_gpu_build; @@ -152,6 +160,8 @@ struct RegisteredQueryHint { RegisteredQueryHint(const RegisteredQueryHint& other) { cpu_mode = other.cpu_mode; + columnar_output = other.columnar_output; + rowwise_output = other.rowwise_output; overlaps_bucket_threshold = other.overlaps_bucket_threshold; overlaps_max_size = other.overlaps_max_size; overlaps_allow_gpu_build = other.overlaps_allow_gpu_build; @@ -162,6 +172,8 @@ struct RegisteredQueryHint { // general query execution bool cpu_mode; + bool columnar_output; + bool rowwise_output; // overlaps hash join double overlaps_bucket_threshold; // defined in "OverlapsJoinHashTable.h" diff --git a/QueryEngine/RelAlgDagBuilder.h b/QueryEngine/RelAlgDagBuilder.h index 053910d8ce..3c17523b99 100644 --- a/QueryEngine/RelAlgDagBuilder.h +++ b/QueryEngine/RelAlgDagBuilder.h @@ -2160,6 +2160,8 @@ class RelAlgDagBuilder : public boost::noncopyable { } void registerQueryHints(Hints* hints_delivered) { + bool detect_columnar_output_hint = false; + bool detect_rowwise_output_hint = false; for (auto it = hints_delivered->begin(); it != hints_delivered->end(); it++) { auto target = it->second; auto hint_type = it->first; @@ -2170,6 +2172,14 @@ class RelAlgDagBuilder : public boost::noncopyable { VLOG(1) << "A user forces to run the query on the CPU execution mode"; break; } + case QueryHint::kColumnarOutput: { + detect_columnar_output_hint = true; + break; + } + case QueryHint::kRowwiseOutput: { + detect_rowwise_output_hint = true; + break; + } case QueryHint::kOverlapsBucketThreshold: { CHECK(target.getListOptions().size() == 1); double overlaps_bucket_threshold = std::stod(target.getListOptions()[0]); @@ -2227,6 +2237,42 @@ class RelAlgDagBuilder : public boost::noncopyable { break; } } + // we have four cases depending on 1) g_enable_columnar_output flag + // and 2) query hint status: columnar_output and rowwise_output + // case 1. g_enable_columnar_output = true + // case 1.a) columnar_output = true (so rowwise_output = false); + // case 1.b) rowwise_output = true (so columnar_output = false); + // case 2. g_enable_columnar_output = false + // case 2.a) columnar_output = true (so rowwise_output = false); + // case 2.b) rowwise_output = true (so columnar_output = false); + // case 1.a --> use columnar output + // case 1.b --> use rowwise output + // case 2.a --> use columnar output + // case 2.b --> use rowwise output + if (detect_columnar_output_hint && detect_rowwise_output_hint) { + VLOG(1) + << "Two hints 1) columnar output and 2) rowwise output are enabled together, " + << "so skip them and use the runtime configuration " + "\"g_enable_columnar_output\""; + } else if (detect_columnar_output_hint && !detect_rowwise_output_hint) { + if (g_enable_columnar_output) { + VLOG(1) << "We already enable columnar output by default " + "(g_enable_columnar_output = true), so skip this columnar output hint"; + } else { + query_hint_.registerHint(QueryHint::kColumnarOutput); + query_hint_.columnar_output = true; + VLOG(1) << "A user forces the query to run with columnar output"; + } + } else if (!detect_columnar_output_hint && detect_rowwise_output_hint) { + if (!g_enable_columnar_output) { + VLOG(1) << "We already use the default rowwise output (g_enable_columnar_output " + "= false), so skip this rowwise output hint"; + } else { + query_hint_.registerHint(QueryHint::kRowwiseOutput); + query_hint_.rowwise_output = true; + VLOG(1) << "A user forces the query to run with rowwise output"; + } + } } const RegisteredQueryHint getQueryHints() const { return query_hint_; } diff --git a/QueryRunner/QueryRunner.cpp b/QueryRunner/QueryRunner.cpp index 1324d240b1..8455b4ae0e 100644 --- a/QueryRunner/QueryRunner.cpp +++ b/QueryRunner/QueryRunner.cpp @@ -582,6 +582,11 @@ std::shared_ptr QueryRunner::runSQLWithAllowingInterrupt( if (cpu_mode_enabled) { co.device_type = ExecutorDeviceType::CPU; } + if (query_hints.isHintRegistered(QueryHint::kColumnarOutput)) { + eo.output_columnar_hint = true; + } else if (query_hints.isHintRegistered(QueryHint::kRowwiseOutput)) { + eo.output_columnar_hint = false; + } result = std::make_shared( ra_executor.executeRelAlgQuery(co, eo, false, nullptr)); }); @@ -693,6 +698,11 @@ std::shared_ptr run_select_query_with_filter_push_down( if (cpu_mode_enabled) { co.device_type = ExecutorDeviceType::CPU; } + if (query_hints.isHintRegistered(QueryHint::kColumnarOutput)) { + eo.output_columnar_hint = true; + } else if (query_hints.isHintRegistered(QueryHint::kRowwiseOutput)) { + eo.output_columnar_hint = false; + } auto result = std::make_shared( ra_executor.executeRelAlgQuery(co, eo, false, nullptr)); const auto& filter_push_down_requests = result->getPushedDownFilterInfo(); @@ -782,6 +792,11 @@ std::shared_ptr QueryRunner::runSelectQuery(const std::string& if (cpu_mode_enabled) { co.device_type = ExecutorDeviceType::CPU; } + if (query_hints.isHintRegistered(QueryHint::kColumnarOutput)) { + eo.output_columnar_hint = true; + } else if (query_hints.isHintRegistered(QueryHint::kRowwiseOutput)) { + eo.output_columnar_hint = false; + } result = std::make_shared( ra_executor.executeRelAlgQuery(co, eo, false, nullptr)); }); diff --git a/Tests/SQLHintTest.cpp b/Tests/SQLHintTest.cpp index 3e6b61f2b5..288edf386e 100644 --- a/Tests/SQLHintTest.cpp +++ b/Tests/SQLHintTest.cpp @@ -227,6 +227,138 @@ TEST(QueryHint, CheckQueryHintForOverlapsJoin) { } } +TEST(QueryHint, checkQueryLayoutHintWithEnablingColumnarOutput) { + const auto enable_columnar_output = g_enable_columnar_output; + g_enable_columnar_output = true; + ScopeGuard reset_columnar_output = [&enable_columnar_output] { + g_enable_columnar_output = enable_columnar_output; + }; + + const auto create_table_ddl = "CREATE TABLE SQL_HINT_DUMMY(key int)"; + const auto drop_table_ddl = "DROP TABLE IF EXISTS SQL_HINT_DUMMY"; + const auto q1 = "SELECT /*+ columnar_output */ * FROM SQL_HINT_DUMMY"; + const auto q2 = "SELECT /*+ rowwise_output */ * FROM SQL_HINT_DUMMY"; + const auto q3 = "SELECT /*+ columnar_output, rowwise_output */ * FROM SQL_HINT_DUMMY"; + const auto q4 = "SELECT /*+ rowwise_output, columnar_output */ * FROM SQL_HINT_DUMMY"; + const auto q5 = + "SELECT /*+ rowwise_output, columnar_output, rowwise_output */ * FROM " + "SQL_HINT_DUMMY"; + const auto q6 = "SELECT /*+ rowwise_output, rowwise_output */ * FROM SQL_HINT_DUMMY"; + const auto q7 = "SELECT /*+ columnar_output, columnar_output */ * FROM SQL_HINT_DUMMY"; + QR::get()->runDDLStatement(drop_table_ddl); + QR::get()->runDDLStatement(create_table_ddl); + + { + auto query_hints = QR::get()->getParsedQueryHint(q1); + auto hint_enabled = query_hints.isHintRegistered(QueryHint::kColumnarOutput); + CHECK(!hint_enabled); + } + + { + auto query_hints = QR::get()->getParsedQueryHint(q2); + auto hint_enabled = query_hints.isHintRegistered(QueryHint::kRowwiseOutput); + CHECK(hint_enabled); + } + + { + auto query_hints = QR::get()->getParsedQueryHint(q3); + auto hint_enabled = query_hints.isAnyQueryHintDelivered(); + CHECK(!hint_enabled); + } + + { + auto query_hints = QR::get()->getParsedQueryHint(q4); + auto hint_enabled = query_hints.isAnyQueryHintDelivered(); + CHECK(!hint_enabled); + } + + { + auto query_hints = QR::get()->getParsedQueryHint(q5); + auto hint_enabled = query_hints.isAnyQueryHintDelivered(); + CHECK(!hint_enabled); + } + + { + auto query_hints = QR::get()->getParsedQueryHint(q6); + auto hint_enabled = query_hints.isHintRegistered(QueryHint::kRowwiseOutput); + CHECK(hint_enabled); + } + + { + auto query_hints = QR::get()->getParsedQueryHint(q7); + auto hint_enabled = query_hints.isHintRegistered(QueryHint::kColumnarOutput); + CHECK(!hint_enabled); + } + + QR::get()->runDDLStatement(drop_table_ddl); +} + +TEST(QueryHint, checkQueryLayoutHintWithoutEnablingColumnarOutput) { + const auto enable_columnar_output = g_enable_columnar_output; + g_enable_columnar_output = false; + ScopeGuard reset_columnar_output = [&enable_columnar_output] { + g_enable_columnar_output = enable_columnar_output; + }; + + const auto create_table_ddl = "CREATE TABLE SQL_HINT_DUMMY(key int)"; + const auto drop_table_ddl = "DROP TABLE IF EXISTS SQL_HINT_DUMMY"; + const auto q1 = "SELECT /*+ columnar_output */ * FROM SQL_HINT_DUMMY"; + const auto q2 = "SELECT /*+ rowwise_output */ * FROM SQL_HINT_DUMMY"; + const auto q3 = "SELECT /*+ columnar_output, rowwise_output */ * FROM SQL_HINT_DUMMY"; + const auto q4 = "SELECT /*+ rowwise_output, columnar_output */ * FROM SQL_HINT_DUMMY"; + const auto q5 = + "SELECT /*+ rowwise_output, columnar_output, rowwise_output */ * FROM " + "SQL_HINT_DUMMY"; + const auto q6 = "SELECT /*+ rowwise_output, rowwise_output */ * FROM SQL_HINT_DUMMY"; + const auto q7 = "SELECT /*+ columnar_output, columnar_output */ * FROM SQL_HINT_DUMMY"; + QR::get()->runDDLStatement(drop_table_ddl); + QR::get()->runDDLStatement(create_table_ddl); + + { + auto query_hints = QR::get()->getParsedQueryHint(q1); + auto hint_enabled = query_hints.isHintRegistered(QueryHint::kColumnarOutput); + CHECK(hint_enabled); + } + + { + auto query_hints = QR::get()->getParsedQueryHint(q2); + auto hint_enabled = query_hints.isHintRegistered(QueryHint::kRowwiseOutput); + CHECK(!hint_enabled); + } + + { + auto query_hints = QR::get()->getParsedQueryHint(q3); + auto hint_enabled = query_hints.isAnyQueryHintDelivered(); + CHECK(!hint_enabled); + } + + { + auto query_hints = QR::get()->getParsedQueryHint(q4); + auto hint_enabled = query_hints.isAnyQueryHintDelivered(); + CHECK(!hint_enabled); + } + + { + auto query_hints = QR::get()->getParsedQueryHint(q5); + auto hint_enabled = query_hints.isAnyQueryHintDelivered(); + CHECK(!hint_enabled); + } + + { + auto query_hints = QR::get()->getParsedQueryHint(q6); + auto hint_enabled = query_hints.isHintRegistered(QueryHint::kRowwiseOutput); + CHECK(!hint_enabled); + } + + { + auto query_hints = QR::get()->getParsedQueryHint(q7); + auto hint_enabled = query_hints.isHintRegistered(QueryHint::kColumnarOutput); + CHECK(hint_enabled); + } + + QR::get()->runDDLStatement(drop_table_ddl); +} + int main(int argc, char** argv) { TestHelpers::init_logger_stderr_only(argc, argv); testing::InitGoogleTest(&argc, argv); diff --git a/ThriftHandler/DBHandler.cpp b/ThriftHandler/DBHandler.cpp index 8e6d5b3c67..4d7587b374 100644 --- a/ThriftHandler/DBHandler.cpp +++ b/ThriftHandler/DBHandler.cpp @@ -5665,7 +5665,13 @@ std::vector DBHandler::execute_rel_alg( intel_jit_profile_}; auto validate_or_explain_query = explain_info.justExplain() || explain_info.justCalciteExplain() || just_validate; - ExecutionOptions eo = {g_enable_columnar_output, + auto columnar_output_enabled = g_enable_columnar_output; + if (query_hints.isHintRegistered(QueryHint::kColumnarOutput)) { + columnar_output_enabled = true; + } else if (query_hints.isHintRegistered(QueryHint::kRowwiseOutput)) { + columnar_output_enabled = false; + } + ExecutionOptions eo = {columnar_output_enabled, allow_multifrag_, explain_info.justExplain(), allow_loop_joins_ || just_validate, @@ -5755,6 +5761,11 @@ void DBHandler::execute_rel_alg_df(TDataFrame& _return, .empty(), g_running_query_interrupt_freq, g_pending_query_interrupt_freq}; + if (query_hints.isHintRegistered(QueryHint::kColumnarOutput)) { + eo.output_columnar_hint = true; + } else if (query_hints.isHintRegistered(QueryHint::kRowwiseOutput)) { + eo.output_columnar_hint = false; + } ExecutionResult result{std::make_shared(std::vector{}, ExecutorDeviceType::CPU, QueryMemoryDescriptor(), diff --git a/java/calcite/src/main/java/com/mapd/parser/hint/OmniSciHintStrategyTable.java b/java/calcite/src/main/java/com/mapd/parser/hint/OmniSciHintStrategyTable.java index f1b8a2bafa..23470adde3 100644 --- a/java/calcite/src/main/java/com/mapd/parser/hint/OmniSciHintStrategyTable.java +++ b/java/calcite/src/main/java/com/mapd/parser/hint/OmniSciHintStrategyTable.java @@ -12,6 +12,8 @@ private static HintStrategyTable createHintStrategies() { static HintStrategyTable createHintStrategies(HintStrategyTable.Builder builder) { return builder.hintStrategy("cpu_mode", HintPredicates.SET_VAR) + .hintStrategy("columnar_output", HintPredicates.SET_VAR) + .hintStrategy("rowwise_output", HintPredicates.SET_VAR) .hintStrategy("overlaps_bucket_threshold", HintPredicates.SET_VAR) .hintStrategy("overlaps_max_size", HintPredicates.SET_VAR) .hintStrategy("overlaps_allow_gpu_build", HintPredicates.SET_VAR)