Skip to content

Commit

Permalink
Add query hint flag to enable columnar output on a per-query basis
Browse files Browse the repository at this point in the history 
* Add rowwise and columnar output query hint

* Add tests

Co-authored-by: Todd Mostak <[email protected]>
  • Loading branch information
@yoonminnam @tmostak @andrewseidl
2 people authored and andrewseidl committed Aug 12, 2021
1 parent 8e1940f commit 4ef9775
Show file tree
Hide file tree
Showing 7 changed files with 227 additions and 1 deletion.
8 changes: 8 additions & 0 deletions Parser/ParserNode.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2643,6 +2643,10 @@ std::shared_ptr<ResultSet> getResultSet(QueryStateProxy query_state_proxy,
g_pending_query_interrupt_freq,
ExecutorType::Native,
outer_fragment_indices};
// we can skip to check rowwise query hint since we assume false by default in this eo
const auto columnar_output_enabled =
query_hints.isHintRegistered(QueryHint::kColumnarOutput);
eo.output_columnar_hint = columnar_output_enabled;
ExecutionResult result{std::make_shared<ResultSet>(std::vector<TargetInfo>{},
ExecutorDeviceType::CPU,
QueryMemoryDescriptor(),
Expand Down Expand Up @@ -2699,6 +2703,10 @@ size_t LocalConnector::getOuterFragmentCount(QueryStateProxy query_state_proxy,
false,
0.9,
false};
// we can skip to check rowwise query hint since we assume false by default in this eo
const auto columnar_output_enabled =
query_hints.isHintRegistered(QueryHint::kColumnarOutput);
eo.output_columnar_hint = columnar_output_enabled;
return ra_executor.getOuterFragmentCount(co, eo);
}

Expand Down
12 changes: 12 additions & 0 deletions QueryEngine/QueryHint.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
// and let remaining enum value to be auto-incremented
enum QueryHint {
kCpuMode = 0,
kColumnarOutput,
kRowwiseOutput,
kOverlapsBucketThreshold,
kOverlapsMaxSize,
kOverlapsAllowGpuBuild,
Expand All @@ -39,6 +41,8 @@ enum QueryHint {

static const std::unordered_map<std::string, QueryHint> SupportedQueryHints = {
{"cpu_mode", QueryHint::kCpuMode},
{"columnar_output", QueryHint::kColumnarOutput},
{"rowwise_output", QueryHint::kRowwiseOutput},
{"overlaps_bucket_threshold", QueryHint::kOverlapsBucketThreshold},
{"overlaps_max_size", QueryHint::kOverlapsMaxSize},
{"overlaps_allow_gpu_build", QueryHint::kOverlapsAllowGpuBuild},
Expand Down Expand Up @@ -132,6 +136,8 @@ struct RegisteredQueryHint {
// registered and its detailed info such as the hint's parameter values given by user
RegisteredQueryHint()
: cpu_mode(false)
, columnar_output(g_enable_columnar_output)
, rowwise_output(!g_enable_columnar_output)
, overlaps_bucket_threshold(std::numeric_limits<double>::max())
, overlaps_max_size(g_overlaps_max_table_size_bytes)
, overlaps_allow_gpu_build(true)
Expand All @@ -141,6 +147,8 @@ struct RegisteredQueryHint {

RegisteredQueryHint& operator=(const RegisteredQueryHint& other) {
cpu_mode = other.cpu_mode;
columnar_output = other.columnar_output;
rowwise_output = other.rowwise_output;
overlaps_bucket_threshold = other.overlaps_bucket_threshold;
overlaps_max_size = other.overlaps_max_size;
overlaps_allow_gpu_build = other.overlaps_allow_gpu_build;
Expand All @@ -152,6 +160,8 @@ struct RegisteredQueryHint {

RegisteredQueryHint(const RegisteredQueryHint& other) {
cpu_mode = other.cpu_mode;
columnar_output = other.columnar_output;
rowwise_output = other.rowwise_output;
overlaps_bucket_threshold = other.overlaps_bucket_threshold;
overlaps_max_size = other.overlaps_max_size;
overlaps_allow_gpu_build = other.overlaps_allow_gpu_build;
Expand All @@ -162,6 +172,8 @@ struct RegisteredQueryHint {

// general query execution
bool cpu_mode;
bool columnar_output;
bool rowwise_output;

// overlaps hash join
double overlaps_bucket_threshold; // defined in "OverlapsJoinHashTable.h"
Expand Down
46 changes: 46 additions & 0 deletions QueryEngine/RelAlgDagBuilder.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2160,6 +2160,8 @@ class RelAlgDagBuilder : public boost::noncopyable {
}
void registerQueryHints(Hints* hints_delivered) {
bool detect_columnar_output_hint = false;
bool detect_rowwise_output_hint = false;
for (auto it = hints_delivered->begin(); it != hints_delivered->end(); it++) {
auto target = it->second;
auto hint_type = it->first;
Expand All @@ -2170,6 +2172,14 @@ class RelAlgDagBuilder : public boost::noncopyable {
VLOG(1) << "A user forces to run the query on the CPU execution mode";
break;
}
case QueryHint::kColumnarOutput: {
detect_columnar_output_hint = true;
break;
}
case QueryHint::kRowwiseOutput: {
detect_rowwise_output_hint = true;
break;
}
case QueryHint::kOverlapsBucketThreshold: {
CHECK(target.getListOptions().size() == 1);
double overlaps_bucket_threshold = std::stod(target.getListOptions()[0]);
Expand Down Expand Up @@ -2227,6 +2237,42 @@ class RelAlgDagBuilder : public boost::noncopyable {
break;
}
}
// we have four cases depending on 1) g_enable_columnar_output flag
// and 2) query hint status: columnar_output and rowwise_output
// case 1. g_enable_columnar_output = true
// case 1.a) columnar_output = true (so rowwise_output = false);
// case 1.b) rowwise_output = true (so columnar_output = false);
// case 2. g_enable_columnar_output = false
// case 2.a) columnar_output = true (so rowwise_output = false);
// case 2.b) rowwise_output = true (so columnar_output = false);
// case 1.a --> use columnar output
// case 1.b --> use rowwise output
// case 2.a --> use columnar output
// case 2.b --> use rowwise output
if (detect_columnar_output_hint && detect_rowwise_output_hint) {
VLOG(1)
<< "Two hints 1) columnar output and 2) rowwise output are enabled together, "
<< "so skip them and use the runtime configuration "
"\"g_enable_columnar_output\"";
} else if (detect_columnar_output_hint && !detect_rowwise_output_hint) {
if (g_enable_columnar_output) {
VLOG(1) << "We already enable columnar output by default "
"(g_enable_columnar_output = true), so skip this columnar output hint";
} else {
query_hint_.registerHint(QueryHint::kColumnarOutput);
query_hint_.columnar_output = true;
VLOG(1) << "A user forces the query to run with columnar output";
}
} else if (!detect_columnar_output_hint && detect_rowwise_output_hint) {
if (!g_enable_columnar_output) {
VLOG(1) << "We already use the default rowwise output (g_enable_columnar_output "
"= false), so skip this rowwise output hint";
} else {
query_hint_.registerHint(QueryHint::kRowwiseOutput);
query_hint_.rowwise_output = true;
VLOG(1) << "A user forces the query to run with rowwise output";
}
}
}

const RegisteredQueryHint getQueryHints() const { return query_hint_; }
Expand Down
15 changes: 15 additions & 0 deletions QueryRunner/QueryRunner.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -582,6 +582,11 @@ std::shared_ptr<ResultSet> QueryRunner::runSQLWithAllowingInterrupt(
if (cpu_mode_enabled) {
co.device_type = ExecutorDeviceType::CPU;
}
if (query_hints.isHintRegistered(QueryHint::kColumnarOutput)) {
eo.output_columnar_hint = true;
} else if (query_hints.isHintRegistered(QueryHint::kRowwiseOutput)) {
eo.output_columnar_hint = false;
}
result = std::make_shared<ExecutionResult>(
ra_executor.executeRelAlgQuery(co, eo, false, nullptr));
});
Expand Down Expand Up @@ -693,6 +698,11 @@ std::shared_ptr<ExecutionResult> run_select_query_with_filter_push_down(
if (cpu_mode_enabled) {
co.device_type = ExecutorDeviceType::CPU;
}
if (query_hints.isHintRegistered(QueryHint::kColumnarOutput)) {
eo.output_columnar_hint = true;
} else if (query_hints.isHintRegistered(QueryHint::kRowwiseOutput)) {
eo.output_columnar_hint = false;
}
auto result = std::make_shared<ExecutionResult>(
ra_executor.executeRelAlgQuery(co, eo, false, nullptr));
const auto& filter_push_down_requests = result->getPushedDownFilterInfo();
Expand Down Expand Up @@ -782,6 +792,11 @@ std::shared_ptr<ExecutionResult> QueryRunner::runSelectQuery(const std::string&
if (cpu_mode_enabled) {
co.device_type = ExecutorDeviceType::CPU;
}
if (query_hints.isHintRegistered(QueryHint::kColumnarOutput)) {
eo.output_columnar_hint = true;
} else if (query_hints.isHintRegistered(QueryHint::kRowwiseOutput)) {
eo.output_columnar_hint = false;
}
result = std::make_shared<ExecutionResult>(
ra_executor.executeRelAlgQuery(co, eo, false, nullptr));
});
Expand Down
132 changes: 132 additions & 0 deletions Tests/SQLHintTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,138 @@ TEST(QueryHint, CheckQueryHintForOverlapsJoin) {
}
}

TEST(QueryHint, checkQueryLayoutHintWithEnablingColumnarOutput) {
const auto enable_columnar_output = g_enable_columnar_output;
g_enable_columnar_output = true;
ScopeGuard reset_columnar_output = [&enable_columnar_output] {
g_enable_columnar_output = enable_columnar_output;
};

const auto create_table_ddl = "CREATE TABLE SQL_HINT_DUMMY(key int)";
const auto drop_table_ddl = "DROP TABLE IF EXISTS SQL_HINT_DUMMY";
const auto q1 = "SELECT /*+ columnar_output */ * FROM SQL_HINT_DUMMY";
const auto q2 = "SELECT /*+ rowwise_output */ * FROM SQL_HINT_DUMMY";
const auto q3 = "SELECT /*+ columnar_output, rowwise_output */ * FROM SQL_HINT_DUMMY";
const auto q4 = "SELECT /*+ rowwise_output, columnar_output */ * FROM SQL_HINT_DUMMY";
const auto q5 =
"SELECT /*+ rowwise_output, columnar_output, rowwise_output */ * FROM "
"SQL_HINT_DUMMY";
const auto q6 = "SELECT /*+ rowwise_output, rowwise_output */ * FROM SQL_HINT_DUMMY";
const auto q7 = "SELECT /*+ columnar_output, columnar_output */ * FROM SQL_HINT_DUMMY";
QR::get()->runDDLStatement(drop_table_ddl);
QR::get()->runDDLStatement(create_table_ddl);

{
auto query_hints = QR::get()->getParsedQueryHint(q1);
auto hint_enabled = query_hints.isHintRegistered(QueryHint::kColumnarOutput);
CHECK(!hint_enabled);
}

{
auto query_hints = QR::get()->getParsedQueryHint(q2);
auto hint_enabled = query_hints.isHintRegistered(QueryHint::kRowwiseOutput);
CHECK(hint_enabled);
}

{
auto query_hints = QR::get()->getParsedQueryHint(q3);
auto hint_enabled = query_hints.isAnyQueryHintDelivered();
CHECK(!hint_enabled);
}

{
auto query_hints = QR::get()->getParsedQueryHint(q4);
auto hint_enabled = query_hints.isAnyQueryHintDelivered();
CHECK(!hint_enabled);
}

{
auto query_hints = QR::get()->getParsedQueryHint(q5);
auto hint_enabled = query_hints.isAnyQueryHintDelivered();
CHECK(!hint_enabled);
}

{
auto query_hints = QR::get()->getParsedQueryHint(q6);
auto hint_enabled = query_hints.isHintRegistered(QueryHint::kRowwiseOutput);
CHECK(hint_enabled);
}

{
auto query_hints = QR::get()->getParsedQueryHint(q7);
auto hint_enabled = query_hints.isHintRegistered(QueryHint::kColumnarOutput);
CHECK(!hint_enabled);
}

QR::get()->runDDLStatement(drop_table_ddl);
}

TEST(QueryHint, checkQueryLayoutHintWithoutEnablingColumnarOutput) {
const auto enable_columnar_output = g_enable_columnar_output;
g_enable_columnar_output = false;
ScopeGuard reset_columnar_output = [&enable_columnar_output] {
g_enable_columnar_output = enable_columnar_output;
};

const auto create_table_ddl = "CREATE TABLE SQL_HINT_DUMMY(key int)";
const auto drop_table_ddl = "DROP TABLE IF EXISTS SQL_HINT_DUMMY";
const auto q1 = "SELECT /*+ columnar_output */ * FROM SQL_HINT_DUMMY";
const auto q2 = "SELECT /*+ rowwise_output */ * FROM SQL_HINT_DUMMY";
const auto q3 = "SELECT /*+ columnar_output, rowwise_output */ * FROM SQL_HINT_DUMMY";
const auto q4 = "SELECT /*+ rowwise_output, columnar_output */ * FROM SQL_HINT_DUMMY";
const auto q5 =
"SELECT /*+ rowwise_output, columnar_output, rowwise_output */ * FROM "
"SQL_HINT_DUMMY";
const auto q6 = "SELECT /*+ rowwise_output, rowwise_output */ * FROM SQL_HINT_DUMMY";
const auto q7 = "SELECT /*+ columnar_output, columnar_output */ * FROM SQL_HINT_DUMMY";
QR::get()->runDDLStatement(drop_table_ddl);
QR::get()->runDDLStatement(create_table_ddl);

{
auto query_hints = QR::get()->getParsedQueryHint(q1);
auto hint_enabled = query_hints.isHintRegistered(QueryHint::kColumnarOutput);
CHECK(hint_enabled);
}

{
auto query_hints = QR::get()->getParsedQueryHint(q2);
auto hint_enabled = query_hints.isHintRegistered(QueryHint::kRowwiseOutput);
CHECK(!hint_enabled);
}

{
auto query_hints = QR::get()->getParsedQueryHint(q3);
auto hint_enabled = query_hints.isAnyQueryHintDelivered();
CHECK(!hint_enabled);
}

{
auto query_hints = QR::get()->getParsedQueryHint(q4);
auto hint_enabled = query_hints.isAnyQueryHintDelivered();
CHECK(!hint_enabled);
}

{
auto query_hints = QR::get()->getParsedQueryHint(q5);
auto hint_enabled = query_hints.isAnyQueryHintDelivered();
CHECK(!hint_enabled);
}

{
auto query_hints = QR::get()->getParsedQueryHint(q6);
auto hint_enabled = query_hints.isHintRegistered(QueryHint::kRowwiseOutput);
CHECK(!hint_enabled);
}

{
auto query_hints = QR::get()->getParsedQueryHint(q7);
auto hint_enabled = query_hints.isHintRegistered(QueryHint::kColumnarOutput);
CHECK(hint_enabled);
}

QR::get()->runDDLStatement(drop_table_ddl);
}

int main(int argc, char** argv) {
TestHelpers::init_logger_stderr_only(argc, argv);
testing::InitGoogleTest(&argc, argv);
Expand Down
13 changes: 12 additions & 1 deletion ThriftHandler/DBHandler.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5665,7 +5665,13 @@ std::vector<PushedDownFilterInfo> DBHandler::execute_rel_alg(
intel_jit_profile_};
auto validate_or_explain_query =
explain_info.justExplain() || explain_info.justCalciteExplain() || just_validate;
ExecutionOptions eo = {g_enable_columnar_output,
auto columnar_output_enabled = g_enable_columnar_output;
if (query_hints.isHintRegistered(QueryHint::kColumnarOutput)) {
columnar_output_enabled = true;
} else if (query_hints.isHintRegistered(QueryHint::kRowwiseOutput)) {
columnar_output_enabled = false;
}
ExecutionOptions eo = {columnar_output_enabled,
allow_multifrag_,
explain_info.justExplain(),
allow_loop_joins_ || just_validate,
Expand Down Expand Up @@ -5755,6 +5761,11 @@ void DBHandler::execute_rel_alg_df(TDataFrame& _return,
.empty(),
g_running_query_interrupt_freq,
g_pending_query_interrupt_freq};
if (query_hints.isHintRegistered(QueryHint::kColumnarOutput)) {
eo.output_columnar_hint = true;
} else if (query_hints.isHintRegistered(QueryHint::kRowwiseOutput)) {
eo.output_columnar_hint = false;
}
ExecutionResult result{std::make_shared<ResultSet>(std::vector<TargetInfo>{},
ExecutorDeviceType::CPU,
QueryMemoryDescriptor(),
Expand Down
2 changes: 2 additions & 0 deletions java/calcite/src/main/java/com/mapd/parser/hint/OmniSciHintStrategyTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ private static HintStrategyTable createHintStrategies() {

static HintStrategyTable createHintStrategies(HintStrategyTable.Builder builder) {
return builder.hintStrategy("cpu_mode", HintPredicates.SET_VAR)
.hintStrategy("columnar_output", HintPredicates.SET_VAR)
.hintStrategy("rowwise_output", HintPredicates.SET_VAR)
.hintStrategy("overlaps_bucket_threshold", HintPredicates.SET_VAR)
.hintStrategy("overlaps_max_size", HintPredicates.SET_VAR)
.hintStrategy("overlaps_allow_gpu_build", HintPredicates.SET_VAR)
Expand Down

0 comments on commit 4ef9775

Please sign in to comment.