Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add a no-op runner for saved_query #8937

Merged
merged 5 commits into from
Oct 31, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changes/unreleased/Under the Hood-20231027-140048.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Under the Hood
body: Add a no-op runner for Saved Qeury
time: 2023-10-27T14:00:48.4755-07:00
custom:
Author: ChenyuLInx
Issue: "8893"
1 change: 1 addition & 0 deletions core/dbt/cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,7 @@ def cli(ctx, **kwargs):
@p.favor_state
@p.deprecated_favor_state
@p.full_refresh
@p.include_saved_query
@p.indirect_selection
@p.profile
@p.profiles_dir
Expand Down
8 changes: 8 additions & 0 deletions core/dbt/cli/params.py
Original file line number Diff line number Diff line change
Expand Up @@ -407,6 +407,14 @@
default=(),
)

include_saved_query = click.option(
"--include-saved-query/--no-include-saved-query",
envvar="DBT_INCLUDE_SAVED_QUERY",
help="Include saved queries in the list of resources to be selected for build command",
is_flag=True,
hidden=True,
aranke marked this conversation as resolved.
Show resolved Hide resolved
)

model_decls = ("-m", "--models", "--model")
select_decls = ("-s", "--select")
select_attrs = {
Expand Down
2 changes: 2 additions & 0 deletions core/dbt/compilation.py
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,8 @@ def link_graph(self, manifest: Manifest):
self.link_node(exposure, manifest)
for metric in manifest.metrics.values():
self.link_node(metric, manifest)
for saved_query in manifest.saved_queries.values():
self.link_node(saved_query, manifest)

cycle = self.find_cycles()

Expand Down
6 changes: 5 additions & 1 deletion core/dbt/contracts/graph/nodes.py
Original file line number Diff line number Diff line change
Expand Up @@ -1744,10 +1744,14 @@ def same_contents(self, old: Optional["SemanticModel"]) -> bool:


@dataclass
class SavedQuery(GraphNode):
class SavedQueryMandatory(GraphNode):
aranke marked this conversation as resolved.
Show resolved Hide resolved
metrics: List[str]
group_bys: List[str]
where: Optional[WhereFilterIntersection]


@dataclass
class SavedQuery(NodeInfoMixin, SavedQueryMandatory):
description: Optional[str] = None
label: Optional[str] = None
metadata: Optional[SourceFileMetadata] = None
Expand Down
2 changes: 2 additions & 0 deletions core/dbt/graph/selector.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,8 @@ def _is_match(self, unique_id: UniqueId) -> bool:
node = self.manifest.metrics[unique_id]
elif unique_id in self.manifest.semantic_models:
node = self.manifest.semantic_models[unique_id]
elif unique_id in self.manifest.saved_queries:
node = self.manifest.saved_queries[unique_id]
else:
raise DbtInternalError(f"Node {unique_id} not found in the manifest!")
return self.node_is_match(node)
Expand Down
1 change: 1 addition & 0 deletions core/dbt/graph/selector_methods.py
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,7 @@ def non_source_nodes(
self.exposure_nodes(included_nodes),
self.metric_nodes(included_nodes),
self.semantic_model_nodes(included_nodes),
self.saved_query_nodes(included_nodes),
)

def groupable_nodes(
Expand Down
62 changes: 62 additions & 0 deletions core/dbt/task/build.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import threading

from .run import RunTask, ModelRunner as run_model_runner
from .snapshot import SnapshotRunner as snapshot_model_runner
from .seed import SeedRunner as seed_runner
Expand All @@ -9,6 +11,62 @@
from dbt.graph import ResourceTypeSelector
from dbt.node_types import NodeType
from dbt.task.test import TestSelector
from dbt.task.base import BaseRunner
from dbt.contracts.results import RunResult, RunStatus
from dbt.events.functions import fire_event
from dbt.events.types import LogStartLine, LogModelResult
from dbt.events.base_types import EventLevel


class SavedQueryRunner(BaseRunner):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could this class go in its own file?

# A no-op Runner for Saved Queries
@property
def description(self):
return "Saved Query {}".format(self.node.unique_id)

def before_execute(self):
fire_event(
LogStartLine(
description=self.description,
index=self.node_index,
total=self.num_nodes,
node_info=self.node.node_info,
)
)

def compile(self, manifest):
return self.node

def after_execute(self, result):
if result.status == NodeStatus.Error:
level = EventLevel.ERROR

Check warning on line 42 in core/dbt/task/build.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/task/build.py#L42

Added line #L42 was not covered by tests
else:
level = EventLevel.INFO
fire_event(
LogModelResult(
description=self.description,
status=result.status,
index=self.node_index,
total=self.num_nodes,
execution_time=result.execution_time,
node_info=self.node.node_info,
),
level=level,
)

def execute(self, compiled_node, manifest):
# no-op
return RunResult(
node=compiled_node,
status=RunStatus.Success,
timing=[],
thread_id=threading.current_thread().name,
execution_time=0.1,
message="done",
adapter_response={},
failures=0,
agate_table=None,
)


class BuildTask(RunTask):
Expand All @@ -31,6 +89,10 @@

@property
def resource_types(self):
if self.args.include_saved_query:
self.RUNNER_MAP[NodeType.SavedQuery] = SavedQueryRunner
self.ALL_RESOURCE_VALUES = self.ALL_RESOURCE_VALUES.union({NodeType.SavedQuery})

if not self.args.resource_types:
return list(self.ALL_RESOURCE_VALUES)

Expand Down
2 changes: 2 additions & 0 deletions core/dbt/task/runnable.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,8 @@ def _runtime_initialize(self):
self._flattened_nodes.append(self.manifest.nodes[uid])
elif uid in self.manifest.sources:
self._flattened_nodes.append(self.manifest.sources[uid])
elif uid in self.manifest.saved_queries:
self._flattened_nodes.append(self.manifest.saved_queries[uid])
else:
raise DbtInternalError(
f"Node selection returned {uid}, expected a node or a source"
Expand Down
38 changes: 38 additions & 0 deletions tests/functional/saved_queries/test_saved_query_build.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
import pytest

from dbt.tests.util import run_dbt
from tests.functional.saved_queries.fixtures import saved_queries_yml, saved_query_description
from tests.functional.semantic_models.fixtures import (
fct_revenue_sql,
metricflow_time_spine_sql,
schema_yml,
)


class TestSavedQueryBuildNoOp:
@pytest.fixture(scope="class")
def models(self):
return {
"saved_queries.yml": saved_queries_yml,
"schema.yml": schema_yml,
"fct_revenue.sql": fct_revenue_sql,
"metricflow_time_spine.sql": metricflow_time_spine_sql,
"docs.md": saved_query_description,
}

@pytest.fixture(scope="class")
def packages(self):
return """
packages:
- package: dbt-labs/dbt_utils
version: 1.1.1
"""

def test_semantic_model_parsing(self, project):
run_dbt(["deps"])
result = run_dbt(["build"])
assert len(result.results) == 2
assert "test_saved_query" not in [r.node.name for r in result.results]
result = run_dbt(["build", "--include-saved-query"])
assert len(result.results) == 3
assert "test_saved_query" in [r.node.name for r in result.results]
Loading