diff --git a/.changes/unreleased/Under the Hood-20231027-140048.yaml b/.changes/unreleased/Under the Hood-20231027-140048.yaml new file mode 100644 index 00000000000..1baa6adf97f --- /dev/null +++ b/.changes/unreleased/Under the Hood-20231027-140048.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add a no-op runner for Saved Qeury +time: 2023-10-27T14:00:48.4755-07:00 +custom: + Author: ChenyuLInx + Issue: "8893" diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 5b62f7f1448..7d4560a7910 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -186,6 +186,7 @@ def cli(ctx, **kwargs): @p.favor_state @p.deprecated_favor_state @p.full_refresh +@p.include_saved_query @p.indirect_selection @p.profile @p.profiles_dir diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index 2214152407c..1898815a724 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -407,6 +407,14 @@ default=(), ) +include_saved_query = click.option( + "--include-saved-query/--no-include-saved-query", + envvar="DBT_INCLUDE_SAVED_QUERY", + help="Include saved queries in the list of resources to be selected for build command", + is_flag=True, + hidden=True, +) + model_decls = ("-m", "--models", "--model") select_decls = ("-s", "--select") select_attrs = { diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index f48caa809be..6a5a32f39db 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -191,6 +191,8 @@ def link_graph(self, manifest: Manifest): self.link_node(exposure, manifest) for metric in manifest.metrics.values(): self.link_node(metric, manifest) + for saved_query in manifest.saved_queries.values(): + self.link_node(saved_query, manifest) cycle = self.find_cycles() diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index e49a7244f3c..4da91653561 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -1744,10 +1744,14 @@ def same_contents(self, old: Optional["SemanticModel"]) -> bool: @dataclass -class SavedQuery(GraphNode): +class SavedQueryMandatory(GraphNode): metrics: List[str] group_bys: List[str] where: Optional[WhereFilterIntersection] + + +@dataclass +class SavedQuery(NodeInfoMixin, SavedQueryMandatory): description: Optional[str] = None label: Optional[str] = None metadata: Optional[SourceFileMetadata] = None diff --git a/core/dbt/graph/selector.py b/core/dbt/graph/selector.py index da07a86529d..6d2d7b42b1f 100644 --- a/core/dbt/graph/selector.py +++ b/core/dbt/graph/selector.py @@ -199,6 +199,8 @@ def _is_match(self, unique_id: UniqueId) -> bool: node = self.manifest.metrics[unique_id] elif unique_id in self.manifest.semantic_models: node = self.manifest.semantic_models[unique_id] + elif unique_id in self.manifest.saved_queries: + node = self.manifest.saved_queries[unique_id] else: raise DbtInternalError(f"Node {unique_id} not found in the manifest!") return self.node_is_match(node) diff --git a/core/dbt/graph/selector_methods.py b/core/dbt/graph/selector_methods.py index 3ce3f0d8b2e..1bbae255309 100644 --- a/core/dbt/graph/selector_methods.py +++ b/core/dbt/graph/selector_methods.py @@ -193,6 +193,7 @@ def non_source_nodes( self.exposure_nodes(included_nodes), self.metric_nodes(included_nodes), self.semantic_model_nodes(included_nodes), + self.saved_query_nodes(included_nodes), ) def groupable_nodes( diff --git a/core/dbt/task/build.py b/core/dbt/task/build.py index 8a5dc39c9b7..b1f1c1beb56 100644 --- a/core/dbt/task/build.py +++ b/core/dbt/task/build.py @@ -1,3 +1,5 @@ +import threading + from .run import RunTask, ModelRunner as run_model_runner from .snapshot import SnapshotRunner as snapshot_model_runner from .seed import SeedRunner as seed_runner @@ -9,6 +11,62 @@ from dbt.graph import ResourceTypeSelector from dbt.node_types import NodeType from dbt.task.test import TestSelector +from dbt.task.base import BaseRunner +from dbt.contracts.results import RunResult, RunStatus +from dbt.events.functions import fire_event +from dbt.events.types import LogStartLine, LogModelResult +from dbt.events.base_types import EventLevel + + +class SavedQueryRunner(BaseRunner): + # A no-op Runner for Saved Queries + @property + def description(self): + return "Saved Query {}".format(self.node.unique_id) + + def before_execute(self): + fire_event( + LogStartLine( + description=self.description, + index=self.node_index, + total=self.num_nodes, + node_info=self.node.node_info, + ) + ) + + def compile(self, manifest): + return self.node + + def after_execute(self, result): + if result.status == NodeStatus.Error: + level = EventLevel.ERROR + else: + level = EventLevel.INFO + fire_event( + LogModelResult( + description=self.description, + status=result.status, + index=self.node_index, + total=self.num_nodes, + execution_time=result.execution_time, + node_info=self.node.node_info, + ), + level=level, + ) + + def execute(self, compiled_node, manifest): + # no-op + return RunResult( + node=compiled_node, + status=RunStatus.Success, + timing=[], + thread_id=threading.current_thread().name, + execution_time=0.1, + message="done", + adapter_response={}, + failures=0, + agate_table=None, + ) class BuildTask(RunTask): @@ -31,6 +89,10 @@ class BuildTask(RunTask): @property def resource_types(self): + if self.args.include_saved_query: + self.RUNNER_MAP[NodeType.SavedQuery] = SavedQueryRunner + self.ALL_RESOURCE_VALUES = self.ALL_RESOURCE_VALUES.union({NodeType.SavedQuery}) + if not self.args.resource_types: return list(self.ALL_RESOURCE_VALUES) diff --git a/core/dbt/task/runnable.py b/core/dbt/task/runnable.py index 561b97bdef4..a5c3a5153bd 100644 --- a/core/dbt/task/runnable.py +++ b/core/dbt/task/runnable.py @@ -153,6 +153,8 @@ def _runtime_initialize(self): self._flattened_nodes.append(self.manifest.nodes[uid]) elif uid in self.manifest.sources: self._flattened_nodes.append(self.manifest.sources[uid]) + elif uid in self.manifest.saved_queries: + self._flattened_nodes.append(self.manifest.saved_queries[uid]) else: raise DbtInternalError( f"Node selection returned {uid}, expected a node or a source" diff --git a/tests/functional/saved_queries/test_saved_query_build.py b/tests/functional/saved_queries/test_saved_query_build.py new file mode 100644 index 00000000000..ffe37761521 --- /dev/null +++ b/tests/functional/saved_queries/test_saved_query_build.py @@ -0,0 +1,38 @@ +import pytest + +from dbt.tests.util import run_dbt +from tests.functional.saved_queries.fixtures import saved_queries_yml, saved_query_description +from tests.functional.semantic_models.fixtures import ( + fct_revenue_sql, + metricflow_time_spine_sql, + schema_yml, +) + + +class TestSavedQueryBuildNoOp: + @pytest.fixture(scope="class") + def models(self): + return { + "saved_queries.yml": saved_queries_yml, + "schema.yml": schema_yml, + "fct_revenue.sql": fct_revenue_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "docs.md": saved_query_description, + } + + @pytest.fixture(scope="class") + def packages(self): + return """ +packages: + - package: dbt-labs/dbt_utils + version: 1.1.1 +""" + + def test_semantic_model_parsing(self, project): + run_dbt(["deps"]) + result = run_dbt(["build"]) + assert len(result.results) == 2 + assert "test_saved_query" not in [r.node.name for r in result.results] + result = run_dbt(["build", "--include-saved-query"]) + assert len(result.results) == 3 + assert "test_saved_query" in [r.node.name for r in result.results]