From ff76596b7bbb6d6cf5825b1396fc3849019374f0 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 26 Sep 2024 10:12:29 -0400 Subject: [PATCH] Attempt to skip saved query processing when no semantic manifest changes --- .changes/unreleased/Fixes-20240926-101220.yaml | 6 ++++++ core/dbt/parser/manifest.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 .changes/unreleased/Fixes-20240926-101220.yaml diff --git a/.changes/unreleased/Fixes-20240926-101220.yaml b/.changes/unreleased/Fixes-20240926-101220.yaml new file mode 100644 index 00000000000..677ee458048 --- /dev/null +++ b/.changes/unreleased/Fixes-20240926-101220.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Attempt to skip saved query processing when no semantic manifest changes +time: 2024-09-26T10:12:20.193453-04:00 +custom: + Author: gshank + Issue: "10563" diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 5536bf39270..ce396b97fef 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -1141,6 +1141,24 @@ def process_metrics(self, config: RuntimeConfig): def process_saved_queries(self, config: RuntimeConfig): """Processes SavedQuery nodes to populate their `depends_on`.""" + # Note: This will also capture various nodes which have been re-parsed + # because they refer to some other changed node, so there will be + # false positives. Ideally we would compare actual changes. + semantic_manifest_changed = False + SemanticManifestNode = Union[SavedQuery, SemanticModel, Metric] + semantic_manifest_nodes: chain[SemanticManifestNode] = chain( + self.manifest.saved_queries.values(), + self.manifest.semantic_models.values(), + self.manifest.metrics.values(), + ) + for node in semantic_manifest_nodes: + # Check if this node has been modified in this parsing run + if node.created_at > self.started_at: + semantic_manifest_changed = True + break # as soon as we run into one changed node we can stop + if semantic_manifest_changed is False: + return + current_project = config.project_name for saved_query in self.manifest.saved_queries.values(): # TODO: