From ff76596b7bbb6d6cf5825b1396fc3849019374f0 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 26 Sep 2024 10:12:29 -0400 Subject: [PATCH 1/3] Attempt to skip saved query processing when no semantic manifest changes --- .changes/unreleased/Fixes-20240926-101220.yaml | 6 ++++++ core/dbt/parser/manifest.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 .changes/unreleased/Fixes-20240926-101220.yaml diff --git a/.changes/unreleased/Fixes-20240926-101220.yaml b/.changes/unreleased/Fixes-20240926-101220.yaml new file mode 100644 index 00000000000..677ee458048 --- /dev/null +++ b/.changes/unreleased/Fixes-20240926-101220.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Attempt to skip saved query processing when no semantic manifest changes +time: 2024-09-26T10:12:20.193453-04:00 +custom: + Author: gshank + Issue: "10563" diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 5536bf39270..ce396b97fef 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -1141,6 +1141,24 @@ def process_metrics(self, config: RuntimeConfig): def process_saved_queries(self, config: RuntimeConfig): """Processes SavedQuery nodes to populate their `depends_on`.""" + # Note: This will also capture various nodes which have been re-parsed + # because they refer to some other changed node, so there will be + # false positives. Ideally we would compare actual changes. + semantic_manifest_changed = False + SemanticManifestNode = Union[SavedQuery, SemanticModel, Metric] + semantic_manifest_nodes: chain[SemanticManifestNode] = chain( + self.manifest.saved_queries.values(), + self.manifest.semantic_models.values(), + self.manifest.metrics.values(), + ) + for node in semantic_manifest_nodes: + # Check if this node has been modified in this parsing run + if node.created_at > self.started_at: + semantic_manifest_changed = True + break # as soon as we run into one changed node we can stop + if semantic_manifest_changed is False: + return + current_project = config.project_name for saved_query in self.manifest.saved_queries.values(): # TODO: From ca72d0009f31a2806f62323a7d9e6b7f50d92fde Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 26 Sep 2024 12:33:59 -0400 Subject: [PATCH 2/3] Move SemanticManifestNode and simplify --- core/dbt/contracts/graph/nodes.py | 1 + core/dbt/parser/manifest.py | 10 ++-------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index 609e9abcd1b..efdf2bc07ef 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -1693,6 +1693,7 @@ class ParsedSingularTestPatch(ParsedPatch): TestNode = Union[SingularTestNode, GenericTestNode] +SemanticManifestNode = Union[SavedQuery, SemanticModel, Metric] RESOURCE_CLASS_TO_NODE_CLASS: Dict[Type[BaseResource], Type[BaseNode]] = { node_class.resource_class(): node_class diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index ce396b97fef..6ae49e9d76b 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -57,6 +57,7 @@ ResultNode, SavedQuery, SeedNode, + SemanticManifestNode, SemanticModel, SourceDefinition, ) @@ -1144,19 +1145,12 @@ def process_saved_queries(self, config: RuntimeConfig): # Note: This will also capture various nodes which have been re-parsed # because they refer to some other changed node, so there will be # false positives. Ideally we would compare actual changes. - semantic_manifest_changed = False - SemanticManifestNode = Union[SavedQuery, SemanticModel, Metric] semantic_manifest_nodes: chain[SemanticManifestNode] = chain( self.manifest.saved_queries.values(), self.manifest.semantic_models.values(), self.manifest.metrics.values(), ) - for node in semantic_manifest_nodes: - # Check if this node has been modified in this parsing run - if node.created_at > self.started_at: - semantic_manifest_changed = True - break # as soon as we run into one changed node we can stop - if semantic_manifest_changed is False: + if any(node.created_at > self.started_at for node in semantic_manifest_nodes): return current_project = config.project_name From cebb1ceaef218d84e1fbbec1a635dab790329b8f Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 26 Sep 2024 12:40:37 -0400 Subject: [PATCH 3/3] Put back original logic --- core/dbt/parser/manifest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 6ae49e9d76b..7ffd00febc5 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -1145,12 +1145,18 @@ def process_saved_queries(self, config: RuntimeConfig): # Note: This will also capture various nodes which have been re-parsed # because they refer to some other changed node, so there will be # false positives. Ideally we would compare actual changes. + semantic_manifest_changed = False semantic_manifest_nodes: chain[SemanticManifestNode] = chain( self.manifest.saved_queries.values(), self.manifest.semantic_models.values(), self.manifest.metrics.values(), ) - if any(node.created_at > self.started_at for node in semantic_manifest_nodes): + for node in semantic_manifest_nodes: + # Check if this node has been modified in this parsing run + if node.created_at > self.started_at: + semantic_manifest_changed = True + break # as soon as we run into one changed node we can stop + if semantic_manifest_changed is False: return current_project = config.project_name