merge from main

.changes/unreleased/Features-20240925-165002.yaml

@@ -3,4 +3,4 @@ body: Enable `retry` support for microbatch models
 time: 2024-09-25T16:50:02.105069-05:00
 custom:
   Author: QMalcolm MichelleArk
-  Issue: "10624"
+  Issue: 10715 10729

.changes/unreleased/Features-20240926-151057.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Adds validations for custom_granularities to ensure unique naming.
+time: 2024-09-26T15:10:57.907694-07:00
+custom:
+  Author: courtneyholcomb
+  Issue: "9265"

.changes/unreleased/Features-20240926-153210.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Ensure microbatch models respect `full_refresh` model config
+time: 2024-09-26T15:32:10.202789-05:00
+custom:
+  Author: QMalcolm
+  Issue: "10785"

.changes/unreleased/Fixes-20240926-101220.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Attempt to skip saved query processing when no semantic manifest changes
+time: 2024-09-26T10:12:20.193453-04:00
+custom:
+  Author: gshank
+  Issue: "10563"

.changes/unreleased/Under the Hood-20240926-143448.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Add group metadata info to LogModelResult and LogTestResult
+time: 2024-09-26T14:34:48.334703+01:00
+custom:
+    Author: aranke
+    Issue: "10775"

core/dbt/config/project.py

@@ -158,8 +158,14 @@ def _parse_versions(versions: Union[List[str], str]) -> List[VersionSpecifier]:
     return [VersionSpecifier.from_version_string(v) for v in versions]
 
 
-def _all_source_paths(*args: List[str]) -> List[str]:
-    paths = chain(*args)
+def _all_source_paths(
+    model_paths: List[str],
+    seed_paths: List[str],
+    snapshot_paths: List[str],
+    analysis_paths: List[str],
+    macro_paths: List[str],
+) -> List[str]:
+    paths = chain(model_paths, seed_paths, snapshot_paths, analysis_paths, macro_paths)
     # Strip trailing slashes since the path is the same even though the name is not
     stripped_paths = map(lambda s: s.rstrip("/"), paths)
     return list(set(stripped_paths))
@@ -403,7 +409,7 @@ def create_project(self, rendered: RenderComponents) -> "Project":
         snapshot_paths: List[str] = value_or(cfg.snapshot_paths, ["snapshots"])
 
         all_source_paths: List[str] = _all_source_paths(
-            model_paths, seed_paths, snapshot_paths, analysis_paths, macro_paths, test_paths
+            model_paths, seed_paths, snapshot_paths, analysis_paths, macro_paths
         )
 
         docs_paths: List[str] = value_or(cfg.docs_paths, all_source_paths)
@@ -646,7 +652,6 @@ def all_source_paths(self) -> List[str]:
             self.snapshot_paths,
             self.analysis_paths,
             self.macro_paths,
-            self.test_paths,
         )
 
     @property

core/dbt/contracts/graph/manifest.py

@@ -58,7 +58,6 @@
     SavedQuery,
     SeedNode,
     SemanticModel,
-    SingularTestNode,
     SourceDefinition,
     UnitTestDefinition,
     UnitTestFileFixture,
@@ -90,7 +89,7 @@
 RefName = str
 
 
-def find_unique_id_for_package(storage, key, package: Optional[PackageName]) -> Optional[UniqueID]:
+def find_unique_id_for_package(storage, key, package: Optional[PackageName]):
     if key not in storage:
         return None
 
@@ -471,43 +470,6 @@ class AnalysisLookup(RefableLookup):
     _versioned_types: ClassVar[set] = set()
 
 
-class SingularTestLookup(dbtClassMixin):
-    def __init__(self, manifest: "Manifest") -> None:
-        self.storage: Dict[str, Dict[PackageName, UniqueID]] = {}
-        self.populate(manifest)
-
-    def get_unique_id(self, search_name, package: Optional[PackageName]) -> Optional[UniqueID]:
-        return find_unique_id_for_package(self.storage, search_name, package)
-
-    def find(
-        self, search_name, package: Optional[PackageName], manifest: "Manifest"
-    ) -> Optional[SingularTestNode]:
-        unique_id = self.get_unique_id(search_name, package)
-        if unique_id is not None:
-            return self.perform_lookup(unique_id, manifest)
-        return None
-
-    def add_singular_test(self, source: SingularTestNode) -> None:
-        if source.search_name not in self.storage:
-            self.storage[source.search_name] = {}
-
-        self.storage[source.search_name][source.package_name] = source.unique_id
-
-    def populate(self, manifest: "Manifest") -> None:
-        for node in manifest.nodes.values():
-            if isinstance(node, SingularTestNode):
-                self.add_singular_test(node)
-
-    def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> SingularTestNode:
-        if unique_id not in manifest.nodes:
-            raise dbt_common.exceptions.DbtInternalError(
-                f"Singular test {unique_id} found in cache but not found in manifest"
-            )
-        node = manifest.nodes[unique_id]
-        assert isinstance(node, SingularTestNode)
-        return node
-
-
 def _packages_to_search(
     current_project: str,
     node_package: str,
@@ -907,9 +869,6 @@ class Manifest(MacroMethods, dbtClassMixin):
     _analysis_lookup: Optional[AnalysisLookup] = field(
         default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
     )
-    _singular_test_lookup: Optional[SingularTestLookup] = field(
-        default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
-    )
     _parsing_info: ParsingInfo = field(
         default_factory=ParsingInfo,
         metadata={"serialize": lambda x: None, "deserialize": lambda x: None},
@@ -1305,12 +1264,6 @@ def analysis_lookup(self) -> AnalysisLookup:
             self._analysis_lookup = AnalysisLookup(self)
         return self._analysis_lookup
 
-    @property
-    def singular_test_lookup(self) -> SingularTestLookup:
-        if self._singular_test_lookup is None:
-            self._singular_test_lookup = SingularTestLookup(self)
-        return self._singular_test_lookup
-
     @property
     def external_node_unique_ids(self):
         return [node.unique_id for node in self.nodes.values() if node.is_external_node]
@@ -1755,7 +1708,6 @@ def __reduce_ex__(self, protocol):
             self._semantic_model_by_measure_lookup,
             self._disabled_lookup,
             self._analysis_lookup,
-            self._singular_test_lookup,
         )
         return self.__class__, args
 

core/dbt/contracts/graph/nodes.py

@@ -1473,7 +1473,7 @@ def to_logging_dict(self) -> Dict[str, Union[str, Dict[str, str]]]:
         return {
             "name": self.name,
             "package_name": self.package_name,
-            "owner": self.owner.to_dict(),
+            "owner": self.owner.to_dict(omit_none=True),
         }
 
 
@@ -1642,11 +1642,6 @@ class ParsedMacroPatch(ParsedPatch):
     arguments: List[MacroArgument] = field(default_factory=list)
 
 
-@dataclass
-class ParsedSingularTestPatch(ParsedPatch):
-    pass
-
-
 # ====================================
 # Node unions/categories
 # ====================================
@@ -1697,6 +1692,7 @@ class ParsedSingularTestPatch(ParsedPatch):
 
 TestNode = Union[SingularTestNode, GenericTestNode]
 
+SemanticManifestNode = Union[SavedQuery, SemanticModel, Metric]
 
 RESOURCE_CLASS_TO_NODE_CLASS: Dict[Type[BaseResource], Type[BaseNode]] = {
     node_class.resource_class(): node_class

core/dbt/contracts/graph/unparsed.py

@@ -202,11 +202,6 @@ class UnparsedAnalysisUpdate(HasConfig, HasColumnDocs, HasColumnProps, HasYamlMe
     access: Optional[str] = None
 
 
-@dataclass
-class UnparsedSingularTestUpdate(HasConfig, HasColumnProps, HasYamlMetadata):
-    pass
-
-
 @dataclass
 class UnparsedNodeUpdate(HasConfig, HasColumnTests, HasColumnAndTestProps, HasYamlMetadata):
     quote_columns: Optional[bool] = None

core/dbt/events/core_types.proto

@@ -1271,6 +1271,12 @@ message SQLRunnerExceptionMsg {
     SQLRunnerException data = 2;
 }
 
+message Group {
+    string name = 1;
+    string package_name = 3;
+    map<string, string> owner = 7;
+}
+
 // Q007
 message LogTestResult {
     NodeInfo node_info = 1;
@@ -1280,6 +1286,8 @@ message LogTestResult {
     int32 num_models = 5;
     float execution_time = 6;
     int32 num_failures = 7;
+    Group group = 8;
+    string attached_node = 9;
 }
 
 message LogTestResultMsg {
@@ -1312,6 +1320,7 @@ message LogModelResult {
     int32 index = 4;
     int32 total = 5;
     float execution_time = 6;
+    Group group = 7;
 }
 
 message LogModelResultMsg {
@@ -1373,7 +1382,7 @@ message LogFreshnessResultMsg {
     LogFreshnessResult data = 2;
 }
 
-// Q018
+// Q019
 message LogNodeNoOpResult {
     NodeInfo node_info = 1;
     string description = 2;
@@ -1820,12 +1829,6 @@ message ServingDocsExitInfoMsg {
     ServingDocsExitInfo data = 2;
 }
 
-message Group {
-    string name = 1;
-    string package_name = 3;
-    map<string, string> owner = 7;
-}
-
 // Z021
 message RunResultWarning {
     string resource_type = 1;

core/dbt/parser/common.py

@@ -13,7 +13,6 @@
     UnparsedMacroUpdate,
     UnparsedModelUpdate,
     UnparsedNodeUpdate,
-    UnparsedSingularTestUpdate,
 )
 from dbt.exceptions import ParsingError
 from dbt.node_types import NodeType
@@ -59,7 +58,6 @@ def trimmed(inp: str) -> str:
     UnpatchedSourceDefinition,
     UnparsedExposure,
     UnparsedModelUpdate,
-    UnparsedSingularTestUpdate,
 )
 
 

core/dbt/parser/manifest.py

@@ -57,6 +57,7 @@
     ResultNode,
     SavedQuery,
     SeedNode,
+    SemanticManifestNode,
     SemanticModel,
     SourceDefinition,
 )
@@ -1141,6 +1142,23 @@ def process_metrics(self, config: RuntimeConfig):
 
     def process_saved_queries(self, config: RuntimeConfig):
         """Processes SavedQuery nodes to populate their `depends_on`."""
+        # Note: This will also capture various nodes which have been re-parsed
+        # because they refer to some other changed node, so there will be
+        # false positives. Ideally we would compare actual changes.
+        semantic_manifest_changed = False
+        semantic_manifest_nodes: chain[SemanticManifestNode] = chain(
+            self.manifest.saved_queries.values(),
+            self.manifest.semantic_models.values(),
+            self.manifest.metrics.values(),
+        )
+        for node in semantic_manifest_nodes:
+            # Check if this node has been modified in this parsing run
+            if node.created_at > self.started_at:
+                semantic_manifest_changed = True
+                break  # as soon as we run into one changed node we can stop
+        if semantic_manifest_changed is False:
+            return
+
         current_project = config.project_name
         for saved_query in self.manifest.saved_queries.values():
             # TODO: