Modify snapshot config to allow using schema/database/alias macros (#…

…10439)

.changes/unreleased/Features-20240712-214546.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Support standard schema/database fields for snapshots
+time: 2024-07-12T21:45:46.06011-04:00
+custom:
+  Author: gshank
+  Issue: "10301"

core/dbt/artifacts/resources/v1/snapshot.py

@@ -19,10 +19,9 @@ class SnapshotConfig(NodeConfig):
     check_cols: Union[str, List[str], None] = None
 
     def final_validate(self):
-        if not self.strategy or not self.unique_key or not self.target_schema:
+        if not self.strategy or not self.unique_key:
             raise ValidationError(
-                "Snapshots must be configured with a 'strategy', 'unique_key', "
-                "and 'target_schema'."
+                "Snapshots must be configured with a 'strategy' and 'unique_key'."
             )
         if self.strategy == "check":
             if not self.check_cols:

core/dbt/parser/base.py

@@ -72,6 +72,7 @@ def __init__(
 
 
 class RelationUpdate:
+    # "component" is database, schema or alias
     def __init__(self, config: RuntimeConfig, manifest: Manifest, component: str) -> None:
         default_macro = manifest.find_generate_macro_by_name(
             component=component,
@@ -127,6 +128,7 @@ def __init__(
     ) -> None:
         super().__init__(project, manifest, root_project)
 
+        # this sets callables from RelationUpdate
         self._update_node_database = RelationUpdate(
             manifest=manifest, config=root_project, component="database"
         )
@@ -288,7 +290,10 @@ def update_parsed_node_relation_names(
         self._update_node_schema(parsed_node, config_dict.get("schema"))
         self._update_node_alias(parsed_node, config_dict.get("alias"))
 
-        # Snapshot nodes use special "target_database" and "target_schema" fields for some reason
+        # Snapshot nodes use special "target_database" and "target_schema" fields
+        # for backward compatibility
+        # We have to do getattr here because saved_query parser calls this method with
+        # Export object instead of a node.
         if getattr(parsed_node, "resource_type", None) == NodeType.Snapshot:
             if "target_database" in config_dict and config_dict["target_database"]:
                 parsed_node.database = config_dict["target_database"]
@@ -443,9 +448,8 @@ def parse_node(self, block: ConfiguredBlockType) -> FinalNode:
             fqn=fqn,
         )
         self.render_update(node, config)
-        result = self.transform(node)
-        self.add_result_node(block, result)
-        return result
+        self.add_result_node(block, node)
+        return node
 
     def _update_node_relation_name(self, node: ManifestNode):
         # Seed and Snapshot nodes and Models that are not ephemeral,
@@ -464,17 +468,12 @@ def _update_node_relation_name(self, node: ManifestNode):
     def parse_file(self, file_block: FileBlock) -> None:
         pass
 
-    @abc.abstractmethod
-    def transform(self, node: FinalNode) -> FinalNode:
-        pass
-
 
 class SimpleParser(
     ConfiguredParser[ConfiguredBlockType, FinalNode],
     Generic[ConfiguredBlockType, FinalNode],
 ):
-    def transform(self, node):
-        return node
+    pass
 
 
 class SQLParser(ConfiguredParser[FileBlock, FinalNode], Generic[FinalNode]):
@@ -483,5 +482,4 @@ def parse_file(self, file_block: FileBlock) -> None:
 
 
 class SimpleSQLParser(SQLParser[FinalNode]):
-    def transform(self, node):
-        return node
+    pass

core/dbt/parser/hooks.py

@@ -66,8 +66,6 @@ def __iter__(self) -> Iterator[HookBlock]:
 
 
 class HookParser(SimpleParser[HookBlock, HookNode]):
-    def transform(self, node):
-        return node
 
     # Hooks are only in the dbt_project.yml file for the project
     def get_path(self) -> FilePath:

core/dbt/parser/snapshots.py

@@ -2,12 +2,10 @@
 from typing import List
 
 from dbt.contracts.graph.nodes import SnapshotNode
-from dbt.exceptions import SnapshopConfigError
 from dbt.node_types import NodeType
 from dbt.parser.base import SQLParser
 from dbt.parser.search import BlockContents, BlockSearcher, FileBlock
 from dbt.utils import split_path
-from dbt_common.dataclass_schema import ValidationError
 
 
 class SnapshotParser(SQLParser[SnapshotNode]):
@@ -24,24 +22,6 @@ def resource_type(self) -> NodeType:
     def get_compiled_path(cls, block: FileBlock):
         return block.path.relative_path
 
-    def set_snapshot_attributes(self, node):
-        # use the target_database setting if we got it, otherwise the
-        # `database` value of the node (ultimately sourced from the `database`
-        # config value), and if that is not set, use the database defined in
-        # the adapter's credentials.
-        if node.config.target_database:
-            node.database = node.config.target_database
-        elif not node.database:
-            node.database = self.root_project.credentials.database
-
-        # the target schema must be set if we got here, so overwrite the node's
-        # schema
-        node.schema = node.config.target_schema
-        # We need to set relation_name again, since database/schema might have changed
-        self._update_node_relation_name(node)
-
-        return node
-
     def get_fqn(self, path: str, name: str) -> List[str]:
         """Get the FQN for the node. This impacts node selection and config
         application.
@@ -54,13 +34,6 @@ def get_fqn(self, path: str, name: str) -> List[str]:
         fqn.append(name)
         return fqn
 
-    def transform(self, node: SnapshotNode) -> SnapshotNode:
-        try:
-            self.set_snapshot_attributes(node)
-            return node
-        except ValidationError as exc:
-            raise SnapshopConfigError(exc, node)
-
     def parse_file(self, file_block: FileBlock) -> None:
         blocks = BlockSearcher(
             source=[file_block],

tests/functional/simple_snapshot/test_basic_snapshot.py → tests/functional/snapshots/test_basic_snapshot.py

@@ -10,7 +10,7 @@
     run_dbt,
     write_file,
 )
-from tests.functional.simple_snapshot.fixtures import (
+from tests.functional.snapshots.fixtures import (
     macros__test_no_overlaps_sql,
     macros_custom_snapshot__custom_sql,
     models__ref_snapshot_sql,
@@ -142,7 +142,7 @@ def project_config_update(self, unique_schema):
         return {
             "snapshots": {
                 "test": {
-                    "target_schema": unique_schema + "_alt",
+                    "schema": "alt",
                 }
             }
         }
@@ -153,6 +153,8 @@ def test_target_schema(self, project):
         # ensure that the schema in the snapshot node is the same as target_schema
         snapshot_id = "snapshot.test.snapshot_actual"
         snapshot_node = manifest.nodes[snapshot_id]
+        # The schema field be changed by the default "generate_schema_name"
+        # to append an underscore plus the configured schema of "alt".
         assert snapshot_node.schema == f"{project.test_schema}_alt"
         assert (
             snapshot_node.relation_name

tests/functional/simple_snapshot/test_changing_strategy_snapshot.py → tests/functional/snapshots/test_changing_strategy_snapshot.py

@@ -1,7 +1,7 @@
 import pytest
 
 from dbt.tests.util import run_dbt
-from tests.functional.simple_snapshot.fixtures import models_slow__gen_sql
+from tests.functional.snapshots.fixtures import models_slow__gen_sql
 
 test_snapshots_changing_strategy__test_snapshot_sql = """
 

tests/functional/simple_snapshot/test_check_cols_snapshot.py → tests/functional/snapshots/test_check_cols_snapshot.py

@@ -100,7 +100,7 @@ def tests():
     return {"my_test.sql": snapshot_test_sql}
 
 
-def test_simple_snapshot(project):
+def test_snapshots(project):
 
     results = run_dbt(["snapshot", "--vars", "version: 1"])
     assert len(results) == 1

tests/functional/simple_snapshot/test_check_cols_updated_at_snapshot.py → tests/functional/snapshots/test_check_cols_updated_at_snapshot.py

@@ -74,7 +74,7 @@ def project_config_update():
     }
 
 
-def test_simple_snapshot(project):
+def test_snapshots(project):
     """
     Test that the `dbt_updated_at` column reflects the `updated_at` timestamp expression in the config.
 

tests/functional/simple_snapshot/test_cross_schema_snapshot.py → tests/functional/snapshots/test_cross_schema_snapshot.py

@@ -3,7 +3,7 @@
 import pytest
 
 from dbt.tests.util import run_dbt
-from tests.functional.simple_snapshot.fixtures import (
+from tests.functional.snapshots.fixtures import (
     macros__test_no_overlaps_sql,
     models__ref_snapshot_sql,
     models__schema_yml,

tests/functional/simple_snapshot/test_hard_delete_snapshot.py → tests/functional/snapshots/test_hard_delete_snapshot.py

@@ -5,7 +5,7 @@
 import pytz
 
 from dbt.tests.util import check_relations_equal, run_dbt
-from tests.functional.simple_snapshot.fixtures import (
+from tests.functional.snapshots.fixtures import (
     macros__test_no_overlaps_sql,
     models__ref_snapshot_sql,
     models__schema_yml,

tests/functional/simple_snapshot/test_invalid_namespace_snapshot.py → tests/functional/snapshots/test_invalid_namespace_snapshot.py

@@ -3,7 +3,7 @@
 import pytest
 
 from dbt.tests.util import run_dbt
-from tests.functional.simple_snapshot.fixtures import (
+from tests.functional.snapshots.fixtures import (
     macros__test_no_overlaps_sql,
     macros_custom_snapshot__custom_sql,
     models__ref_snapshot_sql,

tests/functional/simple_snapshot/test_long_text_snapshot.py → tests/functional/snapshots/test_long_text_snapshot.py

@@ -1,7 +1,7 @@
 import pytest
 
 from dbt.tests.util import run_dbt
-from tests.functional.simple_snapshot.fixtures import (
+from tests.functional.snapshots.fixtures import (
     macros__test_no_overlaps_sql,
     models__ref_snapshot_sql,
     models__schema_yml,

tests/functional/simple_snapshot/test_missing_strategy_snapshot.py → tests/functional/snapshots/test_missing_strategy_snapshot.py

@@ -2,20 +2,18 @@
 
 from dbt.tests.util import run_dbt
 from dbt_common.dataclass_schema import ValidationError
-from tests.functional.simple_snapshot.fixtures import (
+from tests.functional.snapshots.fixtures import (
     macros__test_no_overlaps_sql,
     models__ref_snapshot_sql,
     models__schema_yml,
 )
 
 snapshots_invalid__snapshot_sql = """
-{# make sure to never name this anything with `target_schema` in the name, or the test will be invalid! #}
 {% snapshot snapshot_actual %}
-    {# missing the mandatory target_schema parameter #}
+    {# missing the mandatory strategy parameter #}
     {{
         config(
             unique_key='id || ' ~ "'-'" ~ ' || first_name',
-            strategy='timestamp',
             updated_at='updated_at',
         )
     }}
@@ -47,7 +45,4 @@ def test_missing_strategy(project):
     with pytest.raises(ValidationError) as exc:
         run_dbt(["compile"], expect_pass=False)
 
-    assert (
-        "Snapshots must be configured with a 'strategy', 'unique_key', and 'target_schema'"
-        in str(exc.value)
-    )
+    assert "Snapshots must be configured with a 'strategy' and 'unique_key'" in str(exc.value)

tests/functional/simple_snapshot/test_renamed_source_snapshot.py → tests/functional/snapshots/test_renamed_source_snapshot.py

@@ -1,7 +1,7 @@
 import pytest
 
 from dbt.tests.util import run_dbt
-from tests.functional.simple_snapshot.fixtures import (
+from tests.functional.snapshots.fixtures import (
     macros__test_no_overlaps_sql,
     macros_custom_snapshot__custom_sql,
     seeds__seed_csv,

tests/functional/simple_snapshot/test_select_exclude_snapshot.py → tests/functional/snapshots/test_select_exclude_snapshot.py

@@ -3,7 +3,7 @@
 import pytest
 
 from dbt.tests.util import check_relations_equal, check_table_does_not_exist, run_dbt
-from tests.functional.simple_snapshot.fixtures import (
+from tests.functional.snapshots.fixtures import (
     macros__test_no_overlaps_sql,
     models__ref_snapshot_sql,
     models__schema_yml,

tests/functional/simple_snapshot/test_slow_query_snapshot.py → tests/functional/snapshots/test_slow_query_snapshot.py

@@ -1,7 +1,7 @@
 import pytest
 
 from dbt.tests.util import run_dbt
-from tests.functional.simple_snapshot.fixtures import models_slow__gen_sql
+from tests.functional.snapshots.fixtures import models_slow__gen_sql
 
 snapshots_slow__snapshot_sql = """
 

tests/unit/contracts/graph/test_nodes_parsed.py

@@ -1462,13 +1462,6 @@ def test_missing_snapshot_configs(basic_check_snapshot_config_dict):
         cfg = SnapshotConfig.from_dict(wrong_fields)
         cfg.final_validate()
 
-    wrong_fields["unique_key"] = "id"
-    del wrong_fields["target_schema"]
-    with pytest.raises(ValidationError, match=r"Snapshots must be configured with a 'strategy'"):
-        SnapshotConfig.validate(wrong_fields)
-        cfg = SnapshotConfig.from_dict(wrong_fields)
-        cfg.final_validate()
-
 
 def assert_snapshot_config_fails_validation(dct):
     with pytest.raises(ValidationError):