diff --git a/.changes/unreleased/Features-20240712-214546.yaml b/.changes/unreleased/Features-20240712-214546.yaml new file mode 100644 index 00000000000..3c9c5b8dd26 --- /dev/null +++ b/.changes/unreleased/Features-20240712-214546.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support standard schema/database fields for snapshots +time: 2024-07-12T21:45:46.06011-04:00 +custom: + Author: gshank + Issue: "10301" diff --git a/core/dbt/artifacts/resources/v1/snapshot.py b/core/dbt/artifacts/resources/v1/snapshot.py index dee235a19df..c9f1acdb50f 100644 --- a/core/dbt/artifacts/resources/v1/snapshot.py +++ b/core/dbt/artifacts/resources/v1/snapshot.py @@ -19,10 +19,9 @@ class SnapshotConfig(NodeConfig): check_cols: Union[str, List[str], None] = None def final_validate(self): - if not self.strategy or not self.unique_key or not self.target_schema: + if not self.strategy or not self.unique_key: raise ValidationError( - "Snapshots must be configured with a 'strategy', 'unique_key', " - "and 'target_schema'." + "Snapshots must be configured with a 'strategy' and 'unique_key'." ) if self.strategy == "check": if not self.check_cols: diff --git a/core/dbt/parser/base.py b/core/dbt/parser/base.py index 05d5b7d114d..a68f7384c0b 100644 --- a/core/dbt/parser/base.py +++ b/core/dbt/parser/base.py @@ -72,6 +72,7 @@ def __init__( class RelationUpdate: + # "component" is database, schema or alias def __init__(self, config: RuntimeConfig, manifest: Manifest, component: str) -> None: default_macro = manifest.find_generate_macro_by_name( component=component, @@ -127,6 +128,7 @@ def __init__( ) -> None: super().__init__(project, manifest, root_project) + # this sets callables from RelationUpdate self._update_node_database = RelationUpdate( manifest=manifest, config=root_project, component="database" ) @@ -288,7 +290,10 @@ def update_parsed_node_relation_names( self._update_node_schema(parsed_node, config_dict.get("schema")) self._update_node_alias(parsed_node, config_dict.get("alias")) - # Snapshot nodes use special "target_database" and "target_schema" fields for some reason + # Snapshot nodes use special "target_database" and "target_schema" fields + # for backward compatibility + # We have to do getattr here because saved_query parser calls this method with + # Export object instead of a node. if getattr(parsed_node, "resource_type", None) == NodeType.Snapshot: if "target_database" in config_dict and config_dict["target_database"]: parsed_node.database = config_dict["target_database"] @@ -443,9 +448,8 @@ def parse_node(self, block: ConfiguredBlockType) -> FinalNode: fqn=fqn, ) self.render_update(node, config) - result = self.transform(node) - self.add_result_node(block, result) - return result + self.add_result_node(block, node) + return node def _update_node_relation_name(self, node: ManifestNode): # Seed and Snapshot nodes and Models that are not ephemeral, @@ -464,17 +468,12 @@ def _update_node_relation_name(self, node: ManifestNode): def parse_file(self, file_block: FileBlock) -> None: pass - @abc.abstractmethod - def transform(self, node: FinalNode) -> FinalNode: - pass - class SimpleParser( ConfiguredParser[ConfiguredBlockType, FinalNode], Generic[ConfiguredBlockType, FinalNode], ): - def transform(self, node): - return node + pass class SQLParser(ConfiguredParser[FileBlock, FinalNode], Generic[FinalNode]): @@ -483,5 +482,4 @@ def parse_file(self, file_block: FileBlock) -> None: class SimpleSQLParser(SQLParser[FinalNode]): - def transform(self, node): - return node + pass diff --git a/core/dbt/parser/hooks.py b/core/dbt/parser/hooks.py index f8efd6a5a75..bcc25c0d937 100644 --- a/core/dbt/parser/hooks.py +++ b/core/dbt/parser/hooks.py @@ -66,8 +66,6 @@ def __iter__(self) -> Iterator[HookBlock]: class HookParser(SimpleParser[HookBlock, HookNode]): - def transform(self, node): - return node # Hooks are only in the dbt_project.yml file for the project def get_path(self) -> FilePath: diff --git a/core/dbt/parser/snapshots.py b/core/dbt/parser/snapshots.py index b4d1c7e5349..3f9a7326c74 100644 --- a/core/dbt/parser/snapshots.py +++ b/core/dbt/parser/snapshots.py @@ -2,12 +2,10 @@ from typing import List from dbt.contracts.graph.nodes import SnapshotNode -from dbt.exceptions import SnapshopConfigError from dbt.node_types import NodeType from dbt.parser.base import SQLParser from dbt.parser.search import BlockContents, BlockSearcher, FileBlock from dbt.utils import split_path -from dbt_common.dataclass_schema import ValidationError class SnapshotParser(SQLParser[SnapshotNode]): @@ -24,24 +22,6 @@ def resource_type(self) -> NodeType: def get_compiled_path(cls, block: FileBlock): return block.path.relative_path - def set_snapshot_attributes(self, node): - # use the target_database setting if we got it, otherwise the - # `database` value of the node (ultimately sourced from the `database` - # config value), and if that is not set, use the database defined in - # the adapter's credentials. - if node.config.target_database: - node.database = node.config.target_database - elif not node.database: - node.database = self.root_project.credentials.database - - # the target schema must be set if we got here, so overwrite the node's - # schema - node.schema = node.config.target_schema - # We need to set relation_name again, since database/schema might have changed - self._update_node_relation_name(node) - - return node - def get_fqn(self, path: str, name: str) -> List[str]: """Get the FQN for the node. This impacts node selection and config application. @@ -54,13 +34,6 @@ def get_fqn(self, path: str, name: str) -> List[str]: fqn.append(name) return fqn - def transform(self, node: SnapshotNode) -> SnapshotNode: - try: - self.set_snapshot_attributes(node) - return node - except ValidationError as exc: - raise SnapshopConfigError(exc, node) - def parse_file(self, file_block: FileBlock) -> None: blocks = BlockSearcher( source=[file_block], diff --git a/tests/functional/simple_snapshot/data/invalidate_postgres.sql b/tests/functional/snapshots/data/invalidate_postgres.sql similarity index 100% rename from tests/functional/simple_snapshot/data/invalidate_postgres.sql rename to tests/functional/snapshots/data/invalidate_postgres.sql diff --git a/tests/functional/simple_snapshot/data/seed_pg.sql b/tests/functional/snapshots/data/seed_pg.sql similarity index 100% rename from tests/functional/simple_snapshot/data/seed_pg.sql rename to tests/functional/snapshots/data/seed_pg.sql diff --git a/tests/functional/simple_snapshot/data/shared_macros.sql b/tests/functional/snapshots/data/shared_macros.sql similarity index 100% rename from tests/functional/simple_snapshot/data/shared_macros.sql rename to tests/functional/snapshots/data/shared_macros.sql diff --git a/tests/functional/simple_snapshot/data/update.sql b/tests/functional/snapshots/data/update.sql similarity index 100% rename from tests/functional/simple_snapshot/data/update.sql rename to tests/functional/snapshots/data/update.sql diff --git a/tests/functional/simple_snapshot/fixtures.py b/tests/functional/snapshots/fixtures.py similarity index 100% rename from tests/functional/simple_snapshot/fixtures.py rename to tests/functional/snapshots/fixtures.py diff --git a/tests/functional/simple_snapshot/test_basic_snapshot.py b/tests/functional/snapshots/test_basic_snapshot.py similarity index 97% rename from tests/functional/simple_snapshot/test_basic_snapshot.py rename to tests/functional/snapshots/test_basic_snapshot.py index b1a290f6b99..ac6c3831642 100644 --- a/tests/functional/simple_snapshot/test_basic_snapshot.py +++ b/tests/functional/snapshots/test_basic_snapshot.py @@ -10,7 +10,7 @@ run_dbt, write_file, ) -from tests.functional.simple_snapshot.fixtures import ( +from tests.functional.snapshots.fixtures import ( macros__test_no_overlaps_sql, macros_custom_snapshot__custom_sql, models__ref_snapshot_sql, @@ -142,7 +142,7 @@ def project_config_update(self, unique_schema): return { "snapshots": { "test": { - "target_schema": unique_schema + "_alt", + "schema": "alt", } } } @@ -153,6 +153,8 @@ def test_target_schema(self, project): # ensure that the schema in the snapshot node is the same as target_schema snapshot_id = "snapshot.test.snapshot_actual" snapshot_node = manifest.nodes[snapshot_id] + # The schema field be changed by the default "generate_schema_name" + # to append an underscore plus the configured schema of "alt". assert snapshot_node.schema == f"{project.test_schema}_alt" assert ( snapshot_node.relation_name diff --git a/tests/functional/simple_snapshot/test_changing_check_cols_snapshot.py b/tests/functional/snapshots/test_changing_check_cols_snapshot.py similarity index 100% rename from tests/functional/simple_snapshot/test_changing_check_cols_snapshot.py rename to tests/functional/snapshots/test_changing_check_cols_snapshot.py diff --git a/tests/functional/simple_snapshot/test_changing_strategy_snapshot.py b/tests/functional/snapshots/test_changing_strategy_snapshot.py similarity index 97% rename from tests/functional/simple_snapshot/test_changing_strategy_snapshot.py rename to tests/functional/snapshots/test_changing_strategy_snapshot.py index b553485e674..e02df65938d 100644 --- a/tests/functional/simple_snapshot/test_changing_strategy_snapshot.py +++ b/tests/functional/snapshots/test_changing_strategy_snapshot.py @@ -1,7 +1,7 @@ import pytest from dbt.tests.util import run_dbt -from tests.functional.simple_snapshot.fixtures import models_slow__gen_sql +from tests.functional.snapshots.fixtures import models_slow__gen_sql test_snapshots_changing_strategy__test_snapshot_sql = """ diff --git a/tests/functional/simple_snapshot/test_check_cols_snapshot.py b/tests/functional/snapshots/test_check_cols_snapshot.py similarity index 98% rename from tests/functional/simple_snapshot/test_check_cols_snapshot.py rename to tests/functional/snapshots/test_check_cols_snapshot.py index 8ee2817c45d..2b38d78ceab 100644 --- a/tests/functional/simple_snapshot/test_check_cols_snapshot.py +++ b/tests/functional/snapshots/test_check_cols_snapshot.py @@ -100,7 +100,7 @@ def tests(): return {"my_test.sql": snapshot_test_sql} -def test_simple_snapshot(project): +def test_snapshots(project): results = run_dbt(["snapshot", "--vars", "version: 1"]) assert len(results) == 1 diff --git a/tests/functional/simple_snapshot/test_check_cols_updated_at_snapshot.py b/tests/functional/snapshots/test_check_cols_updated_at_snapshot.py similarity index 99% rename from tests/functional/simple_snapshot/test_check_cols_updated_at_snapshot.py rename to tests/functional/snapshots/test_check_cols_updated_at_snapshot.py index 31265c4121f..73a76ffe716 100644 --- a/tests/functional/simple_snapshot/test_check_cols_updated_at_snapshot.py +++ b/tests/functional/snapshots/test_check_cols_updated_at_snapshot.py @@ -74,7 +74,7 @@ def project_config_update(): } -def test_simple_snapshot(project): +def test_snapshots(project): """ Test that the `dbt_updated_at` column reflects the `updated_at` timestamp expression in the config. diff --git a/tests/functional/simple_snapshot/test_comment_ending_snapshot.py b/tests/functional/snapshots/test_comment_ending_snapshot.py similarity index 100% rename from tests/functional/simple_snapshot/test_comment_ending_snapshot.py rename to tests/functional/snapshots/test_comment_ending_snapshot.py diff --git a/tests/functional/simple_snapshot/test_cross_schema_snapshot.py b/tests/functional/snapshots/test_cross_schema_snapshot.py similarity index 95% rename from tests/functional/simple_snapshot/test_cross_schema_snapshot.py rename to tests/functional/snapshots/test_cross_schema_snapshot.py index 562bb6e0c61..07fc8020773 100644 --- a/tests/functional/simple_snapshot/test_cross_schema_snapshot.py +++ b/tests/functional/snapshots/test_cross_schema_snapshot.py @@ -3,7 +3,7 @@ import pytest from dbt.tests.util import run_dbt -from tests.functional.simple_snapshot.fixtures import ( +from tests.functional.snapshots.fixtures import ( macros__test_no_overlaps_sql, models__ref_snapshot_sql, models__schema_yml, diff --git a/tests/functional/simple_snapshot/test_hard_delete_snapshot.py b/tests/functional/snapshots/test_hard_delete_snapshot.py similarity index 99% rename from tests/functional/simple_snapshot/test_hard_delete_snapshot.py rename to tests/functional/snapshots/test_hard_delete_snapshot.py index befd06097f4..93cb524c59a 100644 --- a/tests/functional/simple_snapshot/test_hard_delete_snapshot.py +++ b/tests/functional/snapshots/test_hard_delete_snapshot.py @@ -5,7 +5,7 @@ import pytz from dbt.tests.util import check_relations_equal, run_dbt -from tests.functional.simple_snapshot.fixtures import ( +from tests.functional.snapshots.fixtures import ( macros__test_no_overlaps_sql, models__ref_snapshot_sql, models__schema_yml, diff --git a/tests/functional/simple_snapshot/test_invalid_namespace_snapshot.py b/tests/functional/snapshots/test_invalid_namespace_snapshot.py similarity index 96% rename from tests/functional/simple_snapshot/test_invalid_namespace_snapshot.py rename to tests/functional/snapshots/test_invalid_namespace_snapshot.py index c4f6b88f247..31060bbba3f 100644 --- a/tests/functional/simple_snapshot/test_invalid_namespace_snapshot.py +++ b/tests/functional/snapshots/test_invalid_namespace_snapshot.py @@ -3,7 +3,7 @@ import pytest from dbt.tests.util import run_dbt -from tests.functional.simple_snapshot.fixtures import ( +from tests.functional.snapshots.fixtures import ( macros__test_no_overlaps_sql, macros_custom_snapshot__custom_sql, models__ref_snapshot_sql, diff --git a/tests/functional/simple_snapshot/test_long_text_snapshot.py b/tests/functional/snapshots/test_long_text_snapshot.py similarity index 96% rename from tests/functional/simple_snapshot/test_long_text_snapshot.py rename to tests/functional/snapshots/test_long_text_snapshot.py index 453c4164be3..495bdca46b2 100644 --- a/tests/functional/simple_snapshot/test_long_text_snapshot.py +++ b/tests/functional/snapshots/test_long_text_snapshot.py @@ -1,7 +1,7 @@ import pytest from dbt.tests.util import run_dbt -from tests.functional.simple_snapshot.fixtures import ( +from tests.functional.snapshots.fixtures import ( macros__test_no_overlaps_sql, models__ref_snapshot_sql, models__schema_yml, diff --git a/tests/functional/simple_snapshot/test_missing_strategy_snapshot.py b/tests/functional/snapshots/test_missing_strategy_snapshot.py similarity index 71% rename from tests/functional/simple_snapshot/test_missing_strategy_snapshot.py rename to tests/functional/snapshots/test_missing_strategy_snapshot.py index 46543da8f4b..6f550d05a28 100644 --- a/tests/functional/simple_snapshot/test_missing_strategy_snapshot.py +++ b/tests/functional/snapshots/test_missing_strategy_snapshot.py @@ -2,20 +2,18 @@ from dbt.tests.util import run_dbt from dbt_common.dataclass_schema import ValidationError -from tests.functional.simple_snapshot.fixtures import ( +from tests.functional.snapshots.fixtures import ( macros__test_no_overlaps_sql, models__ref_snapshot_sql, models__schema_yml, ) snapshots_invalid__snapshot_sql = """ -{# make sure to never name this anything with `target_schema` in the name, or the test will be invalid! #} {% snapshot snapshot_actual %} - {# missing the mandatory target_schema parameter #} + {# missing the mandatory strategy parameter #} {{ config( unique_key='id || ' ~ "'-'" ~ ' || first_name', - strategy='timestamp', updated_at='updated_at', ) }} @@ -47,7 +45,4 @@ def test_missing_strategy(project): with pytest.raises(ValidationError) as exc: run_dbt(["compile"], expect_pass=False) - assert ( - "Snapshots must be configured with a 'strategy', 'unique_key', and 'target_schema'" - in str(exc.value) - ) + assert "Snapshots must be configured with a 'strategy' and 'unique_key'" in str(exc.value) diff --git a/tests/functional/simple_snapshot/test_renamed_source_snapshot.py b/tests/functional/snapshots/test_renamed_source_snapshot.py similarity index 97% rename from tests/functional/simple_snapshot/test_renamed_source_snapshot.py rename to tests/functional/snapshots/test_renamed_source_snapshot.py index e59f2192051..95441aeaaeb 100644 --- a/tests/functional/simple_snapshot/test_renamed_source_snapshot.py +++ b/tests/functional/snapshots/test_renamed_source_snapshot.py @@ -1,7 +1,7 @@ import pytest from dbt.tests.util import run_dbt -from tests.functional.simple_snapshot.fixtures import ( +from tests.functional.snapshots.fixtures import ( macros__test_no_overlaps_sql, macros_custom_snapshot__custom_sql, seeds__seed_csv, diff --git a/tests/functional/simple_snapshot/test_select_exclude_snapshot.py b/tests/functional/snapshots/test_select_exclude_snapshot.py similarity index 98% rename from tests/functional/simple_snapshot/test_select_exclude_snapshot.py rename to tests/functional/snapshots/test_select_exclude_snapshot.py index 6055ebbff04..b460f1fbc25 100644 --- a/tests/functional/simple_snapshot/test_select_exclude_snapshot.py +++ b/tests/functional/snapshots/test_select_exclude_snapshot.py @@ -3,7 +3,7 @@ import pytest from dbt.tests.util import check_relations_equal, check_table_does_not_exist, run_dbt -from tests.functional.simple_snapshot.fixtures import ( +from tests.functional.snapshots.fixtures import ( macros__test_no_overlaps_sql, models__ref_snapshot_sql, models__schema_yml, diff --git a/tests/functional/simple_snapshot/test_slow_query_snapshot.py b/tests/functional/snapshots/test_slow_query_snapshot.py similarity index 95% rename from tests/functional/simple_snapshot/test_slow_query_snapshot.py rename to tests/functional/snapshots/test_slow_query_snapshot.py index 19a3b83c0e7..ac01105c6cf 100644 --- a/tests/functional/simple_snapshot/test_slow_query_snapshot.py +++ b/tests/functional/snapshots/test_slow_query_snapshot.py @@ -1,7 +1,7 @@ import pytest from dbt.tests.util import run_dbt -from tests.functional.simple_snapshot.fixtures import models_slow__gen_sql +from tests.functional.snapshots.fixtures import models_slow__gen_sql snapshots_slow__snapshot_sql = """ diff --git a/tests/functional/simple_snapshot/test_snapshot_config.py b/tests/functional/snapshots/test_snapshot_config.py similarity index 100% rename from tests/functional/simple_snapshot/test_snapshot_config.py rename to tests/functional/snapshots/test_snapshot_config.py diff --git a/tests/unit/contracts/graph/test_nodes_parsed.py b/tests/unit/contracts/graph/test_nodes_parsed.py index bd72529e272..ebbe2443771 100644 --- a/tests/unit/contracts/graph/test_nodes_parsed.py +++ b/tests/unit/contracts/graph/test_nodes_parsed.py @@ -1462,13 +1462,6 @@ def test_missing_snapshot_configs(basic_check_snapshot_config_dict): cfg = SnapshotConfig.from_dict(wrong_fields) cfg.final_validate() - wrong_fields["unique_key"] = "id" - del wrong_fields["target_schema"] - with pytest.raises(ValidationError, match=r"Snapshots must be configured with a 'strategy'"): - SnapshotConfig.validate(wrong_fields) - cfg = SnapshotConfig.from_dict(wrong_fields) - cfg.final_validate() - def assert_snapshot_config_fails_validation(dct): with pytest.raises(ValidationError):