From 493381a5b8e043d20abda0f69f11ffa4e89c6266 Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Mon, 4 Nov 2024 12:01:08 -0500 Subject: [PATCH 1/3] Add changelog entry. --- .changes/unreleased/Features-20241104-120053.yaml | 6 ++++++ core/dbt/artifacts/resources/v1/snapshot.py | 2 ++ 2 files changed, 8 insertions(+) create mode 100644 .changes/unreleased/Features-20241104-120053.yaml diff --git a/.changes/unreleased/Features-20241104-120053.yaml b/.changes/unreleased/Features-20241104-120053.yaml new file mode 100644 index 00000000000..9451fa49884 --- /dev/null +++ b/.changes/unreleased/Features-20241104-120053.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add new hard_deletes="new_record" mode for snapshots. +time: 2024-11-04T12:00:53.95191-05:00 +custom: + Author: peterallenwebb + Issue: "10235" diff --git a/core/dbt/artifacts/resources/v1/snapshot.py b/core/dbt/artifacts/resources/v1/snapshot.py index 1a7b9344ca0..976c93a4d60 100644 --- a/core/dbt/artifacts/resources/v1/snapshot.py +++ b/core/dbt/artifacts/resources/v1/snapshot.py @@ -13,6 +13,7 @@ class SnapshotMetaColumnNames(dbtClassMixin): dbt_valid_from: Optional[str] = None dbt_scd_id: Optional[str] = None dbt_updated_at: Optional[str] = None + dbt_is_deleted: Optional[str] = None @dataclass @@ -37,6 +38,7 @@ def snapshot_table_column_names(self): "dbt_valid_to": self.snapshot_meta_column_names.dbt_valid_to or "dbt_valid_to", "dbt_scd_id": self.snapshot_meta_column_names.dbt_scd_id or "dbt_scd_id", "dbt_updated_at": self.snapshot_meta_column_names.dbt_updated_at or "dbt_updated_at", + "dbt_is_deleted": self.snapshot_meta_column_names.dbt_is_deleted or "dbt_is_deleted", } def final_validate(self): From 0ce9f9045bedcfe6a3414ab02cadd01af76edb91 Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Thu, 21 Nov 2024 16:57:08 -0500 Subject: [PATCH 2/3] Update schemas and test fixtures for new snapshot meta-column --- schemas/dbt/manifest/v12.json | 22 +++++++++++++++++++ .../functional/artifacts/expected_manifest.py | 1 + tests/functional/list/test_list.py | 9 +------- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/schemas/dbt/manifest/v12.json b/schemas/dbt/manifest/v12.json index 8a43f2e978b..892c0014e41 100644 --- a/schemas/dbt/manifest/v12.json +++ b/schemas/dbt/manifest/v12.json @@ -6754,6 +6754,17 @@ } ], "default": null + }, + "dbt_is_deleted": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": false @@ -16672,6 +16683,17 @@ } ], "default": null + }, + "dbt_is_deleted": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": false diff --git a/tests/functional/artifacts/expected_manifest.py b/tests/functional/artifacts/expected_manifest.py index aafe093d86e..c7deb2e8ea8 100644 --- a/tests/functional/artifacts/expected_manifest.py +++ b/tests/functional/artifacts/expected_manifest.py @@ -112,6 +112,7 @@ def get_rendered_snapshot_config(**updates): "dbt_valid_from": None, "dbt_updated_at": None, "dbt_scd_id": None, + "dbt_is_deleted": None, }, "dbt_valid_to_current": None, "tags": [], diff --git a/tests/functional/list/test_list.py b/tests/functional/list/test_list.py index 5442436271e..b669c95077e 100644 --- a/tests/functional/list/test_list.py +++ b/tests/functional/list/test_list.py @@ -69,6 +69,7 @@ def expect_snapshot_output(self, happy_path_project): # noqa: F811 "dbt_updated_at": None, "dbt_valid_from": None, "dbt_valid_to": None, + "dbt_is_deleted": None, }, "unique_key": "id", "strategy": "timestamp", @@ -923,12 +924,4 @@ def test_ls(self, happy_path_project): # noqa: F811 def normalize(path): - """On windows, neither is enough on its own: - >>> normcase('C:\\documents/ALL CAPS/subdir\\..') - 'c:\\documents\\all caps\\subdir\\..' - >>> normpath('C:\\documents/ALL CAPS/subdir\\..') - 'C:\\documents\\ALL CAPS' - >>> normpath(normcase('C:\\documents/ALL CAPS/subdir\\..')) - 'c:\\documents\\all caps' - """ return os.path.normcase(os.path.normpath(path)) From 29ca70fd1a86015bdece165911d550ef78b64753 Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Thu, 21 Nov 2024 17:18:27 -0500 Subject: [PATCH 3/3] Add back comment. --- tests/functional/list/test_list.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/functional/list/test_list.py b/tests/functional/list/test_list.py index b669c95077e..4a08cfad478 100644 --- a/tests/functional/list/test_list.py +++ b/tests/functional/list/test_list.py @@ -924,4 +924,12 @@ def test_ls(self, happy_path_project): # noqa: F811 def normalize(path): + """On windows, neither is enough on its own: + >>> normcase('C:\\documents/ALL CAPS/subdir\\..') + 'c:\\documents\\all caps\\subdir\\..' + >>> normpath('C:\\documents/ALL CAPS/subdir\\..') + 'C:\\documents\\ALL CAPS' + >>> normpath(normcase('C:\\documents/ALL CAPS/subdir\\..')) + 'c:\\documents\\all caps' + """ return os.path.normcase(os.path.normpath(path))