From 4e74e6949747247af36e232f7fd3cf2b59aaa374 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 5 Dec 2024 16:42:34 -0500 Subject: [PATCH] Implement partial parsing for singular data tests configs in yaml files (#11100) (#11101) (cherry picked from commit e32b8a90ac7ffafecc98f26646a6e6c3ff49eba0) Co-authored-by: Gerda Shank --- .../unreleased/Fixes-20241205-145307.yaml | 6 +++++ core/dbt/parser/partial.py | 18 ++++++++++++++ core/dbt/parser/schemas.py | 2 -- tests/functional/data_test_patch/fixtures.py | 14 +++++++++-- .../test_singular_test_patch.py | 24 +++++++++++++------ 5 files changed, 53 insertions(+), 11 deletions(-) create mode 100644 .changes/unreleased/Fixes-20241205-145307.yaml diff --git a/.changes/unreleased/Fixes-20241205-145307.yaml b/.changes/unreleased/Fixes-20241205-145307.yaml new file mode 100644 index 00000000000..d9d33b3cdf5 --- /dev/null +++ b/.changes/unreleased/Fixes-20241205-145307.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Implement partial parsing for singular data test configs in yaml files +time: 2024-12-05T14:53:07.295536-05:00 +custom: + Author: gshank + Issue: "10801" diff --git a/core/dbt/parser/partial.py b/core/dbt/parser/partial.py index d4e20a617e1..e011c57670a 100644 --- a/core/dbt/parser/partial.py +++ b/core/dbt/parser/partial.py @@ -725,6 +725,7 @@ def handle_change(key: str, delete: Callable): handle_change("semantic_models", self.delete_schema_semantic_model) handle_change("unit_tests", self.delete_schema_unit_test) handle_change("saved_queries", self.delete_schema_saved_query) + handle_change("data_tests", self.delete_schema_data_test_patch) def _handle_element_change( self, schema_file, saved_yaml_dict, new_yaml_dict, env_var_changes, dict_key: str, delete @@ -919,6 +920,23 @@ def delete_schema_macro_patch(self, schema_file, macro): self.saved_files[macro_file_id] = deepcopy(self.new_files[macro_file_id]) self.add_to_pp_files(self.saved_files[macro_file_id]) + def delete_schema_data_test_patch(self, schema_file, data_test): + data_test_unique_id = None + for unique_id in schema_file.node_patches: + if not unique_id.startswith("test"): + continue + parts = unique_id.split(".") + elem_name = parts[2] + if elem_name == data_test["name"]: + data_test_unique_id = unique_id + break + if data_test_unique_id and data_test_unique_id in self.saved_manifest.nodes: + singular_data_test = self.saved_manifest.nodes.pop(data_test_unique_id) + file_id = singular_data_test.file_id + if file_id in self.new_files: + self.saved_files[file_id] = deepcopy(self.new_files[file_id]) + self.add_to_pp_files(self.saved_files[file_id]) + # exposures are created only from schema files, so just delete # the exposure or the disabled exposure. def delete_schema_exposure(self, schema_file, exposure_dict): diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index e0c94b8b444..12bb84a5037 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -1199,8 +1199,6 @@ def parse_patch(self, block: TargetBlock[UnparsedSingularTestUpdate], refs: Pars node.patch_path = patch.file_id node.description = patch.description node.created_at = time.time() - node.meta = patch.meta - node.docs = patch.docs class MacroPatchParser(PatchParser[UnparsedMacroUpdate, ParsedMacroPatch]): diff --git a/tests/functional/data_test_patch/fixtures.py b/tests/functional/data_test_patch/fixtures.py index be056f32680..5bac70969c5 100644 --- a/tests/functional/data_test_patch/fixtures.py +++ b/tests/functional/data_test_patch/fixtures.py @@ -13,8 +13,18 @@ description: "{{ doc('my_singular_test_documentation') }}" config: error_if: ">10" - meta: - some_key: some_val + meta: + some_key: some_val +""" + +tests__schema_2_yml = """ +data_tests: + - name: my_singular_test + description: "My singular test description" + config: + error_if: ">10" + meta: + some_key: another_val """ tests__doc_block_md = """ diff --git a/tests/functional/data_test_patch/test_singular_test_patch.py b/tests/functional/data_test_patch/test_singular_test_patch.py index df359c5e645..e7004bab9e2 100644 --- a/tests/functional/data_test_patch/test_singular_test_patch.py +++ b/tests/functional/data_test_patch/test_singular_test_patch.py @@ -2,12 +2,13 @@ import pytest -from dbt.tests.util import get_artifact, run_dbt, run_dbt_and_capture +from dbt.tests.util import get_manifest, run_dbt, run_dbt_and_capture, write_file from tests.functional.data_test_patch.fixtures import ( tests__doc_block_md, tests__invalid_name_schema_yml, tests__malformed_schema_yml, tests__my_singular_test_sql, + tests__schema_2_yml, tests__schema_yml, ) @@ -23,13 +24,22 @@ def tests(self): def test_compile(self, project): run_dbt(["compile"]) - manifest = get_artifact(project.project_root, "target", "manifest.json") - assert len(manifest["nodes"]) == 1 + manifest = get_manifest(project.project_root) + assert len(manifest.nodes) == 1 - my_singular_test_node = manifest["nodes"]["test.test.my_singular_test"] - assert my_singular_test_node["description"] == "Some docs from a doc block" - assert my_singular_test_node["config"]["error_if"] == ">10" - assert my_singular_test_node["config"]["meta"] == {"some_key": "some_val"} + my_singular_test_node = manifest.nodes["test.test.my_singular_test"] + assert my_singular_test_node.description == "Some docs from a doc block" + assert my_singular_test_node.config.error_if == ">10" + assert my_singular_test_node.config.meta == {"some_key": "some_val"} + assert my_singular_test_node.meta == {"some_key": "some_val"} + + # partial parsing test + write_file(tests__schema_2_yml, project.project_root, "tests", "schema.yml") + manifest = run_dbt(["parse"]) + test_node = manifest.nodes["test.test.my_singular_test"] + assert test_node.description == "My singular test description" + assert test_node.config.meta == {"some_key": "another_val"} + assert test_node.meta == {"some_key": "another_val"} class TestPatchSingularTestInvalidName: