From 3ca0ecd33ada7c9069536597be9163b5246b3226 Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Fri, 31 May 2024 23:25:43 +0530 Subject: [PATCH] revert altimate_unique_id --- src/datapilot/core/insights/report.py | 2 +- .../core/platforms/dbt/insights/base.py | 4 +- .../checks/check_column_name_contract.py | 12 ++--- .../checks/check_model_has_all_columns.py | 12 ++--- .../checks/check_model_has_labels_keys.py | 14 ++--- .../checks/check_model_has_meta_keys.py | 12 ++--- .../checks/check_model_has_properties_file.py | 6 +-- .../checks/check_model_has_tests_by_group.py | 8 ++- .../checks/check_model_has_tests_by_name.py | 6 +-- .../checks/check_model_has_tests_by_type.py | 6 +-- .../checks/check_model_name_contract.py | 8 +-- .../checks/check_model_parents_and_childs.py | 12 ++--- .../checks/check_model_parents_database.py | 4 +- .../checks/check_model_parents_schema.py | 4 +- .../checks/check_source_columns_have_desc.py | 6 +-- .../checks/check_source_has_labels_keys.py | 14 ++--- .../dbt_test/missing_primary_key_tests.py | 12 ++--- .../documentation_on_stale_columns.py | 16 +++--- .../public_models_without_contracts.py | 16 +++--- .../governance/undocumented_columns.py | 18 +++---- .../governance/undocumented_public_models.py | 12 ++--- .../modelling/direct_join_to_source.py | 16 +++--- .../downstream_models_dependent_on_source.py | 16 +++--- .../modelling/hard_coded_references.py | 14 ++--- .../dbt/insights/modelling/model_fanout.py | 14 ++--- .../dbt/insights/modelling/root_model.py | 16 +++--- ...ng_model_dependent_on_downstream_models.py | 12 ++--- ...aging_model_dependent_on_staging_models.py | 12 ++--- .../structure/model_directories_structure.py | 14 ++--- .../structure/model_naming_conventions.py | 14 ++--- .../structure/source_directories_structure.py | 10 ++-- .../structure/test_directory_structure.py | 12 ++--- tests/data/manifest_v11.json | 52 +++++++++---------- tests/data/manifest_v11macroargs.json | 52 +++++++++---------- 34 files changed, 228 insertions(+), 230 deletions(-) diff --git a/src/datapilot/core/insights/report.py b/src/datapilot/core/insights/report.py index e11fd5b5..9fbc8036 100644 --- a/src/datapilot/core/insights/report.py +++ b/src/datapilot/core/insights/report.py @@ -8,7 +8,7 @@ def generate_ci_cd_report(insights_data): for insight in insights_data: print(divider) print(f"Project: {insight.package_name}") - print(f"Model ID: {insight.altimate_unique_id}") + print(f"Model ID: {insight.model_unique_id}") print(f"Name: {insight.metadata['model']}") print(f"Message: {insight.message}") print(f"Reason: {insight.reason_to_flag}") diff --git a/src/datapilot/core/platforms/dbt/insights/base.py b/src/datapilot/core/platforms/dbt/insights/base.py index 271614b6..50f0c5a7 100644 --- a/src/datapilot/core/platforms/dbt/insights/base.py +++ b/src/datapilot/core/platforms/dbt/insights/base.py @@ -108,10 +108,10 @@ def build_chain(node_id, current_chain): return long_chains - def should_skip_model(self, altimate_unique_id): + def should_skip_model(self, model_unique_id): """Check if a model is in the excluded models list.""" if self.selected_models: - return altimate_unique_id not in self.selected_models + return model_unique_id not in self.selected_models return False diff --git a/src/datapilot/core/platforms/dbt/insights/checks/check_column_name_contract.py b/src/datapilot/core/platforms/dbt/insights/checks/check_column_name_contract.py index b7273a7c..e099b9d0 100644 --- a/src/datapilot/core/platforms/dbt/insights/checks/check_column_name_contract.py +++ b/src/datapilot/core/platforms/dbt/insights/checks/check_column_name_contract.py @@ -22,11 +22,11 @@ class CheckColumnNameContract(ChecksInsight): "modeling and analysis. It's important to maintain consistent column naming conventions." ) FAILURE_MESSAGE = ( - "The following columns in the model `{altimate_unique_id}` do not adhere to the contract:\n{columns}. " + "The following columns in the model `{model_unique_id}` do not adhere to the contract:\n{columns}. " "Inconsistent column naming conventions can impede understanding and usage of the model." ) RECOMMENDATION = ( - "Update the column names listed above in the model `{altimate_unique_id}` to adhere to the contract. " + "Update the column names listed above in the model `{model_unique_id}` to adhere to the contract. " "Consistent column naming conventions provide valuable context and aids in data understanding and collaboration." ) PATTERN_STR = "pattern" @@ -75,12 +75,12 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]: ) return insights - def _build_failure_result(self, altimate_unique_id: str, columns: Sequence[str]) -> DBTInsightResult: + def _build_failure_result(self, model_unique_id: str, columns: Sequence[str]) -> DBTInsightResult: failure_message = self.FAILURE_MESSAGE.format( columns=numbered_list(columns), - altimate_unique_id=altimate_unique_id, + model_unique_id=model_unique_id, ) - recommendation = self.RECOMMENDATION.format(altimate_unique_id=altimate_unique_id) + recommendation = self.RECOMMENDATION.format(model_unique_id=model_unique_id) return DBTInsightResult( type=self.TYPE, @@ -88,7 +88,7 @@ def _build_failure_result(self, altimate_unique_id: str, columns: Sequence[str]) message=failure_message, recommendation=recommendation, reason_to_flag=self.REASON_TO_FLAG, - metadata={"columns": columns, "altimate_unique_id": altimate_unique_id}, + metadata={"columns": columns, "model_unique_id": model_unique_id}, ) def _get_columns_in_model(self, node_id) -> List[str]: diff --git a/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_all_columns.py b/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_all_columns.py index 07639bff..fcf46835 100644 --- a/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_all_columns.py +++ b/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_all_columns.py @@ -48,13 +48,13 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]: ) return insights - def _build_failure_result(self, altimate_unique_id: str, columns: Sequence[str]) -> DBTInsightResult: + def _build_failure_result(self, model_unique_id: str, columns: Sequence[str]) -> DBTInsightResult: failure_message = ( - "The following columns in the model `{altimate_unique_id}` are missing:\n{columns}. " + "The following columns in the model `{model_unique_id}` are missing:\n{columns}. " "Ensure that the model includes all the required columns." ) recommendation = ( - "Add the missing columns listed above in the model `{altimate_unique_id}`. " + "Add the missing columns listed above in the model `{model_unique_id}`. " "Ensuring that the model has all the required columns helps in maintaining data integrity and consistency." ) @@ -63,11 +63,11 @@ def _build_failure_result(self, altimate_unique_id: str, columns: Sequence[str]) name=self.NAME, message=failure_message.format( columns=numbered_list(columns), - altimate_unique_id=altimate_unique_id, + model_unique_id=model_unique_id, ), - recommendation=recommendation.format(altimate_unique_id=altimate_unique_id), + recommendation=recommendation.format(model_unique_id=model_unique_id), reason_to_flag=self.REASON_TO_FLAG, - metadata={"columns": columns, "altimate_unique_id": altimate_unique_id}, + metadata={"columns": columns, "model_unique_id": model_unique_id}, ) def _check_model_columns(self, node_id) -> Tuple[int, Set[str]]: diff --git a/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_labels_keys.py b/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_labels_keys.py index f077fcfd..48616e76 100644 --- a/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_labels_keys.py +++ b/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_labels_keys.py @@ -46,18 +46,18 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]: ) return insights - def _build_failure_result(self, altimate_unique_id: str, missing_keys: Sequence[str], extra_labels: Sequence[str]) -> DBTInsightResult: + def _build_failure_result(self, model_unique_id: str, missing_keys: Sequence[str], extra_labels: Sequence[str]) -> DBTInsightResult: failure_message = ( - f"The following labels keys are missing in the model `{altimate_unique_id}`:\n{missing_keys}. " + f"The following labels keys are missing in the model `{model_unique_id}`:\n{missing_keys}. " "Ensure that the model includes all the required labels keys." ) if not self.allow_extra_keys: failure_message += ( - f"The following extra labels keys are present in the model `{altimate_unique_id}`:\n{extra_labels}. " + f"The following extra labels keys are present in the model `{model_unique_id}`:\n{extra_labels}. " "Ensure that the model does not include any extra labels keys." ) recommendation = ( - f"Add the missing labels keys listed above in the model `{altimate_unique_id}`. " + f"Add the missing labels keys listed above in the model `{model_unique_id}`. " "Ensuring that the model has all the required labels keys helps in maintaining metadata consistency and understanding." ) @@ -66,11 +66,11 @@ def _build_failure_result(self, altimate_unique_id: str, missing_keys: Sequence[ name=self.NAME, message=failure_message.format( missing_keys=numbered_list(missing_keys), - altimate_unique_id=altimate_unique_id, + model_unique_id=model_unique_id, ), - recommendation=recommendation.format(altimate_unique_id=altimate_unique_id), + recommendation=recommendation.format(model_unique_id=model_unique_id), reason_to_flag=self.REASON_TO_FLAG, - metadata={"missing_keys": missing_keys, "altimate_unique_id": altimate_unique_id, "extra_keys": extra_labels}, + metadata={"missing_keys": missing_keys, "model_unique_id": model_unique_id, "extra_keys": extra_labels}, ) def _check_labels_keys(self, node_id) -> Tuple[int, Set[str]]: diff --git a/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_meta_keys.py b/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_meta_keys.py index 79259974..f45d2e2e 100644 --- a/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_meta_keys.py +++ b/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_meta_keys.py @@ -45,18 +45,18 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]: ) return insights - def _build_failure_result(self, altimate_unique_id: str, missing_keys: Sequence[str], extra_keys: Set[str]) -> DBTInsightResult: + def _build_failure_result(self, model_unique_id: str, missing_keys: Sequence[str], extra_keys: Set[str]) -> DBTInsightResult: failure_message = ( - f"The following meta keys are missing in the model `{altimate_unique_id}`:\n{numbered_list(missing_keys)}. " + f"The following meta keys are missing in the model `{model_unique_id}`:\n{numbered_list(missing_keys)}. " "Ensure that the model includes all the required meta keys." ) if not self.allow_extra_keys: failure_message += ( - f"The following extra meta keys are present in the model `{altimate_unique_id}`:\n{numbered_list(extra_keys)}. " + f"The following extra meta keys are present in the model `{model_unique_id}`:\n{numbered_list(extra_keys)}. " "Ensure that the model does not include any extra meta keys." ) recommendation = ( - f"Add the missing meta keys listed above in the model `{altimate_unique_id}`. " + f"Add the missing meta keys listed above in the model `{model_unique_id}`. " "Ensuring that the model has all the required meta keys helps in maintaining metadata consistency and understanding." ) @@ -64,9 +64,9 @@ def _build_failure_result(self, altimate_unique_id: str, missing_keys: Sequence[ type=self.TYPE, name=self.NAME, message=failure_message, - recommendation=recommendation.format(altimate_unique_id=altimate_unique_id), + recommendation=recommendation.format(model_unique_id=model_unique_id), reason_to_flag=self.REASON_TO_FLAG, - metadata={"missing_keys": missing_keys, "altimate_unique_id": altimate_unique_id, "extra_keys": extra_keys}, + metadata={"missing_keys": missing_keys, "model_unique_id": model_unique_id, "extra_keys": extra_keys}, ) def _check_meta_keys(self, node_id) -> Tuple[int, Set[str], Set[str]]: diff --git a/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_properties_file.py b/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_properties_file.py index 6c7932fc..657ff478 100644 --- a/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_properties_file.py +++ b/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_properties_file.py @@ -37,9 +37,9 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]: ) return insights - def _build_failure_result(self, altimate_unique_id: str) -> DBTInsightResult: + def _build_failure_result(self, model_unique_id: str) -> DBTInsightResult: failure_message = ( - f"The model {altimate_unique_id} do not have a properties file (.yml) defined." + f"The model {model_unique_id} do not have a properties file (.yml) defined." "Ensure that each model has a corresponding .yml file for additional configuration and documentation." ) recommendation = ( @@ -53,7 +53,7 @@ def _build_failure_result(self, altimate_unique_id: str) -> DBTInsightResult: message=failure_message, recommendation=recommendation, reason_to_flag=self.REASON_TO_FLAG, - metadata={"altimate_unique_id": altimate_unique_id}, + metadata={"model_unique_id": model_unique_id}, ) def _check_properties_file(self, node_id) -> int: diff --git a/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_tests_by_group.py b/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_tests_by_group.py index 5757c6ba..66dd1fdb 100644 --- a/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_tests_by_group.py +++ b/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_tests_by_group.py @@ -47,14 +47,12 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]: ) return insights - def _build_failure_result(self, altimate_unique_id: str, missing_test_groups: List[Dict]) -> DBTInsightResult: + def _build_failure_result(self, model_unique_id: str, missing_test_groups: List[Dict]) -> DBTInsightResult: missing_test_group_str = "" for test in missing_test_groups: missing_test_group_str += f"Test Group: {test.get(self.TEST_GROUP_STR)}, Min Count: {test.get(self.TEST_COUNT_STR)}, Actual Count: {test.get('actual_count')}\n" - failure_message = ( - f"The model `{altimate_unique_id}` does not have enough tests for the following groups:\n{missing_test_group_str}. " - ) + failure_message = f"The model `{model_unique_id}` does not have enough tests for the following groups:\n{missing_test_group_str}. " recommendation = ( "Add tests with the specified groups for each model listed above. " "Having tests with specific groups ensures proper validation and data integrity." @@ -66,7 +64,7 @@ def _build_failure_result(self, altimate_unique_id: str, missing_test_groups: Li message=failure_message, recommendation=recommendation, reason_to_flag=self.REASON_TO_FLAG, - metadata={"altimate_unique_id": altimate_unique_id, "missing_test_groups": missing_test_groups}, + metadata={"model_unique_id": model_unique_id, "missing_test_groups": missing_test_groups}, ) def _model_has_tests_by_group(self, node_id) -> List[Dict]: diff --git a/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_tests_by_name.py b/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_tests_by_name.py index f53aa68d..3e5eb04d 100644 --- a/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_tests_by_name.py +++ b/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_tests_by_name.py @@ -45,12 +45,12 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]: ) return insights - def _build_failure_result(self, altimate_unique_id: str, missing_tests: List[Dict]) -> DBTInsightResult: + def _build_failure_result(self, model_unique_id: str, missing_tests: List[Dict]) -> DBTInsightResult: tests_str = "" for test in missing_tests: tests_str += f"Test Name: {test.get(self.TEST_NAME_STR)}, Min Count: {test.get(self.TEST_COUNT_STR)}, Actual Count: {test.get('actual_count')}\n" - failure_message = f"The model `{altimate_unique_id}` does not have enough tests:\n{tests_str}. " + failure_message = f"The model `{model_unique_id}` does not have enough tests:\n{tests_str}. " recommendation = ( "Add tests with the specified names for each model listed above. " "Having tests with specific names ensures proper validation and data integrity." @@ -62,7 +62,7 @@ def _build_failure_result(self, altimate_unique_id: str, missing_tests: List[Dic message=failure_message, recommendation=recommendation, reason_to_flag=self.REASON_TO_FLAG, - metadata={"altimate_unique_id": altimate_unique_id}, + metadata={"model_unique_id": model_unique_id}, ) def _model_has_tests_by_name(self, node_id) -> bool: diff --git a/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_tests_by_type.py b/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_tests_by_type.py index c8744f7f..a5ce95c8 100644 --- a/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_tests_by_type.py +++ b/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_tests_by_type.py @@ -46,12 +46,12 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]: ) return insights - def _build_failure_result(self, altimate_unique_id: str, missing_tests: List[Dict]) -> DBTInsightResult: + def _build_failure_result(self, model_unique_id: str, missing_tests: List[Dict]) -> DBTInsightResult: missing_test_type_str = "" for test in missing_tests: missing_test_type_str += f"Test type: {test.get(self.TEST_TYPE_STR)}, Min Count: {test.get(self.TEST_COUNT_STR)}, Actual Count: {test.get('actual_count')}\n" - failure_message = f"The model `{altimate_unique_id}` does not have enough tests for the following types:\n{missing_test_type_str}. " + failure_message = f"The model `{model_unique_id}` does not have enough tests for the following types:\n{missing_test_type_str}. " recommendation = ( "Add tests with the specified names for each model listed above. " "Having tests with specific names ensures proper validation and data integrity." @@ -63,7 +63,7 @@ def _build_failure_result(self, altimate_unique_id: str, missing_tests: List[Dic message=failure_message, recommendation=recommendation, reason_to_flag=self.REASON_TO_FLAG, - metadata={"altimate_unique_id": altimate_unique_id, "missing_tests": missing_tests}, + metadata={"model_unique_id": model_unique_id, "missing_tests": missing_tests}, ) def _model_has_tests_by_type(self, node_id) -> bool: diff --git a/src/datapilot/core/platforms/dbt/insights/checks/check_model_name_contract.py b/src/datapilot/core/platforms/dbt/insights/checks/check_model_name_contract.py index a7b0d4e7..988c5be9 100644 --- a/src/datapilot/core/platforms/dbt/insights/checks/check_model_name_contract.py +++ b/src/datapilot/core/platforms/dbt/insights/checks/check_model_name_contract.py @@ -50,7 +50,7 @@ def _build_failure_result( message=failure_message, recommendation=recommendation, reason_to_flag=self.REASON_TO_FLAG, - metadata={"altimate_unique_id": node_id, **failure}, + metadata={"model_unique_id": node_id, **failure}, ) def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]: @@ -88,12 +88,12 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]: ) return insights - def _check_model_name_contract(self, altimate_unique_id: str) -> bool: + def _check_model_name_contract(self, model_unique_id: str) -> bool: """ Check if the model name abides to the contract. """ - model_name = self.get_node(altimate_unique_id).name - model_path = self.get_node(altimate_unique_id).original_file_path + model_name = self.get_node(model_unique_id).name + model_path = self.get_node(model_unique_id).original_file_path for folder, pattern in self.patterns.items(): if is_superset_path(folder, model_path): if re.match(pattern, model_name, re.IGNORECASE) is None: diff --git a/src/datapilot/core/platforms/dbt/insights/checks/check_model_parents_and_childs.py b/src/datapilot/core/platforms/dbt/insights/checks/check_model_parents_and_childs.py index 0aec46a3..2ab27ed3 100644 --- a/src/datapilot/core/platforms/dbt/insights/checks/check_model_parents_and_childs.py +++ b/src/datapilot/core/platforms/dbt/insights/checks/check_model_parents_and_childs.py @@ -47,7 +47,7 @@ def _build_failure_result( "max_parents": self.max_parents, "min_childs": self.min_childs, "max_childs": self.max_childs, - "altimate_unique_id": node_id, + "model_unique_id": node_id, }, ) @@ -89,19 +89,19 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]: ) return insights - def _check_model_parents_and_childs(self, altimate_unique_id: str) -> Optional[str]: + def _check_model_parents_and_childs(self, model_unique_id: str) -> Optional[str]: """ Check if the model has a specific number (max/min) of parents or/and childs. """ - children = self.children_map.get(altimate_unique_id, []) - node = self.get_node(altimate_unique_id) + children = self.children_map.get(model_unique_id, []) + node = self.get_node(model_unique_id) parents = node.depends_on.nodes message = "" if len(parents) < self.min_parents or len(parents) > self.max_parents: - message += f"The model:{altimate_unique_id} doesn't have the required number of parents.\n Min parents: {self.min_parents}, Max parents: {self.max_parents}. It has f{len(parents)} parents\n" + message += f"The model:{model_unique_id} doesn't have the required number of parents.\n Min parents: {self.min_parents}, Max parents: {self.max_parents}. It has f{len(parents)} parents\n" if len(children) < self.min_childs or len(children) > self.max_childs: - message += f"The model:{altimate_unique_id} doesn't have the required number of childs.\n Min childs: {self.min_childs}, Max childs: {self.max_childs}. It has f{len(children)} childs\n" + message += f"The model:{model_unique_id} doesn't have the required number of childs.\n Min childs: {self.min_childs}, Max childs: {self.max_childs}. It has f{len(children)} childs\n" return message diff --git a/src/datapilot/core/platforms/dbt/insights/checks/check_model_parents_database.py b/src/datapilot/core/platforms/dbt/insights/checks/check_model_parents_database.py index ed35f63e..44880e8a 100644 --- a/src/datapilot/core/platforms/dbt/insights/checks/check_model_parents_database.py +++ b/src/datapilot/core/platforms/dbt/insights/checks/check_model_parents_database.py @@ -66,11 +66,11 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]: ) return insights - def _check_model_parents_database(self, altimate_unique_id: str) -> bool: + def _check_model_parents_database(self, model_unique_id: str) -> bool: """ Check if the parent models or sources are from certain database. """ - model = self.get_node(altimate_unique_id) + model = self.get_node(model_unique_id) if model.resource_type == AltimateResourceType.model: for parent in getattr(model.depends_on, "nodes", []): parent_model = self.get_node(parent) diff --git a/src/datapilot/core/platforms/dbt/insights/checks/check_model_parents_schema.py b/src/datapilot/core/platforms/dbt/insights/checks/check_model_parents_schema.py index ee7f5d6c..697f0ace 100644 --- a/src/datapilot/core/platforms/dbt/insights/checks/check_model_parents_schema.py +++ b/src/datapilot/core/platforms/dbt/insights/checks/check_model_parents_schema.py @@ -66,11 +66,11 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]: ) return insights - def _check_model_parents_schema(self, altimate_unique_id: str) -> bool: + def _check_model_parents_schema(self, model_unique_id: str) -> bool: """ Check if the parent models or sources are from certain schema. """ - model = self.get_node(altimate_unique_id) + model = self.get_node(model_unique_id) if model.resource_type == AltimateResourceType.model: for parent in getattr(model.depends_on, "nodes", []): parent_model = self.get_node(parent) diff --git a/src/datapilot/core/platforms/dbt/insights/checks/check_source_columns_have_desc.py b/src/datapilot/core/platforms/dbt/insights/checks/check_source_columns_have_desc.py index 7a3ec49f..a39cf0ea 100644 --- a/src/datapilot/core/platforms/dbt/insights/checks/check_source_columns_have_desc.py +++ b/src/datapilot/core/platforms/dbt/insights/checks/check_source_columns_have_desc.py @@ -22,11 +22,11 @@ def __init__(self, catalog_wrapper: BaseCatalogWrapper, *args, **kwargs): self.catalog = catalog_wrapper super().__init__(*args, **kwargs) - def _build_failure_result(self, altimate_unique_id: str, columns: Sequence[str]) -> DBTInsightResult: + def _build_failure_result(self, model_unique_id: str, columns: Sequence[str]) -> DBTInsightResult: """ Build failure result for the insight if a source has columns without descriptions. """ - failure_message = f"The source:{altimate_unique_id} has columns without descriptions:\n" + failure_message = f"The source:{model_unique_id} has columns without descriptions:\n" failure_message += numbered_list(columns) recommendation = "Update the source to include descriptions for all columns." @@ -36,7 +36,7 @@ def _build_failure_result(self, altimate_unique_id: str, columns: Sequence[str]) message=failure_message, recommendation=recommendation, reason_to_flag=self.REASON_TO_FLAG, - metadata={"source_unique_id": altimate_unique_id, "columns": columns}, + metadata={"source_unique_id": model_unique_id, "columns": columns}, ) def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]: diff --git a/src/datapilot/core/platforms/dbt/insights/checks/check_source_has_labels_keys.py b/src/datapilot/core/platforms/dbt/insights/checks/check_source_has_labels_keys.py index 7755c1b1..7caaf1ff 100644 --- a/src/datapilot/core/platforms/dbt/insights/checks/check_source_has_labels_keys.py +++ b/src/datapilot/core/platforms/dbt/insights/checks/check_source_has_labels_keys.py @@ -49,26 +49,26 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]: ) return insights - def _build_failure_result(self, altimate_unique_id: str, missing_keys: Sequence[str], extra_keys: Sequence[str]) -> DBTInsightResult: + def _build_failure_result(self, model_unique_id: str, missing_keys: Sequence[str], extra_keys: Sequence[str]) -> DBTInsightResult: failure_message = "" if missing_keys: failure_message += ( - f"The model `{altimate_unique_id}` is missing the following labels keys: {missing_keys}. " + f"The model `{model_unique_id}` is missing the following labels keys: {missing_keys}. " "Ensure that the model has the required labels keys." ) if extra_keys: failure_message += ( - f"The model `{altimate_unique_id}` has the following extra labels keys: {extra_keys}. " + f"The model `{model_unique_id}` has the following extra labels keys: {extra_keys}. " "Ensure that the model does not include any extra labels keys." ) recommendation = ( - "Add the following labels keys to the model `{altimate_unique_id}`: {missing_keys}. " + "Add the following labels keys to the model `{model_unique_id}`: {missing_keys}. " "Ensuring that the model has the required labels keys helps in maintaining metadata consistency and understanding." ) return DBTInsightResult( - failure_message=failure_message.format(altimate_unique_id=altimate_unique_id, missing_keys=numbered_list(missing_keys)), - recommendation=recommendation.format(altimate_unique_id=altimate_unique_id, missing_keys=numbered_list(missing_keys)), - metadata={"altimate_unique_id": altimate_unique_id, "missing_keys": missing_keys}, + failure_message=failure_message.format(model_unique_id=model_unique_id, missing_keys=numbered_list(missing_keys)), + recommendation=recommendation.format(model_unique_id=model_unique_id, missing_keys=numbered_list(missing_keys)), + metadata={"model_unique_id": model_unique_id, "missing_keys": missing_keys}, ) def _check_labels_keys(self, node_id) -> Tuple[int, Set[str]]: diff --git a/src/datapilot/core/platforms/dbt/insights/dbt_test/missing_primary_key_tests.py b/src/datapilot/core/platforms/dbt/insights/dbt_test/missing_primary_key_tests.py index 72a23e98..e64a4726 100644 --- a/src/datapilot/core/platforms/dbt/insights/dbt_test/missing_primary_key_tests.py +++ b/src/datapilot/core/platforms/dbt/insights/dbt_test/missing_primary_key_tests.py @@ -29,7 +29,7 @@ class MissingPrimaryKeyTests(DBTTestInsight): "the risk of data integrity issues, affecting project reliability and scalability." ) FAILURE_MESSAGE = ( - "dbt model `{altimate_unique_id}` does not have a primary key test. " "This omission may lead to data integrity challenges." + "dbt model `{model_unique_id}` does not have a primary key test. " "This omission may lead to data integrity challenges." ) RECOMMENDATION = ( "To address this, apply a uniqueness test and a not-null test to the column representing the model's grain. " @@ -38,22 +38,22 @@ class MissingPrimaryKeyTests(DBTTestInsight): " and unique_combination_of_columns test." ) - def _build_failure_result(self, altimate_unique_id: str) -> DBTInsightResult: + def _build_failure_result(self, model_unique_id: str) -> DBTInsightResult: """ Constructs a failure result for a given model. - :param altimate_unique_id: Unique ID of the model being evaluated. + :param model_unique_id: Unique ID of the model being evaluated. :return: An instance of DBTInsightResult containing failure details. """ - self.logger.debug(f"Building failure result for model {altimate_unique_id}") - failure = self.FAILURE_MESSAGE.format(altimate_unique_id=altimate_unique_id) + self.logger.debug(f"Building failure result for model {model_unique_id}") + failure = self.FAILURE_MESSAGE.format(model_unique_id=model_unique_id) return DBTInsightResult( type=self.TYPE, name=self.NAME, message=failure, recommendation=self.RECOMMENDATION, reason_to_flag=self.REASON_TO_FLAG, - metadata={"altimate_unique_id": altimate_unique_id}, + metadata={"model_unique_id": model_unique_id}, ) def _has_primary_key_test(self, column_tests: Optional[Dict[str, List]]) -> bool: diff --git a/src/datapilot/core/platforms/dbt/insights/governance/documentation_on_stale_columns.py b/src/datapilot/core/platforms/dbt/insights/governance/documentation_on_stale_columns.py index bf985304..e1f1eb01 100644 --- a/src/datapilot/core/platforms/dbt/insights/governance/documentation_on_stale_columns.py +++ b/src/datapilot/core/platforms/dbt/insights/governance/documentation_on_stale_columns.py @@ -27,11 +27,11 @@ class DBTDocumentationStaleColumns(DBTGovernanceInsight): "This discrepancy can cause confusion and mislead users of the dbt project." ) FAILURE_MESSAGE = ( - "The following documented columns are no longer present in the model `{altimate_unique_id}`:\n{stale_columns}. " + "The following documented columns are no longer present in the model `{model_unique_id}`:\n{stale_columns}. " "This inconsistency can lead to confusion regarding the model's current structure." ) RECOMMENDATION = ( - "Review and update the documentation for model `{altimate_unique_id}`. Remove documentation entries for columns " + "Review and update the documentation for model `{model_unique_id}`. Remove documentation entries for columns " "that are no longer present to maintain clarity and accuracy in the project documentation." ) FILES_REQUIRED: ClassVar = ["Manifest", "Catalog"] @@ -40,22 +40,22 @@ def __init__(self, catalog_wrapper: BaseCatalogWrapper, *args, **kwargs): self.catalog = catalog_wrapper super().__init__(*args, **kwargs) - def _build_failure_result(self, altimate_unique_id: str, columns: List[str]) -> DBTInsightResult: + def _build_failure_result(self, model_unique_id: str, columns: List[str]) -> DBTInsightResult: """ Build failure result for the insight if a model is a root model with 0 direct parents. - :param altimate_unique_id: Unique ID of the current model being evaluated. + :param model_unique_id: Unique ID of the current model being evaluated. :param columns: List of columns that are documented but no longer present in the model. :return: An instance of InsightResult containing failure message and recommendation. """ - self.logger.debug(f"Building failure result for model {altimate_unique_id} with stale columns {columns}") + self.logger.debug(f"Building failure result for model {model_unique_id} with stale columns {columns}") failure = self.FAILURE_MESSAGE.format( stale_columns=numbered_list(columns), - altimate_unique_id=altimate_unique_id, + model_unique_id=model_unique_id, ) - recommendation = self.RECOMMENDATION.format(altimate_unique_id=altimate_unique_id) + recommendation = self.RECOMMENDATION.format(model_unique_id=model_unique_id) return DBTInsightResult( type=self.TYPE, @@ -63,7 +63,7 @@ def _build_failure_result(self, altimate_unique_id: str, columns: List[str]) -> message=failure, recommendation=recommendation, reason_to_flag=self.REASON_TO_FLAG, - metadata={"stale_columns": columns, "altimate_unique_id": altimate_unique_id}, + metadata={"stale_columns": columns, "model_unique_id": model_unique_id}, ) def _get_columns_documented(self, node_id) -> List[str]: diff --git a/src/datapilot/core/platforms/dbt/insights/governance/public_models_without_contracts.py b/src/datapilot/core/platforms/dbt/insights/governance/public_models_without_contracts.py index caa4b952..282284fe 100644 --- a/src/datapilot/core/platforms/dbt/insights/governance/public_models_without_contracts.py +++ b/src/datapilot/core/platforms/dbt/insights/governance/public_models_without_contracts.py @@ -22,31 +22,31 @@ class DBTPublicModelWithoutContracts(DBTGovernanceInsight): "in data consumption." ) FAILURE_MESSAGE = ( - "Model `{altimate_unique_id}` is marked as public but does not have a contract. " + "Model `{model_unique_id}` is marked as public but does not have a contract. " "This can lead to ambiguity regarding data types and columns, impacting downstream consumers." ) RECOMMENDATION = ( - "Enhance the model `{altimate_unique_id}` by adding clear contract entries for columns along " + "Enhance the model `{model_unique_id}` by adding clear contract entries for columns along " "with their data types. Contracts provide essential documentation and guarantees for downstream consumers." ) def _build_failure_result( self, - altimate_unique_id: str, + model_unique_id: str, ) -> DBTInsightResult: """ Build failure result for the insight if a model is a root model with 0 direct parents. - :param altimate_unique_id: Unique ID of the current model being evaluated. + :param model_unique_id: Unique ID of the current model being evaluated. :return: An instance of InsightResult containing failure message and recommendation. """ - self.logger.debug(f"Building failure result model {altimate_unique_id} is public but not documented.") + self.logger.debug(f"Building failure result model {model_unique_id} is public but not documented.") failure = self.FAILURE_MESSAGE.format( - altimate_unique_id=altimate_unique_id, + model_unique_id=model_unique_id, ) recommendation = self.RECOMMENDATION.format( - altimate_unique_id=altimate_unique_id, + model_unique_id=model_unique_id, ) return DBTInsightResult( @@ -56,7 +56,7 @@ def _build_failure_result( recommendation=recommendation, reason_to_flag=self.REASON_TO_FLAG, metadata={ - "model": altimate_unique_id, + "model": model_unique_id, }, ) diff --git a/src/datapilot/core/platforms/dbt/insights/governance/undocumented_columns.py b/src/datapilot/core/platforms/dbt/insights/governance/undocumented_columns.py index 174eb1cf..f6bcae1f 100644 --- a/src/datapilot/core/platforms/dbt/insights/governance/undocumented_columns.py +++ b/src/datapilot/core/platforms/dbt/insights/governance/undocumented_columns.py @@ -27,11 +27,11 @@ class DBTMissingDocumentation(DBTGovernanceInsight): "modeling and analysis. It's important to document data structures comprehensively." ) FAILURE_MESSAGE = ( - "The following columns in the model `{altimate_unique_id}` are missing documentation:\n{columns}. " + "The following columns in the model `{model_unique_id}` are missing documentation:\n{columns}. " "Lack of documentation can impede understanding and usage of the model." ) RECOMMENDATION = ( - "Enhance the documentation for the columns listed above in the model `{altimate_unique_id}`. " + "Enhance the documentation for the columns listed above in the model `{model_unique_id}`. " "Documentation provides valuable context and aids in data understanding and collaboration." ) FILES_REQUIRED: ClassVar = ["Manifest", "Catalog"] @@ -42,29 +42,29 @@ def __init__(self, catalog_wrapper: BaseCatalogWrapper, *args, **kwargs): def _build_failure_result( self, - altimate_unique_id: str, + model_unique_id: str, model_description_is_missing: bool, columns: List[str], ) -> DBTInsightResult: """ Build failure result for the insight if a model is a root model with 0 direct parents. - :param altimate_unique_id: Unique ID of the current model being evaluated. + :param model_unique_id: Unique ID of the current model being evaluated. :param columns: List of columns that are documented but no longer present in the model. :return: An instance of InsightResult containing failure message and recommendation. """ - self.logger.debug(f"Building failure result for model {altimate_unique_id} with stale columns {columns}") + self.logger.debug(f"Building failure result for model {model_unique_id} with stale columns {columns}") failure_message = "" if model_description_is_missing: - failure_message += f"The model {altimate_unique_id} is missing a description.\n" + failure_message += f"The model {model_unique_id} is missing a description.\n" if columns: failure_message += self.FAILURE_MESSAGE.format( columns=numbered_list(columns), - altimate_unique_id=altimate_unique_id, + model_unique_id=model_unique_id, ) - recommendation = self.RECOMMENDATION.format(altimate_unique_id=altimate_unique_id) + recommendation = self.RECOMMENDATION.format(model_unique_id=model_unique_id) return DBTInsightResult( type=self.TYPE, @@ -74,7 +74,7 @@ def _build_failure_result( reason_to_flag=self.REASON_TO_FLAG, metadata={ "columns": columns, - "altimate_unique_id": altimate_unique_id, + "model_unique_id": model_unique_id, "model_description_is_missing": model_description_is_missing, }, ) diff --git a/src/datapilot/core/platforms/dbt/insights/governance/undocumented_public_models.py b/src/datapilot/core/platforms/dbt/insights/governance/undocumented_public_models.py index 95a541c2..7c4d49c5 100644 --- a/src/datapilot/core/platforms/dbt/insights/governance/undocumented_public_models.py +++ b/src/datapilot/core/platforms/dbt/insights/governance/undocumented_public_models.py @@ -25,7 +25,7 @@ class DBTUndocumentedPublicModels(DBTGovernanceInsight): "it's essential to document these models comprehensively." ) FAILURE_MESSAGE = ( - "Model `{altimate_unique_id}` is marked as public but is not documented. " + "Model `{model_unique_id}` is marked as public but is not documented. " "Lack of documentation can lead to confusion for data consumers." ) RECOMMENDATION = ( @@ -35,22 +35,22 @@ class DBTUndocumentedPublicModels(DBTGovernanceInsight): def _build_failure_result( self, - altimate_unique_id: str, + model_unique_id: str, model_description_is_missing: bool, columns: Optional[List[str]] = None, ) -> DBTInsightResult: """ Build failure result for the insight if a model is a root model with 0 direct parents. - :param altimate_unique_id: Unique ID of the current model being evaluated. + :param model_unique_id: Unique ID of the current model being evaluated. :param model_description_is_missing: Whether the model description is missing. :param columns: List of columns that are missing documentation. :return: An instance of InsightResult containing failure message and recommendation. """ - self.logger.debug(f"Building failure result model {altimate_unique_id} is public but not documented.") + self.logger.debug(f"Building failure result model {model_unique_id} is public but not documented.") failure = self.FAILURE_MESSAGE.format( - altimate_unique_id=altimate_unique_id, + model_unique_id=model_unique_id, ) failure += "Missing Model documentation." if model_description_is_missing else "" @@ -63,7 +63,7 @@ def _build_failure_result( recommendation=self.RECOMMENDATION, reason_to_flag=self.REASON_TO_FLAG, metadata={ - "model": altimate_unique_id, + "model": model_unique_id, "columns_without_documentation": columns, "model_description_missing": model_description_is_missing, }, diff --git a/src/datapilot/core/platforms/dbt/insights/modelling/direct_join_to_source.py b/src/datapilot/core/platforms/dbt/insights/modelling/direct_join_to_source.py index 9f5cb438..0da5569c 100644 --- a/src/datapilot/core/platforms/dbt/insights/modelling/direct_join_to_source.py +++ b/src/datapilot/core/platforms/dbt/insights/modelling/direct_join_to_source.py @@ -28,30 +28,30 @@ class DBTDirectJoinSource(DBTModellingInsight): "Direct source-model joins bypass the staging layer, leading to potential inconsistencies in data handling." ) FAILURE_MESSAGE = ( - "Model `{current_altimate_unique_id}` has direct joins to both sources and other models. " + "Model `{current_model_unique_id}` has direct joins to both sources and other models. " "\n### Detected Sources\n{sources}\n\n### Connected Models \n{models}" ) RECOMMENDATION = ( - "Create a dedicated staging model for the source(s) and modify `{current_altimate_unique_id}` " + "Create a dedicated staging model for the source(s) and modify `{current_model_unique_id}` " "to depend on this staging model. This ensures consistent initial data processing steps." ) - def _build_failure_result(self, current_altimate_unique_id: str, dependencies: Dict) -> DBTInsightResult: + def _build_failure_result(self, current_model_unique_id: str, dependencies: Dict) -> DBTInsightResult: """ Build failure result for the insight if a model is directly joining to a source and other models. - :param current_altimate_unique_id: Unique ID of the current model being evaluated. + :param current_model_unique_id: Unique ID of the current model being evaluated. :param dependencies: A dictionary of dependencies categorized as 'source' and 'model'. :return: An instance of InsightResult containing failure message and recommendation and metadata. """ - self.logger.debug(f"Found multiple sources and models for {current_altimate_unique_id}") + self.logger.debug(f"Found multiple sources and models for {current_model_unique_id}") failure = self.FAILURE_MESSAGE.format( - current_altimate_unique_id=current_altimate_unique_id, + current_model_unique_id=current_model_unique_id, sources=numbered_list(dependencies["source"]), models=numbered_list(dependencies["model"]), ) - recommendation = self.RECOMMENDATION.format(current_altimate_unique_id=current_altimate_unique_id) + recommendation = self.RECOMMENDATION.format(current_model_unique_id=current_model_unique_id) return DBTInsightResult( type=self.TYPE, @@ -60,7 +60,7 @@ def _build_failure_result(self, current_altimate_unique_id: str, dependencies: D recommendation=recommendation, reason_to_flag=self.REASON_TO_FLAG, metadata={ - "model": current_altimate_unique_id, + "model": current_model_unique_id, "dependencies": dependencies, }, ) diff --git a/src/datapilot/core/platforms/dbt/insights/modelling/downstream_models_dependent_on_source.py b/src/datapilot/core/platforms/dbt/insights/modelling/downstream_models_dependent_on_source.py index 45514112..1025fe5a 100644 --- a/src/datapilot/core/platforms/dbt/insights/modelling/downstream_models_dependent_on_source.py +++ b/src/datapilot/core/platforms/dbt/insights/modelling/downstream_models_dependent_on_source.py @@ -28,34 +28,34 @@ class DBTDownstreamModelsDependentOnSource(DBTModellingInsight): "while providing a consistent format for downstream consumption." ) FAILURE_MESSAGE = ( - "Downstream model `{current_altimate_unique_id}` of type {model_type} is directly dependent on a source nodes." + "Downstream model `{current_model_unique_id}` of type {model_type} is directly dependent on a source nodes." "Direct source dependencies bypass the critical staging layer, leading to potential data consistency issues." " Source dependencies: {source_dependencies}" ) RECOMMENDATION = ( "Introduce or utilize an existing staging model for the source node involved. Refactor the downstream model " - "`{current_altimate_unique_id}` to select from this staging layer, ensuring a proper abstraction layer between " + "`{current_model_unique_id}` to select from this staging layer, ensuring a proper abstraction layer between " "raw data and downstream data artifacts." ) MODEL_TYPES: ClassVar[List[str]] = [INTERMEDIATE, MART] def _build_failure_result( self, - current_altimate_unique_id: str, + current_model_unique_id: str, source_dependencies: List[str], model_type: str, ) -> DBTInsightResult: """ Build failure result for the insight if a downstream model depends directly on a source node. - :param current_altimate_unique_id: Unique ID of the current model being evaluated. + :param current_model_unique_id: Unique ID of the current model being evaluated. :param source_dependencies: List of source dependencies for the current model. :return: An instance of InsightResult containing failure message and recommendation. """ - self.logger.debug(f"Building failure result for model {current_altimate_unique_id} with direct source dependencies") + self.logger.debug(f"Building failure result for model {current_model_unique_id} with direct source dependencies") failure = self.FAILURE_MESSAGE.format( - current_altimate_unique_id=current_altimate_unique_id, + current_model_unique_id=current_model_unique_id, model_type=model_type, source_dependencies=numbered_list(source_dependencies), ) @@ -64,10 +64,10 @@ def _build_failure_result( type=self.TYPE, name=self.NAME, message=failure, - recommendation=self.RECOMMENDATION.format(current_altimate_unique_id=current_altimate_unique_id), + recommendation=self.RECOMMENDATION.format(current_model_unique_id=current_model_unique_id), reason_to_flag=self.REASON_TO_FLAG, metadata={ - "model": current_altimate_unique_id, + "model": current_model_unique_id, "source_dependencies": source_dependencies, "model_type": model_type, }, diff --git a/src/datapilot/core/platforms/dbt/insights/modelling/hard_coded_references.py b/src/datapilot/core/platforms/dbt/insights/modelling/hard_coded_references.py index 55cdd239..068f0f42 100644 --- a/src/datapilot/core/platforms/dbt/insights/modelling/hard_coded_references.py +++ b/src/datapilot/core/platforms/dbt/insights/modelling/hard_coded_references.py @@ -24,27 +24,27 @@ class DBTHardCodedReferences(DBTModellingInsight): ) SOURCE_FANOUT_THRESHOLD = 1 # Default threshold, can be overridden as needed FAILURE_MESSAGE = ( - "Model `{altimate_unique_id}` contains hard-coded references, which may obscure data lineage. " + "Model `{model_unique_id}` contains hard-coded references, which may obscure data lineage. " "Detected hard-coded references: \n{hard_coded_references}" ) RECOMMENDATION = ( - "Replace hard-coded references in `{altimate_unique_id}` with dbt sources or model references to " + "Replace hard-coded references in `{model_unique_id}` with dbt sources or model references to " "improve clarity and maintainability of data lineage." ) - def _build_failure_result(self, altimate_unique_id: str, hard_coded_references: List[str]) -> DBTInsightResult: + def _build_failure_result(self, model_unique_id: str, hard_coded_references: List[str]) -> DBTInsightResult: failure_message = self.FAILURE_MESSAGE.format( - altimate_unique_id=altimate_unique_id, + model_unique_id=model_unique_id, hard_coded_references=numbered_list(hard_coded_references), ) return DBTInsightResult( name=self.NAME, type=self.TYPE, message=failure_message, - recommendation=self.RECOMMENDATION.format(altimate_unique_id=altimate_unique_id), + recommendation=self.RECOMMENDATION.format(model_unique_id=model_unique_id), reason_to_flag=self.REASON_TO_FLAG, metadata={ - "model": altimate_unique_id, + "model": model_unique_id, "hard_coded_references": hard_coded_references, }, ) @@ -63,7 +63,7 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]: hard_coded_references = get_hard_coded_references(raw_code) if hard_coded_references: insight_result = self._build_failure_result( - altimate_unique_id=node.unique_id, + model_unique_id=node.unique_id, hard_coded_references=hard_coded_references, ) insights.append( diff --git a/src/datapilot/core/platforms/dbt/insights/modelling/model_fanout.py b/src/datapilot/core/platforms/dbt/insights/modelling/model_fanout.py index 49828e45..cf581ca8 100644 --- a/src/datapilot/core/platforms/dbt/insights/modelling/model_fanout.py +++ b/src/datapilot/core/platforms/dbt/insights/modelling/model_fanout.py @@ -22,11 +22,11 @@ class DBTModelFanout(DBTModellingInsight): ) FANOUT_THRESHOLD = 3 # Default threshold, can be overridden as needed FAILURE_MESSAGE = ( - "Model `{parent_altimate_unique_id}` has `{leaf_children}` leaf children, " + "Model `{parent_model_unique_id}` has `{leaf_children}` leaf children, " "exceeding the fanout threshold of `{fanout_threshold}`. This level of fanout may lead to increased complexity." ) RECOMMENDATION = ( - "Consider reviewing and restructuring `{parent_altimate_unique_id}` to simplify its dependencies. " + "Consider reviewing and restructuring `{parent_model_unique_id}` to simplify its dependencies. " "Reducing the number of leaf children can lead to a more streamlined and maintainable data pipeline." ) @@ -34,20 +34,20 @@ class DBTModelFanout(DBTModellingInsight): def _build_failure_result( self, - parent_altimate_unique_id: str, + parent_model_unique_id: str, leaf_children: List[str], fanout_threshold: int, ) -> DBTInsightResult: # Logic to build the failure result - self.logger.debug(f"Found {len(leaf_children)} leaf children for {parent_altimate_unique_id}") + self.logger.debug(f"Found {len(leaf_children)} leaf children for {parent_model_unique_id}") failure_message = self.FAILURE_MESSAGE.format( - parent_altimate_unique_id=parent_altimate_unique_id, + parent_model_unique_id=parent_model_unique_id, leaf_children=len(leaf_children), fanout_threshold=fanout_threshold, ) recommendation = self.RECOMMENDATION.format( - parent_altimate_unique_id=parent_altimate_unique_id, + parent_model_unique_id=parent_model_unique_id, ) return DBTInsightResult( @@ -57,7 +57,7 @@ def _build_failure_result( recommendation=recommendation, reason_to_flag=self.REASON_TO_FLAG, metadata={ - "model": parent_altimate_unique_id, + "model": parent_model_unique_id, "leaf_children_count": len(leaf_children), "leaf_children": leaf_children, }, diff --git a/src/datapilot/core/platforms/dbt/insights/modelling/root_model.py b/src/datapilot/core/platforms/dbt/insights/modelling/root_model.py index 2ef6582d..b6c4d04d 100644 --- a/src/datapilot/core/platforms/dbt/insights/modelling/root_model.py +++ b/src/datapilot/core/platforms/dbt/insights/modelling/root_model.py @@ -22,25 +22,25 @@ class DBTRootModel(DBTModellingInsight): " the overall data model." ) FAILURE_MESSAGE = ( - "Model `{current_altimate_unique_id}` is identified as a root model with no direct parents. " + "Model `{current_model_unique_id}` is identified as a root model with no direct parents. " "This can hinder traceability and clarity in the data model." ) RECOMMENDATION = ( - "Ensure that model `{current_altimate_unique_id}` is appropriately linked to a source or another model " + "Ensure that model `{current_model_unique_id}` is appropriately linked to a source or another model " "within the dbt project. This linkage is crucial for maintaining clear data lineage and project coherence." ) - def _build_failure_result(self, current_altimate_unique_id: str) -> DBTInsightResult: + def _build_failure_result(self, current_model_unique_id: str) -> DBTInsightResult: """ Build failure result for the insight if a model is a root model with 0 direct parents. - :param current_altimate_unique_id: Unique ID of the current model being evaluated. + :param current_model_unique_id: Unique ID of the current model being evaluated. :return: An instance of InsightResult containing failure message and recommendation. """ - self.logger.debug(f"Building failure result for root model {current_altimate_unique_id}") + self.logger.debug(f"Building failure result for root model {current_model_unique_id}") - failure = self.FAILURE_MESSAGE.format(current_altimate_unique_id=current_altimate_unique_id) - recommendation = self.RECOMMENDATION.format(current_altimate_unique_id=current_altimate_unique_id) + failure = self.FAILURE_MESSAGE.format(current_model_unique_id=current_model_unique_id) + recommendation = self.RECOMMENDATION.format(current_model_unique_id=current_model_unique_id) return DBTInsightResult( type=self.TYPE, @@ -48,7 +48,7 @@ def _build_failure_result(self, current_altimate_unique_id: str) -> DBTInsightRe message=failure, recommendation=recommendation, reason_to_flag=self.REASON_TO_FLAG, - metadata={"model": current_altimate_unique_id}, + metadata={"model": current_model_unique_id}, ) def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]: diff --git a/src/datapilot/core/platforms/dbt/insights/modelling/staging_model_dependent_on_downstream_models.py b/src/datapilot/core/platforms/dbt/insights/modelling/staging_model_dependent_on_downstream_models.py index f31a1de0..579cc449 100644 --- a/src/datapilot/core/platforms/dbt/insights/modelling/staging_model_dependent_on_downstream_models.py +++ b/src/datapilot/core/platforms/dbt/insights/modelling/staging_model_dependent_on_downstream_models.py @@ -28,22 +28,22 @@ class DBTStagingModelsDependentOnDownstreamModels(DBTModellingInsight): "Dependencies in the wrong direction can lead to complications in data processing and lineage tracing." ) FAILURE_MESSAGE = ( - "Staging model `{current_altimate_unique_id}` has dependencies on downstream models, " + "Staging model `{current_model_unique_id}` has dependencies on downstream models, " "which is against best practices: \n{downstream_dependencies}" ) RECOMMENDATION = ( - "Refactor the staging model `{current_altimate_unique_id}` to ensure it depends on source or raw data models. " + "Refactor the staging model `{current_model_unique_id}` to ensure it depends on source or raw data models. " "This will align the model with best practices, enhancing data flow clarity and lineage tracing." ) DOWNSTREAM_MODEL_TYPES_STR = "downstream_model_types" DOWNSTREAM_MODEL_TYPES: ClassVar[List[str]] = [MART, INTERMEDIATE] - def _build_failure_result(self, current_altimate_unique_id: str, downstream_dependencies: List[str]) -> DBTInsightResult: + def _build_failure_result(self, current_model_unique_id: str, downstream_dependencies: List[str]) -> DBTInsightResult: failure = self.FAILURE_MESSAGE.format( - current_altimate_unique_id=current_altimate_unique_id, + current_model_unique_id=current_model_unique_id, downstream_dependencies=numbered_list(downstream_dependencies), ) - recommendation = self.RECOMMENDATION.format(current_altimate_unique_id=current_altimate_unique_id) + recommendation = self.RECOMMENDATION.format(current_model_unique_id=current_model_unique_id) return DBTInsightResult( type=self.TYPE, @@ -52,7 +52,7 @@ def _build_failure_result(self, current_altimate_unique_id: str, downstream_depe recommendation=recommendation, reason_to_flag=self.REASON_TO_FLAG, metadata={ - "model": current_altimate_unique_id, + "model": current_model_unique_id, "downstream_dependencies": downstream_dependencies, }, ) diff --git a/src/datapilot/core/platforms/dbt/insights/modelling/staging_model_dependent_on_staging_models.py b/src/datapilot/core/platforms/dbt/insights/modelling/staging_model_dependent_on_staging_models.py index fe091357..abe555bf 100644 --- a/src/datapilot/core/platforms/dbt/insights/modelling/staging_model_dependent_on_staging_models.py +++ b/src/datapilot/core/platforms/dbt/insights/modelling/staging_model_dependent_on_staging_models.py @@ -24,20 +24,20 @@ class DBTStagingModelsDependentOnStagingModels(DBTModellingInsight): "Dependencies among staging models can lead to complicated data flows and hinder data lineage tracking." ) FAILURE_MESSAGE = ( - "Staging model `{current_altimate_unique_id}` has dependencies on other staging models, " + "Staging model `{current_model_unique_id}` has dependencies on other staging models, " "which is against best practices: \n{downstream_dependencies}" ) RECOMMENDATION = ( - "Refactor staging model `{current_altimate_unique_id}` to ensure it depends on source or raw data models, " + "Refactor staging model `{current_model_unique_id}` to ensure it depends on source or raw data models, " "not on other staging models. This realignment with best practices promotes clear and effective data flow." ) - def _build_failure_result(self, current_altimate_unique_id: str, downstream_dependencies: List[str]) -> DBTInsightResult: + def _build_failure_result(self, current_model_unique_id: str, downstream_dependencies: List[str]) -> DBTInsightResult: failure = self.FAILURE_MESSAGE.format( - current_altimate_unique_id=current_altimate_unique_id, + current_model_unique_id=current_model_unique_id, downstream_dependencies=numbered_list(downstream_dependencies), ) - recommendation = self.RECOMMENDATION.format(current_altimate_unique_id=current_altimate_unique_id) + recommendation = self.RECOMMENDATION.format(current_model_unique_id=current_model_unique_id) return DBTInsightResult( type=self.TYPE, @@ -46,7 +46,7 @@ def _build_failure_result(self, current_altimate_unique_id: str, downstream_depe recommendation=recommendation, reason_to_flag=self.REASON_TO_FLAG, metadata={ - "model": current_altimate_unique_id, + "model": current_model_unique_id, "downstream_dependencies": downstream_dependencies, }, ) diff --git a/src/datapilot/core/platforms/dbt/insights/structure/model_directories_structure.py b/src/datapilot/core/platforms/dbt/insights/structure/model_directories_structure.py index 972547c5..d886c83c 100644 --- a/src/datapilot/core/platforms/dbt/insights/structure/model_directories_structure.py +++ b/src/datapilot/core/platforms/dbt/insights/structure/model_directories_structure.py @@ -26,19 +26,19 @@ class DBTModelDirectoryStructure(DBTStructureInsight): "discoverability, and complicate maintenance and scaling of the dbt project." ) FAILURE_MESSAGE = ( - "Incorrect Directory Placement Detected: The model `{altimate_unique_id}` is incorrectly " + "Incorrect Directory Placement Detected: The model `{model_unique_id}` is incorrectly " "placed in the current directory. As a `{model_type}` model, it should be located in " "the `{convention}` directory." ) RECOMMENDATION = ( - "To resolve this issue, please move the model `{altimate_unique_id}` to the `{convention}` " + "To resolve this issue, please move the model `{model_unique_id}` to the `{convention}` " "directory. This change will align the model's location with the established directory " "structure, improving organization and ease of access in your dbt project." ) - def _build_failure_result(self, altimate_unique_id: str, model_type: str, convention: Optional[str]) -> DBTInsightResult: + def _build_failure_result(self, model_unique_id: str, model_type: str, convention: Optional[str]) -> DBTInsightResult: failure_message = self.FAILURE_MESSAGE.format( - altimate_unique_id=altimate_unique_id, + model_unique_id=model_unique_id, model_type=model_type, convention=convention, ) @@ -46,10 +46,10 @@ def _build_failure_result(self, altimate_unique_id: str, model_type: str, conven name=self.NAME, type=self.TYPE, message=failure_message, - recommendation=self.RECOMMENDATION.format(altimate_unique_id=altimate_unique_id, convention=convention), + recommendation=self.RECOMMENDATION.format(model_unique_id=model_unique_id, convention=convention), reason_to_flag=self.REASON_TO_FLAG, metadata={ - "model": altimate_unique_id, + "model": model_unique_id, "model_type": model_type, "convention": convention, }, @@ -82,7 +82,7 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]: path=node.path, original_file_path=node.original_file_path, insight=self._build_failure_result( - altimate_unique_id=node.unique_id, + model_unique_id=node.unique_id, model_type=model_type, convention=message, ), diff --git a/src/datapilot/core/platforms/dbt/insights/structure/model_naming_conventions.py b/src/datapilot/core/platforms/dbt/insights/structure/model_naming_conventions.py index bd83310e..3f2f1094 100644 --- a/src/datapilot/core/platforms/dbt/insights/structure/model_naming_conventions.py +++ b/src/datapilot/core/platforms/dbt/insights/structure/model_naming_conventions.py @@ -27,23 +27,23 @@ class DBTModelNamingConvention(DBTStructureInsight): "and effective data management. This rule flags models that deviate from established naming standards." ) FAILURE_MESSAGE = ( - "Naming Convention Violation Detected: The model `{altimate_unique_id}` does not comply with the " + "Naming Convention Violation Detected: The model `{model_unique_id}` does not comply with the " "established naming convention. It is identified as a `{model_type}` model, but its name does not " "reflect the required prefix or convention `{convention}`. Please update the model name to align " "with the naming standards." ) - RECOMMENDATION = "Please rename the model `{altimate_unique_id}` to follow the appropriate naming convention. " + RECOMMENDATION = "Please rename the model `{model_unique_id}` to follow the appropriate naming convention. " - def _build_failure_result(self, altimate_unique_id: str, model_type: str, convention: Optional[str]) -> DBTInsightResult: + def _build_failure_result(self, model_unique_id: str, model_type: str, convention: Optional[str]) -> DBTInsightResult: if model_type != OTHER: failure_message = self.FAILURE_MESSAGE.format( - altimate_unique_id=altimate_unique_id, + model_unique_id=model_unique_id, model_type=model_type, convention=convention, ) else: failure_message = ( - f"The model `{altimate_unique_id}` was not classified as any of the known model types. " + f"The model `{model_unique_id}` was not classified as any of the known model types. " "The naming conventions for it may not be appropriate" ) @@ -51,10 +51,10 @@ def _build_failure_result(self, altimate_unique_id: str, model_type: str, conven name=self.NAME, type=self.TYPE, message=failure_message, - recommendation=self.RECOMMENDATION.format(altimate_unique_id=altimate_unique_id), + recommendation=self.RECOMMENDATION.format(model_unique_id=model_unique_id), reason_to_flag=self.REASON_TO_FLAG, metadata={ - "model": altimate_unique_id, + "model": model_unique_id, "model_type": model_type, "convention": convention, }, diff --git a/src/datapilot/core/platforms/dbt/insights/structure/source_directories_structure.py b/src/datapilot/core/platforms/dbt/insights/structure/source_directories_structure.py index 35b353a0..fa3ef85b 100644 --- a/src/datapilot/core/platforms/dbt/insights/structure/source_directories_structure.py +++ b/src/datapilot/core/platforms/dbt/insights/structure/source_directories_structure.py @@ -33,18 +33,18 @@ class DBTSourceDirectoryStructure(DBTStructureInsight): "practices for organizing source files in dbt projects." ) - def _build_failure_result(self, altimate_unique_id: str, convention: Optional[str]) -> DBTInsightResult: + def _build_failure_result(self, model_unique_id: str, convention: Optional[str]) -> DBTInsightResult: failure_message = self.FAILURE_MESSAGE.format( - source_id=altimate_unique_id, + source_id=model_unique_id, ) return DBTInsightResult( name=self.NAME, type=self.TYPE, message=failure_message, - recommendation=self.RECOMMENDATION.format(source_id=altimate_unique_id, convention=convention), + recommendation=self.RECOMMENDATION.format(source_id=model_unique_id, convention=convention), reason_to_flag=self.REASON_TO_FLAG, metadata={ - "source_id": altimate_unique_id, + "source_id": model_unique_id, "convention": convention, }, ) @@ -63,7 +63,7 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]: ) if not valid_convention: insight = self._build_failure_result( - altimate_unique_id=source_id, + model_unique_id=source_id, convention=expected_directory, ) insights.append( diff --git a/src/datapilot/core/platforms/dbt/insights/structure/test_directory_structure.py b/src/datapilot/core/platforms/dbt/insights/structure/test_directory_structure.py index ecf3d35e..6ca7f243 100644 --- a/src/datapilot/core/platforms/dbt/insights/structure/test_directory_structure.py +++ b/src/datapilot/core/platforms/dbt/insights/structure/test_directory_structure.py @@ -22,28 +22,28 @@ class DBTTestDirectoryStructure(DBTStructureInsight): "and updating tests in parallel with model changes." ) FAILURE_MESSAGE = ( - "Incorrect Test Placement Detected: The test `{altimate_unique_id}` is not in the correct directory. " + "Incorrect Test Placement Detected: The test `{model_unique_id}` is not in the correct directory. " "For consistent project structure and easy maintenance, it should be placed in the same directory as " "its corresponding model." ) RECOMMENDATION = ( - "To rectify this, move the test `{altimate_unique_id}` to the directory `{convention}`, where its corresponding " + "To rectify this, move the test `{model_unique_id}` to the directory `{convention}`, where its corresponding " "model is located. This adjustment will align your test's location with best practices for" " project organization." ) - def _build_failure_result(self, altimate_unique_id: str, convention: Optional[str]) -> DBTInsightResult: + def _build_failure_result(self, model_unique_id: str, convention: Optional[str]) -> DBTInsightResult: failure_message = self.FAILURE_MESSAGE.format( - altimate_unique_id=altimate_unique_id, + model_unique_id=model_unique_id, ) return DBTInsightResult( name=self.NAME, type=self.TYPE, message=failure_message, - recommendation=self.RECOMMENDATION.format(altimate_unique_id=altimate_unique_id, convention=convention), + recommendation=self.RECOMMENDATION.format(model_unique_id=model_unique_id, convention=convention), reason_to_flag=self.REASON_TO_FLAG, metadata={ - "model": altimate_unique_id, + "model": model_unique_id, "convention": convention, }, ) diff --git a/tests/data/manifest_v11.json b/tests/data/manifest_v11.json index bb545c3d..b8690403 100644 --- a/tests/data/manifest_v11.json +++ b/tests/data/manifest_v11.json @@ -1690,7 +1690,7 @@ }, "compiled_path": "target/compiled/elementary/models/edr/run_results/elementary_test_results.sql", "compiled": true, - "compiled_code": "\n\n\n select * from (\n select\n \n \n cast('this_is_just_a_long_dummy_string' as varchar) as id\n\n,\n \n cast('dummy_string' as varchar) as data_issue_id\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_execution_id\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_unique_id\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as altimate_unique_id\n\n,\n \n cast('dummy_string' as varchar) as invocation_id\n\n,\n cast('2091-02-17' as timestamp) as detected_at\n\n,\n cast('2091-02-17' as timestamp) as created_at\n\n,\n \n cast('dummy_string' as varchar) as database_name\n\n,\n \n cast('dummy_string' as varchar) as schema_name\n\n,\n \n cast('dummy_string' as varchar) as table_name\n\n,\n \n cast('dummy_string' as varchar) as column_name\n\n,\n \n cast('dummy_string' as varchar) as test_type\n\n,\n \n cast('dummy_string' as varchar) as test_sub_type\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_results_description\n\n,\n \n cast('dummy_string' as varchar) as owners\n\n,\n \n cast('dummy_string' as varchar) as tags\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_results_query\n\n,\n \n cast('dummy_string' as varchar) as other\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_name\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_params\n\n,\n \n cast('dummy_string' as varchar) as severity\n\n,\n \n cast('dummy_string' as varchar) as status\n\n,\n \n cast(31474836478 as bigint) as failures\n\n,\n \n cast('dummy_string' as varchar) as test_short_name\n\n,\n \n cast('dummy_string' as varchar) as test_alias\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as result_rows\n\n,\n \n cast(31474836478 as bigint) as failed_row_count\n\n\n ) as empty_table\n where 1 = 0\n", + "compiled_code": "\n\n\n select * from (\n select\n \n \n cast('this_is_just_a_long_dummy_string' as varchar) as id\n\n,\n \n cast('dummy_string' as varchar) as data_issue_id\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_execution_id\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_unique_id\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as model_unique_id\n\n,\n \n cast('dummy_string' as varchar) as invocation_id\n\n,\n cast('2091-02-17' as timestamp) as detected_at\n\n,\n cast('2091-02-17' as timestamp) as created_at\n\n,\n \n cast('dummy_string' as varchar) as database_name\n\n,\n \n cast('dummy_string' as varchar) as schema_name\n\n,\n \n cast('dummy_string' as varchar) as table_name\n\n,\n \n cast('dummy_string' as varchar) as column_name\n\n,\n \n cast('dummy_string' as varchar) as test_type\n\n,\n \n cast('dummy_string' as varchar) as test_sub_type\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_results_description\n\n,\n \n cast('dummy_string' as varchar) as owners\n\n,\n \n cast('dummy_string' as varchar) as tags\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_results_query\n\n,\n \n cast('dummy_string' as varchar) as other\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_name\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_params\n\n,\n \n cast('dummy_string' as varchar) as severity\n\n,\n \n cast('dummy_string' as varchar) as status\n\n,\n \n cast(31474836478 as bigint) as failures\n\n,\n \n cast('dummy_string' as varchar) as test_short_name\n\n,\n \n cast('dummy_string' as varchar) as test_alias\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as result_rows\n\n,\n \n cast(31474836478 as bigint) as failed_row_count\n\n\n ) as empty_table\n where 1 = 0\n", "extra_ctes_injected": true, "extra_ctes": [], "contract": { @@ -1890,7 +1890,7 @@ }, "created_at": 1705031518.705544, "relation_name": "analytics.jaffle_shop_elementary.alerts_dbt_tests", - "raw_code": "{{\n config(\n materialized = 'view',\n bind=False\n )\n}}\n\nwith elementary_test_results as (\n select * from {{ ref('elementary_test_results') }}\n),\n\nalerts_dbt_tests as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n altimate_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where {{ not elementary.get_config_var('disable_test_alerts') }} and lower(status) != 'pass' {% if elementary.get_config_var('disable_warn_alerts') %} and lower(status) != 'warn' {% endif %} {% if elementary.get_config_var('disable_skipped_test_alerts') %} and lower(status) != 'skipped' {% endif %} and test_type = 'dbt_test'\n)\n\nselect * from alerts_dbt_tests", + "raw_code": "{{\n config(\n materialized = 'view',\n bind=False\n )\n}}\n\nwith elementary_test_results as (\n select * from {{ ref('elementary_test_results') }}\n),\n\nalerts_dbt_tests as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n model_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where {{ not elementary.get_config_var('disable_test_alerts') }} and lower(status) != 'pass' {% if elementary.get_config_var('disable_warn_alerts') %} and lower(status) != 'warn' {% endif %} {% if elementary.get_config_var('disable_skipped_test_alerts') %} and lower(status) != 'skipped' {% endif %} and test_type = 'dbt_test'\n)\n\nselect * from alerts_dbt_tests", "language": "sql", "refs": [ { @@ -1911,7 +1911,7 @@ }, "compiled_path": "target/compiled/elementary/models/edr/alerts/alerts_dbt_tests.sql", "compiled": true, - "compiled_code": "\n\nwith elementary_test_results as (\n select * from analytics.jaffle_shop_elementary.elementary_test_results\n),\n\nalerts_dbt_tests as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n altimate_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where True and lower(status) != 'pass' and lower(status) != 'skipped' and test_type = 'dbt_test'\n)\n\nselect * from alerts_dbt_tests", + "compiled_code": "\n\nwith elementary_test_results as (\n select * from analytics.jaffle_shop_elementary.elementary_test_results\n),\n\nalerts_dbt_tests as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n model_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where True and lower(status) != 'pass' and lower(status) != 'skipped' and test_type = 'dbt_test'\n)\n\nselect * from alerts_dbt_tests", "extra_ctes_injected": true, "extra_ctes": [], "contract": { @@ -1996,7 +1996,7 @@ }, "created_at": 1705031518.705947, "relation_name": "analytics.jaffle_shop_elementary.alerts_schema_changes", - "raw_code": "{{\n config(\n materialized = 'view',\n bind=False,\n )\n}}\n\n\nwith elementary_test_results as (\n select * from {{ ref('elementary_test_results') }}\n),\n\nalerts_schema_changes as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n altimate_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where {{ not elementary.get_config_var('disable_test_alerts') }} and lower(status) != 'pass' {%- if elementary.get_config_var('disable_warn_alerts') -%} and lower(status) != 'warn' {%- endif -%} {%- if elementary.get_config_var('disable_skipped_test_alerts') -%} and lower(status) != 'skipped' {%- endif -%} and test_type = 'schema_change'\n)\n\nselect * from alerts_schema_changes", + "raw_code": "{{\n config(\n materialized = 'view',\n bind=False,\n )\n}}\n\n\nwith elementary_test_results as (\n select * from {{ ref('elementary_test_results') }}\n),\n\nalerts_schema_changes as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n model_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where {{ not elementary.get_config_var('disable_test_alerts') }} and lower(status) != 'pass' {%- if elementary.get_config_var('disable_warn_alerts') -%} and lower(status) != 'warn' {%- endif -%} {%- if elementary.get_config_var('disable_skipped_test_alerts') -%} and lower(status) != 'skipped' {%- endif -%} and test_type = 'schema_change'\n)\n\nselect * from alerts_schema_changes", "language": "sql", "refs": [ { @@ -2017,7 +2017,7 @@ }, "compiled_path": "target/compiled/elementary/models/edr/alerts/alerts_schema_changes.sql", "compiled": true, - "compiled_code": "\n\n\nwith elementary_test_results as (\n select * from analytics.jaffle_shop_elementary.elementary_test_results\n),\n\nalerts_schema_changes as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n altimate_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where True and lower(status) != 'pass'and lower(status) != 'skipped'and test_type = 'schema_change'\n)\n\nselect * from alerts_schema_changes", + "compiled_code": "\n\n\nwith elementary_test_results as (\n select * from analytics.jaffle_shop_elementary.elementary_test_results\n),\n\nalerts_schema_changes as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n model_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where True and lower(status) != 'pass'and lower(status) != 'skipped'and test_type = 'schema_change'\n)\n\nselect * from alerts_schema_changes", "extra_ctes_injected": true, "extra_ctes": [], "contract": { @@ -2215,7 +2215,7 @@ }, "created_at": 1705031518.705749, "relation_name": "analytics.jaffle_shop_elementary.alerts_anomaly_detection", - "raw_code": "{{\n config(\n materialized = 'view',\n bind =False\n )\n}}\n\nwith elementary_test_results as (\n select * from {{ ref('elementary_test_results') }}\n),\n\nalerts_anomaly_detection as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n altimate_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where {{ not elementary.get_config_var('disable_test_alerts') }} and lower(status) != 'pass' {%- if elementary.get_config_var('disable_warn_alerts') -%} and lower(status) != 'warn' {%- endif -%} {%- if elementary.get_config_var('disable_skipped_test_alerts') -%} and lower(status) != 'skipped' {%- endif -%} and test_type = 'anomaly_detection'\n)\n\nselect * from alerts_anomaly_detection", + "raw_code": "{{\n config(\n materialized = 'view',\n bind =False\n )\n}}\n\nwith elementary_test_results as (\n select * from {{ ref('elementary_test_results') }}\n),\n\nalerts_anomaly_detection as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n model_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where {{ not elementary.get_config_var('disable_test_alerts') }} and lower(status) != 'pass' {%- if elementary.get_config_var('disable_warn_alerts') -%} and lower(status) != 'warn' {%- endif -%} {%- if elementary.get_config_var('disable_skipped_test_alerts') -%} and lower(status) != 'skipped' {%- endif -%} and test_type = 'anomaly_detection'\n)\n\nselect * from alerts_anomaly_detection", "language": "sql", "refs": [ { @@ -2236,7 +2236,7 @@ }, "compiled_path": "target/compiled/elementary/models/edr/alerts/alerts_anomaly_detection.sql", "compiled": true, - "compiled_code": "\n\nwith elementary_test_results as (\n select * from analytics.jaffle_shop_elementary.elementary_test_results\n),\n\nalerts_anomaly_detection as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n altimate_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where True and lower(status) != 'pass'and lower(status) != 'skipped'and test_type = 'anomaly_detection'\n)\n\nselect * from alerts_anomaly_detection", + "compiled_code": "\n\nwith elementary_test_results as (\n select * from analytics.jaffle_shop_elementary.elementary_test_results\n),\n\nalerts_anomaly_detection as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n model_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where True and lower(status) != 'pass'and lower(status) != 'skipped'and test_type = 'anomaly_detection'\n)\n\nselect * from alerts_anomaly_detection", "extra_ctes_injected": true, "extra_ctes": [], "contract": { @@ -2787,8 +2787,8 @@ "quote": null, "tags": [] }, - "parent_altimate_unique_id": { - "name": "parent_altimate_unique_id", + "parent_model_unique_id": { + "name": "parent_model_unique_id", "description": "", "meta": {}, "data_type": "string", @@ -2892,7 +2892,7 @@ }, "compiled_path": "target/compiled/elementary/models/edr/dbt_artifacts/dbt_tests.sql", "compiled": true, - "compiled_code": "\n\nselect * from (\n select\n \n \n cast('dummy_string' as varchar) as unique_id\n\n,\n \n cast('dummy_string' as varchar) as database_name\n\n,\n \n cast('dummy_string' as varchar) as schema_name\n\n,\n \n cast('dummy_string' as varchar) as name\n\n,\n \n cast('dummy_string' as varchar) as short_name\n\n,\n \n cast('dummy_string' as varchar) as alias\n\n,\n \n cast('dummy_string' as varchar) as test_column_name\n\n,\n \n cast('dummy_string' as varchar) as severity\n\n,\n \n cast('dummy_string' as varchar) as warn_if\n\n,\n \n cast('dummy_string' as varchar) as error_if\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_params\n\n,\n \n cast('dummy_string' as varchar) as test_namespace\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as tags\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as model_tags\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as model_owners\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as meta\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as depends_on_macros\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as depends_on_nodes\n\n,\n \n cast('dummy_string' as varchar) as parent_altimate_unique_id\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as description\n\n,\n \n cast('dummy_string' as varchar) as package_name\n\n,\n \n cast('dummy_string' as varchar) as type\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as original_path\n\n,\n \n cast('dummy_string' as varchar) as path\n\n,\n \n cast('dummy_string' as varchar) as generated_at\n\n,\n \n cast('dummy_string' as varchar) as metadata_hash\n\n,\n \n cast('dummy_string' as varchar) as quality_dimension\n\n\n ) as empty_table\n where 1 = 0", + "compiled_code": "\n\nselect * from (\n select\n \n \n cast('dummy_string' as varchar) as unique_id\n\n,\n \n cast('dummy_string' as varchar) as database_name\n\n,\n \n cast('dummy_string' as varchar) as schema_name\n\n,\n \n cast('dummy_string' as varchar) as name\n\n,\n \n cast('dummy_string' as varchar) as short_name\n\n,\n \n cast('dummy_string' as varchar) as alias\n\n,\n \n cast('dummy_string' as varchar) as test_column_name\n\n,\n \n cast('dummy_string' as varchar) as severity\n\n,\n \n cast('dummy_string' as varchar) as warn_if\n\n,\n \n cast('dummy_string' as varchar) as error_if\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_params\n\n,\n \n cast('dummy_string' as varchar) as test_namespace\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as tags\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as model_tags\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as model_owners\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as meta\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as depends_on_macros\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as depends_on_nodes\n\n,\n \n cast('dummy_string' as varchar) as parent_model_unique_id\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as description\n\n,\n \n cast('dummy_string' as varchar) as package_name\n\n,\n \n cast('dummy_string' as varchar) as type\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as original_path\n\n,\n \n cast('dummy_string' as varchar) as path\n\n,\n \n cast('dummy_string' as varchar) as generated_at\n\n,\n \n cast('dummy_string' as varchar) as metadata_hash\n\n,\n \n cast('dummy_string' as varchar) as quality_dimension\n\n\n ) as empty_table\n where 1 = 0", "extra_ctes_injected": true, "extra_ctes": [], "contract": { @@ -22765,7 +22765,7 @@ "path": "macros/edr/materializations/test/test.sql", "original_file_path": "macros/edr/materializations/test/test.sql", "unique_id": "macro.elementary.get_anomaly_test_result_row", - "macro_sql": "{% macro get_anomaly_test_result_row(flattened_test, anomaly_scores_rows) %}\n {%- set latest_row = anomaly_scores_rows[-1] %}\n {%- set full_table_name = elementary.insensitive_get_dict_value(latest_row, 'full_table_name') %}\n {%- set test_unique_id = flattened_test.unique_id %}\n {%- set test_configuration = elementary.get_cache(test_unique_id) %}\n {%- set test_params = elementary.insensitive_get_dict_value(flattened_test, 'test_params') %}\n {%- do test_params.update(test_configuration) %}\n {%- set parent_altimate_unique_id = elementary.insensitive_get_dict_value(flattened_test, 'parent_altimate_unique_id') %}\n {%- set column_name = elementary.insensitive_get_dict_value(latest_row, 'column_name') %}\n {%- set metric_name = elementary.insensitive_get_dict_value(latest_row, 'metric_name') %}\n {%- set test_unique_id = elementary.insensitive_get_dict_value(latest_row, 'test_unique_id') %}\n {%- set has_anomaly_score = elementary.insensitive_get_dict_value(latest_row, 'anomaly_score') is not none %}\n {%- if not has_anomaly_score %}\n {% do elementary.edr_log(\"Not enough data to calculate anomaly scores on `{}`\".format(test_unique_id)) %}\n {% endif %}\n {%- set test_results_query -%}\n select * from ({{ sql }}) results\n where\n anomaly_score is not null and\n upper(full_table_name) = upper({{ elementary.const_as_string(full_table_name) }}) and\n metric_name = {{ elementary.const_as_string(metric_name) }}\n {%- if column_name %}\n and upper(column_name) = upper({{ elementary.const_as_string(column_name) }})\n {%- endif %}\n {%- endset -%}\n {% set test_results_description %}\n {% if has_anomaly_score %}\n {{ elementary.insensitive_get_dict_value(latest_row, 'anomaly_description') }}\n {% else %}\n Not enough data to calculate anomaly score.\n {% endif %}\n {% endset %}\n {% set failures = namespace(data=0) %}\n {% set filtered_anomaly_scores_rows = [] %}\n {% for row in anomaly_scores_rows %}\n {% if row.anomaly_score is not none %}\n {% do filtered_anomaly_scores_rows.append(row) %}\n {% if row.is_anomalous %}\n {% set failures.data = failures.data + 1 %}\n {% endif %}\n {% endif %}\n {% endfor %}\n {% set test_result_dict = {\n 'id': elementary.insensitive_get_dict_value(latest_row, 'id'),\n 'data_issue_id': elementary.insensitive_get_dict_value(latest_row, 'metric_id'),\n 'altimate_unique_id': parent_altimate_unique_id,\n 'column_name': column_name,\n 'test_type': 'anomaly_detection',\n 'test_sub_type': metric_name,\n 'test_results_description': test_results_description,\n 'other': elementary.insensitive_get_dict_value(latest_row, 'anomalous_value'),\n 'test_results_query': test_results_query,\n 'test_params': test_params,\n 'result_rows': filtered_anomaly_scores_rows,\n 'failures': failures.data\n } %}\n {% set elementary_test_row = elementary.get_dbt_test_result_row(flattened_test) %}\n {% do elementary_test_row.update(test_result_dict) %}\n {% do return(elementary_test_row) %}\n{% endmacro %}", + "macro_sql": "{% macro get_anomaly_test_result_row(flattened_test, anomaly_scores_rows) %}\n {%- set latest_row = anomaly_scores_rows[-1] %}\n {%- set full_table_name = elementary.insensitive_get_dict_value(latest_row, 'full_table_name') %}\n {%- set test_unique_id = flattened_test.unique_id %}\n {%- set test_configuration = elementary.get_cache(test_unique_id) %}\n {%- set test_params = elementary.insensitive_get_dict_value(flattened_test, 'test_params') %}\n {%- do test_params.update(test_configuration) %}\n {%- set parent_model_unique_id = elementary.insensitive_get_dict_value(flattened_test, 'parent_model_unique_id') %}\n {%- set column_name = elementary.insensitive_get_dict_value(latest_row, 'column_name') %}\n {%- set metric_name = elementary.insensitive_get_dict_value(latest_row, 'metric_name') %}\n {%- set test_unique_id = elementary.insensitive_get_dict_value(latest_row, 'test_unique_id') %}\n {%- set has_anomaly_score = elementary.insensitive_get_dict_value(latest_row, 'anomaly_score') is not none %}\n {%- if not has_anomaly_score %}\n {% do elementary.edr_log(\"Not enough data to calculate anomaly scores on `{}`\".format(test_unique_id)) %}\n {% endif %}\n {%- set test_results_query -%}\n select * from ({{ sql }}) results\n where\n anomaly_score is not null and\n upper(full_table_name) = upper({{ elementary.const_as_string(full_table_name) }}) and\n metric_name = {{ elementary.const_as_string(metric_name) }}\n {%- if column_name %}\n and upper(column_name) = upper({{ elementary.const_as_string(column_name) }})\n {%- endif %}\n {%- endset -%}\n {% set test_results_description %}\n {% if has_anomaly_score %}\n {{ elementary.insensitive_get_dict_value(latest_row, 'anomaly_description') }}\n {% else %}\n Not enough data to calculate anomaly score.\n {% endif %}\n {% endset %}\n {% set failures = namespace(data=0) %}\n {% set filtered_anomaly_scores_rows = [] %}\n {% for row in anomaly_scores_rows %}\n {% if row.anomaly_score is not none %}\n {% do filtered_anomaly_scores_rows.append(row) %}\n {% if row.is_anomalous %}\n {% set failures.data = failures.data + 1 %}\n {% endif %}\n {% endif %}\n {% endfor %}\n {% set test_result_dict = {\n 'id': elementary.insensitive_get_dict_value(latest_row, 'id'),\n 'data_issue_id': elementary.insensitive_get_dict_value(latest_row, 'metric_id'),\n 'model_unique_id': parent_model_unique_id,\n 'column_name': column_name,\n 'test_type': 'anomaly_detection',\n 'test_sub_type': metric_name,\n 'test_results_description': test_results_description,\n 'other': elementary.insensitive_get_dict_value(latest_row, 'anomalous_value'),\n 'test_results_query': test_results_query,\n 'test_params': test_params,\n 'result_rows': filtered_anomaly_scores_rows,\n 'failures': failures.data\n } %}\n {% set elementary_test_row = elementary.get_dbt_test_result_row(flattened_test) %}\n {% do elementary_test_row.update(test_result_dict) %}\n {% do return(elementary_test_row) %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.elementary.insensitive_get_dict_value", @@ -22817,7 +22817,7 @@ "path": "macros/edr/materializations/test/test.sql", "original_file_path": "macros/edr/materializations/test/test.sql", "unique_id": "macro.elementary.get_dbt_test_result_row", - "macro_sql": "{% macro get_dbt_test_result_row(flattened_test, result_rows=none) %}\n {% if not result_rows %}\n {% set result_rows = [] %}\n {% endif %}\n\n {% set test_execution_id = elementary.get_node_execution_id(flattened_test) %}\n {% set parent_altimate_unique_id = elementary.insensitive_get_dict_value(flattened_test, 'parent_altimate_unique_id') %}\n {% set parent_model = elementary.get_node(parent_altimate_unique_id) %}\n {% set parent_model_name = elementary.get_table_name_from_node(parent_model) %}\n {% set test_result_dict = {\n 'id': test_execution_id,\n 'data_issue_id': none,\n 'test_execution_id': test_execution_id,\n 'test_unique_id': elementary.insensitive_get_dict_value(flattened_test, 'unique_id'),\n 'altimate_unique_id': parent_altimate_unique_id,\n 'detected_at': elementary.insensitive_get_dict_value(flattened_test, 'generated_at'),\n 'database_name': elementary.insensitive_get_dict_value(flattened_test, 'database_name'),\n 'schema_name': elementary.insensitive_get_dict_value(flattened_test, 'schema_name'),\n 'table_name': parent_model_name,\n 'column_name': elementary.insensitive_get_dict_value(flattened_test, 'test_column_name'),\n 'test_type': elementary.get_test_type(flattened_test),\n 'test_sub_type': elementary.insensitive_get_dict_value(flattened_test, 'type'),\n 'other': none,\n 'owners': elementary.insensitive_get_dict_value(flattened_test, 'model_owners'),\n 'tags': elementary.insensitive_get_dict_value(flattened_test, 'model_tags') + elementary.insensitive_get_dict_value(flattened_test, 'tags'),\n 'test_results_query': elementary.get_compiled_code(flattened_test),\n 'test_name': elementary.insensitive_get_dict_value(flattened_test, 'name'),\n 'test_params': elementary.insensitive_get_dict_value(flattened_test, 'test_params'),\n 'severity': elementary.insensitive_get_dict_value(flattened_test, 'severity'),\n 'test_short_name': elementary.insensitive_get_dict_value(flattened_test, 'short_name'),\n 'test_alias': elementary.insensitive_get_dict_value(flattened_test, 'alias'),\n 'result_rows': result_rows\n }%}\n {% do return(test_result_dict) %}\n{% endmacro %}", + "macro_sql": "{% macro get_dbt_test_result_row(flattened_test, result_rows=none) %}\n {% if not result_rows %}\n {% set result_rows = [] %}\n {% endif %}\n\n {% set test_execution_id = elementary.get_node_execution_id(flattened_test) %}\n {% set parent_model_unique_id = elementary.insensitive_get_dict_value(flattened_test, 'parent_model_unique_id') %}\n {% set parent_model = elementary.get_node(parent_model_unique_id) %}\n {% set parent_model_name = elementary.get_table_name_from_node(parent_model) %}\n {% set test_result_dict = {\n 'id': test_execution_id,\n 'data_issue_id': none,\n 'test_execution_id': test_execution_id,\n 'test_unique_id': elementary.insensitive_get_dict_value(flattened_test, 'unique_id'),\n 'model_unique_id': parent_model_unique_id,\n 'detected_at': elementary.insensitive_get_dict_value(flattened_test, 'generated_at'),\n 'database_name': elementary.insensitive_get_dict_value(flattened_test, 'database_name'),\n 'schema_name': elementary.insensitive_get_dict_value(flattened_test, 'schema_name'),\n 'table_name': parent_model_name,\n 'column_name': elementary.insensitive_get_dict_value(flattened_test, 'test_column_name'),\n 'test_type': elementary.get_test_type(flattened_test),\n 'test_sub_type': elementary.insensitive_get_dict_value(flattened_test, 'type'),\n 'other': none,\n 'owners': elementary.insensitive_get_dict_value(flattened_test, 'model_owners'),\n 'tags': elementary.insensitive_get_dict_value(flattened_test, 'model_tags') + elementary.insensitive_get_dict_value(flattened_test, 'tags'),\n 'test_results_query': elementary.get_compiled_code(flattened_test),\n 'test_name': elementary.insensitive_get_dict_value(flattened_test, 'name'),\n 'test_params': elementary.insensitive_get_dict_value(flattened_test, 'test_params'),\n 'severity': elementary.insensitive_get_dict_value(flattened_test, 'severity'),\n 'test_short_name': elementary.insensitive_get_dict_value(flattened_test, 'short_name'),\n 'test_alias': elementary.insensitive_get_dict_value(flattened_test, 'alias'),\n 'result_rows': result_rows\n }%}\n {% do return(test_result_dict) %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.elementary.get_node_execution_id", @@ -26469,7 +26469,7 @@ "path": "macros/edr/system/system_utils/empty_table.sql", "original_file_path": "macros/edr/system/system_utils/empty_table.sql", "unique_id": "macro.elementary.empty_elementary_test_results", - "macro_sql": "{% macro empty_elementary_test_results() %}\n {{ elementary.empty_table([\n ('id','long_string'),\n ('data_issue_id','string'),\n ('test_execution_id','long_string'),\n ('test_unique_id','long_string'),\n ('altimate_unique_id','long_string'),\n ('invocation_id', 'string'),\n ('detected_at','timestamp'),\n ('created_at','timestamp'),\n ('database_name','string'),\n ('schema_name','string'),\n ('table_name','string'),\n ('column_name','string'),\n ('test_type','string'),\n ('test_sub_type','string'),\n ('test_results_description','long_string'),\n ('owners','string'),\n ('tags','string'),\n ('test_results_query','long_string'),\n ('other','string'),\n ('test_name','long_string'),\n ('test_params','long_string'),\n ('severity','string'),\n ('status','string'),\n ('failures', 'bigint'),\n ('test_short_name', 'string'),\n ('test_alias', 'string'),\n ('result_rows', 'long_string'),\n ('failed_row_count', 'bigint')\n ]) }}\n{% endmacro %}", + "macro_sql": "{% macro empty_elementary_test_results() %}\n {{ elementary.empty_table([\n ('id','long_string'),\n ('data_issue_id','string'),\n ('test_execution_id','long_string'),\n ('test_unique_id','long_string'),\n ('model_unique_id','long_string'),\n ('invocation_id', 'string'),\n ('detected_at','timestamp'),\n ('created_at','timestamp'),\n ('database_name','string'),\n ('schema_name','string'),\n ('table_name','string'),\n ('column_name','string'),\n ('test_type','string'),\n ('test_sub_type','string'),\n ('test_results_description','long_string'),\n ('owners','string'),\n ('tags','string'),\n ('test_results_query','long_string'),\n ('other','string'),\n ('test_name','long_string'),\n ('test_params','long_string'),\n ('severity','string'),\n ('status','string'),\n ('failures', 'bigint'),\n ('test_short_name', 'string'),\n ('test_alias', 'string'),\n ('result_rows', 'long_string'),\n ('failed_row_count', 'bigint')\n ]) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.elementary.empty_table" @@ -28554,7 +28554,7 @@ "path": "macros/edr/dbt_artifacts/upload_dbt_tests.sql", "original_file_path": "macros/edr/dbt_artifacts/upload_dbt_tests.sql", "unique_id": "macro.elementary.get_dbt_tests_empty_table_query", - "macro_sql": "{% macro get_dbt_tests_empty_table_query() %}\n {% set dbt_tests_empty_table_query = elementary.empty_table([('unique_id', 'string'),\n ('database_name', 'string'),\n ('schema_name', 'string'),\n ('name', 'string'),\n ('short_name', 'string'),\n ('alias', 'string'),\n ('test_column_name', 'string'),\n ('severity', 'string'),\n ('warn_if', 'string'),\n ('error_if', 'string'),\n ('test_params', 'long_string'),\n ('test_namespace', 'string'),\n ('tags', 'long_string'),\n ('model_tags', 'long_string'),\n ('model_owners', 'long_string'),\n ('meta', 'long_string'),\n ('depends_on_macros', 'long_string'),\n ('depends_on_nodes', 'long_string'),\n ('parent_altimate_unique_id', 'string'),\n ('description', 'long_string'),\n ('package_name', 'string'),\n ('type', 'string'),\n ('original_path', 'long_string'),\n ('path', 'string'),\n ('generated_at', 'string'),\n ('metadata_hash', 'string'),\n ('quality_dimension', 'string')\n ]) %}\n {{ return(dbt_tests_empty_table_query) }}\n{% endmacro %}", + "macro_sql": "{% macro get_dbt_tests_empty_table_query() %}\n {% set dbt_tests_empty_table_query = elementary.empty_table([('unique_id', 'string'),\n ('database_name', 'string'),\n ('schema_name', 'string'),\n ('name', 'string'),\n ('short_name', 'string'),\n ('alias', 'string'),\n ('test_column_name', 'string'),\n ('severity', 'string'),\n ('warn_if', 'string'),\n ('error_if', 'string'),\n ('test_params', 'long_string'),\n ('test_namespace', 'string'),\n ('tags', 'long_string'),\n ('model_tags', 'long_string'),\n ('model_owners', 'long_string'),\n ('meta', 'long_string'),\n ('depends_on_macros', 'long_string'),\n ('depends_on_nodes', 'long_string'),\n ('parent_model_unique_id', 'string'),\n ('description', 'long_string'),\n ('package_name', 'string'),\n ('type', 'string'),\n ('original_path', 'long_string'),\n ('path', 'string'),\n ('generated_at', 'string'),\n ('metadata_hash', 'string'),\n ('quality_dimension', 'string')\n ]) %}\n {{ return(dbt_tests_empty_table_query) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.elementary.empty_table" @@ -28578,14 +28578,14 @@ "path": "macros/edr/dbt_artifacts/upload_dbt_tests.sql", "original_file_path": "macros/edr/dbt_artifacts/upload_dbt_tests.sql", "unique_id": "macro.elementary.flatten_test", - "macro_sql": "{% macro flatten_test(node_dict) %}\n {% set config_dict = elementary.safe_get_with_default(node_dict, 'config', {}) %}\n {% set depends_on_dict = elementary.safe_get_with_default(node_dict, 'depends_on', {}) %}\n\n {% set test_metadata = elementary.safe_get_with_default(node_dict, 'test_metadata', {}) %}\n {% set test_namespace = test_metadata.get('namespace') %}\n {% set test_short_name = elementary.get_test_short_name(node_dict, test_metadata) %}\n\n {% set default_description = elementary.get_default_description(test_short_name, test_namespace) %}\n\n {% set config_meta_dict = elementary.safe_get_with_default(config_dict, 'meta', {}) %}\n {% set meta_dict = {} %}\n {% if default_description %}\n {% set meta_dict = {'description': default_description} %} \n {% endif %}\n {% do meta_dict.update(elementary.safe_get_with_default(node_dict, 'meta', {})) %}\n {% do meta_dict.update(config_meta_dict) %}\n\n {% set config_tags = elementary.safe_get_with_default(config_dict, 'tags', []) %}\n {% set global_tags = elementary.safe_get_with_default(node_dict, 'tags', []) %}\n {% set meta_tags = elementary.safe_get_with_default(meta_dict, 'tags', []) %}\n {% set tags = elementary.union_lists(config_tags, global_tags) %}\n {% set tags = elementary.union_lists(tags, meta_tags) %}\n\n {% set test_altimate_unique_ids = elementary.get_parent_altimate_unique_ids_from_test_node(node_dict) %}\n {% set test_model_nodes = elementary.get_nodes_by_unique_ids(test_altimate_unique_ids) %}\n {% set test_models_owners = [] %}\n {% set test_models_tags = [] %}\n {% for test_model_node in test_model_nodes %}\n {% set flatten_test_model_node = elementary.flatten_model(test_model_node) %}\n {% set test_model_owner = flatten_test_model_node.get('owner') %}\n {% if test_model_owner %}\n {% if test_model_owner is string %}\n {% set owners = test_model_owner.split(',') %}\n {% for owner in owners %}\n {% do test_models_owners.append(owner | trim) %} \n {% endfor %}\n {% elif test_model_owner is iterable %}\n {% do test_models_owners.extend(test_model_owner) %}\n {% endif %}\n {% endif %}\n {% set test_model_tags = flatten_test_model_node.get('tags') %}\n {% if test_model_tags and test_model_tags is sequence %}\n {% do test_models_tags.extend(test_model_tags) %}\n {% endif %}\n {% endfor %}\n {% set test_models_owners = test_models_owners | unique | list %}\n {% set test_models_tags = test_models_tags | unique | list %}\n\n {% set test_kwargs = elementary.safe_get_with_default(test_metadata, 'kwargs', {}) %}\n\n {% set primary_test_model_id = namespace(data=none) %}\n {% if test_altimate_unique_ids | length == 1 %}\n {# if only one parent model for this test, simply use this model #}\n {% set primary_test_model_id.data = test_altimate_unique_ids[0] %}\n {% else %}\n {% set test_model_jinja = test_kwargs.get('model') %}\n {% if test_model_jinja %}\n {% set test_model_name_matches =\n modules.re.findall(\"ref\\(['\\\"](\\w+)['\\\"]\\)\", test_model_jinja) +\n modules.re.findall(\"source\\(['\\\"]\\w+['\\\"], ['\\\"](\\w+)['\\\"]\\)\", test_model_jinja) %}\n {% if test_model_name_matches | length == 1 %}\n {% set test_model_name = test_model_name_matches[0] %}\n {% for test_altimate_unique_id in test_altimate_unique_ids %}\n {% set split_test_altimate_unique_id = test_altimate_unique_id.split('.') %}\n {% if split_test_altimate_unique_id and split_test_altimate_unique_id | length > 0 %}\n {% set test_node_model_name = split_test_altimate_unique_id[-1] %}\n {% if test_node_model_name == test_model_name %}\n {% set primary_test_model_id.data = test_altimate_unique_id %}\n {% endif %}\n {% endif %}\n {% endfor %}\n {% endif %}\n {% endif %}\n {% endif %}\n\n {% set primary_test_model_database = none %}\n {% set primary_test_model_schema = none %}\n {%- if primary_test_model_id.data is not none -%}\n {% set tested_model_node = elementary.get_node(primary_test_model_id.data) %}\n {%- if tested_model_node -%}\n {% set primary_test_model_database = tested_model_node.get('database') %}\n {% set primary_test_model_schema = tested_model_node.get('schema') %}\n {%- endif -%}\n {%- endif -%}\n\n {%- if primary_test_model_database is none or primary_test_model_schema is none -%}\n {# This is mainly here to support singular test cases with multiple referred models, in this case the tested node is being used to extract the db and schema #}\n {% set primary_test_model_database, primary_test_model_schema = elementary.get_model_database_and_schema_from_test_node(node_dict) %}\n {%- endif -%}\n\n {% set original_file_path = node_dict.get('original_file_path') %}\n {% set flatten_test_metadata_dict = {\n 'unique_id': node_dict.get('unique_id'),\n 'short_name': test_short_name,\n 'alias': node_dict.get('alias'),\n 'test_column_name': node_dict.get('column_name'),\n 'severity': config_dict.get('severity'),\n 'warn_if': config_dict.get('warn_if'),\n 'error_if': config_dict.get('error_if'),\n 'test_params': test_kwargs,\n 'test_namespace': test_namespace,\n 'tags': elementary.filter_none_and_sort(tags),\n 'model_tags': elementary.filter_none_and_sort(test_models_tags),\n 'model_owners': elementary.filter_none_and_sort(test_models_owners),\n 'meta': meta_dict,\n 'database_name': primary_test_model_database,\n 'schema_name': primary_test_model_schema,\n 'depends_on_macros': elementary.filter_none_and_sort(depends_on_dict.get('macros', [])),\n 'depends_on_nodes': elementary.filter_none_and_sort(depends_on_dict.get('nodes', [])),\n 'parent_altimate_unique_id': primary_test_model_id.data,\n 'description': meta_dict.get('description'),\n 'name': node_dict.get('name'),\n 'package_name': node_dict.get('package_name'),\n 'type': elementary.get_test_sub_type(original_file_path, test_namespace),\n 'original_path': original_file_path,\n 'compiled_code': elementary.get_compiled_code(node_dict),\n 'path': node_dict.get('path'),\n 'generated_at': elementary.datetime_now_utc_as_string(),\n 'quality_dimension': meta_dict.get('quality_dimension') or elementary.get_quality_dimension(test_short_name, test_namespace)\n }%}\n {% do flatten_test_metadata_dict.update({\"metadata_hash\": elementary.get_artifact_metadata_hash(flatten_test_metadata_dict)}) %}\n {{ return(flatten_test_metadata_dict) }}\n{% endmacro %}", + "macro_sql": "{% macro flatten_test(node_dict) %}\n {% set config_dict = elementary.safe_get_with_default(node_dict, 'config', {}) %}\n {% set depends_on_dict = elementary.safe_get_with_default(node_dict, 'depends_on', {}) %}\n\n {% set test_metadata = elementary.safe_get_with_default(node_dict, 'test_metadata', {}) %}\n {% set test_namespace = test_metadata.get('namespace') %}\n {% set test_short_name = elementary.get_test_short_name(node_dict, test_metadata) %}\n\n {% set default_description = elementary.get_default_description(test_short_name, test_namespace) %}\n\n {% set config_meta_dict = elementary.safe_get_with_default(config_dict, 'meta', {}) %}\n {% set meta_dict = {} %}\n {% if default_description %}\n {% set meta_dict = {'description': default_description} %} \n {% endif %}\n {% do meta_dict.update(elementary.safe_get_with_default(node_dict, 'meta', {})) %}\n {% do meta_dict.update(config_meta_dict) %}\n\n {% set config_tags = elementary.safe_get_with_default(config_dict, 'tags', []) %}\n {% set global_tags = elementary.safe_get_with_default(node_dict, 'tags', []) %}\n {% set meta_tags = elementary.safe_get_with_default(meta_dict, 'tags', []) %}\n {% set tags = elementary.union_lists(config_tags, global_tags) %}\n {% set tags = elementary.union_lists(tags, meta_tags) %}\n\n {% set test_model_unique_ids = elementary.get_parent_model_unique_ids_from_test_node(node_dict) %}\n {% set test_model_nodes = elementary.get_nodes_by_unique_ids(test_model_unique_ids) %}\n {% set test_models_owners = [] %}\n {% set test_models_tags = [] %}\n {% for test_model_node in test_model_nodes %}\n {% set flatten_test_model_node = elementary.flatten_model(test_model_node) %}\n {% set test_model_owner = flatten_test_model_node.get('owner') %}\n {% if test_model_owner %}\n {% if test_model_owner is string %}\n {% set owners = test_model_owner.split(',') %}\n {% for owner in owners %}\n {% do test_models_owners.append(owner | trim) %} \n {% endfor %}\n {% elif test_model_owner is iterable %}\n {% do test_models_owners.extend(test_model_owner) %}\n {% endif %}\n {% endif %}\n {% set test_model_tags = flatten_test_model_node.get('tags') %}\n {% if test_model_tags and test_model_tags is sequence %}\n {% do test_models_tags.extend(test_model_tags) %}\n {% endif %}\n {% endfor %}\n {% set test_models_owners = test_models_owners | unique | list %}\n {% set test_models_tags = test_models_tags | unique | list %}\n\n {% set test_kwargs = elementary.safe_get_with_default(test_metadata, 'kwargs', {}) %}\n\n {% set primary_test_model_id = namespace(data=none) %}\n {% if test_model_unique_ids | length == 1 %}\n {# if only one parent model for this test, simply use this model #}\n {% set primary_test_model_id.data = test_model_unique_ids[0] %}\n {% else %}\n {% set test_model_jinja = test_kwargs.get('model') %}\n {% if test_model_jinja %}\n {% set test_model_name_matches =\n modules.re.findall(\"ref\\(['\\\"](\\w+)['\\\"]\\)\", test_model_jinja) +\n modules.re.findall(\"source\\(['\\\"]\\w+['\\\"], ['\\\"](\\w+)['\\\"]\\)\", test_model_jinja) %}\n {% if test_model_name_matches | length == 1 %}\n {% set test_model_name = test_model_name_matches[0] %}\n {% for test_model_unique_id in test_model_unique_ids %}\n {% set split_test_model_unique_id = test_model_unique_id.split('.') %}\n {% if split_test_model_unique_id and split_test_model_unique_id | length > 0 %}\n {% set test_node_model_name = split_test_model_unique_id[-1] %}\n {% if test_node_model_name == test_model_name %}\n {% set primary_test_model_id.data = test_model_unique_id %}\n {% endif %}\n {% endif %}\n {% endfor %}\n {% endif %}\n {% endif %}\n {% endif %}\n\n {% set primary_test_model_database = none %}\n {% set primary_test_model_schema = none %}\n {%- if primary_test_model_id.data is not none -%}\n {% set tested_model_node = elementary.get_node(primary_test_model_id.data) %}\n {%- if tested_model_node -%}\n {% set primary_test_model_database = tested_model_node.get('database') %}\n {% set primary_test_model_schema = tested_model_node.get('schema') %}\n {%- endif -%}\n {%- endif -%}\n\n {%- if primary_test_model_database is none or primary_test_model_schema is none -%}\n {# This is mainly here to support singular test cases with multiple referred models, in this case the tested node is being used to extract the db and schema #}\n {% set primary_test_model_database, primary_test_model_schema = elementary.get_model_database_and_schema_from_test_node(node_dict) %}\n {%- endif -%}\n\n {% set original_file_path = node_dict.get('original_file_path') %}\n {% set flatten_test_metadata_dict = {\n 'unique_id': node_dict.get('unique_id'),\n 'short_name': test_short_name,\n 'alias': node_dict.get('alias'),\n 'test_column_name': node_dict.get('column_name'),\n 'severity': config_dict.get('severity'),\n 'warn_if': config_dict.get('warn_if'),\n 'error_if': config_dict.get('error_if'),\n 'test_params': test_kwargs,\n 'test_namespace': test_namespace,\n 'tags': elementary.filter_none_and_sort(tags),\n 'model_tags': elementary.filter_none_and_sort(test_models_tags),\n 'model_owners': elementary.filter_none_and_sort(test_models_owners),\n 'meta': meta_dict,\n 'database_name': primary_test_model_database,\n 'schema_name': primary_test_model_schema,\n 'depends_on_macros': elementary.filter_none_and_sort(depends_on_dict.get('macros', [])),\n 'depends_on_nodes': elementary.filter_none_and_sort(depends_on_dict.get('nodes', [])),\n 'parent_model_unique_id': primary_test_model_id.data,\n 'description': meta_dict.get('description'),\n 'name': node_dict.get('name'),\n 'package_name': node_dict.get('package_name'),\n 'type': elementary.get_test_sub_type(original_file_path, test_namespace),\n 'original_path': original_file_path,\n 'compiled_code': elementary.get_compiled_code(node_dict),\n 'path': node_dict.get('path'),\n 'generated_at': elementary.datetime_now_utc_as_string(),\n 'quality_dimension': meta_dict.get('quality_dimension') or elementary.get_quality_dimension(test_short_name, test_namespace)\n }%}\n {% do flatten_test_metadata_dict.update({\"metadata_hash\": elementary.get_artifact_metadata_hash(flatten_test_metadata_dict)}) %}\n {{ return(flatten_test_metadata_dict) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.elementary.safe_get_with_default", "macro.elementary.get_test_short_name", "macro.elementary.get_default_description", "macro.elementary.union_lists", - "macro.elementary.get_parent_altimate_unique_ids_from_test_node", + "macro.elementary.get_parent_model_unique_ids_from_test_node", "macro.elementary.get_nodes_by_unique_ids", "macro.elementary.flatten_model", "macro.elementary.get_node", @@ -34943,14 +34943,14 @@ "created_at": 1705031517.168567, "supported_languages": null }, - "macro.elementary.get_parent_altimate_unique_ids_from_test_node": { - "name": "get_parent_altimate_unique_ids_from_test_node", + "macro.elementary.get_parent_model_unique_ids_from_test_node": { + "name": "get_parent_model_unique_ids_from_test_node", "resource_type": "macro", "package_name": "elementary", - "path": "macros/utils/graph/get_parent_altimate_unique_ids_from_test_node.sql", - "original_file_path": "macros/utils/graph/get_parent_altimate_unique_ids_from_test_node.sql", - "unique_id": "macro.elementary.get_parent_altimate_unique_ids_from_test_node", - "macro_sql": "{% macro get_parent_altimate_unique_ids_from_test_node(test_node) %}\n {% set nodes_in_current_package = [] %}\n {% set test_depends_on = test_node.get('depends_on') %}\n {% if test_depends_on %}\n {% set depends_on_nodes = test_depends_on.get('nodes') %}\n {% if depends_on_nodes %}\n {% set current_package_name = test_node.get('package_name') %}\n {% if current_package_name %}\n {% set current_package_name = '.' ~ current_package_name ~ '.' %}\n {% for node in depends_on_nodes %}\n {% if current_package_name in node %}\n {% do nodes_in_current_package.append(node) %}\n {% endif %}\n {% endfor %}\n {% endif %}\n {% endif %}\n {% endif %}\n {{ return(nodes_in_current_package) }}\n{% endmacro %}", + "path": "macros/utils/graph/get_parent_model_unique_ids_from_test_node.sql", + "original_file_path": "macros/utils/graph/get_parent_model_unique_ids_from_test_node.sql", + "unique_id": "macro.elementary.get_parent_model_unique_ids_from_test_node", + "macro_sql": "{% macro get_parent_model_unique_ids_from_test_node(test_node) %}\n {% set nodes_in_current_package = [] %}\n {% set test_depends_on = test_node.get('depends_on') %}\n {% if test_depends_on %}\n {% set depends_on_nodes = test_depends_on.get('nodes') %}\n {% if depends_on_nodes %}\n {% set current_package_name = test_node.get('package_name') %}\n {% if current_package_name %}\n {% set current_package_name = '.' ~ current_package_name ~ '.' %}\n {% for node in depends_on_nodes %}\n {% if current_package_name in node %}\n {% do nodes_in_current_package.append(node) %}\n {% endif %}\n {% endfor %}\n {% endif %}\n {% endif %}\n {% endif %}\n {{ return(nodes_in_current_package) }}\n{% endmacro %}", "depends_on": { "macros": [] }, @@ -37231,7 +37231,7 @@ "path": "macros/commands/delete_duplicate_rows.sql", "original_file_path": "macros/commands/delete_duplicate_rows.sql", "unique_id": "macro.elementary.delete_duplicate_rows", - "macro_sql": "{% macro delete_duplicate_rows(altimate_unique_id) %}\n {% do return(adapter.dispatch(\"delete_duplicate_rows\", \"elementary\")(altimate_unique_id)) %}\n{% endmacro %}", + "macro_sql": "{% macro delete_duplicate_rows(model_unique_id) %}\n {% do return(adapter.dispatch(\"delete_duplicate_rows\", \"elementary\")(model_unique_id)) %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.elementary.default__delete_duplicate_rows" @@ -37255,7 +37255,7 @@ "path": "macros/commands/delete_duplicate_rows.sql", "original_file_path": "macros/commands/delete_duplicate_rows.sql", "unique_id": "macro.elementary.postgres__delete_duplicate_rows", - "macro_sql": "{% macro postgres__delete_duplicate_rows(altimate_unique_id) %}\n {% set node = graph.nodes[altimate_unique_id] %}\n {% set relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) %}\n {% if relation is none %}\n {% do print(\"Relation '{}' does not exist.\".format(node.relation_name)) %}\n {% do return([]) %}\n {% endif %}\n\n {% set column_names = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | map(\"lower\") | list %}\n\n {% set query %}\n DELETE FROM {{ relation }} AS t1\n USING {{ relation }} AS t2\n WHERE t1.ctid < t2.ctid\n {% for col in column_names %}\n AND t1.{{ col }} = t2.{{ col }}\n {% endfor %}\n {% endset %}\n {% do elementary.run_query(query) %}\n {% do adapter.commit() %}\n{% endmacro %}", + "macro_sql": "{% macro postgres__delete_duplicate_rows(model_unique_id) %}\n {% set node = graph.nodes[model_unique_id] %}\n {% set relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) %}\n {% if relation is none %}\n {% do print(\"Relation '{}' does not exist.\".format(node.relation_name)) %}\n {% do return([]) %}\n {% endif %}\n\n {% set column_names = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | map(\"lower\") | list %}\n\n {% set query %}\n DELETE FROM {{ relation }} AS t1\n USING {{ relation }} AS t2\n WHERE t1.ctid < t2.ctid\n {% for col in column_names %}\n AND t1.{{ col }} = t2.{{ col }}\n {% endfor %}\n {% endset %}\n {% do elementary.run_query(query) %}\n {% do adapter.commit() %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.elementary.run_query" @@ -37279,7 +37279,7 @@ "path": "macros/commands/delete_duplicate_rows.sql", "original_file_path": "macros/commands/delete_duplicate_rows.sql", "unique_id": "macro.elementary.default__delete_duplicate_rows", - "macro_sql": "{% macro default__delete_duplicate_rows(altimate_unique_id) %}\n {{ exceptions.raise_compiler_error(\"This macro is not supported on '{}'.\".format(target.type)) }}\n{% endmacro %}", + "macro_sql": "{% macro default__delete_duplicate_rows(model_unique_id) %}\n {{ exceptions.raise_compiler_error(\"This macro is not supported on '{}'.\".format(target.type)) }}\n{% endmacro %}", "depends_on": { "macros": [] }, @@ -37472,7 +37472,7 @@ "path": "macros/commands/dump_table.sql", "original_file_path": "macros/commands/dump_table.sql", "unique_id": "macro.elementary.dump_table", - "macro_sql": "{% macro dump_table(altimate_unique_id, output_path, exclude_deprecated_columns=true, timestamp_column=none, since=none, days_back=7, dedup=false) %}\n {% set node = graph.nodes.get(altimate_unique_id) %}\n {% if not node %}\n {% do print(\"Node '{}' does not exist.\".format(altimate_unique_id)) %}\n {% do return([]) %}\n {% endif %}\n\n {% set relation = api.Relation.create(database=node.database, schema=node.schema, identifier=node.alias) %}\n {% set column_names = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | map(\"lower\") | list %}\n {% if not column_names %}\n {% do print(\"Relation '{}' does not exist.\".format(node.relation_name)) %}\n {% do return([]) %}\n {% endif %}\n\n {% if exclude_deprecated_columns %}\n {% set deprecated_column_names = node.meta.get(\"deprecated_columns\", []) | map(attribute=\"name\") | map(\"lower\") | list %}\n {% set column_names = column_names | reject(\"in\", deprecated_column_names) | list %}\n {% endif %}\n\n {% set dedup_by_column = node.meta.dedup_by_column or \"unique_id\" %}\n {% set order_by_dedup_column = \"generated_at\" %}\n {% set query %}\n {% if dedup and (dedup_by_column in column_names) and (order_by_dedup_column in column_names) %}\n {{ elementary.dedup_by_column_query(dedup_by_column, order_by_dedup_column, column_names, relation) }}\n {% else %}\n select {{ elementary.escape_select(column_names) }} from {{ relation }}\n {% endif %}\n {% if timestamp_column %}\n {% if since %}\n where {{ elementary.edr_cast_as_timestamp(timestamp_column) }} > {{ elementary.edr_cast_as_timestamp(elementary.edr_quote(since)) }}\n {% else %}\n where {{ elementary.edr_datediff(elementary.edr_cast_as_timestamp(timestamp_column), elementary.edr_current_timestamp(), 'day') }} < {{ days_back }}\n {% endif %}\n {% endif %}\n {% endset %}\n {% set results = elementary.run_query(query) %}\n {% do results.to_csv(output_path) %}\n {% do return(results.column_names) %}\n{% endmacro %}", + "macro_sql": "{% macro dump_table(model_unique_id, output_path, exclude_deprecated_columns=true, timestamp_column=none, since=none, days_back=7, dedup=false) %}\n {% set node = graph.nodes.get(model_unique_id) %}\n {% if not node %}\n {% do print(\"Node '{}' does not exist.\".format(model_unique_id)) %}\n {% do return([]) %}\n {% endif %}\n\n {% set relation = api.Relation.create(database=node.database, schema=node.schema, identifier=node.alias) %}\n {% set column_names = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | map(\"lower\") | list %}\n {% if not column_names %}\n {% do print(\"Relation '{}' does not exist.\".format(node.relation_name)) %}\n {% do return([]) %}\n {% endif %}\n\n {% if exclude_deprecated_columns %}\n {% set deprecated_column_names = node.meta.get(\"deprecated_columns\", []) | map(attribute=\"name\") | map(\"lower\") | list %}\n {% set column_names = column_names | reject(\"in\", deprecated_column_names) | list %}\n {% endif %}\n\n {% set dedup_by_column = node.meta.dedup_by_column or \"unique_id\" %}\n {% set order_by_dedup_column = \"generated_at\" %}\n {% set query %}\n {% if dedup and (dedup_by_column in column_names) and (order_by_dedup_column in column_names) %}\n {{ elementary.dedup_by_column_query(dedup_by_column, order_by_dedup_column, column_names, relation) }}\n {% else %}\n select {{ elementary.escape_select(column_names) }} from {{ relation }}\n {% endif %}\n {% if timestamp_column %}\n {% if since %}\n where {{ elementary.edr_cast_as_timestamp(timestamp_column) }} > {{ elementary.edr_cast_as_timestamp(elementary.edr_quote(since)) }}\n {% else %}\n where {{ elementary.edr_datediff(elementary.edr_cast_as_timestamp(timestamp_column), elementary.edr_current_timestamp(), 'day') }} < {{ days_back }}\n {% endif %}\n {% endif %}\n {% endset %}\n {% set results = elementary.run_query(query) %}\n {% do results.to_csv(output_path) %}\n {% do return(results.column_names) %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.elementary.dedup_by_column_query", diff --git a/tests/data/manifest_v11macroargs.json b/tests/data/manifest_v11macroargs.json index 3453c534..bb74a34f 100644 --- a/tests/data/manifest_v11macroargs.json +++ b/tests/data/manifest_v11macroargs.json @@ -1690,7 +1690,7 @@ }, "compiled_path": "target/compiled/elementary/models/edr/run_results/elementary_test_results.sql", "compiled": true, - "compiled_code": "\n\n\n select * from (\n select\n \n \n cast('this_is_just_a_long_dummy_string' as varchar) as id\n\n,\n \n cast('dummy_string' as varchar) as data_issue_id\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_execution_id\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_unique_id\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as altimate_unique_id\n\n,\n \n cast('dummy_string' as varchar) as invocation_id\n\n,\n cast('2091-02-17' as timestamp) as detected_at\n\n,\n cast('2091-02-17' as timestamp) as created_at\n\n,\n \n cast('dummy_string' as varchar) as database_name\n\n,\n \n cast('dummy_string' as varchar) as schema_name\n\n,\n \n cast('dummy_string' as varchar) as table_name\n\n,\n \n cast('dummy_string' as varchar) as column_name\n\n,\n \n cast('dummy_string' as varchar) as test_type\n\n,\n \n cast('dummy_string' as varchar) as test_sub_type\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_results_description\n\n,\n \n cast('dummy_string' as varchar) as owners\n\n,\n \n cast('dummy_string' as varchar) as tags\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_results_query\n\n,\n \n cast('dummy_string' as varchar) as other\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_name\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_params\n\n,\n \n cast('dummy_string' as varchar) as severity\n\n,\n \n cast('dummy_string' as varchar) as status\n\n,\n \n cast(31474836478 as bigint) as failures\n\n,\n \n cast('dummy_string' as varchar) as test_short_name\n\n,\n \n cast('dummy_string' as varchar) as test_alias\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as result_rows\n\n,\n \n cast(31474836478 as bigint) as failed_row_count\n\n\n ) as empty_table\n where 1 = 0\n", + "compiled_code": "\n\n\n select * from (\n select\n \n \n cast('this_is_just_a_long_dummy_string' as varchar) as id\n\n,\n \n cast('dummy_string' as varchar) as data_issue_id\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_execution_id\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_unique_id\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as model_unique_id\n\n,\n \n cast('dummy_string' as varchar) as invocation_id\n\n,\n cast('2091-02-17' as timestamp) as detected_at\n\n,\n cast('2091-02-17' as timestamp) as created_at\n\n,\n \n cast('dummy_string' as varchar) as database_name\n\n,\n \n cast('dummy_string' as varchar) as schema_name\n\n,\n \n cast('dummy_string' as varchar) as table_name\n\n,\n \n cast('dummy_string' as varchar) as column_name\n\n,\n \n cast('dummy_string' as varchar) as test_type\n\n,\n \n cast('dummy_string' as varchar) as test_sub_type\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_results_description\n\n,\n \n cast('dummy_string' as varchar) as owners\n\n,\n \n cast('dummy_string' as varchar) as tags\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_results_query\n\n,\n \n cast('dummy_string' as varchar) as other\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_name\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_params\n\n,\n \n cast('dummy_string' as varchar) as severity\n\n,\n \n cast('dummy_string' as varchar) as status\n\n,\n \n cast(31474836478 as bigint) as failures\n\n,\n \n cast('dummy_string' as varchar) as test_short_name\n\n,\n \n cast('dummy_string' as varchar) as test_alias\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as result_rows\n\n,\n \n cast(31474836478 as bigint) as failed_row_count\n\n\n ) as empty_table\n where 1 = 0\n", "extra_ctes_injected": true, "extra_ctes": [], "contract": { @@ -1890,7 +1890,7 @@ }, "created_at": 1705031518.705544, "relation_name": "analytics.jaffle_shop_elementary.alerts_dbt_tests", - "raw_code": "{{\n config(\n materialized = 'view',\n bind=False\n )\n}}\n\nwith elementary_test_results as (\n select * from {{ ref('elementary_test_results') }}\n),\n\nalerts_dbt_tests as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n altimate_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where {{ not elementary.get_config_var('disable_test_alerts') }} and lower(status) != 'pass' {% if elementary.get_config_var('disable_warn_alerts') %} and lower(status) != 'warn' {% endif %} {% if elementary.get_config_var('disable_skipped_test_alerts') %} and lower(status) != 'skipped' {% endif %} and test_type = 'dbt_test'\n)\n\nselect * from alerts_dbt_tests", + "raw_code": "{{\n config(\n materialized = 'view',\n bind=False\n )\n}}\n\nwith elementary_test_results as (\n select * from {{ ref('elementary_test_results') }}\n),\n\nalerts_dbt_tests as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n model_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where {{ not elementary.get_config_var('disable_test_alerts') }} and lower(status) != 'pass' {% if elementary.get_config_var('disable_warn_alerts') %} and lower(status) != 'warn' {% endif %} {% if elementary.get_config_var('disable_skipped_test_alerts') %} and lower(status) != 'skipped' {% endif %} and test_type = 'dbt_test'\n)\n\nselect * from alerts_dbt_tests", "language": "sql", "refs": [ { @@ -1911,7 +1911,7 @@ }, "compiled_path": "target/compiled/elementary/models/edr/alerts/alerts_dbt_tests.sql", "compiled": true, - "compiled_code": "\n\nwith elementary_test_results as (\n select * from analytics.jaffle_shop_elementary.elementary_test_results\n),\n\nalerts_dbt_tests as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n altimate_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where True and lower(status) != 'pass' and lower(status) != 'skipped' and test_type = 'dbt_test'\n)\n\nselect * from alerts_dbt_tests", + "compiled_code": "\n\nwith elementary_test_results as (\n select * from analytics.jaffle_shop_elementary.elementary_test_results\n),\n\nalerts_dbt_tests as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n model_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where True and lower(status) != 'pass' and lower(status) != 'skipped' and test_type = 'dbt_test'\n)\n\nselect * from alerts_dbt_tests", "extra_ctes_injected": true, "extra_ctes": [], "contract": { @@ -1996,7 +1996,7 @@ }, "created_at": 1705031518.705947, "relation_name": "analytics.jaffle_shop_elementary.alerts_schema_changes", - "raw_code": "{{\n config(\n materialized = 'view',\n bind=False,\n )\n}}\n\n\nwith elementary_test_results as (\n select * from {{ ref('elementary_test_results') }}\n),\n\nalerts_schema_changes as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n altimate_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where {{ not elementary.get_config_var('disable_test_alerts') }} and lower(status) != 'pass' {%- if elementary.get_config_var('disable_warn_alerts') -%} and lower(status) != 'warn' {%- endif -%} {%- if elementary.get_config_var('disable_skipped_test_alerts') -%} and lower(status) != 'skipped' {%- endif -%} and test_type = 'schema_change'\n)\n\nselect * from alerts_schema_changes", + "raw_code": "{{\n config(\n materialized = 'view',\n bind=False,\n )\n}}\n\n\nwith elementary_test_results as (\n select * from {{ ref('elementary_test_results') }}\n),\n\nalerts_schema_changes as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n model_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where {{ not elementary.get_config_var('disable_test_alerts') }} and lower(status) != 'pass' {%- if elementary.get_config_var('disable_warn_alerts') -%} and lower(status) != 'warn' {%- endif -%} {%- if elementary.get_config_var('disable_skipped_test_alerts') -%} and lower(status) != 'skipped' {%- endif -%} and test_type = 'schema_change'\n)\n\nselect * from alerts_schema_changes", "language": "sql", "refs": [ { @@ -2017,7 +2017,7 @@ }, "compiled_path": "target/compiled/elementary/models/edr/alerts/alerts_schema_changes.sql", "compiled": true, - "compiled_code": "\n\n\nwith elementary_test_results as (\n select * from analytics.jaffle_shop_elementary.elementary_test_results\n),\n\nalerts_schema_changes as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n altimate_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where True and lower(status) != 'pass'and lower(status) != 'skipped'and test_type = 'schema_change'\n)\n\nselect * from alerts_schema_changes", + "compiled_code": "\n\n\nwith elementary_test_results as (\n select * from analytics.jaffle_shop_elementary.elementary_test_results\n),\n\nalerts_schema_changes as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n model_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where True and lower(status) != 'pass'and lower(status) != 'skipped'and test_type = 'schema_change'\n)\n\nselect * from alerts_schema_changes", "extra_ctes_injected": true, "extra_ctes": [], "contract": { @@ -2215,7 +2215,7 @@ }, "created_at": 1705031518.705749, "relation_name": "analytics.jaffle_shop_elementary.alerts_anomaly_detection", - "raw_code": "{{\n config(\n materialized = 'view',\n bind =False\n )\n}}\n\nwith elementary_test_results as (\n select * from {{ ref('elementary_test_results') }}\n),\n\nalerts_anomaly_detection as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n altimate_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where {{ not elementary.get_config_var('disable_test_alerts') }} and lower(status) != 'pass' {%- if elementary.get_config_var('disable_warn_alerts') -%} and lower(status) != 'warn' {%- endif -%} {%- if elementary.get_config_var('disable_skipped_test_alerts') -%} and lower(status) != 'skipped' {%- endif -%} and test_type = 'anomaly_detection'\n)\n\nselect * from alerts_anomaly_detection", + "raw_code": "{{\n config(\n materialized = 'view',\n bind =False\n )\n}}\n\nwith elementary_test_results as (\n select * from {{ ref('elementary_test_results') }}\n),\n\nalerts_anomaly_detection as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n model_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where {{ not elementary.get_config_var('disable_test_alerts') }} and lower(status) != 'pass' {%- if elementary.get_config_var('disable_warn_alerts') -%} and lower(status) != 'warn' {%- endif -%} {%- if elementary.get_config_var('disable_skipped_test_alerts') -%} and lower(status) != 'skipped' {%- endif -%} and test_type = 'anomaly_detection'\n)\n\nselect * from alerts_anomaly_detection", "language": "sql", "refs": [ { @@ -2236,7 +2236,7 @@ }, "compiled_path": "target/compiled/elementary/models/edr/alerts/alerts_anomaly_detection.sql", "compiled": true, - "compiled_code": "\n\nwith elementary_test_results as (\n select * from analytics.jaffle_shop_elementary.elementary_test_results\n),\n\nalerts_anomaly_detection as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n altimate_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where True and lower(status) != 'pass'and lower(status) != 'skipped'and test_type = 'anomaly_detection'\n)\n\nselect * from alerts_anomaly_detection", + "compiled_code": "\n\nwith elementary_test_results as (\n select * from analytics.jaffle_shop_elementary.elementary_test_results\n),\n\nalerts_anomaly_detection as (\n select id as alert_id,\n data_issue_id,\n test_execution_id,\n test_unique_id,\n model_unique_id,\n detected_at,\n database_name,\n schema_name,\n table_name,\n column_name,\n test_type as alert_type,\n test_sub_type as sub_type,\n test_results_description as alert_description,\n owners,\n tags,\n test_results_query as alert_results_query,\n other,\n test_name,\n test_short_name,\n test_params,\n severity,\n status,\n result_rows\n from elementary_test_results\n where True and lower(status) != 'pass'and lower(status) != 'skipped'and test_type = 'anomaly_detection'\n)\n\nselect * from alerts_anomaly_detection", "extra_ctes_injected": true, "extra_ctes": [], "contract": { @@ -2787,8 +2787,8 @@ "quote": null, "tags": [] }, - "parent_altimate_unique_id": { - "name": "parent_altimate_unique_id", + "parent_model_unique_id": { + "name": "parent_model_unique_id", "description": "", "meta": {}, "data_type": "string", @@ -2892,7 +2892,7 @@ }, "compiled_path": "target/compiled/elementary/models/edr/dbt_artifacts/dbt_tests.sql", "compiled": true, - "compiled_code": "\n\nselect * from (\n select\n \n \n cast('dummy_string' as varchar) as unique_id\n\n,\n \n cast('dummy_string' as varchar) as database_name\n\n,\n \n cast('dummy_string' as varchar) as schema_name\n\n,\n \n cast('dummy_string' as varchar) as name\n\n,\n \n cast('dummy_string' as varchar) as short_name\n\n,\n \n cast('dummy_string' as varchar) as alias\n\n,\n \n cast('dummy_string' as varchar) as test_column_name\n\n,\n \n cast('dummy_string' as varchar) as severity\n\n,\n \n cast('dummy_string' as varchar) as warn_if\n\n,\n \n cast('dummy_string' as varchar) as error_if\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_params\n\n,\n \n cast('dummy_string' as varchar) as test_namespace\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as tags\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as model_tags\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as model_owners\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as meta\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as depends_on_macros\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as depends_on_nodes\n\n,\n \n cast('dummy_string' as varchar) as parent_altimate_unique_id\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as description\n\n,\n \n cast('dummy_string' as varchar) as package_name\n\n,\n \n cast('dummy_string' as varchar) as type\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as original_path\n\n,\n \n cast('dummy_string' as varchar) as path\n\n,\n \n cast('dummy_string' as varchar) as generated_at\n\n,\n \n cast('dummy_string' as varchar) as metadata_hash\n\n,\n \n cast('dummy_string' as varchar) as quality_dimension\n\n\n ) as empty_table\n where 1 = 0", + "compiled_code": "\n\nselect * from (\n select\n \n \n cast('dummy_string' as varchar) as unique_id\n\n,\n \n cast('dummy_string' as varchar) as database_name\n\n,\n \n cast('dummy_string' as varchar) as schema_name\n\n,\n \n cast('dummy_string' as varchar) as name\n\n,\n \n cast('dummy_string' as varchar) as short_name\n\n,\n \n cast('dummy_string' as varchar) as alias\n\n,\n \n cast('dummy_string' as varchar) as test_column_name\n\n,\n \n cast('dummy_string' as varchar) as severity\n\n,\n \n cast('dummy_string' as varchar) as warn_if\n\n,\n \n cast('dummy_string' as varchar) as error_if\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as test_params\n\n,\n \n cast('dummy_string' as varchar) as test_namespace\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as tags\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as model_tags\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as model_owners\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as meta\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as depends_on_macros\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as depends_on_nodes\n\n,\n \n cast('dummy_string' as varchar) as parent_model_unique_id\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as description\n\n,\n \n cast('dummy_string' as varchar) as package_name\n\n,\n \n cast('dummy_string' as varchar) as type\n\n,\n \n cast('this_is_just_a_long_dummy_string' as varchar) as original_path\n\n,\n \n cast('dummy_string' as varchar) as path\n\n,\n \n cast('dummy_string' as varchar) as generated_at\n\n,\n \n cast('dummy_string' as varchar) as metadata_hash\n\n,\n \n cast('dummy_string' as varchar) as quality_dimension\n\n\n ) as empty_table\n where 1 = 0", "extra_ctes_injected": true, "extra_ctes": [], "contract": { @@ -22776,7 +22776,7 @@ "path": "macros/edr/materializations/test/test.sql", "original_file_path": "macros/edr/materializations/test/test.sql", "unique_id": "macro.elementary.get_anomaly_test_result_row", - "macro_sql": "{% macro get_anomaly_test_result_row(flattened_test, anomaly_scores_rows) %}\n {%- set latest_row = anomaly_scores_rows[-1] %}\n {%- set full_table_name = elementary.insensitive_get_dict_value(latest_row, 'full_table_name') %}\n {%- set test_unique_id = flattened_test.unique_id %}\n {%- set test_configuration = elementary.get_cache(test_unique_id) %}\n {%- set test_params = elementary.insensitive_get_dict_value(flattened_test, 'test_params') %}\n {%- do test_params.update(test_configuration) %}\n {%- set parent_altimate_unique_id = elementary.insensitive_get_dict_value(flattened_test, 'parent_altimate_unique_id') %}\n {%- set column_name = elementary.insensitive_get_dict_value(latest_row, 'column_name') %}\n {%- set metric_name = elementary.insensitive_get_dict_value(latest_row, 'metric_name') %}\n {%- set test_unique_id = elementary.insensitive_get_dict_value(latest_row, 'test_unique_id') %}\n {%- set has_anomaly_score = elementary.insensitive_get_dict_value(latest_row, 'anomaly_score') is not none %}\n {%- if not has_anomaly_score %}\n {% do elementary.edr_log(\"Not enough data to calculate anomaly scores on `{}`\".format(test_unique_id)) %}\n {% endif %}\n {%- set test_results_query -%}\n select * from ({{ sql }}) results\n where\n anomaly_score is not null and\n upper(full_table_name) = upper({{ elementary.const_as_string(full_table_name) }}) and\n metric_name = {{ elementary.const_as_string(metric_name) }}\n {%- if column_name %}\n and upper(column_name) = upper({{ elementary.const_as_string(column_name) }})\n {%- endif %}\n {%- endset -%}\n {% set test_results_description %}\n {% if has_anomaly_score %}\n {{ elementary.insensitive_get_dict_value(latest_row, 'anomaly_description') }}\n {% else %}\n Not enough data to calculate anomaly score.\n {% endif %}\n {% endset %}\n {% set failures = namespace(data=0) %}\n {% set filtered_anomaly_scores_rows = [] %}\n {% for row in anomaly_scores_rows %}\n {% if row.anomaly_score is not none %}\n {% do filtered_anomaly_scores_rows.append(row) %}\n {% if row.is_anomalous %}\n {% set failures.data = failures.data + 1 %}\n {% endif %}\n {% endif %}\n {% endfor %}\n {% set test_result_dict = {\n 'id': elementary.insensitive_get_dict_value(latest_row, 'id'),\n 'data_issue_id': elementary.insensitive_get_dict_value(latest_row, 'metric_id'),\n 'altimate_unique_id': parent_altimate_unique_id,\n 'column_name': column_name,\n 'test_type': 'anomaly_detection',\n 'test_sub_type': metric_name,\n 'test_results_description': test_results_description,\n 'other': elementary.insensitive_get_dict_value(latest_row, 'anomalous_value'),\n 'test_results_query': test_results_query,\n 'test_params': test_params,\n 'result_rows': filtered_anomaly_scores_rows,\n 'failures': failures.data\n } %}\n {% set elementary_test_row = elementary.get_dbt_test_result_row(flattened_test) %}\n {% do elementary_test_row.update(test_result_dict) %}\n {% do return(elementary_test_row) %}\n{% endmacro %}", + "macro_sql": "{% macro get_anomaly_test_result_row(flattened_test, anomaly_scores_rows) %}\n {%- set latest_row = anomaly_scores_rows[-1] %}\n {%- set full_table_name = elementary.insensitive_get_dict_value(latest_row, 'full_table_name') %}\n {%- set test_unique_id = flattened_test.unique_id %}\n {%- set test_configuration = elementary.get_cache(test_unique_id) %}\n {%- set test_params = elementary.insensitive_get_dict_value(flattened_test, 'test_params') %}\n {%- do test_params.update(test_configuration) %}\n {%- set parent_model_unique_id = elementary.insensitive_get_dict_value(flattened_test, 'parent_model_unique_id') %}\n {%- set column_name = elementary.insensitive_get_dict_value(latest_row, 'column_name') %}\n {%- set metric_name = elementary.insensitive_get_dict_value(latest_row, 'metric_name') %}\n {%- set test_unique_id = elementary.insensitive_get_dict_value(latest_row, 'test_unique_id') %}\n {%- set has_anomaly_score = elementary.insensitive_get_dict_value(latest_row, 'anomaly_score') is not none %}\n {%- if not has_anomaly_score %}\n {% do elementary.edr_log(\"Not enough data to calculate anomaly scores on `{}`\".format(test_unique_id)) %}\n {% endif %}\n {%- set test_results_query -%}\n select * from ({{ sql }}) results\n where\n anomaly_score is not null and\n upper(full_table_name) = upper({{ elementary.const_as_string(full_table_name) }}) and\n metric_name = {{ elementary.const_as_string(metric_name) }}\n {%- if column_name %}\n and upper(column_name) = upper({{ elementary.const_as_string(column_name) }})\n {%- endif %}\n {%- endset -%}\n {% set test_results_description %}\n {% if has_anomaly_score %}\n {{ elementary.insensitive_get_dict_value(latest_row, 'anomaly_description') }}\n {% else %}\n Not enough data to calculate anomaly score.\n {% endif %}\n {% endset %}\n {% set failures = namespace(data=0) %}\n {% set filtered_anomaly_scores_rows = [] %}\n {% for row in anomaly_scores_rows %}\n {% if row.anomaly_score is not none %}\n {% do filtered_anomaly_scores_rows.append(row) %}\n {% if row.is_anomalous %}\n {% set failures.data = failures.data + 1 %}\n {% endif %}\n {% endif %}\n {% endfor %}\n {% set test_result_dict = {\n 'id': elementary.insensitive_get_dict_value(latest_row, 'id'),\n 'data_issue_id': elementary.insensitive_get_dict_value(latest_row, 'metric_id'),\n 'model_unique_id': parent_model_unique_id,\n 'column_name': column_name,\n 'test_type': 'anomaly_detection',\n 'test_sub_type': metric_name,\n 'test_results_description': test_results_description,\n 'other': elementary.insensitive_get_dict_value(latest_row, 'anomalous_value'),\n 'test_results_query': test_results_query,\n 'test_params': test_params,\n 'result_rows': filtered_anomaly_scores_rows,\n 'failures': failures.data\n } %}\n {% set elementary_test_row = elementary.get_dbt_test_result_row(flattened_test) %}\n {% do elementary_test_row.update(test_result_dict) %}\n {% do return(elementary_test_row) %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.elementary.insensitive_get_dict_value", @@ -22828,7 +22828,7 @@ "path": "macros/edr/materializations/test/test.sql", "original_file_path": "macros/edr/materializations/test/test.sql", "unique_id": "macro.elementary.get_dbt_test_result_row", - "macro_sql": "{% macro get_dbt_test_result_row(flattened_test, result_rows=none) %}\n {% if not result_rows %}\n {% set result_rows = [] %}\n {% endif %}\n\n {% set test_execution_id = elementary.get_node_execution_id(flattened_test) %}\n {% set parent_altimate_unique_id = elementary.insensitive_get_dict_value(flattened_test, 'parent_altimate_unique_id') %}\n {% set parent_model = elementary.get_node(parent_altimate_unique_id) %}\n {% set parent_model_name = elementary.get_table_name_from_node(parent_model) %}\n {% set test_result_dict = {\n 'id': test_execution_id,\n 'data_issue_id': none,\n 'test_execution_id': test_execution_id,\n 'test_unique_id': elementary.insensitive_get_dict_value(flattened_test, 'unique_id'),\n 'altimate_unique_id': parent_altimate_unique_id,\n 'detected_at': elementary.insensitive_get_dict_value(flattened_test, 'generated_at'),\n 'database_name': elementary.insensitive_get_dict_value(flattened_test, 'database_name'),\n 'schema_name': elementary.insensitive_get_dict_value(flattened_test, 'schema_name'),\n 'table_name': parent_model_name,\n 'column_name': elementary.insensitive_get_dict_value(flattened_test, 'test_column_name'),\n 'test_type': elementary.get_test_type(flattened_test),\n 'test_sub_type': elementary.insensitive_get_dict_value(flattened_test, 'type'),\n 'other': none,\n 'owners': elementary.insensitive_get_dict_value(flattened_test, 'model_owners'),\n 'tags': elementary.insensitive_get_dict_value(flattened_test, 'model_tags') + elementary.insensitive_get_dict_value(flattened_test, 'tags'),\n 'test_results_query': elementary.get_compiled_code(flattened_test),\n 'test_name': elementary.insensitive_get_dict_value(flattened_test, 'name'),\n 'test_params': elementary.insensitive_get_dict_value(flattened_test, 'test_params'),\n 'severity': elementary.insensitive_get_dict_value(flattened_test, 'severity'),\n 'test_short_name': elementary.insensitive_get_dict_value(flattened_test, 'short_name'),\n 'test_alias': elementary.insensitive_get_dict_value(flattened_test, 'alias'),\n 'result_rows': result_rows\n }%}\n {% do return(test_result_dict) %}\n{% endmacro %}", + "macro_sql": "{% macro get_dbt_test_result_row(flattened_test, result_rows=none) %}\n {% if not result_rows %}\n {% set result_rows = [] %}\n {% endif %}\n\n {% set test_execution_id = elementary.get_node_execution_id(flattened_test) %}\n {% set parent_model_unique_id = elementary.insensitive_get_dict_value(flattened_test, 'parent_model_unique_id') %}\n {% set parent_model = elementary.get_node(parent_model_unique_id) %}\n {% set parent_model_name = elementary.get_table_name_from_node(parent_model) %}\n {% set test_result_dict = {\n 'id': test_execution_id,\n 'data_issue_id': none,\n 'test_execution_id': test_execution_id,\n 'test_unique_id': elementary.insensitive_get_dict_value(flattened_test, 'unique_id'),\n 'model_unique_id': parent_model_unique_id,\n 'detected_at': elementary.insensitive_get_dict_value(flattened_test, 'generated_at'),\n 'database_name': elementary.insensitive_get_dict_value(flattened_test, 'database_name'),\n 'schema_name': elementary.insensitive_get_dict_value(flattened_test, 'schema_name'),\n 'table_name': parent_model_name,\n 'column_name': elementary.insensitive_get_dict_value(flattened_test, 'test_column_name'),\n 'test_type': elementary.get_test_type(flattened_test),\n 'test_sub_type': elementary.insensitive_get_dict_value(flattened_test, 'type'),\n 'other': none,\n 'owners': elementary.insensitive_get_dict_value(flattened_test, 'model_owners'),\n 'tags': elementary.insensitive_get_dict_value(flattened_test, 'model_tags') + elementary.insensitive_get_dict_value(flattened_test, 'tags'),\n 'test_results_query': elementary.get_compiled_code(flattened_test),\n 'test_name': elementary.insensitive_get_dict_value(flattened_test, 'name'),\n 'test_params': elementary.insensitive_get_dict_value(flattened_test, 'test_params'),\n 'severity': elementary.insensitive_get_dict_value(flattened_test, 'severity'),\n 'test_short_name': elementary.insensitive_get_dict_value(flattened_test, 'short_name'),\n 'test_alias': elementary.insensitive_get_dict_value(flattened_test, 'alias'),\n 'result_rows': result_rows\n }%}\n {% do return(test_result_dict) %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.elementary.get_node_execution_id", @@ -26480,7 +26480,7 @@ "path": "macros/edr/system/system_utils/empty_table.sql", "original_file_path": "macros/edr/system/system_utils/empty_table.sql", "unique_id": "macro.elementary.empty_elementary_test_results", - "macro_sql": "{% macro empty_elementary_test_results() %}\n {{ elementary.empty_table([\n ('id','long_string'),\n ('data_issue_id','string'),\n ('test_execution_id','long_string'),\n ('test_unique_id','long_string'),\n ('altimate_unique_id','long_string'),\n ('invocation_id', 'string'),\n ('detected_at','timestamp'),\n ('created_at','timestamp'),\n ('database_name','string'),\n ('schema_name','string'),\n ('table_name','string'),\n ('column_name','string'),\n ('test_type','string'),\n ('test_sub_type','string'),\n ('test_results_description','long_string'),\n ('owners','string'),\n ('tags','string'),\n ('test_results_query','long_string'),\n ('other','string'),\n ('test_name','long_string'),\n ('test_params','long_string'),\n ('severity','string'),\n ('status','string'),\n ('failures', 'bigint'),\n ('test_short_name', 'string'),\n ('test_alias', 'string'),\n ('result_rows', 'long_string'),\n ('failed_row_count', 'bigint')\n ]) }}\n{% endmacro %}", + "macro_sql": "{% macro empty_elementary_test_results() %}\n {{ elementary.empty_table([\n ('id','long_string'),\n ('data_issue_id','string'),\n ('test_execution_id','long_string'),\n ('test_unique_id','long_string'),\n ('model_unique_id','long_string'),\n ('invocation_id', 'string'),\n ('detected_at','timestamp'),\n ('created_at','timestamp'),\n ('database_name','string'),\n ('schema_name','string'),\n ('table_name','string'),\n ('column_name','string'),\n ('test_type','string'),\n ('test_sub_type','string'),\n ('test_results_description','long_string'),\n ('owners','string'),\n ('tags','string'),\n ('test_results_query','long_string'),\n ('other','string'),\n ('test_name','long_string'),\n ('test_params','long_string'),\n ('severity','string'),\n ('status','string'),\n ('failures', 'bigint'),\n ('test_short_name', 'string'),\n ('test_alias', 'string'),\n ('result_rows', 'long_string'),\n ('failed_row_count', 'bigint')\n ]) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.elementary.empty_table" @@ -28565,7 +28565,7 @@ "path": "macros/edr/dbt_artifacts/upload_dbt_tests.sql", "original_file_path": "macros/edr/dbt_artifacts/upload_dbt_tests.sql", "unique_id": "macro.elementary.get_dbt_tests_empty_table_query", - "macro_sql": "{% macro get_dbt_tests_empty_table_query() %}\n {% set dbt_tests_empty_table_query = elementary.empty_table([('unique_id', 'string'),\n ('database_name', 'string'),\n ('schema_name', 'string'),\n ('name', 'string'),\n ('short_name', 'string'),\n ('alias', 'string'),\n ('test_column_name', 'string'),\n ('severity', 'string'),\n ('warn_if', 'string'),\n ('error_if', 'string'),\n ('test_params', 'long_string'),\n ('test_namespace', 'string'),\n ('tags', 'long_string'),\n ('model_tags', 'long_string'),\n ('model_owners', 'long_string'),\n ('meta', 'long_string'),\n ('depends_on_macros', 'long_string'),\n ('depends_on_nodes', 'long_string'),\n ('parent_altimate_unique_id', 'string'),\n ('description', 'long_string'),\n ('package_name', 'string'),\n ('type', 'string'),\n ('original_path', 'long_string'),\n ('path', 'string'),\n ('generated_at', 'string'),\n ('metadata_hash', 'string'),\n ('quality_dimension', 'string')\n ]) %}\n {{ return(dbt_tests_empty_table_query) }}\n{% endmacro %}", + "macro_sql": "{% macro get_dbt_tests_empty_table_query() %}\n {% set dbt_tests_empty_table_query = elementary.empty_table([('unique_id', 'string'),\n ('database_name', 'string'),\n ('schema_name', 'string'),\n ('name', 'string'),\n ('short_name', 'string'),\n ('alias', 'string'),\n ('test_column_name', 'string'),\n ('severity', 'string'),\n ('warn_if', 'string'),\n ('error_if', 'string'),\n ('test_params', 'long_string'),\n ('test_namespace', 'string'),\n ('tags', 'long_string'),\n ('model_tags', 'long_string'),\n ('model_owners', 'long_string'),\n ('meta', 'long_string'),\n ('depends_on_macros', 'long_string'),\n ('depends_on_nodes', 'long_string'),\n ('parent_model_unique_id', 'string'),\n ('description', 'long_string'),\n ('package_name', 'string'),\n ('type', 'string'),\n ('original_path', 'long_string'),\n ('path', 'string'),\n ('generated_at', 'string'),\n ('metadata_hash', 'string'),\n ('quality_dimension', 'string')\n ]) %}\n {{ return(dbt_tests_empty_table_query) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.elementary.empty_table" @@ -28589,14 +28589,14 @@ "path": "macros/edr/dbt_artifacts/upload_dbt_tests.sql", "original_file_path": "macros/edr/dbt_artifacts/upload_dbt_tests.sql", "unique_id": "macro.elementary.flatten_test", - "macro_sql": "{% macro flatten_test(node_dict) %}\n {% set config_dict = elementary.safe_get_with_default(node_dict, 'config', {}) %}\n {% set depends_on_dict = elementary.safe_get_with_default(node_dict, 'depends_on', {}) %}\n\n {% set test_metadata = elementary.safe_get_with_default(node_dict, 'test_metadata', {}) %}\n {% set test_namespace = test_metadata.get('namespace') %}\n {% set test_short_name = elementary.get_test_short_name(node_dict, test_metadata) %}\n\n {% set default_description = elementary.get_default_description(test_short_name, test_namespace) %}\n\n {% set config_meta_dict = elementary.safe_get_with_default(config_dict, 'meta', {}) %}\n {% set meta_dict = {} %}\n {% if default_description %}\n {% set meta_dict = {'description': default_description} %} \n {% endif %}\n {% do meta_dict.update(elementary.safe_get_with_default(node_dict, 'meta', {})) %}\n {% do meta_dict.update(config_meta_dict) %}\n\n {% set config_tags = elementary.safe_get_with_default(config_dict, 'tags', []) %}\n {% set global_tags = elementary.safe_get_with_default(node_dict, 'tags', []) %}\n {% set meta_tags = elementary.safe_get_with_default(meta_dict, 'tags', []) %}\n {% set tags = elementary.union_lists(config_tags, global_tags) %}\n {% set tags = elementary.union_lists(tags, meta_tags) %}\n\n {% set test_altimate_unique_ids = elementary.get_parent_altimate_unique_ids_from_test_node(node_dict) %}\n {% set test_model_nodes = elementary.get_nodes_by_unique_ids(test_altimate_unique_ids) %}\n {% set test_models_owners = [] %}\n {% set test_models_tags = [] %}\n {% for test_model_node in test_model_nodes %}\n {% set flatten_test_model_node = elementary.flatten_model(test_model_node) %}\n {% set test_model_owner = flatten_test_model_node.get('owner') %}\n {% if test_model_owner %}\n {% if test_model_owner is string %}\n {% set owners = test_model_owner.split(',') %}\n {% for owner in owners %}\n {% do test_models_owners.append(owner | trim) %} \n {% endfor %}\n {% elif test_model_owner is iterable %}\n {% do test_models_owners.extend(test_model_owner) %}\n {% endif %}\n {% endif %}\n {% set test_model_tags = flatten_test_model_node.get('tags') %}\n {% if test_model_tags and test_model_tags is sequence %}\n {% do test_models_tags.extend(test_model_tags) %}\n {% endif %}\n {% endfor %}\n {% set test_models_owners = test_models_owners | unique | list %}\n {% set test_models_tags = test_models_tags | unique | list %}\n\n {% set test_kwargs = elementary.safe_get_with_default(test_metadata, 'kwargs', {}) %}\n\n {% set primary_test_model_id = namespace(data=none) %}\n {% if test_altimate_unique_ids | length == 1 %}\n {# if only one parent model for this test, simply use this model #}\n {% set primary_test_model_id.data = test_altimate_unique_ids[0] %}\n {% else %}\n {% set test_model_jinja = test_kwargs.get('model') %}\n {% if test_model_jinja %}\n {% set test_model_name_matches =\n modules.re.findall(\"ref\\(['\\\"](\\w+)['\\\"]\\)\", test_model_jinja) +\n modules.re.findall(\"source\\(['\\\"]\\w+['\\\"], ['\\\"](\\w+)['\\\"]\\)\", test_model_jinja) %}\n {% if test_model_name_matches | length == 1 %}\n {% set test_model_name = test_model_name_matches[0] %}\n {% for test_altimate_unique_id in test_altimate_unique_ids %}\n {% set split_test_altimate_unique_id = test_altimate_unique_id.split('.') %}\n {% if split_test_altimate_unique_id and split_test_altimate_unique_id | length > 0 %}\n {% set test_node_model_name = split_test_altimate_unique_id[-1] %}\n {% if test_node_model_name == test_model_name %}\n {% set primary_test_model_id.data = test_altimate_unique_id %}\n {% endif %}\n {% endif %}\n {% endfor %}\n {% endif %}\n {% endif %}\n {% endif %}\n\n {% set primary_test_model_database = none %}\n {% set primary_test_model_schema = none %}\n {%- if primary_test_model_id.data is not none -%}\n {% set tested_model_node = elementary.get_node(primary_test_model_id.data) %}\n {%- if tested_model_node -%}\n {% set primary_test_model_database = tested_model_node.get('database') %}\n {% set primary_test_model_schema = tested_model_node.get('schema') %}\n {%- endif -%}\n {%- endif -%}\n\n {%- if primary_test_model_database is none or primary_test_model_schema is none -%}\n {# This is mainly here to support singular test cases with multiple referred models, in this case the tested node is being used to extract the db and schema #}\n {% set primary_test_model_database, primary_test_model_schema = elementary.get_model_database_and_schema_from_test_node(node_dict) %}\n {%- endif -%}\n\n {% set original_file_path = node_dict.get('original_file_path') %}\n {% set flatten_test_metadata_dict = {\n 'unique_id': node_dict.get('unique_id'),\n 'short_name': test_short_name,\n 'alias': node_dict.get('alias'),\n 'test_column_name': node_dict.get('column_name'),\n 'severity': config_dict.get('severity'),\n 'warn_if': config_dict.get('warn_if'),\n 'error_if': config_dict.get('error_if'),\n 'test_params': test_kwargs,\n 'test_namespace': test_namespace,\n 'tags': elementary.filter_none_and_sort(tags),\n 'model_tags': elementary.filter_none_and_sort(test_models_tags),\n 'model_owners': elementary.filter_none_and_sort(test_models_owners),\n 'meta': meta_dict,\n 'database_name': primary_test_model_database,\n 'schema_name': primary_test_model_schema,\n 'depends_on_macros': elementary.filter_none_and_sort(depends_on_dict.get('macros', [])),\n 'depends_on_nodes': elementary.filter_none_and_sort(depends_on_dict.get('nodes', [])),\n 'parent_altimate_unique_id': primary_test_model_id.data,\n 'description': meta_dict.get('description'),\n 'name': node_dict.get('name'),\n 'package_name': node_dict.get('package_name'),\n 'type': elementary.get_test_sub_type(original_file_path, test_namespace),\n 'original_path': original_file_path,\n 'compiled_code': elementary.get_compiled_code(node_dict),\n 'path': node_dict.get('path'),\n 'generated_at': elementary.datetime_now_utc_as_string(),\n 'quality_dimension': meta_dict.get('quality_dimension') or elementary.get_quality_dimension(test_short_name, test_namespace)\n }%}\n {% do flatten_test_metadata_dict.update({\"metadata_hash\": elementary.get_artifact_metadata_hash(flatten_test_metadata_dict)}) %}\n {{ return(flatten_test_metadata_dict) }}\n{% endmacro %}", + "macro_sql": "{% macro flatten_test(node_dict) %}\n {% set config_dict = elementary.safe_get_with_default(node_dict, 'config', {}) %}\n {% set depends_on_dict = elementary.safe_get_with_default(node_dict, 'depends_on', {}) %}\n\n {% set test_metadata = elementary.safe_get_with_default(node_dict, 'test_metadata', {}) %}\n {% set test_namespace = test_metadata.get('namespace') %}\n {% set test_short_name = elementary.get_test_short_name(node_dict, test_metadata) %}\n\n {% set default_description = elementary.get_default_description(test_short_name, test_namespace) %}\n\n {% set config_meta_dict = elementary.safe_get_with_default(config_dict, 'meta', {}) %}\n {% set meta_dict = {} %}\n {% if default_description %}\n {% set meta_dict = {'description': default_description} %} \n {% endif %}\n {% do meta_dict.update(elementary.safe_get_with_default(node_dict, 'meta', {})) %}\n {% do meta_dict.update(config_meta_dict) %}\n\n {% set config_tags = elementary.safe_get_with_default(config_dict, 'tags', []) %}\n {% set global_tags = elementary.safe_get_with_default(node_dict, 'tags', []) %}\n {% set meta_tags = elementary.safe_get_with_default(meta_dict, 'tags', []) %}\n {% set tags = elementary.union_lists(config_tags, global_tags) %}\n {% set tags = elementary.union_lists(tags, meta_tags) %}\n\n {% set test_model_unique_ids = elementary.get_parent_model_unique_ids_from_test_node(node_dict) %}\n {% set test_model_nodes = elementary.get_nodes_by_unique_ids(test_model_unique_ids) %}\n {% set test_models_owners = [] %}\n {% set test_models_tags = [] %}\n {% for test_model_node in test_model_nodes %}\n {% set flatten_test_model_node = elementary.flatten_model(test_model_node) %}\n {% set test_model_owner = flatten_test_model_node.get('owner') %}\n {% if test_model_owner %}\n {% if test_model_owner is string %}\n {% set owners = test_model_owner.split(',') %}\n {% for owner in owners %}\n {% do test_models_owners.append(owner | trim) %} \n {% endfor %}\n {% elif test_model_owner is iterable %}\n {% do test_models_owners.extend(test_model_owner) %}\n {% endif %}\n {% endif %}\n {% set test_model_tags = flatten_test_model_node.get('tags') %}\n {% if test_model_tags and test_model_tags is sequence %}\n {% do test_models_tags.extend(test_model_tags) %}\n {% endif %}\n {% endfor %}\n {% set test_models_owners = test_models_owners | unique | list %}\n {% set test_models_tags = test_models_tags | unique | list %}\n\n {% set test_kwargs = elementary.safe_get_with_default(test_metadata, 'kwargs', {}) %}\n\n {% set primary_test_model_id = namespace(data=none) %}\n {% if test_model_unique_ids | length == 1 %}\n {# if only one parent model for this test, simply use this model #}\n {% set primary_test_model_id.data = test_model_unique_ids[0] %}\n {% else %}\n {% set test_model_jinja = test_kwargs.get('model') %}\n {% if test_model_jinja %}\n {% set test_model_name_matches =\n modules.re.findall(\"ref\\(['\\\"](\\w+)['\\\"]\\)\", test_model_jinja) +\n modules.re.findall(\"source\\(['\\\"]\\w+['\\\"], ['\\\"](\\w+)['\\\"]\\)\", test_model_jinja) %}\n {% if test_model_name_matches | length == 1 %}\n {% set test_model_name = test_model_name_matches[0] %}\n {% for test_model_unique_id in test_model_unique_ids %}\n {% set split_test_model_unique_id = test_model_unique_id.split('.') %}\n {% if split_test_model_unique_id and split_test_model_unique_id | length > 0 %}\n {% set test_node_model_name = split_test_model_unique_id[-1] %}\n {% if test_node_model_name == test_model_name %}\n {% set primary_test_model_id.data = test_model_unique_id %}\n {% endif %}\n {% endif %}\n {% endfor %}\n {% endif %}\n {% endif %}\n {% endif %}\n\n {% set primary_test_model_database = none %}\n {% set primary_test_model_schema = none %}\n {%- if primary_test_model_id.data is not none -%}\n {% set tested_model_node = elementary.get_node(primary_test_model_id.data) %}\n {%- if tested_model_node -%}\n {% set primary_test_model_database = tested_model_node.get('database') %}\n {% set primary_test_model_schema = tested_model_node.get('schema') %}\n {%- endif -%}\n {%- endif -%}\n\n {%- if primary_test_model_database is none or primary_test_model_schema is none -%}\n {# This is mainly here to support singular test cases with multiple referred models, in this case the tested node is being used to extract the db and schema #}\n {% set primary_test_model_database, primary_test_model_schema = elementary.get_model_database_and_schema_from_test_node(node_dict) %}\n {%- endif -%}\n\n {% set original_file_path = node_dict.get('original_file_path') %}\n {% set flatten_test_metadata_dict = {\n 'unique_id': node_dict.get('unique_id'),\n 'short_name': test_short_name,\n 'alias': node_dict.get('alias'),\n 'test_column_name': node_dict.get('column_name'),\n 'severity': config_dict.get('severity'),\n 'warn_if': config_dict.get('warn_if'),\n 'error_if': config_dict.get('error_if'),\n 'test_params': test_kwargs,\n 'test_namespace': test_namespace,\n 'tags': elementary.filter_none_and_sort(tags),\n 'model_tags': elementary.filter_none_and_sort(test_models_tags),\n 'model_owners': elementary.filter_none_and_sort(test_models_owners),\n 'meta': meta_dict,\n 'database_name': primary_test_model_database,\n 'schema_name': primary_test_model_schema,\n 'depends_on_macros': elementary.filter_none_and_sort(depends_on_dict.get('macros', [])),\n 'depends_on_nodes': elementary.filter_none_and_sort(depends_on_dict.get('nodes', [])),\n 'parent_model_unique_id': primary_test_model_id.data,\n 'description': meta_dict.get('description'),\n 'name': node_dict.get('name'),\n 'package_name': node_dict.get('package_name'),\n 'type': elementary.get_test_sub_type(original_file_path, test_namespace),\n 'original_path': original_file_path,\n 'compiled_code': elementary.get_compiled_code(node_dict),\n 'path': node_dict.get('path'),\n 'generated_at': elementary.datetime_now_utc_as_string(),\n 'quality_dimension': meta_dict.get('quality_dimension') or elementary.get_quality_dimension(test_short_name, test_namespace)\n }%}\n {% do flatten_test_metadata_dict.update({\"metadata_hash\": elementary.get_artifact_metadata_hash(flatten_test_metadata_dict)}) %}\n {{ return(flatten_test_metadata_dict) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.elementary.safe_get_with_default", "macro.elementary.get_test_short_name", "macro.elementary.get_default_description", "macro.elementary.union_lists", - "macro.elementary.get_parent_altimate_unique_ids_from_test_node", + "macro.elementary.get_parent_model_unique_ids_from_test_node", "macro.elementary.get_nodes_by_unique_ids", "macro.elementary.flatten_model", "macro.elementary.get_node", @@ -34954,14 +34954,14 @@ "created_at": 1705031517.168567, "supported_languages": null }, - "macro.elementary.get_parent_altimate_unique_ids_from_test_node": { - "name": "get_parent_altimate_unique_ids_from_test_node", + "macro.elementary.get_parent_model_unique_ids_from_test_node": { + "name": "get_parent_model_unique_ids_from_test_node", "resource_type": "macro", "package_name": "elementary", - "path": "macros/utils/graph/get_parent_altimate_unique_ids_from_test_node.sql", - "original_file_path": "macros/utils/graph/get_parent_altimate_unique_ids_from_test_node.sql", - "unique_id": "macro.elementary.get_parent_altimate_unique_ids_from_test_node", - "macro_sql": "{% macro get_parent_altimate_unique_ids_from_test_node(test_node) %}\n {% set nodes_in_current_package = [] %}\n {% set test_depends_on = test_node.get('depends_on') %}\n {% if test_depends_on %}\n {% set depends_on_nodes = test_depends_on.get('nodes') %}\n {% if depends_on_nodes %}\n {% set current_package_name = test_node.get('package_name') %}\n {% if current_package_name %}\n {% set current_package_name = '.' ~ current_package_name ~ '.' %}\n {% for node in depends_on_nodes %}\n {% if current_package_name in node %}\n {% do nodes_in_current_package.append(node) %}\n {% endif %}\n {% endfor %}\n {% endif %}\n {% endif %}\n {% endif %}\n {{ return(nodes_in_current_package) }}\n{% endmacro %}", + "path": "macros/utils/graph/get_parent_model_unique_ids_from_test_node.sql", + "original_file_path": "macros/utils/graph/get_parent_model_unique_ids_from_test_node.sql", + "unique_id": "macro.elementary.get_parent_model_unique_ids_from_test_node", + "macro_sql": "{% macro get_parent_model_unique_ids_from_test_node(test_node) %}\n {% set nodes_in_current_package = [] %}\n {% set test_depends_on = test_node.get('depends_on') %}\n {% if test_depends_on %}\n {% set depends_on_nodes = test_depends_on.get('nodes') %}\n {% if depends_on_nodes %}\n {% set current_package_name = test_node.get('package_name') %}\n {% if current_package_name %}\n {% set current_package_name = '.' ~ current_package_name ~ '.' %}\n {% for node in depends_on_nodes %}\n {% if current_package_name in node %}\n {% do nodes_in_current_package.append(node) %}\n {% endif %}\n {% endfor %}\n {% endif %}\n {% endif %}\n {% endif %}\n {{ return(nodes_in_current_package) }}\n{% endmacro %}", "depends_on": { "macros": [] }, @@ -37242,7 +37242,7 @@ "path": "macros/commands/delete_duplicate_rows.sql", "original_file_path": "macros/commands/delete_duplicate_rows.sql", "unique_id": "macro.elementary.delete_duplicate_rows", - "macro_sql": "{% macro delete_duplicate_rows(altimate_unique_id) %}\n {% do return(adapter.dispatch(\"delete_duplicate_rows\", \"elementary\")(altimate_unique_id)) %}\n{% endmacro %}", + "macro_sql": "{% macro delete_duplicate_rows(model_unique_id) %}\n {% do return(adapter.dispatch(\"delete_duplicate_rows\", \"elementary\")(model_unique_id)) %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.elementary.default__delete_duplicate_rows" @@ -37266,7 +37266,7 @@ "path": "macros/commands/delete_duplicate_rows.sql", "original_file_path": "macros/commands/delete_duplicate_rows.sql", "unique_id": "macro.elementary.postgres__delete_duplicate_rows", - "macro_sql": "{% macro postgres__delete_duplicate_rows(altimate_unique_id) %}\n {% set node = graph.nodes[altimate_unique_id] %}\n {% set relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) %}\n {% if relation is none %}\n {% do print(\"Relation '{}' does not exist.\".format(node.relation_name)) %}\n {% do return([]) %}\n {% endif %}\n\n {% set column_names = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | map(\"lower\") | list %}\n\n {% set query %}\n DELETE FROM {{ relation }} AS t1\n USING {{ relation }} AS t2\n WHERE t1.ctid < t2.ctid\n {% for col in column_names %}\n AND t1.{{ col }} = t2.{{ col }}\n {% endfor %}\n {% endset %}\n {% do elementary.run_query(query) %}\n {% do adapter.commit() %}\n{% endmacro %}", + "macro_sql": "{% macro postgres__delete_duplicate_rows(model_unique_id) %}\n {% set node = graph.nodes[model_unique_id] %}\n {% set relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) %}\n {% if relation is none %}\n {% do print(\"Relation '{}' does not exist.\".format(node.relation_name)) %}\n {% do return([]) %}\n {% endif %}\n\n {% set column_names = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | map(\"lower\") | list %}\n\n {% set query %}\n DELETE FROM {{ relation }} AS t1\n USING {{ relation }} AS t2\n WHERE t1.ctid < t2.ctid\n {% for col in column_names %}\n AND t1.{{ col }} = t2.{{ col }}\n {% endfor %}\n {% endset %}\n {% do elementary.run_query(query) %}\n {% do adapter.commit() %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.elementary.run_query" @@ -37290,7 +37290,7 @@ "path": "macros/commands/delete_duplicate_rows.sql", "original_file_path": "macros/commands/delete_duplicate_rows.sql", "unique_id": "macro.elementary.default__delete_duplicate_rows", - "macro_sql": "{% macro default__delete_duplicate_rows(altimate_unique_id) %}\n {{ exceptions.raise_compiler_error(\"This macro is not supported on '{}'.\".format(target.type)) }}\n{% endmacro %}", + "macro_sql": "{% macro default__delete_duplicate_rows(model_unique_id) %}\n {{ exceptions.raise_compiler_error(\"This macro is not supported on '{}'.\".format(target.type)) }}\n{% endmacro %}", "depends_on": { "macros": [] }, @@ -37483,7 +37483,7 @@ "path": "macros/commands/dump_table.sql", "original_file_path": "macros/commands/dump_table.sql", "unique_id": "macro.elementary.dump_table", - "macro_sql": "{% macro dump_table(altimate_unique_id, output_path, exclude_deprecated_columns=true, timestamp_column=none, since=none, days_back=7, dedup=false) %}\n {% set node = graph.nodes.get(altimate_unique_id) %}\n {% if not node %}\n {% do print(\"Node '{}' does not exist.\".format(altimate_unique_id)) %}\n {% do return([]) %}\n {% endif %}\n\n {% set relation = api.Relation.create(database=node.database, schema=node.schema, identifier=node.alias) %}\n {% set column_names = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | map(\"lower\") | list %}\n {% if not column_names %}\n {% do print(\"Relation '{}' does not exist.\".format(node.relation_name)) %}\n {% do return([]) %}\n {% endif %}\n\n {% if exclude_deprecated_columns %}\n {% set deprecated_column_names = node.meta.get(\"deprecated_columns\", []) | map(attribute=\"name\") | map(\"lower\") | list %}\n {% set column_names = column_names | reject(\"in\", deprecated_column_names) | list %}\n {% endif %}\n\n {% set dedup_by_column = node.meta.dedup_by_column or \"unique_id\" %}\n {% set order_by_dedup_column = \"generated_at\" %}\n {% set query %}\n {% if dedup and (dedup_by_column in column_names) and (order_by_dedup_column in column_names) %}\n {{ elementary.dedup_by_column_query(dedup_by_column, order_by_dedup_column, column_names, relation) }}\n {% else %}\n select {{ elementary.escape_select(column_names) }} from {{ relation }}\n {% endif %}\n {% if timestamp_column %}\n {% if since %}\n where {{ elementary.edr_cast_as_timestamp(timestamp_column) }} > {{ elementary.edr_cast_as_timestamp(elementary.edr_quote(since)) }}\n {% else %}\n where {{ elementary.edr_datediff(elementary.edr_cast_as_timestamp(timestamp_column), elementary.edr_current_timestamp(), 'day') }} < {{ days_back }}\n {% endif %}\n {% endif %}\n {% endset %}\n {% set results = elementary.run_query(query) %}\n {% do results.to_csv(output_path) %}\n {% do return(results.column_names) %}\n{% endmacro %}", + "macro_sql": "{% macro dump_table(model_unique_id, output_path, exclude_deprecated_columns=true, timestamp_column=none, since=none, days_back=7, dedup=false) %}\n {% set node = graph.nodes.get(model_unique_id) %}\n {% if not node %}\n {% do print(\"Node '{}' does not exist.\".format(model_unique_id)) %}\n {% do return([]) %}\n {% endif %}\n\n {% set relation = api.Relation.create(database=node.database, schema=node.schema, identifier=node.alias) %}\n {% set column_names = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | map(\"lower\") | list %}\n {% if not column_names %}\n {% do print(\"Relation '{}' does not exist.\".format(node.relation_name)) %}\n {% do return([]) %}\n {% endif %}\n\n {% if exclude_deprecated_columns %}\n {% set deprecated_column_names = node.meta.get(\"deprecated_columns\", []) | map(attribute=\"name\") | map(\"lower\") | list %}\n {% set column_names = column_names | reject(\"in\", deprecated_column_names) | list %}\n {% endif %}\n\n {% set dedup_by_column = node.meta.dedup_by_column or \"unique_id\" %}\n {% set order_by_dedup_column = \"generated_at\" %}\n {% set query %}\n {% if dedup and (dedup_by_column in column_names) and (order_by_dedup_column in column_names) %}\n {{ elementary.dedup_by_column_query(dedup_by_column, order_by_dedup_column, column_names, relation) }}\n {% else %}\n select {{ elementary.escape_select(column_names) }} from {{ relation }}\n {% endif %}\n {% if timestamp_column %}\n {% if since %}\n where {{ elementary.edr_cast_as_timestamp(timestamp_column) }} > {{ elementary.edr_cast_as_timestamp(elementary.edr_quote(since)) }}\n {% else %}\n where {{ elementary.edr_datediff(elementary.edr_cast_as_timestamp(timestamp_column), elementary.edr_current_timestamp(), 'day') }} < {{ days_back }}\n {% endif %}\n {% endif %}\n {% endset %}\n {% set results = elementary.run_query(query) %}\n {% do results.to_csv(output_path) %}\n {% do return(results.column_names) %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.elementary.dedup_by_column_query",