Skip to content

Commit

Permalink
revert altimate_unique_id
Browse files Browse the repository at this point in the history
  • Loading branch information
gaurpulkit committed May 31, 2024
1 parent ec7401d commit 3ca0ecd
Show file tree
Hide file tree
Showing 34 changed files with 228 additions and 230 deletions.
2 changes: 1 addition & 1 deletion src/datapilot/core/insights/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ def generate_ci_cd_report(insights_data):
for insight in insights_data:
print(divider)
print(f"Project: {insight.package_name}")
print(f"Model ID: {insight.altimate_unique_id}")
print(f"Model ID: {insight.model_unique_id}")
print(f"Name: {insight.metadata['model']}")
print(f"Message: {insight.message}")
print(f"Reason: {insight.reason_to_flag}")
Expand Down
4 changes: 2 additions & 2 deletions src/datapilot/core/platforms/dbt/insights/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,10 +108,10 @@ def build_chain(node_id, current_chain):

return long_chains

def should_skip_model(self, altimate_unique_id):
def should_skip_model(self, model_unique_id):
"""Check if a model is in the excluded models list."""
if self.selected_models:
return altimate_unique_id not in self.selected_models
return model_unique_id not in self.selected_models

return False

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,11 @@ class CheckColumnNameContract(ChecksInsight):
"modeling and analysis. It's important to maintain consistent column naming conventions."
)
FAILURE_MESSAGE = (
"The following columns in the model `{altimate_unique_id}` do not adhere to the contract:\n{columns}. "
"The following columns in the model `{model_unique_id}` do not adhere to the contract:\n{columns}. "
"Inconsistent column naming conventions can impede understanding and usage of the model."
)
RECOMMENDATION = (
"Update the column names listed above in the model `{altimate_unique_id}` to adhere to the contract. "
"Update the column names listed above in the model `{model_unique_id}` to adhere to the contract. "
"Consistent column naming conventions provide valuable context and aids in data understanding and collaboration."
)
PATTERN_STR = "pattern"
Expand Down Expand Up @@ -75,20 +75,20 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]:
)
return insights

def _build_failure_result(self, altimate_unique_id: str, columns: Sequence[str]) -> DBTInsightResult:
def _build_failure_result(self, model_unique_id: str, columns: Sequence[str]) -> DBTInsightResult:
failure_message = self.FAILURE_MESSAGE.format(
columns=numbered_list(columns),
altimate_unique_id=altimate_unique_id,
model_unique_id=model_unique_id,
)
recommendation = self.RECOMMENDATION.format(altimate_unique_id=altimate_unique_id)
recommendation = self.RECOMMENDATION.format(model_unique_id=model_unique_id)

return DBTInsightResult(
type=self.TYPE,
name=self.NAME,
message=failure_message,
recommendation=recommendation,
reason_to_flag=self.REASON_TO_FLAG,
metadata={"columns": columns, "altimate_unique_id": altimate_unique_id},
metadata={"columns": columns, "model_unique_id": model_unique_id},
)

def _get_columns_in_model(self, node_id) -> List[str]:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,13 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]:
)
return insights

def _build_failure_result(self, altimate_unique_id: str, columns: Sequence[str]) -> DBTInsightResult:
def _build_failure_result(self, model_unique_id: str, columns: Sequence[str]) -> DBTInsightResult:
failure_message = (
"The following columns in the model `{altimate_unique_id}` are missing:\n{columns}. "
"The following columns in the model `{model_unique_id}` are missing:\n{columns}. "
"Ensure that the model includes all the required columns."
)
recommendation = (
"Add the missing columns listed above in the model `{altimate_unique_id}`. "
"Add the missing columns listed above in the model `{model_unique_id}`. "
"Ensuring that the model has all the required columns helps in maintaining data integrity and consistency."
)

Expand All @@ -63,11 +63,11 @@ def _build_failure_result(self, altimate_unique_id: str, columns: Sequence[str])
name=self.NAME,
message=failure_message.format(
columns=numbered_list(columns),
altimate_unique_id=altimate_unique_id,
model_unique_id=model_unique_id,
),
recommendation=recommendation.format(altimate_unique_id=altimate_unique_id),
recommendation=recommendation.format(model_unique_id=model_unique_id),
reason_to_flag=self.REASON_TO_FLAG,
metadata={"columns": columns, "altimate_unique_id": altimate_unique_id},
metadata={"columns": columns, "model_unique_id": model_unique_id},
)

def _check_model_columns(self, node_id) -> Tuple[int, Set[str]]:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,18 +46,18 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]:
)
return insights

def _build_failure_result(self, altimate_unique_id: str, missing_keys: Sequence[str], extra_labels: Sequence[str]) -> DBTInsightResult:
def _build_failure_result(self, model_unique_id: str, missing_keys: Sequence[str], extra_labels: Sequence[str]) -> DBTInsightResult:
failure_message = (
f"The following labels keys are missing in the model `{altimate_unique_id}`:\n{missing_keys}. "
f"The following labels keys are missing in the model `{model_unique_id}`:\n{missing_keys}. "
"Ensure that the model includes all the required labels keys."
)
if not self.allow_extra_keys:
failure_message += (
f"The following extra labels keys are present in the model `{altimate_unique_id}`:\n{extra_labels}. "
f"The following extra labels keys are present in the model `{model_unique_id}`:\n{extra_labels}. "
"Ensure that the model does not include any extra labels keys."
)
recommendation = (
f"Add the missing labels keys listed above in the model `{altimate_unique_id}`. "
f"Add the missing labels keys listed above in the model `{model_unique_id}`. "
"Ensuring that the model has all the required labels keys helps in maintaining metadata consistency and understanding."
)

Expand All @@ -66,11 +66,11 @@ def _build_failure_result(self, altimate_unique_id: str, missing_keys: Sequence[
name=self.NAME,
message=failure_message.format(
missing_keys=numbered_list(missing_keys),
altimate_unique_id=altimate_unique_id,
model_unique_id=model_unique_id,
),
recommendation=recommendation.format(altimate_unique_id=altimate_unique_id),
recommendation=recommendation.format(model_unique_id=model_unique_id),
reason_to_flag=self.REASON_TO_FLAG,
metadata={"missing_keys": missing_keys, "altimate_unique_id": altimate_unique_id, "extra_keys": extra_labels},
metadata={"missing_keys": missing_keys, "model_unique_id": model_unique_id, "extra_keys": extra_labels},
)

def _check_labels_keys(self, node_id) -> Tuple[int, Set[str]]:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,28 +45,28 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]:
)
return insights

def _build_failure_result(self, altimate_unique_id: str, missing_keys: Sequence[str], extra_keys: Set[str]) -> DBTInsightResult:
def _build_failure_result(self, model_unique_id: str, missing_keys: Sequence[str], extra_keys: Set[str]) -> DBTInsightResult:
failure_message = (
f"The following meta keys are missing in the model `{altimate_unique_id}`:\n{numbered_list(missing_keys)}. "
f"The following meta keys are missing in the model `{model_unique_id}`:\n{numbered_list(missing_keys)}. "
"Ensure that the model includes all the required meta keys."
)
if not self.allow_extra_keys:
failure_message += (
f"The following extra meta keys are present in the model `{altimate_unique_id}`:\n{numbered_list(extra_keys)}. "
f"The following extra meta keys are present in the model `{model_unique_id}`:\n{numbered_list(extra_keys)}. "
"Ensure that the model does not include any extra meta keys."
)
recommendation = (
f"Add the missing meta keys listed above in the model `{altimate_unique_id}`. "
f"Add the missing meta keys listed above in the model `{model_unique_id}`. "
"Ensuring that the model has all the required meta keys helps in maintaining metadata consistency and understanding."
)

return DBTInsightResult(
type=self.TYPE,
name=self.NAME,
message=failure_message,
recommendation=recommendation.format(altimate_unique_id=altimate_unique_id),
recommendation=recommendation.format(model_unique_id=model_unique_id),
reason_to_flag=self.REASON_TO_FLAG,
metadata={"missing_keys": missing_keys, "altimate_unique_id": altimate_unique_id, "extra_keys": extra_keys},
metadata={"missing_keys": missing_keys, "model_unique_id": model_unique_id, "extra_keys": extra_keys},
)

def _check_meta_keys(self, node_id) -> Tuple[int, Set[str], Set[str]]:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,9 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]:
)
return insights

def _build_failure_result(self, altimate_unique_id: str) -> DBTInsightResult:
def _build_failure_result(self, model_unique_id: str) -> DBTInsightResult:
failure_message = (
f"The model {altimate_unique_id} do not have a properties file (.yml) defined."
f"The model {model_unique_id} do not have a properties file (.yml) defined."
"Ensure that each model has a corresponding .yml file for additional configuration and documentation."
)
recommendation = (
Expand All @@ -53,7 +53,7 @@ def _build_failure_result(self, altimate_unique_id: str) -> DBTInsightResult:
message=failure_message,
recommendation=recommendation,
reason_to_flag=self.REASON_TO_FLAG,
metadata={"altimate_unique_id": altimate_unique_id},
metadata={"model_unique_id": model_unique_id},
)

def _check_properties_file(self, node_id) -> int:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,14 +47,12 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]:
)
return insights

def _build_failure_result(self, altimate_unique_id: str, missing_test_groups: List[Dict]) -> DBTInsightResult:
def _build_failure_result(self, model_unique_id: str, missing_test_groups: List[Dict]) -> DBTInsightResult:
missing_test_group_str = ""
for test in missing_test_groups:
missing_test_group_str += f"Test Group: {test.get(self.TEST_GROUP_STR)}, Min Count: {test.get(self.TEST_COUNT_STR)}, Actual Count: {test.get('actual_count')}\n"

failure_message = (
f"The model `{altimate_unique_id}` does not have enough tests for the following groups:\n{missing_test_group_str}. "
)
failure_message = f"The model `{model_unique_id}` does not have enough tests for the following groups:\n{missing_test_group_str}. "
recommendation = (
"Add tests with the specified groups for each model listed above. "
"Having tests with specific groups ensures proper validation and data integrity."
Expand All @@ -66,7 +64,7 @@ def _build_failure_result(self, altimate_unique_id: str, missing_test_groups: Li
message=failure_message,
recommendation=recommendation,
reason_to_flag=self.REASON_TO_FLAG,
metadata={"altimate_unique_id": altimate_unique_id, "missing_test_groups": missing_test_groups},
metadata={"model_unique_id": model_unique_id, "missing_test_groups": missing_test_groups},
)

def _model_has_tests_by_group(self, node_id) -> List[Dict]:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,12 +45,12 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]:
)
return insights

def _build_failure_result(self, altimate_unique_id: str, missing_tests: List[Dict]) -> DBTInsightResult:
def _build_failure_result(self, model_unique_id: str, missing_tests: List[Dict]) -> DBTInsightResult:
tests_str = ""
for test in missing_tests:
tests_str += f"Test Name: {test.get(self.TEST_NAME_STR)}, Min Count: {test.get(self.TEST_COUNT_STR)}, Actual Count: {test.get('actual_count')}\n"

failure_message = f"The model `{altimate_unique_id}` does not have enough tests:\n{tests_str}. "
failure_message = f"The model `{model_unique_id}` does not have enough tests:\n{tests_str}. "
recommendation = (
"Add tests with the specified names for each model listed above. "
"Having tests with specific names ensures proper validation and data integrity."
Expand All @@ -62,7 +62,7 @@ def _build_failure_result(self, altimate_unique_id: str, missing_tests: List[Dic
message=failure_message,
recommendation=recommendation,
reason_to_flag=self.REASON_TO_FLAG,
metadata={"altimate_unique_id": altimate_unique_id},
metadata={"model_unique_id": model_unique_id},
)

def _model_has_tests_by_name(self, node_id) -> bool:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,12 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]:
)
return insights

def _build_failure_result(self, altimate_unique_id: str, missing_tests: List[Dict]) -> DBTInsightResult:
def _build_failure_result(self, model_unique_id: str, missing_tests: List[Dict]) -> DBTInsightResult:
missing_test_type_str = ""
for test in missing_tests:
missing_test_type_str += f"Test type: {test.get(self.TEST_TYPE_STR)}, Min Count: {test.get(self.TEST_COUNT_STR)}, Actual Count: {test.get('actual_count')}\n"

failure_message = f"The model `{altimate_unique_id}` does not have enough tests for the following types:\n{missing_test_type_str}. "
failure_message = f"The model `{model_unique_id}` does not have enough tests for the following types:\n{missing_test_type_str}. "
recommendation = (
"Add tests with the specified names for each model listed above. "
"Having tests with specific names ensures proper validation and data integrity."
Expand All @@ -63,7 +63,7 @@ def _build_failure_result(self, altimate_unique_id: str, missing_tests: List[Dic
message=failure_message,
recommendation=recommendation,
reason_to_flag=self.REASON_TO_FLAG,
metadata={"altimate_unique_id": altimate_unique_id, "missing_tests": missing_tests},
metadata={"model_unique_id": model_unique_id, "missing_tests": missing_tests},
)

def _model_has_tests_by_type(self, node_id) -> bool:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ def _build_failure_result(
message=failure_message,
recommendation=recommendation,
reason_to_flag=self.REASON_TO_FLAG,
metadata={"altimate_unique_id": node_id, **failure},
metadata={"model_unique_id": node_id, **failure},
)

def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]:
Expand Down Expand Up @@ -88,12 +88,12 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]:
)
return insights

def _check_model_name_contract(self, altimate_unique_id: str) -> bool:
def _check_model_name_contract(self, model_unique_id: str) -> bool:
"""
Check if the model name abides to the contract.
"""
model_name = self.get_node(altimate_unique_id).name
model_path = self.get_node(altimate_unique_id).original_file_path
model_name = self.get_node(model_unique_id).name
model_path = self.get_node(model_unique_id).original_file_path
for folder, pattern in self.patterns.items():
if is_superset_path(folder, model_path):
if re.match(pattern, model_name, re.IGNORECASE) is None:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ def _build_failure_result(
"max_parents": self.max_parents,
"min_childs": self.min_childs,
"max_childs": self.max_childs,
"altimate_unique_id": node_id,
"model_unique_id": node_id,
},
)

Expand Down Expand Up @@ -89,19 +89,19 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]:
)
return insights

def _check_model_parents_and_childs(self, altimate_unique_id: str) -> Optional[str]:
def _check_model_parents_and_childs(self, model_unique_id: str) -> Optional[str]:
"""
Check if the model has a specific number (max/min) of parents or/and childs.
"""
children = self.children_map.get(altimate_unique_id, [])
node = self.get_node(altimate_unique_id)
children = self.children_map.get(model_unique_id, [])
node = self.get_node(model_unique_id)
parents = node.depends_on.nodes
message = ""
if len(parents) < self.min_parents or len(parents) > self.max_parents:
message += f"The model:{altimate_unique_id} doesn't have the required number of parents.\n Min parents: {self.min_parents}, Max parents: {self.max_parents}. It has f{len(parents)} parents\n"
message += f"The model:{model_unique_id} doesn't have the required number of parents.\n Min parents: {self.min_parents}, Max parents: {self.max_parents}. It has f{len(parents)} parents\n"

if len(children) < self.min_childs or len(children) > self.max_childs:
message += f"The model:{altimate_unique_id} doesn't have the required number of childs.\n Min childs: {self.min_childs}, Max childs: {self.max_childs}. It has f{len(children)} childs\n"
message += f"The model:{model_unique_id} doesn't have the required number of childs.\n Min childs: {self.min_childs}, Max childs: {self.max_childs}. It has f{len(children)} childs\n"

return message

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,11 +66,11 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]:
)
return insights

def _check_model_parents_database(self, altimate_unique_id: str) -> bool:
def _check_model_parents_database(self, model_unique_id: str) -> bool:
"""
Check if the parent models or sources are from certain database.
"""
model = self.get_node(altimate_unique_id)
model = self.get_node(model_unique_id)
if model.resource_type == AltimateResourceType.model:
for parent in getattr(model.depends_on, "nodes", []):
parent_model = self.get_node(parent)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,11 +66,11 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]:
)
return insights

def _check_model_parents_schema(self, altimate_unique_id: str) -> bool:
def _check_model_parents_schema(self, model_unique_id: str) -> bool:
"""
Check if the parent models or sources are from certain schema.
"""
model = self.get_node(altimate_unique_id)
model = self.get_node(model_unique_id)
if model.resource_type == AltimateResourceType.model:
for parent in getattr(model.depends_on, "nodes", []):
parent_model = self.get_node(parent)
Expand Down
Loading

0 comments on commit 3ca0ecd

Please sign in to comment.