Merge pull request #555 from elementary-data/ele-1811-normalize-data-…

…type-on-schema-changes-test Normalizing data type on schema changes test.
elementary-data · Sep 27, 2023 · 70fc7e3 · 70fc7e3
2 parents 2fd9b2c + a1e0f07
commit 70fc7e3
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 12 deletions.
diff --git a/integration_tests/tests/test_schema_changes.py b/integration_tests/tests/test_schema_changes.py
@@ -4,31 +4,40 @@
 from dbt_project import DbtProject
 
 DATASET1 = [
-    {"id": 1, "name": "Elon"},
+    {"id": 1, "name": "Elon", "nick": "EGK"},
     {"id": 2, "name": "Itamar"},
     {"id": 3, "name": "Ofek"},
 ]
 
 DATASET2 = [
-    {"id": "a", "first_name": "Elon", "age": 22},
+    {"id": "a", "first_name": "Elon", "age": 22, "nick": "EGK"},
     {"id": "b", "first_name": "Itamar", "age": None},
     {"id": "c", "first_name": "Ofek", "age": None},
 ]
 
 EXPECTED_FAILURES = [
-    "column_added",  # first_name
-    "column_added",  # age
-    "type_changed",  # id
-    "column_removed",  # name
+    ("first_name", "column_added"),
+    ("age", "column_added"),
+    ("id", "type_changed"),
+    ("name", "column_removed"),
 ]
 
+STRING_JINJA = r"{{ 'STRING' if (target.type == 'bigquery' or target.type == 'databricks') else 'character varying' if target.type == 'redshift' else 'TEXT' }}"
+
 
 def assert_test_results(test_results: List[dict]):
     expected_failures = EXPECTED_FAILURES.copy()
-    for test_result in test_results:
-        test_sub_type = test_result["test_sub_type"]
-        assert test_result["status"] == "fail" and test_sub_type in expected_failures
-        expected_failures.remove(test_sub_type)
+    failed_test_results = [
+        test_result for test_result in test_results if test_result["status"] == "fail"
+    ]
+    assert len(failed_test_results) == len(expected_failures)
+
+    for test_result in failed_test_results:
+        test_failure = (
+            test_result["column_name"].lower(),
+            test_result["test_sub_type"].lower(),
+        )
+        expected_failures.remove(test_failure)
     assert not expected_failures
 
 
@@ -54,7 +63,8 @@ def test_schema_changes_from_baseline(test_id: str, dbt_project: DbtProject):
         test_args={"fail_on_added": True, "enforce_types": True},
         columns=[
             {"name": "id", "data_type": "integer"},
-            {"name": "name", "data_type": "string"},
+            {"name": "name", "data_type": STRING_JINJA},
+            {"name": "nick", "data_type": STRING_JINJA},
         ],
         data=DATASET2,
         multiple_results=True,

diff --git a/macros/edr/data_monitoring/schema_changes/get_columns_snapshot_query.sql b/macros/edr/data_monitoring/schema_changes/get_columns_snapshot_query.sql
@@ -38,7 +38,7 @@
                 {% for column in columns %}
                     select
                         {{ elementary.edr_cast_as_string(elementary.edr_quote(column.name)) }} as column_name,
-                        {{ elementary.edr_cast_as_string(elementary.edr_quote(column.data_type)) }} as data_type
+                        {{ elementary.edr_cast_as_string(elementary.edr_quote(elementary.get_normalized_data_type(column.dtype))) }} as data_type
                     {% if not loop.last %}
                         union all
                     {% endif %}