Add union view

Updates ELE-1819
elementary-data · Oct 1, 2023 · 9ce379e · 9ce379e
1 parent bdbb6de
commit 9ce379e
Show file tree

Hide file tree

Showing 4 changed files with 44 additions and 18 deletions.
diff --git a/macros/edr/dbt_artifacts/upload_dbt_exposures.sql b/macros/edr/dbt_artifacts/upload_dbt_exposures.sql
@@ -28,6 +28,7 @@
                                                                      ('generated_at', 'string'),
                                                                      ('metadata_hash', 'string'),
                                                                      ('label', 'string'),
+                                                                     ('raw_queries', 'long_string'),
                                                                      ]) %}
     {{ return(dbt_exposures_empty_table_query) }}
 {% endmacro %}
@@ -58,4 +59,4 @@
       }%}
     {% do flatten_exposure_metadata_dict.update({"metadata_hash": elementary.get_artifact_metadata_hash(flatten_exposure_metadata_dict)}) %}
     {{ return(flatten_exposure_metadata_dict) }}
-{% endmacro %}
+{% endmacro %}
diff --git a/macros/edr/system/system_utils/empty_table.sql b/macros/edr/system/system_utils/empty_table.sql
@@ -75,18 +75,26 @@
 {% endmacro %}
 
 {% macro empty_elementary_exposures() %}
-    {% set columns = [('id','string'),
-                      ('label','string'),
-                      ('url','string'),
-                      ('type','string'),
-                      ('maturity','float'),
-                      ('depends_on','longstring'),
-                      ('owner_name','string'),
-                      ('owner_email','string'),
-                      ('created_at', 'timestamp'),
-                      ('updated_at','timestamp'),
-                      ('raw_queries','longstring')]
-    %}
+    {% set columns = [('unique_id', 'string'),
+                      ('name', 'string'),
+                      ('maturity', 'string'),
+                      ('type', 'string'),
+                      ('owner_email', 'string'),
+                      ('owner_name', 'string'),
+                      ('url', 'long_string'),
+                      ('depends_on_macros', 'long_string'),
+                      ('depends_on_nodes', 'long_string'),
+                      ('description', 'long_string'),
+                      ('tags', 'long_string'),
+                      ('meta', 'long_string'),
+                      ('package_name', 'string'),
+                      ('original_path', 'long_string'),
+                      ('path', 'string'),
+                      ('generated_at', 'string'),
+                      ('metadata_hash', 'string'),
+                      ('label', 'string'),
+                      ('raw_queries', 'long_string'),
+                     ] %}
     {{ elementary.empty_table(columns) }}
 {% endmacro %}
 

diff --git a/models/edr/elementary_exposures/elementary_exposures.sql b/models/edr/elementary_exposures/elementary_exposures.sql
@@ -1,12 +1,10 @@
 {{
   config(
     materialized='incremental',
-    unique_key='id',
-    on_schema_change='append_new_columns',
+    transient=False,
+    unique_key='unique_id',
     full_refresh=elementary.get_config_var('elementary_full_refresh'),
-    meta={
-      "timestamp_column": "created_at",
-    }
+    on_schema_change='sync_all_columns',
   )
 }}
 

diff --git a/models/edr/elementary_exposures/enriched_exposures.sql b/models/edr/elementary_exposures/enriched_exposures.sql
@@ -0,0 +1,19 @@
+{{
+  config(
+    materialized = 'view',
+    bind=False,
+    unique_key='unique_id',
+  )
+}}
+
+with dbt_exposures as (
+  select * from {{ ref('dbt_exposures') }}
+),
+
+elementary_exposures as (
+  select * from {{ ref('elementary_exposures') }}
+)
+
+{# Union without duplicates where elementary_exposures has prio #}
+select * from dbt_exposures where not exists (select 1 from elementary_exposures where dbt_exposures.unique_id = elementary_exposures.unique_id)
+union select * from elementary_exposures