Ele 3521 add collected by to metric properties (#747)

* add collected_by flag to collect_metrics * test_collect_metrics: add cloud_monitored parameter * bugfixes * validate_unique_metric_names - bugfix
elementary-data · Aug 13, 2024 · 79fd56c · 79fd56c
1 parent 26e24f9
commit 79fd56c
Show file tree

Hide file tree

Showing 5 changed files with 19 additions and 11 deletions.
diff --git a/macros/edr/metadata_collection/get_metric_properties.sql b/macros/edr/metadata_collection/get_metric_properties.sql
@@ -5,20 +5,25 @@
   time_bucket,
   dimensions=none,
   freshness_column=none,
-  event_timestamp_column=none
+  event_timestamp_column=none,
+  collected_by=none
 ) %}
     {% set timestamp_column = elementary.get_test_argument('timestamp_column', timestamp_column, model_graph_node) %}
     {% set where_expression = elementary.get_test_argument('where_expression', where_expression, model_graph_node) %}
     {% set time_bucket = elementary.get_time_bucket(time_bucket, model_graph_node) %}
     {% set freshness_column = elementary.get_test_argument('freshness_column', freshness_column, model_graph_node) %}
     {% set event_timestamp_column = elementary.get_test_argument('event_timestamp_column', event_timestamp_column, model_graph_node) %}
     {% set dimensions = elementary.get_test_argument('dimensions', dimensions, model_graph_node) %}
-    {% do return({
+    {% set metric_props = {
       'timestamp_column': timestamp_column,
       'where_expression': where_expression,
       'time_bucket': time_bucket,
       'freshness_column': freshness_column,
       'event_timestamp_column': event_timestamp_column,
       'dimensions': dimensions
-    }) %}
+    } %}
+    {% if collected_by %}
+      {% do metric_props.update({'collected_by': collected_by}) %}
+    {% endif %}
+    {% do return(metric_props) %}
 {% endmacro %}
diff --git a/macros/edr/tests/test_collect_metrics.sql b/macros/edr/tests/test_collect_metrics.sql
@@ -6,7 +6,8 @@
     days_back=64,
     backfill_days=none,
     where_expression=none,
-    dimensions=none
+    dimensions=none,
+    cloud_monitored=false
 ) %}
 
     {{ config(
@@ -73,11 +74,11 @@
     {% endfor %}
 
     {% if table_metrics %}
-        {% do elementary.collect_table_metrics(table_metrics, model, model_relation, timestamp_column, time_bucket, days_back, backfill_days, where_expression, dimensions) %}
+        {% do elementary.collect_table_metrics(table_metrics, model, model_relation, timestamp_column, time_bucket, days_back, backfill_days, where_expression, dimensions, collected_by="collect_metrics") %}
     {% endif %}
 
     {% for col_name, col_metrics in col_to_metrics.items() %}
-        {% do elementary.collect_column_metrics(col_metrics, model, model_relation, col_name, timestamp_column, time_bucket, days_back, backfill_days, where_expression, dimensions) %}
+        {% do elementary.collect_column_metrics(col_metrics, model, model_relation, col_name, timestamp_column, time_bucket, days_back, backfill_days, where_expression, dimensions, collected_by="collect_metrics") %}
     {% endfor %}
 
     {# This test always passes. #}

diff --git a/macros/edr/tests/test_utils/collect_column_metrics.sql b/macros/edr/tests/test_utils/collect_column_metrics.sql
@@ -8,10 +8,11 @@
     days_back,
     backfill_days,
     where_expression,
-    dimensions
+    dimensions,
+    collected_by=none
 ) %}
     {% set model_graph_node = elementary.get_model_graph_node(model_relation) %}
-    {% set metric_props = elementary.get_metric_properties(model_graph_node, timestamp_column, where_expression, time_bucket, dimensions) %}
+    {% set metric_props = elementary.get_metric_properties(model_graph_node, timestamp_column, where_expression, time_bucket, dimensions, collected_by=collected_by) %}
     {% set days_back = elementary.get_test_argument('days_back', days_back, model_graph_node) %}
     {% set backfill_days = elementary.get_test_argument('backfill_days', backfill_days, model_graph_node) %}
 

diff --git a/macros/edr/tests/test_utils/collect_table_metrics.sql b/macros/edr/tests/test_utils/collect_table_metrics.sql
@@ -7,10 +7,11 @@
     days_back,
     backfill_days,
     where_expression,
-    dimensions
+    dimensions,
+    collected_by=none
 ) %}
     {% set model_graph_node = elementary.get_model_graph_node(model_relation) %}
-    {% set metric_props = elementary.get_metric_properties(model_graph_node, timestamp_column, where_expression, time_bucket, dimensions) %}
+    {% set metric_props = elementary.get_metric_properties(model_graph_node, timestamp_column, where_expression, time_bucket, dimensions, collected_by=collected_by) %}
     {% set days_back = elementary.get_test_argument('days_back', days_back, model_graph_node) %}
     {% set backfill_days = elementary.get_test_argument('backfill_days', backfill_days, model_graph_node) %}
 

diff --git a/macros/edr/tests/test_utils/validate_unique_metric_names.sql b/macros/edr/tests/test_utils/validate_unique_metric_names.sql
@@ -15,7 +15,7 @@
 
     {% for graph_node in graph.nodes.values() %}
         {% if test_node.unique_id != graph_node.unique_id and graph_node.resource_type == "test" %}
-            {% set test_metadata = graph_node.test_metadata %}
+            {% set test_metadata = elementary.safe_get_with_default(graph_node, 'test_metadata', {}) %}
             {% if test_metadata.namespace == "elementary" and test_metadata.name == "collect_metrics" %}
                 {% set test_parent_model_unique_ids = elementary.get_parent_model_unique_ids_from_test_node(graph_node) %}
                 {% if parent_model_unique_ids == test_parent_model_unique_ids %}