Merge pull request #600 from elementary-data/ELE-1946-dbt-columns-flags

Ele 1946 dbt columns flags
elementary-data · Nov 2, 2023 · 44bb408 · 44bb408
2 parents 2f517ed + cc8cc0e
commit 44bb408
Show file tree

Hide file tree

Showing 5 changed files with 12 additions and 3 deletions.
diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml
@@ -105,7 +105,7 @@ jobs:
         run: |
           mkdir -p ~/.dbt
           DBT_VERSION=$(pip show dbt-core | grep -i version | awk '{print $2}' | sed 's/\.//g')
-          UNDERSCORED_REF_NAME=$(echo "${{ inputs.warehouse-type }}_dbt_${DBT_VERSION}_${BRANCH_NAME}" | head -c 40 | sed "s/-/_/g")
+          UNDERSCORED_REF_NAME=$(echo "${{ inputs.warehouse-type }}_dbt_${DBT_VERSION}_${BRANCH_NAME}" | awk '{print tolower($0)}' | head -c 40 | sed "s/-/_/g")
           echo "$PROFILES_YML" | base64 -d | sed "s/<SCHEMA_NAME>/dbt_pkg_$UNDERSCORED_REF_NAME/g" > ~/.dbt/profiles.yml
 
       - name: Check DWH connection

diff --git a/integration_tests/tests/dbt_project.py b/integration_tests/tests/dbt_project.py
@@ -16,6 +16,7 @@
 _DEFAULT_VARS = {
     "disable_dbt_invocation_autoupload": True,
     "disable_dbt_artifacts_autoupload": True,
+    "disable_dbt_columns_autoupload": True,
     "disable_run_results": True,
     "debug_logs": True,
     "collect_metrics": False,

diff --git a/macros/edr/dbt_artifacts/upload_dbt_artifacts.sql b/macros/edr/dbt_artifacts/upload_dbt_artifacts.sql
@@ -8,10 +8,13 @@
       "dbt_metrics": elementary.upload_dbt_metrics,
       "dbt_exposures": elementary.upload_dbt_exposures,
       "dbt_seeds": elementary.upload_dbt_seeds,
-      "dbt_columns": elementary.upload_dbt_columns,
       }
     %}
 
+    {% if not elementary.get_config_var('disable_dbt_columns_autoupload') %}
+      {% do model_upload_func_map.update({"dbt_columns": elementary.upload_dbt_columns}) %}
+    {% endif %}
+
     {% set artifacts_hashes = elementary.get_artifacts_hashes() %}
     {% do elementary.file_log("Uploading dbt artifacts.") %}
     {% for artifacts_model, upload_artifacts_func in model_upload_func_map.items() %}

diff --git a/macros/edr/dbt_artifacts/upload_dbt_columns.sql b/macros/edr/dbt_artifacts/upload_dbt_columns.sql
@@ -35,7 +35,10 @@
 
     {% set flattened_columns = [] %}
     {% for column_node in column_nodes.values() %}
-        {% do flattened_columns.append(elementary.flatten_column(table_node, column_node)) %}
+        {% set flat_column = elementary.flatten_column(table_node, column_node) %}
+        {% if not elementary.get_config_var('upload_only_columns_with_descriptions') or flat_column['description'] %}
+            {% do flattened_columns.append(flat_column) %}
+        {% endif %}
     {% endfor %}
     {% do return(flattened_columns) %}
 {% endmacro %}

diff --git a/macros/edr/system/system_utils/get_config_var.sql b/macros/edr/system/system_utils/get_config_var.sql
@@ -31,6 +31,8 @@
     'disable_run_results': false,
     'disable_tests_results': false,
     'disable_dbt_artifacts_autoupload': false,
+    'disable_dbt_columns_autoupload': false,
+    'upload_only_columns_with_descriptions': false,
     'disable_dbt_invocation_autoupload': false,
     'disable_skipped_model_alerts': true,
     'disable_skipped_test_alerts': true,