Make updates for dbt unit test Github Action and allow for use of env…

…ironment variables instead of project variables
Velir · Oct 31, 2024 · f198262 · f198262
1 parent 555671e
commit f198262
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 11 deletions.
diff --git a/.github/workflows/run_unit_tests_on_pr.yml b/.github/workflows/run_unit_tests_on_pr.yml
@@ -1,10 +1,6 @@
 name: Run Unit Tests on Pull Request
 
-# on: [pull_request_target,workflow_dispatch]
-on:
-  push:
-    branches:
-      - 'feature/dbt-unit-tests'
+on: [pull_request_target,workflow_dispatch]
 env:
   BIGQUERY_PROJECT: ${{ secrets.BIGQUERY_PROJECT }}
   BIGQUERY_PROPERTY_ID: ${{ secrets.BIGQUERY_PROPERTY_ID }}
@@ -68,5 +64,10 @@ jobs:
           pip install dbt-bigquery
           dbt deps
 
+      - name: Materialize necessary dbt resources
+        run: |
+          dbt seed -f
+          dbt run -s +test_type:unit -f --empty
+
       - name: Run dbt unit tests
         run: dbt test -s test_type:unit
diff --git a/dbt_project.yml b/dbt_project.yml
@@ -10,9 +10,9 @@ snapshot-paths: ["snapshots"]
 
 profile: 'default'
 
+# These variables are used for unit tests during CI for the package
+# These variables should either be removed or updated to reflect the needs of your GA data and dbt project
 vars:
-  source_project: "{{ env_var('BIGQUERY_PROJECT') }}"
-  property_ids: ["{{ env_var('BIGQUERY_PROPERTY_ID') }}"]
   start_date: "20230306"
   static_incremental_days: 3
   derived_session_properties:
@@ -24,7 +24,6 @@ vars:
       user_property_name: "most_recent_page_title"
       value_type: "string_value"
   conversion_events: ['large_button_clicked', 'add_to_cart']
-  session_attribution_lookback_window_days: 30
 
 target-path: "target"  # directory which will store compiled SQL files
 clean-targets:         # directories to be removed by `dbt clean`

diff --git a/macros/base_select.sql b/macros/base_select.sql
@@ -36,7 +36,8 @@
     , ecommerce.transaction_id
     , items
     , {%- if  var('combined_dataset', false) != false %} cast(left(regexp_replace(_table_suffix, r'^(intraday_)?\d{8}', ''), 100) as int64)
-        {%- else %} {{ var('property_ids')[0] }}
+        {%- elif var('property_ids', false) != false %} {{ var('property_ids')[0] }}
+        {%- else %} {{ env_var('BIGQUERY_PROPERTY_ID') }}
         {%- endif %} as property_id
 {% endmacro %}
 

diff --git a/models/staging/src_ga4.yml b/models/staging/src_ga4.yml
@@ -4,11 +4,13 @@ sources:
   - name: ga4
     database: | # Source from target.project if multi-property, otherwise source from source_project
       {%- if  var('combined_dataset', false) != false -%} {{target.project}}
-      {%- else -%} {{var('source_project')}}
+      {%- elif var('source_project', false) != false -%} {{var('source_project')}}
+      {%- else -%} {{env_var('BIGQUERY_PROJECT')}}
       {%- endif -%}
     schema: | # Source from combined property dataset if set, otherwise source from original GA4 property
       {%- if  var('combined_dataset', false) != false -%} {{var('combined_dataset')}}
-      {%- else -%} analytics_{{var('property_ids')[0]}}
+      {%- elif var('property_ids', false) != false -%} analytics_{{var('property_ids')[0]}}
+      {%- else -%} analytics_{{env_var('BIGQUERY_PROPERTY_ID')}}
       {%- endif -%}
     tables:
       - name: events