diff --git a/.github/workflows/run_unit_tests_on_pr.yml b/.github/workflows/run_unit_tests_on_pr.yml index 11579394..61e5211f 100644 --- a/.github/workflows/run_unit_tests_on_pr.yml +++ b/.github/workflows/run_unit_tests_on_pr.yml @@ -1,10 +1,6 @@ name: Run Unit Tests on Pull Request -# on: [pull_request_target,workflow_dispatch] -on: - push: - branches: - - 'feature/dbt-unit-tests' +on: [pull_request_target,workflow_dispatch] env: BIGQUERY_PROJECT: ${{ secrets.BIGQUERY_PROJECT }} BIGQUERY_PROPERTY_ID: ${{ secrets.BIGQUERY_PROPERTY_ID }} @@ -68,5 +64,10 @@ jobs: pip install dbt-bigquery dbt deps + - name: Materialize necessary dbt resources + run: | + dbt seed -f + dbt run -s +test_type:unit -f --empty + - name: Run dbt unit tests run: dbt test -s test_type:unit diff --git a/dbt_project.yml b/dbt_project.yml index d8201549..10f76dca 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -10,9 +10,9 @@ snapshot-paths: ["snapshots"] profile: 'default' +# These variables are used for unit tests during CI for the package +# These variables should either be removed or updated to reflect the needs of your GA data and dbt project vars: - source_project: "{{ env_var('BIGQUERY_PROJECT') }}" - property_ids: ["{{ env_var('BIGQUERY_PROPERTY_ID') }}"] start_date: "20230306" static_incremental_days: 3 derived_session_properties: @@ -24,7 +24,6 @@ vars: user_property_name: "most_recent_page_title" value_type: "string_value" conversion_events: ['large_button_clicked', 'add_to_cart'] - session_attribution_lookback_window_days: 30 target-path: "target" # directory which will store compiled SQL files clean-targets: # directories to be removed by `dbt clean` diff --git a/macros/base_select.sql b/macros/base_select.sql index 8919388f..7603d446 100644 --- a/macros/base_select.sql +++ b/macros/base_select.sql @@ -36,7 +36,8 @@ , ecommerce.transaction_id , items , {%- if var('combined_dataset', false) != false %} cast(left(regexp_replace(_table_suffix, r'^(intraday_)?\d{8}', ''), 100) as int64) - {%- else %} {{ var('property_ids')[0] }} + {%- elif var('property_ids', false) != false %} {{ var('property_ids')[0] }} + {%- else %} {{ env_var('BIGQUERY_PROPERTY_ID') }} {%- endif %} as property_id {% endmacro %} diff --git a/models/staging/src_ga4.yml b/models/staging/src_ga4.yml index 29104767..8f29701d 100644 --- a/models/staging/src_ga4.yml +++ b/models/staging/src_ga4.yml @@ -4,11 +4,13 @@ sources: - name: ga4 database: | # Source from target.project if multi-property, otherwise source from source_project {%- if var('combined_dataset', false) != false -%} {{target.project}} - {%- else -%} {{var('source_project')}} + {%- elif var('source_project', false) != false -%} {{var('source_project')}} + {%- else -%} {{env_var('BIGQUERY_PROJECT')}} {%- endif -%} schema: | # Source from combined property dataset if set, otherwise source from original GA4 property {%- if var('combined_dataset', false) != false -%} {{var('combined_dataset')}} - {%- else -%} analytics_{{var('property_ids')[0]}} + {%- elif var('property_ids', false) != false -%} analytics_{{var('property_ids')[0]}} + {%- else -%} analytics_{{env_var('BIGQUERY_PROPERTY_ID')}} {%- endif -%} tables: - name: events