Update partition logic, add source relation (#78)

.github/workflows/auto-release.yml

@@ -0,0 +1,13 @@
+name: 'auto release'
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - main
+
+jobs:
+  call-workflow-passing-data:
+    if: github.event.pull_request.merged
+    uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main
+    secrets: inherit

.gitignore

@@ -2,5 +2,6 @@
 target/
 dbt_modules/
 logs/
-
-dbt_packages/
+package-lock.yml
+dbt_packages/
+env/

CHANGELOG.md

@@ -1,3 +1,18 @@
+# dbt_shopify_source v0.11.0
+[PR #78](https://github.com/fivetran/dbt_shopify_source/pull/78) introduces the following changes: 
+
+## 🚨 Breaking Changes 🚨
+- Added `source_relation` to the `partition_by` clauses that determine the `is_most_recent_record` in the `stg_shopify__metafield` table.
+- Added `source_relation` to the `partition_by` clauses that determines the `index` in the `stg_shopify__abandoned_checkout_discount_code` table. 
+- If the user is leveraging the union feature, this could change data values.
+
+## 🐛 Bug Fixes 🪛 
+- Updated partition logic in `stg_shopify__metafield` and `stg_shopify__abandoned_checkout_discount_code` to account for null table Redshift errors when handling null field cases. 
+
+## 🚘 Under The Hood 🚘
+- Included auto-releaser GitHub Actions workflow to automate future releases.
+- Added additional casting in seed dependencies for above models `integration_tests/dbt_project.yml` to ensure local testing passed on null cases.
+
 # dbt_shopify_source v0.10.0
 ## 🚨 Breaking Changes 🚨
 - This release will be a breaking change due to the removal of below dependencies.

README.md

@@ -44,7 +44,7 @@ If you  are **not** using the [Shopify transformation package](https://github.co
 ```yml
 packages:
   - package: fivetran/shopify_source
-    version: [">=0.10.0", "<0.11.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=0.11.0", "<0.12.0"] # we recommend using ranges to capture non-breaking changes automatically
 ```
 
 ## Step 3: Define database and schema variables

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'shopify_source'
-version: '0.10.0'
+version: '0.11.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: shopify_source_integration_tests_7
+      schema: shopify_source_integration_tests_9
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: shopify_source_integration_tests_7
+      schema: shopify_source_integration_tests_9
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: shopify_source_integration_tests_7
+      schema: shopify_source_integration_tests_9
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: shopify_source_integration_tests_7
+      schema: shopify_source_integration_tests_8
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: shopify_source_integration_tests_7
+      schema: shopify_source_integration_tests_9
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,10 +1,10 @@
 name: 'shopify_source_integration_tests'
-version: '0.10.0'
+version: '0.11.0'
 profile: 'integration_tests'
 config-version: 2
 
 vars:
-  shopify_schema: shopify_source_integration_tests_7
+  shopify_schema: shopify_source_integration_tests_9
   shopify_source:
     shopify_customer_identifier:          "shopify_customer_data"
     shopify_order_line_refund_identifier: "shopify_order_line_refund_data"
@@ -109,9 +109,14 @@ seeds:
         user_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
     shopify_abandoned_checkout_discount_code_data:
       +column_types:
+        checkout_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        discount_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
         amount: float
+        _fivetran_synced: timestamp
         created_at: timestamp
         updated_at: timestamp
+        code: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
+        type: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
     shopify_abandoned_checkout_data:
       +column_types:
         total_line_items_price: float
@@ -150,6 +155,9 @@ seeds:
       +column_types:
         value_type: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
         type: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
+        _fivetran_synced: timestamp
+        created_at: timestamp
+        updated_at: timestamp
     shopify_shop_data:
       +column_types:
         taxes_included: boolean

models/src_shopify.yml

@@ -1437,4 +1437,4 @@ sources:
           - name: rate
             description: The proportion of the line item price that the tax represents as a decimal.
           - name: title
-            description: The name of the tax.
+            description: The name of the tax.

models/stg_shopify__abandoned_checkout_discount_code.sql

@@ -34,7 +34,10 @@ final as (
         {{ dbt_date.convert_timezone(column='cast(updated_at as ' ~ dbt.type_timestamp() ~ ')', target_tz=var('shopify_timezone', "UTC"), source_tz="UTC") }} as updated_at,
         {{ dbt_date.convert_timezone(column='cast(_fivetran_synced as ' ~ dbt.type_timestamp() ~ ')', target_tz=var('shopify_timezone', "UTC"), source_tz="UTC") }} as _fivetran_synced,
         source_relation, 
-        row_number() over(partition by checkout_id, upper(code) order by index desc) as index
+        case when checkout_id is null and code is null and index is null
+            then row_number() over(partition by source_relation order by source_relation)
+            else row_number() over(partition by checkout_id, upper(code), source_relation order by index desc)
+        end as index
 
     from fields
 

models/stg_shopify__metafield.sql

@@ -38,7 +38,10 @@ final as (
         {{ dbt_date.convert_timezone(column='cast(updated_at as ' ~ dbt.type_timestamp() ~ ')', target_tz=var('shopify_timezone', "UTC"), source_tz="UTC") }} as updated_at,
         {{ dbt_date.convert_timezone(column='cast(_fivetran_synced as ' ~ dbt.type_timestamp() ~ ')', target_tz=var('shopify_timezone', "UTC"), source_tz="UTC") }} as _fivetran_synced,
         lower({{ dbt.concat(["namespace","'_'","key"]) }}) as metafield_reference,
-        row_number() over (partition by id order by updated_at desc) = 1 as is_most_recent_record,
+        case when id is null and updated_at is null
+            then row_number() over(partition by source_relation order by source_relation) = 1
+            else row_number() over(partition by id, source_relation order by updated_at desc) = 1
+        end as is_most_recent_record,
         source_relation
 
     from fields