Merge pull request #4 from TasmanAnalytics/fix/incremental-logic

🚑 Fix incremental logic to accommodate for window functions
TasmanAnalytics · Mar 4, 2024 · 76a1226 · 76a1226
2 parents 1d1cb15 + 07c3d80
commit 76a1226
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 3 deletions.
diff --git a/models/core/_core.yml b/models/core/_core.yml
@@ -202,6 +202,8 @@ models:
         description: 'The unique order ID for the transaction.'
         tests:
           - not_null
+          - unique:
+              where: "valid_to is null"
 
       - name: takehome_percentage
         data_type: varchar

diff --git a/models/core/revenuecat_subscription_transactions.sql b/models/core/revenuecat_subscription_transactions.sql
@@ -2,10 +2,21 @@
   config(
     materialized = 'incremental',
     unique_key = 'transaction_row_id',
+    merge_update_columns = ['valid_to'],
     )
 }}
 
-with
+with 
+{% if is_incremental() %} 
+merge_identifiers as (
+  select distinct store_transaction_id 
+        from {{ source('revenuecat', 'transactions') }}
+        where regexp_substr(_file_name, '[0-9]{10}')::timestamp_ntz > (
+            select max(_exported_at) 
+            from {{ this }}
+        )
+),
+{% endif %}
 
 source as (
     select 
@@ -17,8 +28,11 @@ source as (
         {% if var('revenuecat_filter') %}
         and {{ var('revenuecat_filter') }}
         {% endif %}
-        {% if is_incremental() %}
-        and regexp_substr(_file_name, '[0-9]{10}')::timestamp_ntz > (select max(_exported_at) from {{ this }})
+        {% if is_incremental() %} 
+        and store_transaction_id in (
+                select store_transaction_id 
+                from merge_identifiers
+        )
         {% endif %}
 
 ),