Merge pull request #42 from TasmanAnalytics/feature/test_incremental

Testing update for incremental models

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'tasman_dbt_mta'
-version: '0.0.1'
+version: '1.0.2'
 
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 

integration_tests/data/dummy_data/conversion_events.csv

@@ -2,4 +2,5 @@ event,event_timestamp,user,type
 739982bb-f169-40d0-814d-ff633db562c0,2022-05-12 00:54:19,[email protected],purchase
 1e0c4442-03ef-40bd-a5b8-2cfc77457903,2022-05-21 11:31:37,[email protected],purchase
 da9360ef-f52e-4572-aa98-128c17d97a41,2022-06-07 07:08:37,[email protected],lead
-be62daba-e4d2-4998-a057-8a3b25f2e9e3,2022-06-17 06:48:26,[email protected],purchase
+be62daba-e4d2-4998-a057-8a3b25f2e9e3,2022-06-17 06:48:26,[email protected],purchase
+3olgh3bs-05hf-40bd-a5b8-2cfc77457903,2024-09-12 11:31:37,[email protected],purchase

integration_tests/data/dummy_data/touch_events.csv

@@ -17,4 +17,7 @@ e158c32b-642e-4133-84e7-2bfb6fb4ed42,2022-06-10 20:50:26,[email protected],paid-so
 dff554e6-2f53-4eb1-a663-70d33658be63,2022-06-17 06:30:15,[email protected],direct
 a36aa6c8-ecab-4c4b-a2be-6c615cc69c5d,2022-04-17 10:30:25,[email protected],paid-search
 50f9def3-6e57-4e28-ae58-2c887e6e9e52,2022-04-18 10:30:25,[email protected],organic-search
-dedd99d9-7f58-4a92-b406-32e60c1ecbdf,2022-04-19 10:30:25,[email protected],paid-search
+dedd99d9-7f58-4a92-b406-32e60c1ecbdf,2022-04-19 10:30:25,[email protected],paid-search
+2ab3jfy0-7516-46d2-af13-fe0e309b997b,2024-09-10 09:15:56,[email protected],paid-search
+ef9028hg-84ee-484c-b4d1-cf4fd7d0b867,2024-09-11 11:30:12,[email protected],organic-search
+789nghfs-847c-45d0-97c1-d2acd1d23593,2024-09-11 11:30:00,[email protected],paid-search

integration_tests/dbt_project.yml

@@ -35,5 +35,6 @@ vars:
     attribution_rules: "{{ ref('attribution_rules') }}"
     conversion_shares: "{{ ref('conversion_shares') }}"
     attribution_windows: "{{ ref('attribution_windows') }}"
+    test_hours: "36"
     snowflake_prod_warehouse: ""
-    snowflake_dev_warehouse: ""
+    snowflake_dev_warehouse: ""

models/schema.yml

@@ -182,66 +182,98 @@ models:
       - name: surrogate_key
         description: "A unique key for the table, generated from the hash of the model ID and conversion event ID"
         tests:
-          - not_null
-          - unique
+          - not_null:
+              config: 
+                where: "conversion_timestamp >= DATEADD(HOUR, -{{ var('test_hours') }}, current_timestamp())"
+          - unique:
+              config: 
+                where: "conversion_timestamp >= DATEADD(HOUR, -{{ var('test_hours') }}, current_timestamp())"
 
       - name: conversion_user_id
         description: "Identifier for the user associated with the conversion"
         tests:
-          - not_null
+          - not_null:
+              config: 
+                where: "conversion_timestamp >= DATEADD(HOUR, -{{ var('test_hours') }}, current_timestamp())"
 
       - name: conversion_event_id
         description: "Identifer that defines a unique conversion event."
         tests:
-          - not_null
+          - not_null:
+              config: 
+                where: "conversion_timestamp >= DATEADD(HOUR, -{{ var('test_hours') }}, current_timestamp())"
 
       - name: conversion_timestamp
         description: "The timestamp when the conversion event happened."
         tests:
-          - not_null
-          - in_past
+          - not_null:
+              config: 
+                where: "conversion_timestamp >= DATEADD(HOUR, -{{ var('test_hours') }}, current_timestamp())"
+          - in_past:
+              config: 
+                where: "conversion_timestamp >= DATEADD(HOUR, -{{ var('test_hours') }}, current_timestamp())"
 
       - name: model_id
         description: "Identifer that specifies the attribution model that the attribution relates to."
         tests:
-          - not_null
+          - not_null:
+              config: 
+                where: "conversion_timestamp >= DATEADD(HOUR, -{{ var('test_hours') }}, current_timestamp())"
 
       - name: conversion_category
         description: "The category of conversion event, as defined in the conversion rules seed."
         tests:
-          - not_null
+          - not_null:
+              config: 
+                where: "conversion_timestamp >= DATEADD(HOUR, -{{ var('test_hours') }}, current_timestamp())"
 
   - name: tasman_mta__filtered_touch_events
     description: "This model applies the touch rules seed to the model defined in the 'touches_model' variable with the project file."
     columns:
       - name: surrogate_key
         description: "A unique key for the table, generated from the hash of the model ID and touch event ID"
         tests:
-          - not_null
-          - unique
+          - not_null:
+              config: 
+                where: "touch_timestamp >= DATEADD(HOUR, -{{ var('test_hours') }}, current_timestamp())"
+          - unique:
+              config: 
+                where: "touch_timestamp >= DATEADD(HOUR, -{{ var('test_hours') }}, current_timestamp())"
 
       - name: touch_user_id
         description: "Identifier for the user associated with the touch"
         tests:
-          - not_null
+          - not_null:
+              config: 
+                where: "touch_timestamp >= DATEADD(HOUR, -{{ var('test_hours') }}, current_timestamp())"
 
       - name: touch_event_id
         description: "Identifer that defines a unique touch event."
         tests:
-          - not_null
+          - not_null:
+              config: 
+                where: "touch_timestamp >= DATEADD(HOUR, -{{ var('test_hours') }}, current_timestamp())"
 
       - name: touch_timestamp
         description: "The timestamp when the touch event happened."
         tests:
-          - not_null
-          - in_past
+          - not_null:
+              config: 
+                where: "touch_timestamp >= DATEADD(HOUR, -{{ var('test_hours') }}, current_timestamp())"
+          - in_past:
+              config: 
+                where: "touch_timestamp >= DATEADD(HOUR, -{{ var('test_hours') }}, current_timestamp())"
 
       - name: model_id
         description: "Identifer that specifies the attribution model that the attribution relates to."
         tests:
-          - not_null
+          - not_null:
+              config: 
+                where: "touch_timestamp >= DATEADD(HOUR, -{{ var('test_hours') }}, current_timestamp())"
 
       - name: touch_category
         description: "The category of the touch, as defined in the touch rules seed."
         tests:
-          - not_null
+          - not_null:
+              config: 
+                where: "touch_timestamp >= DATEADD(HOUR, -{{ var('test_hours') }}, current_timestamp())"