diff --git a/integration_tests/data/materializations/bigquery/data_incremental_expected.csv b/integration_tests/data/materializations/bigquery/data_incremental_expected.csv index e90c4830..5bad3085 100644 --- a/integration_tests/data/materializations/bigquery/data_incremental_expected.csv +++ b/integration_tests/data/materializations/bigquery/data_incremental_expected.csv @@ -1,8 +1,8 @@ -id,start_tstamp -1,2021-01-01 00:00:00 -1,2021-03-05 00:00:00 -2,2021-03-02 00:00:00 -2,2021-03-02 00:00:00 -3,2021-03-07 00:00:00 -4,2021-03-04 00:00:00 -5,2021-03-08 00:00:00 +id,id2,start_tstamp +1,1,2021-01-01 00:00:00 +1,1,2021-03-05 00:00:00 +2,2,2021-03-02 00:00:00 +2,2,2021-03-02 00:00:00 +3,3,2021-03-07 00:00:00 +4,4,2021-03-04 00:00:00 +5,5,2021-03-08 00:00:00 diff --git a/integration_tests/data/materializations/data_incremental.csv b/integration_tests/data/materializations/data_incremental.csv index d11ba2ba..851501be 100644 --- a/integration_tests/data/materializations/data_incremental.csv +++ b/integration_tests/data/materializations/data_incremental.csv @@ -1,10 +1,10 @@ -run,id,start_tstamp -1,1,2021-01-01 00:00:00 -1,2,2021-03-01 00:00:00 -1,2,2021-03-03 00:00:00 -1,3,2021-03-03 00:00:00 -1,4,2021-03-04 00:00:00 -2,1,2021-03-05 00:00:00 -2,2,2021-03-02 00:00:00 -2,3,2021-03-07 00:00:00 -2,5,2021-03-08 00:00:00 +run,id,id2,start_tstamp +1,1,1,2021-01-01 00:00:00 +1,2,2,2021-03-01 00:00:00 +1,2,2,2021-03-03 00:00:00 +1,3,3,2021-03-03 00:00:00 +1,4,4,2021-03-04 00:00:00 +2,1,1,2021-03-05 00:00:00 +2,2,2,2021-03-02 00:00:00 +2,3,3,2021-03-07 00:00:00 +2,5,5,2021-03-08 00:00:00 diff --git a/integration_tests/data/materializations/databricks/data_incremental_expected.csv b/integration_tests/data/materializations/databricks/data_incremental_expected.csv index e90c4830..5bad3085 100644 --- a/integration_tests/data/materializations/databricks/data_incremental_expected.csv +++ b/integration_tests/data/materializations/databricks/data_incremental_expected.csv @@ -1,8 +1,8 @@ -id,start_tstamp -1,2021-01-01 00:00:00 -1,2021-03-05 00:00:00 -2,2021-03-02 00:00:00 -2,2021-03-02 00:00:00 -3,2021-03-07 00:00:00 -4,2021-03-04 00:00:00 -5,2021-03-08 00:00:00 +id,id2,start_tstamp +1,1,2021-01-01 00:00:00 +1,1,2021-03-05 00:00:00 +2,2,2021-03-02 00:00:00 +2,2,2021-03-02 00:00:00 +3,3,2021-03-07 00:00:00 +4,4,2021-03-04 00:00:00 +5,5,2021-03-08 00:00:00 diff --git a/integration_tests/data/materializations/default/data_incremental_expected.csv b/integration_tests/data/materializations/default/data_incremental_expected.csv index d350319d..2c1eb458 100644 --- a/integration_tests/data/materializations/default/data_incremental_expected.csv +++ b/integration_tests/data/materializations/default/data_incremental_expected.csv @@ -1,7 +1,7 @@ -id,start_tstamp -1,2021-01-01 00:00:00 -4,2021-03-04 00:00:00 -5,2021-03-08 00:00:00 -1,2021-03-05 00:00:00 -2,2021-03-02 00:00:00 -3,2021-03-07 00:00:00 +id,id2,start_tstamp +1,1,2021-01-01 00:00:00 +4,4,2021-03-04 00:00:00 +5,5,2021-03-08 00:00:00 +1,1,2021-03-05 00:00:00 +2,2,2021-03-02 00:00:00 +3,3,2021-03-07 00:00:00 diff --git a/integration_tests/data/materializations/snowflake/data_incremental_expected.csv b/integration_tests/data/materializations/snowflake/data_incremental_expected.csv index 1fa3b843..3173b9b6 100644 --- a/integration_tests/data/materializations/snowflake/data_incremental_expected.csv +++ b/integration_tests/data/materializations/snowflake/data_incremental_expected.csv @@ -1,7 +1,7 @@ -id,start_tstamp -1,2021-01-01 00:00:00 -1,2021-03-05 00:00:00 -2,2021-03-02 00:00:00 -3,2021-03-07 00:00:00 -4,2021-03-04 00:00:00 -5,2021-03-08 00:00:00 +id,id2,start_tstamp +1,1,2021-01-01 00:00:00 +1,1,2021-03-05 00:00:00 +2,2,2021-03-02 00:00:00 +3,3,2021-03-07 00:00:00 +4,4,2021-03-04 00:00:00 +5,5,2021-03-08 00:00:00 diff --git a/integration_tests/models/materializations/default_strategy/test_incremental.sql b/integration_tests/models/materializations/default_strategy/test_incremental.sql index 01ff137f..9b08b3dc 100644 --- a/integration_tests/models/materializations/default_strategy/test_incremental.sql +++ b/integration_tests/models/materializations/default_strategy/test_incremental.sql @@ -12,7 +12,7 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0 {{ config( materialized='incremental', - unique_key='id', + unique_key=['id', 'id2'], upsert_date_key='start_tstamp', partition_by = snowplow_utils.get_value_by_target_type(bigquery_val={ "field": "start_tstamp", @@ -36,6 +36,7 @@ with data as ( select id, + id2, start_tstamp from data @@ -45,6 +46,7 @@ with data as ( select id, + id2, start_tstamp from data diff --git a/integration_tests/models/materializations/snowflake_delete_insert/test_incremental_delete_insert.sql b/integration_tests/models/materializations/snowflake_delete_insert/test_incremental_delete_insert.sql index 3c55c04c..d405d09b 100644 --- a/integration_tests/models/materializations/snowflake_delete_insert/test_incremental_delete_insert.sql +++ b/integration_tests/models/materializations/snowflake_delete_insert/test_incremental_delete_insert.sql @@ -13,7 +13,7 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0 config( materialized='incremental', incremental_strategy='delete+insert', - unique_key='id', + unique_key=['id', 'id2'], upsert_date_key='start_tstamp', tags=["requires_script"], snowplow_optimize=true @@ -28,6 +28,7 @@ with data as ( select id, + id2, start_tstamp from data @@ -37,6 +38,7 @@ with data as ( select id, + id2, start_tstamp from data diff --git a/macros/materializations/base_incremental/common/get_merge_sql.sql b/macros/materializations/base_incremental/common/get_merge_sql.sql index 33689e93..23145616 100644 --- a/macros/materializations/base_incremental/common/get_merge_sql.sql +++ b/macros/materializations/base_incremental/common/get_merge_sql.sql @@ -99,7 +99,7 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0 {% set lower_limit, upper_limit = limits[0], limits[1] %} -- use those calculated min + max values to limit 'target' scan, to only the days with new data {% set predicate_override %} - {{ date_column }} between '{{ lower_limit }}' and '{{ upper_limit }}' + {{target_tb}}.{{ date_column }} between '{{ lower_limit }}' and '{{ upper_limit }}' {% endset %} {% endif %} {# Combine predicates with user provided ones #}