Remove latest_shard_to_retrieve

Velir · Mar 26, 2024 · 16e830e · 16e830e
1 parent 5d829f1
commit 16e830e
Showing 1 changed file with 3 additions and 5 deletions.
diff --git a/macros/combine_property_data.sql b/macros/combine_property_data.sql
@@ -11,8 +11,6 @@
         {%- set earliest_shard_to_retrieve = var('start_date')|int -%}
     {% endif %}
 
-    {%- set latest_shard_to_retrieve = var('end_date', modules.datetime.date.today()|string|replace("-", ""))|int -%}
-
     {% for property_id in var('property_ids') %}
         {%- set schema_name = "analytics_" + property_id|string -%}
 
@@ -23,7 +21,7 @@
             {%- set relations = dbt_utils.get_relations_by_pattern(schema_pattern=schema_name, table_pattern='events_intraday_%', database=var('source_project')) -%}
             {% for relation in relations %}
                 {%- set relation_suffix = relation.identifier|replace('events_intraday_', '') -%}
-                {%- if earliest_shard_to_retrieve|int <= relation_suffix|int <= latest_shard_to_retrieve|int -%}
+                {%- if relation_suffix|int >= earliest_shard_to_retrieve|int -%}
                     create or replace table `{{target.project}}.{{var('combined_dataset')}}.events_intraday_{{relation_suffix}}{{property_id}}` clone `{{var('source_project')}}.analytics_{{property_id}}.events_intraday_{{relation_suffix}}`;
                 {%- endif -%}
             {% endfor %}
@@ -32,7 +30,7 @@
             {%- set relations = dbt_utils.get_relations_by_pattern(schema_pattern=schema_name, table_pattern='events_%', exclude='events_intraday_%', database=var('source_project')) -%}
             {% for relation in relations %}
                 {%- set relation_suffix = relation.identifier|replace('events_', '') -%}
-                {%- if earliest_shard_to_retrieve|int <= relation_suffix|int <= latest_shard_to_retrieve|int -%}
+                {%- if relation_suffix|int >= earliest_shard_to_retrieve|int -%}
                     create or replace table `{{target.project}}.{{var('combined_dataset')}}.events_{{relation_suffix}}{{property_id}}` clone `{{var('source_project')}}.analytics_{{property_id}}.events_{{relation_suffix}}`;
                     drop table if exists `{{target.project}}.{{var('combined_dataset')}}.events_intraday_{{relation_suffix}}{{property_id}}`;
                 {%- endif -%}
@@ -42,7 +40,7 @@
         {% do run_query(combine_specified_property_data_query) %}
 
         {% if execute %}
-            {{ log("Cloned from `" ~ var('source_project') ~ ".analytics_" ~ property_id ~ ".events_*[" ~ earliest_shard_to_retrieve ~ "-" ~ latest_shard_to_retrieve ~ "]` to `" ~ target.project ~ "." ~ var('combined_dataset') ~ ".events_YYYYMMDD" ~ property_id ~ "`.", True) }}
+            {{ log("Cloned from `" ~ var('source_project') ~ ".analytics_" ~ property_id ~ ".events_*` to `" ~ target.project ~ "." ~ var('combined_dataset') ~ ".events_YYYYMMDD" ~ property_id ~ "`.", True) }}
         {% endif %}
     {% endfor %}
 {% endmacro %}