From afa37e63f0f7cd1980529de5b0635dce7b8b1f7e Mon Sep 17 00:00:00 2001 From: Leopoldo Araujo <68853890+poloaraujo@users.noreply.github.com> Date: Mon, 2 Oct 2023 12:30:34 -0400 Subject: [PATCH 1/8] Formatted staging models --- models/staging/stg_dbt__exposures.sql | 45 ++++++++------- models/staging/stg_dbt__invocations.sql | 55 +++++++++--------- models/staging/stg_dbt__model_executions.sql | 57 ++++++++++--------- models/staging/stg_dbt__models.sql | 47 +++++++-------- models/staging/stg_dbt__seed_executions.sql | 51 +++++++++-------- models/staging/stg_dbt__seeds.sql | 41 ++++++------- .../staging/stg_dbt__snapshot_executions.sql | 51 +++++++++-------- models/staging/stg_dbt__snapshots.sql | 45 ++++++++------- models/staging/stg_dbt__sources.sql | 41 ++++++------- models/staging/stg_dbt__test_executions.sql | 43 +++++++------- models/staging/stg_dbt__tests.sql | 35 ++++++------ 11 files changed, 261 insertions(+), 250 deletions(-) diff --git a/models/staging/stg_dbt__exposures.sql b/models/staging/stg_dbt__exposures.sql index d78417db..c6d20053 100644 --- a/models/staging/stg_dbt__exposures.sql +++ b/models/staging/stg_dbt__exposures.sql @@ -1,29 +1,30 @@ -with base as ( +with + base as ( - select * - from {{ ref('exposures') }} + select * + from {{ ref('exposures') }} -), + ) -enhanced as ( + , enhanced as ( - select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as exposure_execution_id, - command_invocation_id, - node_id, - run_started_at, - name, - type, - owner, - maturity, - path, - description, - url, - package_name, - depends_on_nodes, - tags - from base + select + {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as exposure_execution_id + , command_invocation_id + , node_id + , run_started_at + , name + , type + , owner + , maturity + , path + , description + , url + , package_name + , depends_on_nodes + , tags + from base -) + ) select * from enhanced diff --git a/models/staging/stg_dbt__invocations.sql b/models/staging/stg_dbt__invocations.sql index 824c13bd..7537d700 100644 --- a/models/staging/stg_dbt__invocations.sql +++ b/models/staging/stg_dbt__invocations.sql @@ -1,34 +1,35 @@ -with base as ( +with + base as ( - select * - from {{ ref('invocations') }} + select * + from {{ ref('invocations') }} -), + ) -enhanced as ( + , enhanced as ( - select - command_invocation_id, - dbt_version, - project_name, - run_started_at, - dbt_command, - full_refresh_flag, - target_profile_name, - target_name, - target_schema, - target_threads, - dbt_cloud_project_id, - dbt_cloud_job_id, - dbt_cloud_run_id, - dbt_cloud_run_reason_category, - dbt_cloud_run_reason, - env_vars, - dbt_vars, - invocation_args, - dbt_custom_envs - from base + select + command_invocation_id + , dbt_version + , project_name + , run_started_at + , dbt_command + , full_refresh_flag + , target_profile_name + , target_name + , target_schema + , target_threads + , dbt_cloud_project_id + , dbt_cloud_job_id + , dbt_cloud_run_id + , dbt_cloud_run_reason_category + , dbt_cloud_run_reason + , env_vars + , dbt_vars + , invocation_args + , dbt_custom_envs + from base -) + ) select * from enhanced diff --git a/models/staging/stg_dbt__model_executions.sql b/models/staging/stg_dbt__model_executions.sql index 1ab0b5a5..02becca5 100644 --- a/models/staging/stg_dbt__model_executions.sql +++ b/models/staging/stg_dbt__model_executions.sql @@ -1,35 +1,36 @@ -with base as ( +with + base as ( - select * - from {{ ref('model_executions') }} + select * + from {{ ref('model_executions') }} -), + ) -enhanced as ( + , enhanced as ( - select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as model_execution_id, - command_invocation_id, - node_id, - run_started_at, - was_full_refresh, - {{ split_part('thread_id', "'-'", 2) }} as thread_id, - status, - compile_started_at, - query_completed_at, - total_node_runtime, - rows_affected, - {% if target.type == 'bigquery' %} - bytes_processed, - {% endif %} - materialization, - schema, -- noqa - name, - alias, - message, - adapter_response - from base + select + {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as model_execution_id + , command_invocation_id + , node_id + , run_started_at + , was_full_refresh + , {{ split_part('thread_id', "'-'", 2) }} as thread_id + , status + , compile_started_at + , query_completed_at + , total_node_runtime + , rows_affected + , {% if target.type == 'bigquery' % + , bytes_processed + , {% endif % + , materialization + , schema -- noqa + , name + , alias + , message + , adapter_response + from base -) + ) select * from enhanced diff --git a/models/staging/stg_dbt__models.sql b/models/staging/stg_dbt__models.sql index db896a4e..0af3b491 100644 --- a/models/staging/stg_dbt__models.sql +++ b/models/staging/stg_dbt__models.sql @@ -1,30 +1,31 @@ -with base as ( +with + base as ( - select * - from {{ ref('models') }} + select * + from {{ ref('models') }} -), + ) -enhanced as ( + , enhanced as ( - select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as model_execution_id, - command_invocation_id, - node_id, - run_started_at, - database, - schema, - name, - depends_on_nodes, - package_name, - path, - checksum, - materialization, - tags, - meta, - alias - from base + select + {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as model_execution_id + , command_invocation_id + , node_id + , run_started_at + , database + , schema + , name + , depends_on_nodes + , package_name + , path + , checksum + , materialization + , tags + , meta + , alias + from base -) + ) select * from enhanced diff --git a/models/staging/stg_dbt__seed_executions.sql b/models/staging/stg_dbt__seed_executions.sql index a8bba7a7..ec90aaf0 100644 --- a/models/staging/stg_dbt__seed_executions.sql +++ b/models/staging/stg_dbt__seed_executions.sql @@ -1,32 +1,33 @@ -with base as ( +with + base as ( - select * - from {{ ref('seed_executions') }} + select * + from {{ ref('seed_executions') }} -), + ) -enhanced as ( + , enhanced as ( - select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as seed_execution_id, - command_invocation_id, - node_id, - run_started_at, - was_full_refresh, - {{ split_part('thread_id', "'-'", 2) }} as thread_id, - status, - compile_started_at, - query_completed_at, - total_node_runtime, - rows_affected, - materialization, - schema, -- noqa - name, - alias, - message, - adapter_response - from base + select + {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as seed_execution_id + , command_invocation_id + , node_id + , run_started_at + , was_full_refresh + , {{ split_part('thread_id', "'-'", 2) }} as thread_id + , status + , compile_started_at + , query_completed_at + , total_node_runtime + , rows_affected + , materialization + , schema -- noqa + , name + , alias + , message + , adapter_response + from base -) + ) select * from enhanced diff --git a/models/staging/stg_dbt__seeds.sql b/models/staging/stg_dbt__seeds.sql index 3a2a66fa..6e652631 100644 --- a/models/staging/stg_dbt__seeds.sql +++ b/models/staging/stg_dbt__seeds.sql @@ -1,27 +1,28 @@ -with base as ( +with + base as ( - select * - from {{ ref('seeds') }} + select * + from {{ ref('seeds') }} -), + ) -enhanced as ( + , enhanced as ( - select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as seed_execution_id, - command_invocation_id, - node_id, - run_started_at, - database, - schema, - name, - package_name, - path, - checksum, - meta, - alias - from base + select + {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as seed_execution_id + , command_invocation_id + , node_id + , run_started_at + , database + , schema + , name + , package_name + , path + , checksum + , meta + , alias + from base -) + ) select * from enhanced diff --git a/models/staging/stg_dbt__snapshot_executions.sql b/models/staging/stg_dbt__snapshot_executions.sql index 3ee0c050..207e7691 100644 --- a/models/staging/stg_dbt__snapshot_executions.sql +++ b/models/staging/stg_dbt__snapshot_executions.sql @@ -1,32 +1,33 @@ -with base as ( +with + base as ( - select * - from {{ ref('snapshot_executions') }} + select * + from {{ ref('snapshot_executions') }} -), + ) -enhanced as ( + , enhanced as ( - select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as snapshot_execution_id, - command_invocation_id, - node_id, - run_started_at, - was_full_refresh, - {{ split_part('thread_id', "'-'", 2) }} as thread_id, - status, - compile_started_at, - query_completed_at, - total_node_runtime, - rows_affected, - materialization, - schema, -- noqa - name, - alias, - message, - adapter_response - from base + select + , {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as snapshot_execution_id + , command_invocation_id + , node_id + , run_started_at + , was_full_refresh + , {{ split_part('thread_id', "'-'", 2) }} as thread_id + , status + , compile_started_at + , query_completed_at + , total_node_runtime + , rows_affected + , materialization + , schema -- noqa + , name + , alias + , message + , adapter_response + from base -) + ) select * from enhanced diff --git a/models/staging/stg_dbt__snapshots.sql b/models/staging/stg_dbt__snapshots.sql index f5236e87..7571e14b 100644 --- a/models/staging/stg_dbt__snapshots.sql +++ b/models/staging/stg_dbt__snapshots.sql @@ -1,29 +1,30 @@ -with base as ( +with + base as ( - select * - from {{ ref('snapshots') }} + select * + from {{ ref('snapshots') }} -), + ) -enhanced as ( + , enhanced as ( - select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as snapshot_execution_id, - command_invocation_id, - node_id, - run_started_at, - database, - schema, - name, - depends_on_nodes, - package_name, - path, - checksum, - strategy, - meta, - alias - from base + select + {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as snapshot_execution_id + , command_invocation_id + , node_id + , run_started_at + , database + , schema + , name + , depends_on_nodes + , package_name + , path + , checksum + , strategy + , meta + , alias + from base -) + ) select * from enhanced diff --git a/models/staging/stg_dbt__sources.sql b/models/staging/stg_dbt__sources.sql index f18a80e9..aca60a43 100644 --- a/models/staging/stg_dbt__sources.sql +++ b/models/staging/stg_dbt__sources.sql @@ -1,27 +1,28 @@ -with base as ( +with + base as ( - select * - from {{ ref('sources') }} + select * + from {{ ref('sources') }} -), + ) -enhanced as ( + , enhanced as ( - select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as source_execution_id, - command_invocation_id, - node_id, - run_started_at, - database, - schema, - source_name, - loader, - name, - identifier, - loaded_at_field, - freshness - from base + select + {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as source_execution_id + , command_invocation_id + , node_id + , run_started_at + , database + , schema + , source_name + , loader + , name + , identifier + , loaded_at_field + , freshness + from base -) + ) select * from enhanced diff --git a/models/staging/stg_dbt__test_executions.sql b/models/staging/stg_dbt__test_executions.sql index ca7ea868..9fdb3a5f 100644 --- a/models/staging/stg_dbt__test_executions.sql +++ b/models/staging/stg_dbt__test_executions.sql @@ -1,28 +1,29 @@ -with base as ( +with + base as ( - select * - from {{ ref('test_executions') }} + select * + from {{ ref('test_executions') }} -), + ) -enhanced as ( + , enhanced as ( - select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as test_execution_id, - command_invocation_id, - node_id, - run_started_at, - was_full_refresh, - {{ split_part('thread_id', "'-'", 2) }} as thread_id, - status, - compile_started_at, - query_completed_at, - total_node_runtime, - rows_affected, - failures, - message - from base + select + {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as test_execution_id + , command_invocation_id + , node_id + , run_started_at + , was_full_refresh + , {{ split_part('thread_id', "'-'", 2) }} as thread_id + , status + , compile_started_at + , query_completed_at + , total_node_runtime + , rows_affected + , failures + , message + from base -) + ) select * from enhanced diff --git a/models/staging/stg_dbt__tests.sql b/models/staging/stg_dbt__tests.sql index 1e726de7..5f4a1b10 100644 --- a/models/staging/stg_dbt__tests.sql +++ b/models/staging/stg_dbt__tests.sql @@ -1,24 +1,25 @@ -with base as ( +with + base as ( - select * - from {{ ref('tests') }} + select * + from {{ ref('tests') }} -), + ) -enhanced as ( + , enhanced as ( - select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as test_execution_id, - command_invocation_id, - node_id, - run_started_at, - name, - depends_on_nodes, - package_name, - test_path, - tags - from base + select + {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as test_execution_id + , command_invocation_id + , node_id + , run_started_at + , name + , depends_on_nodes + , package_name + , test_path + , tags + from base -) + ) select * from enhanced From 7ab30f14bfe8c62d53d5857e1ca70b651c5ccbe3 Mon Sep 17 00:00:00 2001 From: Leopoldo Araujo <68853890+poloaraujo@users.noreply.github.com> Date: Mon, 2 Oct 2023 13:06:12 -0400 Subject: [PATCH 2/8] Formatted dim models --- models/dim_dbt__current_models.sql | 203 ++++++++++--------- models/dim_dbt__exposures.sql | 45 ++-- models/dim_dbt__models.sql | 47 ++--- models/dim_dbt__seeds.sql | 41 ++-- models/dim_dbt__snapshots.sql | 45 ++-- models/dim_dbt__sources.sql | 41 ++-- models/dim_dbt__tests.sql | 35 ++-- models/staging/stg_dbt__model_executions.sql | 6 +- 8 files changed, 241 insertions(+), 222 deletions(-) diff --git a/models/dim_dbt__current_models.sql b/models/dim_dbt__current_models.sql index 4d7a3c46..cb935256 100644 --- a/models/dim_dbt__current_models.sql +++ b/models/dim_dbt__current_models.sql @@ -1,97 +1,110 @@ -with base as ( - select * - from {{ ref('stg_dbt__models') }} -), - -model_executions as ( - select * - from {{ ref('stg_dbt__model_executions') }} -), - -latest_models as ( - /* Retrieves the models present in the most recent run */ - select * - from base - where run_started_at = (select max(run_started_at) from base) -), - -latest_models_runs as ( - /* Retreives all successful run information for the models present in the most - recent run and ranks them based on query completion time */ - select - model_executions.node_id - , model_executions.was_full_refresh - , model_executions.query_completed_at - , model_executions.total_node_runtime - , model_executions.rows_affected - {% if target.type == 'bigquery' %} - , model_executions.bytes_processed - {% endif %} - /* Row number by refresh and node ID */ - , row_number() over ( - partition by latest_models.node_id, model_executions.was_full_refresh - order by model_executions.query_completed_at desc /* most recent ranked first */ - ) as run_idx - /* Row number by node ID */ - , row_number() over ( - partition by latest_models.node_id - order by model_executions.query_completed_at desc /* most recent ranked first */ - ) as run_idx_id_only - from model_executions - inner join latest_models on model_executions.node_id = latest_models.node_id - where model_executions.status = 'success' -), - -latest_model_stats as ( - select - node_id - , max(case when was_full_refresh then query_completed_at end) as last_full_refresh_run_completed_at - , max(case when was_full_refresh then total_node_runtime end) as last_full_refresh_run_total_runtime - , max(case when was_full_refresh then rows_affected end) as last_full_refresh_run_rows_affected - {% if target.type == 'bigquery' %} - , max(case when was_full_refresh then bytes_processed end) as last_full_refresh_run_bytes_processed - {% endif %} - , max(case when run_idx_id_only = 1 then query_completed_at end) as last_run_completed_at - , max(case when run_idx_id_only = 1 then total_node_runtime end) as last_run_total_runtime - , max(case when run_idx_id_only = 1 then rows_affected end) as last_run_rows_affected - {% if target.type == 'bigquery' %} - , max(case when run_idx_id_only = 1 then bytes_processed end) as last_run_bytes_processed - {% endif %} - , max(case when not was_full_refresh then query_completed_at end) as last_incremental_run_completed_at - , max(case when not was_full_refresh then total_node_runtime end) as last_incremental_run_total_runtime - , max(case when not was_full_refresh then rows_affected end) as last_incremental_run_rows_affected - {% if target.type == 'bigquery' %} - , max(case when not was_full_refresh then bytes_processed end) as last_incremental_run_bytes_processed - {% endif %} - from latest_models_runs - where run_idx = 1 - group by 1 -), - -final as ( - select - latest_models.* - , latest_model_stats.last_full_refresh_run_completed_at - , latest_model_stats.last_full_refresh_run_total_runtime - , latest_model_stats.last_full_refresh_run_rows_affected - {% if target.type == 'bigquery' %} - , latest_model_stats.last_full_refresh_run_bytes_processed - {% endif %} - , latest_model_stats.last_run_completed_at - , latest_model_stats.last_run_total_runtime - , latest_model_stats.last_run_rows_affected - {% if target.type == 'bigquery' %} - , latest_model_stats.last_run_bytes_processed - {% endif %} - , latest_model_stats.last_incremental_run_completed_at - , latest_model_stats.last_incremental_run_total_runtime - , latest_model_stats.last_incremental_run_rows_affected - {% if target.type == 'bigquery' %} - , latest_model_stats.last_incremental_run_bytes_processed - {% endif %} - from latest_models - left join latest_model_stats - on latest_models.node_id = latest_model_stats.node_id -) +with + base as ( + + select * + from {{ ref('stg_dbt__models') }} + + ) + + , model_executions as ( + + select * + from {{ ref('stg_dbt__model_executions') }} + + ) + + , latest_models as ( + + /* Retrieves the models present in the most recent run */ + select * + from base + where run_started_at = (select max(run_started_at) from base) + + ) + + , latest_models_runs as ( + + /* Retreives all successful run information for the models present in the most + recent run and ranks them based on query completion time */ + select + model_executions.node_id + , model_executions.was_full_refresh + , model_executions.query_completed_at + , model_executions.total_node_runtime + , model_executions.rows_affected + {% if target.type == 'bigquery' %} + , model_executions.bytes_processed + {% endif %} + /* Row number by refresh and node ID */ + , row_number() over ( + partition by latest_models.node_id, model_executions.was_full_refresh + order by model_executions.query_completed_at desc /* most recent ranked first */ + ) as run_idx + /* Row number by node ID */ + , row_number() over ( + partition by latest_models.node_id + order by model_executions.query_completed_at desc /* most recent ranked first */ + ) as run_idx_id_only + from model_executions + inner join latest_models on model_executions.node_id = latest_models.node_id + where model_executions.status = 'success' + + ) + + , latest_model_stats as ( + + select + node_id + , max(case when was_full_refresh then query_completed_at end) as last_full_refresh_run_completed_at + , max(case when was_full_refresh then total_node_runtime end) as last_full_refresh_run_total_runtime + , max(case when was_full_refresh then rows_affected end) as last_full_refresh_run_rows_affected + {% if target.type == 'bigquery' %} + , max(case when was_full_refresh then bytes_processed end) as last_full_refresh_run_bytes_processed + {% endif %} + , max(case when run_idx_id_only = 1 then query_completed_at end) as last_run_completed_at + , max(case when run_idx_id_only = 1 then total_node_runtime end) as last_run_total_runtime + , max(case when run_idx_id_only = 1 then rows_affected end) as last_run_rows_affected + {% if target.type == 'bigquery' %} + , max(case when run_idx_id_only = 1 then bytes_processed end) as last_run_bytes_processed + {% endif %} + , max(case when not was_full_refresh then query_completed_at end) as last_incremental_run_completed_at + , max(case when not was_full_refresh then total_node_runtime end) as last_incremental_run_total_runtime + , max(case when not was_full_refresh then rows_affected end) as last_incremental_run_rows_affected + {% if target.type == 'bigquery' %} + , max(case when not was_full_refresh then bytes_processed end) as last_incremental_run_bytes_processed + {% endif %} + from latest_models_runs + where run_idx = 1 + group by 1 + + ) + + , final as ( + + select + latest_models.* + , latest_model_stats.last_full_refresh_run_completed_at + , latest_model_stats.last_full_refresh_run_total_runtime + , latest_model_stats.last_full_refresh_run_rows_affected + {% if target.type == 'bigquery' %} + , latest_model_stats.last_full_refresh_run_bytes_processed + {% endif %} + , latest_model_stats.last_run_completed_at + , latest_model_stats.last_run_total_runtime + , latest_model_stats.last_run_rows_affected + {% if target.type == 'bigquery' %} + , latest_model_stats.last_run_bytes_processed + {% endif %} + , latest_model_stats.last_incremental_run_completed_at + , latest_model_stats.last_incremental_run_total_runtime + , latest_model_stats.last_incremental_run_rows_affected + {% if target.type == 'bigquery' %} + , latest_model_stats.last_incremental_run_bytes_processed + {% endif %} + from latest_models + left join latest_model_stats + on latest_models.node_id = latest_model_stats.node_id + + ) select * from final diff --git a/models/dim_dbt__exposures.sql b/models/dim_dbt__exposures.sql index 6a2202b7..dc653382 100644 --- a/models/dim_dbt__exposures.sql +++ b/models/dim_dbt__exposures.sql @@ -1,29 +1,30 @@ -with base as ( +with + base as ( - select * - from {{ ref('stg_dbt__exposures') }} + select * + from {{ ref('stg_dbt__exposures') }} -), + ) -exposures as ( + , exposures as ( - select - exposure_execution_id, - command_invocation_id, - node_id, - run_started_at, - name, - type, - owner, - maturity, - path, - description, - url, - package_name, - depends_on_nodes, - tags - from base + select + exposure_execution_id + command_invocation_id + node_id + run_started_at + name + type + owner + maturity + path + description + url + package_name + depends_on_nodes + tags + from base -) + ) select * from exposures diff --git a/models/dim_dbt__models.sql b/models/dim_dbt__models.sql index 2722ce83..8ed90ce0 100644 --- a/models/dim_dbt__models.sql +++ b/models/dim_dbt__models.sql @@ -1,30 +1,31 @@ -with base as ( +with + base as ( - select * - from {{ ref('stg_dbt__models') }} + select * + from {{ ref('stg_dbt__models') }} -), + ) -models as ( + , models as ( - select - model_execution_id, - command_invocation_id, - node_id, - run_started_at, - database, - schema, - name, - depends_on_nodes, - package_name, - path, - checksum, - materialization, - tags, - meta, - alias - from base + select + model_execution_id + , command_invocation_id + , node_id + , run_started_at + , database + , schema + , name + , depends_on_nodes + , package_name + , path + , checksum + , materialization + , tags + , meta + , alias + from base -) + ) select * from models diff --git a/models/dim_dbt__seeds.sql b/models/dim_dbt__seeds.sql index d7a461ab..6478d309 100644 --- a/models/dim_dbt__seeds.sql +++ b/models/dim_dbt__seeds.sql @@ -1,27 +1,28 @@ -with base as ( +with + base as ( - select * - from {{ ref('stg_dbt__seeds') }} + select * + from {{ ref('stg_dbt__seeds') }} -), + ) -seeds as ( + , seeds as ( - select - seed_execution_id, - command_invocation_id, - node_id, - run_started_at, - database, - schema, - name, - package_name, - path, - checksum, - meta, - alias - from base + select + seed_execution_id + command_invocation_id + node_id + run_started_at + database + schema + name + package_name + path + checksum + meta + alias + from base -) + ) select * from seeds diff --git a/models/dim_dbt__snapshots.sql b/models/dim_dbt__snapshots.sql index 13f05d0e..a3270714 100644 --- a/models/dim_dbt__snapshots.sql +++ b/models/dim_dbt__snapshots.sql @@ -1,29 +1,30 @@ -with base as ( +with + base as ( - select * - from {{ ref('stg_dbt__snapshots') }} + select * + from {{ ref('stg_dbt__snapshots') }} -), + ) -snapshots as ( + , snapshots as ( - select - snapshot_execution_id, - command_invocation_id, - node_id, - run_started_at, - database, - schema, - name, - depends_on_nodes, - package_name, - path, - checksum, - strategy, - meta, - alias - from base + select + snapshot_execution_id + , command_invocation_id + , node_id + , run_started_at + , database + , schema + , name + , depends_on_nodes + , package_name + , path + , checksum + , strategy + , meta + , alias + from base -) + ) select * from snapshots diff --git a/models/dim_dbt__sources.sql b/models/dim_dbt__sources.sql index 84b76d4d..d06bccb2 100644 --- a/models/dim_dbt__sources.sql +++ b/models/dim_dbt__sources.sql @@ -1,27 +1,28 @@ -with base as ( +with + base as ( - select * - from {{ ref('stg_dbt__sources') }} + select * + from {{ ref('stg_dbt__sources') }} -), + ) -sources as ( + sources as ( - select - source_execution_id, - command_invocation_id, - node_id, - run_started_at, - database, - schema, - source_name, - loader, - name, - identifier, - loaded_at_field, - freshness - from base + select + source_execution_id + , command_invocation_id + , node_id + , run_started_at + , database + , schema + , source_name + , loader + , name + , identifier + , loaded_at_field + , freshness + from base -) + ) select * from sources diff --git a/models/dim_dbt__tests.sql b/models/dim_dbt__tests.sql index c56504e9..b873786c 100644 --- a/models/dim_dbt__tests.sql +++ b/models/dim_dbt__tests.sql @@ -1,24 +1,25 @@ -with base as ( +with + base as ( - select * - from {{ ref('stg_dbt__tests') }} + select * + from {{ ref('stg_dbt__tests') }} -), + ) -tests as ( + , tests as ( - select - test_execution_id, - command_invocation_id, - node_id, - run_started_at, - name, - depends_on_nodes, - package_name, - test_path, - tags - from base + select + test_execution_id + , command_invocation_id + , node_id + , run_started_at + , name + , depends_on_nodes + , package_name + , test_path + , tags + from base -) + ) select * from tests diff --git a/models/staging/stg_dbt__model_executions.sql b/models/staging/stg_dbt__model_executions.sql index 02becca5..e264f4a4 100644 --- a/models/staging/stg_dbt__model_executions.sql +++ b/models/staging/stg_dbt__model_executions.sql @@ -20,9 +20,9 @@ with , query_completed_at , total_node_runtime , rows_affected - , {% if target.type == 'bigquery' % - , bytes_processed - , {% endif % + , {% if target.type == 'bigquery' %} + , bytes_processed + {% endif %} , materialization , schema -- noqa , name From bb4a42ab505d086779e996ed327e5228fd4ab413 Mon Sep 17 00:00:00 2001 From: Leopoldo Araujo <68853890+poloaraujo@users.noreply.github.com> Date: Mon, 2 Oct 2023 13:16:17 -0400 Subject: [PATCH 3/8] Formatted fact models --- models/dim_dbt__exposures.sql | 26 ++++----- models/dim_dbt__seeds.sql | 22 ++++---- models/dim_dbt__sources.sql | 2 +- models/fct_dbt__invocations.sql | 55 ++++++++++---------- models/fct_dbt__model_executions.sql | 55 ++++++++++---------- models/fct_dbt__seed_executions.sql | 49 ++++++++--------- models/fct_dbt__snapshot_executions.sql | 49 ++++++++--------- models/fct_dbt__test_executions.sql | 43 +++++++-------- models/staging/stg_dbt__model_executions.sql | 6 +-- 9 files changed, 156 insertions(+), 151 deletions(-) diff --git a/models/dim_dbt__exposures.sql b/models/dim_dbt__exposures.sql index dc653382..71a70c99 100644 --- a/models/dim_dbt__exposures.sql +++ b/models/dim_dbt__exposures.sql @@ -10,19 +10,19 @@ with select exposure_execution_id - command_invocation_id - node_id - run_started_at - name - type - owner - maturity - path - description - url - package_name - depends_on_nodes - tags + , command_invocation_id + , node_id + , run_started_at + , name + , type + , owner + , maturity + , path + , description + , url + , package_name + , depends_on_nodes + , tags from base ) diff --git a/models/dim_dbt__seeds.sql b/models/dim_dbt__seeds.sql index 6478d309..d07c288d 100644 --- a/models/dim_dbt__seeds.sql +++ b/models/dim_dbt__seeds.sql @@ -10,17 +10,17 @@ with select seed_execution_id - command_invocation_id - node_id - run_started_at - database - schema - name - package_name - path - checksum - meta - alias + , command_invocation_id + , node_id + , run_started_at + , database + , schema + , name + , package_name + , path + , checksum + , meta + , alias from base ) diff --git a/models/dim_dbt__sources.sql b/models/dim_dbt__sources.sql index d06bccb2..affe5b99 100644 --- a/models/dim_dbt__sources.sql +++ b/models/dim_dbt__sources.sql @@ -6,7 +6,7 @@ with ) - sources as ( + , sources as ( select source_execution_id diff --git a/models/fct_dbt__invocations.sql b/models/fct_dbt__invocations.sql index 6a52890c..1ac51d41 100644 --- a/models/fct_dbt__invocations.sql +++ b/models/fct_dbt__invocations.sql @@ -1,34 +1,35 @@ -with base as ( +with + base as ( - select * - from {{ ref('stg_dbt__invocations') }} + select * + from {{ ref('stg_dbt__invocations') }} -), + ) -invocations as ( + , invocations as ( - select - command_invocation_id, - dbt_version, - project_name, - run_started_at, - dbt_command, - full_refresh_flag, - target_profile_name, - target_name, - target_schema, - target_threads, - dbt_cloud_project_id, - dbt_cloud_job_id, - dbt_cloud_run_id, - dbt_cloud_run_reason_category, - dbt_cloud_run_reason, - env_vars, - dbt_vars, - invocation_args, - dbt_custom_envs - from base + select + command_invocation_id + , dbt_version + , project_name + , run_started_at + , dbt_command + , full_refresh_flag + , target_profile_name + , target_name + , target_schema + , target_threads + , dbt_cloud_project_id + , dbt_cloud_job_id + , dbt_cloud_run_id + , dbt_cloud_run_reason_category + , dbt_cloud_run_reason + , env_vars + , dbt_vars + , invocation_args + , dbt_custom_envs + from base -) + ) select * from invocations diff --git a/models/fct_dbt__model_executions.sql b/models/fct_dbt__model_executions.sql index f78a7f93..c6c93548 100644 --- a/models/fct_dbt__model_executions.sql +++ b/models/fct_dbt__model_executions.sql @@ -1,34 +1,35 @@ -with base as ( +with + base as ( - select * - from {{ ref('stg_dbt__model_executions') }} + select * + from {{ ref('stg_dbt__model_executions') }} -), + ) -model_executions as ( + , model_executions as ( - select - model_execution_id, - command_invocation_id, - node_id, - run_started_at, - was_full_refresh, - thread_id, - status, - compile_started_at, - query_completed_at, - total_node_runtime, - rows_affected, - {% if target.type == 'bigquery' %} - bytes_processed, - {% endif %} - materialization, - schema, -- noqa - name, - alias, - message - from base + select + model_execution_id + , command_invocation_id + , node_id + , run_started_at + , was_full_refresh + , thread_id + , status + , compile_started_at + , query_completed_at + , total_node_runtime + , rows_affected + {% if target.type == 'bigquery' %} + , bytes_processed + {% endif %} + , materialization + , schema -- noqa + , name + , alias + , message + from base -) + ) select * from model_executions diff --git a/models/fct_dbt__seed_executions.sql b/models/fct_dbt__seed_executions.sql index e8e38365..39d2abb5 100644 --- a/models/fct_dbt__seed_executions.sql +++ b/models/fct_dbt__seed_executions.sql @@ -1,31 +1,32 @@ -with base as ( +with + base as ( - select * - from {{ ref('stg_dbt__seed_executions') }} + select * + from {{ ref('stg_dbt__seed_executions') }} -), + ) -seed_executions as ( + , seed_executions as ( - select - seed_execution_id, - command_invocation_id, - node_id, - run_started_at, - was_full_refresh, - thread_id, - status, - compile_started_at, - query_completed_at, - total_node_runtime, - rows_affected, - materialization, - schema, - name, - alias, - message - from base + select + seed_execution_id + , command_invocation_id + , node_id + , run_started_at + , was_full_refresh + , thread_id + , status + , compile_started_at + , query_completed_at + , total_node_runtime + , rows_affected + , materialization + , schema + , name + , alias + , message + from base -) + ) select * from seed_executions diff --git a/models/fct_dbt__snapshot_executions.sql b/models/fct_dbt__snapshot_executions.sql index 83c656f1..91613db0 100644 --- a/models/fct_dbt__snapshot_executions.sql +++ b/models/fct_dbt__snapshot_executions.sql @@ -1,31 +1,32 @@ -with base as ( +with + base as ( - select * - from {{ ref('stg_dbt__snapshot_executions') }} + select * + from {{ ref('stg_dbt__snapshot_executions') }} -), + ) -snapshot_executions as ( + , snapshot_executions as ( - select - snapshot_execution_id, - command_invocation_id, - node_id, - run_started_at, - was_full_refresh, - thread_id, - status, - compile_started_at, - query_completed_at, - total_node_runtime, - rows_affected, - materialization, - schema, - name, - alias, - message - from base + select + snapshot_execution_id + , command_invocation_id + , node_id + , run_started_at + , was_full_refresh + , thread_id + , status + , compile_started_at + , query_completed_at + , total_node_runtime + , rows_affected + , materialization + , schema + , name + , alias + , message + from base -) + ) select * from snapshot_executions diff --git a/models/fct_dbt__test_executions.sql b/models/fct_dbt__test_executions.sql index b921df43..43b87d33 100644 --- a/models/fct_dbt__test_executions.sql +++ b/models/fct_dbt__test_executions.sql @@ -1,28 +1,29 @@ -with base as ( +with + base as ( - select * - from {{ ref('stg_dbt__test_executions') }} + select * + from {{ ref('stg_dbt__test_executions') }} -), + ) -test_executions as ( + , test_executions as ( - select - test_execution_id, - command_invocation_id, - node_id, - run_started_at, - was_full_refresh, - thread_id, - status, - compile_started_at, - query_completed_at, - total_node_runtime, - rows_affected, - failures, - message - from base + select + test_execution_id + , command_invocation_id + , node_id + , run_started_at + , was_full_refresh + , thread_id + , status + , compile_started_at + , query_completed_at + , total_node_runtime + , rows_affected + , failures + , message + from base -) + ) select * from test_executions diff --git a/models/staging/stg_dbt__model_executions.sql b/models/staging/stg_dbt__model_executions.sql index e264f4a4..1381f6c4 100644 --- a/models/staging/stg_dbt__model_executions.sql +++ b/models/staging/stg_dbt__model_executions.sql @@ -20,9 +20,9 @@ with , query_completed_at , total_node_runtime , rows_affected - , {% if target.type == 'bigquery' %} - , bytes_processed - {% endif %} + {% if target.type == 'bigquery' %} + , bytes_processed + {% endif %} , materialization , schema -- noqa , name From b825bdfaad798345914fe55040ba455cb06f8a1e Mon Sep 17 00:00:00 2001 From: Leopoldo Araujo <68853890+poloaraujo@users.noreply.github.com> Date: Mon, 2 Oct 2023 13:48:51 -0400 Subject: [PATCH 4/8] rolled back fct_dbt__snapshot_executions --- models/fct_dbt__snapshot_executions.sql | 51 ++++++++++++------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/models/fct_dbt__snapshot_executions.sql b/models/fct_dbt__snapshot_executions.sql index 91613db0..bd0957a8 100644 --- a/models/fct_dbt__snapshot_executions.sql +++ b/models/fct_dbt__snapshot_executions.sql @@ -1,32 +1,31 @@ -with - base as ( +with base as ( - select * - from {{ ref('stg_dbt__snapshot_executions') }} + select * + from {{ ref('stg_dbt__snapshot_executions') }} - ) +), - , snapshot_executions as ( +snapshot_executions as ( - select - snapshot_execution_id - , command_invocation_id - , node_id - , run_started_at - , was_full_refresh - , thread_id - , status - , compile_started_at - , query_completed_at - , total_node_runtime - , rows_affected - , materialization - , schema - , name - , alias - , message - from base + select + snapshot_execution_id, + command_invocation_id, + node_id, + run_started_at, + was_full_refresh, + thread_id, + status, + compile_started_at, + query_completed_at, + total_node_runtime, + rows_affected, + materialization, + schema, + name, + alias, + message + from base - ) +) -select * from snapshot_executions +select * from snapshot_executions \ No newline at end of file From 504a683e852c5c56f43e0f14d4767fcd2aa97740 Mon Sep 17 00:00:00 2001 From: Leopoldo Araujo <68853890+poloaraujo@users.noreply.github.com> Date: Mon, 2 Oct 2023 13:59:00 -0400 Subject: [PATCH 5/8] Formatted sources Formatted sources and corrected snapshot_executions --- models/fct_dbt__snapshot_executions.sql | 51 ++++++++++--------- models/sources/exposures.sql | 37 +++++++------- models/sources/invocations.sql | 47 +++++++++-------- models/sources/model_executions.sql | 43 ++++++++-------- models/sources/models.sql | 39 +++++++------- models/sources/seed_executions.sql | 41 ++++++++------- models/sources/seeds.sql | 33 ++++++------ models/sources/snapshot_executions.sql | 41 ++++++++------- models/sources/snapshots.sql | 36 ++++++------- models/sources/sources.sql | 33 ++++++------ models/sources/test_executions.sql | 34 +++++++------ models/sources/tests.sql | 27 +++++----- .../staging/stg_dbt__snapshot_executions.sql | 2 +- 13 files changed, 247 insertions(+), 217 deletions(-) diff --git a/models/fct_dbt__snapshot_executions.sql b/models/fct_dbt__snapshot_executions.sql index bd0957a8..91613db0 100644 --- a/models/fct_dbt__snapshot_executions.sql +++ b/models/fct_dbt__snapshot_executions.sql @@ -1,31 +1,32 @@ -with base as ( +with + base as ( - select * - from {{ ref('stg_dbt__snapshot_executions') }} + select * + from {{ ref('stg_dbt__snapshot_executions') }} -), + ) -snapshot_executions as ( + , snapshot_executions as ( - select - snapshot_execution_id, - command_invocation_id, - node_id, - run_started_at, - was_full_refresh, - thread_id, - status, - compile_started_at, - query_completed_at, - total_node_runtime, - rows_affected, - materialization, - schema, - name, - alias, - message - from base + select + snapshot_execution_id + , command_invocation_id + , node_id + , run_started_at + , was_full_refresh + , thread_id + , status + , compile_started_at + , query_completed_at + , total_node_runtime + , rows_affected + , materialization + , schema + , name + , alias + , message + from base -) + ) -select * from snapshot_executions \ No newline at end of file +select * from snapshot_executions diff --git a/models/sources/exposures.sql b/models/sources/exposures.sql index 9b6e48b3..6d593bcf 100644 --- a/models/sources/exposures.sql +++ b/models/sources/exposures.sql @@ -1,22 +1,25 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with dummy_cte as ( - select 1 as foo -) +with + dummy_cte as ( + + select 1 as foo + + ) select - cast(null as {{ type_string() }}) as command_invocation_id, - cast(null as {{ type_string() }}) as node_id, - cast(null as {{ type_timestamp() }}) as run_started_at, - cast(null as {{ type_string() }}) as name, - cast(null as {{ type_string() }}) as type, - cast(null as {{ type_json() }}) as owner, - cast(null as {{ type_string() }}) as maturity, - cast(null as {{ type_string() }}) as path, - cast(null as {{ type_string() }}) as description, - cast(null as {{ type_string() }}) as url, - cast(null as {{ type_string() }}) as package_name, - cast(null as {{ type_array() }}) as depends_on_nodes, - cast(null as {{ type_array() }}) as tags, - cast(null as {{ type_json() }}) as all_results + cast(null as {{ type_string() }}) as command_invocation_id + , cast(null as {{ type_string() }}) as node_id + , cast(null as {{ type_timestamp() }}) as run_started_at + , cast(null as {{ type_string() }}) as name + , cast(null as {{ type_string() }}) as type + , cast(null as {{ type_json() }}) as owner + , cast(null as {{ type_string() }}) as maturity + , cast(null as {{ type_string() }}) as path + , cast(null as {{ type_string() }}) as description + , cast(null as {{ type_string() }}) as url + , cast(null as {{ type_string() }}) as package_name + , cast(null as {{ type_array() }}) as depends_on_nodes + , cast(null as {{ type_array() }}) as tags + , cast(null as {{ type_json() }}) as all_results from dummy_cte where 1 = 0 diff --git a/models/sources/invocations.sql b/models/sources/invocations.sql index 7cdabf94..d92c8bfb 100644 --- a/models/sources/invocations.sql +++ b/models/sources/invocations.sql @@ -1,27 +1,30 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with dummy_cte as ( - select 1 as foo -) +with + dummy_cte as ( + + select 1 as foo + + ) select - cast(null as {{ type_string() }}) as command_invocation_id, - cast(null as {{ type_string() }}) as dbt_version, - cast(null as {{ type_string() }}) as project_name, - cast(null as {{ type_timestamp() }}) as run_started_at, - cast(null as {{ type_string() }}) as dbt_command, - cast(null as {{ type_boolean() }}) as full_refresh_flag, - cast(null as {{ type_string() }}) as target_profile_name, - cast(null as {{ type_string() }}) as target_name, - cast(null as {{ type_string() }}) as target_schema, - cast(null as {{ type_int() }}) as target_threads, - cast(null as {{ type_string() }}) as dbt_cloud_project_id, - cast(null as {{ type_string() }}) as dbt_cloud_job_id, - cast(null as {{ type_string() }}) as dbt_cloud_run_id, - cast(null as {{ type_string() }}) as dbt_cloud_run_reason_category, - cast(null as {{ type_string() }}) as dbt_cloud_run_reason, - cast(null as {{ type_json() }}) as env_vars, - cast(null as {{ type_json() }}) as dbt_vars, - cast(null as {{ type_json() }}) as invocation_args, - cast(null as {{ type_json() }}) as dbt_custom_envs + cast(null as {{ type_string() }}) as command_invocation_id + , cast(null as {{ type_string() }}) as dbt_version + , cast(null as {{ type_string() }}) as project_name + , cast(null as {{ type_timestamp() }}) as run_started_at + , cast(null as {{ type_string() }}) as dbt_command + , cast(null as {{ type_boolean() }}) as full_refresh_flag + , cast(null as {{ type_string() }}) as target_profile_name + , cast(null as {{ type_string() }}) as target_name + , cast(null as {{ type_string() }}) as target_schema + , cast(null as {{ type_int() }}) as target_threads + , cast(null as {{ type_string() }}) as dbt_cloud_project_id + , cast(null as {{ type_string() }}) as dbt_cloud_job_id + , cast(null as {{ type_string() }}) as dbt_cloud_run_id + , cast(null as {{ type_string() }}) as dbt_cloud_run_reason_category + , cast(null as {{ type_string() }}) as dbt_cloud_run_reason + , cast(null as {{ type_json() }}) as env_vars + , cast(null as {{ type_json() }}) as dbt_vars + , cast(null as {{ type_json() }}) as invocation_args + , cast(null as {{ type_json() }}) as dbt_custom_envs from dummy_cte where 1 = 0 diff --git a/models/sources/model_executions.sql b/models/sources/model_executions.sql index 09bec105..14d6c6b2 100644 --- a/models/sources/model_executions.sql +++ b/models/sources/model_executions.sql @@ -1,27 +1,30 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with dummy_cte as ( - select 1 as foo -) +with + dummy_cte as ( + + select 1 as foo + + ) select - cast(null as {{ type_string() }}) as command_invocation_id, - cast(null as {{ type_string() }}) as node_id, - cast(null as {{ type_timestamp() }}) as run_started_at, - cast(null as {{ type_boolean() }}) as was_full_refresh, - cast(null as {{ type_string() }}) as thread_id, - cast(null as {{ type_string() }}) as status, - cast(null as {{ type_timestamp() }}) as compile_started_at, - cast(null as {{ type_timestamp() }}) as query_completed_at, - cast(null as {{ type_float() }}) as total_node_runtime, - cast(null as {{ type_int() }}) as rows_affected, + cast(null as {{ type_string() }}) as command_invocation_id + , cast(null as {{ type_string() }}) as node_id + , cast(null as {{ type_timestamp() }}) as run_started_at + , cast(null as {{ type_boolean() }}) as was_full_refresh + , cast(null as {{ type_string() }}) as thread_id + , cast(null as {{ type_string() }}) as status + , cast(null as {{ type_timestamp() }}) as compile_started_at + , cast(null as {{ type_timestamp() }}) as query_completed_at + , cast(null as {{ type_float() }}) as total_node_runtime + , cast(null as {{ type_int() }}) as rows_affected {% if target.type == 'bigquery' %} - cast(null as {{ type_int() }}) as bytes_processed, + , cast(null as {{ type_int() }}) as bytes_processed {% endif %} - cast(null as {{ type_string() }}) as materialization, - cast(null as {{ type_string() }}) as schema, - cast(null as {{ type_string() }}) as name, - cast(null as {{ type_string() }}) as alias, - cast(null as {{ type_string() }}) as message, - cast(null as {{ type_json() }}) as adapter_response + , cast(null as {{ type_string() }}) as materialization + , cast(null as {{ type_string() }}) as schema + , cast(null as {{ type_string() }}) as name + , cast(null as {{ type_string() }}) as alias + , cast(null as {{ type_string() }}) as message + , cast(null as {{ type_json() }}) as adapter_response from dummy_cte where 1 = 0 diff --git a/models/sources/models.sql b/models/sources/models.sql index f4ec8f01..debeaf05 100644 --- a/models/sources/models.sql +++ b/models/sources/models.sql @@ -1,23 +1,26 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with dummy_cte as ( - select 1 as foo -) +with + dummy_cte as ( + + select 1 as foo + + ) select - cast(null as {{ type_string() }}) as command_invocation_id, - cast(null as {{ type_string() }}) as node_id, - cast(null as {{ type_timestamp() }}) as run_started_at, - cast(null as {{ type_string() }}) as database, - cast(null as {{ type_string() }}) as schema, - cast(null as {{ type_string() }}) as name, - cast(null as {{ type_array() }}) as depends_on_nodes, - cast(null as {{ type_string() }}) as package_name, - cast(null as {{ type_string() }}) as path, - cast(null as {{ type_string() }}) as checksum, - cast(null as {{ type_string() }}) as materialization, - cast(null as {{ type_array() }}) as tags, - cast(null as {{ type_json() }}) as meta, - cast(null as {{ type_string() }}) as alias, - cast(null as {{ type_json() }}) as all_results + cast(null as {{ type_string() }}) as command_invocation_id + , cast(null as {{ type_string() }}) as node_id + , cast(null as {{ type_timestamp() }}) as run_started_at + , cast(null as {{ type_string() }}) as database + , cast(null as {{ type_string() }}) as schema + , cast(null as {{ type_string() }}) as name + , cast(null as {{ type_array() }}) as depends_on_nodes + , cast(null as {{ type_string() }}) as package_name + , cast(null as {{ type_string() }}) as path + , cast(null as {{ type_string() }}) as checksum + , cast(null as {{ type_string() }}) as materialization + , cast(null as {{ type_array() }}) as tags + , cast(null as {{ type_json() }}) as meta + , cast(null as {{ type_string() }}) as alias + , cast(null as {{ type_json() }}) as all_results from dummy_cte where 1 = 0 diff --git a/models/sources/seed_executions.sql b/models/sources/seed_executions.sql index dbb93977..6d9aee35 100644 --- a/models/sources/seed_executions.sql +++ b/models/sources/seed_executions.sql @@ -1,24 +1,27 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with dummy_cte as ( - select 1 as foo -) +with + dummy_cte as ( + + select 1 as foo + + ) select - cast(null as {{ type_string() }}) as command_invocation_id, - cast(null as {{ type_string() }}) as node_id, - cast(null as {{ type_timestamp() }}) as run_started_at, - cast(null as {{ type_boolean() }}) as was_full_refresh, - cast(null as {{ type_string() }}) as thread_id, - cast(null as {{ type_string() }}) as status, - cast(null as {{ type_timestamp() }}) as compile_started_at, - cast(null as {{ type_timestamp() }}) as query_completed_at, - cast(null as {{ type_float() }}) as total_node_runtime, - cast(null as {{ type_int() }}) as rows_affected, - cast(null as {{ type_string() }}) as materialization, - cast(null as {{ type_string() }}) as schema, - cast(null as {{ type_string() }}) as name, - cast(null as {{ type_string() }}) as alias, - cast(null as {{ type_string() }}) as message, - cast(null as {{ type_json() }}) as adapter_response + cast(null as {{ type_string() }}) as command_invocation_id + , cast(null as {{ type_string() }}) as node_id + , cast(null as {{ type_timestamp() }}) as run_started_at + , cast(null as {{ type_boolean() }}) as was_full_refresh + , cast(null as {{ type_string() }}) as thread_id + , cast(null as {{ type_string() }}) as status + , cast(null as {{ type_timestamp() }}) as compile_started_at + , cast(null as {{ type_timestamp() }}) as query_completed_at + , cast(null as {{ type_float() }}) as total_node_runtime + , cast(null as {{ type_int() }}) as rows_affected + , cast(null as {{ type_string() }}) as materialization + , cast(null as {{ type_string() }}) as schema + , cast(null as {{ type_string() }}) as name + , cast(null as {{ type_string() }}) as alias + , cast(null as {{ type_string() }}) as message + , cast(null as {{ type_json() }}) as adapter_responss from dummy_cte where 1 = 0 diff --git a/models/sources/seeds.sql b/models/sources/seeds.sql index 7c52ecf7..3c99ef01 100644 --- a/models/sources/seeds.sql +++ b/models/sources/seeds.sql @@ -1,20 +1,23 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with dummy_cte as ( - select 1 as foo -) +with + dummy_cte as ( + + select 1 as foo + + ) select - cast(null as {{ type_string() }}) as command_invocation_id, - cast(null as {{ type_string() }}) as node_id, - cast(null as {{ type_timestamp() }}) as run_started_at, - cast(null as {{ type_string() }}) as database, - cast(null as {{ type_string() }}) as schema, - cast(null as {{ type_string() }}) as name, - cast(null as {{ type_string() }}) as package_name, - cast(null as {{ type_string() }}) as path, - cast(null as {{ type_string() }}) as checksum, - cast(null as {{ type_json() }}) as meta, - cast(null as {{ type_string() }}) as alias, - cast(null as {{ type_json() }}) as all_results + cast(null as {{ type_string() }}) as command_invocation_id + , cast(null as {{ type_string() }}) as node_id + , cast(null as {{ type_timestamp() }}) as run_started_at + , cast(null as {{ type_string() }}) as database + , cast(null as {{ type_string() }}) as schema + , cast(null as {{ type_string() }}) as name + , cast(null as {{ type_string() }}) as package_name + , cast(null as {{ type_string() }}) as path + , cast(null as {{ type_string() }}) as checksum + , cast(null as {{ type_json() }}) as meta + , cast(null as {{ type_string() }}) as alias + , cast(null as {{ type_json() }}) as all_results from dummy_cte where 1 = 0 diff --git a/models/sources/snapshot_executions.sql b/models/sources/snapshot_executions.sql index dbb93977..c3e9465d 100644 --- a/models/sources/snapshot_executions.sql +++ b/models/sources/snapshot_executions.sql @@ -1,24 +1,27 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with dummy_cte as ( - select 1 as foo -) +with + dummy_cte as ( + + select 1 as foo + + ) select - cast(null as {{ type_string() }}) as command_invocation_id, - cast(null as {{ type_string() }}) as node_id, - cast(null as {{ type_timestamp() }}) as run_started_at, - cast(null as {{ type_boolean() }}) as was_full_refresh, - cast(null as {{ type_string() }}) as thread_id, - cast(null as {{ type_string() }}) as status, - cast(null as {{ type_timestamp() }}) as compile_started_at, - cast(null as {{ type_timestamp() }}) as query_completed_at, - cast(null as {{ type_float() }}) as total_node_runtime, - cast(null as {{ type_int() }}) as rows_affected, - cast(null as {{ type_string() }}) as materialization, - cast(null as {{ type_string() }}) as schema, - cast(null as {{ type_string() }}) as name, - cast(null as {{ type_string() }}) as alias, - cast(null as {{ type_string() }}) as message, - cast(null as {{ type_json() }}) as adapter_response + cast(null as {{ type_string() }}) as command_invocation_id + , cast(null as {{ type_string() }}) as node_id + , cast(null as {{ type_timestamp() }}) as run_started_at + , cast(null as {{ type_boolean() }}) as was_full_refresh + , cast(null as {{ type_string() }}) as thread_id + , cast(null as {{ type_string() }}) as status + , cast(null as {{ type_timestamp() }}) as compile_started_at + , cast(null as {{ type_timestamp() }}) as query_completed_at + , cast(null as {{ type_float() }}) as total_node_runtime + , cast(null as {{ type_int() }}) as rows_affected + , cast(null as {{ type_string() }}) as materialization + , cast(null as {{ type_string() }}) as schema + , cast(null as {{ type_string() }}) as name + , cast(null as {{ type_string() }}) as alias + , cast(null as {{ type_string() }}) as message + , cast(null as {{ type_json() }}) as adapter_response from dummy_cte where 1 = 0 diff --git a/models/sources/snapshots.sql b/models/sources/snapshots.sql index e7832de6..c47ef812 100644 --- a/models/sources/snapshots.sql +++ b/models/sources/snapshots.sql @@ -1,22 +1,24 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with dummy_cte as ( - select 1 as foo -) +with + dummy_cte as ( + + select 1 as foo + ) select - cast(null as {{ type_string() }}) as command_invocation_id, - cast(null as {{ type_string() }}) as node_id, - cast(null as {{ type_timestamp() }}) as run_started_at, - cast(null as {{ type_string() }}) as database, - cast(null as {{ type_string() }}) as schema, - cast(null as {{ type_string() }}) as name, - cast(null as {{ type_array() }}) as depends_on_nodes, - cast(null as {{ type_string() }}) as package_name, - cast(null as {{ type_string() }}) as path, - cast(null as {{ type_string() }}) as checksum, - cast(null as {{ type_string() }}) as strategy, - cast(null as {{ type_json() }}) as meta, - cast(null as {{ type_string() }}) as alias, - cast(null as {{ type_json() }}) as all_results + cast(null as {{ type_string() }}) as command_invocation_id + , cast(null as {{ type_string() }}) as node_id + , cast(null as {{ type_timestamp() }}) as run_started_at + , cast(null as {{ type_string() }}) as database + , cast(null as {{ type_string() }}) as schema + , cast(null as {{ type_string() }}) as name + , cast(null as {{ type_array() }}) as depends_on_nodes + , cast(null as {{ type_string() }}) as package_name + , cast(null as {{ type_string() }}) as path + , cast(null as {{ type_string() }}) as checksum + , cast(null as {{ type_string() }}) as strategy + , cast(null as {{ type_json() }}) as meta + , cast(null as {{ type_string() }}) as alias + , cast(null as {{ type_json() }}) as all_results from dummy_cte where 1 = 0 diff --git a/models/sources/sources.sql b/models/sources/sources.sql index ad08f0d7..d152ad22 100644 --- a/models/sources/sources.sql +++ b/models/sources/sources.sql @@ -1,24 +1,25 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with dummy_cte as ( - select 1 as foo -) +with + dummy_cte as ( + select 1 as foo + ) select - cast(null as {{ type_string() }}) as command_invocation_id, - cast(null as {{ type_string() }}) as node_id, - cast(null as {{ type_timestamp() }}) as run_started_at, - cast(null as {{ type_string() }}) as database, - cast(null as {{ type_string() }}) as schema, - cast(null as {{ type_string() }}) as source_name, - cast(null as {{ type_string() }}) as loader, - cast(null as {{ type_string() }}) as name, - cast(null as {{ type_string() }}) as identifier, - cast(null as {{ type_string() }}) as loaded_at_field, + cast(null as {{ type_string() }}) as command_invocation_id + , cast(null as {{ type_string() }}) as node_id + , cast(null as {{ type_timestamp() }}) as run_started_at + , cast(null as {{ type_string() }}) as database + , cast(null as {{ type_string() }}) as schema + , cast(null as {{ type_string() }}) as source_name + , cast(null as {{ type_string() }}) as loader + , cast(null as {{ type_string() }}) as name + , cast(null as {{ type_string() }}) as identifier + , cast(null as {{ type_string() }}) as loaded_at_field {% if target.type == 'snowflake' %} - cast(null as {{ type_array() }}) as freshness, + , cast(null as {{ type_array() }}) as freshness {% else %} - cast(null as {{ type_json() }}) as freshness, + , cast(null as {{ type_json() }}) as freshness {% endif %} - cast(null as {{ type_json() }}) as all_results + , cast(null as {{ type_json() }}) as all_results from dummy_cte where 1 = 0 diff --git a/models/sources/test_executions.sql b/models/sources/test_executions.sql index 50d37980..c7d59121 100644 --- a/models/sources/test_executions.sql +++ b/models/sources/test_executions.sql @@ -1,21 +1,23 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with dummy_cte as ( - select 1 as foo -) +with + dummy_cte as ( + + select 1 as foo + ) select - cast(null as {{ type_string() }}) as command_invocation_id, - cast(null as {{ type_string() }}) as node_id, - cast(null as {{ type_timestamp() }}) as run_started_at, - cast(null as {{ type_boolean() }}) as was_full_refresh, - cast(null as {{ type_string() }}) as thread_id, - cast(null as {{ type_string() }}) as status, - cast(null as {{ type_timestamp() }}) as compile_started_at, - cast(null as {{ type_timestamp() }}) as query_completed_at, - cast(null as {{ type_float() }}) as total_node_runtime, - cast(null as {{ type_int() }}) as rows_affected, - cast(null as {{ type_int() }}) as failures, - cast(null as {{ type_string() }}) as message, - cast(null as {{ type_json() }}) as adapter_response + cast(null as {{ type_string() }}) as command_invocation_id + , cast(null as {{ type_string() }}) as node_id + , cast(null as {{ type_timestamp() }}) as run_started_at + , cast(null as {{ type_boolean() }}) as was_full_refresh + , cast(null as {{ type_string() }}) as thread_id + , cast(null as {{ type_string() }}) as status + , cast(null as {{ type_timestamp() }}) as compile_started_at + , cast(null as {{ type_timestamp() }}) as query_completed_at + , cast(null as {{ type_float() }}) as total_node_runtime + , cast(null as {{ type_int() }}) as rows_affected + , cast(null as {{ type_int() }}) as failures + , cast(null as {{ type_string() }}) as message + , cast(null as {{ type_json() }}) as adapter_response from dummy_cte where 1 = 0 diff --git a/models/sources/tests.sql b/models/sources/tests.sql index 1a4b5d9d..d46ed1dc 100644 --- a/models/sources/tests.sql +++ b/models/sources/tests.sql @@ -1,17 +1,20 @@ /* Bigquery won't let us `where` without `from` so we use this workaround */ -with dummy_cte as ( - select 1 as foo -) +with + dummy_cte as ( + + select 1 as foo + + ) select - cast(null as {{ type_string() }}) as command_invocation_id, - cast(null as {{ type_string() }}) as node_id, - cast(null as {{ type_timestamp() }}) as run_started_at, - cast(null as {{ type_string() }}) as name, - cast(null as {{ type_array() }}) as depends_on_nodes, - cast(null as {{ type_string() }}) as package_name, - cast(null as {{ type_string() }}) as test_path, - cast(null as {{ type_array() }}) as tags, - cast(null as {{ type_json() }}) as all_results + cast(null as {{ type_string() }}) as command_invocation_id + , cast(null as {{ type_string() }}) as node_id + , cast(null as {{ type_timestamp() }}) as run_started_at + , cast(null as {{ type_string() }}) as name + , cast(null as {{ type_array() }}) as depends_on_nodes + , cast(null as {{ type_string() }}) as package_name + , cast(null as {{ type_string() }}) as test_path + , cast(null as {{ type_array() }}) as tags + , cast(null as {{ type_json() }}) as all_results from dummy_cte where 1 = 0 diff --git a/models/staging/stg_dbt__snapshot_executions.sql b/models/staging/stg_dbt__snapshot_executions.sql index 207e7691..5818133d 100644 --- a/models/staging/stg_dbt__snapshot_executions.sql +++ b/models/staging/stg_dbt__snapshot_executions.sql @@ -9,7 +9,7 @@ with , enhanced as ( select - , {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as snapshot_execution_id + {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as snapshot_execution_id , command_invocation_id , node_id , run_started_at From 0374a31cd2b4a10f11191d0859516b0efdda3f82 Mon Sep 17 00:00:00 2001 From: Leopoldo Araujo <68853890+poloaraujo@users.noreply.github.com> Date: Mon, 2 Oct 2023 14:04:01 -0400 Subject: [PATCH 6/8] Fixed typo --- models/sources/seed_executions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/sources/seed_executions.sql b/models/sources/seed_executions.sql index 6d9aee35..c3e9465d 100644 --- a/models/sources/seed_executions.sql +++ b/models/sources/seed_executions.sql @@ -22,6 +22,6 @@ select , cast(null as {{ type_string() }}) as name , cast(null as {{ type_string() }}) as alias , cast(null as {{ type_string() }}) as message - , cast(null as {{ type_json() }}) as adapter_responss + , cast(null as {{ type_json() }}) as adapter_response from dummy_cte where 1 = 0 From bf46483302f3d069f799aef17e5f88913dc2f64c Mon Sep 17 00:00:00 2001 From: Leopoldo Araujo <68853890+poloaraujo@users.noreply.github.com> Date: Wed, 4 Oct 2023 13:31:43 -0400 Subject: [PATCH 7/8] SQL Fluff changes - Removed L02 from tox.ini - Formatted dim_dbt__current_models --- models/dim_dbt__current_models.sql | 9 ++++++--- tox.ini | 3 +-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/models/dim_dbt__current_models.sql b/models/dim_dbt__current_models.sql index cb935256..d63ab5b3 100644 --- a/models/dim_dbt__current_models.sql +++ b/models/dim_dbt__current_models.sql @@ -1,14 +1,16 @@ with base as ( - select * + select + * from {{ ref('stg_dbt__models') }} ) , model_executions as ( - select * + select + * from {{ ref('stg_dbt__model_executions') }} ) @@ -16,7 +18,8 @@ with , latest_models as ( /* Retrieves the models present in the most recent run */ - select * + select + * from base where run_started_at = (select max(run_started_at) from base) diff --git a/tox.ini b/tox.ini index 8add76ba..0b4e2403 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ envlist = lint_all [sqlfluff] dialect = snowflake templater = dbt -rules = LT01,LT02,LT03,CP01,AL01,AL02,CP02,ST08,LT06,LT07,AM01,LT08,AL05,RF02,RF03,CP03,ST01,LT09,AM03,CP04,LT10,ST05,ST03,JJ01,AM05,CV08 +rules = LT01,LT03,CP01,AL01,AL02,CP02,ST08,LT06,LT07,AM01,LT08,AL05,RF02,RF03,CP03,ST01,LT09,AM03,CP04,LT10,ST05,ST03,JJ01,AM05,CV08 # LT01: [aliasing.table] Implicit/explicit aliasing of table. # AL02: [aliasing.column] Implicit/explicit aliasing of columns. @@ -20,7 +20,6 @@ rules = LT01,LT02,LT03,CP01,AL01,AL02,CP02,ST08,LT06,LT07,AM01,LT08,AL05,RF02,RF # CV08: [convention.left_join] Use 'LEFT JOIN' instead of 'RIGHT JOIN'. # JJ01: [jinja.padding] Jinja tags should have a single whitespace on either side. # LT01: [layout.spacing] Inappropriate Spacing. -# LT02: [layout.indent] Incorrect Indentation. # LT03: [layout.operators] Operators should follow a standard for being before/after newlines. # LT06: [layout.functions] Function name not immediately followed by parenthesis. # LT07: [layout.functions] 'WITH' clause closing bracket should be on a new line. From e94b400541daa48a1656ea88f0bf7c1eb1427cd9 Mon Sep 17 00:00:00 2001 From: Leopoldo Araujo <68853890+poloaraujo@users.noreply.github.com> Date: Fri, 30 Aug 2024 14:21:45 -0400 Subject: [PATCH 8/8] Changed single select lines --- models/dim_dbt__current_models.sql | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/models/dim_dbt__current_models.sql b/models/dim_dbt__current_models.sql index d63ab5b3..3e24ae79 100644 --- a/models/dim_dbt__current_models.sql +++ b/models/dim_dbt__current_models.sql @@ -1,25 +1,20 @@ with base as ( - select - * - from {{ ref('stg_dbt__models') }} + select * from {{ ref('stg_dbt__models') }} ) , model_executions as ( - select - * - from {{ ref('stg_dbt__model_executions') }} + select * from {{ ref('stg_dbt__model_executions') }} ) , latest_models as ( /* Retrieves the models present in the most recent run */ - select - * + select * from base where run_started_at = (select max(run_started_at) from base)