Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Format all models for consistency #390

Merged
merged 10 commits into from
Aug 30, 2024
206 changes: 111 additions & 95 deletions models/dim_dbt__current_models.sql
Original file line number Diff line number Diff line change
@@ -1,97 +1,113 @@
with base as (
select *
from {{ ref('stg_dbt__models') }}
),

model_executions as (
select *
from {{ ref('stg_dbt__model_executions') }}
),

latest_models as (
/* Retrieves the models present in the most recent run */
select *
from base
where run_started_at = (select max(run_started_at) from base)
),

latest_models_runs as (
/* Retreives all successful run information for the models present in the most
recent run and ranks them based on query completion time */
select
model_executions.node_id
, model_executions.was_full_refresh
, model_executions.query_completed_at
, model_executions.total_node_runtime
, model_executions.rows_affected
{% if target.type == 'bigquery' %}
, model_executions.bytes_processed
{% endif %}
/* Row number by refresh and node ID */
, row_number() over (
partition by latest_models.node_id, model_executions.was_full_refresh
order by model_executions.query_completed_at desc /* most recent ranked first */
) as run_idx
/* Row number by node ID */
, row_number() over (
partition by latest_models.node_id
order by model_executions.query_completed_at desc /* most recent ranked first */
) as run_idx_id_only
from model_executions
inner join latest_models on model_executions.node_id = latest_models.node_id
where model_executions.status = 'success'
),

latest_model_stats as (
select
node_id
, max(case when was_full_refresh then query_completed_at end) as last_full_refresh_run_completed_at
, max(case when was_full_refresh then total_node_runtime end) as last_full_refresh_run_total_runtime
, max(case when was_full_refresh then rows_affected end) as last_full_refresh_run_rows_affected
{% if target.type == 'bigquery' %}
, max(case when was_full_refresh then bytes_processed end) as last_full_refresh_run_bytes_processed
{% endif %}
, max(case when run_idx_id_only = 1 then query_completed_at end) as last_run_completed_at
, max(case when run_idx_id_only = 1 then total_node_runtime end) as last_run_total_runtime
, max(case when run_idx_id_only = 1 then rows_affected end) as last_run_rows_affected
{% if target.type == 'bigquery' %}
, max(case when run_idx_id_only = 1 then bytes_processed end) as last_run_bytes_processed
{% endif %}
, max(case when not was_full_refresh then query_completed_at end) as last_incremental_run_completed_at
, max(case when not was_full_refresh then total_node_runtime end) as last_incremental_run_total_runtime
, max(case when not was_full_refresh then rows_affected end) as last_incremental_run_rows_affected
{% if target.type == 'bigquery' %}
, max(case when not was_full_refresh then bytes_processed end) as last_incremental_run_bytes_processed
{% endif %}
from latest_models_runs
where run_idx = 1
group by 1
),

final as (
select
latest_models.*
, latest_model_stats.last_full_refresh_run_completed_at
, latest_model_stats.last_full_refresh_run_total_runtime
, latest_model_stats.last_full_refresh_run_rows_affected
{% if target.type == 'bigquery' %}
, latest_model_stats.last_full_refresh_run_bytes_processed
{% endif %}
, latest_model_stats.last_run_completed_at
, latest_model_stats.last_run_total_runtime
, latest_model_stats.last_run_rows_affected
{% if target.type == 'bigquery' %}
, latest_model_stats.last_run_bytes_processed
{% endif %}
, latest_model_stats.last_incremental_run_completed_at
, latest_model_stats.last_incremental_run_total_runtime
, latest_model_stats.last_incremental_run_rows_affected
{% if target.type == 'bigquery' %}
, latest_model_stats.last_incremental_run_bytes_processed
{% endif %}
from latest_models
left join latest_model_stats
on latest_models.node_id = latest_model_stats.node_id
)
with
base as (

select
*
from {{ ref('stg_dbt__models') }}
llifoawing marked this conversation as resolved.
Show resolved Hide resolved

)

, model_executions as (

select
*
from {{ ref('stg_dbt__model_executions') }}
llifoawing marked this conversation as resolved.
Show resolved Hide resolved

)

, latest_models as (

/* Retrieves the models present in the most recent run */
select
*
from base
llifoawing marked this conversation as resolved.
Show resolved Hide resolved
where run_started_at = (select max(run_started_at) from base)

)

, latest_models_runs as (

/* Retreives all successful run information for the models present in the most
recent run and ranks them based on query completion time */
select
model_executions.node_id
, model_executions.was_full_refresh
, model_executions.query_completed_at
, model_executions.total_node_runtime
, model_executions.rows_affected
{% if target.type == 'bigquery' %}
, model_executions.bytes_processed
{% endif %}
/* Row number by refresh and node ID */
, row_number() over (
partition by latest_models.node_id, model_executions.was_full_refresh
order by model_executions.query_completed_at desc /* most recent ranked first */
) as run_idx
/* Row number by node ID */
, row_number() over (
partition by latest_models.node_id
order by model_executions.query_completed_at desc /* most recent ranked first */
) as run_idx_id_only
from model_executions
inner join latest_models on model_executions.node_id = latest_models.node_id
where model_executions.status = 'success'

)

, latest_model_stats as (

select
node_id
, max(case when was_full_refresh then query_completed_at end) as last_full_refresh_run_completed_at
, max(case when was_full_refresh then total_node_runtime end) as last_full_refresh_run_total_runtime
, max(case when was_full_refresh then rows_affected end) as last_full_refresh_run_rows_affected
{% if target.type == 'bigquery' %}
, max(case when was_full_refresh then bytes_processed end) as last_full_refresh_run_bytes_processed
{% endif %}
, max(case when run_idx_id_only = 1 then query_completed_at end) as last_run_completed_at
, max(case when run_idx_id_only = 1 then total_node_runtime end) as last_run_total_runtime
, max(case when run_idx_id_only = 1 then rows_affected end) as last_run_rows_affected
{% if target.type == 'bigquery' %}
, max(case when run_idx_id_only = 1 then bytes_processed end) as last_run_bytes_processed
{% endif %}
, max(case when not was_full_refresh then query_completed_at end) as last_incremental_run_completed_at
, max(case when not was_full_refresh then total_node_runtime end) as last_incremental_run_total_runtime
, max(case when not was_full_refresh then rows_affected end) as last_incremental_run_rows_affected
{% if target.type == 'bigquery' %}
, max(case when not was_full_refresh then bytes_processed end) as last_incremental_run_bytes_processed
{% endif %}
from latest_models_runs
where run_idx = 1
group by 1

)

, final as (

select
latest_models.*
, latest_model_stats.last_full_refresh_run_completed_at
, latest_model_stats.last_full_refresh_run_total_runtime
, latest_model_stats.last_full_refresh_run_rows_affected
{% if target.type == 'bigquery' %}
, latest_model_stats.last_full_refresh_run_bytes_processed
{% endif %}
, latest_model_stats.last_run_completed_at
, latest_model_stats.last_run_total_runtime
, latest_model_stats.last_run_rows_affected
{% if target.type == 'bigquery' %}
, latest_model_stats.last_run_bytes_processed
{% endif %}
, latest_model_stats.last_incremental_run_completed_at
, latest_model_stats.last_incremental_run_total_runtime
, latest_model_stats.last_incremental_run_rows_affected
{% if target.type == 'bigquery' %}
, latest_model_stats.last_incremental_run_bytes_processed
{% endif %}
from latest_models
left join latest_model_stats
on latest_models.node_id = latest_model_stats.node_id

)

select * from final
45 changes: 23 additions & 22 deletions models/dim_dbt__exposures.sql
Original file line number Diff line number Diff line change
@@ -1,29 +1,30 @@
with base as (
with
base as (

select *
from {{ ref('stg_dbt__exposures') }}
select *
from {{ ref('stg_dbt__exposures') }}

),
)

exposures as (
, exposures as (

select
exposure_execution_id,
command_invocation_id,
node_id,
run_started_at,
name,
type,
owner,
maturity,
path,
description,
url,
package_name,
depends_on_nodes,
tags
from base
select
exposure_execution_id
, command_invocation_id
, node_id
, run_started_at
, name
, type
, owner
, maturity
, path
, description
, url
, package_name
, depends_on_nodes
, tags
from base

)
)

select * from exposures
47 changes: 24 additions & 23 deletions models/dim_dbt__models.sql
Original file line number Diff line number Diff line change
@@ -1,30 +1,31 @@
with base as (
with
base as (

select *
from {{ ref('stg_dbt__models') }}
select *
from {{ ref('stg_dbt__models') }}

),
)

models as (
, models as (

select
model_execution_id,
command_invocation_id,
node_id,
run_started_at,
database,
schema,
name,
depends_on_nodes,
package_name,
path,
checksum,
materialization,
tags,
meta,
alias
from base
select
model_execution_id
, command_invocation_id
, node_id
, run_started_at
, database
, schema
, name
, depends_on_nodes
, package_name
, path
, checksum
, materialization
, tags
, meta
, alias
from base

)
)

select * from models
41 changes: 21 additions & 20 deletions models/dim_dbt__seeds.sql
Original file line number Diff line number Diff line change
@@ -1,27 +1,28 @@
with base as (
with
base as (

select *
from {{ ref('stg_dbt__seeds') }}
select *
from {{ ref('stg_dbt__seeds') }}

),
)

seeds as (
, seeds as (

select
seed_execution_id,
command_invocation_id,
node_id,
run_started_at,
database,
schema,
name,
package_name,
path,
checksum,
meta,
alias
from base
select
seed_execution_id
, command_invocation_id
, node_id
, run_started_at
, database
, schema
, name
, package_name
, path
, checksum
, meta
, alias
from base

)
)

select * from seeds
Loading
Loading