From 0f9a3f3a2767ee5904488bb60ef30cf2d0d0fe84 Mon Sep 17 00:00:00 2001 From: jelstongreen Date: Thu, 10 Nov 2022 11:09:22 +0000 Subject: [PATCH] Gatekeep Repo Rename ymls --- .../macros/sensitive/{macros.md => _docs.md} | 0 .../sensitive/{macros.yml => _macros.yml} | 0 jaffle_shop/models/final/finance/_models.yml | 25 +++++++++++++++++ .../finance/fnl_finance_customerreturns.sql | 6 +++++ jaffle_shop/models/final/sales/_models.yml | 25 +++++++++++++++++ .../final/sales/fnl_sales_newcustomers.sql | 5 ++++ .../{schema.yml => src_seed/_models.yml} | 14 ++++++++-- .../staging/src_seed/sensitive/_models.yml | 19 +++++++++++++ .../sensitive/stg_customers_pii.sql} | 0 .../models/staging/src_seed/stg_customers.sql | 3 +++ .../staging/{ => src_seed}/stg_orders.sql | 0 .../staging/{ => src_seed}/stg_payments.sql | 0 .../models/{docs.md => warehouse/_docs.md} | 0 .../{schema.yml => warehouse/_models.yml} | 27 +++++-------------- .../wh_customers.sql} | 4 +-- .../{orders.sql => warehouse/wh_orders.sql} | 0 .../dbt_project_evaluator_exceptions.csv | 2 ++ 17 files changed, 106 insertions(+), 24 deletions(-) rename jaffle_shop/macros/sensitive/{macros.md => _docs.md} (100%) rename jaffle_shop/macros/sensitive/{macros.yml => _macros.yml} (100%) create mode 100644 jaffle_shop/models/final/finance/_models.yml create mode 100644 jaffle_shop/models/final/finance/fnl_finance_customerreturns.sql create mode 100644 jaffle_shop/models/final/sales/_models.yml create mode 100644 jaffle_shop/models/final/sales/fnl_sales_newcustomers.sql rename jaffle_shop/models/staging/{schema.yml => src_seed/_models.yml} (63%) create mode 100644 jaffle_shop/models/staging/src_seed/sensitive/_models.yml rename jaffle_shop/models/staging/{stg_customers.sql => src_seed/sensitive/stg_customers_pii.sql} (100%) create mode 100644 jaffle_shop/models/staging/src_seed/stg_customers.sql rename jaffle_shop/models/staging/{ => src_seed}/stg_orders.sql (100%) rename jaffle_shop/models/staging/{ => src_seed}/stg_payments.sql (100%) rename jaffle_shop/models/{docs.md => warehouse/_docs.md} (100%) rename jaffle_shop/models/{schema.yml => warehouse/_models.yml} (92%) rename jaffle_shop/models/{customers.sql => warehouse/wh_customers.sql} (94%) rename jaffle_shop/models/{orders.sql => warehouse/wh_orders.sql} (100%) create mode 100644 jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv diff --git a/jaffle_shop/macros/sensitive/macros.md b/jaffle_shop/macros/sensitive/_docs.md similarity index 100% rename from jaffle_shop/macros/sensitive/macros.md rename to jaffle_shop/macros/sensitive/_docs.md diff --git a/jaffle_shop/macros/sensitive/macros.yml b/jaffle_shop/macros/sensitive/_macros.yml similarity index 100% rename from jaffle_shop/macros/sensitive/macros.yml rename to jaffle_shop/macros/sensitive/_macros.yml diff --git a/jaffle_shop/models/final/finance/_models.yml b/jaffle_shop/models/final/finance/_models.yml new file mode 100644 index 000000000..a9d590607 --- /dev/null +++ b/jaffle_shop/models/final/finance/_models.yml @@ -0,0 +1,25 @@ +version: 2 + +exposures: + - name: Customer returns dashboard + description: A dashboard for showing total value of returned goods by customer. + type: dashboard + url: https://an-example-dashboard-site.com + owner: + email: jaffle.mcjaffleson@gmail.com + depends_on: + - ref('fnl_finance_customerreturns') + +models: + - name: fnl_finance_customerreturns + meta: + owner: 'jaffle.mcjaffleson@gmail.com' + description: This table has the total lifetime return value by customer id. + columns: + - name: customer_id + description: This is a unique identifier for a customer + tests: + - unique + - not_null + - name: returned_value + description: Total value of orders returned. \ No newline at end of file diff --git a/jaffle_shop/models/final/finance/fnl_finance_customerreturns.sql b/jaffle_shop/models/final/finance/fnl_finance_customerreturns.sql new file mode 100644 index 000000000..2357252ae --- /dev/null +++ b/jaffle_shop/models/final/finance/fnl_finance_customerreturns.sql @@ -0,0 +1,6 @@ +SELECT customer_id + , SUM(amount) as returned_value +FROM {{ ref('wh_orders') }} +WHERE status = 'returned' +GROUP BY customer_id +ORDER BY SUM(amount) DESC \ No newline at end of file diff --git a/jaffle_shop/models/final/sales/_models.yml b/jaffle_shop/models/final/sales/_models.yml new file mode 100644 index 000000000..7e89208d3 --- /dev/null +++ b/jaffle_shop/models/final/sales/_models.yml @@ -0,0 +1,25 @@ +version: 2 + +exposures: + - name: New customer dashboard + description: A dashboard for showing new customers each month. + type: dashboard + url: https://an-example-dashboard-site.com + owner: + email: jaffle.mcjaffleson@gmail.com + depends_on: + - ref('fnl_sales_newcustomers') + +models: + - name: fnl_sales_newcustomers + meta: + owner: 'jaffle.mcjaffleson@gmail.com' + description: This table has the count of customers making their first order by month. + columns: + - name: first_order_month + description: This is the month in which a customers first order was placed. + tests: + - unique + - not_null + - name: new_customer_count + description: Total count of new customers placing their first order. \ No newline at end of file diff --git a/jaffle_shop/models/final/sales/fnl_sales_newcustomers.sql b/jaffle_shop/models/final/sales/fnl_sales_newcustomers.sql new file mode 100644 index 000000000..112c08c97 --- /dev/null +++ b/jaffle_shop/models/final/sales/fnl_sales_newcustomers.sql @@ -0,0 +1,5 @@ +SELECT COALESCE(date_trunc('MM', first_order),'2018-01-01') as first_order_month + , COUNT(customer_id) as new_customer_count +FROM {{ ref('wh_customers') }} +GROUP BY date_trunc('MM', first_order) +ORDER BY date_trunc('MM', first_order) ASC \ No newline at end of file diff --git a/jaffle_shop/models/staging/schema.yml b/jaffle_shop/models/staging/src_seed/_models.yml similarity index 63% rename from jaffle_shop/models/staging/schema.yml rename to jaffle_shop/models/staging/src_seed/_models.yml index c207e4cf5..856e494f1 100644 --- a/jaffle_shop/models/staging/schema.yml +++ b/jaffle_shop/models/staging/src_seed/_models.yml @@ -2,13 +2,20 @@ version: 2 models: - name: stg_customers + meta: + owner: 'jaffle.mcjaffleson@gmail.com' + description: > + Staging layer for customers' source data. Sensitive data is hashed. columns: - name: customer_id tests: - unique - not_null - - name: stg_orders + meta: + owner: 'jaffle.mcjaffleson@gmail.com' + description: > + Staging layer for orders' source data. columns: - name: order_id tests: @@ -18,8 +25,11 @@ models: tests: - accepted_values: values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] - - name: stg_payments + meta: + owner: 'jaffle.mcjaffleson@gmail.com' + description: > + Staging layer for payments' source data. columns: - name: payment_id tests: diff --git a/jaffle_shop/models/staging/src_seed/sensitive/_models.yml b/jaffle_shop/models/staging/src_seed/sensitive/_models.yml new file mode 100644 index 000000000..59aef1aab --- /dev/null +++ b/jaffle_shop/models/staging/src_seed/sensitive/_models.yml @@ -0,0 +1,19 @@ +version: 2 + +models: + - name: stg_customers_pii + meta: + owner: 'jaffle.mcjaffleson@gmail.com' + description: > + Staging layer for customers' source data. + columns: + - name: customer_id + tests: + - unique + - not_null + - name: first_name + meta: + sensitive: true + - name: last_name + meta: + sensitive: true diff --git a/jaffle_shop/models/staging/stg_customers.sql b/jaffle_shop/models/staging/src_seed/sensitive/stg_customers_pii.sql similarity index 100% rename from jaffle_shop/models/staging/stg_customers.sql rename to jaffle_shop/models/staging/src_seed/sensitive/stg_customers_pii.sql diff --git a/jaffle_shop/models/staging/src_seed/stg_customers.sql b/jaffle_shop/models/staging/src_seed/stg_customers.sql new file mode 100644 index 000000000..2bd98141c --- /dev/null +++ b/jaffle_shop/models/staging/src_seed/stg_customers.sql @@ -0,0 +1,3 @@ +SELECT + {{hash_sensitive_columns('stg_customers_pii')}} +from {{ref('stg_customers_pii')}} diff --git a/jaffle_shop/models/staging/stg_orders.sql b/jaffle_shop/models/staging/src_seed/stg_orders.sql similarity index 100% rename from jaffle_shop/models/staging/stg_orders.sql rename to jaffle_shop/models/staging/src_seed/stg_orders.sql diff --git a/jaffle_shop/models/staging/stg_payments.sql b/jaffle_shop/models/staging/src_seed/stg_payments.sql similarity index 100% rename from jaffle_shop/models/staging/stg_payments.sql rename to jaffle_shop/models/staging/src_seed/stg_payments.sql diff --git a/jaffle_shop/models/docs.md b/jaffle_shop/models/warehouse/_docs.md similarity index 100% rename from jaffle_shop/models/docs.md rename to jaffle_shop/models/warehouse/_docs.md diff --git a/jaffle_shop/models/schema.yml b/jaffle_shop/models/warehouse/_models.yml similarity index 92% rename from jaffle_shop/models/schema.yml rename to jaffle_shop/models/warehouse/_models.yml index 381349cfd..eb4a2baaf 100644 --- a/jaffle_shop/models/schema.yml +++ b/jaffle_shop/models/warehouse/_models.yml @@ -1,81 +1,68 @@ version: 2 models: - - name: customers + - name: wh_customers + meta: + owner: 'jaffle.mcjaffleson@gmail.com' description: This table has basic information about a customer, as well as some derived facts based on a customer's orders - columns: - name: customer_id description: This is a unique identifier for a customer tests: - unique - not_null - - name: first_name description: Customer's first name. PII. - - name: last_name description: Customer's last name. PII. - - name: first_order description: Date (UTC) of a customer's first order - - name: most_recent_order description: Date (UTC) of a customer's most recent order - - name: number_of_orders description: Count of the number of orders a customer has placed - - name: total_order_amount description: Total value (AUD) of a customer's orders - - - name: orders + - name: wh_orders + meta: + owner: 'jaffle.mcjaffleson@gmail.com' description: This table has basic information about orders, as well as some derived facts based on payments - columns: - name: order_id tests: - unique - not_null description: This is a unique identifier for an order - - name: customer_id description: Foreign key to the customers table tests: - not_null - relationships: - to: ref('customers') + to: ref('wh_customers') field: customer_id - - name: order_date description: Date (UTC) that the order was placed - - name: status description: '{{ doc("orders_status") }}' tests: - accepted_values: values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] - - name: amount description: Total amount (AUD) of the order tests: - not_null - - name: credit_card_amount description: Amount of the order (AUD) paid for by credit card tests: - not_null - - name: coupon_amount description: Amount of the order (AUD) paid for by coupon tests: - not_null - - name: bank_transfer_amount description: Amount of the order (AUD) paid for by bank transfer tests: - not_null - - name: gift_card_amount description: Amount of the order (AUD) paid for by gift card tests: diff --git a/jaffle_shop/models/customers.sql b/jaffle_shop/models/warehouse/wh_customers.sql similarity index 94% rename from jaffle_shop/models/customers.sql rename to jaffle_shop/models/warehouse/wh_customers.sql index 016a004fe..dd7c60ef4 100644 --- a/jaffle_shop/models/customers.sql +++ b/jaffle_shop/models/warehouse/wh_customers.sql @@ -49,8 +49,8 @@ final as ( select customers.customer_id, - customers.first_name, - customers.last_name, + customers.first_name_hash, + customers.last_name_hash, customer_orders.first_order, customer_orders.most_recent_order, customer_orders.number_of_orders, diff --git a/jaffle_shop/models/orders.sql b/jaffle_shop/models/warehouse/wh_orders.sql similarity index 100% rename from jaffle_shop/models/orders.sql rename to jaffle_shop/models/warehouse/wh_orders.sql diff --git a/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv b/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv new file mode 100644 index 000000000..b9f94e579 --- /dev/null +++ b/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv @@ -0,0 +1,2 @@ +fct_name,column_name,id_to_exclude,comment +fct_staging_dependent_on_staging,parent,stg_customers_pii,Scrubbing pii permitted in staging layer. \ No newline at end of file