Refactor Octopus Jaffle Shop

jaffle_shop/models/final/finance/_models.yml

@@ -0,0 +1,25 @@
+version: 2
+
+exposures:
+  - name: Customer returns dashboard
+    description: A dashboard for showing total value of returned goods by customer.
+    type: dashboard
+    url: https://an-example-dashboard-site.com
+    owner:
+      email: [email protected]
+    depends_on:
+      - ref('fnl_finance_customerreturns')
+
+models:
+  - name: fnl_finance_customerreturns
+    meta:
+      owner: '[email protected]'
+    description: This table has the total lifetime return value by customer id.
+    columns:
+      - name: customer_id
+        description: This is a unique identifier for a customer
+        tests:
+          - unique
+          - not_null
+      - name: returned_value
+        description: Total value of orders returned.

jaffle_shop/models/final/finance/fnl_finance_customerreturns.sql

@@ -0,0 +1,6 @@
+SELECT customer_id
+       , SUM(amount) as returned_value
+FROM {{ ref('wh_orders') }}
+WHERE status = 'returned'
+GROUP BY customer_id
+ORDER BY SUM(amount) DESC

jaffle_shop/models/final/sales/_models.yml

@@ -0,0 +1,25 @@
+version: 2
+
+exposures:
+  - name: New customer dashboard
+    description: A dashboard for showing new customers each month.
+    type: dashboard
+    url: https://an-example-dashboard-site.com
+    owner:
+      email: [email protected]
+    depends_on:
+      - ref('fnl_sales_newcustomers')
+
+models:
+  - name: fnl_sales_newcustomers
+    meta:
+      owner: '[email protected]'
+    description: This table has the count of customers making their first order by month.
+    columns:
+      - name: first_order_month
+        description: This is the month in which a customers first order was placed.
+        tests:
+          - unique
+          - not_null
+      - name: new_customer_count
+        description: Total count of new customers placing their first order.

jaffle_shop/models/final/sales/fnl_sales_newcustomers.sql

@@ -0,0 +1,5 @@
+SELECT COALESCE(date_trunc('MM', first_order),'2018-01-01') as first_order_month
+     , COUNT(customer_id) as new_customer_count
+FROM {{ ref('wh_customers') }}
+GROUP BY date_trunc('MM', first_order)
+ORDER BY date_trunc('MM', first_order) ASC

jaffle_shop/models/staging/schema.yml → jaffle_shop/models/staging/src_seed/_models.yml

@@ -2,13 +2,20 @@ version: 2
 
 models:
   - name: stg_customers
+    meta:
+      owner: '[email protected]'
+    description: >
+      Staging layer for customers' source data. Sensitive data is hashed.
     columns:
       - name: customer_id
         tests:
           - unique
           - not_null
-
   - name: stg_orders
+    meta:
+      owner: '[email protected]'
+    description: >
+      Staging layer for orders' source data.
     columns:
       - name: order_id
         tests:
@@ -18,8 +25,11 @@ models:
         tests:
           - accepted_values:
               values: ['placed', 'shipped', 'completed', 'return_pending', 'returned']
-
   - name: stg_payments
+    meta:
+      owner: '[email protected]'
+    description: >
+      Staging layer for payments' source data.
     columns:
       - name: payment_id
         tests:

jaffle_shop/models/staging/src_seed/sensitive/_models.yml

@@ -0,0 +1,19 @@
+version: 2
+
+models:
+  - name: stg_customers_pii
+    meta:
+      owner: '[email protected]'
+    description: >
+      Staging layer for customers' source data.
+    columns:
+      - name: customer_id
+        tests:
+          - unique
+          - not_null
+      - name: first_name
+        meta:
+          sensitive: true
+      - name: last_name
+        meta:
+          sensitive: true

jaffle_shop/models/staging/src_seed/stg_customers.sql

@@ -0,0 +1,3 @@
+SELECT
+    {{hash_sensitive_columns('stg_customers_pii')}}
+from {{ref('stg_customers_pii')}}

jaffle_shop/models/schema.yml → jaffle_shop/models/warehouse/_models.yml

@@ -1,81 +1,68 @@
 version: 2
 
 models:
-  - name: customers
+  - name: wh_customers
+    meta:
+      owner: '[email protected]'
     description: This table has basic information about a customer, as well as some derived facts based on a customer's orders
-
     columns:
       - name: customer_id
         description: This is a unique identifier for a customer
         tests:
           - unique
           - not_null
-
       - name: first_name
         description: Customer's first name. PII.
-
       - name: last_name
         description: Customer's last name. PII.
-
       - name: first_order
         description: Date (UTC) of a customer's first order
-
       - name: most_recent_order
         description: Date (UTC) of a customer's most recent order
-
       - name: number_of_orders
         description: Count of the number of orders a customer has placed
-
       - name: total_order_amount
         description: Total value (AUD) of a customer's orders
-
-  - name: orders
+  - name: wh_orders
+    meta:
+      owner: '[email protected]'
     description: This table has basic information about orders, as well as some derived facts based on payments
-
     columns:
       - name: order_id
         tests:
           - unique
           - not_null
         description: This is a unique identifier for an order
-
       - name: customer_id
         description: Foreign key to the customers table
         tests:
           - not_null
           - relationships:
-              to: ref('customers')
+              to: ref('wh_customers')
               field: customer_id
-
       - name: order_date
         description: Date (UTC) that the order was placed
-
       - name: status
         description: '{{ doc("orders_status") }}'
         tests:
           - accepted_values:
               values: ['placed', 'shipped', 'completed', 'return_pending', 'returned']
-
       - name: amount
         description: Total amount (AUD) of the order
         tests:
           - not_null
-
       - name: credit_card_amount
         description: Amount of the order (AUD) paid for by credit card
         tests:
           - not_null
-
       - name: coupon_amount
         description: Amount of the order (AUD) paid for by coupon
         tests:
           - not_null
-
       - name: bank_transfer_amount
         description: Amount of the order (AUD) paid for by bank transfer
         tests:
           - not_null
-
       - name: gift_card_amount
         description: Amount of the order (AUD) paid for by gift card
         tests:

jaffle_shop/models/customers.sql → jaffle_shop/models/warehouse/wh_customers.sql

@@ -49,8 +49,8 @@ final as (
 
     select
         customers.customer_id,
-        customers.first_name,
-        customers.last_name,
+        customers.first_name_hash,
+        customers.last_name_hash,
         customer_orders.first_order,
         customer_orders.most_recent_order,
         customer_orders.number_of_orders,

jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv

@@ -0,0 +1,2 @@
+fct_name,column_name,id_to_exclude,comment
+fct_staging_dependent_on_staging,parent,stg_customers_pii,Scrubbing pii permitted in staging layer.