Maja dbt certification

.user.yml

@@ -0,0 +1 @@
+id: 292d2798-5b54-468e-a539-9b71028dd88f

jaffle_shop/models/.sqlfluffignore

@@ -0,0 +1,3 @@
+# sqlfluff incompatible with models using hash_sensitive_columns macro - disabled till resolved.
+
+stg_customers.sql

jaffle_shop/models/final/_exposures.yml

@@ -0,0 +1,22 @@
+version: 2
+
+exposures:
+  - name: new_customers
+    label: New customers
+    description: Inksacio dashboard
+    type: dashboard
+    url: https://inksacio.eks.octopus.engineering/my_certification_dashboard/
+    owner:
+      email: [email protected]
+    depends_on:
+      - ref('fnl_sales_newcustomers')
+  - name: returns_value
+    label: Value of returns per customer
+    description: Inksacio dashboard
+    type: dashboard
+    url: https://inksacio.eks.octopus.engineering/my_certification_dashboard/
+    owner:
+      email: [email protected]
+    depends_on:
+      - ref('fnl_finance_returnsvalue')
+

jaffle_shop/models/final/finance/_models.yml

@@ -0,0 +1,16 @@
+version: 2
+
+models:
+  - name: fnl_finance_returnsvalue
+    meta:
+      owner: "[email protected]"
+    description: |
+      Value of returns by customer id
+    columns:
+      - name: customer_id
+        description: This is a unique identifier for a customer
+        tests:
+          - unique
+          - not_null
+      - name: total_return_value
+        description: Total value of a customer's returns

jaffle_shop/models/final/finance/fnl_finance_returnsvalue.sql

@@ -0,0 +1,10 @@
+WITH fnl_finance_returnsvalue AS (
+    SELECT
+        orders.customer_id AS customer_id
+        , SUM(orders.amount) AS total_return_value
+    FROM {{ref('wh_orders')}} as orders
+    WHERE orders.status = 'returned'
+    GROUP BY orders.customer_id
+)
+
+SELECT * FROM fnl_finance_returnsvalue

jaffle_shop/models/final/sales/_models.yml

@@ -0,0 +1,17 @@
+version: 2
+
+models:
+  - name: fnl_sales_newcustomers
+    meta:
+      owner: "[email protected]"
+    description: |
+      Number of new customers by month
+    columns:
+      - name: first_order_month
+        description: |
+          This is a unique identifier for month and year
+        tests:
+          - unique
+          - not_null
+      - name: new_customers_count
+        description: Total number of new customers

jaffle_shop/models/final/sales/fnl_sales_newcustomers.sql

@@ -0,0 +1,12 @@
+WITH first_orders AS (
+    SELECT
+      customers.customer_id
+      ,  DATE_FORMAT(customers.first_order, '%y-%m') AS first_order_month
+    FROM {{ ref('wh_customers') }} AS customers
+)
+
+SELECT
+  first_order_month
+  , COUNT(*) AS new_customers_count
+FROM first_orders
+GROUP BY first_order_month

jaffle_shop/models/staging/src_seed/_models.yml

@@ -0,0 +1,69 @@
+version: 2
+
+models:
+  - name: stg_customers_pii
+    meta:
+      owner: "[email protected]"
+      sensitive: true
+    description: List of jaffle shop customers
+    columns:
+      - name: customer_id
+        tests:
+          - unique
+          - not_null
+      - name: first_name
+        meta:
+          sensitive: true
+      - name: last_name
+        meta:
+          sensitive: true
+
+  - name: stg_customers
+    meta:
+      owner: "[email protected]"
+    description: List of customer ids
+    columns:
+      - name: customer_id
+        tests:
+          - unique
+          - not_null
+      - name: first_name_hash
+        tests:
+          - dbt_expectations.expect_columns_to_exist
+      - name: last_name_hash
+        tests:
+          - dbt_expectations.expect_columns_to_exist
+
+  - name: stg_orders
+    meta:
+      owner: "[email protected]"
+    description: List of customer orders
+    columns:
+      - name: order_id
+        tests:
+          - unique
+          - not_null
+      - name: customer_id
+      - name: order_date
+      - name: status
+        tests:
+          - accepted_values:
+              values: ['placed', 'shipped', 'completed', 'return_pending', 'returned']
+
+
+  - name: stg_payments
+    meta:
+      owner: "[email protected]"
+    description: List of payments
+    columns:
+      - name: payment_id
+        tests:
+          - unique
+          - not_null
+      - name: order_id
+      - name: payment_method
+        tests:
+          - accepted_values:
+              values: ['credit_card', 'coupon', 'bank_transfer', 'gift_card']
+      - name: amount
+

jaffle_shop/models/staging/src_seed/stg_customers.sql

@@ -0,0 +1,3 @@
+SELECT
+  {{ hash_sensitive_columns('stg_customers_pii') }}
+FROM {{ ref('stg_customers_pii') }}

jaffle_shop/models/schema.yml → jaffle_shop/models/warehouse/_models.yml

@@ -1,9 +1,12 @@
 version: 2
 
 models:
-  - name: customers
-    description: This table has basic information about a customer, as well as some derived facts based on a customer's orders
-
+  - name: wh_customers
+    meta:
+      owner: "[email protected]"
+    description: |
+      This table has basic information about a customer, as well as some derived facts based on a customer's orders
+
     columns:
       - name: customer_id
         description: This is a unique identifier for a customer
@@ -29,9 +32,12 @@ models:
       - name: total_order_amount
         description: Total value (AUD) of a customer's orders
 
-  - name: orders
-    description: This table has basic information about orders, as well as some derived facts based on payments
-
+  - name: wh_orders
+    meta:
+        owner: "[email protected]"
+    description: |
+      This table has basic information about orders, as well as some derived facts based on payments
+
     columns:
       - name: order_id
         tests:
@@ -44,7 +50,7 @@ models:
         tests:
           - not_null
           - relationships:
-              to: ref('customers')
+              to: ref('wh_jaffle_customers')
               field: customer_id
 
       - name: order_date

jaffle_shop/models/customers.sql → jaffle_shop/models/warehouse/wh_customers.sql

@@ -1,6 +1,6 @@
 with customers as (
 
-    select * from {{ ref('stg_customers') }}
+    select * from {{ ref('stg_customers_pii') }}
 
 ),
 

jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv

@@ -0,0 +1,2 @@
+fct_name,column_name,id_to_exclude,comment
+fct_staging_dependent_on_staging,parent,stg_customers_pii,Scrubbing pii permitted in staging layer.

packages.yml

@@ -1,9 +1,9 @@
 packages:
   - package: dbt-labs/dbt_utils
-    version: 0.8.6
+    version: 1.1.1
   - package: dbt-labs/spark_utils
     version: 0.3.0
   - package: dbt-labs/codegen
-    version: 0.7.0
+    version: 0.12.1
   - package: dbt-labs/dbt_project_evaluator
-    version: 0.1.3
+    version: 0.6.2