Merge pull request #90 from Synthetixio/feat/scheduler-add-test-dag

Add test DAG to Scheduler
Synthetixio · Jul 30, 2024 · 3b37679 · 3b37679
2 parents c11f447 + de6cfc8
commit 3b37679
Show file tree

Hide file tree

Showing 7 changed files with 13 additions and 9 deletions.
diff --git a/scheduler/dags/v3_etl.py b/scheduler/dags/v3_etl.py
@@ -80,7 +80,17 @@ def create_dag(network, rpc_var):
         network_env_var=rpc_var,
     )
 
-    latest_only_task >> extract_task >> transform_task
+    test_task_id = f"test_{network}"
+    test_task = create_docker_operator(
+        dag=dag,
+        task_id=test_task_id,
+        config_file=None,
+        image="data-transformer",
+        command=f"dbt test --target {'prod' if network != 'optimism_mainnet' else 'prod-op'} --select tag:{network} --profiles-dir profiles --profile synthetix",
+        network_env_var=rpc_var
+    )
+
+    latest_only_task >> extract_task >> transform_task >> test_task
 
     return dag
 

diff --git a/transformers/synthetix/models/marts/arbitrum/mainnet/prices/schema.yml b/transformers/synthetix/models/marts/arbitrum/mainnet/prices/schema.yml
@@ -9,8 +9,6 @@ models:
       - name: market_address
         description: "Market address"
         data_type: text
-        tests:
-          - not_null
       - name: market_symbol
         description: "Market symbol"
         data_type: text

diff --git a/transformers/synthetix/models/marts/arbitrum/sepolia/prices/schema.yml b/transformers/synthetix/models/marts/arbitrum/sepolia/prices/schema.yml
@@ -9,8 +9,6 @@ models:
       - name: market_address
         description: "Market address"
         data_type: text
-        tests:
-          - not_null
       - name: market_symbol
         description: "Market symbol"
         data_type: text

diff --git a/transformers/synthetix/models/marts/base/mainnet/perp/schema.yml b/transformers/synthetix/models/marts/base/mainnet/perp/schema.yml
@@ -290,6 +290,7 @@ models:
         tests:
           - not_null
           - dbt_utils.accepted_range:
+              severity: warn
               min_value: 0
               inclusive: true
       - name: exchange_fees

diff --git a/transformers/synthetix/models/marts/base/mainnet/prices/schema.yml b/transformers/synthetix/models/marts/base/mainnet/prices/schema.yml
@@ -9,8 +9,6 @@ models:
       - name: market_address
         description: "Market address"
         data_type: text
-        tests:
-          - not_null
       - name: market_symbol
         description: "Market symbol"
         data_type: text

diff --git a/transformers/synthetix/models/marts/base/sepolia/perp/schema.yml b/transformers/synthetix/models/marts/base/sepolia/perp/schema.yml
@@ -290,6 +290,7 @@ models:
         tests:
           - not_null
           - dbt_utils.accepted_range:
+              severity: warn
               min_value: 0
               inclusive: true
       - name: exchange_fees

diff --git a/transformers/synthetix/models/marts/base/sepolia/prices/schema.yml b/transformers/synthetix/models/marts/base/sepolia/prices/schema.yml
@@ -9,8 +9,6 @@ models:
       - name: market_address
         description: "Market address"
         data_type: text
-        tests:
-          - not_null
       - name: market_symbol
         description: "Market symbol"
         data_type: text