From 4afd41b87591c70abed60141d7481a639e0af685 Mon Sep 17 00:00:00 2001 From: Valentin Matton Date: Mon, 9 Oct 2023 15:06:29 +0200 Subject: [PATCH] fix(pipeline): run mednum intermediate models --- pipeline/dags/main.py | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/pipeline/dags/main.py b/pipeline/dags/main.py index c88b23b0..db073911 100644 --- a/pipeline/dags/main.py +++ b/pipeline/dags/main.py @@ -107,28 +107,10 @@ def _geocode(): select=f"path:models/staging/sources/**/stg_{dbt_source_id}__*.sql", ) - dbt_run_intermediate = dbt_operator_factory( - task_id="dbt_run_intermediate", - command="run", - select=( - f"path:models/intermediate/sources/**/int_{dbt_source_id}__*.sql" - ), - ) - - dbt_test_intermediate = dbt_operator_factory( - task_id="dbt_test_intermediate", - command="test", - select=( - f"path:models/intermediate/sources/**/int_{dbt_source_id}__*.sql" - ), - ) + # TODO: add intermediate models here + # this would require to refactor mediation numerique models - ( - dbt_run_staging - >> dbt_test_staging - >> dbt_run_intermediate - >> dbt_test_intermediate - ) + (dbt_run_staging >> dbt_test_staging) dbt_staging_tasks_list += [source_task_group] @@ -139,7 +121,7 @@ def _geocode(): [ # FIXME: handle odspep as other sources (add to dags/settings.py) "path:models/staging/sources/odspep", - "path:models/intermediate/sources/odspep", + "path:models/intermediate/sources/**/*", "path:models/intermediate/int__union_adresses.sql", "path:models/intermediate/int__union_services.sql", "path:models/intermediate/int__union_structures.sql",