From 9a46dd37a2f55fae5a93987720abc6d399944bbe Mon Sep 17 00:00:00 2001 From: nlarge-google Date: Fri, 8 Nov 2024 19:34:11 +0000 Subject: [PATCH] Fix: Resolved configuration issue in pipeline.yaml. --- .../pipelines/thelook_ecommerce/pipeline.yaml | 19 ++++++++++--------- .../thelook_ecommerce_dag.py | 6 ++++-- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/datasets/thelook_ecommerce/pipelines/thelook_ecommerce/pipeline.yaml b/datasets/thelook_ecommerce/pipelines/thelook_ecommerce/pipeline.yaml index b0454bd8d..082452847 100644 --- a/datasets/thelook_ecommerce/pipelines/thelook_ecommerce/pipeline.yaml +++ b/datasets/thelook_ecommerce/pipelines/thelook_ecommerce/pipeline.yaml @@ -402,15 +402,16 @@ dag: args: task_id: "create_distribution_center_geom_column" configuration: - query: |- - ALTER TABLE `{{ var.value.gcp_project }}.thelook_ecommerce.distribution_centers` - ADD COLUMN IF NOT EXISTS distribution_center_geom GEOGRAPHY; - UPDATE `{{ var.value.gcp_project }}.thelook_ecommerce.distribution_centers` - SET distribution_center_geom = SAFE.ST_GeogFromText(CONCAT('POINT(',CAST(longitude AS STRING), ' ', CAST(latitude as STRING), ')')) - WHERE longitude IS NOT NULL - AND latitude IS NOT NULL; - # Use Legacy SQL should be false for any query that uses a DML statement - useLegacySql: False + query: + query: |- + ALTER TABLE `{{ var.value.gcp_project }}.thelook_ecommerce.distribution_centers` + ADD COLUMN IF NOT EXISTS distribution_center_geom GEOGRAPHY; + UPDATE `{{ var.value.gcp_project }}.thelook_ecommerce.distribution_centers` + SET distribution_center_geom = SAFE.ST_GeogFromText(CONCAT('POINT(',CAST(longitude AS STRING), ' ', CAST(latitude as STRING), ')')) + WHERE longitude IS NOT NULL + AND latitude IS NOT NULL; + # Use Legacy SQL should be false for any query that uses a DML statement + useLegacySql: False graph_paths: - "create_cluster >> generate_thelook >> delete_cluster >> [load_products_to_bq, load_events_to_bq, load_inventory_items_to_bq, load_order_items_to_bq, load_orders_to_bq, load_users_to_bq, load_distribution_centers_to_bq] >> create_user_geom_column >> create_distribution_center_geom_column" diff --git a/datasets/thelook_ecommerce/pipelines/thelook_ecommerce/thelook_ecommerce_dag.py b/datasets/thelook_ecommerce/pipelines/thelook_ecommerce/thelook_ecommerce_dag.py index adb4b459d..1c47b471a 100644 --- a/datasets/thelook_ecommerce/pipelines/thelook_ecommerce/thelook_ecommerce_dag.py +++ b/datasets/thelook_ecommerce/pipelines/thelook_ecommerce/thelook_ecommerce_dag.py @@ -260,8 +260,10 @@ create_distribution_center_geom_column = bigquery.BigQueryInsertJobOperator( task_id="create_distribution_center_geom_column", configuration={ - "query": "ALTER TABLE `{{ var.value.gcp_project }}.thelook_ecommerce.distribution_centers`\n ADD COLUMN IF NOT EXISTS distribution_center_geom GEOGRAPHY;\nUPDATE `{{ var.value.gcp_project }}.thelook_ecommerce.distribution_centers`\n SET distribution_center_geom = SAFE.ST_GeogFromText(CONCAT('POINT(',CAST(longitude AS STRING), ' ', CAST(latitude as STRING), ')'))\n WHERE longitude IS NOT NULL\n AND latitude IS NOT NULL;\n# Use Legacy SQL should be false for any query that uses a DML statement", - "useLegacySql": False, + "query": { + "query": "ALTER TABLE `{{ var.value.gcp_project }}.thelook_ecommerce.distribution_centers`\n ADD COLUMN IF NOT EXISTS distribution_center_geom GEOGRAPHY;\nUPDATE `{{ var.value.gcp_project }}.thelook_ecommerce.distribution_centers`\n SET distribution_center_geom = SAFE.ST_GeogFromText(CONCAT('POINT(',CAST(longitude AS STRING), ' ', CAST(latitude as STRING), ')'))\nWHERE longitude IS NOT NULL\n AND latitude IS NOT NULL;\n# Use Legacy SQL should be false for any query that uses a DML statement", + "useLegacySql": False, + } }, )