Fix: Resolved configuration issue in pipeline.yaml.

GoogleCloudPlatform · Nov 8, 2024 · 9a46dd3 · 9a46dd3
1 parent ca1d39e
commit 9a46dd3
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 11 deletions.
diff --git a/datasets/thelook_ecommerce/pipelines/thelook_ecommerce/pipeline.yaml b/datasets/thelook_ecommerce/pipelines/thelook_ecommerce/pipeline.yaml
@@ -402,15 +402,16 @@ dag:
       args:
         task_id: "create_distribution_center_geom_column"
         configuration:
-          query: |-
-            ALTER TABLE `{{ var.value.gcp_project }}.thelook_ecommerce.distribution_centers`
-              ADD COLUMN IF NOT EXISTS distribution_center_geom GEOGRAPHY;
-            UPDATE `{{ var.value.gcp_project }}.thelook_ecommerce.distribution_centers`
-               SET distribution_center_geom = SAFE.ST_GeogFromText(CONCAT('POINT(',CAST(longitude AS STRING), ' ', CAST(latitude as STRING), ')'))
-             WHERE longitude IS NOT NULL
-               AND latitude IS NOT NULL;
-            # Use Legacy SQL should be false for any query that uses a DML statement
-          useLegacySql: False
+          query:
+            query: |-
+              ALTER TABLE `{{ var.value.gcp_project }}.thelook_ecommerce.distribution_centers`
+                ADD COLUMN IF NOT EXISTS distribution_center_geom GEOGRAPHY;
+              UPDATE `{{ var.value.gcp_project }}.thelook_ecommerce.distribution_centers`
+                SET distribution_center_geom = SAFE.ST_GeogFromText(CONCAT('POINT(',CAST(longitude AS STRING), ' ', CAST(latitude as STRING), ')'))
+              WHERE longitude IS NOT NULL
+                AND latitude IS NOT NULL;
+              # Use Legacy SQL should be false for any query that uses a DML statement
+            useLegacySql: False
 
   graph_paths:
     - "create_cluster >> generate_thelook >> delete_cluster >> [load_products_to_bq, load_events_to_bq, load_inventory_items_to_bq, load_order_items_to_bq, load_orders_to_bq, load_users_to_bq, load_distribution_centers_to_bq] >> create_user_geom_column >> create_distribution_center_geom_column"
diff --git a/datasets/thelook_ecommerce/pipelines/thelook_ecommerce/thelook_ecommerce_dag.py b/datasets/thelook_ecommerce/pipelines/thelook_ecommerce/thelook_ecommerce_dag.py
@@ -260,8 +260,10 @@
     create_distribution_center_geom_column = bigquery.BigQueryInsertJobOperator(
         task_id="create_distribution_center_geom_column",
         configuration={
-            "query": "ALTER TABLE `{{ var.value.gcp_project }}.thelook_ecommerce.distribution_centers`\n  ADD COLUMN IF NOT EXISTS distribution_center_geom GEOGRAPHY;\nUPDATE `{{ var.value.gcp_project }}.thelook_ecommerce.distribution_centers`\n   SET distribution_center_geom = SAFE.ST_GeogFromText(CONCAT('POINT(',CAST(longitude AS STRING), ' ', CAST(latitude as STRING), ')'))\n WHERE longitude IS NOT NULL\n   AND latitude IS NOT NULL;\n# Use Legacy SQL should be false for any query that uses a DML statement",
-            "useLegacySql": False,
+            "query": {
+                "query": "ALTER TABLE `{{ var.value.gcp_project }}.thelook_ecommerce.distribution_centers`\n  ADD COLUMN IF NOT EXISTS distribution_center_geom GEOGRAPHY;\nUPDATE `{{ var.value.gcp_project }}.thelook_ecommerce.distribution_centers`\n  SET distribution_center_geom = SAFE.ST_GeogFromText(CONCAT('POINT(',CAST(longitude AS STRING), ' ', CAST(latitude as STRING), ')'))\nWHERE longitude IS NOT NULL\n  AND latitude IS NOT NULL;\n# Use Legacy SQL should be false for any query that uses a DML statement",
+                "useLegacySql": False,
+            }
         },
     )