Fix: Migrate open_buildings to new environment.

GoogleCloudPlatform · Nov 8, 2024 · 5490dd1 · 5490dd1
1 parent 4df1a02
commit 5490dd1
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 13 deletions.
diff --git a/datasets/open_buildings/pipelines/open_buildings/open_buildings_dag.py b/datasets/open_buildings/pipelines/open_buildings/open_buildings_dag.py
@@ -74,8 +74,9 @@
         task_id="py_gcs_to_bq",
         startup_timeout_seconds=1000,
         name="load_data",
-        namespace="composer",
-        service_account_name="datasets",
+        namespace="composer-user-workloads",
+        service_account_name="default",
+        config_file="/home/airflow/composer_kube_config",
         image_pull_policy="Always",
         image="{{ var.json.open_buildings.container_registry.run_script_kub }}",
         env_vars={
@@ -85,11 +86,6 @@
             "GCS_BUCKET": "{{ var.value.composer_bucket }}",
             "SCHEMA_FILEPATH": "schema.json",
         },
-        resources={
-            "request_memory": "2G",
-            "request_cpu": "1",
-            "request_ephemeral_storage": "10G",
-        },
     )
 
     (

diff --git a/datasets/open_buildings/pipelines/open_buildings/pipeline.yaml b/datasets/open_buildings/pipelines/open_buildings/pipeline.yaml
@@ -201,8 +201,9 @@ dag:
         task_id: "py_gcs_to_bq"
         startup_timeout_seconds: 1000
         name: "load_data"
-        namespace: "composer"
-        service_account_name: "datasets"
+        namespace: "composer-user-workloads"
+        service_account_name: "default"
+        config_file: "/home/airflow/composer_kube_config"
         image_pull_policy: "Always"
         image: "{{ var.json.open_buildings.container_registry.run_script_kub }}"
         env_vars:
@@ -211,10 +212,6 @@ dag:
           DATASET_ID: "{{ var.json.open_buildings.dataset_id }}"
           GCS_BUCKET: "{{ var.value.composer_bucket }}"
           SCHEMA_FILEPATH: "schema.json"
-        resources:
-          request_memory: "2G"
-          request_cpu: "1"
-          request_ephemeral_storage: "10G"
 
   graph_paths:
     - "bash_gcs_to_gcs >> batch1_bash_gunzip >> batch2_bash_gunzip >> batch3_bash_gunzip >> batch4_bash_gunzip >> batch5_bash_gunzip >> py_gcs_to_bq"