Fix: Migrate NOAA Passive Bioacoustic DAG to new environment

GoogleCloudPlatform · Nov 7, 2024 · 7b643fb · 7b643fb
1 parent c0526ac
commit 7b643fb
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 14 deletions.
diff --git a/...aa_passive_bioacoustic/pipelines/noaa_passive_bioacoustic/noaa_passive_bioacoustic_dag.py b/...aa_passive_bioacoustic/pipelines/noaa_passive_bioacoustic/noaa_passive_bioacoustic_dag.py
@@ -49,8 +49,9 @@
         task_id="kub_gcs_to_bq",
         startup_timeout_seconds=1000,
         name="load_data",
-        namespace="composer",
-        service_account_name="datasets",
+        namespace="composer-user-workloads",
+        service_account_name="default",
+        config_file="/home/airflow/composer_kube_config",
         image_pull_policy="Always",
         image="{{ var.json.noaa_passive_bioacoustic.container_registry.run_csv_transform_kub }}",
         env_vars={
@@ -62,11 +63,6 @@
             "SCHEMA_FILEPATH": "schema.json",
             "HEADER": '["CLOUD_PATH" ,"FILE_NAME" ,"START_DATE" ,"START_TIME" ,"DATA_COLLECTION_NAME" ,"SOURCE_ORGANIZATION" ,"FUNDING_ORGANIZATION" ,"PROJECT_NAME" ,"SENSOR_DEPTH" ,"SAMPLE_RATE" ,"DURATION" ,"PLATFORM_NAME" ,"SHAPE1" ,"SHAPE2" ,"SHAPE3" ,"SHAPE4" ,"SHAPE5" ,"SHAPE6" ,"SHAPE7" ,"SHAPE8" ,"SHAPE9" ,"SHAPE10" ]',
         },
-        resources={
-            "request_memory": "2G",
-            "request_cpu": "1",
-            "request_ephemeral_storage": "10G",
-        },
     )
 
     gcs_to_gcs_operator >> kub_gcs_to_bq
diff --git a/datasets/noaa_passive_bioacoustic/pipelines/noaa_passive_bioacoustic/pipeline.yaml b/datasets/noaa_passive_bioacoustic/pipelines/noaa_passive_bioacoustic/pipeline.yaml
@@ -48,8 +48,9 @@ dag:
         task_id: "kub_gcs_to_bq"
         startup_timeout_seconds: 1000
         name: "load_data"
-        namespace: "composer"
-        service_account_name: "datasets"
+        namespace: "composer-user-workloads"
+        service_account_name: "default"
+        config_file: "/home/airflow/composer_kube_config"
         image_pull_policy: "Always"
         image: "{{ var.json.noaa_passive_bioacoustic.container_registry.run_csv_transform_kub }}"
         env_vars:
@@ -62,10 +63,5 @@ dag:
           HEADER: >-
             ["CLOUD_PATH" ,"FILE_NAME" ,"START_DATE" ,"START_TIME" ,"DATA_COLLECTION_NAME" ,"SOURCE_ORGANIZATION" ,"FUNDING_ORGANIZATION" ,"PROJECT_NAME" ,"SENSOR_DEPTH" ,"SAMPLE_RATE" ,"DURATION" ,"PLATFORM_NAME" ,"SHAPE1" ,"SHAPE2" ,"SHAPE3" ,"SHAPE4" ,"SHAPE5" ,"SHAPE6" ,"SHAPE7" ,"SHAPE8" ,"SHAPE9" ,"SHAPE10" ]
 
-        resources:
-          request_memory: "2G"
-          request_cpu: "1"
-          request_ephemeral_storage: "10G"
-
   graph_paths:
     - "gcs_to_gcs_operator >> kub_gcs_to_bq"