Fix: Migrate mimic_iii dag (#864)

GoogleCloudPlatform · Nov 11, 2024 · 95bfb72 · 95bfb72
1 parent f8a0632
commit 95bfb72
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 8 deletions.
diff --git a/datasets/mimic_iii/pipelines/mimic_iii/mimic_iii_dag.py b/datasets/mimic_iii/pipelines/mimic_iii/mimic_iii_dag.py
@@ -36,8 +36,9 @@
     copy_bq_dataset = kubernetes_pod.KubernetesPodOperator(
         task_id="copy_bq_dataset",
         name="copy_bq_dataset",
-        namespace="composer",
-        service_account_name="datasets",
+        namespace="composer-user-workloads",
+        service_account_name="default",
+        config_file="/home/airflow/composer_kube_config",
         image_pull_policy="Always",
         image="{{ var.json.mimic_iii.container_registry.run_script_kub }}",
         env_vars={
@@ -46,7 +47,6 @@
             "TARGET_PROJECT_ID": "{{ var.json.mimic_iii.target_project_id }}",
             "TARGET_BQ_DATASET": "{{ var.json.mimic_iii.target_bq_dataset }}",
         },
-        resources={"request_memory": "128M", "request_cpu": "200m"},
     )
 
     copy_bq_dataset
diff --git a/datasets/mimic_iii/pipelines/mimic_iii/pipeline.yaml b/datasets/mimic_iii/pipelines/mimic_iii/pipeline.yaml
@@ -34,18 +34,16 @@ dag:
       args:
         task_id: "copy_bq_dataset"
         name: "copy_bq_dataset"
-        namespace: "composer"
-        service_account_name: "datasets"
+        namespace: "composer-user-workloads"
+        service_account_name: "default"
+        config_file: "/home/airflow/composer_kube_config"
         image_pull_policy: "Always"
         image: "{{ var.json.mimic_iii.container_registry.run_script_kub }}"
         env_vars:
           SOURCE_PROJECT_ID: "{{ var.json.mimic_iii.source_project_id }}"
           SOURCE_BQ_DATASET: "{{ var.json.mimic_iii.source_bq_dataset }}"
           TARGET_PROJECT_ID: "{{ var.json.mimic_iii.target_project_id }}"
           TARGET_BQ_DATASET: "{{ var.json.mimic_iii.target_bq_dataset }}"
-        resources:
-          request_memory: "128M"
-          request_cpu: "200m"
 
   graph_paths:
     - "copy_bq_dataset"