Fix: Migration changes for human_variant_annotation DAG

GoogleCloudPlatform · Nov 14, 2024 · 2d8160a · 2d8160a
1 parent 5d8ed02
commit 2d8160a
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 18 deletions.
diff --git a/datasets/human_variant_annotation/pipelines/clinvar/clinvar_dag.py b/datasets/human_variant_annotation/pipelines/clinvar/clinvar_dag.py
@@ -1,4 +1,4 @@
-# Copyright 2021 Google LLC
+# Copyright 2022 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -38,8 +38,9 @@
         task_id="clinvar_vcf_grch37",
         startup_timeout_seconds=600,
         name="name_basics",
-        namespace="composer",
-        service_account_name="datasets",
+        namespace="composer-user-workloads",
+        service_account_name="default",
+        config_file="/home/airflow/composer_kube_config",
         image_pull_policy="Always",
         image="{{ var.json.human_variant_annotation.container_registry.run_csv_transform_kub }}",
         env_vars={
@@ -50,7 +51,11 @@
             "TARGET_GCS_FOLDER": "data/human_variant_annotation/clinVar-vcf_GRCh37/",
             "PIPELINE": "clinvar",
         },
-        resources={"limit_memory": "1G", "limit_cpu": "1"},
+        container_resources={
+            "memory": {"request": "32Gi"},
+            "cpu": {"request": "2"},
+            "ephemeral-storage": {"request": "10Gi"},
+        },
     )
 
     # Task to run a GoogleCloudStorageToGoogleCloudStorageOperator
@@ -69,8 +74,9 @@
         task_id="clinvar_vcf_grch38",
         startup_timeout_seconds=600,
         name="name_basics",
-        namespace="composer",
-        service_account_name="datasets",
+        namespace="composer-user-workloads",
+        service_account_name="default",
+        config_file="/home/airflow/composer_kube_config",
         image_pull_policy="Always",
         image="{{ var.json.human_variant_annotation.container_registry.run_csv_transform_kub }}",
         env_vars={
@@ -81,7 +87,11 @@
             "TARGET_GCS_FOLDER": "data/human_variant_annotation/clinVar-vcf_GRCh38/",
             "PIPELINE": "db_snp",
         },
-        resources={"limit_memory": "1G", "limit_cpu": "1"},
+        container_resources={
+            "memory": {"request": "32Gi"},
+            "cpu": {"request": "2"},
+            "ephemeral-storage": {"request": "10Gi"},
+        },
     )
 
     # Task to run a GoogleCloudStorageToGoogleCloudStorageOperator

diff --git a/datasets/human_variant_annotation/pipelines/clinvar/pipeline.yaml b/datasets/human_variant_annotation/pipelines/clinvar/pipeline.yaml
@@ -35,8 +35,9 @@ dag:
         task_id: "clinvar_vcf_grch37"
         startup_timeout_seconds: 600
         name: "name_basics"
-        namespace: "composer"
-        service_account_name: "datasets"
+        namespace: "composer-user-workloads"
+        service_account_name: "default"
+        config_file: "/home/airflow/composer_kube_config"
         image_pull_policy: "Always"
         image: "{{ var.json.human_variant_annotation.container_registry.run_csv_transform_kub }}"
         env_vars:
@@ -46,9 +47,13 @@ dag:
           GCS_BUCKET: "{{ var.value.composer_bucket }}"
           TARGET_GCS_FOLDER: "data/human_variant_annotation/clinVar-vcf_GRCh37/"
           PIPELINE: "clinvar"
-        resources:
-          limit_memory: "1G"
-          limit_cpu: "1"
+        container_resources:
+          memory:
+            request: "32Gi"
+          cpu:
+            request: "2"
+          ephemeral-storage:
+            request: "10Gi"
 
     - operator: "GoogleCloudStorageToGoogleCloudStorageOperator"
       description: "Task to run a GoogleCloudStorageToGoogleCloudStorageOperator"
@@ -67,8 +72,9 @@ dag:
         task_id: "clinvar_vcf_grch38"
         startup_timeout_seconds: 600
         name: "name_basics"
-        namespace: "composer"
-        service_account_name: "datasets"
+        namespace: "composer-user-workloads"
+        service_account_name: "default"
+        config_file: "/home/airflow/composer_kube_config"
         image_pull_policy: "Always"
         image: "{{ var.json.human_variant_annotation.container_registry.run_csv_transform_kub }}"
         env_vars:
@@ -78,9 +84,13 @@ dag:
           GCS_BUCKET: "{{ var.value.composer_bucket }}"
           TARGET_GCS_FOLDER: "data/human_variant_annotation/clinVar-vcf_GRCh38/"
           PIPELINE: "db_snp"
-        resources:
-          limit_memory: "1G"
-          limit_cpu: "1"
+        container_resources:
+          memory:
+            request: "32Gi"
+          cpu:
+            request: "2"
+          ephemeral-storage:
+            request: "10Gi"
 
     - operator: "GoogleCloudStorageToGoogleCloudStorageOperator"
       description: "Task to run a GoogleCloudStorageToGoogleCloudStorageOperator"

diff --git a/datasets/human_variant_annotation/pipelines/db_snp/db_snp_dag.py b/datasets/human_variant_annotation/pipelines/db_snp/db_snp_dag.py
@@ -1,4 +1,4 @@
-# Copyright 2021 Google LLC
+# Copyright 2022 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.