From 4fcc0eb969fa69d40f95028e2105623d38020ef4 Mon Sep 17 00:00:00 2001 From: Nicholas Large Date: Thu, 14 Nov 2024 10:36:51 -0600 Subject: [PATCH] Fix: Migrate Human Variant Annotation to new environment. (#871) --- .../pipelines/clinvar/clinvar_dag.py | 14 +++++++------- .../pipelines/clinvar/pipeline.yaml | 16 ++++++---------- .../pipelines/db_snp/db_snp_dag.py | 2 +- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/datasets/human_variant_annotation/pipelines/clinvar/clinvar_dag.py b/datasets/human_variant_annotation/pipelines/clinvar/clinvar_dag.py index 0ad21eda1..0e40e7b6a 100644 --- a/datasets/human_variant_annotation/pipelines/clinvar/clinvar_dag.py +++ b/datasets/human_variant_annotation/pipelines/clinvar/clinvar_dag.py @@ -1,4 +1,4 @@ -# Copyright 2021 Google LLC +# Copyright 2022 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -38,8 +38,9 @@ task_id="clinvar_vcf_grch37", startup_timeout_seconds=600, name="name_basics", - namespace="composer", - service_account_name="datasets", + namespace="composer-user-workloads", + service_account_name="default", + config_file="/home/airflow/composer_kube_config", image_pull_policy="Always", image="{{ var.json.human_variant_annotation.container_registry.run_csv_transform_kub }}", env_vars={ @@ -50,7 +51,6 @@ "TARGET_GCS_FOLDER": "data/human_variant_annotation/clinVar-vcf_GRCh37/", "PIPELINE": "clinvar", }, - resources={"limit_memory": "1G", "limit_cpu": "1"}, ) # Task to run a GoogleCloudStorageToGoogleCloudStorageOperator @@ -69,8 +69,9 @@ task_id="clinvar_vcf_grch38", startup_timeout_seconds=600, name="name_basics", - namespace="composer", - service_account_name="datasets", + namespace="composer-user-workloads", + service_account_name="default", + config_file="/home/airflow/composer_kube_config", image_pull_policy="Always", image="{{ var.json.human_variant_annotation.container_registry.run_csv_transform_kub }}", env_vars={ @@ -81,7 +82,6 @@ "TARGET_GCS_FOLDER": "data/human_variant_annotation/clinVar-vcf_GRCh38/", "PIPELINE": "db_snp", }, - resources={"limit_memory": "1G", "limit_cpu": "1"}, ) # Task to run a GoogleCloudStorageToGoogleCloudStorageOperator diff --git a/datasets/human_variant_annotation/pipelines/clinvar/pipeline.yaml b/datasets/human_variant_annotation/pipelines/clinvar/pipeline.yaml index 0a316cc89..8e86a3fc1 100644 --- a/datasets/human_variant_annotation/pipelines/clinvar/pipeline.yaml +++ b/datasets/human_variant_annotation/pipelines/clinvar/pipeline.yaml @@ -35,8 +35,9 @@ dag: task_id: "clinvar_vcf_grch37" startup_timeout_seconds: 600 name: "name_basics" - namespace: "composer" - service_account_name: "datasets" + namespace: "composer-user-workloads" + service_account_name: "default" + config_file: "/home/airflow/composer_kube_config" image_pull_policy: "Always" image: "{{ var.json.human_variant_annotation.container_registry.run_csv_transform_kub }}" env_vars: @@ -46,9 +47,6 @@ dag: GCS_BUCKET: "{{ var.value.composer_bucket }}" TARGET_GCS_FOLDER: "data/human_variant_annotation/clinVar-vcf_GRCh37/" PIPELINE: "clinvar" - resources: - limit_memory: "1G" - limit_cpu: "1" - operator: "GoogleCloudStorageToGoogleCloudStorageOperator" description: "Task to run a GoogleCloudStorageToGoogleCloudStorageOperator" @@ -67,8 +65,9 @@ dag: task_id: "clinvar_vcf_grch38" startup_timeout_seconds: 600 name: "name_basics" - namespace: "composer" - service_account_name: "datasets" + namespace: "composer-user-workloads" + service_account_name: "default" + config_file: "/home/airflow/composer_kube_config" image_pull_policy: "Always" image: "{{ var.json.human_variant_annotation.container_registry.run_csv_transform_kub }}" env_vars: @@ -78,9 +77,6 @@ dag: GCS_BUCKET: "{{ var.value.composer_bucket }}" TARGET_GCS_FOLDER: "data/human_variant_annotation/clinVar-vcf_GRCh38/" PIPELINE: "db_snp" - resources: - limit_memory: "1G" - limit_cpu: "1" - operator: "GoogleCloudStorageToGoogleCloudStorageOperator" description: "Task to run a GoogleCloudStorageToGoogleCloudStorageOperator" diff --git a/datasets/human_variant_annotation/pipelines/db_snp/db_snp_dag.py b/datasets/human_variant_annotation/pipelines/db_snp/db_snp_dag.py index 00f5ce899..a26cf9dae 100644 --- a/datasets/human_variant_annotation/pipelines/db_snp/db_snp_dag.py +++ b/datasets/human_variant_annotation/pipelines/db_snp/db_snp_dag.py @@ -1,4 +1,4 @@ -# Copyright 2021 Google LLC +# Copyright 2022 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.