From 051287f41cad671b25d04d082245fb3d4fa9a16d Mon Sep 17 00:00:00 2001 From: Lydia Andreyevna Krasilnikova Date: Tue, 12 Nov 2024 14:47:05 -0500 Subject: [PATCH 1/9] removed refine_augur_tree task from augur_from_mltree workflow --- pipes/WDL/workflows/augur_from_mltree.wdl | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/pipes/WDL/workflows/augur_from_mltree.wdl b/pipes/WDL/workflows/augur_from_mltree.wdl index aa7f50015..0c24ee6a1 100644 --- a/pipes/WDL/workflows/augur_from_mltree.wdl +++ b/pipes/WDL/workflows/augur_from_mltree.wdl @@ -54,35 +54,29 @@ workflow augur_from_mltree { } } - call nextstrain.refine_augur_tree { - input: - raw_tree = raw_tree, - msa_or_vcf = msa_or_vcf, - metadata = sample_metadata - } if(defined(ancestral_traits_to_infer) && length(select_first([ancestral_traits_to_infer,[]]))>0) { call nextstrain.ancestral_traits { input: - tree = refine_augur_tree.tree_refined, + tree = raw_tree, metadata = sample_metadata, columns = select_first([ancestral_traits_to_infer,[]]) } } call nextstrain.ancestral_tree { input: - tree = refine_augur_tree.tree_refined, + tree = raw_tree, msa_or_vcf = msa_or_vcf } call nextstrain.translate_augur_tree { input: - tree = refine_augur_tree.tree_refined, + tree = raw_tree, nt_muts = ancestral_tree.nt_muts_json, genbank_gb = genbank_gb } if(defined(clades_tsv)) { call nextstrain.assign_clades_to_nodes { input: - tree_nwk = refine_augur_tree.tree_refined, + tree_nwk = raw_tree, nt_muts_json = ancestral_tree.nt_muts_json, aa_muts_json = translate_augur_tree.aa_muts_json, ref_fasta = ref_fasta, @@ -91,10 +85,9 @@ workflow augur_from_mltree { } call nextstrain.export_auspice_json { input: - tree = refine_augur_tree.tree_refined, + tree = raw_tree, sample_metadata = sample_metadata, node_data_jsons = select_all([ - refine_augur_tree.branch_lengths, ancestral_traits.node_data_json, ancestral_tree.nt_muts_json, translate_augur_tree.aa_muts_json, @@ -103,7 +96,7 @@ workflow augur_from_mltree { } output { - File time_tree = refine_augur_tree.tree_refined + File time_tree = raw_tree File auspice_input_json = export_auspice_json.virus_json } -} \ No newline at end of file +} From 72264bc578756a03c3ba77d0fbe995a4e07a5b73 Mon Sep 17 00:00:00 2001 From: Lydia Andreyevna Krasilnikova Date: Tue, 12 Nov 2024 17:08:17 -0500 Subject: [PATCH 2/9] removed translate_augur_tree --- pipes/WDL/workflows/augur_from_mltree.wdl | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pipes/WDL/workflows/augur_from_mltree.wdl b/pipes/WDL/workflows/augur_from_mltree.wdl index 0c24ee6a1..8ae58c7e0 100644 --- a/pipes/WDL/workflows/augur_from_mltree.wdl +++ b/pipes/WDL/workflows/augur_from_mltree.wdl @@ -67,18 +67,11 @@ workflow augur_from_mltree { tree = raw_tree, msa_or_vcf = msa_or_vcf } - call nextstrain.translate_augur_tree { - input: - tree = raw_tree, - nt_muts = ancestral_tree.nt_muts_json, - genbank_gb = genbank_gb - } if(defined(clades_tsv)) { call nextstrain.assign_clades_to_nodes { input: tree_nwk = raw_tree, nt_muts_json = ancestral_tree.nt_muts_json, - aa_muts_json = translate_augur_tree.aa_muts_json, ref_fasta = ref_fasta, clades_tsv = select_first([clades_tsv]) } @@ -90,7 +83,6 @@ workflow augur_from_mltree { node_data_jsons = select_all([ ancestral_traits.node_data_json, ancestral_tree.nt_muts_json, - translate_augur_tree.aa_muts_json, assign_clades_to_nodes.node_clade_data_json]), auspice_config = auspice_config } From 91a56d164afa58185699ab9eaee221cac23620d5 Mon Sep 17 00:00:00 2001 From: Lydia Andreyevna Krasilnikova Date: Wed, 13 Nov 2024 13:58:55 -0500 Subject: [PATCH 3/9] reverted augur_from_mltree back to how it was --- pipes/WDL/workflows/augur_from_mltree.wdl | 25 ++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/pipes/WDL/workflows/augur_from_mltree.wdl b/pipes/WDL/workflows/augur_from_mltree.wdl index 8ae58c7e0..19f9ca18d 100644 --- a/pipes/WDL/workflows/augur_from_mltree.wdl +++ b/pipes/WDL/workflows/augur_from_mltree.wdl @@ -54,41 +54,56 @@ workflow augur_from_mltree { } } + call nextstrain.refine_augur_tree { + input: + raw_tree = raw_tree, + msa_or_vcf = msa_or_vcf, + metadata = sample_metadata + } if(defined(ancestral_traits_to_infer) && length(select_first([ancestral_traits_to_infer,[]]))>0) { call nextstrain.ancestral_traits { input: - tree = raw_tree, + tree = refine_augur_tree.tree_refined, metadata = sample_metadata, columns = select_first([ancestral_traits_to_infer,[]]) } } call nextstrain.ancestral_tree { input: - tree = raw_tree, + tree = refine_augur_tree.tree_refined, msa_or_vcf = msa_or_vcf } + call nextstrain.translate_augur_tree { + input: + tree = refine_augur_tree.tree_refined, + nt_muts = ancestral_tree.nt_muts_json, + genbank_gb = genbank_gb + } if(defined(clades_tsv)) { call nextstrain.assign_clades_to_nodes { input: - tree_nwk = raw_tree, + tree_nwk = refine_augur_tree.tree_refined, nt_muts_json = ancestral_tree.nt_muts_json, + aa_muts_json = translate_augur_tree.aa_muts_json, ref_fasta = ref_fasta, clades_tsv = select_first([clades_tsv]) } } call nextstrain.export_auspice_json { input: - tree = raw_tree, + tree = refine_augur_tree.tree_refined, sample_metadata = sample_metadata, node_data_jsons = select_all([ + refine_augur_tree.branch_lengths, ancestral_traits.node_data_json, ancestral_tree.nt_muts_json, + translate_augur_tree.aa_muts_json, assign_clades_to_nodes.node_clade_data_json]), auspice_config = auspice_config } output { - File time_tree = raw_tree + File time_tree = refine_augur_tree.tree_refined File auspice_input_json = export_auspice_json.virus_json } } From 6c0ba5af1d9da7cb002140b59bd64151c16da200 Mon Sep 17 00:00:00 2001 From: Lydia Andreyevna Krasilnikova Date: Wed, 13 Nov 2024 14:00:31 -0500 Subject: [PATCH 4/9] removed --timetree from refine_augur_tree command --- pipes/WDL/tasks/tasks_nextstrain.wdl | 1 - 1 file changed, 1 deletion(-) diff --git a/pipes/WDL/tasks/tasks_nextstrain.wdl b/pipes/WDL/tasks/tasks_nextstrain.wdl index 52d874982..8d7fdacb7 100644 --- a/pipes/WDL/tasks/tasks_nextstrain.wdl +++ b/pipes/WDL/tasks/tasks_nextstrain.wdl @@ -1409,7 +1409,6 @@ task refine_augur_tree { --metadata "~{metadata}" \ --output-tree "~{out_basename}_timetree.nwk" \ --output-node-data "~{out_basename}_branch_lengths.json" \ - --timetree \ ~{"--clock-rate " + clock_rate} \ ~{"--clock-std-dev " + clock_std_dev} \ ~{"--coalescent " + coalescent} \ From a4a0667248be950f95f96c21d4777189a12d7739 Mon Sep 17 00:00:00 2001 From: Lydia Andreyevna Krasilnikova Date: Mon, 18 Nov 2024 14:13:32 -0500 Subject: [PATCH 5/9] exposed --timetree through generate_timetree parameter --- pipes/WDL/tasks/tasks_nextstrain.wdl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pipes/WDL/tasks/tasks_nextstrain.wdl b/pipes/WDL/tasks/tasks_nextstrain.wdl index 8d7fdacb7..8d0ad85be 100644 --- a/pipes/WDL/tasks/tasks_nextstrain.wdl +++ b/pipes/WDL/tasks/tasks_nextstrain.wdl @@ -1373,6 +1373,7 @@ task refine_augur_tree { File msa_or_vcf File metadata + Boolean? generate_timetree = true Int? gen_per_year Float? clock_rate Float? clock_std_dev @@ -1409,6 +1410,7 @@ task refine_augur_tree { --metadata "~{metadata}" \ --output-tree "~{out_basename}_timetree.nwk" \ --output-node-data "~{out_basename}_branch_lengths.json" \ + ~{true="--timetree" false="" generate_timetree} \ ~{"--clock-rate " + clock_rate} \ ~{"--clock-std-dev " + clock_std_dev} \ ~{"--coalescent " + coalescent} \ From b7cf475e00a1003b78993d6656b59838f4df44ae Mon Sep 17 00:00:00 2001 From: Lydia Andreyevna Krasilnikova Date: Tue, 19 Nov 2024 10:59:44 -0500 Subject: [PATCH 6/9] make generate_timetree not optional Co-authored-by: Daniel Park --- pipes/WDL/tasks/tasks_nextstrain.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipes/WDL/tasks/tasks_nextstrain.wdl b/pipes/WDL/tasks/tasks_nextstrain.wdl index 8d0ad85be..c18d75e78 100644 --- a/pipes/WDL/tasks/tasks_nextstrain.wdl +++ b/pipes/WDL/tasks/tasks_nextstrain.wdl @@ -1373,7 +1373,7 @@ task refine_augur_tree { File msa_or_vcf File metadata - Boolean? generate_timetree = true + Boolean generate_timetree = true Int? gen_per_year Float? clock_rate Float? clock_std_dev From 5487410030c53b68a553aae13b360ed7029a8090 Mon Sep 17 00:00:00 2001 From: Chris Tomkins-Tinch Date: Thu, 21 Nov 2024 18:14:33 -0500 Subject: [PATCH 7/9] remove exemption from global version pin for viral-core image used in check_terra_env; update viral-core spec of the task to 2.4.0 remove exemption from global version pin for viral-core image used in check_terra_env; update viral-core spec of the task to 2.4.0 --- pipes/WDL/tasks/tasks_terra.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipes/WDL/tasks/tasks_terra.wdl b/pipes/WDL/tasks/tasks_terra.wdl index a920ad548..1e421dcb1 100644 --- a/pipes/WDL/tasks/tasks_terra.wdl +++ b/pipes/WDL/tasks/tasks_terra.wdl @@ -32,7 +32,7 @@ task gcs_copy { task check_terra_env { input { - String docker = "quay.io/broadinstitute/viral-core:2.2.2" #skip-global-version-pin + String docker = "quay.io/broadinstitute/viral-core:2.4.0" } meta { description: "task for inspection of backend to determine whether the task is running on Terra and/or GCP" From 0e1b4257696016fd31f6c3486c9c6f5a02c880d8 Mon Sep 17 00:00:00 2001 From: Danny Park Date: Fri, 22 Nov 2024 14:42:04 -0500 Subject: [PATCH 8/9] WDL cleanups --- pipes/WDL/workflows/demux_deplete.wdl | 64 ++++++++++++++------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/pipes/WDL/workflows/demux_deplete.wdl b/pipes/WDL/workflows/demux_deplete.wdl index 3f2936b44..e6b1deb2f 100644 --- a/pipes/WDL/workflows/demux_deplete.wdl +++ b/pipes/WDL/workflows/demux_deplete.wdl @@ -26,7 +26,7 @@ workflow demux_deplete { Boolean insert_demux_outputs_into_terra_tables=false File? sample_rename_map - Array[File]? biosample_map_tsvs + Array[File] biosample_map_tsvs = [] Int min_reads_per_bam = 100 String? instrument_model_user_specified @@ -183,22 +183,43 @@ workflow demux_deplete { Pair[String,Int] count_cleaned = (basename(raw_reads, '.bam'), read_count_post_depletion) } - if (length(flatten(select_all([biosample_map_tsvs]))) > 0) { + if(insert_demux_outputs_into_terra_tables) { + call terra.check_terra_env + + if(check_terra_env.is_running_on_terra) { + call terra.create_or_update_sample_tables { + input: + flowcell_run_id = illumina_demux.run_info[0]['run_id'], + workspace_name = check_terra_env.workspace_name, + workspace_namespace = check_terra_env.workspace_namespace, + + raw_reads_unaligned_bams = flatten(illumina_demux.raw_reads_unaligned_bams), + cleaned_reads_unaligned_bams = select_all(cleaned_bam_passing), + meta_by_filename_json = meta_filename.merged_json, + read_counts_raw_json = write_json(count_raw), + read_counts_cleaned_json = write_json(count_cleaned) + } + } + } + + + if (length(biosample_map_tsvs) > 0) { #### merge biosample attribute tsvs (iff provided with more than one) - if (length(flatten(select_all([biosample_map_tsvs]))) > 1) { + if (length(biosample_map_tsvs) > 1) { call utils.tsv_join as biosample_map_tsv_join { input: - input_tsvs = flatten([select_first([biosample_map_tsvs,[]])]), + input_tsvs = biosample_map_tsvs, id_col = 'accession', out_suffix = ".tsv", out_basename = "biosample-attributes-merged" } } + File biosample_map_tsv = select_first(flatten([[biosample_map_tsv_join.out_tsv], biosample_map_tsvs])) #### biosample metadata mapping call ncbi.biosample_to_table { input: - biosample_attributes_tsv = select_first([biosample_map_tsv_join.out_tsv, biosample_map_tsvs]), + biosample_attributes_tsv = biosample_map_tsv, cleaned_bam_filepaths = select_all(cleaned_bam_passing), demux_meta_json = meta_filename.merged_json } @@ -207,7 +228,7 @@ workflow demux_deplete { call ncbi.sra_meta_prep { input: cleaned_bam_filepaths = select_all(cleaned_bam_passing), - biosample_map = select_first([biosample_map_tsv_join.out_tsv, biosample_map_tsvs]), + biosample_map = biosample_map_tsv, library_metadata = samplesheet_rename_ids.new_sheet, platform = "ILLUMINA", paired = (illumina_demux.run_info[0]['indexes'] == '2'), @@ -216,32 +237,13 @@ workflow demux_deplete { instrument_model = select_first(flatten([[instrument_model_user_specified],[illumina_demux.run_info[0]['sequencer_model']]])), title = select_first([sra_title]) } - } - if(insert_demux_outputs_into_terra_tables) { - call terra.check_terra_env - - if(check_terra_env.is_running_on_terra) { - call terra.create_or_update_sample_tables { - input: - flowcell_run_id = illumina_demux.run_info[0]['run_id'], - workspace_name = check_terra_env.workspace_name, - workspace_namespace = check_terra_env.workspace_namespace, - - raw_reads_unaligned_bams = flatten(illumina_demux.raw_reads_unaligned_bams), - cleaned_reads_unaligned_bams = select_all(cleaned_bam_passing), - meta_by_filename_json = meta_filename.merged_json, - read_counts_raw_json = write_json(count_raw), - read_counts_cleaned_json = write_json(count_cleaned) - } - - if (length(flatten(select_all([biosample_map_tsvs]))) > 0) { - call terra.upload_entities_tsv as terra_load_biosample_data { - input: - workspace_name = check_terra_env.workspace_name, - terra_project = check_terra_env.workspace_namespace, - tsv_file = select_first([biosample_to_table.sample_meta_tsv]) - } + if(insert_demux_outputs_into_terra_tables && select_first([check_terra_env.is_running_on_terra])) { + call terra.upload_entities_tsv as terra_load_biosample_data { + input: + workspace_name = select_first([check_terra_env.workspace_name]), + terra_project = select_first([check_terra_env.workspace_namespace]), + tsv_file = biosample_map_tsv } } } From 9017a8093277913ce42f72835ad4edf60ed70932 Mon Sep 17 00:00:00 2001 From: Danny Park Date: Mon, 25 Nov 2024 08:19:11 -0500 Subject: [PATCH 9/9] bugfix --- pipes/WDL/workflows/demux_deplete.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipes/WDL/workflows/demux_deplete.wdl b/pipes/WDL/workflows/demux_deplete.wdl index e6b1deb2f..1d3e775ab 100644 --- a/pipes/WDL/workflows/demux_deplete.wdl +++ b/pipes/WDL/workflows/demux_deplete.wdl @@ -243,7 +243,7 @@ workflow demux_deplete { input: workspace_name = select_first([check_terra_env.workspace_name]), terra_project = select_first([check_terra_env.workspace_namespace]), - tsv_file = biosample_map_tsv + tsv_file = biosample_to_table.sample_meta_tsv } } }