From 803ed164ef4f513fa4e183c2a02f4fcbb66caa8f Mon Sep 17 00:00:00 2001 From: Jennifer Chang Date: Wed, 11 Dec 2024 16:34:55 -0800 Subject: [PATCH 1/3] Ingest: Derive URL column during ingest --- ingest/defaults/config.yaml | 3 +++ ingest/rules/curate.smk | 23 ++++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/ingest/defaults/config.yaml b/ingest/defaults/config.yaml index 4a51e7e..919b698 100644 --- a/ingest/defaults/config.yaml +++ b/ingest/defaults/config.yaml @@ -94,6 +94,8 @@ curate: output_id_field: 'genbank_accession' # The field in the NDJSON record that contains the actual genomic sequence output_sequence_field: 'sequence' + # The field in the NDJSON record that contains the actual GenBank accession + genbank_accession: 'genbank_accession' # The list of metadata columns to keep in the final output of the curation pipeline. metadata_columns: [ 'genbank_accession', @@ -111,4 +113,5 @@ curate: 'sra_accessions', 'authors', 'institution', + 'url', ] diff --git a/ingest/rules/curate.smk b/ingest/rules/curate.smk index 2ec4ab9..d890038 100644 --- a/ingest/rules/curate.smk +++ b/ingest/rules/curate.smk @@ -102,9 +102,30 @@ rule curate: """ +rule add_metadata_columns: + """Add columns to metadata + Notable columns: + - [NEW] url: URL linking to the NCBI GenBank record ('https://www.ncbi.nlm.nih.gov/nuccore/*'). + """ + input: + metadata = "data/all_metadata.tsv" + output: + metadata = temp("data/all_metadata_added.tsv") + params: + accession=config['curate']['genbank_accession'] + shell: + """ + csvtk mutate2 -t \ + -n url \ + -e '"https://www.ncbi.nlm.nih.gov/nuccore/" + ${params.accession}' \ + {input.metadata} \ + > {output.metadata} + """ + + rule subset_metadata: input: - metadata="data/all_metadata.tsv", + metadata="data/all_metadata_added.tsv", output: subset_metadata="results/metadata.tsv", params: From 53545a56188cadeff442bb188722dddd33127016 Mon Sep 17 00:00:00 2001 From: Jennifer Chang Date: Wed, 11 Dec 2024 16:44:13 -0800 Subject: [PATCH 2/3] Replace genbank_accession with accession This simplifies USVI data merge --- ingest/defaults/config.yaml | 16 ++++++------- ingest/rules/fetch_from_ncbi.smk | 6 ++--- phylogenetic/rules/merge_sequences_usvi.smk | 26 +-------------------- 3 files changed, 12 insertions(+), 36 deletions(-) diff --git a/ingest/defaults/config.yaml b/ingest/defaults/config.yaml index 919b698..8cc4a43 100644 --- a/ingest/defaults/config.yaml +++ b/ingest/defaults/config.yaml @@ -45,8 +45,8 @@ curate: # The original field names should match the ncbi_datasets_fields provided above. # This is the first step in the pipeline, so any references to field names in the configs below should use the new field names field_map: - accession: genbank_accession - accession-rev: genbank_accession_rev + accession: accession + accession_version: accession_version isolate-lineage: strain sourcedb: database geo-region: region @@ -62,7 +62,7 @@ curate: # Currently accepts any characters because we do not have a clear standard for strain names across pathogens strain_regex: '^.+$' # Back up strain name field to use if 'strain' doesn't match regex above - strain_backup_fields: ['genbank_accession'] + strain_backup_fields: ['accession'] # List of date fields to standardize to ISO format YYYY-MM-DD date_fields: ['date', 'release_date', 'update_date'] # List of expected date formats that are present in the date fields provided above @@ -89,17 +89,17 @@ curate: # The path should be relative to the ingest directory annotations: "defaults/annotations.tsv" # The ID field in the metadata to use to merge the manual annotations - annotations_id: 'genbank_accession' + annotations_id: 'accession' # The ID field in the metadata to use as the sequence id in the output FASTA file - output_id_field: 'genbank_accession' + output_id_field: 'accession' # The field in the NDJSON record that contains the actual genomic sequence output_sequence_field: 'sequence' # The field in the NDJSON record that contains the actual GenBank accession - genbank_accession: 'genbank_accession' + genbank_accession: 'accession' # The list of metadata columns to keep in the final output of the curation pipeline. metadata_columns: [ - 'genbank_accession', - 'genbank_accession_rev', + 'accession', + 'accession_version', 'strain', 'date', 'region', diff --git a/ingest/rules/fetch_from_ncbi.smk b/ingest/rules/fetch_from_ncbi.smk index 3c32e42..ca6cedb 100644 --- a/ingest/rules/fetch_from_ncbi.smk +++ b/ingest/rules/fetch_from_ncbi.smk @@ -66,8 +66,8 @@ rule format_ncbi_dataset_report: --elide-header \ | csvtk fix-quotes -Ht \ | csvtk add-header -t -n {params.ncbi_datasets_fields:q} \ - | csvtk rename -t -f accession -n accession-rev \ - | csvtk -t mutate -f accession-rev -n accession -p "^(.+?)\." \ + | csvtk rename -t -f accession -n accession_version \ + | csvtk -t mutate -f accession_version -n accession -p "^(.+?)\." \ | csvtk del-quotes -t \ | tsv-select -H -f accession --rest last \ > {output.ncbi_dataset_tsv} @@ -89,7 +89,7 @@ rule format_ncbi_datasets_ndjson: augur curate passthru \ --metadata {input.ncbi_dataset_tsv} \ --fasta {input.ncbi_dataset_sequences} \ - --seq-id-column accession-rev \ + --seq-id-column accession_version \ --seq-field sequence \ --unmatched-reporting warn \ --duplicate-reporting warn \ diff --git a/phylogenetic/rules/merge_sequences_usvi.smk b/phylogenetic/rules/merge_sequences_usvi.smk index ffc7a50..3765937 100644 --- a/phylogenetic/rules/merge_sequences_usvi.smk +++ b/phylogenetic/rules/merge_sequences_usvi.smk @@ -21,35 +21,11 @@ This part of the workflow usually includes the following steps: """ -rule add_metadata_columns: - """Add columns to metadata - - Notable columns: - - genbank_accession: GenBank accession for Auspice to generate a URL to the NCBI GenBank record. - - [NEW] accession: The GenBank accession. Added to go alongside USVI accession. - - [NEW] url: URL linking to the NCBI GenBank record ('https://www.ncbi.nlm.nih.gov/nuccore/*'). Added to go alongside USVI url. - """ - input: - metadata = "data/metadata.tsv" - output: - metadata = "data/metadata_modified.tsv" - shell: - """ - csvtk mutate2 -tl \ - -n url \ - -e '"https://www.ncbi.nlm.nih.gov/nuccore/" + $genbank_accession' \ - {input.metadata} \ - | csvtk mutate2 -tl \ - -n accession \ - -e '$genbank_accession' \ - > {output.metadata} - """ - rule append_usvi: """Appending USVI sequences""" input: sequences = "data/sequences.fasta", - metadata = "data/metadata_modified.tsv", + metadata = "data/metadata.tsv", usvi_sequences = "data/sequences_usvi.fasta", usvi_metadata = "data/metadata_usvi.tsv" output: From 77fa1a495052a78983e2c415d3789eddd813234f Mon Sep 17 00:00:00 2001 From: Jennifer Chang Date: Wed, 11 Dec 2024 17:26:52 -0800 Subject: [PATCH 3/3] Update example data --- phylogenetic/example_data/metadata.tsv | 70 ++++++++++----------- phylogenetic/example_data/metadata_usvi.tsv | 4 +- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/phylogenetic/example_data/metadata.tsv b/phylogenetic/example_data/metadata.tsv index 3d39cf9..928289c 100644 --- a/phylogenetic/example_data/metadata.tsv +++ b/phylogenetic/example_data/metadata.tsv @@ -1,35 +1,35 @@ -strain virus genbank_accession date region country division city db segment authors -PAN/CDC_259359_V1_V3/2015 zika KX156774 2015-12-18 North America Panama Panama Panama genbank genome Shabman et al -COL/FLR_00024/2015 zika MF574569 2015-12-XX South America Colombia Colombia Colombia genbank genome Pickett et al -PRVABC59 zika KU501215 2015-12-XX North America Puerto Rico Puerto Rico Puerto Rico genbank genome Lanciotti et al -COL/FLR_00008/2015 zika MF574562 2015-12-XX South America Colombia Colombia Colombia genbank genome Pickett et al -Colombia/2016/ZC204Se zika KY317939 2016-01-06 South America Colombia Colombia Colombia genbank genome Quick et al -ZKC2/2016 zika KX253996 2016-02-16 Oceania American Samoa American Samoa American Samoa genbank genome Wu et al -VEN/UF_1/2016 zika KX702400 2016-03-25 South America Venezuela Venezuela Venezuela genbank genome Blohm et al -DOM/2016/BB_0059 zika KY785425 2016-04-04 North America Dominican Republic Dominican Republic Dominican Republic genbank genome Metsky et al -BRA/2016/FC_6706 zika KY785433 2016-04-08 South America Brazil Brazil Brazil genbank genome Metsky et al -DOM/2016/BB_0183 zika KY785420 2016-04-18 North America Dominican Republic Dominican Republic Dominican Republic genbank genome Metsky et al -EcEs062_16 zika KX879603 2016-04-XX South America Ecuador Ecuador Ecuador genbank genome Marquez et al -HND/2016/HU_ME59 zika KY785418 2016-05-13 North America Honduras Honduras Honduras genbank genome Metsky et al -DOM/2016/MA_WGS16_011 zika KY785484 2016-06-06 North America Dominican Republic Dominican Republic Dominican Republic genbank genome Metsky et al -DOM/2016/BB_0433 zika KY785441 2016-06-13 North America Dominican Republic Dominican Republic Dominican Republic genbank genome Metsky et al -USA/2016/FL022 zika KY075935 2016-07-22 North America Usa Usa Usa genbank genome Grubaugh et al -SG_027 zika KY241697 2016-08-27 Southeast Asia Singapore Singapore Singapore genbank genome Ho et al -SG_074 zika KY241744 2016-08-28 Southeast Asia Singapore Singapore Singapore genbank genome Ho et al -SG_056 zika KY241726 2016-08-28 Southeast Asia Singapore Singapore Singapore genbank genome Ho et al -USA/2016/FLUR022 zika KY325473 2016-08-31 North America Usa Usa Usa genbank genome Grubaugh et al -Aedes_aegypti/USA/2016/FL05 zika KY075937 2016-09-09 North America Usa Usa Usa genbank genome Grubaugh et al -SG_018 zika KY241688 2016-09-13 Southeast Asia Singapore Singapore Singapore genbank genome Ho et al -USA/2016/FLWB042 zika KY325478 2016-09-26 North America Usa Usa Usa genbank genome Grubaugh et al -COL/PRV_00028/2015 zika MF574578 2016-12-XX South America Colombia Colombia Colombia genbank genome Pickett et al -Thailand/1610acTw zika MF692778 2016-10-XX Southeast Asia Thailand Thailand Thailand genbank genome Lin et al -1_0087_PF zika KX447509 2013-12-XX Oceania French Polynesia French Polynesia French Polynesia genbank genome Pettersson et al -1_0199_PF zika KX447519 2013-11-XX Oceania French Polynesia French Polynesia French Polynesia genbank genome Pettersson et al -1_0181_PF zika KX447512 2013-12-XX Oceania French Polynesia French Polynesia French Polynesia genbank genome Pettersson et al -Brazil/2015/ZBRC301 zika KY558995 2015-05-13 South America Brazil Brazil Brazil genbank genome Faria et al -Brazil/2015/ZBRA105 zika KY558989 2015-02-23 South America Brazil Brazil Brazil genbank genome Faria et al -Brazil/2016/ZBRC16 zika KY558991 2016-01-19 South America Brazil Brazil Brazil genbank genome Faria et al -V8375 zika KU501217 2015-11-01 North America Guatemala Guatemala Guatemala genbank genome Lanciotti et al -Nica1_16 zika KX421195 2016-01-19 North America Nicaragua Nicaragua Nicaragua genbank genome Tabata et al -Brazil/2015/ZBRC303 zika KY558997 2015-05-14 South America Brazil Brazil Brazil genbank genome Faria et al -SMGC_1 zika KX266255 2016-02-14 Oceania American Samoa American Samoa American Samoa genbank genome Bi et al +strain virus accession date region country division city db segment authors url +PAN/CDC_259359_V1_V3/2015 zika KX156774 2015-12-18 North America Panama Panama Panama genbank genome Shabman et al https://www.ncbi.nlm.nih.gov/nuccore/KX156774 +COL/FLR_00024/2015 zika MF574569 2015-12-XX South America Colombia Colombia Colombia genbank genome Pickett et al https://www.ncbi.nlm.nih.gov/nuccore/MF574569 +PRVABC59 zika KU501215 2015-12-XX North America Puerto Rico Puerto Rico Puerto Rico genbank genome Lanciotti et al https://www.ncbi.nlm.nih.gov/nuccore/KU501215 +COL/FLR_00008/2015 zika MF574562 2015-12-XX South America Colombia Colombia Colombia genbank genome Pickett et al https://www.ncbi.nlm.nih.gov/nuccore/MF574562 +Colombia/2016/ZC204Se zika KY317939 2016-01-06 South America Colombia Colombia Colombia genbank genome Quick et al https://www.ncbi.nlm.nih.gov/nuccore/KY317939 +ZKC2/2016 zika KX253996 2016-02-16 Oceania American Samoa American Samoa American Samoa genbank genome Wu et al https://www.ncbi.nlm.nih.gov/nuccore/KX253996 +VEN/UF_1/2016 zika KX702400 2016-03-25 South America Venezuela Venezuela Venezuela genbank genome Blohm et al https://www.ncbi.nlm.nih.gov/nuccore/KX702400 +DOM/2016/BB_0059 zika KY785425 2016-04-04 North America Dominican Republic Dominican Republic Dominican Republic genbank genome Metsky et al https://www.ncbi.nlm.nih.gov/nuccore/KY785425 +BRA/2016/FC_6706 zika KY785433 2016-04-08 South America Brazil Brazil Brazil genbank genome Metsky et al https://www.ncbi.nlm.nih.gov/nuccore/KY785433 +DOM/2016/BB_0183 zika KY785420 2016-04-18 North America Dominican Republic Dominican Republic Dominican Republic genbank genome Metsky et al https://www.ncbi.nlm.nih.gov/nuccore/KY785420 +EcEs062_16 zika KX879603 2016-04-XX South America Ecuador Ecuador Ecuador genbank genome Marquez et al https://www.ncbi.nlm.nih.gov/nuccore/KX879603 +HND/2016/HU_ME59 zika KY785418 2016-05-13 North America Honduras Honduras Honduras genbank genome Metsky et al https://www.ncbi.nlm.nih.gov/nuccore/KY785418 +DOM/2016/MA_WGS16_011 zika KY785484 2016-06-06 North America Dominican Republic Dominican Republic Dominican Republic genbank genome Metsky et al https://www.ncbi.nlm.nih.gov/nuccore/KY785484 +DOM/2016/BB_0433 zika KY785441 2016-06-13 North America Dominican Republic Dominican Republic Dominican Republic genbank genome Metsky et al https://www.ncbi.nlm.nih.gov/nuccore/KY785441 +USA/2016/FL022 zika KY075935 2016-07-22 North America Usa Usa Usa genbank genome Grubaugh et al https://www.ncbi.nlm.nih.gov/nuccore/KY075935 +SG_027 zika KY241697 2016-08-27 Southeast Asia Singapore Singapore Singapore genbank genome Ho et al https://www.ncbi.nlm.nih.gov/nuccore/KY241697 +SG_074 zika KY241744 2016-08-28 Southeast Asia Singapore Singapore Singapore genbank genome Ho et al https://www.ncbi.nlm.nih.gov/nuccore/KY241744 +SG_056 zika KY241726 2016-08-28 Southeast Asia Singapore Singapore Singapore genbank genome Ho et al https://www.ncbi.nlm.nih.gov/nuccore/KY241726 +USA/2016/FLUR022 zika KY325473 2016-08-31 North America Usa Usa Usa genbank genome Grubaugh et al https://www.ncbi.nlm.nih.gov/nuccore/KY325473 +Aedes_aegypti/USA/2016/FL05 zika KY075937 2016-09-09 North America Usa Usa Usa genbank genome Grubaugh et al https://www.ncbi.nlm.nih.gov/nuccore/KY075937 +SG_018 zika KY241688 2016-09-13 Southeast Asia Singapore Singapore Singapore genbank genome Ho et al https://www.ncbi.nlm.nih.gov/nuccore/KY241688 +USA/2016/FLWB042 zika KY325478 2016-09-26 North America Usa Usa Usa genbank genome Grubaugh et al https://www.ncbi.nlm.nih.gov/nuccore/KY325478 +COL/PRV_00028/2015 zika MF574578 2016-12-XX South America Colombia Colombia Colombia genbank genome Pickett et al https://www.ncbi.nlm.nih.gov/nuccore/MF574578 +Thailand/1610acTw zika MF692778 2016-10-XX Southeast Asia Thailand Thailand Thailand genbank genome Lin et al https://www.ncbi.nlm.nih.gov/nuccore/MF692778 +1_0087_PF zika KX447509 2013-12-XX Oceania French Polynesia French Polynesia French Polynesia genbank genome Pettersson et al https://www.ncbi.nlm.nih.gov/nuccore/KX447509 +1_0199_PF zika KX447519 2013-11-XX Oceania French Polynesia French Polynesia French Polynesia genbank genome Pettersson et al https://www.ncbi.nlm.nih.gov/nuccore/KX447519 +1_0181_PF zika KX447512 2013-12-XX Oceania French Polynesia French Polynesia French Polynesia genbank genome Pettersson et al https://www.ncbi.nlm.nih.gov/nuccore/KX447512 +Brazil/2015/ZBRC301 zika KY558995 2015-05-13 South America Brazil Brazil Brazil genbank genome Faria et al https://www.ncbi.nlm.nih.gov/nuccore/KY558995 +Brazil/2015/ZBRA105 zika KY558989 2015-02-23 South America Brazil Brazil Brazil genbank genome Faria et al https://www.ncbi.nlm.nih.gov/nuccore/KY558989 +Brazil/2016/ZBRC16 zika KY558991 2016-01-19 South America Brazil Brazil Brazil genbank genome Faria et al https://www.ncbi.nlm.nih.gov/nuccore/KY558991 +V8375 zika KU501217 2015-11-01 North America Guatemala Guatemala Guatemala genbank genome Lanciotti et al https://www.ncbi.nlm.nih.gov/nuccore/KU501217 +Nica1_16 zika KX421195 2016-01-19 North America Nicaragua Nicaragua Nicaragua genbank genome Tabata et al https://www.ncbi.nlm.nih.gov/nuccore/KX421195 +Brazil/2015/ZBRC303 zika KY558997 2015-05-14 South America Brazil Brazil Brazil genbank genome Faria et al https://www.ncbi.nlm.nih.gov/nuccore/KY558997 +SMGC_1 zika KX266255 2016-02-14 Oceania American Samoa American Samoa American Samoa genbank genome Bi et al https://www.ncbi.nlm.nih.gov/nuccore/KX266255 diff --git a/phylogenetic/example_data/metadata_usvi.tsv b/phylogenetic/example_data/metadata_usvi.tsv index 96d3d52..42a3a08 100644 --- a/phylogenetic/example_data/metadata_usvi.tsv +++ b/phylogenetic/example_data/metadata_usvi.tsv @@ -1,2 +1,2 @@ -genbank_accession genbank_accession_rev accession strain date region country division location length host release_date update_date sra_accessions authors institution url -USVI/37/2016 VI37 USVI/37/2016 2016-10-06 North America Usvi Saint Croix Saint Croix 10807 Homo sapiens Black et al FH https://github.com/blab/zika-usvi/ +accession accession_version strain date region country division location length host release_date update_date sra_accessions authors institution url +VI37 VI37 USVI/37/2016 2016-10-06 North America Usvi Saint Croix Saint Croix 10807 Homo sapiens Black et al FH https://github.com/blab/zika-usvi/ \ No newline at end of file