diff --git a/.github/workflows/phylogenetic-fauna.yaml b/.github/workflows/phylogenetic-fauna.yaml new file mode 100644 index 0000000..482de73 --- /dev/null +++ b/.github/workflows/phylogenetic-fauna.yaml @@ -0,0 +1,56 @@ +name: Phylogenetic Fauna + +defaults: + run: + # This is the same as GitHub Action's `bash` keyword as of 20 June 2023: + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsshell + # + # Completely spelling it out here so that GitHub can't change it out from under us + # and we don't have to refer to the docs to know the expected behavior. + shell: bash --noprofile --norc -eo pipefail {0} + +on: + workflow_dispatch: + inputs: + image: + description: 'Specific container image to use for ingest workflow (will override the default of "nextstrain build")' + required: false + type: string + trial-name: + description: | + Trial name for deploying builds. + If not set, builds will overwrite existing builds at s3://nextstrain-data/avian-flu* + If set, builds will be deployed to s3://nextstrain-staging/avian-flu_trials__* + required: false + type: string + +jobs: + phylogenetic: + permissions: + id-token: write + uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master + secrets: inherit + with: + runtime: aws-batch + run: | + declare -a config; + + if [[ "$TRIAL_NAME" ]]; then + config+=( + deploy_url="s3://nextstrain-staging/avian-flu_trials_${TRIAL_NAME}_" + ) + fi; + + nextstrain build \ + --detach \ + --no-download \ + --cpus 16 \ + --memory 28800mib \ + . \ + deploy_all \ + --config "${config[@]}" + + env: | + NEXTSTRAIN_DOCKER_IMAGE: ${{ inputs.image }} + TRIAL_NAME: ${{ inputs.trial-name }} + artifact-name: phylogenetic-fauna-build-output diff --git a/.github/workflows/phylogenetic-ncbi.yaml b/.github/workflows/phylogenetic-ncbi.yaml new file mode 100644 index 0000000..45e286b --- /dev/null +++ b/.github/workflows/phylogenetic-ncbi.yaml @@ -0,0 +1,57 @@ +name: Phylogenetic NCBI + +defaults: + run: + # This is the same as GitHub Action's `bash` keyword as of 20 June 2023: + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsshell + # + # Completely spelling it out here so that GitHub can't change it out from under us + # and we don't have to refer to the docs to know the expected behavior. + shell: bash --noprofile --norc -eo pipefail {0} + +on: + workflow_dispatch: + inputs: + image: + description: 'Specific container image to use for ingest workflow (will override the default of "nextstrain build")' + required: false + type: string + trial-name: + description: | + Trial name for deploying builds. + If not set, builds will overwrite existing builds at s3://nextstrain-data/avian-flu* + If set, builds will be deployed to s3://nextstrain-staging/avian-flu_trials__* + required: false + type: string + +jobs: + phylogenetic: + permissions: + id-token: write + uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master + secrets: inherit + with: + runtime: docker + run: | + declare -a config; + + config+=( + s3_src="s3://nextstrain-data/files/workflows/avian-flu/h5n1" + ); + + if [[ "$TRIAL_NAME" ]]; then + config+=( + deploy_url="s3://nextstrain-staging/avian-flu_trials_${TRIAL_NAME}_" + ) + fi; + + nextstrain build \ + . \ + deploy_all \ + --snakefile Snakefile.genome \ + --config "${config[@]}" + + env: | + NEXTSTRAIN_DOCKER_IMAGE: ${{ inputs.image }} + TRIAL_NAME: ${{ inputs.trial-name }} + artifact-name: phylogenetic-full-genome-build-output diff --git a/README.md b/README.md index 66f749a..3ba2d86 100755 --- a/README.md +++ b/README.md @@ -16,6 +16,15 @@ nextstrain build --aws-batch --aws-batch-cpus 16 --aws-batch-memory 28800 . --jo Please see [nextstrain.org/docs](https://nextstrain.org/docs) for details about augur and pathogen builds. +### Deploying builds + +The pipeline can automatically deploy resulting builds within the auspice folder +to nextstrain.org by running: + +``` +nextstrain build . deploy_all +``` + ## Creating a custom build The easiest way to generate your own, custom avian-flu build is to use the quickstart-build as a starting template. Simply clone the quickstart-build, run with the example data, and edit the Snakefile to customize. This build includes example data and a simplified, heavily annotated Snakefile that goes over the structure of Snakefiles and annotates rules and inputs/outputs that can be modified. This build, with it's own readme, is available [here](https://github.com/nextstrain/avian-flu/tree/master/quickstart-build). diff --git a/Snakefile b/Snakefile index 9f1a9a1..0b19545 100755 --- a/Snakefile +++ b/Snakefile @@ -22,6 +22,9 @@ rule all: input: auspice_json = all_targets() +# This must be after the `all` rule above since it depends on its inputs +include: "rules/deploy.smk" + rule test_target: """ For testing purposes such as CI workflows. diff --git a/Snakefile.genome b/Snakefile.genome index 27bd8be..a1a63e9 100644 --- a/Snakefile.genome +++ b/Snakefile.genome @@ -33,6 +33,9 @@ def subtype(build_name): rule all: input: expand("auspice/avian-flu_{build_name}_genome.json", build_name=BUILD_NAME) +# This must be after the `all` rule above since it depends on its inputs +include: "rules/deploy.smk" + rule files: params: reference = lambda w: f"config/reference_{subtype(w.build_name)}_{{segment}}.gb", diff --git a/rules/deploy.smk b/rules/deploy.smk new file mode 100644 index 0000000..101b8be --- /dev/null +++ b/rules/deploy.smk @@ -0,0 +1,15 @@ +DEPLOY_URL = config.get('deploy_url', "s3://nextstrain-data") + + +rule deploy_all: + """ + Upload all builds to AWS S3 + Depends on indendent Snakemake workflow's defined `all` rule + """ + input: rules.all.input + params: + s3_dst = DEPLOY_URL + shell: + """ + nextstrain remote upload {params.s3_dst:q} {input} + """