Skip to content

Commit

Permalink
ingest-fauna/ingest-ncbi: Add inputs for trial_name and image
Browse files Browse the repository at this point in the history
Motivated by my own need to test the ingest workflows for the latest
addition of Nextclade outputs in previous commits.
  • Loading branch information
joverlee521 committed Jun 24, 2024
1 parent 245f152 commit 58df478
Show file tree
Hide file tree
Showing 2 changed files with 61 additions and 2 deletions.
28 changes: 27 additions & 1 deletion .github/workflows/ingest-fauna.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,18 @@ defaults:

on:
workflow_dispatch:
inputs:
image:
description: 'Specific container image to use for ingest workflow (will override the default of "nextstrain build")'
required: false
type: string
trial-name:
description: |
Trial name for outputs.
If not set, outputs will overwrite files at s3://nextstrain-data-private/files/workflows/avian-flu/
If set, outputs will be uploaded to s3://nextstrain-data-private/files/workflows/avian-flu/trials/<trial_name>/
required: false
type: string

jobs:
ingest:
Expand All @@ -24,11 +36,25 @@ jobs:
# the job runs longer than the GH Action limit of 6 hours.
runtime: docker
run: |
declare -a config;
if [[ "$TRIAL_NAME" ]]; then
# Create JSON string for the nested upload config
S3_DST="s3://nextstrain-data-private/files/workflows/avian-flu/trial/$TRIAL_NAME"
config+=(
s3_dst=$(jq -cn --arg S3_DST "$S3_DST" '{"fauna": $S3_DST}')
)
fi;
nextstrain build \
--env RETHINK_HOST \
--env RETHINK_AUTH_KEY \
ingest \
upload_all
upload_all \
--config "${config[@]}"
env: |
NEXTSTRAIN_DOCKER_IMAGE: ${{ inputs.image }}
TRIAL_NAME: ${{ inputs.trial-name }}
# Specifying artifact name to differentiate ingest build outputs from
# the phylogenetic build outputs
artifact-name: ingest-fauna-build-output
Expand Down
35 changes: 34 additions & 1 deletion .github/workflows/ingest-ncbi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,18 @@ on:
- cron: '0 17 * * *'

workflow_dispatch:
inputs:
image:
description: 'Specific container image to use for ingest workflow (will override the default of "nextstrain build")'
required: false
type: string
trial-name:
description: |
Trial name for outputs.
If not set, outputs will overwrite files at s3://nextstrain-data/files/workflows/avian-flu/
If set, outputs will be uploaded to s3://nextstrain-data/files/workflows/avian-flu/trials/<trial_name>/
required: false
type: string

jobs:
ingest:
Expand All @@ -45,10 +57,31 @@ jobs:
# the job runs longer than the GH Action limit of 6 hours.
runtime: docker
run: |
declare -a config;
if [[ "$TRIAL_NAME" ]]; then
# Create JSON string for the nested upload config
S3_DST_BASE="s3://nextstrain-data/files/workflows/avian-flu/trial/$TRIAL_NAME"
config+=(
s3_dst=$(
jq -cn --arg S3_DST_BASE "$S3_DST_BASE" '{
"joined-ncbi": "\($S3_DST_BASE)/h5n1",
"ncbi": "\($S3_DST_BASE)/h5n1/ncbi",
"andersen-lab": "\($S3_DST_BASE)/h5n1/andersen-lab"
}'
)
)
fi;
nextstrain build \
ingest \
upload_all_ncbi \
--configfile build-configs/ncbi/defaults/config.yaml
--configfile build-configs/ncbi/defaults/config.yaml \
--config "${config[@]}"
env: |
NEXTSTRAIN_DOCKER_IMAGE: ${{ inputs.image }}
TRIAL_NAME: ${{ inputs.trial-name }}
# Specifying artifact name to differentiate ingest build outputs from
# the phylogenetic build outputs
artifact-name: ingest-ncbi-build-output
Expand Down

0 comments on commit 58df478

Please sign in to comment.