Merge pull request #63 from nextstrain/gh-action-ingest-trial

ingest-fauna/ingest-ncbi: Add inputs for trial_name and image
nextstrain · Jun 24, 2024 · 8d65962 · 8d65962
2 parents f864158 + 377759d
commit 8d65962
Show file tree

Hide file tree

Showing 2 changed files with 61 additions and 2 deletions.
diff --git a/.github/workflows/ingest-fauna.yaml b/.github/workflows/ingest-fauna.yaml
@@ -11,6 +11,18 @@ defaults:
 
 on:
   workflow_dispatch:
+    inputs:
+      image:
+        description: 'Specific container image to use for ingest workflow (will override the default of "nextstrain build")'
+        required: false
+        type: string
+      trial-name:
+        description: |
+          Trial name for outputs.
+          If not set, outputs will overwrite files at s3://nextstrain-data-private/files/workflows/avian-flu/
+          If set, outputs will be uploaded to s3://nextstrain-data-private/files/workflows/avian-flu/trials/<trial_name>/
+        required: false
+        type: string
 
 jobs:
   ingest:
@@ -24,11 +36,25 @@ jobs:
       # the job runs longer than the GH Action limit of 6 hours.
       runtime: docker
       run: |
+        declare -a config;
+
+        if [[ "$TRIAL_NAME" ]]; then
+          # Create JSON string for the nested upload config
+          S3_DST="s3://nextstrain-data-private/files/workflows/avian-flu/trial/$TRIAL_NAME"
+          config+=(
+            s3_dst=$(jq -cn --arg S3_DST "$S3_DST" '{"fauna": $S3_DST}')
+          )
+        fi;
+
         nextstrain build \
           --env RETHINK_HOST \
           --env RETHINK_AUTH_KEY \
           ingest \
-            upload_all
+            upload_all \
+            --config "${config[@]}"
+      env: |
+        NEXTSTRAIN_DOCKER_IMAGE: ${{ inputs.image }}
+        TRIAL_NAME: ${{ inputs.trial-name }}
       # Specifying artifact name to differentiate ingest build outputs from
       # the phylogenetic build outputs
       artifact-name: ingest-fauna-build-output

diff --git a/.github/workflows/ingest-ncbi.yaml b/.github/workflows/ingest-ncbi.yaml
@@ -32,6 +32,18 @@ on:
     - cron: '0 17 * * *'
 
   workflow_dispatch:
+    inputs:
+      image:
+        description: 'Specific container image to use for ingest workflow (will override the default of "nextstrain build")'
+        required: false
+        type: string
+      trial-name:
+        description: |
+          Trial name for outputs.
+          If not set, outputs will overwrite files at s3://nextstrain-data/files/workflows/avian-flu/
+          If set, outputs will be uploaded to s3://nextstrain-data/files/workflows/avian-flu/trials/<trial_name>/
+        required: false
+        type: string
 
 jobs:
   ingest:
@@ -45,10 +57,31 @@ jobs:
       # the job runs longer than the GH Action limit of 6 hours.
       runtime: docker
       run: |
+        declare -a config;
+
+        if [[ "$TRIAL_NAME" ]]; then
+          # Create JSON string for the nested upload config
+          S3_DST_BASE="s3://nextstrain-data/files/workflows/avian-flu/trial/$TRIAL_NAME"
+
+          config+=(
+            s3_dst=$(
+              jq -cn --arg S3_DST_BASE "$S3_DST_BASE" '{
+                "joined-ncbi": "\($S3_DST_BASE)/h5n1",
+                "ncbi": "\($S3_DST_BASE)/h5n1/ncbi",
+                "andersen-lab": "\($S3_DST_BASE)/h5n1/andersen-lab"
+              }'
+            )
+          )
+        fi;
+
         nextstrain build \
           ingest \
             upload_all_ncbi \
-            --configfile build-configs/ncbi/defaults/config.yaml
+            --configfile build-configs/ncbi/defaults/config.yaml \
+            --config "${config[@]}"
+      env: |
+        NEXTSTRAIN_DOCKER_IMAGE: ${{ inputs.image }}
+        TRIAL_NAME: ${{ inputs.trial-name }}
       # Specifying artifact name to differentiate ingest build outputs from
       # the phylogenetic build outputs
       artifact-name: ingest-ncbi-build-output