Merge branch 'aws-role-policies'

nextstrain · May 22, 2024 · d1f65a9 · d1f65a9
2 parents f996c9b + 3a2d597
commit d1f65a9
Show file tree

Hide file tree

Showing 4 changed files with 139 additions and 17 deletions.
diff --git a/.github/workflows/pathogen-repo-build.yaml b/.github/workflows/pathogen-repo-build.yaml
@@ -47,12 +47,17 @@ on:
 
           If your build runs longer than the 6 hour limit for a single GitHub Action job, then use the aws-batch runtime and the `--detach` flag. Subsequent chained jobs will be automatically used to wait on the remote build for up to 24 hours total.
 
-          All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option.  If AWS credentials were acquired by the GitHub Action job via role assumption, the following environment variables are also available to be passed:
+          All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option.
 
+          It is assumed that the pathogen repo build requires AWS credentials for read/write access to S3 buckets. These may come directly from secrets or indirectly from assuming a role via GitHub Actions' OIDC provider.
 
-            - AWS_ACCESS_KEY_ID
-            - AWS_SECRET_ACCESS_KEY
-            - AWS_SESSION_TOKEN
+          The following secrets are used if present:
+
+          - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY
+
+          They must be defined in the repo's Actions secrets and passed to this workflow with `secrets: inherit`.
+
+          If no secrets are present, the GitHubActionsRoleNextstrainRepo@<repo> role is assumed (in both senses of the verb).  The <repo> here is always the _calling workflow's_ repository name (without owner), regardless of the "repo" input.  The repository must already be configured in nextstrain/infra for the role to exist; see documentation there for how to add the repository if necessary.
         type: string
         default: nextstrain build .
         required: false
@@ -138,12 +143,17 @@ on:
 
           If your build runs longer than the 6 hour limit for a single GitHub Action job, then use the aws-batch runtime and the `--detach` flag. Subsequent chained jobs will be automatically used to wait on the remote build for up to 24 hours total.
 
-          All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option.  If AWS credentials were acquired by the GitHub Action job via role assumption, the following environment variables are also available to be passed:
+          All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option.
+
+          It is assumed that the pathogen repo build requires AWS credentials for read/write access to S3 buckets. These may come directly from secrets or indirectly from assuming a role via GitHub Actions' OIDC provider.
+
+          The following secrets are used if present:
+
+          - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY
 
+          They must be defined in the repo's Actions secrets and passed to this workflow with `secrets: inherit`.
 
-            - AWS_ACCESS_KEY_ID
-            - AWS_SECRET_ACCESS_KEY
-            - AWS_SESSION_TOKEN
+          If no secrets are present, the GitHubActionsRoleNextstrainRepo@<repo> role is assumed (in both senses of the verb).  The <repo> here is always the _calling workflow's_ repository name (without owner), regardless of the "repo" input.  The repository must already be configured in nextstrain/infra for the role to exist; see documentation there for how to add the repository if necessary.
         type: string
         default: nextstrain build .
         required: false
@@ -210,6 +220,7 @@ on:
 env:
   NEXTSTRAIN_GITHUB_DIR: .git/nextstrain/.github
   NEXTSTRAIN_BUILD_LOG: build.log
+  NEXTSTRAIN_RUNTIME_ENVDIR: .git/nextstrain/env.d
 permissions:
   id-token: write
 jobs:
@@ -255,14 +266,45 @@ jobs:
 
           echo "$secrets" | jq 'del(.github_token)' | "$NEXTSTRAIN_GITHUB_DIR"/bin/json-to-envvars | tee -a "$GITHUB_ENV"
 
+      - id: role
+        name: Set repo-specific role to (potentially) assume for runtime access to AWS
+        env:
+          REPO_FULL_NAME: ${{ github.repository }}
+        run: |
+          echo "arn=arn:aws:iam::827581582529:role/GitHubActionsRoleNextstrainRepo@${REPO_FULL_NAME#*/}" | tee -a "$GITHUB_OUTPUT"
+      - name: Configure credentials for runtime access to AWS
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: us-east-1
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && steps.role.outputs.arn || '' }}
+          role-duration-seconds: 43200 # seconds, or 12 hours
+      - name: Report identity of runtime access to AWS
+        run: aws sts get-caller-identity
+      - name: Save runtime AWS credentials to ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }}
+        run: |
+          "$NEXTSTRAIN_GITHUB_DIR"/bin/write-envdir "$NEXTSTRAIN_RUNTIME_ENVDIR" \
+            AWS_ACCESS_KEY_ID \
+            AWS_SECRET_ACCESS_KEY \
+            AWS_SESSION_TOKEN
+        # This will overwrite the runtime AWS credential envvars configured above
+        # so if the build is using the aws-batch runtime, the Nextstrain CLI will
+        # have access to the AWS Batch session credentials
+        # Comment only applies to this first use of the `&setup-aws-batch-credentials`, so
+        # outdenting comments to not repeat it with expanded YAML
       - if: inputs.runtime == 'aws-batch'
+        name: Configure credentials for GitHub Actions job access to AWS Batch
         uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-region: us-east-1
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && 'arn:aws:iam::827581582529:role/GitHubActionsRoleNextstrainBatchJobs' || '' }}
           role-duration-seconds: 43200 # seconds, or 12 hours
+      - if: inputs.runtime == 'aws-batch'
+        name: Report identity of GitHub Actions job access to AWS Batch
+        run: aws sts get-caller-identity
       - name: Setup runtime ${{ inputs.runtime }}
         uses: ./.git/nextstrain/.github/actions/setup-nextstrain-cli
         with:
@@ -271,6 +313,7 @@ jobs:
       - name: Run build via ${{ inputs.runtime }}
         env:
           NEXTSTRAIN_BUILD_COMMAND: ${{ inputs.run }}
+          NEXTSTRAIN_RUNTIME_ENVDIRS: ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }}
         run: |
           # shellcheck disable=SC2154
           set -x
@@ -331,6 +374,7 @@ jobs:
           ref: ${{ needs.workflow-context.outputs.sha }}
           path: ${{ env.NEXTSTRAIN_GITHUB_DIR }}
       - if: inputs.runtime == 'aws-batch'
+        name: Configure credentials for GitHub Actions job access to AWS Batch
         uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-region: us-east-1
@@ -388,6 +432,7 @@ jobs:
           ref: ${{ needs.workflow-context.outputs.sha }}
           path: ${{ env.NEXTSTRAIN_GITHUB_DIR }}
       - if: inputs.runtime == 'aws-batch'
+        name: Configure credentials for GitHub Actions job access to AWS Batch
         uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-region: us-east-1
@@ -446,6 +491,7 @@ jobs:
           ref: ${{ needs.workflow-context.outputs.sha }}
           path: ${{ env.NEXTSTRAIN_GITHUB_DIR }}
       - if: inputs.runtime == 'aws-batch'
+        name: Configure credentials for GitHub Actions job access to AWS Batch
         uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-region: us-east-1
@@ -504,6 +550,7 @@ jobs:
           ref: ${{ needs.workflow-context.outputs.sha }}
           path: ${{ env.NEXTSTRAIN_GITHUB_DIR }}
       - if: inputs.runtime == 'aws-batch'
+        name: Configure credentials for GitHub Actions job access to AWS Batch
         uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-region: us-east-1
@@ -585,6 +632,7 @@ jobs:
           ref: ${{ needs.workflow-context.outputs.sha }}
           path: ${{ env.NEXTSTRAIN_GITHUB_DIR }}
       - if: inputs.runtime == 'aws-batch'
+        name: Configure credentials for GitHub Actions job access to AWS Batch
         uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-region: us-east-1

diff --git a/.github/workflows/pathogen-repo-build.yaml.in b/.github/workflows/pathogen-repo-build.yaml.in
@@ -66,13 +66,26 @@ on:
 
           All environment variables provided via the env input and all secrets
           provided via `secrets: inherit` can be passed to the build runtime
-          via the `--env` option.  If AWS credentials were acquired by the
-          GitHub Action job via role assumption, the following environment
-          variables are also available to be passed:
+          via the `--env` option.
 
-            - AWS_ACCESS_KEY_ID
-            - AWS_SECRET_ACCESS_KEY
-            - AWS_SESSION_TOKEN
+          It is assumed that the pathogen repo build requires AWS credentials for
+          read/write access to S3 buckets. These may come directly from secrets
+          or indirectly from assuming a role via GitHub Actions' OIDC provider.
+
+          The following secrets are used if present:
+
+          - AWS_ACCESS_KEY_ID
+          - AWS_SECRET_ACCESS_KEY
+
+          They must be defined in the repo's Actions secrets and passed to this
+          workflow with `secrets: inherit`.
+
+          If no secrets are present, the GitHubActionsRoleNextstrainRepo@<repo>
+          role is assumed (in both senses of the verb).  The <repo> here is
+          always the _calling workflow's_ repository name (without owner),
+          regardless of the "repo" input.  The repository must already be
+          configured in nextstrain/infra for the role to exist; see
+          documentation there for how to add the repository if necessary.
         type: string
         default: nextstrain build .
         required: false
@@ -162,6 +175,7 @@ on:
 env:
   NEXTSTRAIN_GITHUB_DIR: .git/nextstrain/.github
   NEXTSTRAIN_BUILD_LOG: build.log
+  NEXTSTRAIN_RUNTIME_ENVDIR: .git/nextstrain/env.d
 
 permissions:
   id-token: write
@@ -218,8 +232,40 @@ jobs:
           | "$NEXTSTRAIN_GITHUB_DIR"/bin/json-to-envvars
           | tee -a "$GITHUB_ENV"
 
-      - &setup-aws-credentials
+      - id: role
+        name: Set repo-specific role to (potentially) assume for runtime access to AWS
+        env:
+          REPO_FULL_NAME: ${{ github.repository }}
+        run: |
+          echo "arn=arn:aws:iam::827581582529:role/GitHubActionsRoleNextstrainRepo@${REPO_FULL_NAME#*/}" | tee -a "$GITHUB_OUTPUT"
+
+      - name: Configure credentials for runtime access to AWS
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: us-east-1
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && steps.role.outputs.arn || '' }}
+          role-duration-seconds: 43200 # seconds, or 12 hours
+
+      - name: Report identity of runtime access to AWS
+        run: aws sts get-caller-identity
+
+      - name: Save runtime AWS credentials to ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }}
+        run: |
+          "$NEXTSTRAIN_GITHUB_DIR"/bin/write-envdir "$NEXTSTRAIN_RUNTIME_ENVDIR" \
+            AWS_ACCESS_KEY_ID \
+            AWS_SECRET_ACCESS_KEY \
+            AWS_SESSION_TOKEN
+
+    # This will overwrite the runtime AWS credential envvars configured above
+    # so if the build is using the aws-batch runtime, the Nextstrain CLI will
+    # have access to the AWS Batch session credentials
+    # Comment only applies to this first use of the `&setup-aws-batch-credentials`, so
+    # outdenting comments to not repeat it with expanded YAML
+      - &setup-aws-batch-credentials
         if: inputs.runtime == 'aws-batch'
+        name: Configure credentials for GitHub Actions job access to AWS Batch
         uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-region: us-east-1
@@ -228,6 +274,10 @@ jobs:
           role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && 'arn:aws:iam::827581582529:role/GitHubActionsRoleNextstrainBatchJobs' || '' }}
           role-duration-seconds: 43200 # seconds, or 12 hours
 
+      - if: inputs.runtime == 'aws-batch'
+        name: Report identity of GitHub Actions job access to AWS Batch
+        run: aws sts get-caller-identity
+
       - &setup-runtime
         name: Setup runtime ${{ inputs.runtime }}
         uses: ./.git/nextstrain/.github/actions/setup-nextstrain-cli
@@ -238,6 +288,7 @@ jobs:
       - name: Run build via ${{ inputs.runtime }}
         env:
           NEXTSTRAIN_BUILD_COMMAND: ${{ inputs.run }}
+          NEXTSTRAIN_RUNTIME_ENVDIRS: ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }}
         run: |
           # shellcheck disable=SC2154
           set -x
@@ -294,7 +345,7 @@ jobs:
     steps:
       # Uses needs.workflow-context.outputs
       - *checkout-workflow-support
-      - *setup-aws-credentials
+      - *setup-aws-batch-credentials
       - *setup-runtime
 
       - id: attach
@@ -379,7 +430,7 @@ jobs:
     steps:
       # Uses needs.workflow-context.outputs
       - *checkout-workflow-support
-      - *setup-aws-credentials
+      - *setup-aws-batch-credentials
       - *setup-runtime
 
       - id: cancel

diff --git a/README.md b/README.md
@@ -106,6 +106,7 @@ Executable scripts that are used in our workflows.
 
 - [interpolate-env](bin/interpolate-env)
 - [json-to-envvars](bin/json-to-envvars)
+- [write-envdir](bin/write-envdir)
 - [yaml-to-envvars](bin/yaml-to-envvars)
 
 ## Workflow text templates

diff --git a/bin/write-envdir b/bin/write-envdir
@@ -0,0 +1,22 @@
+#!/bin/bash
+# usage: write-envdir <envdir> [[var1 [var2 [var3 […]]]]
+#
+# Writes the current value for each environment variable name given into the
+# directory <envdir>, one file per variable.  Creates <envdir> if it doesn't
+# already exist.
+#
+# This was copied from the ncov-ingest repo:
+# <https://github.com/nextstrain/ncov-ingest/blob/20b432624ad2c4bf24c85756bd04bae1e067bde8/bin/write-envdir>
+#
+set -eou pipefail
+
+dir="${1:?no envdir path}"
+shift
+
+mkdir -pv "$dir"
+cd "$dir"
+
+for name in "$@"; do
+    echo "${!name}" > "$name"
+    echo "Wrote $dir/$name"
+done