From 68c5c72f50fe30304a3d2284382ee9e8d32690c3 Mon Sep 17 00:00:00 2001 From: Jover Lee Date: Fri, 10 May 2024 12:37:09 -0700 Subject: [PATCH] pathogen-repo-build: support assuming AWS role for runtime permissions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assumes repo-specific roles, `GitHubActionsRoleNextstrainRepo@zika` for example, which are managed by the Terraform configuration in the nextstrain/infra repo. The repo here is always the _calling_ repository, regardless of if the "repo" input was provided to use workflows from another place. The runtime credentials are then saved in an envdir that is passed to the build command via `NEXTSTRAIN_RUNTIME_ENVDIRS`. Namespaced `NEXTSTRAIN_RUNTIME_ENVDIR` with `./git/nextstrain` as suggested by @tsibley in review¹ ¹ Related-to: Co-authored-by: Thomas Sibley --- .github/workflows/pathogen-repo-build.yaml | 52 +++++++++++++++--- .github/workflows/pathogen-repo-build.yaml.in | 54 ++++++++++++++++--- 2 files changed, 92 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pathogen-repo-build.yaml b/.github/workflows/pathogen-repo-build.yaml index a3a418d..883dc15 100644 --- a/.github/workflows/pathogen-repo-build.yaml +++ b/.github/workflows/pathogen-repo-build.yaml @@ -47,12 +47,17 @@ on: If your build runs longer than the 6 hour limit for a single GitHub Action job, then use the aws-batch runtime and the `--detach` flag. Subsequent chained jobs will be automatically used to wait on the remote build for up to 24 hours total. - All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option. If AWS credentials were acquired by the GitHub Action job via role assumption, the following environment variables are also available to be passed: + All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option. + It is assumed that the pathogen repo build requires AWS credentials for read/write access to S3 buckets. These may come directly from secrets or indirectly from assuming a role via GitHub Actions' OIDC provider. - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN + The following secrets are used if present: + + - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY + + They must be defined in the repo's Actions secrets and passed to this workflow with `secrets: inherit`. + + If no secrets are present, the GitHubActionsRoleNextstrainRepo@ role is assumed (in both senses of the verb). The here is always the _calling workflow's_ repository name (without owner), regardless of the "repo" input. The repository must already be configured in nextstrain/infra for the role to exist; see documentation there for how to add the repository if necessary. type: string default: nextstrain build . required: false @@ -138,12 +143,17 @@ on: If your build runs longer than the 6 hour limit for a single GitHub Action job, then use the aws-batch runtime and the `--detach` flag. Subsequent chained jobs will be automatically used to wait on the remote build for up to 24 hours total. - All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option. If AWS credentials were acquired by the GitHub Action job via role assumption, the following environment variables are also available to be passed: + All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option. + + It is assumed that the pathogen repo build requires AWS credentials for read/write access to S3 buckets. These may come directly from secrets or indirectly from assuming a role via GitHub Actions' OIDC provider. + + The following secrets are used if present: + + - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY + They must be defined in the repo's Actions secrets and passed to this workflow with `secrets: inherit`. - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN + If no secrets are present, the GitHubActionsRoleNextstrainRepo@ role is assumed (in both senses of the verb). The here is always the _calling workflow's_ repository name (without owner), regardless of the "repo" input. The repository must already be configured in nextstrain/infra for the role to exist; see documentation there for how to add the repository if necessary. type: string default: nextstrain build . required: false @@ -210,6 +220,7 @@ on: env: NEXTSTRAIN_GITHUB_DIR: .git/nextstrain/.github NEXTSTRAIN_BUILD_LOG: build.log + NEXTSTRAIN_RUNTIME_ENVDIR: .git/nextstrain/env.d permissions: id-token: write jobs: @@ -255,6 +266,30 @@ jobs: echo "$secrets" | jq 'del(.github_token)' | "$NEXTSTRAIN_GITHUB_DIR"/bin/json-to-envvars | tee -a "$GITHUB_ENV" + - id: role + name: Set repo-specific role to (potentially) assume for runtime access to AWS + env: + REPO_FULL_NAME: ${{ github.repository }} + run: | + echo "arn=arn:aws:iam::827581582529:role/GitHubActionsRoleNextstrainRepo@${REPO_FULL_NAME#*/}" | tee -a "$GITHUB_OUTPUT" + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-east-1 + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && steps.role.outputs.arn || '' }} + role-duration-seconds: 43200 # seconds, or 12 hours + - name: Save runtime AWS credentials to ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} + run: | + "$NEXTSTRAIN_GITHUB_DIR"/bin/write-envdir "$NEXTSTRAIN_RUNTIME_ENVDIR" \ + AWS_ACCESS_KEY_ID \ + AWS_SECRET_ACCESS_KEY \ + AWS_SESSION_TOKEN + # This will overwrite the runtime AWS credential envvars configured above + # so if the build is using the aws-batch runtime, the Nextstrain CLI will + # have access to the AWS Batch session credentials + # Comment only applies to this first use of the `&setup-aws-batch-credentials`, so + # outdenting comments to not repeat it with expanded YAML - if: inputs.runtime == 'aws-batch' uses: aws-actions/configure-aws-credentials@v4 with: @@ -271,6 +306,7 @@ jobs: - name: Run build via ${{ inputs.runtime }} env: NEXTSTRAIN_BUILD_COMMAND: ${{ inputs.run }} + NEXTSTRAIN_RUNTIME_ENVDIRS: ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} run: | # shellcheck disable=SC2154 set -x diff --git a/.github/workflows/pathogen-repo-build.yaml.in b/.github/workflows/pathogen-repo-build.yaml.in index 315ffe8..8becae3 100644 --- a/.github/workflows/pathogen-repo-build.yaml.in +++ b/.github/workflows/pathogen-repo-build.yaml.in @@ -66,13 +66,26 @@ on: All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime - via the `--env` option. If AWS credentials were acquired by the - GitHub Action job via role assumption, the following environment - variables are also available to be passed: + via the `--env` option. - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN + It is assumed that the pathogen repo build requires AWS credentials for + read/write access to S3 buckets. These may come directly from secrets + or indirectly from assuming a role via GitHub Actions' OIDC provider. + + The following secrets are used if present: + + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + + They must be defined in the repo's Actions secrets and passed to this + workflow with `secrets: inherit`. + + If no secrets are present, the GitHubActionsRoleNextstrainRepo@ + role is assumed (in both senses of the verb). The here is + always the _calling workflow's_ repository name (without owner), + regardless of the "repo" input. The repository must already be + configured in nextstrain/infra for the role to exist; see + documentation there for how to add the repository if necessary. type: string default: nextstrain build . required: false @@ -162,6 +175,7 @@ on: env: NEXTSTRAIN_GITHUB_DIR: .git/nextstrain/.github NEXTSTRAIN_BUILD_LOG: build.log + NEXTSTRAIN_RUNTIME_ENVDIR: .git/nextstrain/env.d permissions: id-token: write @@ -218,6 +232,33 @@ jobs: | "$NEXTSTRAIN_GITHUB_DIR"/bin/json-to-envvars | tee -a "$GITHUB_ENV" + - id: role + name: Set repo-specific role to (potentially) assume for runtime access to AWS + env: + REPO_FULL_NAME: ${{ github.repository }} + run: | + echo "arn=arn:aws:iam::827581582529:role/GitHubActionsRoleNextstrainRepo@${REPO_FULL_NAME#*/}" | tee -a "$GITHUB_OUTPUT" + + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-east-1 + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && steps.role.outputs.arn || '' }} + role-duration-seconds: 43200 # seconds, or 12 hours + + - name: Save runtime AWS credentials to ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} + run: | + "$NEXTSTRAIN_GITHUB_DIR"/bin/write-envdir "$NEXTSTRAIN_RUNTIME_ENVDIR" \ + AWS_ACCESS_KEY_ID \ + AWS_SECRET_ACCESS_KEY \ + AWS_SESSION_TOKEN + + # This will overwrite the runtime AWS credential envvars configured above + # so if the build is using the aws-batch runtime, the Nextstrain CLI will + # have access to the AWS Batch session credentials + # Comment only applies to this first use of the `&setup-aws-batch-credentials`, so + # outdenting comments to not repeat it with expanded YAML - &setup-aws-batch-credentials if: inputs.runtime == 'aws-batch' uses: aws-actions/configure-aws-credentials@v4 @@ -238,6 +279,7 @@ jobs: - name: Run build via ${{ inputs.runtime }} env: NEXTSTRAIN_BUILD_COMMAND: ${{ inputs.run }} + NEXTSTRAIN_RUNTIME_ENVDIRS: ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} run: | # shellcheck disable=SC2154 set -x