From 009b8d8bd2171082c4db2168cda0acc032345360 Mon Sep 17 00:00:00 2001 From: Jover Lee Date: Fri, 10 May 2024 12:37:09 -0700 Subject: [PATCH] pathogen-repo-build: support assuming AWS role for runtime permissions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assumes repo-specific roles, `GitHubActionsRoleNextstrainRepo@zika` for example, which are managed by the Terraform configuration in the nextstrain/infra repo. The repo here is always the _calling_ repository, regardless of if the "repo" input was provided to use workflows from another place. The runtime credentials are then saved in an envdir that is passed to the build command via `NEXTSTRAIN_RUNTIME_ENVDIRS`. Namespaced `NEXTSTRAIN_RUNTIME_ENVDIR` with `./git/nextstrain` as suggested by @tsibley in review¹ ¹ Related-to: Co-authored-by: Thomas Sibley --- .github/workflows/pathogen-repo-build.yaml | 52 ++++++++++++++++--- .github/workflows/pathogen-repo-build.yaml.in | 51 +++++++++++++++--- 2 files changed, 89 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pathogen-repo-build.yaml b/.github/workflows/pathogen-repo-build.yaml index a3a418d..c0e6329 100644 --- a/.github/workflows/pathogen-repo-build.yaml +++ b/.github/workflows/pathogen-repo-build.yaml @@ -47,12 +47,17 @@ on: If your build runs longer than the 6 hour limit for a single GitHub Action job, then use the aws-batch runtime and the `--detach` flag. Subsequent chained jobs will be automatically used to wait on the remote build for up to 24 hours total. - All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option. If AWS credentials were acquired by the GitHub Action job via role assumption, the following environment variables are also available to be passed: + All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option. + It is assumed that the pathogen repo build requires AWS credentials for read/write access to S3 buckets. These may come directly from secrets or indirectly from assuming a role via GitHub Actions' OIDC provider. - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN + The following secrets are used if present: + + - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY + + They must be defined in the repo's Actions secrets and passed to this workflow with `secrets: inherit`. + + If no secrets are present, the GitHubActionsRoleNextstrainS3Access role is assumed (in both senses of the verb) with additional repo specific inline session policies defined in text-templates/repo-aws-iam-policy.json type: string default: nextstrain build . required: false @@ -138,12 +143,17 @@ on: If your build runs longer than the 6 hour limit for a single GitHub Action job, then use the aws-batch runtime and the `--detach` flag. Subsequent chained jobs will be automatically used to wait on the remote build for up to 24 hours total. - All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option. If AWS credentials were acquired by the GitHub Action job via role assumption, the following environment variables are also available to be passed: + All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option. + + It is assumed that the pathogen repo build requires AWS credentials for read/write access to S3 buckets. These may come directly from secrets or indirectly from assuming a role via GitHub Actions' OIDC provider. + + The following secrets are used if present: + + - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY + They must be defined in the repo's Actions secrets and passed to this workflow with `secrets: inherit`. - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN + If no secrets are present, the GitHubActionsRoleNextstrainS3Access role is assumed (in both senses of the verb) with additional repo specific inline session policies defined in text-templates/repo-aws-iam-policy.json type: string default: nextstrain build . required: false @@ -210,6 +220,7 @@ on: env: NEXTSTRAIN_GITHUB_DIR: .git/nextstrain/.github NEXTSTRAIN_BUILD_LOG: build.log + NEXTSTRAIN_RUNTIME_ENVDIR: .git/nextstrain/env.d permissions: id-token: write jobs: @@ -255,6 +266,30 @@ jobs: echo "$secrets" | jq 'del(.github_token)' | "$NEXTSTRAIN_GITHUB_DIR"/bin/json-to-envvars | tee -a "$GITHUB_ENV" + - id: role + name: Set repo-specific role to (potentially) assume for runtime access to AWS + env: + REPO_FULL_NAME: ${{ github.repository }} + run: | + echo "arn=arn:aws:iam::827581582529:role/GitHubActionsRoleNextstrainRepo@${REPO_FULL_NAME#*/}" | tee -a "$GITHUB_OUTPUT" + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-east-1 + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && steps.role.outputs.arn || '' }} + role-duration-seconds: 43200 # seconds, or 12 hours + - name: Save runtime AWS credentials to ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} + run: | + "$NEXTSTRAIN_GITHUB_DIR"/bin/write-envdir "$NEXTSTRAIN_RUNTIME_ENVDIR" \ + AWS_ACCESS_KEY_ID \ + AWS_SECRET_ACCESS_KEY \ + AWS_SESSION_TOKEN + # This will overwrite the runtime AWS credential envvars configured above + # so if the build is using the aws-batch runtime, the Nextstrain CLI will + # have access to the AWS Batch session credentials + # Comment only applies to this first use of the `&setup-aws-batch-credentials`, so + # outdenting comments to not repeat it with expanded YAML - if: inputs.runtime == 'aws-batch' uses: aws-actions/configure-aws-credentials@v4 with: @@ -271,6 +306,7 @@ jobs: - name: Run build via ${{ inputs.runtime }} env: NEXTSTRAIN_BUILD_COMMAND: ${{ inputs.run }} + NEXTSTRAIN_RUNTIME_ENVDIRS: ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} run: | # shellcheck disable=SC2154 set -x diff --git a/.github/workflows/pathogen-repo-build.yaml.in b/.github/workflows/pathogen-repo-build.yaml.in index 315ffe8..b9f2137 100644 --- a/.github/workflows/pathogen-repo-build.yaml.in +++ b/.github/workflows/pathogen-repo-build.yaml.in @@ -66,13 +66,23 @@ on: All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime - via the `--env` option. If AWS credentials were acquired by the - GitHub Action job via role assumption, the following environment - variables are also available to be passed: + via the `--env` option. - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN + It is assumed that the pathogen repo build requires AWS credentials for + read/write access to S3 buckets. These may come directly from secrets + or indirectly from assuming a role via GitHub Actions' OIDC provider. + + The following secrets are used if present: + + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + + They must be defined in the repo's Actions secrets and passed to this + workflow with `secrets: inherit`. + + If no secrets are present, the GitHubActionsRoleNextstrainS3Access + role is assumed (in both senses of the verb) with additional repo specific + inline session policies defined in text-templates/repo-aws-iam-policy.json type: string default: nextstrain build . required: false @@ -162,6 +172,7 @@ on: env: NEXTSTRAIN_GITHUB_DIR: .git/nextstrain/.github NEXTSTRAIN_BUILD_LOG: build.log + NEXTSTRAIN_RUNTIME_ENVDIR: .git/nextstrain/env.d permissions: id-token: write @@ -218,6 +229,33 @@ jobs: | "$NEXTSTRAIN_GITHUB_DIR"/bin/json-to-envvars | tee -a "$GITHUB_ENV" + - id: role + name: Set repo-specific role to (potentially) assume for runtime access to AWS + env: + REPO_FULL_NAME: ${{ github.repository }} + run: | + echo "arn=arn:aws:iam::827581582529:role/GitHubActionsRoleNextstrainRepo@${REPO_FULL_NAME#*/}" | tee -a "$GITHUB_OUTPUT" + + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-east-1 + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && steps.role.outputs.arn || '' }} + role-duration-seconds: 43200 # seconds, or 12 hours + + - name: Save runtime AWS credentials to ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} + run: | + "$NEXTSTRAIN_GITHUB_DIR"/bin/write-envdir "$NEXTSTRAIN_RUNTIME_ENVDIR" \ + AWS_ACCESS_KEY_ID \ + AWS_SECRET_ACCESS_KEY \ + AWS_SESSION_TOKEN + + # This will overwrite the runtime AWS credential envvars configured above + # so if the build is using the aws-batch runtime, the Nextstrain CLI will + # have access to the AWS Batch session credentials + # Comment only applies to this first use of the `&setup-aws-batch-credentials`, so + # outdenting comments to not repeat it with expanded YAML - &setup-aws-batch-credentials if: inputs.runtime == 'aws-batch' uses: aws-actions/configure-aws-credentials@v4 @@ -238,6 +276,7 @@ jobs: - name: Run build via ${{ inputs.runtime }} env: NEXTSTRAIN_BUILD_COMMAND: ${{ inputs.run }} + NEXTSTRAIN_RUNTIME_ENVDIRS: ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} run: | # shellcheck disable=SC2154 set -x