From b2c38a04e15c817dba16b360eefbd2d89ecaaf11 Mon Sep 17 00:00:00 2001 From: Jover Lee Date: Fri, 10 May 2024 12:37:09 -0700 Subject: [PATCH] pathogen-repo-build: support assuming AWS role for runtime permissions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assumes the role `GitHubActionsRoleNextstrainS3Access` (just a placeholder name, final name TDB in https://github.com/nextstrain/infra/issues/4) and scopes repo specific permissions with an inline session policy. The runtime credentials are then saved in an envdir that is passed to the build command via `NEXTSTRAIN_RUNTIME_ENVDIRS`. Namespaced `NEXTSTRAIN_RUNTIME_ENVDIR` with `./git/nextstrain` as suggested by @tsibley in review¹ ¹ pathogen-repo-build: --- .github/workflows/pathogen-repo-build.yaml | 49 +++++++++++++++---- .github/workflows/pathogen-repo-build.yaml.in | 47 +++++++++++++++--- 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/.github/workflows/pathogen-repo-build.yaml b/.github/workflows/pathogen-repo-build.yaml index bfc4699..5b2083e 100644 --- a/.github/workflows/pathogen-repo-build.yaml +++ b/.github/workflows/pathogen-repo-build.yaml @@ -47,12 +47,17 @@ on: If your build runs longer than the 6 hour limit for a single GitHub Action job, then use the aws-batch runtime and the `--detach` flag. Subsequent chained jobs will be automatically used to wait on the remote build for up to 24 hours total. - All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option. If AWS credentials were acquired by the GitHub Action job via role assumption, the following environment variables are also available to be passed: + All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option. + It is assumed that the pathogen repo build requires AWS credentials for read/write access to S3 buckets. These may come directly from secrets or indirectly from assuming a role via GitHub Actions' OIDC provider. - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN + The following secrets are used if present: + + - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY + + They must be defined in the repo's Actions secrets and passed to this workflow with `secrets: inherit`. + + If no secrets are present, the GitHubActionsRoleNextstrainS3Access role is assumed (in both senses of the verb) with additional repo specific inline session policies defined in text-templates/repo-aws-iam-policy.json type: string default: nextstrain build . required: false @@ -138,12 +143,17 @@ on: If your build runs longer than the 6 hour limit for a single GitHub Action job, then use the aws-batch runtime and the `--detach` flag. Subsequent chained jobs will be automatically used to wait on the remote build for up to 24 hours total. - All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option. If AWS credentials were acquired by the GitHub Action job via role assumption, the following environment variables are also available to be passed: + All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option. + + It is assumed that the pathogen repo build requires AWS credentials for read/write access to S3 buckets. These may come directly from secrets or indirectly from assuming a role via GitHub Actions' OIDC provider. + + The following secrets are used if present: + + - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY + They must be defined in the repo's Actions secrets and passed to this workflow with `secrets: inherit`. - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN + If no secrets are present, the GitHubActionsRoleNextstrainS3Access role is assumed (in both senses of the verb) with additional repo specific inline session policies defined in text-templates/repo-aws-iam-policy.json type: string default: nextstrain build . required: false @@ -210,6 +220,7 @@ on: env: NEXTSTRAIN_GITHUB_DIR: .git/nextstrain/.github NEXTSTRAIN_BUILD_LOG: build.log + NEXTSTRAIN_RUNTIME_ENVDIR: .git/nextstrain/env.d permissions: id-token: write jobs: @@ -263,7 +274,26 @@ jobs: export PREFIX="${REPO_NAME}_" aws_inline_session_policy=$("$NEXTSTRAIN_GITHUB_DIR"/bin/interpolate-env < "$NEXTSTRAIN_GITHUB_DIR"/text-templates/repo-aws-iam-policy.json | jq -c .) - echo "AWS_INLINE_SESSION_POLICY=$aws_inline_session_policy" | tee -a $GITHUB_ENV + echo "AWS_INLINE_SESSION_POLICY=$aws_inline_session_policy" | tee -a "$GITHUB_ENV" + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-east-1 + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && 'arn:aws:iam::827581582529:role/GitHubActionsRoleNextstrainS3Access' || '' }} + role-duration-seconds: 43200 # seconds, or 12 hours + inline-session-policy: ${{ secrets.AWS_ACCESS_KEY_ID == '' && env.AWS_INLINE_SESSION_POLICY || '' }} + - name: Save runtime AWS credentials to ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} + run: | + ./bin/write-envdir "$NEXTSTRAIN_RUNTIME_ENVDIR" \ + AWS_ACCESS_KEY_ID \ + AWS_SECRET_ACCESS_KEY \ + AWS_SESSION_TOKEN + # This will overwrite the runtime AWS credential envvars configured above + # so if the build is using the aws-batch runtime, the Nextstrain CLI will + # have access to the AWS Batch session credentials + # Comment only applies to this first use of the `&setup-aws-batch-credentials`, so + # outdenting comments to not repeat it with expanded YAML - if: inputs.runtime == 'aws-batch' uses: aws-actions/configure-aws-credentials@v4 with: @@ -280,6 +310,7 @@ jobs: - name: Run build via ${{ inputs.runtime }} env: NEXTSTRAIN_BUILD_COMMAND: ${{ inputs.run }} + NEXTSTRAIN_RUNTIME_ENVDIRS: ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} run: | # shellcheck disable=SC2154 set -x diff --git a/.github/workflows/pathogen-repo-build.yaml.in b/.github/workflows/pathogen-repo-build.yaml.in index 09ff1c0..bf30f79 100644 --- a/.github/workflows/pathogen-repo-build.yaml.in +++ b/.github/workflows/pathogen-repo-build.yaml.in @@ -66,13 +66,23 @@ on: All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime - via the `--env` option. If AWS credentials were acquired by the - GitHub Action job via role assumption, the following environment - variables are also available to be passed: + via the `--env` option. - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN + It is assumed that the pathogen repo build requires AWS credentials for + read/write access to S3 buckets. These may come directly from secrets + or indirectly from assuming a role via GitHub Actions' OIDC provider. + + The following secrets are used if present: + + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + + They must be defined in the repo's Actions secrets and passed to this + workflow with `secrets: inherit`. + + If no secrets are present, the GitHubActionsRoleNextstrainS3Access + role is assumed (in both senses of the verb) with additional repo specific + inline session policies defined in text-templates/repo-aws-iam-policy.json type: string default: nextstrain build . required: false @@ -162,6 +172,7 @@ on: env: NEXTSTRAIN_GITHUB_DIR: .git/nextstrain/.github NEXTSTRAIN_BUILD_LOG: build.log + NEXTSTRAIN_RUNTIME_ENVDIR: .git/nextstrain/env.d permissions: id-token: write @@ -226,8 +237,29 @@ jobs: export PREFIX="${REPO_NAME}_" aws_inline_session_policy=$("$NEXTSTRAIN_GITHUB_DIR"/bin/interpolate-env < "$NEXTSTRAIN_GITHUB_DIR"/text-templates/repo-aws-iam-policy.json | jq -c .) - echo "AWS_INLINE_SESSION_POLICY=$aws_inline_session_policy" | tee -a $GITHUB_ENV + echo "AWS_INLINE_SESSION_POLICY=$aws_inline_session_policy" | tee -a "$GITHUB_ENV" + + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-east-1 + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && 'arn:aws:iam::827581582529:role/GitHubActionsRoleNextstrainS3Access' || '' }} + role-duration-seconds: 43200 # seconds, or 12 hours + inline-session-policy: ${{ secrets.AWS_ACCESS_KEY_ID == '' && env.AWS_INLINE_SESSION_POLICY || '' }} + - name: Save runtime AWS credentials to ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} + run: | + ./bin/write-envdir "$NEXTSTRAIN_RUNTIME_ENVDIR" \ + AWS_ACCESS_KEY_ID \ + AWS_SECRET_ACCESS_KEY \ + AWS_SESSION_TOKEN + + # This will overwrite the runtime AWS credential envvars configured above + # so if the build is using the aws-batch runtime, the Nextstrain CLI will + # have access to the AWS Batch session credentials + # Comment only applies to this first use of the `&setup-aws-batch-credentials`, so + # outdenting comments to not repeat it with expanded YAML - &setup-aws-batch-credentials if: inputs.runtime == 'aws-batch' uses: aws-actions/configure-aws-credentials@v4 @@ -248,6 +280,7 @@ jobs: - name: Run build via ${{ inputs.runtime }} env: NEXTSTRAIN_BUILD_COMMAND: ${{ inputs.run }} + NEXTSTRAIN_RUNTIME_ENVDIRS: ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} run: | # shellcheck disable=SC2154 set -x