CI #1449
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: [push, pull_request, workflow_dispatch] | |
| # Prevent intermediate images from being used by another run. | |
| # Concurrency group is unique: | |
| # 1. to this workflow (github.workflow) | |
| # 2. per event ref (github.ref) | |
| # 3. per run on the default branch (github.run_id) | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref_type == 'branch' && github.ref_name == github.event.repository.default_branch && github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: nextstrain/.github/actions/shellcheck@master | |
| # Build multi-platform builder and final images with caching from Docker Hub | |
| # and GitHub Container Registry; push to GitHub Container Registry. | |
| build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '>=3.8' | |
| - name: Set $CACHE_DATE | |
| run: echo "CACHE_DATE=$(date --utc +%Y%m%dT%H%M%SZ)" | tee -a "$GITHUB_ENV" | |
| - if: github.event_name != 'pull_request' && github.ref_type == 'branch' && github.ref_name == github.event.repository.default_branch | |
| name: Set $TAG (default branch) | |
| run: echo "TAG=build-$CACHE_DATE" | tee -a "$GITHUB_ENV" | |
| - if: env.TAG == '' | |
| name: Set $TAG (PRs, non-default branch) | |
| # From `man docker-image-tag`: The tag name must be valid ASCII and may | |
| # contain lowercase and uppercase letters, digits, underscores, periods | |
| # and hyphens. | |
| run: echo "TAG=branch-${GITHUB_REF_NAME//[^A-Za-z0-9._-]/-}" | tee -a "$GITHUB_ENV" | |
| - uses: docker/setup-qemu-action@v3 | |
| # GITHUB_TOKEN is unreliable¹ so use a token from nextstrain-bot. | |
| # ¹ https://github.com/docker/build-push-action/issues/463#issuecomment-939394233 | |
| - uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: nextstrain-bot | |
| password: ${{ secrets.GH_TOKEN_NEXTSTRAIN_BOT_MANAGE_PACKAGES }} | |
| - run: ./devel/build -p linux/amd64,linux/arm64 -r ghcr.io -t "$TAG" -l logs/ | |
| - if: always() | |
| name: Upload build logs as artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: build-logs | |
| path: logs/ | |
| - if: always() | |
| name: Summarize build logs for the GitHub Actions run summary page | |
| run: | | |
| for log in logs/*; do | |
| { | |
| echo "## $(basename "$log")" | |
| echo '' | |
| echo '```' | |
| ./devel/summarize-buildkit-output "$log" 2>&1 | |
| echo '```' | |
| echo '' | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| done | |
| outputs: | |
| tag: ${{ env.TAG }} | |
| # Run tests with the final image from GitHub Container Registry. | |
| test: | |
| name: test (${{ matrix.platform }}) | |
| needs: build | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| platform: | |
| - linux/amd64 | |
| - linux/arm64 | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: nextstrain-bot | |
| password: ${{ secrets.GH_TOKEN_NEXTSTRAIN_BOT_MANAGE_PACKAGES }} | |
| - uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: us-east-1 | |
| role-to-assume: arn:aws:iam::827581582529:role/GitHubActionsRoleNextstrainTmpBucket | |
| - run: aws sts get-caller-identity | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ~3 | |
| # The ubuntu-latest runner is linux/amd64 so anything else will | |
| # run with emulation, which is still better than nothing. | |
| - if: matrix.platform != 'linux/amd64' | |
| uses: docker/setup-qemu-action@v3 | |
| - uses: actions/checkout@v4 | |
| - run: pip install cram | |
| - run: make test | |
| env: | |
| IMAGE: ghcr.io/nextstrain/base:${{ needs.build.outputs.tag }} | |
| DOCKER_DEFAULT_PLATFORM: ${{ matrix.platform }} | |
| - uses: nextstrain/.github/actions/setup-nextstrain-cli@master | |
| - name: Run zika-tutorial | |
| run: | | |
| git clone https://github.com/nextstrain/zika-tutorial | |
| nextstrain build --image ghcr.io/nextstrain/base:${{ needs.build.outputs.tag }} zika-tutorial -F | |
| env: | |
| DOCKER_DEFAULT_PLATFORM: ${{ matrix.platform }} | |
| validate-platforms: | |
| name: Validate platforms | |
| needs: build | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: docker/setup-qemu-action@v3 | |
| - uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: nextstrain-bot | |
| password: ${{ secrets.GH_TOKEN_NEXTSTRAIN_BOT_MANAGE_PACKAGES }} | |
| - name: Validate final images | |
| run: ./devel/validate-platforms -r ghcr.io -t ${{ needs.build.outputs.tag }} | |
| # "Push" (copy) the builder and final images from GitHub Container Registry to | |
| # Docker Hub, where they will persist. Do this regardless of test results. | |
| push-branch: | |
| if: startsWith(needs.build.outputs.tag, 'branch-') && github.event_name != 'pull_request' | |
| needs: build | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: nextstrain-bot | |
| password: ${{ secrets.GH_TOKEN_NEXTSTRAIN_BOT_MANAGE_PACKAGES }} | |
| - uses: docker/login-action@v3 | |
| with: | |
| registry: docker.io | |
| username: nextstrainbot | |
| password: ${{ secrets.DOCKER_TOKEN }} | |
| - name: Copy $TAG images to Docker Hub | |
| run: ./devel/copy-images -i ghcr.io -o docker.io -t ${{ needs.build.outputs.tag }} | |
| # "Push" (copy) the builder and final images from GitHub Container Registry to | |
| # Docker Hub, where they will persist. Only do this if tests pass. | |
| push-build: | |
| if: startsWith(needs.build.outputs.tag, 'build-') | |
| needs: [build, test, validate-platforms] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: nextstrain-bot | |
| password: ${{ secrets.GH_TOKEN_NEXTSTRAIN_BOT_MANAGE_PACKAGES }} | |
| - uses: docker/login-action@v3 | |
| with: | |
| registry: docker.io | |
| username: nextstrainbot | |
| password: ${{ secrets.DOCKER_TOKEN }} | |
| - name: Copy $TAG + latest images to Docker Hub | |
| run: | | |
| ./devel/copy-images -i ghcr.io -o docker.io -t ${{ needs.build.outputs.tag }} -l | |
| # Run pathogen repo CI builds with the final image | |
| # This is running pathogen-repo-ci@v0 for pathogen repos that do not conform | |
| # to the standard pathogen repo structure and is not expected to be updated. | |
| # Any new pathogen repos should be added to the job using the latest version | |
| # of the pathogen-repo-ci below. | |
| test-pathogen-repo-ci-v0: | |
| # Only one of push-{branch,build} runs for any given workflow run, and | |
| # we're ok with either of them. | |
| needs: [build, push-branch, push-build] | |
| if: |2 | |
| success() | |
| || needs.push-branch.result == 'success' | |
| || needs.push-build.result == 'success' | |
| strategy: | |
| matrix: | |
| include: | |
| - { pathogen: avian-flu, build-args: --configfile config/gisaid.yaml -pf test_target } | |
| - { pathogen: ebola } | |
| - { pathogen: mumps } | |
| - { pathogen: ncov, build-args: all_regions -j 2 --profile nextstrain_profiles/nextstrain-ci } | |
| - { pathogen: rsv } | |
| - { pathogen: seasonal-flu, build-args: --configfile profiles/ci/builds.yaml -p } | |
| name: test-pathogen-repo-ci-v0 (${{ matrix.pathogen }}) | |
| uses: nextstrain/.github/.github/workflows/pathogen-repo-ci.yaml@v0 | |
| with: | |
| repo: nextstrain/${{ matrix.pathogen }} | |
| build-args: ${{ matrix.build-args }} | |
| runtimes: | | |
| - docker | |
| env: | | |
| NEXTSTRAIN_DOCKER_IMAGE: nextstrain/base:${{ needs.build.outputs.tag }} | |
| artifact-name: ${{ matrix.pathogen }}-outputs | |
| continue-on-error: true | |
| secrets: inherit | |
| # Run pathogen repo CI builds with the final image | |
| # This is running pathogen-repo-ci@master for pathogen repos that _do_ follow | |
| # standard pathogen repo structure and new pathogens should be added here | |
| # to be supported for future updates such as testing on multiple platforms. | |
| test-pathogen-repo-ci: | |
| # Only one of push-{branch,build} runs for any given workflow run, and | |
| # we're ok with either of them. | |
| needs: [build, push-branch, push-build] | |
| if: |2 | |
| success() | |
| || needs.push-branch.result == 'success' | |
| || needs.push-build.result == 'success' | |
| strategy: | |
| # XXX TODO: Test on multiple platforms via the matrix too, as above? | |
| matrix: | |
| pathogen: | |
| - dengue | |
| - lassa | |
| - measles | |
| - mpox | |
| - seasonal-cov | |
| - zika | |
| name: test-pathogen-repo-ci (${{ matrix.pathogen }}) | |
| uses: nextstrain/.github/.github/workflows/pathogen-repo-ci.yaml@master | |
| with: | |
| repo: nextstrain/${{ matrix.pathogen }} | |
| runtimes: | | |
| - docker | |
| env: | | |
| NEXTSTRAIN_DOCKER_IMAGE: nextstrain/base:${{ needs.build.outputs.tag }} | |
| artifact-name: ${{ matrix.pathogen }}-outputs | |
| continue-on-error: true | |
| secrets: inherit | |
| # Delete the builder and final images from GitHub Container Registry. | |
| cleanup-registry: | |
| if: always() | |
| needs: [build, test, validate-platforms, push-branch, push-build] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{ secrets.GH_TOKEN_NEXTSTRAIN_BOT_MANAGE_PACKAGES }} | |
| script: | | |
| const script = require('./devel/delete-from-ghcr.js'); | |
| const tag = "${{ needs.build.outputs.tag }}"; | |
| const token = "${{ secrets.GH_TOKEN_NEXTSTRAIN_BOT_MANAGE_PACKAGES }}"; | |
| script({fetch, octokit: github, tag, token}); |