chore: output playwright report in readable form in github actions output #63
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 'Run tests' | |
| on: | |
| pull_request: | |
| branches: [main] | |
| schedule: | |
| - cron: '0 6 * * *' # Run every day at 6am UTC | |
| workflow_dispatch: | |
| inputs: | |
| versions: | |
| description: 'The versions of Next.js to test against (quoted and comma separated)' | |
| required: false | |
| default: 'latest' | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - name: Check PR labels | |
| if: github.event_name == 'pull_request' | |
| id: check-labels | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const { data: labels } = await github.rest.issues.listLabelsOnIssue({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.payload.pull_request.number, | |
| }); | |
| return labels.some(label => label.name === 'autorelease: pending' || label.name === 'test all versions'); | |
| - name: Set Next.js versions to test | |
| id: set-matrix | |
| # If this is the nightly build or a release PR then run the full matrix of versions | |
| run: | | |
| if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then | |
| echo "matrix=${{ github.event.inputs.versions }}" >> $GITHUB_OUTPUT | |
| elif [ "${{ github.event_name }}" = "schedule" ] || [ "${{ steps.check-labels.outputs.result }}" = "true" ]; then | |
| echo "matrix=[\"latest\", \"canary\", \"13.5.1\"]" >> $GITHUB_OUTPUT | |
| else | |
| echo "matrix=[\"latest\"]" >> $GITHUB_OUTPUT | |
| fi | |
| e2e: | |
| needs: setup | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: ${{ fromJson(needs.setup.outputs.matrix) }} | |
| shard: [1, 2, 3, 4, 5] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: 'Install Node' | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '18.x' | |
| cache: 'npm' | |
| cache-dependency-path: '**/package-lock.json' | |
| - uses: oven-sh/setup-bun@v1 | |
| - name: setup pnpm/yarn | |
| run: corepack enable | |
| shell: bash | |
| - name: Install Deno | |
| uses: denoland/setup-deno@v1 | |
| with: | |
| # Should match the `DENO_VERSION_RANGE` from https://github.com/netlify/edge-bundler/blob/main/node/bridge.ts#L17 | |
| deno-version: v1.37.0 | |
| - name: 'Install dependencies' | |
| run: npm ci | |
| - name: 'Netlify Login' | |
| env: | |
| NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} | |
| run: | | |
| # TODO(serhalp) Unpin version once | |
| # https://netlify.slack.com/archives/C07686YAY13/p1718139257978049 is resolved. | |
| npm i -g [email protected] | |
| netlify login | |
| - name: Get installed Playwright version | |
| id: playwright-version | |
| run: echo "version=$(npm view @playwright/test version)" >> $GITHUB_OUTPUT | |
| - uses: actions/cache@v4 | |
| id: playwright-cache | |
| with: | |
| path: '~/.cache/ms-playwright' | |
| key: '${{ runner.os }}-playwright-${{ steps.playwright-version.outputs.version }}' | |
| restore-keys: | | |
| ${{ runner.os }}-playwright- | |
| - name: Install Playwright Browsers | |
| if: steps.playwright-cache.outputs.cache-hit != 'true' | |
| run: npx playwright install --with-deps | |
| - name: Resolve Next.js version | |
| id: resolve-next-version | |
| shell: bash | |
| run: | | |
| RESOLVED_VERSION=$(npm view next@${{ matrix.version }} version) | |
| echo "version=$RESOLVED_VERSION" >> $GITHUB_OUTPUT | |
| echo "Resolved Next.js version for 'next@${{ matrix.version }}' is '$RESOLVED_VERSION'" | |
| - name: Run Playwright tests | |
| run: npm run test:ci:e2e -- --shard=${{ matrix.shard }}/5 | |
| env: | |
| NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} | |
| NEXT_VERSION: ${{ matrix.version }} | |
| NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }} | |
| - name: Upload blob report to GitHub Actions Artifacts | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: blob-report-${{matrix.version}}-${{ matrix.shard }} | |
| path: blob-report | |
| retention-days: 1 | |
| test: | |
| needs: setup | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: [1, 2, 3, 4, 5, 6, 7, 8] | |
| os: [ubuntu-latest, windows-latest] | |
| version: ${{ fromJson(needs.setup.outputs.matrix) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: 'Install Node' | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '18.x' | |
| cache: 'npm' | |
| cache-dependency-path: '**/package-lock.json' | |
| - name: setup pnpm/yarn | |
| run: corepack enable | |
| shell: bash | |
| - name: Install Deno | |
| uses: denoland/setup-deno@v1 | |
| with: | |
| # Should match the `DENO_VERSION_RANGE` from https://github.com/netlify/edge-bundler/blob/e55f825bd985d3c92e21d1b765d71e70d5628fba/node/bridge.ts#L17 | |
| deno-version: v1.37.0 | |
| - name: 'Install dependencies' | |
| run: npm ci | |
| - name: 'Build' | |
| run: npm run build | |
| - name: Resolve Next.js version | |
| id: resolve-next-version | |
| shell: bash | |
| run: | | |
| RESOLVED_VERSION=$(npm view next@${{ matrix.version }} version) | |
| echo "version=$RESOLVED_VERSION" >> $GITHUB_OUTPUT | |
| echo "Resolved Next.js version for 'next@${{ matrix.version }}' is '$RESOLVED_VERSION'" | |
| - name: Compute Fixtures Cache Key | |
| id: fixture-cache-key | |
| # Fixtures only need to be rebuilt if either fixture or support files change, | |
| # so we're using a hash of the fixtures and support files as the cache key. | |
| run: | |
| echo "key=$(git ls-files -s tests/fixtures/ tests/utils/ tests/prepare.mjs | git hash-object --stdin)" | |
| >> "$GITHUB_OUTPUT" | |
| shell: bash | |
| - name: Cache Fixtures | |
| id: cache-fixtures | |
| uses: actions/cache@v4 | |
| with: | |
| path: tests/fixtures | |
| key: | |
| integration-fixtures-${{ runner.os }}-${{steps.resolve-next-version.outputs.version}}-${{ | |
| steps.fixture-cache-key.outputs.key }} | |
| - name: 'Prepare Fixtures' | |
| if: steps.cache-fixtures.outputs.cache-hit != 'true' | |
| run: npm run pretest | |
| env: | |
| NEXT_VERSION: ${{ matrix.version }} | |
| NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }} | |
| - name: 'Unit and integration tests' | |
| run: npm run test:ci:unit-and-integration -- --shard=${{ matrix.shard }}/8 | |
| env: | |
| NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} | |
| NEXT_VERSION: ${{ matrix.version }} | |
| NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }} | |
| TEMP: ${{ github.workspace }}/.. | |
| smoke: | |
| if: always() | |
| needs: setup | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: ${{ fromJson(needs.setup.outputs.matrix) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: 'Install Node' | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '18.x' | |
| cache: 'npm' | |
| cache-dependency-path: '**/package-lock.json' | |
| - name: setup pnpm/yarn | |
| run: corepack enable | |
| shell: bash | |
| - name: Install Deno | |
| uses: denoland/setup-deno@v1 | |
| with: | |
| # Should match the `DENO_VERSION_RANGE` from https://github.com/netlify/edge-bundler/blob/main/node/bridge.ts#L17 | |
| deno-version: v1.37.0 | |
| - name: 'Install dependencies' | |
| run: npm ci | |
| - name: 'Build' | |
| run: npm run build | |
| - name: 'Netlify Login' | |
| env: | |
| NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} | |
| run: | | |
| # TODO(serhalp) Unpin version once | |
| # https://netlify.slack.com/archives/C07686YAY13/p1718139257978049 is resolved. | |
| npm i -g [email protected] | |
| netlify login | |
| - name: Resolve Next.js version | |
| id: resolve-next-version | |
| shell: bash | |
| run: | | |
| RESOLVED_VERSION=$(npm view next@${{ matrix.version }} version) | |
| echo "version=$RESOLVED_VERSION" >> $GITHUB_OUTPUT | |
| echo "Resolved Next.js version for 'next@${{ matrix.version }}' is '$RESOLVED_VERSION'" | |
| - name: 'Smoke tests' | |
| run: npm run test:ci:smoke | |
| env: | |
| NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} | |
| NEXT_VERSION: ${{ matrix.version }} | |
| NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }} | |
| merge-reports: | |
| if: always() | |
| needs: [setup,e2e] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: ${{ fromJson(needs.setup.outputs.matrix) }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 18 | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Download blob reports from GitHub Actions Artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: all-blob-reports | |
| pattern: blob-report-${{ matrix.version }}-* | |
| merge-multiple: true | |
| - name: Merge reports | |
| run: | | |
| npx playwright merge-reports --reporter html ./all-blob-reports | |
| npx playwright merge-reports --reporter json ./all-blob-reports > merged_reports.json | |
| npx playwright merge-reports --reporter list ./all-blob-reports | |
| # FIXME(serhalp) Once we recover access to the SquidlifyBot Slack App, reenable this. | |
| # - name: Notify Slack | |
| # if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.head_ref == 'main' | |
| # run: npx playwright-slack-report -j merged_reports.json -c tests/playwright-slack-conf.json | |
| # env: | |
| # SLACK_BOT_USER_OAUTH_TOKEN: ${{ secrets.SLACK_BOT_USER_OAUTH_TOKEN }} | |
| # RESULTS_VERSION: ${{ matrix.version }} | |
| # RESULTS_URL: ${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}} | |
| - name: Upload HTML report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: html-report-${{ matrix.version }}-attempt-${{ github.run_attempt }} | |
| path: playwright-report | |
| retention-days: 14 |