test-e2e.yml

name: Run Next.js repo tests

on:
  workflow_call:
  pull_request:
    branches: [main]
  schedule:
    - cron: '0 3 * * *' # Run every day at 3am UTC
  workflow_dispatch:
    inputs:
      versions:
        description: 'The versions of Next.js to test against (escape-quoted and comma separated)'
        required: false
        # TODO(serhalp) Ideally this would simply accept bare quotes but we're having trouble
        # parsing that so this will do for now.
        default: "\\\"latest\\\""
concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
  cancel-in-progress: true

env:
  NODE_VERSION: 18.18.0
  PNPM_VERSION: 8.9.0
  NEXT_REPO: vercel/next.js
  NEXT_TEST_MODE: deploy
  NEXT_JUNIT_TEST_REPORT: true
  # In older versions of Next.js both of these need to be set to enable junit reporting
  DATADOG_TRACE_NEXTJS_TEST: true
  DATADOG_API_KEY: foo
  TEST_CONCURRENCY: 2
  NEXT_E2E_TEST_TIMEOUT: 300000
  NEXT_TELEMETRY_DISABLED: 1
  NEXT_SKIP_NATIVE_POSTINSTALL: 1
  TURBO_API: ${{ secrets.TURBO_API }}
  TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
  TURBO_TEAM: next-runtime-minimal
  TURBO_TEAMID: team_netlify
  NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
  NETLIFY_SITE_ID: 1d5a5c76-d445-4ae5-b694-b0d3f2e2c395
  NEXT_TEST_CONTINUE_ON_ERROR: 1
  next-path: next.js
  runtime-path: next-runtime
  GH_TOKEN: ${{ github.token }}
jobs:
  setup:
    runs-on: ubuntu-latest
    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-e2e-tests')) }}
    outputs:
      version_spec: ${{ steps.set-matrix.outputs.version_spec }}
      group: ${{ steps.set-matrix.outputs.group }}
      total: ${{ steps.set-matrix.outputs.total }}
    steps:

      - name: Set Next.js versions to test
        id: set-matrix
        run: |
          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
            VERSION_SELECTORS=[${{ github.event.inputs.versions }}]
            echo "group=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]" >> $GITHUB_OUTPUT
            echo "total=12" >> $GITHUB_OUTPUT
          elif [ "${{ github.event_name }}" == "pull_request" ]; then
            VERSION_SELECTORS=[\"latest\"]
            echo "group=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]" >> $GITHUB_OUTPUT
            echo "total=12" >> $GITHUB_OUTPUT
          else
            VERSION_SELECTORS=[\"latest\",\"canary\",\"13.5.1\"]
            echo "group=[1, 2, 3, 4]" >> $GITHUB_OUTPUT
            echo "total=4" >> $GITHUB_OUTPUT
          fi

          VERSION_SPEC="["
          for QUOTED_SELECTOR in $(echo $VERSION_SELECTORS | jq -c '.[]'); do
            SELECTOR=$(echo $QUOTED_SELECTOR | xargs)
            if [ "${VERSION_SPEC}" != "[" ]; then
              VERSION_SPEC+=","
            fi
            VERSION_SPEC+="{\"selector\":\"$SELECTOR\""

            if [ "$SELECTOR" == "latest" ]; then
              VERSION_SPEC+=",\"tag\":\"$(curl -s https://api.github.com/repos/vercel/next.js/releases/latest | jq -r .tag_name)\""
            elif [ "$SELECTOR" == "canary" ]; then
              VERSION_SPEC+=",\"tag\":\"$(curl -s https://api.github.com/repos/vercel/next.js/releases | jq -r '.[] | select(.prerelease == true) | .tag_name' | head -n 1)\""
            else
              VERSION_SPEC+=",\"tag\":\"v$SELECTOR\""
            fi

            VERSION_SPEC+=",\"version\":\"$(npm view next@$SELECTOR version)\""

            VERSION_SPEC+="}"
          done
          VERSION_SPEC+="]"
          echo "version_spec=$VERSION_SPEC" >> $GITHUB_OUTPUT

  e2e:
    needs: setup
    runs-on: ubuntu-latest
    name: Test next@${{ matrix.version_spec.selector }} group ${{ matrix.group }}/${{ needs.setup.outputs.total }}
    timeout-minutes: 120
    strategy:
      fail-fast: false
      matrix:
        version_spec: ${{ fromJson(needs.setup.outputs.version_spec) }}
        group: ${{ fromJson(needs.setup.outputs.group) }}

    steps:
      - name: checkout Next.js repo
        uses: actions/checkout@v4
        with:
          repository: ${{ env.NEXT_REPO }}
          path: ${{ env.next-path }}
          ref: ${{ matrix.version_spec.tag }}

      - name: checkout runtime repo
        uses: actions/checkout@v4
        with:
          path: ${{ env.runtime-path }}

      - name: setup node
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}

      - name: setup pnpm/yarn
        run: corepack enable

      - name: get pnpm store
        id: pnpm-store
        run: echo "PATH=$(pnpm store path --silent)" >> ${GITHUB_OUTPUT}

      - name: cache pnpm deps
        uses: actions/cache@v4
        with:
          path: ${{ steps.pnpm-store.outputs.PATH }}
          key: pnpm-store-${{ hashFiles('next.js/pnpm-lock.yaml') }}
          restore-keys: |
            pnpm-store-

      - name: get npm cache
        id: npm-cache
        run: echo "PATH=$(npm config get cache)" >> ${GITHUB_OUTPUT}

      - name: cache npm deps
        uses: actions/cache@v4
        with:
          path: ${{ steps.npm-cache.outputs.PATH }}
          key: node-cache-${{ hashFiles('next-runtime/package-lock.json') }}
          restore-keys: |
            node-cache-

      - name: patch Next.js
        run: |
          cp ../${{ env.runtime-path }}/tests/netlify-deploy.ts test/lib/next-modes/
          git apply ../${{ env.runtime-path }}/tests/e2e-utils.patch
        working-directory: ${{ env.next-path }}

      - name: install Next.js
        run: pnpm install
        working-directory: ${{ env.next-path }}

      - name: build Next.js
        run: pnpm build
        working-directory: ${{ env.next-path }}

      - name: install swc
        run: pnpm add --workspace-root @next/swc-linux-x64-gnu@${{ matrix.version_spec.version }}
        working-directory: ${{ env.next-path }}

      - name: Install Deno
        uses: denoland/setup-deno@v1
        with:
          # Should match the `DENO_VERSION_RANGE` from https://github.com/netlify/edge-bundler/blob/main/node/bridge.ts#L17
          deno-version: v1.37.0

      - name: install runtime
        run: npm install --ignore-scripts
        working-directory: ${{ env.runtime-path }}

      - name: build runtime
        run: npm run build
        working-directory: ${{ env.runtime-path }}

      - name: add netlify cli
        run: npm install -g netlify-cli

      - name: Install Playwright Browsers
        run: npx playwright install --with-deps
        working-directory: ${{ env.next-path }}

      - name: Get test filters
        id: test-filters
        run: |
          # This is when the manifest version was changed
          if [ `npx semver -p -r ">=14.0.4-canary.26" ${{ matrix.version_spec.version }}` ]; then
            echo "filters=../next-runtime/tests/netlify-e2e.cjs" >> $GITHUB_OUTPUT
            echo "skip-retry=../next-runtime/tests/e2e-skip-retry.json" >> $GITHUB_OUTPUT
          else
            echo "filters=../next-runtime/tests/netlify-e2e-legacy.json" >> $GITHUB_OUTPUT
            echo "skip-retry=../next-runtime/tests/e2e-skip-retry-legacy.json" >> $GITHUB_OUTPUT
          fi

      - name: run tests
        env:
          NODE_ENV: production
          NEXT_EXTERNAL_TESTS_FILTERS: ${{ steps.test-filters.outputs.filters }}
          NEXT_TEST_SKIP_RETRY_MANIFEST: ${{ steps.test-filters.outputs.skip-retry }}
        run: node run-tests.js -g ${{ matrix.group }}/${{ needs.setup.outputs.total }} -c ${TEST_CONCURRENCY} --type e2e --timings
        working-directory: ${{ env.next-path }}

      - name: Upload Test Results
        if: success() || failure()
        uses: actions/upload-artifact@v4
        with:
          name: test-result-${{matrix.version_spec.selector}}-${{ matrix.group }}
          path: ${{ env.next-path }}/test/test-junit-report/*.xml
  publish-test-results:
    name: 'E2E Test Summary (${{matrix.version_spec.selector}})'
    needs:
      - e2e
      - setup
    runs-on: ubuntu-latest
    permissions:
      checks: write
      contents: read
      issues: read
    if: success() || failure()
    strategy:
      matrix:
        version_spec: ${{ fromJson(needs.setup.outputs.version_spec) }}

    steps:
      - name: checkout runtime repo
        uses: actions/checkout@v4

      - name: Install Deno
        uses: denoland/setup-deno@v1

      - name: Download Artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: test-result-${{matrix.version_spec.selector}}-*
          path: artifacts

      - name: Get and format Github issues
        if: success() || failure()
        run: |
          gh issue list --label "Next.js e2e test failure" --json url,body > report/issues.json
          deno run -A tools/deno/ghIssues2json.ts tests/test-config.json report/issues.json

      - name: Publish Test Report
        id: publish-test-results
        if: success() || failure()
        run: |
          echo "slackEvent<<NETLIFY_EOF" >> $GITHUB_OUTPUT
          deno run -A tools/deno/junit2slack.ts --dir artifacts --version ${{matrix.version_spec.selector}} --runUrl ${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}} >> $GITHUB_OUTPUT
          echo "NETLIFY_EOF" >> $GITHUB_OUTPUT
          deno run -A tools/deno/junit2json.ts artifacts ${{ matrix.version_spec.tag }} > report/test-results.json
          deno run -A tools/deno/generate-md.ts >> $GITHUB_STEP_SUMMARY

      - name: Upload Test JSON
        uses: actions/upload-artifact@v4
        with:
          # TODO(serhalp) Consider renaming this. It's misleading, since it's just an identifier,
          # but it's formatted like a filename, and happens to be almost - but not quite - the
          # actual filename.
          name: ${{matrix.version_spec.selector}}-test-results.json
          path: report/test-results.json

      # FIXME(serhalp) Once we recover access to the SquidlifyBot Slack App, reenable this.
      # - name: Notify Slack
      #   if: ${{ (success() || failure()) && github.event_name == 'schedule' }}
      #   uses: slackapi/slack-github-action@v1.26.0
      #   env:
      #     SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
      #     SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
      #   with:
      #     payload: ${{ steps.publish-test-results.outputs.slackEvent }}