From f0b45c44065f90296c08b5a1e619c7780194f072 Mon Sep 17 00:00:00 2001 From: Rob Stanford Date: Mon, 2 Dec 2024 14:33:20 +0000 Subject: [PATCH 1/7] feat: add circleci config for running tests --- .circleci/config.yml | 250 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..1484b37f3 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,250 @@ +version: 2.1 + +orbs: + node: circleci/node@6.3.0 + bun: ksylvest/bun@1.0.1 + deno: ksylvest/deno@1.0.0 + +executors: + ubuntu: + machine: + image: 'ubuntu-2404:2024.11.1' + resource_class: xlarge + windows: + machine: + image: 'windows-server-2022-gui:2024.07.1' + resource_class: windows.xlarge + shell: powershell.exe -ExecutionPolicy Bypass + +parameters: + # Comma separated list, e.g. canary, latest, 14.2.15, 13.5.1 + next-versions: + type: string + default: latest + # Should match the buildbot node version + node-version: + type: string + default: '18.20' + # Should match the DENO_VERSION_RANGE from https://github.com/netlify/edge-bundler/blob/main/node/bridge.ts#L17 + deno-version: + type: string + default: '1.37.0' + +jobs: + e2e: + parameters: + next-version: + type: string + shard: + type: integer + executor: ubuntu + steps: + - checkout + - node/install: + install-yarn: true + install-pnpm: true + node-version: << pipeline.parameters.node-version >> + - bun/install + - deno/install + - run: + name: Set Deno Version + command: deno upgrade --version << pipeline.parameters.deno-version >> + - node/install-packages + - run: + name: Install Netlify CLI + environment: + NETLIFY_AUTH_TOKEN: $NETLIFY_AUTH_TOKEN + command: | + # Control netlify-cli as a regular dev dep but expose it globally for test fixtures to use + npm install -g "netlify-cli@$(npm list --json --depth=0 netlify-cli | jq -r ".dependencies[\"netlify-cli\"].version")" + npx netlify login + - run: + name: Get Playwright Version + command: | + PLAYWRIGHT_VERSION=$(npm view @playwright/test version) + echo "export PLAYWRIGHT_VERSION=$PLAYWRIGHT_VERSION" >> $BASH_ENV + - restore_cache: + name: Restore Playwright Cache + key: v1-{{ arch }}-playwright-{{ .Environment.PLAYWRIGHT_VERSION }} + - run: + name: Install Playwright Browsers + command: | + if [ ! -d ~/.cache/ms-playwright ]; then + npx playwright install --with-deps + fi + - save_cache: + name: Save Playwright Cache + paths: + - ~/.cache/ms-playwright + key: v1-{{ arch }}-playwright-{{ .Environment.PLAYWRIGHT_VERSION }} + - run: + name: Resolve Next.js version + command: | + RESOLVED_VERSION=$(npm view next@${NEXT_VERSION} version) + echo "export RESOLVED_VERSION=$RESOLVED_VERSION" >> $BASH_ENV + echo "Resolved Next.js version for 'next@${NEXT_VERSION}' is '$RESOLVED_VERSION'" + - run: + name: Run Playwright tests + command: npm run test:ci:e2e -- --shard=<< parameters.shard >>/5 + environment: + NETLIFY_AUTH_TOKEN: $NETLIFY_AUTH_TOKEN + NEXT_VERSION: << parameters.next-version >> + NEXT_RESOLVED_VERSION: $RESOLVED_VERSION + PLAYWRIGHT_BLOB_OUTPUT_NAME: + report-<< parameters.next-version >>-<< parameters.shard >>.zip + - persist_to_workspace: + name: Upload Blob Report to CircleCI Artifacts + root: . + paths: + - ./blob-reports + + test: + parameters: + next-version: + type: string + os: + type: executor + default: ubuntu + shard: + type: integer + executor: << parameters.os >> + steps: + - checkout + - node/install: + install-yarn: true + install-pnpm: true + node-version: << pipeline.parameters.node-version >> + - deno/install + - run: + name: Set Deno Version + command: deno upgrade --version << pipeline.parameters.deno-version >> + - node/install-packages + - run: + name: Build + command: npm run build + - run: + name: Resolve Next.js version + command: | + RESOLVED_VERSION=$(npm view next@${NEXT_VERSION} version) + echo "export RESOLVED_VERSION=$RESOLVED_VERSION" >> $BASH_ENV + echo "Resolved Next.js version for 'next@${NEXT_VERSION}' is '$RESOLVED_VERSION'" + - run: + name: Compute Fixtures Cache Key + command: | + CACHE_KEY=$(git ls-files -s tests/fixtures/ tests/utils/ tests/prepare.mjs | git hash-object --stdin) + echo "export CACHE_KEY=$CACHE_KEY" >> $BASH_ENV + - restore_cache: + name: Restore Fixtures Cache + key: + v1-{{ arch }}-fixtures-{{ .Environment.RESOLVED_VERSION }}-{{ .Environment.CACHE_KEY }} + - run: + name: Prepare Fixtures + command: | + if [ ! -d tests/fixtures ]; then + npm run pretest + fi + environment: + NEXT_VERSION: << parameters.next-version >> + NEXT_RESOLVED_VERSION: $RESOLVED_VERSION + - save_cache: + name: Save Fixtures Cache + paths: + - tests/fixtures + key: + v1-{{ arch }}-fixtures-{{ .Environment.RESOLVED_VERSION }}-{{ .Environment.CACHE_KEY }} + - run: + name: Run Unit and Integration Tests + command: npm run test:ci:unit-and-integration -- --shard=<< parameters.shard >>/8 + environment: + NETLIFY_AUTH_TOKEN: $NETLIFY_AUTH_TOKEN + NEXT_VERSION: << parameters.next-version >> + NEXT_RESOLVED_VERSION: $RESOLVED_VERSION + TEMP: /tmp + + smoke: + parameters: + next-version: + type: string + executor: ubuntu + steps: + - checkout + - node/install: + install-yarn: true + install-pnpm: true + node-version: << pipeline.parameters.node-version >> + - deno/install + - run: + name: Set Deno Version + command: deno upgrade --version << pipeline.parameters.deno-version >> + - node/install-packages + - run: + name: Build + command: npm run build + - run: + name: Install Netlify CLI + environment: + NETLIFY_AUTH_TOKEN: $NETLIFY_AUTH_TOKEN + command: | + # Control netlify-cli as a regular dev dep but expose it globally for test fixtures to use + npm install -g "netlify-cli@$(npm list --json --depth=0 netlify-cli | jq -r ".dependencies[\"netlify-cli\"].version")" + npx netlify login + - run: + name: Resolve Next.js version + command: | + RESOLVED_VERSION=$(npm view next@${NEXT_VERSION} version) + echo "export RESOLVED_VERSION=$RESOLVED_VERSION" >> $BASH_ENV + echo "Resolved Next.js version for 'next@${NEXT_VERSION}' is '$RESOLVED_VERSION'" + - run: + name: Run Smoke Tests + command: npm run test:ci:smoke + environment: + NETLIFY_AUTH_TOKEN: $NETLIFY_AUTH_TOKEN + NEXT_VERSION: << parameters.next-version >> + NEXT_RESOLVED_VERSION: $RESOLVED_VERSION + + merge-reports: + parameters: + next-version: + type: string + executor: ubuntu + steps: + - checkout + - node/install: + node-version: << pipeline.parameters.node-version >> + - node/install-packages + - attach_workspace: + at: . + - run: + name: Merge reports + command: npx playwright merge-reports --reporter html ./blob-reports + - store_artifacts: + path: playwright-report + destination: html-report-<< parameters.next-version >>-${ .Environment.CIRCLE_BUILD_NUM} + +workflows: + 'Run adapter tests': + jobs: + - e2e: + context: [shared-secrets] + matrix: + parameters: + next-version: [<< pipeline.parameters.next-versions >>] + shard: [1, 2, 3, 4, 5] + - test: + context: [shared-secrets] + matrix: + parameters: + next-version: [<< pipeline.parameters.next-versions >>] + os: [ubuntu, windows] + shard: [1, 2, 3, 4, 5, 6, 7, 8] + - smoke: + context: [shared-secrets] + matrix: + parameters: + next-version: [<< pipeline.parameters.next-versions >>] + - merge-reports: + requires: [e2e] + context: [shared-secrets] + matrix: + parameters: + next-version: [<< pipeline.parameters.next-versions >>] From 1969bab373cfae7cd52b84d42827bbc142ab7b7c Mon Sep 17 00:00:00 2001 From: Rob Stanford Date: Wed, 4 Dec 2024 15:27:05 +0000 Subject: [PATCH 2/7] fix: circleci config edits --- .circleci/config.yml | 50 +++++++++++++++++++------------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1484b37f3..efc73d50b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,9 +35,8 @@ jobs: parameters: next-version: type: string - shard: - type: integer executor: ubuntu + parallelism: 2 steps: - checkout - node/install: @@ -62,10 +61,10 @@ jobs: name: Get Playwright Version command: | PLAYWRIGHT_VERSION=$(npm view @playwright/test version) - echo "export PLAYWRIGHT_VERSION=$PLAYWRIGHT_VERSION" >> $BASH_ENV + echo $PLAYWRIGHT_VERSION > /tmp/playwright-version - restore_cache: name: Restore Playwright Cache - key: v1-{{ arch }}-playwright-{{ .Environment.PLAYWRIGHT_VERSION }} + key: v1-{{ arch }}-playwright-{{ checksum "/tmp/playwright-version" }} - run: name: Install Playwright Browsers command: | @@ -76,22 +75,24 @@ jobs: name: Save Playwright Cache paths: - ~/.cache/ms-playwright - key: v1-{{ arch }}-playwright-{{ .Environment.PLAYWRIGHT_VERSION }} + key: v1-{{ arch }}-playwright-{{ checksum "/tmp/playwright-version" }} - run: name: Resolve Next.js version command: | - RESOLVED_VERSION=$(npm view next@${NEXT_VERSION} version) + RESOLVED_VERSION=$(npm view next@<< parameters.next-version >> version) echo "export RESOLVED_VERSION=$RESOLVED_VERSION" >> $BASH_ENV - echo "Resolved Next.js version for 'next@${NEXT_VERSION}' is '$RESOLVED_VERSION'" + echo "Resolved Next.js version for 'next@<< parameters.next-version >>' is '$RESOLVED_VERSION'" - run: name: Run Playwright tests - command: npm run test:ci:e2e -- --shard=<< parameters.shard >>/5 + command: | + SHARD=$((${CIRCLE_NODE_INDEX}+1)) + npm run test:ci:e2e -- --shard=${SHARD}/${CIRCLE_NODE_TOTAL} environment: NETLIFY_AUTH_TOKEN: $NETLIFY_AUTH_TOKEN NEXT_VERSION: << parameters.next-version >> NEXT_RESOLVED_VERSION: $RESOLVED_VERSION PLAYWRIGHT_BLOB_OUTPUT_NAME: - report-<< parameters.next-version >>-<< parameters.shard >>.zip + report-<< parameters.next-version >>-{{ .Environment.CIRCLE_NODE_INDEX }}.zip - persist_to_workspace: name: Upload Blob Report to CircleCI Artifacts root: . @@ -105,9 +106,8 @@ jobs: os: type: executor default: ubuntu - shard: - type: integer executor: << parameters.os >> + parallelism: 4 steps: - checkout - node/install: @@ -125,18 +125,17 @@ jobs: - run: name: Resolve Next.js version command: | - RESOLVED_VERSION=$(npm view next@${NEXT_VERSION} version) + RESOLVED_VERSION=$(npm view next@<< parameters.next-version >> version) echo "export RESOLVED_VERSION=$RESOLVED_VERSION" >> $BASH_ENV - echo "Resolved Next.js version for 'next@${NEXT_VERSION}' is '$RESOLVED_VERSION'" + echo "Resolved Next.js version for 'next@<< parameters.next-version >>' is '$RESOLVED_VERSION'" - run: name: Compute Fixtures Cache Key command: | CACHE_KEY=$(git ls-files -s tests/fixtures/ tests/utils/ tests/prepare.mjs | git hash-object --stdin) - echo "export CACHE_KEY=$CACHE_KEY" >> $BASH_ENV + echo "{$RESOLVED_VERSION}-{$CACHE_KEY}" > /tmp/fixtures-cache-key - restore_cache: name: Restore Fixtures Cache - key: - v1-{{ arch }}-fixtures-{{ .Environment.RESOLVED_VERSION }}-{{ .Environment.CACHE_KEY }} + key: v1-{{ arch }}-fixtures-{{ checksum "/tmp/fixtures-cache-key" }} - run: name: Prepare Fixtures command: | @@ -150,11 +149,12 @@ jobs: name: Save Fixtures Cache paths: - tests/fixtures - key: - v1-{{ arch }}-fixtures-{{ .Environment.RESOLVED_VERSION }}-{{ .Environment.CACHE_KEY }} + key: v1-{{ arch }}-fixtures-{{ checksum "/tmp/fixtures-cache-key" }} - run: name: Run Unit and Integration Tests - command: npm run test:ci:unit-and-integration -- --shard=<< parameters.shard >>/8 + command: | + SHARD=$((${CIRCLE_NODE_INDEX}+1)) + npm run test:ci:unit-and-integration -- --shard=${SHARD}/${CIRCLE_NODE_TOTAL} environment: NETLIFY_AUTH_TOKEN: $NETLIFY_AUTH_TOKEN NEXT_VERSION: << parameters.next-version >> @@ -191,9 +191,9 @@ jobs: - run: name: Resolve Next.js version command: | - RESOLVED_VERSION=$(npm view next@${NEXT_VERSION} version) + RESOLVED_VERSION=$(npm view next@<< parameters.next-version >> version) echo "export RESOLVED_VERSION=$RESOLVED_VERSION" >> $BASH_ENV - echo "Resolved Next.js version for 'next@${NEXT_VERSION}' is '$RESOLVED_VERSION'" + echo "Resolved Next.js version for 'next@<< parameters.next-version >>' is '$RESOLVED_VERSION'" - run: name: Run Smoke Tests command: npm run test:ci:smoke @@ -222,29 +222,23 @@ jobs: destination: html-report-<< parameters.next-version >>-${ .Environment.CIRCLE_BUILD_NUM} workflows: - 'Run adapter tests': + build-and-test-adapter: jobs: - e2e: - context: [shared-secrets] matrix: parameters: next-version: [<< pipeline.parameters.next-versions >>] - shard: [1, 2, 3, 4, 5] - test: - context: [shared-secrets] matrix: parameters: next-version: [<< pipeline.parameters.next-versions >>] os: [ubuntu, windows] - shard: [1, 2, 3, 4, 5, 6, 7, 8] - smoke: - context: [shared-secrets] matrix: parameters: next-version: [<< pipeline.parameters.next-versions >>] - merge-reports: requires: [e2e] - context: [shared-secrets] matrix: parameters: next-version: [<< pipeline.parameters.next-versions >>] From 19f6e2790e31ad5ba07425ff1b9f51edd3ecac58 Mon Sep 17 00:00:00 2001 From: Rob Stanford Date: Wed, 4 Dec 2024 15:31:06 +0000 Subject: [PATCH 3/7] feat: downgrade circleci machine class for testing --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index efc73d50b..625bb73aa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,11 +9,11 @@ executors: ubuntu: machine: image: 'ubuntu-2404:2024.11.1' - resource_class: xlarge + resource_class: large windows: machine: image: 'windows-server-2022-gui:2024.07.1' - resource_class: windows.xlarge + resource_class: windows.large shell: powershell.exe -ExecutionPolicy Bypass parameters: From c174ffb3d6ac74c8a412ee70b6f8332d776ef9c6 Mon Sep 17 00:00:00 2001 From: Rob Stanford Date: Wed, 4 Dec 2024 16:01:02 +0000 Subject: [PATCH 4/7] fix: corepack enable in circleci --- .circleci/config.yml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 625bb73aa..6d2364c32 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,9 +40,8 @@ jobs: steps: - checkout - node/install: - install-yarn: true - install-pnpm: true node-version: << pipeline.parameters.node-version >> + - run: corepack enable - bun/install - deno/install - run: @@ -88,9 +87,7 @@ jobs: SHARD=$((${CIRCLE_NODE_INDEX}+1)) npm run test:ci:e2e -- --shard=${SHARD}/${CIRCLE_NODE_TOTAL} environment: - NETLIFY_AUTH_TOKEN: $NETLIFY_AUTH_TOKEN NEXT_VERSION: << parameters.next-version >> - NEXT_RESOLVED_VERSION: $RESOLVED_VERSION PLAYWRIGHT_BLOB_OUTPUT_NAME: report-<< parameters.next-version >>-{{ .Environment.CIRCLE_NODE_INDEX }}.zip - persist_to_workspace: @@ -111,9 +108,8 @@ jobs: steps: - checkout - node/install: - install-yarn: true - install-pnpm: true node-version: << pipeline.parameters.node-version >> + - run: corepack enable - deno/install - run: name: Set Deno Version @@ -156,9 +152,7 @@ jobs: SHARD=$((${CIRCLE_NODE_INDEX}+1)) npm run test:ci:unit-and-integration -- --shard=${SHARD}/${CIRCLE_NODE_TOTAL} environment: - NETLIFY_AUTH_TOKEN: $NETLIFY_AUTH_TOKEN NEXT_VERSION: << parameters.next-version >> - NEXT_RESOLVED_VERSION: $RESOLVED_VERSION TEMP: /tmp smoke: @@ -169,9 +163,8 @@ jobs: steps: - checkout - node/install: - install-yarn: true - install-pnpm: true node-version: << pipeline.parameters.node-version >> + - run: corepack enable - deno/install - run: name: Set Deno Version @@ -198,9 +191,7 @@ jobs: name: Run Smoke Tests command: npm run test:ci:smoke environment: - NETLIFY_AUTH_TOKEN: $NETLIFY_AUTH_TOKEN NEXT_VERSION: << parameters.next-version >> - NEXT_RESOLVED_VERSION: $RESOLVED_VERSION merge-reports: parameters: From 889edd95d5a0553cd679adc212f322275436cfcf Mon Sep 17 00:00:00 2001 From: Rob Stanford Date: Wed, 4 Dec 2024 16:18:14 +0000 Subject: [PATCH 5/7] fix: circleci parallelism and windows shell --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6d2364c32..284ea60a9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ executors: machine: image: 'windows-server-2022-gui:2024.07.1' resource_class: windows.large - shell: powershell.exe -ExecutionPolicy Bypass + shell: bash.exe parameters: # Comma separated list, e.g. canary, latest, 14.2.15, 13.5.1 @@ -36,7 +36,7 @@ jobs: next-version: type: string executor: ubuntu - parallelism: 2 + parallelism: 4 steps: - checkout - node/install: @@ -104,7 +104,7 @@ jobs: type: executor default: ubuntu executor: << parameters.os >> - parallelism: 4 + parallelism: 8 steps: - checkout - node/install: From ed45834cd97a7491b3ebe2d538c09c20e6ffc15a Mon Sep 17 00:00:00 2001 From: Rob Stanford Date: Thu, 5 Dec 2024 16:26:38 +0000 Subject: [PATCH 6/7] fix: circleci fixtures fix --- .circleci/config.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 284ea60a9..59546106b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -88,13 +88,14 @@ jobs: npm run test:ci:e2e -- --shard=${SHARD}/${CIRCLE_NODE_TOTAL} environment: NEXT_VERSION: << parameters.next-version >> + NETLIFY_AUTH_TOKEN: $NETLIFY_AUTH_TOKEN PLAYWRIGHT_BLOB_OUTPUT_NAME: report-<< parameters.next-version >>-{{ .Environment.CIRCLE_NODE_INDEX }}.zip - persist_to_workspace: name: Upload Blob Report to CircleCI Artifacts root: . paths: - - ./blob-reports + - ./blob-report test: parameters: @@ -135,7 +136,7 @@ jobs: - run: name: Prepare Fixtures command: | - if [ ! -d tests/fixtures ]; then + if [ ! -d tests/fixtures/simple/.next ]; then npm run pretest fi environment: @@ -153,6 +154,7 @@ jobs: npm run test:ci:unit-and-integration -- --shard=${SHARD}/${CIRCLE_NODE_TOTAL} environment: NEXT_VERSION: << parameters.next-version >> + NETLIFY_AUTH_TOKEN: $NETLIFY_AUTH_TOKEN TEMP: /tmp smoke: @@ -192,6 +194,7 @@ jobs: command: npm run test:ci:smoke environment: NEXT_VERSION: << parameters.next-version >> + NETLIFY_AUTH_TOKEN: $NETLIFY_AUTH_TOKEN merge-reports: parameters: From 0ebac585a89df665e168b61baba40a01c7c467ec Mon Sep 17 00:00:00 2001 From: Rob Stanford Date: Fri, 6 Dec 2024 13:04:54 +0000 Subject: [PATCH 7/7] fix: circleci attempt to fix missing site id --- .circleci/config.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 59546106b..dad036d02 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -51,6 +51,7 @@ jobs: - run: name: Install Netlify CLI environment: + NETLIFY_SITE_ID: $NETLIFY_SITE_ID NETLIFY_AUTH_TOKEN: $NETLIFY_AUTH_TOKEN command: | # Control netlify-cli as a regular dev dep but expose it globally for test fixtures to use @@ -88,6 +89,7 @@ jobs: npm run test:ci:e2e -- --shard=${SHARD}/${CIRCLE_NODE_TOTAL} environment: NEXT_VERSION: << parameters.next-version >> + NETLIFY_SITE_ID: $NETLIFY_SITE_ID NETLIFY_AUTH_TOKEN: $NETLIFY_AUTH_TOKEN PLAYWRIGHT_BLOB_OUTPUT_NAME: report-<< parameters.next-version >>-{{ .Environment.CIRCLE_NODE_INDEX }}.zip @@ -154,6 +156,7 @@ jobs: npm run test:ci:unit-and-integration -- --shard=${SHARD}/${CIRCLE_NODE_TOTAL} environment: NEXT_VERSION: << parameters.next-version >> + NETLIFY_SITE_ID: $NETLIFY_SITE_ID NETLIFY_AUTH_TOKEN: $NETLIFY_AUTH_TOKEN TEMP: /tmp @@ -178,6 +181,7 @@ jobs: - run: name: Install Netlify CLI environment: + NETLIFY_SITE_ID: $NETLIFY_SITE_ID NETLIFY_AUTH_TOKEN: $NETLIFY_AUTH_TOKEN command: | # Control netlify-cli as a regular dev dep but expose it globally for test fixtures to use @@ -194,6 +198,7 @@ jobs: command: npm run test:ci:smoke environment: NEXT_VERSION: << parameters.next-version >> + NETLIFY_SITE_ID: $NETLIFY_SITE_ID NETLIFY_AUTH_TOKEN: $NETLIFY_AUTH_TOKEN merge-reports: