Module ID global optimization #1789
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Turbopack Test | |
on: | |
push: | |
branches: [main] | |
pull_request: | |
concurrency: | |
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
permissions: | |
actions: write | |
contents: read | |
pull-requests: read | |
jobs: | |
determine_jobs: | |
name: Determine jobs to run | |
runs-on: ubuntu-latest | |
permissions: | |
contents: read | |
pull-requests: write | |
steps: | |
- name: Find PR Comment | |
id: comment | |
if: github.event_name == 'pull_request' | |
uses: peter-evans/find-comment@v2 | |
with: | |
issue-number: ${{ github.event.pull_request.number }} | |
comment-author: "github-actions[bot]" | |
body-includes: "<!-- CI COMMENT -->" | |
- name: Create or update PR comment | |
if: github.event_name == 'pull_request' && steps.comment.outputs.comment-id != '' | |
uses: peter-evans/create-or-update-comment@v2 | |
continue-on-error: true | |
with: | |
comment-id: ${{ steps.comment.outputs.comment-id }} | |
issue-number: ${{ github.event.pull_request.number }} | |
body: | | |
## ⏳ CI is running again... ⏳ | |
[Wait for it...](https://github.com/vercel/turbo/actions/runs/${{ github.run_id }}) | |
<!-- CI COMMENT --> | |
edit-mode: replace | |
- name: Checkout | |
uses: actions/checkout@v3 | |
- name: CI related changes | |
id: ci | |
uses: technote-space/get-diff-action@v6 | |
with: | |
PATTERNS: | | |
.github/actions/** | |
.github/workflows/turbopack-test.yml | |
.github/workflows/test-turbopack-rust-bench-test.yml | |
.github/workflows/pr-clean-caches.yml | |
- name: Root cargo related changes | |
id: cargo | |
uses: technote-space/get-diff-action@v6 | |
with: | |
PATTERNS: | | |
Cargo.* | |
rust-toolchain | |
- name: Rust related changes | |
id: rust | |
uses: technote-space/get-diff-action@v6 | |
with: | |
PATTERNS: | | |
pnpm-lock.yaml | |
package.json | |
Cargo.** | |
crates/** | |
shim/** | |
xtask/** | |
.cargo/** | |
rust-toolchain | |
!**.md | |
!**.mdx | |
- name: Turbopack related changes | |
id: turbopack | |
uses: technote-space/get-diff-action@v6 | |
with: | |
PATTERNS: | | |
pnpm-lock.yaml | |
package.json | |
crates/** | |
xtask/** | |
.cargo/** | |
rust-toolchain | |
!crates/turborepo*/** | |
!**.md | |
!**.mdx | |
- name: Turbopack Benchmark related changes | |
id: turbopack_bench | |
uses: technote-space/get-diff-action@v6 | |
with: | |
PATTERNS: | | |
crates/turbopack-bench/** | |
!*.md | |
- name: Turbopack TypeScript related changes | |
id: turbopack_typescript | |
uses: technote-space/get-diff-action@v6 | |
with: | |
PATTERNS: | | |
crates/turbopack*/js/** | |
- name: Turborepo Rust related changes | |
id: turborepo_rust | |
uses: technote-space/get-diff-action@v6 | |
with: | |
PATTERNS: | | |
pnpm-lock.yaml | |
package.json | |
crates/turborepo*/** | |
.cargo/** | |
rust-toolchain | |
!**.md | |
!**.mdx | |
- name: Formatting related changes | |
id: format | |
uses: technote-space/get-diff-action@v6 | |
with: | |
PATTERNS: | | |
**/*.{yml,yaml,md,mdx,js,jsx,ts,tsx,json,toml,css} | |
outputs: | |
turbopack_typescript: ${{ steps.ci.outputs.diff != '' || steps.turbopack_typescript.outputs.diff != '' }} | |
rust: ${{ steps.ci.outputs.diff != '' || steps.rust.outputs.diff != '' }} | |
turbopack: ${{ steps.ci.outputs.diff != '' || steps.turbopack.outputs.diff != '' }} | |
# We only test workspace dependency changes on main, not on PRs to speed up CI | |
cargo_on_main: ${{ steps.ci.outputs.diff != '' || (steps.cargo.outputs.diff != '' && github.event_name == 'push' && github.ref == 'refs/heads/main') }} | |
cargo_only: ${{ steps.ci.outputs.diff != '' || (steps.cargo.outputs.diff != '' && steps.turbopack.outputs.diff == '' && steps.turborepo_rust.outputs.diff == '') }} | |
turbopack_bench: ${{ steps.ci.outputs.diff != '' || steps.turbopack_bench.outputs.diff != '' }} | |
turbopack_typescript: | |
name: Turbopack TypeScript files | |
runs-on: | |
- "self-hosted" | |
- "linux" | |
- "x64" | |
- "metal" | |
needs: determine_jobs | |
if: needs.determine_jobs.outputs.turbopack_typescript == 'true' | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v3 | |
- name: Setup Node.js | |
uses: ./.github/actions/setup-node | |
- name: Install dependencies | |
run: pnpm install -r --side-effects-cache false | |
- name: Check turbopack-node types | |
working-directory: crates/turbopack-node/js | |
run: pnpm run check | |
- name: Check turbopack-cli types | |
working-directory: crates/turbopack-cli/js | |
run: pnpm run check | |
- name: Check turbopack-ecmascript-runtime types | |
working-directory: crates/turbopack-ecmascript-runtime/js | |
run: pnpm run check | |
turbopack_rust_check: | |
needs: [determine_jobs] | |
# We test dependency changes only on main | |
if: | | |
(needs.determine_jobs.outputs.rust == 'true' && needs.determine_jobs.outputs.turbopack == 'true') || | |
needs.determine_jobs.outputs.cargo_on_main == 'true' || | |
needs.determine_jobs.outputs.cargo_only == 'true' | |
name: Turbopack rust check | |
runs-on: | |
- "self-hosted" | |
- "linux" | |
- "x64" | |
- "metal" | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v3 | |
- name: Setup Rust | |
uses: ./.github/actions/setup-rust | |
with: | |
targets: wasm32-unknown-unknown,wasm32-wasip1-threads | |
github-token: "${{ secrets.GITHUB_TOKEN }}" | |
- name: Run cargo check release | |
run: | | |
RUSTFLAGS="-D warnings -A deprecated" cargo groups check turbopack --features rustls-tls --release | |
- name: Run cargo check for the wasi targets | |
run: | | |
CARGO_BUILD_TARGET="wasm32-wasip1-threads" RUSTFLAGS="-D warnings -A deprecated" cargo groups check turbopack-wasi --release | |
turbopack_rust_clippy: | |
needs: [turbopack_rust_check] | |
name: Turbopack rust clippy | |
runs-on: | |
- "self-hosted" | |
- "linux" | |
- "x64" | |
- "metal" | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v3 | |
- name: Setup Rust | |
uses: ./.github/actions/setup-rust | |
with: | |
targets: wasm32-unknown-unknown | |
github-token: "${{ secrets.GITHUB_TOKEN }}" | |
- name: Run cargo clippy | |
run: | | |
RUSTFLAGS="-D warnings -A deprecated" cargo groups clippy turbopack --features rustls-tls | |
- name: Run ast-grep lints | |
run: | | |
npx --package @ast-grep/cli -- ast-grep scan $(cargo groups list turbopack | awk '{ print $2 }' | tr '\n' ' ') | |
next_dev_check: | |
needs: [determine_jobs] | |
if: needs.determine_jobs.outputs.turbopack == 'true' || needs.determine_jobs.outputs.cargo_on_main == 'true' | |
name: Check next-swc | |
runs-on: | |
- "self-hosted" | |
- "linux" | |
- "x64" | |
- "metal" | |
permissions: | |
pull-requests: write | |
steps: | |
- name: Checkout Next.js | |
uses: actions/checkout@v3 | |
with: | |
repository: vercel/next.js | |
- name: Setup Rust | |
uses: ./.github/actions/setup-rust | |
with: | |
github-token: "${{ secrets.GITHUB_TOKEN }}" | |
- name: Build next-swc | |
run: | | |
export TURBOPACK_REMOTE="https://github.com/vercel/turbo" | |
# patch the turbo git dependency with the commit of the current PR. | |
export BINDING=$(printf 'patch."%s".%s.git="%s?rev=%s"' "$TURBOPACK_REMOTE" "turbopack-binding" "$TURBOPACK_REMOTE" "$GITHUB_SHA") | |
export TASKS=$(printf 'patch."%s".%s.git="%s?rev=%s"' "$TURBOPACK_REMOTE" "turbo-tasks" "$TURBOPACK_REMOTE" "$GITHUB_SHA") | |
export TASKS_FS=$(printf 'patch."%s".%s.git="%s?rev=%s"' "$TURBOPACK_REMOTE" "turbo-tasks-fs" "$TURBOPACK_REMOTE" "$GITHUB_SHA") | |
# set pipefail so the exit code of `cargo check` gets preserved | |
(set -o pipefail && \ | |
cargo check \ | |
--config $BINDING --config $TASKS --config $TASKS_FS \ | |
--all -p next-swc-api \ | |
--no-default-features --features rustls-tls \ | |
--message-format short --quiet \ | |
2>&1 | tee cargo_output.log) | |
- name: Success Log | |
run: | | |
printf ":white_check_mark: This change can build \`next-swc\`" > out.log | |
- name: Post logs if there are errors | |
if: failure() | |
env: | |
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
run: | | |
job_url=$(gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/jobs --paginate --jq '.jobs.[] | select(.name == "Check next-swc").html_url') | |
# strip ansi colors with `sed` | |
echo "\ | |
⚠️ This change may fail to build \`next-swc\`. | |
<details><summary>Logs</summary> | |
<p> | |
\`\`\` | |
$(cat cargo_output.log | sed -r "s/\x1B\[([0-9]{1,3}(;[0-9]{1,2};?)?)?[mGK]//g") | |
\`\`\` | |
</p> | |
</details> | |
See [job summary]($job_url) for details | |
" > out.log | |
- name: PR comment with file | |
uses: thollander/actions-comment-pull-request@v2 | |
if: always() && github.repository == 'vercel/turbo' | |
# We'll not block CI on this step | |
continue-on-error: true | |
with: | |
filePath: ./out.log | |
comment_tag: check_next_swc_turbopack | |
turbopack_rust_test1: | |
needs: [turbopack_rust_check] | |
runs-on: | |
- "self-hosted" | |
- "linux" | |
- "x64" | |
- "metal" | |
name: Turbopack Rust testing on ubuntu | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v3 | |
- name: Setup Rust | |
uses: ./.github/actions/setup-rust | |
with: | |
save-cache: true | |
github-token: "${{ secrets.GITHUB_TOKEN }}" | |
- name: Setup Node.js | |
uses: ./.github/actions/setup-node | |
with: | |
node-version: 18 | |
- name: Install tests dependencies | |
working-directory: crates/turbopack/tests/node-file-trace | |
run: pnpm install -r --side-effects-cache false | |
- name: Install tests dependencies in examples/with-yarn | |
working-directory: examples/with-yarn | |
run: npm install | |
- name: Install nextest | |
uses: taiki-e/install-action@v2 | |
env: | |
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
with: | |
tool: [email protected] | |
- name: Build nextest | |
timeout-minutes: 120 | |
run: | | |
cargo tp-pre-test | |
- name: Run nextest | |
timeout-minutes: 120 | |
run: | | |
cargo tp-test --profile tp-test-linux | |
- name: Upload results | |
if: always() | |
uses: actions/upload-artifact@v3 | |
with: | |
name: test_reports | |
path: target/nextest/**/*.xml | |
turbopack_rust_test2: | |
needs: [turbopack_rust_check] | |
strategy: | |
fail-fast: false | |
matrix: | |
os: | |
- name: macos | |
runner: macos-12 | |
- name: windows | |
runner: windows-latest | |
runs-on: ${{ matrix.os.runner }} | |
name: Turbopack Rust testing on ${{ matrix.os.name }} | |
steps: | |
- name: Set git to use LF line endings | |
run: | | |
git config --global core.autocrlf false | |
git config --global core.eol lf | |
if: matrix.os.name == 'windows' | |
- name: Checkout | |
uses: actions/checkout@v3 | |
- name: Set up Python | |
uses: actions/setup-python@v4 | |
with: | |
# TODO: Remove this specific version constraint once pnpm ships with | |
# node-gyp that's compatible with Python 3.12+ | |
python-version: 3.11 | |
- name: Setup Rust | |
uses: ./.github/actions/setup-rust | |
with: | |
save-cache: true | |
github-token: "${{ secrets.GITHUB_TOKEN }}" | |
- name: Setup Node.js | |
uses: ./.github/actions/setup-node | |
with: | |
node-version: 18 | |
- name: Prepare toolchain on Windows | |
run: | | |
pnpx node-gyp install | |
echo 'node-linker = "hoisted"' > crates/turbopack/tests/node-file-trace/.npmrc | |
if: matrix.os.name == 'windows' | |
- name: Install tests dependencies | |
working-directory: crates/turbopack/tests/node-file-trace | |
run: pnpm install -r --side-effects-cache false | |
- name: Install tests dependencies in examples/with-yarn | |
working-directory: examples/with-yarn | |
run: npm install | |
- name: Install nextest | |
uses: taiki-e/install-action@v2 | |
env: | |
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
with: | |
tool: [email protected] | |
- name: Build nextest | |
timeout-minutes: 120 | |
run: | | |
cargo tp-pre-test | |
- name: Run nextest (Mac) | |
timeout-minutes: 120 | |
if: matrix.os.name == 'macos' | |
run: | | |
cargo tp-test --profile tp-test-mac | |
- name: Run nextest (Windows) | |
timeout-minutes: 120 | |
if: matrix.os.name == 'windows' | |
run: | | |
cargo tp-test --profile tp-test-win | |
- name: Upload results | |
if: always() | |
uses: actions/upload-artifact@v3 | |
with: | |
name: test_reports | |
path: target/nextest/**/*.xml | |
turbopack_rust_test_bench1: | |
needs: [determine_jobs, turbopack_rust_check] | |
name: Turbopack Rust testing benchmarks on linux | |
uses: ./.github/workflows/test-turbopack-rust-bench-test.yml | |
with: | |
# custom runner label specific to workflow | |
runner: ubuntu-latest-metal | |
os: linux | |
all: ${{ needs.determine_jobs.outputs.turbopack_bench == 'true' }} | |
turbopack_rust_test_bench2: | |
needs: [determine_jobs, turbopack_rust_check] | |
strategy: | |
fail-fast: false | |
matrix: | |
os: | |
- name: macos | |
runner: macos-12 | |
# Temporarily disable windows bench due to consistent timeouts | |
# - name: windows | |
# runner: windows-2019 | |
name: Turbopack Rust testing benchmarks on ${{ matrix.os.name }} | |
uses: ./.github/workflows/test-turbopack-rust-bench-test.yml | |
with: | |
runner: ${{ matrix.os.runner }} | |
os: ${{ matrix.os.name }} | |
all: ${{ needs.determine_jobs.outputs.turbopack_bench == 'true' }} | |
final: | |
name: Ok | |
needs: | |
- determine_jobs | |
- turbopack_rust_check | |
- turbopack_rust_clippy | |
- turbopack_rust_test1 | |
- turbopack_rust_test_bench1 | |
- turbopack_typescript | |
if: always() | |
permissions: | |
contents: read | |
pull-requests: write | |
runs-on: ubuntu-latest | |
steps: | |
- name: Compute info | |
id: info | |
if: always() | |
run: | | |
cancelled=false | |
failure=false | |
subjob () { | |
local result=$1 | |
local name=$2 | |
if [ "$result" = "cancelled" ]; then | |
cancelled=true | |
elif [ "$result" != "success" ] && [ "$result" != "skipped" ]; then | |
echo "- $name" >> failures.md | |
failure=true | |
fi | |
} | |
subjob ${{needs.determine_jobs.result}} "Determining jobs" | |
subjob ${{needs.turbopack_rust_check.result}} "Turbopack Rust checks" | |
subjob ${{needs.turbopack_rust_clippy.result}} "Turbopack Rust clippy" | |
subjob ${{needs.turbopack_rust_test1.result}} "Turbopack Rust tests (linux)" | |
subjob ${{needs.turbopack_rust_test_bench1.result}} "Turbopack Rust benchmark tests (linux)" | |
subjob ${{needs.turbopack_typescript.result}} "Turbopack Typescript checks" | |
if [ "$cancelled" = "true" ]; then | |
echo "cancelled=true" >> $GITHUB_OUTPUT | |
elif [ "$failure" = "true" ]; then | |
echo "failure=true" >> $GITHUB_OUTPUT | |
else | |
echo "success=true" >> $GITHUB_OUTPUT | |
fi | |
- name: Add failure prose text | |
if: steps.info.outputs.failure == 'true' | |
run: | | |
echo "## ⚠ CI failed ⚠" > comment.md | |
echo >> comment.md | |
echo "The following steps have failed in CI:" >> comment.md | |
echo >> comment.md | |
cat failures.md >> comment.md | |
echo >> comment.md | |
echo "See [workflow summary](https://github.com/vercel/turbo/actions/runs/${{ github.run_id }}) for details">> comment.md | |
echo >> comment.md | |
echo "<!-- CI COMMENT -->" >> comment.md | |
- name: Add success prose text | |
if: steps.info.outputs.success == 'true' | |
run: | | |
echo "## 🟢 CI likely successful 🟢" > comment.md | |
echo >> comment.md | |
echo "A few longer running steps are still running, but they should not be considered as blocking." >> comment.md | |
echo >> comment.md | |
echo "See [workflow summary](https://github.com/vercel/turbo/actions/runs/${{ github.run_id }}) for details">> comment.md | |
echo >> comment.md | |
echo "<!-- CI COMMENT -->" >> comment.md | |
- name: Find PR Comment | |
id: comment | |
if: always() && github.event_name == 'pull_request' && steps.info.outputs.cancelled != 'true' | |
uses: peter-evans/find-comment@v2 | |
with: | |
issue-number: ${{ github.event.pull_request.number }} | |
comment-author: "github-actions[bot]" | |
body-includes: "<!-- CI COMMENT -->" | |
- name: Create or update PR comment | |
if: always() && github.event_name == 'pull_request' && steps.info.outputs.cancelled != 'true' | |
uses: peter-evans/create-or-update-comment@v2 | |
continue-on-error: true | |
with: | |
comment-id: ${{ steps.comment.outputs.comment-id }} | |
issue-number: ${{ github.event.pull_request.number }} | |
body-file: "comment.md" | |
edit-mode: replace | |
- name: It's not fine | |
if: steps.info.outputs.failure == 'true' | |
run: exit 1 | |
- name: It's fine | |
if: steps.info.outputs.success == 'true' | |
run: echo Ok | |
# Upload Turbopack's test results into datadog. | |
upload_test_results: | |
name: Upload Test results | |
needs: [determine_jobs, turbopack_rust_test1, turbopack_rust_test2] | |
# We have to set condition to always, since we want to upload test results for the failed tests as well. | |
if: ${{ always() }} | |
runs-on: ubuntu-latest | |
# Do not block CI if upload fails for some reason | |
continue-on-error: true | |
steps: | |
# Uploading test results does not require turbopack's src codes, but this | |
# allows datadog uploader to sync with git information. | |
- name: Checkout | |
uses: actions/checkout@v3 | |
- name: Setup Node.js | |
uses: actions/[email protected] | |
with: | |
node-version: 18 | |
- name: Install datadog | |
run: npm install -g @datadog/[email protected] | |
- name: Download test results | |
uses: actions/download-artifact@v3 | |
with: | |
path: artifacts | |
- name: Upload | |
continue-on-error: true | |
env: | |
DATADOG_API_KEY: ${{ secrets.DD_KEY_TURBOPACK }} | |
run: | | |
DD_ENV=ci datadog-ci junit upload --service Turbopack ./artifacts/test_reports/**/*.xml | |
done: | |
name: Done | |
needs: | |
- final | |
- determine_jobs | |
- turbopack_rust_check | |
- turbopack_rust_clippy | |
- turbopack_rust_test1 | |
- turbopack_rust_test2 | |
- turbopack_rust_test_bench1 | |
- turbopack_rust_test_bench2 | |
- turbopack_typescript | |
if: always() | |
permissions: | |
contents: read | |
pull-requests: write | |
runs-on: ubuntu-latest | |
steps: | |
- name: Compute info | |
id: info | |
if: always() | |
run: | | |
cancelled=false | |
failure=false | |
subjob () { | |
local result=$1 | |
local name=$2 | |
echo "$name: $result" | |
if [ "$result" = "cancelled" ]; then | |
cancelled=true | |
elif [ "$result" != "success" ] && [ "$result" != "skipped" ]; then | |
echo "- $name" >> failures.md | |
failure=true | |
fi | |
} | |
subjob ${{needs.determine_jobs.result}} "Determining jobs" | |
subjob ${{needs.turbopack_rust_check.result}} "Turbopack Rust checks" | |
subjob ${{needs.turbopack_rust_clippy.result}} "Turbopack Rust clippy" | |
subjob ${{needs.turbopack_rust_test1.result}} "Turbopack Rust tests (linux)" | |
subjob ${{needs.turbopack_rust_test2.result}} "Turbopack Rust tests (mac/win, non-blocking)" | |
subjob ${{needs.turbopack_rust_test_bench1.result}} "Turbopack Rust benchmark tests (linux)" | |
subjob ${{needs.turbopack_rust_test_bench2.result}} "Turbopack Rust benchmark tests (mac/win, non-blocking)" | |
subjob ${{needs.turbopack_typescript.result}} "Turbopack Typescript checks" | |
if [ "$cancelled" = "true" ]; then | |
echo "cancelled=true" >> $GITHUB_OUTPUT | |
elif [ "$failure" = "true" ]; then | |
echo "failure=true" >> $GITHUB_OUTPUT | |
else | |
echo "success=true" >> $GITHUB_OUTPUT | |
fi | |
- name: Add failure prose text | |
if: steps.info.outputs.failure == 'true' | |
run: | | |
echo "## ⚠️ CI failed ⚠️" > comment.md | |
echo >> comment.md | |
echo "The following steps have failed in CI:" >> comment.md | |
echo >> comment.md | |
cat failures.md >> comment.md | |
echo >> comment.md | |
echo "See [workflow summary](https://github.com/vercel/turbo/actions/runs/${{ github.run_id }}) for details">> comment.md | |
echo >> comment.md | |
echo "<!-- CI COMMENT -->" >> comment.md | |
- name: Add success prose text | |
if: steps.info.outputs.success == 'true' | |
run: | | |
echo "## 🟢 CI successful 🟢" > comment.md | |
echo >> comment.md | |
echo "Thanks" >> comment.md | |
echo >> comment.md | |
echo "<!-- CI COMMENT -->" >> comment.md | |
- name: Find PR Comment | |
id: comment | |
if: always() && github.event_name == 'pull_request' && steps.info.outputs.cancelled != 'true' | |
uses: peter-evans/find-comment@v2 | |
with: | |
issue-number: ${{ github.event.pull_request.number }} | |
comment-author: "github-actions[bot]" | |
body-includes: "<!-- CI COMMENT -->" | |
- name: Create or update PR comment | |
if: always() && github.event_name == 'pull_request' && steps.info.outputs.cancelled != 'true' | |
uses: peter-evans/create-or-update-comment@v2 | |
continue-on-error: true | |
with: | |
comment-id: ${{ steps.comment.outputs.comment-id }} | |
issue-number: ${{ github.event.pull_request.number }} | |
body-file: "comment.md" | |
edit-mode: replace | |
- name: It's not fine | |
if: steps.info.outputs.failure == 'true' | |
run: exit 1 | |
- name: It's fine | |
if: steps.info.outputs.success == 'true' | |
run: echo Ok | |
cleanup: | |
name: Cleanup | |
needs: [done] | |
if: always() | |
uses: ./.github/workflows/pr-clean-caches.yml | |
secrets: inherit |