From fed3248d33b828229bca59b6b6669f3b621fc3a7 Mon Sep 17 00:00:00 2001 From: Anthony Shew Date: Mon, 18 Nov 2024 21:42:11 -0700 Subject: [PATCH 1/7] chore: Require semantic comments on PR titles. (#9459) ### Description We're looking to use semantic titles for PRs so that they release notes look a little tidier. This action will enforce this for us. --- .github/workflows/semantic-release.yml | 33 ++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/semantic-release.yml diff --git a/.github/workflows/semantic-release.yml b/.github/workflows/semantic-release.yml new file mode 100644 index 0000000000000..a3eacf59078db --- /dev/null +++ b/.github/workflows/semantic-release.yml @@ -0,0 +1,33 @@ +name: Lint pull request title + +on: + pull_request_target: + types: + - opened + - edited + - synchronize + - reopened + +permissions: + pull-requests: read + +jobs: + main: + name: Validate PR title + runs-on: ubuntu-latest + steps: + - uses: amannn/action-semantic-pull-request + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + # Configure that a scope must always be provided. + requireScope: false + # Configure additional validation for the subject based on a regex. + # Ensures that the subject doesn't start with an uppercase character. + subjectPattern: ^[A-Z].*$ + # If `subjectPattern` is configured, you can use this property to override + # the default error message that is shown when the pattern doesn't match. + # The variables `subject` and `title` can be used within the message. + subjectPatternError: | + The subject "{subject}" found in the pull request title "{title}" doesn't match the configured pattern. + Please ensure that the subject doesn't start with a lowercase character. From e0380d6a5d989479fa51c03d6ce51650bed4e640 Mon Sep 17 00:00:00 2001 From: Anthony Shew Date: Mon, 18 Nov 2024 22:19:15 -0700 Subject: [PATCH 2/7] chore: Renaming conventional commits file. (#9461) --- .github/workflows/{semantic-release.yml => lint-pr-title.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{semantic-release.yml => lint-pr-title.yml} (100%) diff --git a/.github/workflows/semantic-release.yml b/.github/workflows/lint-pr-title.yml similarity index 100% rename from .github/workflows/semantic-release.yml rename to .github/workflows/lint-pr-title.yml From 7fb5b21fe3034c29affef8e35a8c90ee75c0cb67 Mon Sep 17 00:00:00 2001 From: Maksim Bondarenkov <119937608+ognevny@users.noreply.github.com> Date: Tue, 19 Nov 2024 17:48:17 +0300 Subject: [PATCH 3/7] deps: Update psm (#9443) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit psm@0.1.23 contains a bug which breaks build for gnullvm targets, with 0.1.24 it's fixed ### Description ### Testing Instructions build for gnullvm targets fixed --- Cargo.lock | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eb3a70c080949..c5a658ed6f9d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1036,12 +1036,13 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.83" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" dependencies = [ "jobserver", "libc", + "shlex", ] [[package]] @@ -2802,9 +2803,9 @@ checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" [[package]] name = "jobserver" -version = "0.1.26" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" dependencies = [ "libc", ] @@ -4109,9 +4110,9 @@ dependencies = [ [[package]] name = "psm" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa37f80ca58604976033fae9515a8a2989fc13797d953f7c04fb8fa36a11f205" +checksum = "200b9ff220857e53e184257720a14553b2f4aa02577d2ed9842d45d4b9654810" dependencies = [ "cc", ] @@ -4880,6 +4881,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook" version = "0.3.17" From 405ccc1251e348cb80af7dc69224c1170d6d3e05 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Tue, 19 Nov 2024 12:58:29 -0500 Subject: [PATCH 4/7] chore(release): fix release by avoiding old glibc version (#9464) ### Description https://github.com/vercel/turborepo/pull/9447 bumped us to `upload-artifact` to v4 which uses Node 20. Node 20 required a newer glibc version than was available on our build that targeted AWS's Lambda2's glibc version. See [this job](https://github.com/vercel/turborepo/actions/runs/11897321603/job/33151832964#step:12:21) for an example failure. Requiring this glibc version isn't necessary for the musl builds since it uses musl instead of glibc for libc. I do not think this was intentional to have the musl binary building in this container as we changed this back when we were cross compiling: https://github.com/vercel/turborepo/commit/8ca67c3c0cd9601909730953b9d03699a01b1da8 Future work: This does not fix the `@turbo/repository` release. ### Testing Instructions Ran test release and x86 msul build now passes: [job](https://github.com/vercel/turborepo/actions/runs/11917567457/job/33214049836) --- .github/workflows/turborepo-release.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/turborepo-release.yml b/.github/workflows/turborepo-release.yml index 1c0c60e223088..7aad56e181af4 100644 --- a/.github/workflows/turborepo-release.yml +++ b/.github/workflows/turborepo-release.yml @@ -116,11 +116,9 @@ jobs: target: "aarch64-apple-darwin" container-options: "--rm" - host: ubuntu-latest - container: ubuntu:xenial container-options: "--platform=linux/amd64 --rm" - container-setup: "apt-get update && apt-get install -y curl musl-tools sudo unzip" target: "x86_64-unknown-linux-musl" - setup: "apt-get install -y build-essential clang-5.0 lldb-5.0 llvm-5.0-dev libclang-5.0-dev" + setup: "sudo apt-get update && sudo apt-get install -y build-essential clang lldb llvm libclang-dev curl musl-tools sudo unzip" - host: ubuntu-latest container-options: "--rm" target: "aarch64-unknown-linux-musl" From 316b94e45ffd2ef893f92407d4da52db5408293b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 19 Nov 2024 13:47:30 -0500 Subject: [PATCH 5/7] release(turborepo): 2.3.1-canary.1 (#9465) Co-authored-by: Turbobot --- packages/create-turbo/package.json | 2 +- packages/eslint-config-turbo/package.json | 2 +- packages/eslint-plugin-turbo/package.json | 2 +- packages/turbo-codemod/package.json | 2 +- packages/turbo-gen/package.json | 2 +- packages/turbo-ignore/package.json | 2 +- packages/turbo-types/package.json | 2 +- packages/turbo-workspaces/package.json | 2 +- packages/turbo/package.json | 14 +++++++------- version.txt | 2 +- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/create-turbo/package.json b/packages/create-turbo/package.json index 66f5b913078ec..cb294655a59ec 100644 --- a/packages/create-turbo/package.json +++ b/packages/create-turbo/package.json @@ -1,6 +1,6 @@ { "name": "create-turbo", - "version": "2.3.1-canary.0", + "version": "2.3.1-canary.1", "description": "Create a new Turborepo", "homepage": "https://turbo.build/repo", "license": "MIT", diff --git a/packages/eslint-config-turbo/package.json b/packages/eslint-config-turbo/package.json index 82d80b87e4b76..2cf72c53ee082 100644 --- a/packages/eslint-config-turbo/package.json +++ b/packages/eslint-config-turbo/package.json @@ -1,6 +1,6 @@ { "name": "eslint-config-turbo", - "version": "2.3.1-canary.0", + "version": "2.3.1-canary.1", "description": "ESLint config for Turborepo", "repository": { "type": "git", diff --git a/packages/eslint-plugin-turbo/package.json b/packages/eslint-plugin-turbo/package.json index 9287b76ee3f5c..82375a95f3448 100644 --- a/packages/eslint-plugin-turbo/package.json +++ b/packages/eslint-plugin-turbo/package.json @@ -1,6 +1,6 @@ { "name": "eslint-plugin-turbo", - "version": "2.3.1-canary.0", + "version": "2.3.1-canary.1", "description": "ESLint plugin for Turborepo", "keywords": [ "turbo", diff --git a/packages/turbo-codemod/package.json b/packages/turbo-codemod/package.json index d10ebfafe1387..27386914a7cbb 100644 --- a/packages/turbo-codemod/package.json +++ b/packages/turbo-codemod/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/codemod", - "version": "2.3.1-canary.0", + "version": "2.3.1-canary.1", "description": "Provides Codemod transformations to help upgrade your Turborepo codebase when a feature is deprecated.", "homepage": "https://turbo.build/repo", "license": "MIT", diff --git a/packages/turbo-gen/package.json b/packages/turbo-gen/package.json index cd93b3b58014b..1e63bd43d64ae 100644 --- a/packages/turbo-gen/package.json +++ b/packages/turbo-gen/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/gen", - "version": "2.3.1-canary.0", + "version": "2.3.1-canary.1", "description": "Extend a Turborepo", "homepage": "https://turbo.build/repo", "license": "MIT", diff --git a/packages/turbo-ignore/package.json b/packages/turbo-ignore/package.json index 919604bb3694e..31b80461d1e2d 100644 --- a/packages/turbo-ignore/package.json +++ b/packages/turbo-ignore/package.json @@ -1,6 +1,6 @@ { "name": "turbo-ignore", - "version": "2.3.1-canary.0", + "version": "2.3.1-canary.1", "description": "", "homepage": "https://turbo.build/repo", "keywords": [], diff --git a/packages/turbo-types/package.json b/packages/turbo-types/package.json index 5997a85ebeb11..c6d711b372be5 100644 --- a/packages/turbo-types/package.json +++ b/packages/turbo-types/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/types", - "version": "2.3.1-canary.0", + "version": "2.3.1-canary.1", "description": "Turborepo types", "homepage": "https://turbo.build/repo", "license": "MIT", diff --git a/packages/turbo-workspaces/package.json b/packages/turbo-workspaces/package.json index ceb6ff553b7c7..67768804b30c7 100644 --- a/packages/turbo-workspaces/package.json +++ b/packages/turbo-workspaces/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/workspaces", - "version": "2.3.1-canary.0", + "version": "2.3.1-canary.1", "description": "Tools for working with package managers", "homepage": "https://turbo.build/repo", "license": "MIT", diff --git a/packages/turbo/package.json b/packages/turbo/package.json index 80ec7ac0f61c6..08ad168d08752 100644 --- a/packages/turbo/package.json +++ b/packages/turbo/package.json @@ -1,6 +1,6 @@ { "name": "turbo", - "version": "2.3.1-canary.0", + "version": "2.3.1-canary.1", "description": "Turborepo is a high-performance build system for JavaScript and TypeScript codebases.", "repository": "https://github.com/vercel/turborepo", "bugs": "https://github.com/vercel/turborepo/issues", @@ -17,11 +17,11 @@ "bin" ], "optionalDependencies": { - "turbo-darwin-64": "2.3.1-canary.0", - "turbo-darwin-arm64": "2.3.1-canary.0", - "turbo-linux-64": "2.3.1-canary.0", - "turbo-linux-arm64": "2.3.1-canary.0", - "turbo-windows-64": "2.3.1-canary.0", - "turbo-windows-arm64": "2.3.1-canary.0" + "turbo-darwin-64": "2.3.1-canary.1", + "turbo-darwin-arm64": "2.3.1-canary.1", + "turbo-linux-64": "2.3.1-canary.1", + "turbo-linux-arm64": "2.3.1-canary.1", + "turbo-windows-64": "2.3.1-canary.1", + "turbo-windows-arm64": "2.3.1-canary.1" } } diff --git a/version.txt b/version.txt index 84d8cd78c3355..44d68e49edf88 100644 --- a/version.txt +++ b/version.txt @@ -1,2 +1,2 @@ -2.3.1-canary.0 +2.3.1-canary.1 canary From ccf1c2fd62aa08a956bc2260a9bb7d0c71fed509 Mon Sep 17 00:00:00 2001 From: Nicholas Yang Date: Tue, 19 Nov 2024 15:15:19 -0500 Subject: [PATCH 6/7] feat(link): add `--yes` and `--scope` flags to `link` (#9466) ### Description Adds `--yes` and `--scope` flags ### Testing Instructions Added tests for parsing those flags --- crates/turborepo-lib/src/cli/error.rs | 4 +- crates/turborepo-lib/src/cli/mod.rs | 106 +++++++++++++++++- crates/turborepo-lib/src/commands/link.rs | 20 +++- crates/turborepo-lib/src/diagnostics.rs | 9 +- docs/repo-docs/reference/link.mdx | 10 +- .../integration/tests/command-link.t | 8 ++ .../integration/tests/turbo-help.t | 8 +- 7 files changed, 152 insertions(+), 13 deletions(-) diff --git a/crates/turborepo-lib/src/cli/error.rs b/crates/turborepo-lib/src/cli/error.rs index a85e0103573f4..99631493ca74d 100644 --- a/crates/turborepo-lib/src/cli/error.rs +++ b/crates/turborepo-lib/src/cli/error.rs @@ -8,7 +8,7 @@ use turborepo_telemetry::events::command::CommandEventBuilder; use turborepo_ui::{color, BOLD, GREY}; use crate::{ - commands::{bin, generate, ls, prune, run::get_signal, CommandBase}, + commands::{bin, generate, link, ls, prune, run::get_signal, CommandBase}, daemon::DaemonError, query, rewrite_json::RewriteError, @@ -45,6 +45,8 @@ pub enum Error { #[diagnostic(transparent)] Ls(#[from] ls::Error), #[error(transparent)] + Link(#[from] link::Error), + #[error(transparent)] #[diagnostic(transparent)] Prune(#[from] prune::Error), #[error(transparent)] diff --git a/crates/turborepo-lib/src/cli/mod.rs b/crates/turborepo-lib/src/cli/mod.rs index b857bd34b8f6c..d4025503d2bf1 100644 --- a/crates/turborepo-lib/src/cli/mod.rs +++ b/crates/turborepo-lib/src/cli/mod.rs @@ -569,6 +569,13 @@ pub enum Command { #[clap(long)] no_gitignore: bool, + /// The scope, i.e. Vercel team, to which you are linking + #[clap(long)] + scope: Option, + + /// Answer yes to all prompts (default false) + #[clap(long, short)] + yes: bool, /// Specify what should be linked (default "remote cache") #[clap(long, value_enum, default_value_t = LinkTarget::RemoteCache)] target: LinkTarget, @@ -1299,11 +1306,18 @@ pub async fn run( } Command::Link { no_gitignore, + scope, + yes, target, } => { CommandEventBuilder::new("link") .with_parent(&root_telemetry) .track_call(); + + if cli_args.team.is_some() { + warn!("team flag does not set the scope for linking. Use --scope instead."); + } + if cli_args.test_run { println!("Link test run successful"); return Ok(0); @@ -1311,11 +1325,11 @@ pub async fn run( let modify_gitignore = !*no_gitignore; let to = *target; + let yes = *yes; + let scope = scope.clone(); let mut base = CommandBase::new(cli_args, repo_root, version, color_config); - if let Err(err) = link::link(&mut base, modify_gitignore, to).await { - error!("error: {}", err.to_string()) - } + link::link(&mut base, scope, modify_gitignore, yes, to).await?; Ok(0) } @@ -1469,7 +1483,7 @@ mod test { use itertools::Itertools; use pretty_assertions::assert_eq; - use crate::cli::{ExecutionArgs, RunArgs}; + use crate::cli::{ExecutionArgs, LinkTarget, RunArgs}; struct CommandTestCase { command: &'static str, @@ -2300,6 +2314,90 @@ mod test { .test(); } + #[test] + fn test_parse_link() { + assert_eq!( + Args::try_parse_from(["turbo", "link"]).unwrap(), + Args { + command: Some(Command::Link { + no_gitignore: false, + scope: None, + yes: false, + target: LinkTarget::RemoteCache, + }), + ..Args::default() + } + ); + + CommandTestCase { + command: "link", + command_args: vec![], + global_args: vec![vec!["--cwd", "../examples/with-yarn"]], + expected_output: Args { + command: Some(Command::Link { + no_gitignore: false, + scope: None, + yes: false, + target: LinkTarget::RemoteCache, + }), + cwd: Some(Utf8PathBuf::from("../examples/with-yarn")), + ..Args::default() + }, + } + .test(); + + CommandTestCase { + command: "link", + command_args: vec![vec!["--yes"]], + global_args: vec![vec!["--cwd", "../examples/with-yarn"]], + expected_output: Args { + command: Some(Command::Link { + yes: true, + no_gitignore: false, + scope: None, + target: LinkTarget::RemoteCache, + }), + cwd: Some(Utf8PathBuf::from("../examples/with-yarn")), + ..Args::default() + }, + } + .test(); + + CommandTestCase { + command: "link", + command_args: vec![vec!["--scope", "foo"]], + global_args: vec![vec!["--cwd", "../examples/with-yarn"]], + expected_output: Args { + command: Some(Command::Link { + yes: false, + no_gitignore: false, + scope: Some("foo".to_string()), + target: LinkTarget::RemoteCache, + }), + cwd: Some(Utf8PathBuf::from("../examples/with-yarn")), + ..Args::default() + }, + } + .test(); + + CommandTestCase { + command: "link", + command_args: vec![vec!["--no-gitignore"]], + global_args: vec![vec!["--cwd", "../examples/with-yarn"]], + expected_output: Args { + command: Some(Command::Link { + yes: false, + no_gitignore: true, + scope: None, + target: LinkTarget::RemoteCache, + }), + cwd: Some(Utf8PathBuf::from("../examples/with-yarn")), + ..Args::default() + }, + } + .test(); + } + #[test] fn test_parse_login() { assert_eq!( diff --git a/crates/turborepo-lib/src/commands/link.rs b/crates/turborepo-lib/src/commands/link.rs index 721c3a07458a4..e52a42e653ea1 100644 --- a/crates/turborepo-lib/src/commands/link.rs +++ b/crates/turborepo-lib/src/commands/link.rs @@ -163,7 +163,9 @@ pub(crate) async fn verify_caching_enabled<'a>( pub async fn link( base: &mut CommandBase, + scope: Option, modify_gitignore: bool, + yes: bool, target: LinkTarget, ) -> Result<(), Error> { let homedir_path = home_dir().ok_or_else(|| Error::HomeDirectoryNotFound)?; @@ -183,7 +185,7 @@ pub async fn link( REMOTE_CACHING_URL ); - if !should_link_remote_cache(base, &repo_root_with_tilde)? { + if !yes && !should_link_remote_cache(base, &repo_root_with_tilde)? { return Err(Error::NotLinking); } @@ -203,7 +205,17 @@ pub async fn link( .await .map_err(Error::TeamsRequest)?; - let selected_team = select_team(base, &teams_response.teams)?; + let selected_team = if let Some(team_slug) = scope { + SelectedTeam::Team( + teams_response + .teams + .iter() + .find(|team| team.slug == team_slug) + .ok_or_else(|| Error::TeamNotFound(team_slug.to_string()))?, + ) + } else { + select_team(base, &teams_response.teams)? + }; let team_id = match selected_team { SelectedTeam::User => user_response.user.id.as_str(), @@ -632,7 +644,7 @@ mod test { ) .unwrap(); - link::link(&mut base, false, LinkTarget::RemoteCache) + link::link(&mut base, None, false, false, LinkTarget::RemoteCache) .await .unwrap(); @@ -707,7 +719,7 @@ mod test { ) .unwrap(); - link::link(&mut base, false, LinkTarget::Spaces) + link::link(&mut base, None, false, false, LinkTarget::Spaces) .await .unwrap(); diff --git a/crates/turborepo-lib/src/diagnostics.rs b/crates/turborepo-lib/src/diagnostics.rs index 04f1c989b2c45..c8297f56f32d1 100644 --- a/crates/turborepo-lib/src/diagnostics.rs +++ b/crates/turborepo-lib/src/diagnostics.rs @@ -429,7 +429,14 @@ impl Diagnostic for RemoteCacheDiagnostic { return; }; stopped.await.unwrap(); - let link_res = link(&mut base, false, crate::cli::LinkTarget::RemoteCache).await; + let link_res = link( + &mut base, + None, + false, + false, + crate::cli::LinkTarget::RemoteCache, + ) + .await; resume.send(()).unwrap(); link_res }; diff --git a/docs/repo-docs/reference/link.mdx b/docs/repo-docs/reference/link.mdx index c9a2712182f15..a71997207f18e 100644 --- a/docs/repo-docs/reference/link.mdx +++ b/docs/repo-docs/reference/link.mdx @@ -13,6 +13,14 @@ The selected owner (either a user or an organization) will be able to share [cac Specifies the URL of your Remote Cache provider. +### `--yes` + +Answer yes to all prompts + +### `--scope ` + +The scope, i.e. Vercel team, to which you are linking + ```bash title="Terminal" turbo link --api https://acme.com -``` \ No newline at end of file +``` diff --git a/turborepo-tests/integration/tests/command-link.t b/turborepo-tests/integration/tests/command-link.t index bd0e882f85f32..d3ba844b0b743 100644 --- a/turborepo-tests/integration/tests/command-link.t +++ b/turborepo-tests/integration/tests/command-link.t @@ -6,3 +6,11 @@ Link Test Run $ ${TURBO} link --__test-run Link test run successful + $ ${TURBO} link --__test-run --yes + Link test run successful + + $ ${TURBO} link --__test-run --team=my-team + WARNING team flag does not set the scope for linking. Use --scope instead. + Link test run successful + + diff --git a/turborepo-tests/integration/tests/turbo-help.t b/turborepo-tests/integration/tests/turbo-help.t index b9c7d73bc70af..c484db0230c7a 100644 --- a/turborepo-tests/integration/tests/turbo-help.t +++ b/turborepo-tests/integration/tests/turbo-help.t @@ -329,14 +329,18 @@ Test help flag for link command Do not create or modify .gitignore (default false) --version + --scope + The scope, i.e. Vercel team, to which you are linking --skip-infer Skip any attempts to infer which version of Turbo the project is configured to use - --target - Specify what should be linked (default "remote cache") [default: remote-cache] [possible values: remote-cache, spaces] --no-update-notifier Disable the turbo update notification + -y, --yes + Answer yes to all prompts (default false) --api Override the endpoint for API calls + --target + Specify what should be linked (default "remote cache") [default: remote-cache] [possible values: remote-cache, spaces] --color Force color usage in the terminal --cwd From 3a494fe2e4dadba7b6d8f15f11b6e66047ad7f93 Mon Sep 17 00:00:00 2001 From: Benjamin Woodruff Date: Tue, 19 Nov 2024 15:23:22 -0800 Subject: [PATCH 7/7] docs(turbopack): Add incremental-computation page, intended to replace core-concepts page (#9456) ### Description This depends on some images (these are placeholders, I'll get the real assets from @anthonyshew) ### Testing Instructions ![Screenshot 2024-11-18 at 15-43-55 Incremental computation.png](https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/HAZVitxRNnZz8QMiPn4a/527e54e5-5c71-4593-b06f-a339075a96b4.png) --------- Co-authored-by: Anthony Shew --- docs/pack-docs/features/dev-server.mdx | 2 +- docs/pack-docs/incremental-computation.mdx | 136 +++++++++++++++++++++ docs/pack-docs/meta.json | 2 +- docs/pack-docs/why-turbopack.mdx | 2 +- 4 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 docs/pack-docs/incremental-computation.mdx diff --git a/docs/pack-docs/features/dev-server.mdx b/docs/pack-docs/features/dev-server.mdx index 662af5a2b3a48..62d600b2f3b88 100644 --- a/docs/pack-docs/features/dev-server.mdx +++ b/docs/pack-docs/features/dev-server.mdx @@ -9,7 +9,7 @@ Turbopack is optimized to give you an extremely fast development server. We cons Hot Module Replacement (HMR) gives your dev server the ability to push file updates to the browser without triggering a full refresh. This works for most static assets (including JavaScript files) enabling a smooth and fast developer experience. -Turbopack supports Hot Module Replacement out of the box. Because of our [incremental architecture](/pack/docs/core-concepts), we are hyper-optimized for delivering fast updates. +Turbopack supports Hot Module Replacement out of the box. Because of our [incremental architecture](/pack/docs/incremental-computation), we are hyper-optimized for delivering fast updates. {/* Maybe link to benchmarks here? */} diff --git a/docs/pack-docs/incremental-computation.mdx b/docs/pack-docs/incremental-computation.mdx new file mode 100644 index 0000000000000..b40f14f9d1aac --- /dev/null +++ b/docs/pack-docs/incremental-computation.mdx @@ -0,0 +1,136 @@ +--- +title: Incremental computation +description: Learn about the innovative architecture that powers Turbopack's speed improvements. +--- + +import { ThemeAwareImage } from '#/components/theme-aware-image'; + +Turbopack uses an automatic demand-driven incremental computation architecture to provide [React Fast Refresh](https://nextjs.org/docs/architecture/fast-refresh) with massive Next.js and React applications. + +This architecture uses caching to remember what values functions were called with and what values they returned. Incremental builds scale by the size of your local changes, not the size of your application. + + + +Turbopack’s architecture is based on over a decade of learnings and prior research. It draws inspiration from [webpack](https://webpack.js.org/), [Salsa](https://salsa-rs.netlify.app/overview) (which powers [Rust-Analyzer](https://rust-analyzer.github.io/) and [Ruff](https://docs.astral.sh/ruff/)), [Parcel](https://parceljs.org/), the [Rust compiler’s query system](https://rustc-dev-guide.rust-lang.org/query.html), [Adapton](http://adapton.org/), and many others. + +## Background: Manual incremental computation + +Many build systems include explicit dependency graphs that must be manually populated when evaluating build rules. Explicitly declaring your dependency graph can theoretically give optimal results, but in practice it leaves room for errors. + +The difficulty of specifying an explicit dependency graph means that usually caching is done at a coarse file-level granularity. This granularity does have some benefits: less incremental results means less data to cache, which might be worth it if you have limited disk space or memory. + +An example of such an architecture is [GNU Make](https://www.gnu.org/software/make/), where output targets and prerequisites are manually configured and represented as files. Systems like GNU Make miss caching opportunities due to their coarse granularity: they does not understand and cannot cache internal data structures within the compiler. + +## Function-level fine-grained automatic incremental computation + +In Turbopack, the relationship between input files and resulting build artifacts isn’t straightforward. Bundlers employ whole-program analysis for dead code elimination ("tree shaking") and clustering of common dependencies in the module graph. Consequently, the build artifacts—JavaScript files shared across multiple application routes—form complex many-to-many relationships with input files. + +Turbopack uses a very fine-grained caching architecture. Because manually declaring and adding dependencies to a graph is prone to human errors, Turbopack needs an automated solution that can scale. + +### Value cells + +To facilitate automatic caching and dependency tracking, Turbopack introduces a concept of “value cells” (`Vc<…>`). Each value cell represents a fine-grained piece of execution, like a cell in a spreadsheet. When reading a cell, it records the currently executing function and all of its cells as dependent on that cell. + + + +By not marking cells as dependencies until they are read, Turbopack achieves finer-grained caching than [a traditional top-down memoization approach](https://en.wikipedia.org/wiki/Memoization) would provide. For example, an argument might be an object or mapping of _many_ value cells. Instead of needing to recompute our tracked function when _any part of_ the object or mapping changes, it only needs to recompute the tracked function when a cell that _it has actually read_ changes. + +Value cells represent nearly everything inside of Turbopack, such as a file on disk, an abstract syntax tree (AST), metadata about imports and exports of modules, or clustering information used for chunking and bundling. + + + +### Marking dirty and propagating changes + +When a cell’s value changes, Turbopack must determine what work to recompute. It uses [Adapton’s](http://adapton.org/) two-phase dirty and propagate algorithms. + + + +Typically, source code files are at the bottom of the dependency graph. When the incremental execution engine finds that a file’s contents have changed, it marks the function that read it and its associated value cells as “dirty”. To watch for filesystem changes, Turbopack uses `inotify` (Linux) or the equivalent platform API [via the `notify` crate](https://docs.rs/notify/). + +Next comes propagation, where the bundler is re-run from the bottom up, bubbling up any computations that yield new results. This propagation is "demand-driven," meaning the system only recomputes a dependent cell if it's part of an "active query". An active query could be a currently open webpage with hot reloading enabled, or even a request to build the full production app. + +If a cell isn't part of an active query, propagation of it’s dirty flag is deferred until either the dependency graph changes or a new active query is created. + +### Additional optimization: The aggregation tree + +The dependency graph can contain hundreds of thousands of unique invocations of small tracked functions, and the incremental execution engine must frequently traverse the graph to inspect and update dirty flags. + +Turbopack optimizes these operations using an “aggregation tree”. Each node of the aggregation tree represents a cluster of tracked function calls, reducing some of the memory overhead associated with dependency tracking, and reducing the number of nodes that must be traversed. + +## Parallel and async execution with Rust and Tokio + +To parallelize execution, Turbopack uses Rust with [the Tokio asynchronous runtime](https://tokio.rs/). Each tracked function is spawned into Tokio’s thread pool as [a Tokio task](https://tokio.rs/tokio/tutorial/spawning#tasks). That allows Turbopack to benefit from Rust’s low-overhead parallelism and [Tokio’s work-stealing scheduler](https://tokio.rs/blog/2019-10-scheduler). + +While bundling is CPU-bound in most scenarios, it can become IO-bound when building from slow hard drives, [a network drive](https://github.com/facebook/sapling/blob/main/eden/fs/docs/Overview.md), or when reading from or writing to persistent caches. Tokio allows Turbopack to more gracefully handle these degraded situations than we might otherwise be able to. diff --git a/docs/pack-docs/meta.json b/docs/pack-docs/meta.json index 5de589a068a61..2aa30d5113509 100644 --- a/docs/pack-docs/meta.json +++ b/docs/pack-docs/meta.json @@ -2,7 +2,7 @@ "pages": [ "index", "why-turbopack", - "core-concepts", + "incremental-computation", "roadmap", "features", "benchmarks", diff --git a/docs/pack-docs/why-turbopack.mdx b/docs/pack-docs/why-turbopack.mdx index 058c0d93f40ae..31ce87c599cd7 100644 --- a/docs/pack-docs/why-turbopack.mdx +++ b/docs/pack-docs/why-turbopack.mdx @@ -39,7 +39,7 @@ This more ‘lazy’ approach (only bundling assets when absolutely necessary) i esbuild doesn’t have a concept of ‘lazy’ bundling - it’s all-or-nothing, unless you specifically target only certain entry points. -Turbopack’s development mode builds a minimal graph of your app’s imports and exports based on received requests and only bundles the minimal code necessary. Learn more in the [core concepts docs](/pack/docs/core-concepts). +Turbopack’s development mode builds a minimal graph of your app’s imports and exports based on received requests and only bundles the minimal code necessary. Learn more in the [incremental computation docs](/pack/docs/incremental-computation). ## Summary