From 6a793bc3b1d6bcd41ec571b70d93cc4251e044fd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 4 Nov 2024 14:03:35 -0700 Subject: [PATCH 01/12] release(turborepo): 2.2.4-canary.6 (#9374) Co-authored-by: Turbobot --- packages/create-turbo/package.json | 2 +- packages/eslint-config-turbo/package.json | 2 +- packages/eslint-plugin-turbo/package.json | 2 +- packages/turbo-codemod/package.json | 2 +- packages/turbo-gen/package.json | 2 +- packages/turbo-ignore/package.json | 2 +- packages/turbo-types/package.json | 2 +- packages/turbo-workspaces/package.json | 2 +- packages/turbo/package.json | 14 +++++++------- version.txt | 2 +- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/create-turbo/package.json b/packages/create-turbo/package.json index 55851def58b35..272bf54d4e4c3 100644 --- a/packages/create-turbo/package.json +++ b/packages/create-turbo/package.json @@ -1,6 +1,6 @@ { "name": "create-turbo", - "version": "2.2.4-canary.5", + "version": "2.2.4-canary.6", "description": "Create a new Turborepo", "homepage": "https://turbo.build/repo", "license": "MIT", diff --git a/packages/eslint-config-turbo/package.json b/packages/eslint-config-turbo/package.json index 0b2ecdc96ab08..f5b5698d3cadd 100644 --- a/packages/eslint-config-turbo/package.json +++ b/packages/eslint-config-turbo/package.json @@ -1,6 +1,6 @@ { "name": "eslint-config-turbo", - "version": "2.2.4-canary.5", + "version": "2.2.4-canary.6", "description": "ESLint config for Turborepo", "repository": { "type": "git", diff --git a/packages/eslint-plugin-turbo/package.json b/packages/eslint-plugin-turbo/package.json index 2fc11f20dca7c..a790fb0c242ae 100644 --- a/packages/eslint-plugin-turbo/package.json +++ b/packages/eslint-plugin-turbo/package.json @@ -1,6 +1,6 @@ { "name": "eslint-plugin-turbo", - "version": "2.2.4-canary.5", + "version": "2.2.4-canary.6", "description": "ESLint plugin for Turborepo", "keywords": [ "turbo", diff --git a/packages/turbo-codemod/package.json b/packages/turbo-codemod/package.json index ffcc237279272..fa66c043d8404 100644 --- a/packages/turbo-codemod/package.json +++ b/packages/turbo-codemod/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/codemod", - "version": "2.2.4-canary.5", + "version": "2.2.4-canary.6", "description": "Provides Codemod transformations to help upgrade your Turborepo codebase when a feature is deprecated.", "homepage": "https://turbo.build/repo", "license": "MIT", diff --git a/packages/turbo-gen/package.json b/packages/turbo-gen/package.json index 74feb956bec2e..bd230682b7861 100644 --- a/packages/turbo-gen/package.json +++ b/packages/turbo-gen/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/gen", - "version": "2.2.4-canary.5", + "version": "2.2.4-canary.6", "description": "Extend a Turborepo", "homepage": "https://turbo.build/repo", "license": "MIT", diff --git a/packages/turbo-ignore/package.json b/packages/turbo-ignore/package.json index e7710dcc93fac..260bd3cf7102b 100644 --- a/packages/turbo-ignore/package.json +++ b/packages/turbo-ignore/package.json @@ -1,6 +1,6 @@ { "name": "turbo-ignore", - "version": "2.2.4-canary.5", + "version": "2.2.4-canary.6", "description": "", "homepage": "https://turbo.build/repo", "keywords": [], diff --git a/packages/turbo-types/package.json b/packages/turbo-types/package.json index c655568299438..e6d3f34d9a98a 100644 --- a/packages/turbo-types/package.json +++ b/packages/turbo-types/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/types", - "version": "2.2.4-canary.5", + "version": "2.2.4-canary.6", "description": "Turborepo types", "homepage": "https://turbo.build/repo", "license": "MIT", diff --git a/packages/turbo-workspaces/package.json b/packages/turbo-workspaces/package.json index 248e16fc69def..32e12e8b1f47e 100644 --- a/packages/turbo-workspaces/package.json +++ b/packages/turbo-workspaces/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/workspaces", - "version": "2.2.4-canary.5", + "version": "2.2.4-canary.6", "description": "Tools for working with package managers", "homepage": "https://turbo.build/repo", "license": "MIT", diff --git a/packages/turbo/package.json b/packages/turbo/package.json index d84f7d9e7f27f..39bb37ff87627 100644 --- a/packages/turbo/package.json +++ b/packages/turbo/package.json @@ -1,6 +1,6 @@ { "name": "turbo", - "version": "2.2.4-canary.5", + "version": "2.2.4-canary.6", "description": "Turborepo is a high-performance build system for JavaScript and TypeScript codebases.", "repository": "https://github.com/vercel/turborepo", "bugs": "https://github.com/vercel/turborepo/issues", @@ -17,11 +17,11 @@ "bin" ], "optionalDependencies": { - "turbo-darwin-64": "2.2.4-canary.5", - "turbo-darwin-arm64": "2.2.4-canary.5", - "turbo-linux-64": "2.2.4-canary.5", - "turbo-linux-arm64": "2.2.4-canary.5", - "turbo-windows-64": "2.2.4-canary.5", - "turbo-windows-arm64": "2.2.4-canary.5" + "turbo-darwin-64": "2.2.4-canary.6", + "turbo-darwin-arm64": "2.2.4-canary.6", + "turbo-linux-64": "2.2.4-canary.6", + "turbo-linux-arm64": "2.2.4-canary.6", + "turbo-windows-64": "2.2.4-canary.6", + "turbo-windows-arm64": "2.2.4-canary.6" } } diff --git a/version.txt b/version.txt index afd16ccb13cd1..83c28e547e68d 100644 --- a/version.txt +++ b/version.txt @@ -1,2 +1,2 @@ -2.2.4-canary.5 +2.2.4-canary.6 canary From 6d039f71ef9d57630de81fb81d3ec10dc4e8c86b Mon Sep 17 00:00:00 2001 From: Nicholas Yang Date: Mon, 4 Nov 2024 17:43:14 -0500 Subject: [PATCH 02/12] chore(ci): bump macOS runner to 13 (#9375) --- .github/workflows/lsp.yml | 4 ++-- .github/workflows/test-js-packages.yml | 2 +- .github/workflows/turborepo-test.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lsp.yml b/.github/workflows/lsp.yml index c586ab1b7e4c1..22567bcc1bdec 100644 --- a/.github/workflows/lsp.yml +++ b/.github/workflows/lsp.yml @@ -15,10 +15,10 @@ jobs: fail-fast: false matrix: settings: - - host: macos-12 + - host: macos-13 target: "x86_64-apple-darwin" container-options: "--rm" - - host: macos-12 + - host: macos-13 target: "aarch64-apple-darwin" container-options: "--rm" - host: ubuntu-latest diff --git a/.github/workflows/test-js-packages.yml b/.github/workflows/test-js-packages.yml index 2e4de376bec24..7f22b8cc9532f 100644 --- a/.github/workflows/test-js-packages.yml +++ b/.github/workflows/test-js-packages.yml @@ -68,7 +68,7 @@ jobs: - "x64" - "metal" - name: macos - runner: macos-12 + runner: macos-13 node-version: - 18 - 20 diff --git a/.github/workflows/turborepo-test.yml b/.github/workflows/turborepo-test.yml index abcfeb4c67b30..f505d66eddf44 100644 --- a/.github/workflows/turborepo-test.yml +++ b/.github/workflows/turborepo-test.yml @@ -116,7 +116,7 @@ jobs: matrix: os: - runner: ubuntu-latest - - runner: macos-12 + - runner: macos-13 - runner: windows-latest steps: # On Windows, set autocrlf to input so that when the repo is cloned down @@ -276,7 +276,7 @@ jobs: - "metal" nextest: linux - name: macos - runner: macos-12 + runner: macos-13 nextest: mac - name: windows runner: windows-latest From 21863add07ef52ee4dbfde7cc10a303a773357b8 Mon Sep 17 00:00:00 2001 From: Anthony Shew Date: Mon, 4 Nov 2024 17:06:38 -0700 Subject: [PATCH 03/12] fix: Don't require a name for the root package.json. (#9378) --- .../src/package_graph/mod.rs | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/crates/turborepo-repository/src/package_graph/mod.rs b/crates/turborepo-repository/src/package_graph/mod.rs index 439fabf97e1dd..15cd6351b02cc 100644 --- a/crates/turborepo-repository/src/package_graph/mod.rs +++ b/crates/turborepo-repository/src/package_graph/mod.rs @@ -145,9 +145,20 @@ impl PackageGraph { pub fn validate(&self) -> Result<(), Error> { for info in self.packages.values() { let name = info.package_json.name.as_deref(); - if matches!(name, None | Some("")) { - let package_json_path = self.repo_root.resolve(info.package_json_path()); - return Err(Error::PackageJsonMissingName(package_json_path)); + let package_json_path = self.repo_root.resolve(info.package_json_path()); + match name { + Some("") => { + return Err(Error::PackageJsonMissingName(package_json_path)); + } + None => { + // We don't need to require a name for the root package.json. + if package_json_path == self.repo_root.join_component("package.json") { + continue; + } + + return Err(Error::PackageJsonMissingName(package_json_path)); + } + Some(_) => continue, } } graph::validate_graph(&self.graph).map_err(Error::InvalidPackageGraph)?; @@ -902,4 +913,17 @@ mod test { )) ); } + + #[tokio::test] + async fn test_does_not_require_name_for_root_package_json() { + let root = + AbsoluteSystemPathBuf::new(if cfg!(windows) { r"C:\repo" } else { "/repo" }).unwrap(); + let pkg_graph = PackageGraph::builder(&root, PackageJson::from_value(json!({})).unwrap()) + .with_package_discovery(MockDiscovery) + .build() + .await + .unwrap(); + + assert!(pkg_graph.validate().is_ok()); + } } From 7d996094899209e0dc014a97da8388a1068b39b8 Mon Sep 17 00:00:00 2001 From: Anthony Shew Date: Mon, 4 Nov 2024 22:58:07 -0700 Subject: [PATCH 04/12] refactor: Use simpler root package.json check. (#9382) ### Description Following up on Chris' comment on #9378. I didn't see it before I hit merge because I was on mobile. --- .../src/package_graph/mod.rs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/crates/turborepo-repository/src/package_graph/mod.rs b/crates/turborepo-repository/src/package_graph/mod.rs index 15cd6351b02cc..988c8850cad0f 100644 --- a/crates/turborepo-repository/src/package_graph/mod.rs +++ b/crates/turborepo-repository/src/package_graph/mod.rs @@ -143,19 +143,14 @@ impl PackageGraph { #[tracing::instrument(skip(self))] pub fn validate(&self) -> Result<(), Error> { - for info in self.packages.values() { + for (package_name, info) in self.packages.iter() { + if matches!(package_name, PackageName::Root) { + continue; + } let name = info.package_json.name.as_deref(); - let package_json_path = self.repo_root.resolve(info.package_json_path()); match name { - Some("") => { - return Err(Error::PackageJsonMissingName(package_json_path)); - } - None => { - // We don't need to require a name for the root package.json. - if package_json_path == self.repo_root.join_component("package.json") { - continue; - } - + Some("") | None => { + let package_json_path = self.repo_root.resolve(info.package_json_path()); return Err(Error::PackageJsonMissingName(package_json_path)); } Some(_) => continue, From 8c6d8d547d1f8ec221f140249bbdc56a118be294 Mon Sep 17 00:00:00 2001 From: Anthony Shew Date: Tue, 5 Nov 2024 07:10:37 -0700 Subject: [PATCH 05/12] chore: Stop running unnecessary checks for docs-only changes. (#9387) --- .github/workflows/test-js-packages.yml | 10 +--------- .github/workflows/turborepo-native-lib-test.yml | 2 ++ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-js-packages.yml b/.github/workflows/test-js-packages.yml index 7f22b8cc9532f..47202a6eed8c6 100644 --- a/.github/workflows/test-js-packages.yml +++ b/.github/workflows/test-js-packages.yml @@ -39,22 +39,14 @@ jobs: PATTERNS: | packages/** - - name: Docs related changes - id: docs - uses: technote-space/get-diff-action@v6 - with: - PATTERNS: | - docs/** - outputs: ci: ${{ steps.ci.outputs.diff != ''}} packages: ${{ steps.packages.outputs.diff != '' }} - docs: ${{ steps.docs.outputs.diff != '' }} js_packages: name: "JS Package Tests (${{matrix.os.name}}, Node ${{matrix.node-version}})" timeout-minutes: 30 - if: needs.determine_jobs.outputs.ci == 'true' || needs.determine_jobs.outputs.packages == 'true' || needs.determine_jobs.outputs.docs == 'true' + if: needs.determine_jobs.outputs.ci == 'true' || needs.determine_jobs.outputs.packages == 'true' needs: [determine_jobs] runs-on: ${{ matrix.os.runner }} strategy: diff --git a/.github/workflows/turborepo-native-lib-test.yml b/.github/workflows/turborepo-native-lib-test.yml index 0b6e5a31cbc22..8368d2a55669d 100644 --- a/.github/workflows/turborepo-native-lib-test.yml +++ b/.github/workflows/turborepo-native-lib-test.yml @@ -2,6 +2,8 @@ name: Turborepo Native Library Tests on: push: branches: [main] + paths-ignore: + - "docs/**" pull_request: permissions: From d02463f01d4756e0a8d0b6892ca7dee91db2a51b Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Tue, 5 Nov 2024 09:13:47 -0500 Subject: [PATCH 06/12] fix(mfe): build internal package if present in graph (#9383) ### Description Previously we were relying on the proxy already being built, for non-internal use cases this will be true, but for the internal case we need to make sure it is built before we try to start the proxy. ### Testing Instructions - Edit `packages/micro-frontends/dist/bin/cli.cjs` and add a `throw new Error("oops")` to the top - `turbo vercel-docs#dev` should fail due to the proxy script failing - Now try with the PR `turbo`: `turbo_dev --skip-infer vercel-docs#dev`, you will see the MFE package being built before the proxy is started. --- crates/turborepo-lib/src/turbo_json/loader.rs | 7 ++++++- crates/turborepo-lib/src/turbo_json/mod.rs | 8 +++++++- crates/turborepo-micro-frontend/src/lib.rs | 3 ++- crates/turborepo-unescape/src/lib.rs | 5 +++++ 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/crates/turborepo-lib/src/turbo_json/loader.rs b/crates/turborepo-lib/src/turbo_json/loader.rs index 4a4ddbda55bfc..1b895f6ee690e 100644 --- a/crates/turborepo-lib/src/turbo_json/loader.rs +++ b/crates/turborepo-lib/src/turbo_json/loader.rs @@ -1,8 +1,10 @@ use std::collections::HashMap; +use itertools::Itertools; use tracing::debug; use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf}; use turborepo_errors::Spanned; +use turborepo_micro_frontend::MICRO_FRONTENDS_PACKAGE_INTERNAL; use turborepo_repository::{ package_graph::{PackageInfo, PackageName}, package_json::PackageJson, @@ -184,7 +186,10 @@ impl TurboJsonLoader { Error::NoTurboJSON => Ok(TurboJson::default()), err => Err(err), })?; - turbo_json.with_proxy(); + let needs_proxy_build = packages + .keys() + .contains(&PackageName::from(MICRO_FRONTENDS_PACKAGE_INTERNAL)); + turbo_json.with_proxy(needs_proxy_build); Ok(turbo_json) } else { turbo_json diff --git a/crates/turborepo-lib/src/turbo_json/mod.rs b/crates/turborepo-lib/src/turbo_json/mod.rs index afa025dfc652e..25c5f02f84ba9 100644 --- a/crates/turborepo-lib/src/turbo_json/mod.rs +++ b/crates/turborepo-lib/src/turbo_json/mod.rs @@ -12,6 +12,7 @@ use serde::{Deserialize, Serialize}; use struct_iterable::Iterable; use turbopath::AbsoluteSystemPath; use turborepo_errors::Spanned; +use turborepo_micro_frontend::MICRO_FRONTENDS_PACKAGE_INTERNAL; use turborepo_repository::package_graph::ROOT_PKG_NAME; use turborepo_unescape::UnescapedString; @@ -610,7 +611,7 @@ impl TurboJson { } /// Adds a local proxy task to a workspace TurboJson - pub fn with_proxy(&mut self) { + pub fn with_proxy(&mut self, needs_proxy_build: bool) { if self.extends.is_empty() { self.extends = Spanned::new(vec!["//".into()]); } @@ -619,6 +620,11 @@ impl TurboJson { TaskName::from("proxy"), Spanned::new(RawTaskDefinition { cache: Some(Spanned::new(false)), + depends_on: needs_proxy_build.then(|| { + Spanned::new(vec![Spanned::new(UnescapedString::from(format!( + "{MICRO_FRONTENDS_PACKAGE_INTERNAL}#build" + )))]) + }), ..Default::default() }), ); diff --git a/crates/turborepo-micro-frontend/src/lib.rs b/crates/turborepo-micro-frontend/src/lib.rs index 8a0cabc54ff39..e250d7779088a 100644 --- a/crates/turborepo-micro-frontend/src/lib.rs +++ b/crates/turborepo-micro-frontend/src/lib.rs @@ -13,7 +13,8 @@ use turbopath::AbsoluteSystemPath; /// /// This is subject to change at any time. pub const DEFAULT_MICRO_FRONTENDS_CONFIG: &str = "micro-frontends.jsonc"; -pub const MICRO_FRONTENDS_PACKAGES: &[&str] = ["@vercel/micro-frontends-internal"].as_slice(); +pub const MICRO_FRONTENDS_PACKAGES: &[&str] = [MICRO_FRONTENDS_PACKAGE_INTERNAL].as_slice(); +pub const MICRO_FRONTENDS_PACKAGE_INTERNAL: &str = "@vercel/micro-frontends-internal"; /// The minimal amount of information Turborepo needs to correctly start a local /// proxy server for microfrontends diff --git a/crates/turborepo-unescape/src/lib.rs b/crates/turborepo-unescape/src/lib.rs index 27e1b9a637fd6..bbffef4ac3a9d 100644 --- a/crates/turborepo-unescape/src/lib.rs +++ b/crates/turborepo-unescape/src/lib.rs @@ -57,6 +57,11 @@ impl From for String { } } +impl From for UnescapedString { + fn from(value: String) -> Self { + Self(value) + } +} // For testing purposes impl From<&'static str> for UnescapedString { fn from(value: &'static str) -> Self { From 062874a2c6f13a70f0239ed9b076133ff54f8eba Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 5 Nov 2024 09:45:14 -0500 Subject: [PATCH 07/12] release(turborepo): 2.2.4-canary.7 (#9388) Co-authored-by: Turbobot --- packages/create-turbo/package.json | 2 +- packages/eslint-config-turbo/package.json | 2 +- packages/eslint-plugin-turbo/package.json | 2 +- packages/turbo-codemod/package.json | 2 +- packages/turbo-gen/package.json | 2 +- packages/turbo-ignore/package.json | 2 +- packages/turbo-types/package.json | 2 +- packages/turbo-workspaces/package.json | 2 +- packages/turbo/package.json | 14 +++++++------- version.txt | 2 +- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/create-turbo/package.json b/packages/create-turbo/package.json index 272bf54d4e4c3..6284cec637595 100644 --- a/packages/create-turbo/package.json +++ b/packages/create-turbo/package.json @@ -1,6 +1,6 @@ { "name": "create-turbo", - "version": "2.2.4-canary.6", + "version": "2.2.4-canary.7", "description": "Create a new Turborepo", "homepage": "https://turbo.build/repo", "license": "MIT", diff --git a/packages/eslint-config-turbo/package.json b/packages/eslint-config-turbo/package.json index f5b5698d3cadd..0ed4c379cea78 100644 --- a/packages/eslint-config-turbo/package.json +++ b/packages/eslint-config-turbo/package.json @@ -1,6 +1,6 @@ { "name": "eslint-config-turbo", - "version": "2.2.4-canary.6", + "version": "2.2.4-canary.7", "description": "ESLint config for Turborepo", "repository": { "type": "git", diff --git a/packages/eslint-plugin-turbo/package.json b/packages/eslint-plugin-turbo/package.json index a790fb0c242ae..fb18d2101ccd6 100644 --- a/packages/eslint-plugin-turbo/package.json +++ b/packages/eslint-plugin-turbo/package.json @@ -1,6 +1,6 @@ { "name": "eslint-plugin-turbo", - "version": "2.2.4-canary.6", + "version": "2.2.4-canary.7", "description": "ESLint plugin for Turborepo", "keywords": [ "turbo", diff --git a/packages/turbo-codemod/package.json b/packages/turbo-codemod/package.json index fa66c043d8404..0613a89d65ab7 100644 --- a/packages/turbo-codemod/package.json +++ b/packages/turbo-codemod/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/codemod", - "version": "2.2.4-canary.6", + "version": "2.2.4-canary.7", "description": "Provides Codemod transformations to help upgrade your Turborepo codebase when a feature is deprecated.", "homepage": "https://turbo.build/repo", "license": "MIT", diff --git a/packages/turbo-gen/package.json b/packages/turbo-gen/package.json index bd230682b7861..87a7b3a60a830 100644 --- a/packages/turbo-gen/package.json +++ b/packages/turbo-gen/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/gen", - "version": "2.2.4-canary.6", + "version": "2.2.4-canary.7", "description": "Extend a Turborepo", "homepage": "https://turbo.build/repo", "license": "MIT", diff --git a/packages/turbo-ignore/package.json b/packages/turbo-ignore/package.json index 260bd3cf7102b..166b21cf80929 100644 --- a/packages/turbo-ignore/package.json +++ b/packages/turbo-ignore/package.json @@ -1,6 +1,6 @@ { "name": "turbo-ignore", - "version": "2.2.4-canary.6", + "version": "2.2.4-canary.7", "description": "", "homepage": "https://turbo.build/repo", "keywords": [], diff --git a/packages/turbo-types/package.json b/packages/turbo-types/package.json index e6d3f34d9a98a..e5d81ecb7dc57 100644 --- a/packages/turbo-types/package.json +++ b/packages/turbo-types/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/types", - "version": "2.2.4-canary.6", + "version": "2.2.4-canary.7", "description": "Turborepo types", "homepage": "https://turbo.build/repo", "license": "MIT", diff --git a/packages/turbo-workspaces/package.json b/packages/turbo-workspaces/package.json index 32e12e8b1f47e..5bc1b17bc5624 100644 --- a/packages/turbo-workspaces/package.json +++ b/packages/turbo-workspaces/package.json @@ -1,6 +1,6 @@ { "name": "@turbo/workspaces", - "version": "2.2.4-canary.6", + "version": "2.2.4-canary.7", "description": "Tools for working with package managers", "homepage": "https://turbo.build/repo", "license": "MIT", diff --git a/packages/turbo/package.json b/packages/turbo/package.json index 39bb37ff87627..cca2ee5626d3d 100644 --- a/packages/turbo/package.json +++ b/packages/turbo/package.json @@ -1,6 +1,6 @@ { "name": "turbo", - "version": "2.2.4-canary.6", + "version": "2.2.4-canary.7", "description": "Turborepo is a high-performance build system for JavaScript and TypeScript codebases.", "repository": "https://github.com/vercel/turborepo", "bugs": "https://github.com/vercel/turborepo/issues", @@ -17,11 +17,11 @@ "bin" ], "optionalDependencies": { - "turbo-darwin-64": "2.2.4-canary.6", - "turbo-darwin-arm64": "2.2.4-canary.6", - "turbo-linux-64": "2.2.4-canary.6", - "turbo-linux-arm64": "2.2.4-canary.6", - "turbo-windows-64": "2.2.4-canary.6", - "turbo-windows-arm64": "2.2.4-canary.6" + "turbo-darwin-64": "2.2.4-canary.7", + "turbo-darwin-arm64": "2.2.4-canary.7", + "turbo-linux-64": "2.2.4-canary.7", + "turbo-linux-arm64": "2.2.4-canary.7", + "turbo-windows-64": "2.2.4-canary.7", + "turbo-windows-arm64": "2.2.4-canary.7" } } diff --git a/version.txt b/version.txt index 83c28e547e68d..c1cc433c75e58 100644 --- a/version.txt +++ b/version.txt @@ -1,2 +1,2 @@ -2.2.4-canary.6 +2.2.4-canary.7 canary From 7c9731ba8c670c89737bd85333d98cca6ce1d8c9 Mon Sep 17 00:00:00 2001 From: Anthony Shew Date: Tue, 5 Nov 2024 08:38:06 -0700 Subject: [PATCH 08/12] docs: Clarify that `TURBO_FORCE` is a boolean. (#9380) ### Description According to some user feedback, it was unclear that `TURBO_FORCE` accepts a boolean. Clarifying that here. --- docs/repo-docs/reference/system-environment-variables.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/repo-docs/reference/system-environment-variables.mdx b/docs/repo-docs/reference/system-environment-variables.mdx index 9032bfd473882..9eec6200cf085 100644 --- a/docs/repo-docs/reference/system-environment-variables.mdx +++ b/docs/repo-docs/reference/system-environment-variables.mdx @@ -17,7 +17,7 @@ System environment variables are always overridden by flag values provided direc | `TURBO_CI_VENDOR_ENV_KEY` | Set a prefix for environment variables that you want **excluded** from [Framework Inference](/repo/docs/crafting-your-repository/using-environment-variables#framework-inference). **NOTE**: This does not need to be set by the user and should be configured automatically by supported platforms. | | `TURBO_DANGEROUSLY_DISABLE_PACKAGE_MANAGER_CHECK` | Disable checking the `packageManager` field in `package.json`. You may run into [errors and unexpected caching behavior](/repo/docs/reference/run#--dangerously-disable-package-manager-check) when disabling this check. Use `true` or `1` to disable. | | `TURBO_DOWNLOAD_LOCAL_ENABLED` | Enables global `turbo` to install the correct local version if one is not found. | -| `TURBO_FORCE` | Always force all tasks to run in full, opting out of all caching. | +| `TURBO_FORCE` | Set to `true` to force all tasks to run in full, opting out of all caching. | | `TURBO_GLOBAL_WARNING_DISABLED` | Disable warning when global `turbo` cannot find a local version to use. | | `TURBO_PRINT_VERSION_DISABLED` | Disable printing the version of `turbo` that is being executed. | | `TURBO_LOG_ORDER` | Set the [log order](/repo/docs/reference/run#--log-order-option). Allowed values are `grouped` and `default`. | @@ -32,7 +32,7 @@ System environment variables are always overridden by flag values provided direc | `TURBO_REMOTE_ONLY` | Always ignore the local filesystem cache for all tasks. | | `TURBO_RUN_SUMMARY` | Generate a [Run Summary](/repo/docs/reference/run#--summarize) when you run tasks. | | `TURBO_SCM_BASE` | Base used by `--affected` when calculating what has changed from `base...head` | -| `TURBO_SCM_HEAD` | Head used by `--affected` when calculating what has changed from `base...head` | +| `TURBO_SCM_HEAD` | Head used by `--affected` when calculating what has changed from `base...head` | | `TURBO_TEAM` | The account name associated with your repository. When using [Vercel Remote Cache](https://vercel.com/docs/monorepos/remote-caching#vercel-remote-cache), this is your team's slug. | | `TURBO_TEAMID` | The account identifier associated with your repository. When using [Vercel Remote Cache](https://vercel.com/docs/monorepos/remote-caching#vercel-remote-cache), this is your team's ID. | | `TURBO_TELEMETRY_MESSAGE_DISABLED` | Disable the message notifying you that [Telemetry](/repo/docs/telemetry) is enabled. | From 3d9b2597abd5d6fb39b1e1e1fb853221a09d3b3f Mon Sep 17 00:00:00 2001 From: Nicholas Yang Date: Tue, 5 Nov 2024 11:10:19 -0500 Subject: [PATCH 09/12] fix(query): validate package name (#9370) ### Description Before we'd just trust that a package would exist with the name provided. This would lead to errors when using the package but not when constructing it. Now we actually validate it during construction. ### Testing Instructions Added test for invalid package constructor --------- Co-authored-by: Chris Olszewski --- crates/turborepo-lib/src/commands/query.rs | 9 +- crates/turborepo-lib/src/query/mod.rs | 83 ++++--- crates/turborepo-lib/src/query/package.rs | 22 +- crates/turborepo-lib/src/query/task.rs | 220 ++++++++---------- crates/turborepo/tests/query.rs | 12 + ...ckage_that_doesn't_exist_(npm@10.5.0).snap | 21 ++ 6 files changed, 204 insertions(+), 163 deletions(-) create mode 100644 crates/turborepo/tests/snapshots/query__basic_monorepo_get_package_that_doesn't_exist_(npm@10.5.0).snap diff --git a/crates/turborepo-lib/src/commands/query.rs b/crates/turborepo-lib/src/commands/query.rs index 24fac3e27e66c..c3d79df1f9fe0 100644 --- a/crates/turborepo-lib/src/commands/query.rs +++ b/crates/turborepo-lib/src/commands/query.rs @@ -33,10 +33,10 @@ struct QueryError { impl QueryError { fn get_index_from_row_column(query: &str, row: usize, column: usize) -> usize { let mut index = 0; - for line in query.lines().take(row + 1) { + for line in query.lines().take(row.saturating_sub(1)) { index += line.len() + 1; } - index + column + index + column - 1 } fn new(server_error: ServerError, query: String) -> Self { let span: Option = server_error.locations.first().map(|location| { @@ -106,9 +106,8 @@ pub async fn run( let request = Request::new(&query).variables(variables); let result = schema.execute(request).await; - if result.errors.is_empty() { - println!("{}", serde_json::to_string_pretty(&result)?); - } else { + println!("{}", serde_json::to_string_pretty(&result)?); + if !result.errors.is_empty() { for error in result.errors { let error = QueryError::new(error, query.clone()); eprintln!("{:?}", Report::new(error)); diff --git a/crates/turborepo-lib/src/query/mod.rs b/crates/turborepo-lib/src/query/mod.rs index b2dd00ce63f00..97e02e5208ee4 100644 --- a/crates/turborepo-lib/src/query/mod.rs +++ b/crates/turborepo-lib/src/query/mod.rs @@ -3,11 +3,14 @@ mod package; mod server; mod task; -use std::{io, sync::Arc}; +use std::{ + io, + ops::{Deref, DerefMut}, + sync::Arc, +}; use async_graphql::{http::GraphiQLSource, *}; use axum::{response, response::IntoResponse}; -use itertools::Itertools; use miette::Diagnostic; use package::Package; pub use server::run_server; @@ -74,6 +77,19 @@ pub struct Array { length: usize, } +impl Deref for Array { + type Target = [T]; + fn deref(&self) -> &Self::Target { + &self.items + } +} + +impl DerefMut for Array { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.items + } +} + impl FromIterator for Array { fn from_iter>(iter: I) -> Self { let items: Vec<_> = iter.into_iter().collect(); @@ -120,7 +136,7 @@ struct PackagePredicate { impl PackagePredicate { fn check_equals(pkg: &Package, field: &PackageFields, value: &Any) -> bool { match (field, &value.0) { - (PackageFields::Name, Value::String(name)) => pkg.name.as_ref() == name, + (PackageFields::Name, Value::String(name)) => pkg.get_name().as_ref() == name, (PackageFields::DirectDependencyCount, Value::Number(n)) => { let Some(n) = n.as_u64() else { return false; @@ -247,7 +263,7 @@ impl PackagePredicate { fn check_has(pkg: &Package, field: &PackageFields, value: &Any) -> bool { match (field, &value.0) { - (PackageFields::Name, Value::String(name)) => pkg.name.as_ref() == name, + (PackageFields::Name, Value::String(name)) => pkg.get_name().as_str() == name, (PackageFields::TaskName, Value::String(name)) => pkg.get_tasks().contains_key(name), _ => false, } @@ -491,7 +507,7 @@ impl RepositoryQuery { let mut opts = self.run.opts().clone(); opts.scope_opts.affected_range = Some((base, head)); - Ok(RunBuilder::calculate_filtered_packages( + let mut packages = RunBuilder::calculate_filtered_packages( self.run.repo_root(), &opts, self.run.pkg_dep_graph(), @@ -499,24 +515,28 @@ impl RepositoryQuery { self.run.root_turbo_json(), )? .into_iter() - .map(|(package, reason)| ChangedPackage { - package: Package { - run: self.run.clone(), - name: package, - }, - reason: reason.into(), + .map(|(package, reason)| { + Ok(ChangedPackage { + package: Package::new(self.run.clone(), package)?, + reason: reason.into(), + }) }) - .filter(|package| filter.as_ref().map_or(true, |f| f.check(&package.package))) - .sorted_by(|a, b| a.package.name.cmp(&b.package.name)) - .collect()) + .filter(|package: &Result| { + let Ok(package) = package.as_ref() else { + return true; + }; + filter.as_ref().map_or(true, |f| f.check(&package.package)) + }) + .collect::, _>>()?; + + packages.sort_by(|a, b| a.package.get_name().cmp(b.package.get_name())); + Ok(packages) } + /// Gets a single package by name async fn package(&self, name: String) -> Result { let name = PackageName::from(name); - Ok(Package { - run: self.run.clone(), - name, - }) + Package::new(self.run.clone(), name) } async fn version(&self) -> &'static str { @@ -536,29 +556,26 @@ impl RepositoryQuery { /// Gets a list of packages that match the given filter async fn packages(&self, filter: Option) -> Result, Error> { let Some(filter) = filter else { - return Ok(self + let mut packages = self .run .pkg_dep_graph() .packages() - .map(|(name, _)| Package { - run: self.run.clone(), - name: name.clone(), - }) - .sorted_by(|a, b| a.name.cmp(&b.name)) - .collect()); + .map(|(name, _)| Package::new(self.run.clone(), name.clone())) + .collect::, _>>()?; + packages.sort_by(|a, b| a.get_name().cmp(b.get_name())); + return Ok(packages); }; - Ok(self + let mut packages = self .run .pkg_dep_graph() .packages() - .map(|(name, _)| Package { - run: self.run.clone(), - name: name.clone(), - }) - .filter(|pkg| filter.check(pkg)) - .sorted_by(|a, b| a.name.cmp(&b.name)) - .collect()) + .map(|(name, _)| Package::new(self.run.clone(), name.clone())) + .filter(|pkg| pkg.as_ref().map_or(false, |pkg| filter.check(pkg))) + .collect::, _>>()?; + packages.sort_by(|a, b| a.get_name().cmp(b.get_name())); + + Ok(packages) } } diff --git a/crates/turborepo-lib/src/query/package.rs b/crates/turborepo-lib/src/query/package.rs index 621f6385dfbe2..dc3a3b8d35e89 100644 --- a/crates/turborepo-lib/src/query/package.rs +++ b/crates/turborepo-lib/src/query/package.rs @@ -12,11 +12,29 @@ use crate::{ #[derive(Clone)] pub struct Package { - pub run: Arc, - pub name: PackageName, + run: Arc, + name: PackageName, } impl Package { + pub fn new(run: Arc, name: PackageName) -> Result { + run.pkg_dep_graph() + .package_info(&name) + .ok_or_else(|| Error::PackageNotFound(name.clone()))?; + + Ok(Self { run, name }) + } + + pub fn run(&self) -> &Arc { + &self.run + } + + /// This uses a different naming convention because we already have a + /// `name` resolver defined for GraphQL + pub fn get_name(&self) -> &PackageName { + &self.name + } + pub fn get_tasks(&self) -> HashMap> { self.run .pkg_dep_graph() diff --git a/crates/turborepo-lib/src/query/task.rs b/crates/turborepo-lib/src/query/task.rs index 5fbe69c7c5219..72997203071ec 100644 --- a/crates/turborepo-lib/src/query/task.rs +++ b/crates/turborepo-lib/src/query/task.rs @@ -1,12 +1,11 @@ use std::sync::Arc; use async_graphql::Object; -use itertools::Itertools; use turborepo_errors::Spanned; use crate::{ engine::TaskNode, - query::{package::Package, Array}, + query::{package::Package, Array, Error}, run::{task_id::TaskId, Run}, }; @@ -17,18 +16,37 @@ pub struct RepositoryTask { } impl RepositoryTask { - pub fn new(task_id: &TaskId, run: &Arc) -> Self { - let package = Package { - name: task_id.package().into(), - run: run.clone(), - }; + pub fn new(task_id: &TaskId, run: &Arc) -> Result { + let package = Package::new(run.clone(), task_id.package().into())?; let script = package.get_tasks().get(task_id.task()).cloned(); - RepositoryTask { + Ok(RepositoryTask { name: task_id.task().to_string(), package, script, - } + }) + } + + fn collect_and_sort<'a>( + &self, + task_id: &TaskId<'a>, + tasks: impl IntoIterator, + ) -> Result, Error> { + let mut tasks = tasks + .into_iter() + .filter_map(|task| match task { + TaskNode::Root => None, + TaskNode::Task(task) if task == task_id => None, + TaskNode::Task(task) => Some(RepositoryTask::new(task, self.package.run())), + }) + .collect::, _>>()?; + tasks.sort_by(|a, b| { + a.package + .get_name() + .cmp(b.package.get_name()) + .then_with(|| a.name.cmp(&b.name)) + }); + Ok(tasks) } } @@ -43,96 +61,72 @@ impl RepositoryTask { } async fn full_name(&self) -> String { - format!("{}#{}", self.package.name, self.name) + format!("{}#{}", self.package.get_name(), self.name) } async fn script(&self) -> Option { self.script.as_ref().map(|script| script.value.to_string()) } - async fn direct_dependents(&self) -> Array { - let task_id = TaskId::from_static(self.package.name.to_string(), self.name.clone()); - self.package - .run - .engine() - .dependents(&task_id) - .into_iter() - .flatten() - .filter_map(|task| match task { - TaskNode::Root => None, - TaskNode::Task(task) if task == &task_id => None, - TaskNode::Task(task) => Some(RepositoryTask::new(task, &self.package.run)), - }) - .sorted_by(|a, b| { - a.package - .name - .cmp(&b.package.name) - .then_with(|| a.name.cmp(&b.name)) - }) - .collect() + async fn direct_dependents(&self) -> Result, Error> { + let task_id = TaskId::from_static(self.package.get_name().to_string(), self.name.clone()); + + self.collect_and_sort( + &task_id, + self.package + .run() + .engine() + .dependents(&task_id) + .into_iter() + .flatten(), + ) } - async fn direct_dependencies(&self) -> Array { - let task_id = TaskId::new(self.package.name.as_ref(), &self.name); - - self.package - .run - .engine() - .dependencies(&task_id) - .into_iter() - .flatten() - .filter_map(|task| match task { - TaskNode::Root => None, - TaskNode::Task(task) if task == &task_id => None, - TaskNode::Task(task) => Some(RepositoryTask::new(task, &self.package.run)), - }) - .sorted_by(|a, b| { - a.package - .name - .cmp(&b.package.name) - .then_with(|| a.name.cmp(&b.name)) - }) - .collect() + async fn direct_dependencies(&self) -> Result, Error> { + let task_id = TaskId::new(self.package.get_name().as_ref(), &self.name); + + self.collect_and_sort( + &task_id, + self.package + .run() + .engine() + .dependencies(&task_id) + .into_iter() + .flatten(), + ) } - async fn indirect_dependents(&self) -> Array { - let task_id = TaskId::from_static(self.package.name.to_string(), self.name.clone()); + async fn indirect_dependents(&self) -> Result, Error> { + let task_id = TaskId::from_static(self.package.get_name().to_string(), self.name.clone()); let direct_dependents = self .package - .run + .run() .engine() .dependencies(&task_id) .unwrap_or_default(); - self.package - .run - .engine() - .transitive_dependents(&task_id) - .into_iter() - .filter(|node| !direct_dependents.contains(node)) - .filter_map(|node| match node { - TaskNode::Root => None, - TaskNode::Task(task) if task == &task_id => None, - TaskNode::Task(task) => Some(RepositoryTask::new(task, &self.package.run)), - }) - .sorted_by(|a, b| { - a.package - .name - .cmp(&b.package.name) - .then_with(|| a.name.cmp(&b.name)) - }) - .collect() + + self.collect_and_sort( + &task_id, + self.package + .run() + .engine() + .transitive_dependents(&task_id) + .into_iter() + .filter(|node| !direct_dependents.contains(node)), + ) } - async fn indirect_dependencies(&self) -> Array { - let task_id = TaskId::from_static(self.package.name.to_string(), self.name.clone()); + async fn indirect_dependencies(&self) -> Result, Error> { + let task_id = TaskId::from_static(self.package.get_name().to_string(), self.name.clone()); let direct_dependencies = self .package - .run + .run() .engine() .dependencies(&task_id) .unwrap_or_default(); - self.package - .run + let mut dependencies = self + .package + .run() .engine() .transitive_dependencies(&task_id) .into_iter() @@ -140,56 +134,36 @@ impl RepositoryTask { .filter_map(|node| match node { TaskNode::Root => None, TaskNode::Task(task) if task == &task_id => None, - TaskNode::Task(task) => Some(RepositoryTask::new(task, &self.package.run)), + TaskNode::Task(task) => Some(RepositoryTask::new(task, self.package.run())), }) - .sorted_by(|a, b| { - a.package - .name - .cmp(&b.package.name) - .then_with(|| a.name.cmp(&b.name)) - }) - .collect() + .collect::, _>>()?; + + dependencies.sort_by(|a, b| { + a.package + .get_name() + .cmp(b.package.get_name()) + .then_with(|| a.name.cmp(&b.name)) + }); + + Ok(dependencies) } - async fn all_dependents(&self) -> Array { - let task_id = TaskId::from_static(self.package.name.to_string(), self.name.clone()); - self.package - .run - .engine() - .transitive_dependents(&task_id) - .into_iter() - .filter_map(|node| match node { - TaskNode::Root => None, - TaskNode::Task(task) if task == &task_id => None, - TaskNode::Task(task) => Some(RepositoryTask::new(task, &self.package.run)), - }) - .sorted_by(|a, b| { - a.package - .name - .cmp(&b.package.name) - .then_with(|| a.name.cmp(&b.name)) - }) - .collect() + async fn all_dependents(&self) -> Result, Error> { + let task_id = TaskId::from_static(self.package.get_name().to_string(), self.name.clone()); + self.collect_and_sort( + &task_id, + self.package.run().engine().transitive_dependents(&task_id), + ) } - async fn all_dependencies(&self) -> Array { - let task_id = TaskId::from_static(self.package.name.to_string(), self.name.clone()); - self.package - .run - .engine() - .transitive_dependencies(&task_id) - .into_iter() - .filter_map(|node| match node { - TaskNode::Root => None, - TaskNode::Task(task) if task == &task_id => None, - TaskNode::Task(task) => Some(RepositoryTask::new(task, &self.package.run)), - }) - .sorted_by(|a, b| { - a.package - .name - .cmp(&b.package.name) - .then_with(|| a.name.cmp(&b.name)) - }) - .collect() + async fn all_dependencies(&self) -> Result, Error> { + let task_id = TaskId::from_static(self.package.get_name().to_string(), self.name.clone()); + self.collect_and_sort( + &task_id, + self.package + .run() + .engine() + .transitive_dependencies(&task_id), + ) } } diff --git a/crates/turborepo/tests/query.rs b/crates/turborepo/tests/query.rs index 08414593677b1..4ea85f7df220e 100644 --- a/crates/turborepo/tests/query.rs +++ b/crates/turborepo/tests/query.rs @@ -1,5 +1,17 @@ mod common; +#[test] +fn test_query() -> Result<(), anyhow::Error> { + check_json!( + "basic_monorepo", + "npm@10.5.0", + "query", + "get package that doesn't exist" => "query { package(name: \"doesnotexist\") { path } }", + ); + + Ok(()) +} + #[cfg(not(windows))] #[test] fn test_double_symlink() -> Result<(), anyhow::Error> { diff --git a/crates/turborepo/tests/snapshots/query__basic_monorepo_get_package_that_doesn't_exist_(npm@10.5.0).snap b/crates/turborepo/tests/snapshots/query__basic_monorepo_get_package_that_doesn't_exist_(npm@10.5.0).snap new file mode 100644 index 0000000000000..b1c34e3924874 --- /dev/null +++ b/crates/turborepo/tests/snapshots/query__basic_monorepo_get_package_that_doesn't_exist_(npm@10.5.0).snap @@ -0,0 +1,21 @@ +--- +source: crates/turborepo/tests/query.rs +expression: query_output +--- +{ + "data": null, + "errors": [ + { + "message": "package not found: doesnotexist", + "locations": [ + { + "line": 1, + "column": 9 + } + ], + "path": [ + "package" + ] + } + ] +} From baa4eb5afb7164d9311b85d5b4580633e064b6f1 Mon Sep 17 00:00:00 2001 From: Nicholas Yang Date: Tue, 5 Nov 2024 12:06:34 -0500 Subject: [PATCH 10/12] feat(trace): pretty print trace errors to logs (#9367) ### Description To help debug trace, pretty print the errors to stderr by default. Includes fancy error messages either via SWC or our own miette infrastructure ### Testing Instructions Try using trace and hit an error --- Cargo.lock | 1 + crates/turbo-trace/Cargo.toml | 4 +- crates/turbo-trace/src/tracer.rs | 72 +++++++++++++++---- crates/turborepo-lib/src/commands/query.rs | 4 +- crates/turborepo-lib/src/query/file.rs | 13 +++- crates/turborepo/tests/query.rs | 2 +- ...id.ts`_with_dependencies_(npm@10.5.0).snap | 2 +- .../integration/tests/turbo-trace.t | 5 +- 8 files changed, 78 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bba34a03ca13b..6eaad0a580924 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5223,6 +5223,7 @@ dependencies = [ "swc_atoms", "swc_eq_ignore_macros", "swc_visit", + "termcolor", "tracing", "unicode-width", "url", diff --git a/crates/turbo-trace/Cargo.toml b/crates/turbo-trace/Cargo.toml index 99f7fd50c3357..8fb3ed3bd84c7 100644 --- a/crates/turbo-trace/Cargo.toml +++ b/crates/turbo-trace/Cargo.toml @@ -11,12 +11,12 @@ futures = { workspace = true } globwalk = { version = "0.1.0", path = "../turborepo-globwalk" } miette = { workspace = true, features = ["fancy"] } oxc_resolver = { version = "2.0.0" } -swc_common = { workspace = true, features = ["concurrent"] } +swc_common = { workspace = true, features = ["concurrent", "tty-emitter"] } swc_ecma_ast = { workspace = true } swc_ecma_parser = { workspace = true } swc_ecma_visit = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true } +tokio = { workspace = true, features = ["full"] } tracing = { workspace = true } tracing-subscriber = { workspace = true } turbopath = { workspace = true } diff --git a/crates/turbo-trace/src/tracer.rs b/crates/turbo-trace/src/tracer.rs index 351c6ff0b0e76..5057c22befd1e 100644 --- a/crates/turbo-trace/src/tracer.rs +++ b/crates/turbo-trace/src/tracer.rs @@ -2,23 +2,33 @@ use std::{collections::HashMap, sync::Arc}; use camino::Utf8PathBuf; use globwalk::WalkType; -use miette::{Diagnostic, NamedSource, SourceSpan}; +use miette::{Diagnostic, Report, SourceSpan}; use oxc_resolver::{ EnforceExtension, ResolveError, ResolveOptions, Resolver, TsconfigOptions, TsconfigReferences, }; -use swc_common::{comments::SingleThreadedComments, input::StringInput, FileName, SourceMap}; +use swc_common::{ + comments::SingleThreadedComments, + errors::{ColorConfig, Handler}, + input::StringInput, + FileName, SourceMap, +}; use swc_ecma_ast::EsVersion; use swc_ecma_parser::{lexer::Lexer, Capturing, EsSyntax, Parser, Syntax, TsSyntax}; use swc_ecma_visit::VisitWith; use thiserror::Error; use tokio::task::JoinSet; -use tracing::debug; +use tracing::{debug, error}; use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, PathError}; use crate::import_finder::ImportFinder; #[derive(Debug, Default)] pub struct SeenFile { + // We have to add these because of a Rust bug where dead code analysis + // doesn't work properly in multi-target crates + // (i.e. crates with both a binary and library) + // https://github.com/rust-lang/rust/issues/95513 + #[allow(dead_code)] pub ast: Option, } @@ -31,35 +41,61 @@ pub struct Tracer { import_type: ImportType, } -#[derive(Debug, Error, Diagnostic)] +#[derive(Clone, Debug, Error, Diagnostic)] pub enum TraceError { #[error("failed to parse file {}: {:?}", .0, .1)] ParseError(AbsoluteSystemPathBuf, swc_ecma_parser::error::Error), #[error("failed to read file: {0}")] FileNotFound(AbsoluteSystemPathBuf), #[error(transparent)] - PathEncoding(PathError), + PathEncoding(Arc), #[error("tracing a root file `{0}`, no parent found")] RootFile(AbsoluteSystemPathBuf), - #[error("failed to resolve import to `{path}`")] + #[error("failed to resolve import to `{import}` in `{file_path}`")] Resolve { - path: String, + import: String, + file_path: String, #[label("import here")] span: SourceSpan, #[source_code] - text: NamedSource, + text: String, }, #[error("failed to walk files")] - GlobError(#[from] globwalk::WalkError), + GlobError(Arc), +} + +impl TraceResult { + #[allow(dead_code)] + pub fn emit_errors(&self) { + let handler = Handler::with_tty_emitter( + ColorConfig::Auto, + true, + false, + Some(self.source_map.clone()), + ); + for error in &self.errors { + match error { + TraceError::ParseError(_, e) => { + e.clone().into_diagnostic(&handler).emit(); + } + e => { + eprintln!("{:?}", Report::new(e.clone())); + } + } + } + } } pub struct TraceResult { + #[allow(dead_code)] + source_map: Arc, pub errors: Vec, pub files: HashMap, } /// The type of imports to trace. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[allow(dead_code)] pub enum ImportType { /// Trace all imports. All, @@ -90,6 +126,7 @@ impl Tracer { } } + #[allow(dead_code)] pub fn set_import_type(&mut self, import_type: ImportType) { self.import_type = import_type; } @@ -162,7 +199,7 @@ impl Tracer { match resolver.resolve(file_dir, import) { Ok(resolved) => { debug!("resolved {:?}", resolved); - match resolved.into_path_buf().try_into() { + match resolved.into_path_buf().try_into().map_err(Arc::new) { Ok(path) => files.push(path), Err(err) => { errors.push(TraceError::PathEncoding(err)); @@ -176,11 +213,14 @@ impl Tracer { Err(err) => { debug!("failed to resolve: {:?}", err); let (start, end) = source_map.span_to_char_offset(&source_file, *span); + let start = start as usize; + let end = end as usize; errors.push(TraceError::Resolve { - path: import.to_string(), - span: (start as usize, end as usize).into(), - text: NamedSource::new(file_path.to_string(), file_content.clone()), + import: import.to_string(), + file_path: file_path.to_string(), + span: SourceSpan::new(start.into(), (end - start).into()), + text: file_content.clone(), }); continue; } @@ -299,6 +339,7 @@ impl Tracer { } TraceResult { + source_map: self.source_map.clone(), files: seen, errors: self.errors, } @@ -322,8 +363,9 @@ impl Tracer { Ok(files) => files, Err(e) => { return TraceResult { + source_map: self.source_map.clone(), files: HashMap::new(), - errors: vec![e.into()], + errors: vec![TraceError::GlobError(Arc::new(e))], } } }; @@ -331,6 +373,7 @@ impl Tracer { let mut futures = JoinSet::new(); let resolver = Arc::new(self.create_resolver()); + let source_map = self.source_map.clone(); let shared_self = Arc::new(self); for file in files { @@ -380,6 +423,7 @@ impl Tracer { } TraceResult { + source_map, files: usages, errors, } diff --git a/crates/turborepo-lib/src/commands/query.rs b/crates/turborepo-lib/src/commands/query.rs index c3d79df1f9fe0..4a0a01363922c 100644 --- a/crates/turborepo-lib/src/commands/query.rs +++ b/crates/turborepo-lib/src/commands/query.rs @@ -80,7 +80,9 @@ pub async fn run( // If the arg starts with "query" or "mutation", and ends in a bracket, it's // likely a direct query If it doesn't, it's a file path, so we need to // read it - let query = if (trimmed_query.starts_with("query") || trimmed_query.starts_with("mutation")) + let query = if (trimmed_query.starts_with("query") + || trimmed_query.starts_with("mutation") + || trimmed_query.starts_with('{')) && trimmed_query.ends_with('}') { query diff --git a/crates/turborepo-lib/src/query/file.rs b/crates/turborepo-lib/src/query/file.rs index 2b1d06112122c..200a2c9ffc2d7 100644 --- a/crates/turborepo-lib/src/query/file.rs +++ b/crates/turborepo-lib/src/query/file.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use async_graphql::{Enum, Object, SimpleObject}; use camino::Utf8PathBuf; use itertools::Itertools; +use miette::SourceCode; use swc_ecma_ast::EsVersion; use swc_ecma_parser::{EsSyntax, Syntax, TsSyntax}; use turbo_trace::Tracer; @@ -108,9 +109,13 @@ impl From for TraceError { message: format!("failed to glob files: {}", err), ..Default::default() }, - turbo_trace::TraceError::Resolve { span, text, .. } => { + turbo_trace::TraceError::Resolve { + span, + text, + file_path, + .. + } => { let import = text - .inner() .read_span(&span, 1, 1) .ok() .map(|s| String::from_utf8_lossy(s.data()).to_string()); @@ -118,7 +123,7 @@ impl From for TraceError { TraceError { message, import, - path: Some(text.name().to_string()), + path: Some(file_path), start: Some(span.offset()), end: Some(span.offset() + span.len()), } @@ -204,6 +209,7 @@ impl File { } let mut result = tracer.trace(depth).await; + result.emit_errors(); // Remove the file itself from the result result.files.remove(&self.path); TraceResult::new(result, self.run.clone()) @@ -225,6 +231,7 @@ impl File { } let mut result = tracer.reverse_trace().await; + result.emit_errors(); // Remove the file itself from the result result.files.remove(&self.path); TraceResult::new(result, self.run.clone()) diff --git a/crates/turborepo/tests/query.rs b/crates/turborepo/tests/query.rs index 4ea85f7df220e..46818a5775092 100644 --- a/crates/turborepo/tests/query.rs +++ b/crates/turborepo/tests/query.rs @@ -56,7 +56,7 @@ fn test_trace() -> Result<(), anyhow::Error> { "get `main.ts` with dependencies" => "query { file(path: \"main.ts\") { path, dependencies { files { items { path } } } } }", "get `button.tsx` with dependencies" => "query { file(path: \"button.tsx\") { path, dependencies { files { items { path } } } } }", "get `circular.ts` with dependencies" => "query { file(path: \"circular.ts\") { path dependencies { files { items { path } } } } }", - "get `invalid.ts` with dependencies" => "query { file(path: \"invalid.ts\") { path dependencies { files { items { path } } errors { items { message } } } } }", + "get `invalid.ts` with dependencies" => "query { file(path: \"invalid.ts\") { path dependencies { files { items { path } } errors { items { import } } } } }", "get `main.ts` with depth = 0" => "query { file(path: \"main.ts\") { path dependencies(depth: 1) { files { items { path } } } } }", "get `with_prefix.ts` with dependencies" => "query { file(path: \"with_prefix.ts\") { path dependencies { files { items { path } } } } }", "get `import_value_and_type.ts` with all dependencies" => "query { file(path: \"import_value_and_type.ts\") { path dependencies(importType: ALL) { files { items { path } } } } }", diff --git a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`invalid.ts`_with_dependencies_(npm@10.5.0).snap b/crates/turborepo/tests/snapshots/query__turbo_trace_get_`invalid.ts`_with_dependencies_(npm@10.5.0).snap index c67ae3145901c..963d257ac7d21 100644 --- a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`invalid.ts`_with_dependencies_(npm@10.5.0).snap +++ b/crates/turborepo/tests/snapshots/query__turbo_trace_get_`invalid.ts`_with_dependencies_(npm@10.5.0).snap @@ -23,7 +23,7 @@ expression: query_output "errors": { "items": [ { - "message": "failed to resolve import to `./non-existent-file.js`" + "import": "import foo from \"./non-existent-file.js\";\nimport { Button } from \"./button.tsx\";\n" } ] } diff --git a/turborepo-tests/integration/tests/turbo-trace.t b/turborepo-tests/integration/tests/turbo-trace.t index 56a1f13dea51d..72356cc2e3a79 100644 --- a/turborepo-tests/integration/tests/turbo-trace.t +++ b/turborepo-tests/integration/tests/turbo-trace.t @@ -87,8 +87,7 @@ Setup } Trace file with invalid import - $ ${TURBO} query "query { file(path: \"invalid.ts\") { path dependencies { files { items { path } } errors { items { message } } } } }" - WARNING query command is experimental and may change in the future + $ ${TURBO} query "query { file(path: \"invalid.ts\") { path dependencies { files { items { path } } errors { items { message } } } } }" 2>/dev/null { "data": { "file": { @@ -110,7 +109,7 @@ Trace file with invalid import "errors": { "items": [ { - "message": "failed to resolve import to `./non-existent-file.js`" + "message": "failed to resolve import to `./non-existent-file.js` in `.*`" (re) } ] } From 4cfbc449150abdd676d8503740ec1e6f3bd10352 Mon Sep 17 00:00:00 2001 From: Nicholas Yang Date: Tue, 5 Nov 2024 13:17:28 -0500 Subject: [PATCH 11/12] feat(turbo): add cache flag (#9348) ### Description Adds a cache flag that lets you set cache permissions per cache type and per action (read or write) ### Testing Instructions Adds tests in `turborepo_cache/src/config.rs` for parsing and in `turborepo-lib/src/opts.rs` for loading and resolving --- Cargo.lock | 1 + crates/turborepo-cache/Cargo.toml | 1 + crates/turborepo-cache/src/async_cache.rs | 42 ++- crates/turborepo-cache/src/config.rs | 231 +++++++++++++ crates/turborepo-cache/src/lib.rs | 42 ++- crates/turborepo-cache/src/multiplexer.rs | 94 +++--- crates/turborepo-lib/src/cli/mod.rs | 88 +++-- crates/turborepo-lib/src/commands/mod.rs | 15 +- crates/turborepo-lib/src/config/env.rs | 14 +- crates/turborepo-lib/src/config/mod.rs | 9 + crates/turborepo-lib/src/opts.rs | 168 ++++++++-- crates/turborepo-lib/src/run/builder.rs | 3 +- crates/turborepo-lib/src/run/cache.rs | 14 +- crates/turborepo-lib/src/run/mod.rs | 4 +- .../turborepo_lib__opts__test__force.snap | 16 + ...b__opts__test__force_remote_r,local_r.snap | 7 + .../turborepo_lib__opts__test__no-cache.snap | 16 + ...pts__test__no-cache_remote_w,local_rw.snap | 7 + ...b__opts__test__remote-cache-read-only.snap | 16 + ...ote-cache-read-only_remote_rw,local_r.snap | 7 + ...urborepo_lib__opts__test__remote-only.snap | 16 + ...__test__remote-only_remote_r,local_rw.snap | 7 + .../snapshots/query__common__check_query.snap | 23 -- .../tests/snapshots/query__check_query.snap | 23 -- ...pro_get_dependencies_(npm@10.5.0)_err.snap | 8 - ...c_repro_get_dependencies_(npm@10.5.0).snap | 23 -- ...n.tsx`_with_dependencies_(npm@10.5.0).snap | 16 - ...ar.ts`_with_dependencies_(npm@10.5.0).snap | 20 -- ...id.ts`_with_dependencies_(npm@10.5.0).snap | 27 -- ...urbo_trace_get_`main.ts`_(npm@10.5.0).snap | 11 - ...e_get_`main.ts`_with_ast_(npm@10.5.0).snap | 309 ------------------ ...in.ts`_with_dependencies_(npm@10.5.0).snap | 29 -- ...`main.ts`_with_depth_=_0_(npm@10.5.0).snap | 26 -- ...x`_with_dependencies_(npm@10.5.0)_err.snap | 8 - ...s`_with_dependencies_(npm@10.5.0)_err.snap | 8 - ...s`_with_dependencies_(npm@10.5.0)_err.snap | 8 - ..._trace_get_`main.ts`_(npm@10.5.0)_err.snap | 8 - ...t_`main.ts`_with_ast_(npm@10.5.0)_err.snap | 8 - ...s`_with_dependencies_(npm@10.5.0)_err.snap | 8 - ...n.ts`_with_depth_=_0_(npm@10.5.0)_err.snap | 8 - ...__index`_with_dependents_(npm@10.5.0).snap | 26 -- .../web/test/__snapshots__/page.spec.tsx.snap | 160 --------- turborepo-tests/integration/tests/no-args.t | 18 +- .../integration/tests/turbo-help.t | 57 ++-- 44 files changed, 724 insertions(+), 926 deletions(-) create mode 100644 crates/turborepo-cache/src/config.rs create mode 100644 crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__force.snap create mode 100644 crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__force_remote_r,local_r.snap create mode 100644 crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__no-cache.snap create mode 100644 crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__no-cache_remote_w,local_rw.snap create mode 100644 crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__remote-cache-read-only.snap create mode 100644 crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__remote-cache-read-only_remote_rw,local_r.snap create mode 100644 crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__remote-only.snap create mode 100644 crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__remote-only_remote_r,local_rw.snap delete mode 100644 crates/turborepo/tests/common/snapshots/query__common__check_query.snap delete mode 100644 crates/turborepo/tests/snapshots/query__check_query.snap delete mode 100644 crates/turborepo/tests/snapshots/query__oxc_repro_get_dependencies_(npm@10.5.0)_err.snap delete mode 100644 crates/turborepo/tests/snapshots/query__query_oxc_repro_get_dependencies_(npm@10.5.0).snap delete mode 100644 crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`button.tsx`_with_dependencies_(npm@10.5.0).snap delete mode 100644 crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`circular.ts`_with_dependencies_(npm@10.5.0).snap delete mode 100644 crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`invalid.ts`_with_dependencies_(npm@10.5.0).snap delete mode 100644 crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`main.ts`_(npm@10.5.0).snap delete mode 100644 crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`main.ts`_with_ast_(npm@10.5.0).snap delete mode 100644 crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`main.ts`_with_dependencies_(npm@10.5.0).snap delete mode 100644 crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`main.ts`_with_depth_=_0_(npm@10.5.0).snap delete mode 100644 crates/turborepo/tests/snapshots/query__turbo_trace_get_`button.tsx`_with_dependencies_(npm@10.5.0)_err.snap delete mode 100644 crates/turborepo/tests/snapshots/query__turbo_trace_get_`circular.ts`_with_dependencies_(npm@10.5.0)_err.snap delete mode 100644 crates/turborepo/tests/snapshots/query__turbo_trace_get_`invalid.ts`_with_dependencies_(npm@10.5.0)_err.snap delete mode 100644 crates/turborepo/tests/snapshots/query__turbo_trace_get_`main.ts`_(npm@10.5.0)_err.snap delete mode 100644 crates/turborepo/tests/snapshots/query__turbo_trace_get_`main.ts`_with_ast_(npm@10.5.0)_err.snap delete mode 100644 crates/turborepo/tests/snapshots/query__turbo_trace_get_`main.ts`_with_dependencies_(npm@10.5.0)_err.snap delete mode 100644 crates/turborepo/tests/snapshots/query__turbo_trace_get_`main.ts`_with_depth_=_0_(npm@10.5.0)_err.snap delete mode 100644 crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`packages__utils__index`_with_dependents_(npm@10.5.0).snap delete mode 100644 examples/with-nestjs/apps/web/test/__snapshots__/page.spec.tsx.snap diff --git a/Cargo.lock b/Cargo.lock index 6eaad0a580924..b817469229c5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6242,6 +6242,7 @@ dependencies = [ "hmac", "insta", "libc", + "miette", "os_str_bytes", "path-clean", "petgraph", diff --git a/crates/turborepo-cache/Cargo.toml b/crates/turborepo-cache/Cargo.toml index 5407fb706a531..acfdf821d4010 100644 --- a/crates/turborepo-cache/Cargo.toml +++ b/crates/turborepo-cache/Cargo.toml @@ -30,6 +30,7 @@ bytes.workspace = true camino = { workspace = true } futures = { workspace = true } hmac = "0.12.1" +miette = { workspace = true } os_str_bytes = "6.5.0" path-clean = { workspace = true } petgraph = "0.6.3" diff --git a/crates/turborepo-cache/src/async_cache.rs b/crates/turborepo-cache/src/async_cache.rs index 3d20ea0bb95e5..23126ce07a5c1 100644 --- a/crates/turborepo-cache/src/async_cache.rs +++ b/crates/turborepo-cache/src/async_cache.rs @@ -227,7 +227,8 @@ mod tests { use crate::{ test_cases::{get_test_cases, TestCase}, - AsyncCache, CacheHitMetadata, CacheOpts, CacheSource, RemoteCacheOpts, + AsyncCache, CacheActions, CacheConfig, CacheHitMetadata, CacheOpts, CacheSource, + RemoteCacheOpts, }; #[tokio::test] @@ -255,9 +256,16 @@ mod tests { let opts = CacheOpts { cache_dir: Utf8PathBuf::from(".turbo/cache"), - remote_cache_read_only: false, - skip_remote: false, - skip_filesystem: true, + cache: CacheConfig { + local: CacheActions { + read: false, + write: false, + }, + remote: CacheActions { + read: true, + write: true, + }, + }, workers: 10, remote_cache_opts: Some(RemoteCacheOpts { unused_team_id: Some("my-team".to_string()), @@ -337,9 +345,16 @@ mod tests { let opts = CacheOpts { cache_dir: Utf8PathBuf::from(".turbo/cache"), - remote_cache_read_only: false, - skip_remote: true, - skip_filesystem: false, + cache: CacheConfig { + local: CacheActions { + read: true, + write: true, + }, + remote: CacheActions { + read: false, + write: false, + }, + }, workers: 10, remote_cache_opts: Some(RemoteCacheOpts { unused_team_id: Some("my-team".to_string()), @@ -429,9 +444,16 @@ mod tests { let opts = CacheOpts { cache_dir: Utf8PathBuf::from(".turbo/cache"), - remote_cache_read_only: false, - skip_remote: false, - skip_filesystem: false, + cache: CacheConfig { + local: CacheActions { + read: true, + write: true, + }, + remote: CacheActions { + read: true, + write: true, + }, + }, workers: 10, remote_cache_opts: Some(RemoteCacheOpts { unused_team_id: Some("my-team".to_string()), diff --git a/crates/turborepo-cache/src/config.rs b/crates/turborepo-cache/src/config.rs new file mode 100644 index 0000000000000..3c1289965bbf6 --- /dev/null +++ b/crates/turborepo-cache/src/config.rs @@ -0,0 +1,231 @@ +use std::str::FromStr; + +use miette::{Diagnostic, SourceSpan}; +use thiserror::Error; + +use crate::{CacheActions, CacheConfig}; + +#[derive(Debug, Error, Diagnostic, PartialEq)] +pub enum Error { + #[error("keys cannot be duplicated, found `{key}` multiple times")] + DuplicateKeys { + #[source_code] + text: String, + key: &'static str, + #[label] + span: Option, + }, + #[error("actions cannot be duplicated, found `{action}` multiple times")] + DuplicateActions { + #[source_code] + text: String, + action: &'static str, + #[label] + span: Option, + }, + #[error("invalid cache type and action pair, found `{pair}`, expected colon separated pair")] + InvalidCacheTypeAndAction { + #[source_code] + text: String, + pair: String, + #[label] + span: Option, + }, + #[error("invalid cache action `{c}`")] + InvalidCacheAction { + #[source_code] + text: String, + c: char, + #[label] + span: Option, + }, + #[error("invalid cache type `{s}`, expected `local` or `remote`")] + InvalidCacheType { + #[source_code] + text: String, + s: String, + #[label] + span: Option, + }, +} + +impl Error { + pub fn add_text(mut self, new_text: impl Into) -> Self { + match &mut self { + Self::DuplicateKeys { text, .. } => *text = new_text.into(), + Self::DuplicateActions { text, .. } => *text = new_text.into(), + Self::InvalidCacheTypeAndAction { text, .. } => *text = new_text.into(), + Self::InvalidCacheAction { text, .. } => *text = new_text.into(), + Self::InvalidCacheType { text, .. } => *text = new_text.into(), + } + + self + } + + pub fn add_span(mut self, new_span: SourceSpan) -> Self { + match &mut self { + Self::DuplicateKeys { span, .. } => *span = Some(new_span), + Self::DuplicateActions { span, .. } => *span = Some(new_span), + Self::InvalidCacheTypeAndAction { span, .. } => *span = Some(new_span), + Self::InvalidCacheAction { span, .. } => *span = Some(new_span), + Self::InvalidCacheType { span, .. } => *span = Some(new_span), + } + + self + } +} + +impl FromStr for CacheConfig { + type Err = Error; + fn from_str(s: &str) -> Result { + let mut cache = CacheConfig { + local: CacheActions { + read: false, + write: false, + }, + remote: CacheActions { + read: false, + write: false, + }, + }; + + if s.is_empty() { + return Ok(cache); + } + + let mut seen_local = false; + let mut seen_remote = false; + let mut idx = 0; + + for action in s.split(',') { + let (key, value) = action + .split_once(':') + .ok_or(Error::InvalidCacheTypeAndAction { + text: s.to_string(), + pair: action.to_string(), + span: Some(SourceSpan::new(idx.into(), action.len().into())), + })?; + + match key { + "local" => { + if seen_local { + return Err(Error::DuplicateKeys { + text: s.to_string(), + key: "local", + span: Some(SourceSpan::new(idx.into(), key.len().into())), + }); + } + + seen_local = true; + cache.local = CacheActions::from_str(value).map_err(|err| { + err.add_text(s).add_span(SourceSpan::new( + (idx + key.len() + 1).into(), + key.len().into(), + )) + })?; + } + "remote" => { + if seen_remote { + return Err(Error::DuplicateKeys { + text: s.to_string(), + key: "remote", + span: Some(SourceSpan::new(idx.into(), key.len().into())), + }); + } + + seen_remote = true; + cache.remote = CacheActions::from_str(value).map_err(|err| { + err.add_text(s).add_span(SourceSpan::new( + (idx + key.len() + 1).into(), + value.len().into(), + )) + })? + } + ty => { + return Err(Error::InvalidCacheType { + text: s.to_string(), + s: ty.to_string(), + span: Some(SourceSpan::new(idx.into(), ty.len().into())), + }) + } + } + + idx += action.len() + 1; + } + Ok(cache) + } +} + +impl FromStr for CacheActions { + type Err = Error; + fn from_str(s: &str) -> Result { + let mut cache = CacheActions { + read: false, + write: false, + }; + + for c in s.chars() { + match c { + 'r' => { + if cache.read { + return Err(Error::DuplicateActions { + text: s.to_string(), + action: "r (read)", + span: None, + }); + } + cache.read = true; + } + + 'w' => { + if cache.write { + return Err(Error::DuplicateActions { + text: s.to_string(), + action: "w (write)", + span: None, + }); + } + cache.write = true; + } + _ => { + return Err(Error::InvalidCacheAction { + c, + text: String::new(), + span: None, + }) + } + } + } + + Ok(cache) + } +} + +#[cfg(test)] +mod test { + use test_case::test_case; + + use super::*; + + #[test_case("local:r,remote:w", Ok(CacheConfig { local: CacheActions { read: true, write: false }, remote: CacheActions { read: false, write: true } }) ; "local:r,remote:w" + )] + #[test_case("local:r", Ok(CacheConfig { local: CacheActions { read: true, write: false }, remote: CacheActions { read: false, write: false } }) ; "local:r" + )] + #[test_case("local:", Ok(CacheConfig { local: CacheActions { read: false, write: false }, remote: CacheActions { read: false, write: false } }) ; "empty action" + )] + #[test_case("local:,remote:", Ok(CacheConfig { local: CacheActions { read: false, write: false }, remote: CacheActions { read: false, write: false } }) ; "multiple empty actions" + )] + #[test_case("local:,remote:r", Ok(CacheConfig { local: CacheActions { read: false, write: false }, remote: CacheActions { read: true, write: false } }) ; "local: empty, remote:r" + )] + #[test_case("", Ok(CacheConfig { local: CacheActions { read: false, write: false }, remote: CacheActions { read: false, write: false } }) ; "empty" + )] + #[test_case("local:r,local:w", Err(Error::DuplicateKeys { text: "local:r,local:w".to_string(), key: "local", span: Some(SourceSpan::new(8.into(), 5.into())) }) ; "duplicate local key" + )] + #[test_case("local:rr", Err(Error::DuplicateActions { text: "local:rr".to_string(), action: "r (read)", span: Some(SourceSpan::new(6.into(), 5.into())) }) ; "duplicate action")] + #[test_case("remote:r,local=rx", Err(Error::InvalidCacheTypeAndAction { text: "remote:r,local=rx".to_string(), pair: "local=rx".to_string(), span: Some(SourceSpan::new(9.into(), 8.into())) }) ; "invalid key action pair")] + #[test_case("local:rx", Err(Error::InvalidCacheAction { c: 'x', text: "local:rx".to_string(), span: Some(SourceSpan::new(6.into(), 5.into())) }) ; "invalid action")] + #[test_case("file:r", Err(Error::InvalidCacheType { s: "file".to_string(), text: "file:r".to_string(), span: Some(SourceSpan::new(0.into(), 4.into())) }) ; "invalid cache type")] + fn test_cache_config(s: &str, expected: Result) { + assert_eq!(CacheConfig::from_str(s), expected); + } +} diff --git a/crates/turborepo-cache/src/lib.rs b/crates/turborepo-cache/src/lib.rs index c05f4dcce5edd..65bcadbc7e5db 100644 --- a/crates/turborepo-cache/src/lib.rs +++ b/crates/turborepo-cache/src/lib.rs @@ -7,6 +7,7 @@ mod async_cache; /// The core cache creation and restoration logic. pub mod cache_archive; +pub mod config; /// File system cache pub mod fs; /// Remote cache @@ -61,6 +62,8 @@ pub enum CacheError { LinkTargetDoesNotExist(String, #[backtrace] Backtrace), #[error("Invalid tar, link target does not exist on header")] LinkTargetNotOnHeader(#[backtrace] Backtrace), + #[error(transparent)] + Config(#[from] config::Error), #[error("attempted to restore unsupported file type: {0:?}")] RestoreUnsupportedFileType(tar::EntryType, #[backtrace] Backtrace), // We don't pass the `FileType` because there's no simple @@ -105,12 +108,43 @@ pub struct CacheHitMetadata { pub time_saved: u64, } -#[derive(Clone, Debug, Default)] +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub struct CacheActions { + pub read: bool, + pub write: bool, +} + +impl CacheActions { + pub fn should_use(&self) -> bool { + self.read || self.write + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Default)] +pub struct CacheConfig { + pub local: CacheActions, + pub remote: CacheActions, +} + +impl CacheConfig { + pub fn skip_writes(&self) -> bool { + !self.local.write && !self.remote.write + } +} + +impl Default for CacheActions { + fn default() -> Self { + Self { + read: true, + write: true, + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct CacheOpts { pub cache_dir: Utf8PathBuf, - pub remote_cache_read_only: bool, - pub skip_remote: bool, - pub skip_filesystem: bool, + pub cache: CacheConfig, pub workers: u32, pub remote_cache_opts: Option, } diff --git a/crates/turborepo-cache/src/multiplexer.rs b/crates/turborepo-cache/src/multiplexer.rs index e5b1ab8db661c..44bc51afaeaf8 100644 --- a/crates/turborepo-cache/src/multiplexer.rs +++ b/crates/turborepo-cache/src/multiplexer.rs @@ -11,7 +11,7 @@ use turborepo_api_client::{APIAuth, APIClient}; use crate::{ fs::FSCache, http::{HTTPCache, UploadMap}, - CacheError, CacheHitMetadata, CacheOpts, + CacheConfig, CacheError, CacheHitMetadata, CacheOpts, }; pub struct CacheMultiplexer { @@ -23,7 +23,7 @@ pub struct CacheMultiplexer { // Just for keeping track of whether we've already printed a warning about the remote cache // being read-only should_print_skipping_remote_put: AtomicBool, - remote_cache_read_only: bool, + cache_config: CacheConfig, fs: Option, http: Option, } @@ -37,8 +37,8 @@ impl CacheMultiplexer { api_auth: Option, analytics_recorder: Option, ) -> Result { - let use_fs_cache = !opts.skip_filesystem; - let use_http_cache = !opts.skip_remote; + let use_fs_cache = opts.cache.local.should_use(); + let use_http_cache = opts.cache.remote.should_use(); // Since the above two flags are not mutually exclusive it is possible to // configure yourself out of having a cache. We should tell you about it @@ -67,7 +67,7 @@ impl CacheMultiplexer { Ok(CacheMultiplexer { should_print_skipping_remote_put: AtomicBool::new(true), should_use_http_cache: AtomicBool::new(http_cache.is_some()), - remote_cache_read_only: opts.remote_cache_read_only, + cache_config: opts.cache, fs: fs_cache, http: http_cache, }) @@ -95,14 +95,20 @@ impl CacheMultiplexer { files: &[AnchoredSystemPathBuf], duration: u64, ) -> Result<(), CacheError> { - self.fs - .as_ref() - .map(|fs| fs.put(anchor, key, files, duration)) - .transpose()?; + if self.cache_config.local.write { + self.fs + .as_ref() + .map(|fs| fs.put(anchor, key, files, duration)) + .transpose()?; + } let http_result = match self.get_http_cache() { Some(http) => { - if self.remote_cache_read_only { + if self.cache_config.remote.write { + let http_result = http.put(anchor, key, files, duration).await; + + Some(http_result) + } else { if self .should_print_skipping_remote_put .load(Ordering::Relaxed) @@ -115,10 +121,6 @@ impl CacheMultiplexer { // Cache is functional but running in read-only mode, so we don't want to try to // write to it None - } else { - let http_result = http.put(anchor, key, files, duration).await; - - Some(http_result) } } _ => None, @@ -144,25 +146,31 @@ impl CacheMultiplexer { anchor: &AbsoluteSystemPath, key: &str, ) -> Result)>, CacheError> { - if let Some(fs) = &self.fs { - if let response @ Ok(Some(_)) = fs.fetch(anchor, key) { - return response; + if self.cache_config.local.read { + if let Some(fs) = &self.fs { + if let response @ Ok(Some(_)) = fs.fetch(anchor, key) { + return response; + } } } - if let Some(http) = self.get_http_cache() { - if let Ok(Some((CacheHitMetadata { source, time_saved }, files))) = - http.fetch(key).await - { - // Store this into fs cache. We can ignore errors here because we know - // we have previously successfully stored in HTTP cache, and so the overall - // result is a success at fetching. Storing in lower-priority caches is an - // optimization. - if let Some(fs) = &self.fs { - let _ = fs.put(anchor, key, &files, time_saved); - } + if self.cache_config.remote.read { + if let Some(http) = self.get_http_cache() { + if let Ok(Some((CacheHitMetadata { source, time_saved }, files))) = + http.fetch(key).await + { + // Store this into fs cache. We can ignore errors here because we know + // we have previously successfully stored in HTTP cache, and so the overall + // result is a success at fetching. Storing in lower-priority caches is an + // optimization. + if self.cache_config.local.write { + if let Some(fs) = &self.fs { + let _ = fs.put(anchor, key, &files, time_saved); + } + } - return Ok(Some((CacheHitMetadata { source, time_saved }, files))); + return Ok(Some((CacheHitMetadata { source, time_saved }, files))); + } } } @@ -171,23 +179,27 @@ impl CacheMultiplexer { #[tracing::instrument(skip_all)] pub async fn exists(&self, key: &str) -> Result, CacheError> { - if let Some(fs) = &self.fs { - match fs.exists(key) { - cache_hit @ Ok(Some(_)) => { - return cache_hit; + if self.cache_config.local.read { + if let Some(fs) = &self.fs { + match fs.exists(key) { + cache_hit @ Ok(Some(_)) => { + return cache_hit; + } + Ok(None) => {} + Err(err) => debug!("failed to check fs cache: {:?}", err), } - Ok(None) => {} - Err(err) => debug!("failed to check fs cache: {:?}", err), } } - if let Some(http) = self.get_http_cache() { - match http.exists(key).await { - cache_hit @ Ok(Some(_)) => { - return cache_hit; + if self.cache_config.remote.read { + if let Some(http) = self.get_http_cache() { + match http.exists(key).await { + cache_hit @ Ok(Some(_)) => { + return cache_hit; + } + Ok(None) => {} + Err(err) => debug!("failed to check http cache: {:?}", err), } - Ok(None) => {} - Err(err) => debug!("failed to check http cache: {:?}", err), } } diff --git a/crates/turborepo-lib/src/cli/mod.rs b/crates/turborepo-lib/src/cli/mod.rs index 136cfa7154d0d..a5e7cd9b2a337 100644 --- a/crates/turborepo-lib/src/cli/mod.rs +++ b/crates/turborepo-lib/src/cli/mod.rs @@ -33,7 +33,6 @@ use crate::{ }; mod error; - // Global turbo sets this environment variable to its cwd so that local // turbo can use it for package inference. pub const INVOCATION_DIR_ENV_VAR: &str = "TURBO_INVOCATION_DIR"; @@ -724,9 +723,6 @@ pub struct ExecutionArgs { /// Run turbo in single-package mode #[clap(long)] pub single_package: bool, - /// Ignore the existing cache (to force execution) - #[clap(long, default_missing_value = "true")] - pub force: Option>, /// Specify whether or not to do framework inference for tasks #[clap(long, value_name = "BOOL", action = ArgAction::Set, default_value = "true", default_missing_value = "true", num_args = 0..=1)] pub framework_inference: bool, @@ -769,10 +765,6 @@ pub struct ExecutionArgs { pub only: bool, #[clap(long, hide = true)] pub pkg_inference_root: Option, - /// Ignore the local filesystem cache for all tasks. Only - /// allow reading and caching artifacts using the remote cache. - #[clap(long, default_missing_value = "true")] - pub remote_only: Option>, /// Use "none" to remove prefixes from task logs. Use "task" to get task id /// prefixing. Use "auto" to let turbo decide how to prefix the logs /// based on the execution environment. In most cases this will be the same @@ -791,11 +783,6 @@ pub struct ExecutionArgs { } impl ExecutionArgs { - pub fn remote_only(&self) -> Option { - let remote_only = self.remote_only?; - Some(remote_only.unwrap_or(true)) - } - fn track(&self, telemetry: &CommandEventBuilder) { // default to false track_usage!(telemetry, self.framework_inference, |val: bool| !val); @@ -803,9 +790,7 @@ impl ExecutionArgs { track_usage!(telemetry, self.continue_execution, |val| val); track_usage!(telemetry, self.single_package, |val| val); track_usage!(telemetry, self.only, |val| val); - track_usage!(telemetry, self.remote_only().unwrap_or_default(), |val| val); track_usage!(telemetry, &self.cache_dir, Option::is_some); - track_usage!(telemetry, &self.force, Option::is_some); track_usage!(telemetry, &self.pkg_inference_root, Option::is_some); if let Some(concurrency) = &self.concurrency { @@ -848,6 +833,29 @@ impl ExecutionArgs { ArgGroup::new("daemon-group").multiple(false).required(false), ])] pub struct RunArgs { + /// Set the cache behavior for this run. Pass a list of comma-separated key, + /// value pairs to enable reading and writing to either the local or + /// remote cache. + #[clap(long, conflicts_with_all = &["force", "remote_only", "remote_cache_read_only", "no_cache"])] + pub cache: Option, + /// Ignore the existing cache (to force execution). Equivalent to + /// `--cache=local:w,remote:w` + #[clap(long, default_missing_value = "true")] + pub force: Option>, + /// Ignore the local filesystem cache for all tasks. Only + /// allow reading and caching artifacts using the remote cache. + /// Equivalent to `--cache=remote:rw` + #[clap(long, default_missing_value = "true", group = "cache-group")] + pub remote_only: Option>, + /// Treat remote cache as read only. Equivalent to + /// `--cache=remote:r;local:rw` + #[clap(long, default_missing_value = "true")] + pub remote_cache_read_only: Option>, + /// Avoid saving task results to the cache. Useful for development/watch + /// tasks. Equivalent to `--cache=local:r,remote:r` + #[clap(long)] + pub no_cache: bool, + /// Set the number of concurrent cache operations (default 10) #[clap(long, default_value_t = DEFAULT_NUM_WORKERS)] pub cache_workers: u32, @@ -859,12 +867,6 @@ pub struct RunArgs { /// is provided #[clap(long, num_args = 0..=1, default_missing_value = "", value_parser = validate_graph_extension)] pub graph: Option, - - /// Avoid saving task results to the cache. Useful for development/watch - /// tasks. - #[clap(long)] - pub no_cache: bool, - // clap does not have negation flags such as --daemon and --no-daemon // so we need to use a group to enforce that only one of them is set. // ----------------------- @@ -887,9 +889,6 @@ pub struct RunArgs { /// All identifying data omitted from the profile. #[clap(long, value_parser=NonEmptyStringValueParser::new(), conflicts_with = "profile")] pub anon_profile: Option, - /// Treat remote cache as read only - #[clap(long, default_missing_value = "true")] - pub remote_cache_read_only: Option>, /// Generate a summary of the turbo run #[clap(long, default_missing_value = "true")] pub summarize: Option>, @@ -906,6 +905,9 @@ pub struct RunArgs { impl Default for RunArgs { fn default() -> Self { Self { + remote_only: None, + cache: None, + force: None, cache_workers: DEFAULT_NUM_WORKERS, dry_run: None, graph: None, @@ -923,6 +925,11 @@ impl Default for RunArgs { } impl RunArgs { + pub fn remote_only(&self) -> Option { + let remote_only = self.remote_only?; + Some(remote_only.unwrap_or(true)) + } + /// Some(true) means force the daemon /// Some(false) means force no daemon /// None means use the default detection @@ -957,6 +964,8 @@ impl RunArgs { pub fn track(&self, telemetry: &CommandEventBuilder) { // default to true track_usage!(telemetry, self.no_cache, |val| val); + track_usage!(telemetry, self.remote_only().unwrap_or_default(), |val| val); + track_usage!(telemetry, &self.force, Option::is_some); track_usage!(telemetry, self.daemon, |val| val); track_usage!(telemetry, self.no_daemon, |val| val); track_usage!(telemetry, self.parallel, |val| val); @@ -1443,7 +1452,6 @@ mod test { fn get_default_execution_args() -> ExecutionArgs { ExecutionArgs { output_logs: None, - remote_only: None, framework_inference: true, ..ExecutionArgs::default() } @@ -1779,10 +1787,12 @@ mod test { command: Some(Command::Run { execution_args: Box::new(ExecutionArgs { tasks: vec!["build".to_string()], - force: Some(Some(true)), ..get_default_execution_args() }), - run_args: Box::new(get_default_run_args()) + run_args: Box::new(RunArgs { + force: Some(Some(true)), + ..get_default_run_args() + }) }), ..Args::default() } ; @@ -2078,10 +2088,12 @@ mod test { command: Some(Command::Run { execution_args: Box::new(ExecutionArgs { tasks: vec!["build".to_string()], - remote_only: None, ..get_default_execution_args() }), - run_args: Box::new(get_default_run_args()) + run_args: Box::new(RunArgs { + remote_only: None, + ..get_default_run_args() + }) }), ..Args::default() } ; @@ -2093,10 +2105,12 @@ mod test { command: Some(Command::Run { execution_args: Box::new(ExecutionArgs { tasks: vec!["build".to_string()], - remote_only: Some(Some(true)), ..get_default_execution_args() }), - run_args: Box::new(get_default_run_args()) + run_args: Box::new(RunArgs { + remote_only: Some(Some(true)), + ..get_default_run_args() + }) }), ..Args::default() } ; @@ -2108,10 +2122,12 @@ mod test { command: Some(Command::Run { execution_args: Box::new(ExecutionArgs { tasks: vec!["build".to_string()], - remote_only: Some(Some(true)), ..get_default_execution_args() }), - run_args: Box::new(get_default_run_args()) + run_args: Box::new(RunArgs { + remote_only: Some(Some(true)), + ..get_default_run_args() + }) }), ..Args::default() } ; @@ -2123,10 +2139,12 @@ mod test { command: Some(Command::Run { execution_args: Box::new(ExecutionArgs { tasks: vec!["build".to_string()], - remote_only: Some(Some(false)), ..get_default_execution_args() }), - run_args: Box::new(get_default_run_args()) + run_args: Box::new(RunArgs { + remote_only: Some(Some(false)), + ..get_default_run_args() + }) }), ..Args::default() } ; diff --git a/crates/turborepo-lib/src/commands/mod.rs b/crates/turborepo-lib/src/commands/mod.rs index 04f1568bc0df8..439be0cd13ae4 100644 --- a/crates/turborepo-lib/src/commands/mod.rs +++ b/crates/turborepo-lib/src/commands/mod.rs @@ -93,20 +93,23 @@ impl CommandBase { ) .with_force( self.args - .execution_args() + .run_args() .and_then(|args| args.force.map(|value| value.unwrap_or(true))), ) .with_log_order(self.args.execution_args().and_then(|args| args.log_order)) - .with_remote_only( - self.args - .execution_args() - .and_then(|args| args.remote_only()), - ) + .with_remote_only(self.args.run_args().and_then(|args| args.remote_only())) .with_remote_cache_read_only( self.args .run_args() .and_then(|args| args.remote_cache_read_only()), ) + .with_cache( + self.args + .run_args() + .and_then(|args| args.cache.as_deref()) + .map(|cache| cache.parse()) + .transpose()?, + ) .with_run_summary(self.args.run_args().and_then(|args| args.summarize())) .with_allow_no_turbo_json(self.args.allow_no_turbo_json.then_some(true)) .build() diff --git a/crates/turborepo-lib/src/config/env.rs b/crates/turborepo-lib/src/config/env.rs index 1f81dba3b0640..5d9aedcf7406f 100644 --- a/crates/turborepo-lib/src/config/env.rs +++ b/crates/turborepo-lib/src/config/env.rs @@ -39,6 +39,7 @@ const TURBO_MAPPING: &[(&str, &str)] = [ ("turbo_remote_cache_read_only", "remote_cache_read_only"), ("turbo_run_summary", "run_summary"), ("turbo_allow_no_turbo_json", "allow_no_turbo_json"), + ("turbo_cache", "cache"), ] .as_slice(); @@ -77,12 +78,6 @@ impl ResolvedConfigurationOptions for EnvVars { .map(|value| value.ok_or_else(|| Error::InvalidPreflight)) .transpose()?; - // Process enabled - let enabled = self - .truthy_value("enabled") - .map(|value| value.ok_or_else(|| Error::InvalidRemoteCacheEnabled)) - .transpose()?; - let force = self.truthy_value("force").flatten(); let remote_only = self.truthy_value("remote_only").flatten(); let remote_cache_read_only = self.truthy_value("remote_cache_read_only").flatten(); @@ -162,10 +157,15 @@ impl ResolvedConfigurationOptions for EnvVars { token: self.output_map.get("token").cloned(), scm_base: self.output_map.get("scm_base").cloned(), scm_head: self.output_map.get("scm_head").cloned(), + cache: self + .output_map + .get("cache") + .map(|c| c.parse()) + .transpose()?, // Processed booleans signature, preflight, - enabled, + enabled: None, ui, allow_no_package_manager, daemon, diff --git a/crates/turborepo-lib/src/config/mod.rs b/crates/turborepo-lib/src/config/mod.rs index a32b592d7eb24..55c772a23dbe4 100644 --- a/crates/turborepo-lib/src/config/mod.rs +++ b/crates/turborepo-lib/src/config/mod.rs @@ -19,6 +19,7 @@ use thiserror::Error; use tracing::debug; use turbo_json::TurboJsonReader; use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf}; +use turborepo_cache::CacheConfig; use turborepo_errors::TURBO_SITE; use turborepo_repository::package_graph::PackageName; @@ -81,6 +82,8 @@ pub enum Error { config_path: AbsoluteSystemPathBuf, error: io::Error, }, + #[error(transparent)] + Cache(#[from] turborepo_cache::config::Error), #[error( "Package tasks (#) are not allowed in single-package repositories: found \ {task_id}" @@ -252,6 +255,8 @@ pub struct ConfigurationOptions { pub(crate) root_turbo_json_path: Option, pub(crate) force: Option, pub(crate) log_order: Option, + #[serde(skip)] + pub(crate) cache: Option, pub(crate) remote_only: Option, pub(crate) remote_cache_read_only: Option, pub(crate) run_summary: Option, @@ -368,6 +373,10 @@ impl ConfigurationOptions { }) } + pub fn cache(&self) -> Option { + self.cache + } + pub fn force(&self) -> bool { self.force.unwrap_or_default() } diff --git a/crates/turborepo-lib/src/opts.rs b/crates/turborepo-lib/src/opts.rs index 9b58a2996cd29..91a659f680b98 100644 --- a/crates/turborepo-lib/src/opts.rs +++ b/crates/turborepo-lib/src/opts.rs @@ -32,6 +32,11 @@ pub enum Error { or equal to 1: {1}" )] ConcurrencyOutOfBounds(#[backtrace] backtrace::Backtrace, String), + #[error( + "Cannot set `cache` config and other cache options (`force`, `remoteOnly`, \ + `remoteCacheReadOnly`) at the same time" + )] + OverlappingCacheOptions, #[error(transparent)] Path(#[from] turbopath::PathError), #[error(transparent)] @@ -100,16 +105,16 @@ impl Opts { return Err(Error::ExpectedRun(Backtrace::capture())); }; - let run_and_execution_args = OptsInputs { + let inputs = OptsInputs { run_args: run_args.as_ref(), execution_args: execution_args.as_ref(), config, api_auth: &api_auth, }; - let run_opts = RunOpts::try_from(run_and_execution_args)?; - let cache_opts = CacheOpts::from(run_and_execution_args); - let scope_opts = ScopeOpts::try_from(run_and_execution_args)?; - let runcache_opts = RunCacheOpts::from(run_and_execution_args); + let run_opts = RunOpts::try_from(inputs)?; + let cache_opts = CacheOpts::try_from(inputs)?; + let scope_opts = ScopeOpts::try_from(inputs)?; + let runcache_opts = RunCacheOpts::from(inputs); Ok(Self { run_opts, @@ -128,18 +133,14 @@ struct OptsInputs<'a> { api_auth: &'a Option, } -#[derive(Clone, Debug, Default)] +#[derive(Clone, Copy, Debug, Default)] pub struct RunCacheOpts { - pub(crate) skip_reads: bool, - pub(crate) skip_writes: bool, pub(crate) task_output_logs_override: Option, } impl<'a> From> for RunCacheOpts { fn from(inputs: OptsInputs<'a>) -> Self { RunCacheOpts { - skip_reads: inputs.config.force(), - skip_writes: inputs.run_args.no_cache, task_output_logs_override: inputs.execution_args.output_logs, } } @@ -359,19 +360,52 @@ impl<'a> TryFrom> for ScopeOpts { } } -impl<'a> From> for CacheOpts { - fn from(inputs: OptsInputs<'a>) -> Self { +impl<'a> TryFrom> for CacheOpts { + type Error = self::Error; + + fn try_from(inputs: OptsInputs<'a>) -> Result { let is_linked = turborepo_api_client::is_linked(inputs.api_auth); - let skip_remote = if !is_linked { - true + let cache = inputs.config.cache(); + let has_old_cache_config = inputs.config.remote_only() + || inputs.config.force() + || inputs.run_args.no_cache + || inputs.config.remote_cache_read_only(); + + if has_old_cache_config && cache.is_some() { + return Err(Error::OverlappingCacheOptions); + } + + let mut cache = cache.unwrap_or_default(); + + if inputs.config.remote_only() { + cache.local.read = false; + cache.local.write = false; + } + + if inputs.config.force() { + cache.local.read = false; + cache.remote.read = false; + } + + if inputs.run_args.no_cache { + cache.local.write = false; + cache.remote.write = false; + } + + if !is_linked { + cache.remote.read = false; + cache.remote.write = false; } else if let Some(enabled) = inputs.config.enabled { // We're linked, but if the user has explicitly enabled or disabled, use that // value - !enabled - } else { - false + cache.remote.read = enabled; + cache.remote.write = enabled; }; + if inputs.config.remote_cache_read_only() { + cache.remote.write = false; + } + // Note that we don't currently use the team_id value here. In the future, we // should probably verify that we only use the signature value when the // configured team_id matches the final resolved team_id. @@ -383,14 +417,12 @@ impl<'a> From> for CacheOpts { signature, )); - CacheOpts { + Ok(CacheOpts { cache_dir: inputs.config.cache_dir().into(), - skip_filesystem: inputs.config.remote_only(), - remote_cache_read_only: inputs.config.remote_cache_read_only(), + cache, workers: inputs.run_args.cache_workers, - skip_remote, remote_cache_opts, - } + }) } } @@ -410,14 +442,20 @@ impl ScopeOpts { #[cfg(test)] mod test { + use test_case::test_case; + use turbopath::AbsoluteSystemPathBuf; + use turborepo_api_client::APIAuth; use turborepo_cache::CacheOpts; + use turborepo_ui::ColorConfig; - use super::RunOpts; + use super::{OptsInputs, RunOpts}; use crate::{ - cli::DryRunMode, + cli::{Command, DryRunMode, RunArgs}, + commands::CommandBase, opts::{Opts, RunCacheOpts, ScopeOpts}, turbo_json::UIMode, + Args, }; #[derive(Default)] @@ -551,4 +589,86 @@ mod test { let synthesized = opts.synthesize_command(); assert_eq!(synthesized, expected); } + + #[test_case( + RunArgs { + no_cache: true, + ..Default::default() + }, "no-cache" + )] + #[test_case( + RunArgs { + force: Some(Some(true)), + ..Default::default() + }, "force" + )] + #[test_case( + RunArgs { + remote_only: Some(Some(true)), + ..Default::default() + }, "remote-only" + )] + #[test_case( + RunArgs { + remote_cache_read_only: Some(Some(true)), + ..Default::default() + }, "remote-cache-read-only" + )] + #[test_case( + RunArgs { + no_cache: true, + cache: Some("remote:w,local:rw".to_string()), + ..Default::default() + }, "no-cache_remote_w,local_rw" + )] + #[test_case( + RunArgs { + remote_only: Some(Some(true)), + cache: Some("remote:r,local:rw".to_string()), + ..Default::default() + }, "remote-only_remote_r,local_rw" + )] + #[test_case( + RunArgs { + force: Some(Some(true)), + cache: Some("remote:r,local:r".to_string()), + ..Default::default() + }, "force_remote_r,local_r" + )] + #[test_case( + RunArgs { + remote_cache_read_only: Some(Some(true)), + cache: Some("remote:rw,local:r".to_string()), + ..Default::default() + }, "remote-cache-read-only_remote_rw,local_r" + )] + fn test_resolve_cache_config(run_args: RunArgs, name: &str) -> Result<(), anyhow::Error> { + let mut args = Args::default(); + args.command = Some(Command::Run { + execution_args: Box::default(), + run_args: Box::new(run_args), + }); + let base = CommandBase::new( + args, + AbsoluteSystemPathBuf::default(), + "1.0.0", + ColorConfig::new(true), + ); + + let cache_opts = CacheOpts::try_from(OptsInputs { + run_args: base.args().run_args().unwrap(), + execution_args: base.args().execution_args().unwrap(), + config: base.config()?, + api_auth: &Some(APIAuth { + team_id: Some("my-team".to_string()), + token: "my-token".to_string(), + team_slug: None, + }), + }) + .map(|cache_opts| cache_opts.cache); + + insta::assert_debug_snapshot!(name, cache_opts); + + Ok(()) + } } diff --git a/crates/turborepo-lib/src/run/builder.rs b/crates/turborepo-lib/src/run/builder.rs index aeb4ac081ae5e..9795b65f347f7 100644 --- a/crates/turborepo-lib/src/run/builder.rs +++ b/crates/turborepo-lib/src/run/builder.rs @@ -457,7 +457,8 @@ impl RunBuilder { let run_cache = Arc::new(RunCache::new( async_cache, &self.repo_root, - &self.opts.runcache_opts, + self.opts.runcache_opts, + &self.opts.cache_opts, color_selector, daemon.clone(), self.color_config, diff --git a/crates/turborepo-lib/src/run/cache.rs b/crates/turborepo-lib/src/run/cache.rs index 02c9eedce5b9b..b70abdd6e48c3 100644 --- a/crates/turborepo-lib/src/run/cache.rs +++ b/crates/turborepo-lib/src/run/cache.rs @@ -10,7 +10,9 @@ use tracing::{debug, error, log::warn}; use turbopath::{ AbsoluteSystemPath, AbsoluteSystemPathBuf, AnchoredSystemPath, AnchoredSystemPathBuf, }; -use turborepo_cache::{http::UploadMap, AsyncCache, CacheError, CacheHitMetadata, CacheSource}; +use turborepo_cache::{ + http::UploadMap, AsyncCache, CacheError, CacheHitMetadata, CacheOpts, CacheSource, +}; use turborepo_repository::package_graph::PackageInfo; use turborepo_scm::SCM; use turborepo_telemetry::events::{task::PackageTaskEventBuilder, TrackedErrors}; @@ -65,10 +67,12 @@ pub trait CacheOutput { } impl RunCache { + #[allow(clippy::too_many_arguments)] pub fn new( cache: AsyncCache, repo_root: &AbsoluteSystemPath, - opts: &RunCacheOpts, + run_cache_opts: RunCacheOpts, + cache_opts: &CacheOpts, color_selector: ColorSelector, daemon_client: Option>, ui: ColorConfig, @@ -77,14 +81,14 @@ impl RunCache { let task_output_logs = if is_dry_run { Some(OutputLogsMode::None) } else { - opts.task_output_logs_override + run_cache_opts.task_output_logs_override }; RunCache { task_output_logs, cache, warnings: Default::default(), - reads_disabled: opts.skip_reads, - writes_disabled: opts.skip_writes, + reads_disabled: !cache_opts.cache.remote.read && !cache_opts.cache.local.read, + writes_disabled: !cache_opts.cache.remote.write && !cache_opts.cache.local.write, repo_root: repo_root.to_owned(), color_selector, daemon_client, diff --git a/crates/turborepo-lib/src/run/mod.rs b/crates/turborepo-lib/src/run/mod.rs index 8120f4dcd569b..448b8c02d4726 100644 --- a/crates/turborepo-lib/src/run/mod.rs +++ b/crates/turborepo-lib/src/run/mod.rs @@ -114,7 +114,7 @@ impl Run { ); } - let use_http_cache = !self.opts.cache_opts.skip_remote; + let use_http_cache = self.opts.cache_opts.cache.remote.should_use(); if use_http_cache { cprintln!(self.color_config, GREY, "• Remote caching enabled"); } else { @@ -277,7 +277,7 @@ impl Run { } pub async fn run(&self, ui_sender: Option, is_watch: bool) -> Result { - let skip_cache_writes = self.opts.runcache_opts.skip_writes; + let skip_cache_writes = self.opts.cache_opts.cache.skip_writes(); if let Some(subscriber) = self.signal_handler.subscribe() { let run_cache = self.run_cache.clone(); tokio::spawn(async move { diff --git a/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__force.snap b/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__force.snap new file mode 100644 index 0000000000000..8cb25abf8d6bf --- /dev/null +++ b/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__force.snap @@ -0,0 +1,16 @@ +--- +source: crates/turborepo-lib/src/opts.rs +expression: cache_opts +--- +Ok( + CacheConfig { + local: CacheActions { + read: false, + write: true, + }, + remote: CacheActions { + read: false, + write: true, + }, + }, +) diff --git a/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__force_remote_r,local_r.snap b/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__force_remote_r,local_r.snap new file mode 100644 index 0000000000000..cbc9a96ee2d30 --- /dev/null +++ b/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__force_remote_r,local_r.snap @@ -0,0 +1,7 @@ +--- +source: crates/turborepo-lib/src/opts.rs +expression: cache_opts +--- +Err( + OverlappingCacheOptions, +) diff --git a/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__no-cache.snap b/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__no-cache.snap new file mode 100644 index 0000000000000..418a3f24a6b8a --- /dev/null +++ b/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__no-cache.snap @@ -0,0 +1,16 @@ +--- +source: crates/turborepo-lib/src/opts.rs +expression: cache_opts +--- +Ok( + CacheConfig { + local: CacheActions { + read: true, + write: false, + }, + remote: CacheActions { + read: true, + write: false, + }, + }, +) diff --git a/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__no-cache_remote_w,local_rw.snap b/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__no-cache_remote_w,local_rw.snap new file mode 100644 index 0000000000000..cbc9a96ee2d30 --- /dev/null +++ b/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__no-cache_remote_w,local_rw.snap @@ -0,0 +1,7 @@ +--- +source: crates/turborepo-lib/src/opts.rs +expression: cache_opts +--- +Err( + OverlappingCacheOptions, +) diff --git a/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__remote-cache-read-only.snap b/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__remote-cache-read-only.snap new file mode 100644 index 0000000000000..4cf87e441ba45 --- /dev/null +++ b/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__remote-cache-read-only.snap @@ -0,0 +1,16 @@ +--- +source: crates/turborepo-lib/src/opts.rs +expression: cache_opts +--- +Ok( + CacheConfig { + local: CacheActions { + read: true, + write: true, + }, + remote: CacheActions { + read: true, + write: false, + }, + }, +) diff --git a/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__remote-cache-read-only_remote_rw,local_r.snap b/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__remote-cache-read-only_remote_rw,local_r.snap new file mode 100644 index 0000000000000..cbc9a96ee2d30 --- /dev/null +++ b/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__remote-cache-read-only_remote_rw,local_r.snap @@ -0,0 +1,7 @@ +--- +source: crates/turborepo-lib/src/opts.rs +expression: cache_opts +--- +Err( + OverlappingCacheOptions, +) diff --git a/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__remote-only.snap b/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__remote-only.snap new file mode 100644 index 0000000000000..1c74ad6f2c749 --- /dev/null +++ b/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__remote-only.snap @@ -0,0 +1,16 @@ +--- +source: crates/turborepo-lib/src/opts.rs +expression: cache_opts +--- +Ok( + CacheConfig { + local: CacheActions { + read: false, + write: false, + }, + remote: CacheActions { + read: true, + write: true, + }, + }, +) diff --git a/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__remote-only_remote_r,local_rw.snap b/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__remote-only_remote_r,local_rw.snap new file mode 100644 index 0000000000000..cbc9a96ee2d30 --- /dev/null +++ b/crates/turborepo-lib/src/snapshots/turborepo_lib__opts__test__remote-only_remote_r,local_rw.snap @@ -0,0 +1,7 @@ +--- +source: crates/turborepo-lib/src/opts.rs +expression: cache_opts +--- +Err( + OverlappingCacheOptions, +) diff --git a/crates/turborepo/tests/common/snapshots/query__common__check_query.snap b/crates/turborepo/tests/common/snapshots/query__common__check_query.snap deleted file mode 100644 index 890664e035247..0000000000000 --- a/crates/turborepo/tests/common/snapshots/query__common__check_query.snap +++ /dev/null @@ -1,23 +0,0 @@ ---- -source: crates/turborepo-lib/tests/common/mod.rs -expression: query_output ---- -{ - "data": { - "file": { - "path": "index.js", - "dependencies": { - "files": { - "items": [ - { - "path": "nm/index.js" - } - ] - }, - "errors": { - "items": [] - } - } - } - } -} diff --git a/crates/turborepo/tests/snapshots/query__check_query.snap b/crates/turborepo/tests/snapshots/query__check_query.snap deleted file mode 100644 index 4fbff3206ef3c..0000000000000 --- a/crates/turborepo/tests/snapshots/query__check_query.snap +++ /dev/null @@ -1,23 +0,0 @@ ---- -source: crates/turborepo-lib/tests/query.rs -expression: query_output ---- -{ - "data": { - "file": { - "path": "index.js", - "dependencies": { - "files": { - "items": [ - { - "path": "nm/index.js" - } - ] - }, - "errors": { - "items": [] - } - } - } - } -} diff --git a/crates/turborepo/tests/snapshots/query__oxc_repro_get_dependencies_(npm@10.5.0)_err.snap b/crates/turborepo/tests/snapshots/query__oxc_repro_get_dependencies_(npm@10.5.0)_err.snap deleted file mode 100644 index cd71b526b9bce..0000000000000 --- a/crates/turborepo/tests/snapshots/query__oxc_repro_get_dependencies_(npm@10.5.0)_err.snap +++ /dev/null @@ -1,8 +0,0 @@ ---- -source: crates/turborepo/tests/query.rs -expression: stderr ---- - WARNING No locally installed `turbo` found. Using version: 2.2.4-canary.0. -turbo 2.2.4-canary.0 - - WARNING query command is experimental and may change in the future diff --git a/crates/turborepo/tests/snapshots/query__query_oxc_repro_get_dependencies_(npm@10.5.0).snap b/crates/turborepo/tests/snapshots/query__query_oxc_repro_get_dependencies_(npm@10.5.0).snap deleted file mode 100644 index 2e879b27d8d8f..0000000000000 --- a/crates/turborepo/tests/snapshots/query__query_oxc_repro_get_dependencies_(npm@10.5.0).snap +++ /dev/null @@ -1,23 +0,0 @@ ---- -source: crates/turborepo/tests/query.rs -expression: query_output ---- -{ - "data": { - "file": { - "path": "index.js", - "dependencies": { - "files": { - "items": [ - { - "path": "nm/index.js" - } - ] - }, - "errors": { - "items": [] - } - } - } - } -} diff --git a/crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`button.tsx`_with_dependencies_(npm@10.5.0).snap b/crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`button.tsx`_with_dependencies_(npm@10.5.0).snap deleted file mode 100644 index 183e2c8e75dd7..0000000000000 --- a/crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`button.tsx`_with_dependencies_(npm@10.5.0).snap +++ /dev/null @@ -1,16 +0,0 @@ ---- -source: crates/turborepo/tests/query.rs -expression: query_output ---- -{ - "data": { - "file": { - "path": "button.tsx", - "dependencies": { - "files": { - "items": [] - } - } - } - } -} diff --git a/crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`circular.ts`_with_dependencies_(npm@10.5.0).snap b/crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`circular.ts`_with_dependencies_(npm@10.5.0).snap deleted file mode 100644 index e4950e2aa3f9d..0000000000000 --- a/crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`circular.ts`_with_dependencies_(npm@10.5.0).snap +++ /dev/null @@ -1,20 +0,0 @@ ---- -source: crates/turborepo/tests/query.rs -expression: query_output ---- -{ - "data": { - "file": { - "path": "circular.ts", - "dependencies": { - "files": { - "items": [ - { - "path": "circular2.ts" - } - ] - } - } - } - } -} diff --git a/crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`invalid.ts`_with_dependencies_(npm@10.5.0).snap b/crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`invalid.ts`_with_dependencies_(npm@10.5.0).snap deleted file mode 100644 index 8354df28773f7..0000000000000 --- a/crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`invalid.ts`_with_dependencies_(npm@10.5.0).snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: crates/turborepo/tests/query.rs -expression: query_output ---- -{ - "data": { - "file": { - "path": "invalid.ts", - "dependencies": { - "files": { - "items": [ - { - "path": "button.tsx" - } - ] - }, - "errors": { - "items": [ - { - "message": "failed to resolve import" - } - ] - } - } - } - } -} diff --git a/crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`main.ts`_(npm@10.5.0).snap b/crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`main.ts`_(npm@10.5.0).snap deleted file mode 100644 index 41503d9c76a44..0000000000000 --- a/crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`main.ts`_(npm@10.5.0).snap +++ /dev/null @@ -1,11 +0,0 @@ ---- -source: crates/turborepo/tests/query.rs -expression: query_output ---- -{ - "data": { - "file": { - "path": "main.ts" - } - } -} diff --git a/crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`main.ts`_with_ast_(npm@10.5.0).snap b/crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`main.ts`_with_ast_(npm@10.5.0).snap deleted file mode 100644 index 863d9791335e2..0000000000000 --- a/crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`main.ts`_with_ast_(npm@10.5.0).snap +++ /dev/null @@ -1,309 +0,0 @@ ---- -source: crates/turborepo/tests/query.rs -expression: query_output ---- -{ - "data": { - "file": { - "path": "main.ts", - "ast": { - "type": "Module", - "span": { - "start": 1, - "end": 169 - }, - "body": [ - { - "type": "ImportDeclaration", - "span": { - "start": 1, - "end": 35 - }, - "specifiers": [ - { - "type": "ImportSpecifier", - "span": { - "start": 10, - "end": 16 - }, - "local": { - "type": "Identifier", - "span": { - "start": 10, - "end": 16 - }, - "ctxt": 0, - "value": "Button", - "optional": false - }, - "imported": null, - "isTypeOnly": false - } - ], - "source": { - "type": "StringLiteral", - "span": { - "start": 24, - "end": 34 - }, - "value": "./button", - "raw": "\"./button\"" - }, - "typeOnly": false, - "with": null, - "phase": "evaluation" - }, - { - "type": "ImportDeclaration", - "span": { - "start": 36, - "end": 60 - }, - "specifiers": [ - { - "type": "ImportDefaultSpecifier", - "span": { - "start": 43, - "end": 46 - }, - "local": { - "type": "Identifier", - "span": { - "start": 43, - "end": 46 - }, - "ctxt": 0, - "value": "foo", - "optional": false - } - } - ], - "source": { - "type": "StringLiteral", - "span": { - "start": 52, - "end": 59 - }, - "value": "./foo", - "raw": "\"./foo\"" - }, - "typeOnly": false, - "with": null, - "phase": "evaluation" - }, - { - "type": "ImportDeclaration", - "span": { - "start": 61, - "end": 96 - }, - "specifiers": [ - { - "type": "ImportDefaultSpecifier", - "span": { - "start": 68, - "end": 74 - }, - "local": { - "type": "Identifier", - "span": { - "start": 68, - "end": 74 - }, - "ctxt": 0, - "value": "repeat", - "optional": false - } - } - ], - "source": { - "type": "StringLiteral", - "span": { - "start": 80, - "end": 95 - }, - "value": "repeat-string", - "raw": "\"repeat-string\"" - }, - "typeOnly": false, - "with": null, - "phase": "evaluation" - }, - { - "type": "VariableDeclaration", - "span": { - "start": 98, - "end": 126 - }, - "ctxt": 0, - "kind": "const", - "declare": false, - "declarations": [ - { - "type": "VariableDeclarator", - "span": { - "start": 104, - "end": 125 - }, - "id": { - "type": "Identifier", - "span": { - "start": 104, - "end": 110 - }, - "ctxt": 0, - "value": "button", - "optional": false, - "typeAnnotation": null - }, - "init": { - "type": "NewExpression", - "span": { - "start": 113, - "end": 125 - }, - "ctxt": 0, - "callee": { - "type": "Identifier", - "span": { - "start": 117, - "end": 123 - }, - "ctxt": 0, - "value": "Button", - "optional": false - }, - "arguments": [], - "typeArguments": null - }, - "definite": false - } - ] - }, - { - "type": "ExpressionStatement", - "span": { - "start": 128, - "end": 144 - }, - "expression": { - "type": "CallExpression", - "span": { - "start": 128, - "end": 143 - }, - "ctxt": 0, - "callee": { - "type": "MemberExpression", - "span": { - "start": 128, - "end": 141 - }, - "object": { - "type": "Identifier", - "span": { - "start": 128, - "end": 134 - }, - "ctxt": 0, - "value": "button", - "optional": false - }, - "property": { - "type": "Identifier", - "span": { - "start": 135, - "end": 141 - }, - "value": "render" - } - }, - "arguments": [], - "typeArguments": null - } - }, - { - "type": "ExpressionStatement", - "span": { - "start": 145, - "end": 162 - }, - "expression": { - "type": "CallExpression", - "span": { - "start": 145, - "end": 161 - }, - "ctxt": 0, - "callee": { - "type": "Identifier", - "span": { - "start": 145, - "end": 151 - }, - "ctxt": 0, - "value": "repeat", - "optional": false - }, - "arguments": [ - { - "spread": null, - "expression": { - "type": "StringLiteral", - "span": { - "start": 152, - "end": 157 - }, - "value": "foo", - "raw": "\"foo\"" - } - }, - { - "spread": null, - "expression": { - "type": "NumericLiteral", - "span": { - "start": 159, - "end": 160 - }, - "value": 5.0, - "raw": "5" - } - } - ], - "typeArguments": null - } - }, - { - "type": "ExpressionStatement", - "span": { - "start": 163, - "end": 169 - }, - "expression": { - "type": "CallExpression", - "span": { - "start": 163, - "end": 168 - }, - "ctxt": 0, - "callee": { - "type": "Identifier", - "span": { - "start": 163, - "end": 166 - }, - "ctxt": 0, - "value": "foo", - "optional": false - }, - "arguments": [], - "typeArguments": null - } - } - ], - "interpreter": null - } - } - } -} diff --git a/crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`main.ts`_with_dependencies_(npm@10.5.0).snap b/crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`main.ts`_with_dependencies_(npm@10.5.0).snap deleted file mode 100644 index d4762d093daf4..0000000000000 --- a/crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`main.ts`_with_dependencies_(npm@10.5.0).snap +++ /dev/null @@ -1,29 +0,0 @@ ---- -source: crates/turborepo/tests/query.rs -expression: query_output ---- -{ - "data": { - "file": { - "path": "main.ts", - "dependencies": { - "files": { - "items": [ - { - "path": "bar.js" - }, - { - "path": "button.tsx" - }, - { - "path": "foo.js" - }, - { - "path": "node_modules/repeat-string/index.js" - } - ] - } - } - } - } -} diff --git a/crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`main.ts`_with_depth_=_0_(npm@10.5.0).snap b/crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`main.ts`_with_depth_=_0_(npm@10.5.0).snap deleted file mode 100644 index d555c7a390292..0000000000000 --- a/crates/turborepo/tests/snapshots/query__query_turbo_trace_get_`main.ts`_with_depth_=_0_(npm@10.5.0).snap +++ /dev/null @@ -1,26 +0,0 @@ ---- -source: crates/turborepo/tests/query.rs -expression: query_output ---- -{ - "data": { - "file": { - "path": "main.ts", - "dependencies": { - "files": { - "items": [ - { - "path": "button.tsx" - }, - { - "path": "foo.js" - }, - { - "path": "node_modules/repeat-string/index.js" - } - ] - } - } - } - } -} diff --git a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`button.tsx`_with_dependencies_(npm@10.5.0)_err.snap b/crates/turborepo/tests/snapshots/query__turbo_trace_get_`button.tsx`_with_dependencies_(npm@10.5.0)_err.snap deleted file mode 100644 index cd71b526b9bce..0000000000000 --- a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`button.tsx`_with_dependencies_(npm@10.5.0)_err.snap +++ /dev/null @@ -1,8 +0,0 @@ ---- -source: crates/turborepo/tests/query.rs -expression: stderr ---- - WARNING No locally installed `turbo` found. Using version: 2.2.4-canary.0. -turbo 2.2.4-canary.0 - - WARNING query command is experimental and may change in the future diff --git a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`circular.ts`_with_dependencies_(npm@10.5.0)_err.snap b/crates/turborepo/tests/snapshots/query__turbo_trace_get_`circular.ts`_with_dependencies_(npm@10.5.0)_err.snap deleted file mode 100644 index cd71b526b9bce..0000000000000 --- a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`circular.ts`_with_dependencies_(npm@10.5.0)_err.snap +++ /dev/null @@ -1,8 +0,0 @@ ---- -source: crates/turborepo/tests/query.rs -expression: stderr ---- - WARNING No locally installed `turbo` found. Using version: 2.2.4-canary.0. -turbo 2.2.4-canary.0 - - WARNING query command is experimental and may change in the future diff --git a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`invalid.ts`_with_dependencies_(npm@10.5.0)_err.snap b/crates/turborepo/tests/snapshots/query__turbo_trace_get_`invalid.ts`_with_dependencies_(npm@10.5.0)_err.snap deleted file mode 100644 index cd71b526b9bce..0000000000000 --- a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`invalid.ts`_with_dependencies_(npm@10.5.0)_err.snap +++ /dev/null @@ -1,8 +0,0 @@ ---- -source: crates/turborepo/tests/query.rs -expression: stderr ---- - WARNING No locally installed `turbo` found. Using version: 2.2.4-canary.0. -turbo 2.2.4-canary.0 - - WARNING query command is experimental and may change in the future diff --git a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`main.ts`_(npm@10.5.0)_err.snap b/crates/turborepo/tests/snapshots/query__turbo_trace_get_`main.ts`_(npm@10.5.0)_err.snap deleted file mode 100644 index cd71b526b9bce..0000000000000 --- a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`main.ts`_(npm@10.5.0)_err.snap +++ /dev/null @@ -1,8 +0,0 @@ ---- -source: crates/turborepo/tests/query.rs -expression: stderr ---- - WARNING No locally installed `turbo` found. Using version: 2.2.4-canary.0. -turbo 2.2.4-canary.0 - - WARNING query command is experimental and may change in the future diff --git a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`main.ts`_with_ast_(npm@10.5.0)_err.snap b/crates/turborepo/tests/snapshots/query__turbo_trace_get_`main.ts`_with_ast_(npm@10.5.0)_err.snap deleted file mode 100644 index cd71b526b9bce..0000000000000 --- a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`main.ts`_with_ast_(npm@10.5.0)_err.snap +++ /dev/null @@ -1,8 +0,0 @@ ---- -source: crates/turborepo/tests/query.rs -expression: stderr ---- - WARNING No locally installed `turbo` found. Using version: 2.2.4-canary.0. -turbo 2.2.4-canary.0 - - WARNING query command is experimental and may change in the future diff --git a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`main.ts`_with_dependencies_(npm@10.5.0)_err.snap b/crates/turborepo/tests/snapshots/query__turbo_trace_get_`main.ts`_with_dependencies_(npm@10.5.0)_err.snap deleted file mode 100644 index cd71b526b9bce..0000000000000 --- a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`main.ts`_with_dependencies_(npm@10.5.0)_err.snap +++ /dev/null @@ -1,8 +0,0 @@ ---- -source: crates/turborepo/tests/query.rs -expression: stderr ---- - WARNING No locally installed `turbo` found. Using version: 2.2.4-canary.0. -turbo 2.2.4-canary.0 - - WARNING query command is experimental and may change in the future diff --git a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`main.ts`_with_depth_=_0_(npm@10.5.0)_err.snap b/crates/turborepo/tests/snapshots/query__turbo_trace_get_`main.ts`_with_depth_=_0_(npm@10.5.0)_err.snap deleted file mode 100644 index cd71b526b9bce..0000000000000 --- a/crates/turborepo/tests/snapshots/query__turbo_trace_get_`main.ts`_with_depth_=_0_(npm@10.5.0)_err.snap +++ /dev/null @@ -1,8 +0,0 @@ ---- -source: crates/turborepo/tests/query.rs -expression: stderr ---- - WARNING No locally installed `turbo` found. Using version: 2.2.4-canary.0. -turbo 2.2.4-canary.0 - - WARNING query command is experimental and may change in the future diff --git a/crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`packages__utils__index`_with_dependents_(npm@10.5.0).snap b/crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`packages__utils__index`_with_dependents_(npm@10.5.0).snap deleted file mode 100644 index 4e6e7ed0e3329..0000000000000 --- a/crates/turborepo/tests/snapshots/query__turbo_trace_monorepo_get_`packages__utils__index`_with_dependents_(npm@10.5.0).snap +++ /dev/null @@ -1,26 +0,0 @@ ---- -source: crates/turborepo/tests/query.rs -expression: query_output ---- -{ - "data": { - "file": { - "path": "packages/utils/index.ts", - "dependents": { - "files": { - "items": [ - { - "path": "apps/my-app/index.ts" - }, - { - "path": "packages/another/index.js" - } - ] - }, - "errors": { - "items": [] - } - } - } - } -} diff --git a/examples/with-nestjs/apps/web/test/__snapshots__/page.spec.tsx.snap b/examples/with-nestjs/apps/web/test/__snapshots__/page.spec.tsx.snap deleted file mode 100644 index 0a564fcf428b6..0000000000000 --- a/examples/with-nestjs/apps/web/test/__snapshots__/page.spec.tsx.snap +++ /dev/null @@ -1,160 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`Root page should match the snapshot 1`] = ` -
-
-
-

- examples/ - - with-nestjs - -

- -
- -
-
-
-
- -
-
- -
- -
- -
- - - Turborepo logo - - - - - - - - - - - -
-
-
-
- -

- title - - - -> - -

-

- description -

- -
-
-
-`; diff --git a/turborepo-tests/integration/tests/no-args.t b/turborepo-tests/integration/tests/no-args.t index 72848871ce25c..7236297d88e32 100644 --- a/turborepo-tests/integration/tests/no-args.t +++ b/turborepo-tests/integration/tests/no-args.t @@ -65,14 +65,22 @@ Make sure exit code is 2 when no args are passed Print help (see more with '--help') Run Arguments: + --cache + Set the cache behavior for this run. Pass a list of comma-separated key, value pairs to enable reading and writing to either the local or remote cache + --force [] + Ignore the existing cache (to force execution). Equivalent to `--cache=local:w,remote:w` [possible values: true, false] + --remote-only [] + Ignore the local filesystem cache for all tasks. Only allow reading and caching artifacts using the remote cache. Equivalent to `--cache=remote:rw` [possible values: true, false] + --remote-cache-read-only [] + Treat remote cache as read only. Equivalent to `--cache=remote:r;local:rw` [possible values: true, false] + --no-cache + Avoid saving task results to the cache. Useful for development/watch tasks. Equivalent to `--cache=local:r,remote:r` --cache-workers Set the number of concurrent cache operations (default 10) [default: 10] --dry-run [] [possible values: text, json] --graph [] Generate a graph of the task execution and output to a file when a filename is specified (.svg, .png, .jpg, .pdf, .json, .html, .mermaid, .dot). Outputs dot graph to stdout when if no filename is provided - --no-cache - Avoid saving task results to the cache. Useful for development/watch tasks --daemon Force turbo to use the local daemon. If unset turbo will use the default detection logic --no-daemon @@ -81,8 +89,6 @@ Make sure exit code is 2 when no args are passed File to write turbo's performance profile output into. You can load the file up in chrome://tracing to see which parts of your build were slow --anon-profile File to write turbo's performance profile output into. All identifying data omitted from the profile - --remote-cache-read-only [] - Treat remote cache as read only [possible values: true, false] --summarize [] Generate a summary of the turbo run [possible values: true, false] --parallel @@ -95,8 +101,6 @@ Make sure exit code is 2 when no args are passed Continue execution even if a task exits with an error or non-zero exit code. The default behavior is to bail --single-package Run turbo in single-package mode - --force [] - Ignore the existing cache (to force execution) [possible values: true, false] --framework-inference [] Specify whether or not to do framework inference for tasks [default: true] [possible values: true, false] --global-deps @@ -113,8 +117,6 @@ Make sure exit code is 2 when no args are passed Set type of task output order. Use "stream" to show output as soon as it is available. Use "grouped" to show output when a command has finished execution. Use "auto" to let turbo decide based on its own heuristics. (default auto) [possible values: auto, stream, grouped] --only Only executes the tasks specified, does not execute parent tasks - --remote-only [] - Ignore the local filesystem cache for all tasks. Only allow reading and caching artifacts using the remote cache [possible values: true, false] --log-prefix Use "none" to remove prefixes from task logs. Use "task" to get task id prefixing. Use "auto" to let turbo decide how to prefix the logs based on the execution environment. In most cases this will be the same as "task". Note that tasks running in parallel interleave their logs, so removing prefixes can make it difficult to associate logs with tasks. Use --log-order=grouped to prevent interleaving. (default auto) [default: auto] [possible values: auto, none, task] [1] diff --git a/turborepo-tests/integration/tests/turbo-help.t b/turborepo-tests/integration/tests/turbo-help.t index 0fd66d1c3d7be..78f2ce5208156 100644 --- a/turborepo-tests/integration/tests/turbo-help.t +++ b/turborepo-tests/integration/tests/turbo-help.t @@ -65,14 +65,22 @@ Test help flag Print help (see more with '--help') Run Arguments: + --cache + Set the cache behavior for this run. Pass a list of comma-separated key, value pairs to enable reading and writing to either the local or remote cache + --force [] + Ignore the existing cache (to force execution). Equivalent to `--cache=local:w,remote:w` [possible values: true, false] + --remote-only [] + Ignore the local filesystem cache for all tasks. Only allow reading and caching artifacts using the remote cache. Equivalent to `--cache=remote:rw` [possible values: true, false] + --remote-cache-read-only [] + Treat remote cache as read only. Equivalent to `--cache=remote:r;local:rw` [possible values: true, false] + --no-cache + Avoid saving task results to the cache. Useful for development/watch tasks. Equivalent to `--cache=local:r,remote:r` --cache-workers Set the number of concurrent cache operations (default 10) [default: 10] --dry-run [] [possible values: text, json] --graph [] Generate a graph of the task execution and output to a file when a filename is specified (.svg, .png, .jpg, .pdf, .json, .html, .mermaid, .dot). Outputs dot graph to stdout when if no filename is provided - --no-cache - Avoid saving task results to the cache. Useful for development/watch tasks --daemon Force turbo to use the local daemon. If unset turbo will use the default detection logic --no-daemon @@ -81,8 +89,6 @@ Test help flag File to write turbo's performance profile output into. You can load the file up in chrome://tracing to see which parts of your build were slow --anon-profile File to write turbo's performance profile output into. All identifying data omitted from the profile - --remote-cache-read-only [] - Treat remote cache as read only [possible values: true, false] --summarize [] Generate a summary of the turbo run [possible values: true, false] --parallel @@ -95,8 +101,6 @@ Test help flag Continue execution even if a task exits with an error or non-zero exit code. The default behavior is to bail --single-package Run turbo in single-package mode - --force [] - Ignore the existing cache (to force execution) [possible values: true, false] --framework-inference [] Specify whether or not to do framework inference for tasks [default: true] [possible values: true, false] --global-deps @@ -113,8 +117,6 @@ Test help flag Set type of task output order. Use "stream" to show output as soon as it is available. Use "grouped" to show output when a command has finished execution. Use "auto" to let turbo decide based on its own heuristics. (default auto) [possible values: auto, stream, grouped] --only Only executes the tasks specified, does not execute parent tasks - --remote-only [] - Ignore the local filesystem cache for all tasks. Only allow reading and caching artifacts using the remote cache [possible values: true, false] --log-prefix Use "none" to remove prefixes from task logs. Use "task" to get task id prefixing. Use "auto" to let turbo decide how to prefix the logs based on the execution environment. In most cases this will be the same as "task". Note that tasks running in parallel interleave their logs, so removing prefixes can make it difficult to associate logs with tasks. Use --log-order=grouped to prevent interleaving. (default auto) [default: auto] [possible values: auto, none, task] @@ -211,6 +213,27 @@ Test help flag Print help (see a summary with '-h') Run Arguments: + --cache + Set the cache behavior for this run. Pass a list of comma-separated key, value pairs to enable reading and writing to either the local or remote cache + + --force [] + Ignore the existing cache (to force execution). Equivalent to `--cache=local:w,remote:w` + + [possible values: true, false] + + --remote-only [] + Ignore the local filesystem cache for all tasks. Only allow reading and caching artifacts using the remote cache. Equivalent to `--cache=remote:rw` + + [possible values: true, false] + + --remote-cache-read-only [] + Treat remote cache as read only. Equivalent to `--cache=remote:r;local:rw` + + [possible values: true, false] + + --no-cache + Avoid saving task results to the cache. Useful for development/watch tasks. Equivalent to `--cache=local:r,remote:r` + --cache-workers Set the number of concurrent cache operations (default 10) @@ -222,9 +245,6 @@ Test help flag --graph [] Generate a graph of the task execution and output to a file when a filename is specified (.svg, .png, .jpg, .pdf, .json, .html, .mermaid, .dot). Outputs dot graph to stdout when if no filename is provided - --no-cache - Avoid saving task results to the cache. Useful for development/watch tasks - --daemon Force turbo to use the local daemon. If unset turbo will use the default detection logic @@ -237,11 +257,6 @@ Test help flag --anon-profile File to write turbo's performance profile output into. All identifying data omitted from the profile - --remote-cache-read-only [] - Treat remote cache as read only - - [possible values: true, false] - --summarize [] Generate a summary of the turbo run @@ -262,11 +277,6 @@ Test help flag --single-package Run turbo in single-package mode - --force [] - Ignore the existing cache (to force execution) - - [possible values: true, false] - --framework-inference [] Specify whether or not to do framework inference for tasks @@ -300,11 +310,6 @@ Test help flag --only Only executes the tasks specified, does not execute parent tasks - --remote-only [] - Ignore the local filesystem cache for all tasks. Only allow reading and caching artifacts using the remote cache - - [possible values: true, false] - --log-prefix Use "none" to remove prefixes from task logs. Use "task" to get task id prefixing. Use "auto" to let turbo decide how to prefix the logs based on the execution environment. In most cases this will be the same as "task". Note that tasks running in parallel interleave their logs, so removing prefixes can make it difficult to associate logs with tasks. Use --log-order=grouped to prevent interleaving. (default auto) From 9c0236563211b2c79acf64cd65ff2788def77dec Mon Sep 17 00:00:00 2001 From: Anthony Shew Date: Tue, 5 Nov 2024 15:17:04 -0700 Subject: [PATCH 12/12] docs: Updates for accuracy in Add to Existing Repository page. (#9379) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Description There were a few parts to the copy on this page that needed some touching up, according to user feedback. ### Testing Instructions 👀 --- .../add-to-existing-repository.mdx | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/docs/repo-docs/getting-started/add-to-existing-repository.mdx b/docs/repo-docs/getting-started/add-to-existing-repository.mdx index a561e9c5c9879..f240aa936ece0 100644 --- a/docs/repo-docs/getting-started/add-to-existing-repository.mdx +++ b/docs/repo-docs/getting-started/add-to-existing-repository.mdx @@ -179,13 +179,34 @@ Add `.turbo` to your `.gitignore` file. The `turbo` CLI uses these folders for p -### Run the `build` and `check-types` tasks with `turbo` +### Add a `packageManager` field to root `package.json` + +Turborepo optimizes your repository using information from your package manager. To declare which package manager you're using, add a [`packageManager`](https://nodejs.org/api/packages.html#packagemanager) field to your root `package.json` if you don't have one already. + +```diff title="package.json" +{ ++ "packageManager": "npm@8.5.0" +} +``` + + + Depending on your repository, you may need to use the + [dangerouslyDisablePackageManagerCheck](`/repo/docs/reference/configuration#dangerouslydisablepackagemanagercheck`) + while migrating or in situations where you can't use the `packageManager` key + yet. + + + + +### Run tasks with `turbo` + +You can now run the tasks you added to `turbo.json` earlier using Turborepo. Using the example tasks from above: ```bash title="Terminal" turbo build check-types ``` -This runs `build` and `check-types` at the same time. +This runs the `build` and `check-types` tasks at the same time. The dependency graph of your [Workspace](/repo/docs/crafting-your-repository/structuring-a-repository#anatomy-of-a-workspace) will be used to run tasks in the right order.