From 8856b3be692015095a3efc05d14ae444ec7bbabd Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Tue, 17 Sep 2024 10:47:23 -0400 Subject: [PATCH] feat: allow usage of `turbo` without turbo.json (#9149) ### Description This PR adds the ability to use `turbo` in a monorepo that doesn't have a `turbo.json`. This feature is currently gated behind `--experimental-allow-no-turbo-json`/`TURBO_ALLOW_NO_TURBO_JSON`. A majority of the PR is refactoring the `EngineBuilder` so that it no longer directly loads `TurboJson`s, but delegates to a `TurboJsonLoader`. This allows us to use different strategies for resolving `TurboJson` loads depending on runtime options e.g. single package mode or task access. Reviewing this PR is best done by viewing each commit individually. ### Testing Instructions Unit testing for `turbo.json` loading changes. Integration test for verifying the new loader is activated with the new env var/flag. --- crates/turborepo-lib/src/cli/mod.rs | 2 + crates/turborepo-lib/src/commands/mod.rs | 1 + crates/turborepo-lib/src/config/env.rs | 7 + crates/turborepo-lib/src/config/mod.rs | 8 + crates/turborepo-lib/src/engine/builder.rs | 228 ++---- .../src/package_changes_watcher.rs | 26 +- crates/turborepo-lib/src/run/builder.rs | 65 +- crates/turborepo-lib/src/run/summary/task.rs | 4 + crates/turborepo-lib/src/run/task_access.rs | 23 +- crates/turborepo-lib/src/task_graph/mod.rs | 6 +- .../turborepo-lib/src/task_graph/visitor.rs | 2 +- crates/turborepo-lib/src/turbo_json/loader.rs | 679 ++++++++++++++++++ crates/turborepo-lib/src/turbo_json/mod.rs | 208 +----- .../monorepo_no_turbo_json/.gitignore | 3 + .../apps/my-app/.env.local | 0 .../apps/my-app/package.json | 10 + .../fixtures/monorepo_no_turbo_json/bar.txt | 1 + .../fixtures/monorepo_no_turbo_json/foo.txt | 1 + .../monorepo_no_turbo_json/package.json | 11 + .../packages/another/package.json | 4 + .../packages/util/package.json | 6 + .../tests/run/allow-no-root-turbo.t | 50 ++ 22 files changed, 946 insertions(+), 399 deletions(-) create mode 100644 crates/turborepo-lib/src/turbo_json/loader.rs create mode 100644 turborepo-tests/integration/fixtures/monorepo_no_turbo_json/.gitignore create mode 100644 turborepo-tests/integration/fixtures/monorepo_no_turbo_json/apps/my-app/.env.local create mode 100644 turborepo-tests/integration/fixtures/monorepo_no_turbo_json/apps/my-app/package.json create mode 100644 turborepo-tests/integration/fixtures/monorepo_no_turbo_json/bar.txt create mode 100644 turborepo-tests/integration/fixtures/monorepo_no_turbo_json/foo.txt create mode 100644 turborepo-tests/integration/fixtures/monorepo_no_turbo_json/package.json create mode 100644 turborepo-tests/integration/fixtures/monorepo_no_turbo_json/packages/another/package.json create mode 100644 turborepo-tests/integration/fixtures/monorepo_no_turbo_json/packages/util/package.json create mode 100644 turborepo-tests/integration/tests/run/allow-no-root-turbo.t diff --git a/crates/turborepo-lib/src/cli/mod.rs b/crates/turborepo-lib/src/cli/mod.rs index 3d8a4fa1735b8..2f68ff217322c 100644 --- a/crates/turborepo-lib/src/cli/mod.rs +++ b/crates/turborepo-lib/src/cli/mod.rs @@ -222,6 +222,8 @@ pub struct Args { /// should be used. #[clap(long, global = true)] pub dangerously_disable_package_manager_check: bool, + #[clap(long = "experimental-allow-no-turbo-json", hide = true, global = true)] + pub allow_no_turbo_json: bool, /// Use the `turbo.json` located at the provided path instead of one at the /// root of the repository. #[clap(long, global = true)] diff --git a/crates/turborepo-lib/src/commands/mod.rs b/crates/turborepo-lib/src/commands/mod.rs index 857f5e31b111b..04f1568bc0df8 100644 --- a/crates/turborepo-lib/src/commands/mod.rs +++ b/crates/turborepo-lib/src/commands/mod.rs @@ -108,6 +108,7 @@ impl CommandBase { .and_then(|args| args.remote_cache_read_only()), ) .with_run_summary(self.args.run_args().and_then(|args| args.summarize())) + .with_allow_no_turbo_json(self.args.allow_no_turbo_json.then_some(true)) .build() } diff --git a/crates/turborepo-lib/src/config/env.rs b/crates/turborepo-lib/src/config/env.rs index bc7e08437eda4..84601bf9f1da8 100644 --- a/crates/turborepo-lib/src/config/env.rs +++ b/crates/turborepo-lib/src/config/env.rs @@ -38,6 +38,7 @@ const TURBO_MAPPING: &[(&str, &str)] = [ ("turbo_remote_only", "remote_only"), ("turbo_remote_cache_read_only", "remote_cache_read_only"), ("turbo_run_summary", "run_summary"), + ("turbo_allow_no_turbo_json", "allow_no_turbo_json"), ] .as_slice(); @@ -86,6 +87,7 @@ impl ResolvedConfigurationOptions for EnvVars { let remote_only = self.truthy_value("remote_only").flatten(); let remote_cache_read_only = self.truthy_value("remote_cache_read_only").flatten(); let run_summary = self.truthy_value("run_summary").flatten(); + let allow_no_turbo_json = self.truthy_value("allow_no_turbo_json").flatten(); // Process timeout let timeout = self @@ -171,6 +173,7 @@ impl ResolvedConfigurationOptions for EnvVars { remote_only, remote_cache_read_only, run_summary, + allow_no_turbo_json, // Processed numbers timeout, @@ -317,6 +320,7 @@ mod test { env.insert("turbo_remote_only".into(), "1".into()); env.insert("turbo_remote_cache_read_only".into(), "1".into()); env.insert("turbo_run_summary".into(), "true".into()); + env.insert("turbo_allow_no_turbo_json".into(), "true".into()); let config = EnvVars::new(&env) .unwrap() @@ -328,6 +332,7 @@ mod test { assert!(config.remote_only()); assert!(config.remote_cache_read_only()); assert!(config.run_summary()); + assert!(config.allow_no_turbo_json()); assert_eq!(turbo_api, config.api_url.unwrap()); assert_eq!(turbo_login, config.login_url.unwrap()); assert_eq!(turbo_team, config.team_slug.unwrap()); @@ -365,6 +370,7 @@ mod test { env.insert("turbo_remote_only".into(), "".into()); env.insert("turbo_remote_cache_read_only".into(), "".into()); env.insert("turbo_run_summary".into(), "".into()); + env.insert("turbo_allow_no_turbo_json".into(), "".into()); let config = EnvVars::new(&env) .unwrap() @@ -387,6 +393,7 @@ mod test { assert!(!config.remote_only()); assert!(!config.remote_cache_read_only()); assert!(!config.run_summary()); + assert!(!config.allow_no_turbo_json()); } #[test] diff --git a/crates/turborepo-lib/src/config/mod.rs b/crates/turborepo-lib/src/config/mod.rs index a693b601466c6..1c35949bbef33 100644 --- a/crates/turborepo-lib/src/config/mod.rs +++ b/crates/turborepo-lib/src/config/mod.rs @@ -17,6 +17,7 @@ use thiserror::Error; use turbo_json::TurboJsonReader; use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf}; use turborepo_errors::TURBO_SITE; +use turborepo_repository::package_graph::PackageName; pub use crate::turbo_json::{RawTurboJson, UIMode}; use crate::{ @@ -179,6 +180,8 @@ pub enum Error { #[source_code] text: NamedSource, }, + #[error("Cannot load turbo.json for in {0} single package mode")] + InvalidTurboJsonLoad(PackageName), } const DEFAULT_API_URL: &str = "https://vercel.com/api"; @@ -239,6 +242,7 @@ pub struct ConfigurationOptions { pub(crate) remote_only: Option, pub(crate) remote_cache_read_only: Option, pub(crate) run_summary: Option, + pub(crate) allow_no_turbo_json: Option, } #[derive(Default)] @@ -367,6 +371,10 @@ impl ConfigurationOptions { .clone() .unwrap_or_else(|| repo_root.join_component(CONFIG_FILE)) } + + pub fn allow_no_turbo_json(&self) -> bool { + self.allow_no_turbo_json.unwrap_or_default() + } } // Maps Some("") to None to emulate how Go handles empty strings diff --git a/crates/turborepo-lib/src/engine/builder.rs b/crates/turborepo-lib/src/engine/builder.rs index 00aaf64a6b47a..484cce98974e5 100644 --- a/crates/turborepo-lib/src/engine/builder.rs +++ b/crates/turborepo-lib/src/engine/builder.rs @@ -14,8 +14,7 @@ use crate::{ run::task_id::{TaskId, TaskName}, task_graph::TaskDefinition, turbo_json::{ - validate_extends, validate_no_package_task_syntax, RawTaskDefinition, TurboJson, - CONFIG_FILE, + validate_extends, validate_no_package_task_syntax, RawTaskDefinition, TurboJsonLoader, }, }; @@ -95,8 +94,8 @@ pub enum Error { pub struct EngineBuilder<'a> { repo_root: &'a AbsoluteSystemPath, package_graph: &'a PackageGraph, + turbo_json_loader: Option, is_single: bool, - turbo_jsons: Option>, workspaces: Vec, tasks: Vec>>, root_enabled_tasks: HashSet>, @@ -107,13 +106,14 @@ impl<'a> EngineBuilder<'a> { pub fn new( repo_root: &'a AbsoluteSystemPath, package_graph: &'a PackageGraph, + turbo_json_loader: TurboJsonLoader, is_single: bool, ) -> Self { Self { repo_root, package_graph, + turbo_json_loader: Some(turbo_json_loader), is_single, - turbo_jsons: None, workspaces: Vec::new(), tasks: Vec::new(), root_enabled_tasks: HashSet::new(), @@ -121,14 +121,6 @@ impl<'a> EngineBuilder<'a> { } } - pub fn with_turbo_jsons( - mut self, - turbo_jsons: Option>, - ) -> Self { - self.turbo_jsons = turbo_jsons; - self - } - pub fn with_tasks_only(mut self, tasks_only: bool) -> Self { self.tasks_only = tasks_only; self @@ -186,7 +178,10 @@ impl<'a> EngineBuilder<'a> { return Ok(Engine::default().seal()); } - let mut turbo_jsons = self.turbo_jsons.take().unwrap_or_default(); + let mut turbo_json_loader = self + .turbo_json_loader + .take() + .expect("engine builder cannot be constructed without TurboJsonLoader"); let mut missing_tasks: HashMap<&TaskName<'_>, Spanned<()>> = HashMap::from_iter(self.tasks.iter().map(|spanned| spanned.as_ref().split())); let mut traversal_queue = VecDeque::with_capacity(1); @@ -195,7 +190,7 @@ impl<'a> EngineBuilder<'a> { .task_id() .unwrap_or_else(|| TaskId::new(workspace.as_ref(), task.task())); - if self.has_task_definition(&mut turbo_jsons, workspace, task, &task_id)? { + if Self::has_task_definition(&mut turbo_json_loader, workspace, task, &task_id)? { missing_tasks.remove(task.as_inner()); // Even if a task definition was found, we _only_ want to add it as an entry @@ -274,13 +269,12 @@ impl<'a> EngineBuilder<'a> { task_id: task_id.to_string(), }); } - let raw_task_definition = RawTaskDefinition::from_iter(self.task_definition_chain( - &mut turbo_jsons, + + let task_definition = self.task_definition( + &mut turbo_json_loader, &task_id, &task_id.as_non_workspace_task_name(), - )?); - - let task_definition = TaskDefinition::try_from(raw_task_definition)?; + )?; // Skip this iteration of the loop if we've already seen this taskID if visited.contains(task_id.as_inner()) { @@ -370,25 +364,27 @@ impl<'a> EngineBuilder<'a> { // Helper methods used when building the engine fn has_task_definition( - &self, - turbo_jsons: &mut HashMap, + loader: &mut TurboJsonLoader, workspace: &PackageName, task_name: &TaskName<'static>, task_id: &TaskId, ) -> Result { - let turbo_json = self - .turbo_json(turbo_jsons, workspace) - // If there was no turbo.json in the workspace, fallback to the root turbo.json - .or_else(|e| { - if e.is_missing_turbo_json() && !matches!(workspace, PackageName::Root) { + let turbo_json = loader.load(workspace).map_or_else( + |err| { + if matches!(err, config::Error::NoTurboJSON) + && !matches!(workspace, PackageName::Root) + { Ok(None) } else { - Err(e) + Err(err) } - })?; + }, + |turbo_json| Ok(Some(turbo_json)), + )?; let Some(turbo_json) = turbo_json else { - return self.has_task_definition(turbo_jsons, &PackageName::Root, task_name, task_id); + // If there was no turbo.json in the workspace, fallback to the root turbo.json + return Self::has_task_definition(loader, &PackageName::Root, task_name, task_id); }; let task_id_as_name = task_id.as_task_name(); @@ -397,23 +393,36 @@ impl<'a> EngineBuilder<'a> { { Ok(true) } else if !matches!(workspace, PackageName::Root) { - self.has_task_definition(turbo_jsons, &PackageName::Root, task_name, task_id) + Self::has_task_definition(loader, &PackageName::Root, task_name, task_id) } else { Ok(false) } } + fn task_definition( + &self, + turbo_json_loader: &mut TurboJsonLoader, + task_id: &Spanned, + task_name: &TaskName, + ) -> Result { + let raw_task_definition = RawTaskDefinition::from_iter(self.task_definition_chain( + turbo_json_loader, + task_id, + task_name, + )?); + + Ok(TaskDefinition::try_from(raw_task_definition)?) + } + fn task_definition_chain( &self, - turbo_jsons: &mut HashMap, + turbo_json_loader: &mut TurboJsonLoader, task_id: &Spanned, task_name: &TaskName, ) -> Result, Error> { let mut task_definitions = Vec::new(); - let root_turbo_json = self - .turbo_json(turbo_jsons, &PackageName::Root)? - .ok_or(Error::Config(crate::config::Error::NoTurboJSON))?; + let root_turbo_json = turbo_json_loader.load(&PackageName::Root)?; if let Some(root_definition) = root_turbo_json.task(task_id, task_name) { task_definitions.push(root_definition) @@ -434,8 +443,8 @@ impl<'a> EngineBuilder<'a> { } if task_id.package() != ROOT_PKG_NAME { - match self.turbo_json(turbo_jsons, &PackageName::from(task_id.package())) { - Ok(Some(workspace_json)) => { + match turbo_json_loader.load(&PackageName::from(task_id.package())) { + Ok(workspace_json) => { let validation_errors = workspace_json .validate(&[validate_no_package_task_syntax, validate_extends]); if !validation_errors.is_empty() { @@ -448,11 +457,9 @@ impl<'a> EngineBuilder<'a> { task_definitions.push(workspace_def.value.clone()); } } - Ok(None) => (), - // swallow the error where the config file doesn't exist, but bubble up other things - Err(e) if e.is_missing_turbo_json() => (), + Err(config::Error::NoTurboJSON) => (), Err(e) => { - return Err(e); + return Err(e.into()); } } } @@ -469,42 +476,6 @@ impl<'a> EngineBuilder<'a> { Ok(task_definitions) } - - fn turbo_json<'b>( - &self, - turbo_jsons: &'b mut HashMap, - workspace: &PackageName, - ) -> Result, Error> { - if turbo_jsons.get(workspace).is_none() { - let json = self.load_turbo_json(workspace)?; - turbo_jsons.insert(workspace.clone(), json); - } - Ok(turbo_jsons.get(workspace)) - } - - fn load_turbo_json(&self, workspace: &PackageName) -> Result { - let package_json = self.package_graph.package_json(workspace).ok_or_else(|| { - Error::MissingPackageJson { - workspace: workspace.clone(), - } - })?; - let workspace_dir = - self.package_graph - .package_dir(workspace) - .ok_or_else(|| Error::MissingPackageJson { - workspace: workspace.clone(), - })?; - let workspace_turbo_json = self - .repo_root - .resolve(workspace_dir) - .join_component(CONFIG_FILE); - Ok(TurboJson::load( - self.repo_root, - &workspace_turbo_json, - package_json, - self.is_single, - )?) - } } impl Error { @@ -548,7 +519,10 @@ mod test { }; use super::*; - use crate::{engine::TaskNode, turbo_json::RawTurboJson}; + use crate::{ + engine::TaskNode, + turbo_json::{RawTurboJson, TurboJson}, + }; // Only used to prevent package graph construction from attempting to read // lockfile from disk @@ -650,41 +624,6 @@ mod test { .unwrap() } - #[test] - fn test_turbo_json_loading() { - let repo_root_dir = TempDir::with_prefix("repo").unwrap(); - let repo_root = AbsoluteSystemPathBuf::new(repo_root_dir.path().to_str().unwrap()).unwrap(); - let package_graph = mock_package_graph( - &repo_root, - package_jsons! { - repo_root, - "a" => [], - "b" => [], - "c" => ["a", "b"] - }, - ); - let engine_builder = EngineBuilder::new(&repo_root, &package_graph, false) - .with_turbo_jsons(Some(vec![].into_iter().collect())); - - let a_turbo_json = repo_root.join_components(&["packages", "a", "turbo.json"]); - a_turbo_json.ensure_dir().unwrap(); - - let result = engine_builder.load_turbo_json(&PackageName::from("a")); - assert!( - result.is_err() && result.unwrap_err().is_missing_turbo_json(), - "expected parsing to fail with missing turbo.json" - ); - - a_turbo_json - .create_with_contents(r#"{"tasks": {"build": {}}}"#) - .unwrap(); - - let turbo_json = engine_builder - .load_turbo_json(&PackageName::from("a")) - .unwrap(); - assert_eq!(turbo_json.tasks.len(), 1); - } - fn turbo_json(value: serde_json::Value) -> TurboJson { let json_text = serde_json::to_string(&value).unwrap(); let raw = RawTurboJson::parse(&json_text, "").unwrap(); @@ -702,18 +641,7 @@ mod test { task_id: &'static str, expected: bool, ) { - let repo_root_dir = TempDir::with_prefix("repo").unwrap(); - let repo_root = AbsoluteSystemPathBuf::new(repo_root_dir.path().to_str().unwrap()).unwrap(); - let package_graph = mock_package_graph( - &repo_root, - package_jsons! { - repo_root, - "a" => [], - "b" => [], - "c" => ["a", "b"] - }, - ); - let mut turbo_jsons = vec![ + let turbo_jsons = vec![ ( PackageName::Root, turbo_json(json!({ @@ -735,13 +663,13 @@ mod test { ] .into_iter() .collect(); - let engine_builder = EngineBuilder::new(&repo_root, &package_graph, false); + let mut loader = TurboJsonLoader::noop(turbo_jsons); let task_name = TaskName::from(task_name); let task_id = TaskId::try_from(task_id).unwrap(); - let has_def = engine_builder - .has_task_definition(&mut turbo_jsons, &workspace, &task_name, &task_id) - .unwrap(); + let has_def = + EngineBuilder::has_task_definition(&mut loader, &workspace, &task_name, &task_id) + .unwrap(); assert_eq!(has_def, expected); } @@ -805,8 +733,8 @@ mod test { )] .into_iter() .collect(); - let engine = EngineBuilder::new(&repo_root, &package_graph, false) - .with_turbo_jsons(Some(turbo_jsons)) + let loader = TurboJsonLoader::noop(turbo_jsons); + let engine = EngineBuilder::new(&repo_root, &package_graph, loader, false) .with_tasks(Some(Spanned::new(TaskName::from("test")))) .with_workspaces(vec![ PackageName::from("a"), @@ -862,8 +790,8 @@ mod test { )] .into_iter() .collect(); - let engine = EngineBuilder::new(&repo_root, &package_graph, false) - .with_turbo_jsons(Some(turbo_jsons)) + let loader = TurboJsonLoader::noop(turbo_jsons); + let engine = EngineBuilder::new(&repo_root, &package_graph, loader, false) .with_tasks(Some(Spanned::new(TaskName::from("test")))) .with_workspaces(vec![PackageName::from("app2")]) .build() @@ -901,8 +829,8 @@ mod test { )] .into_iter() .collect(); - let engine = EngineBuilder::new(&repo_root, &package_graph, false) - .with_turbo_jsons(Some(turbo_jsons)) + let loader = TurboJsonLoader::noop(turbo_jsons); + let engine = EngineBuilder::new(&repo_root, &package_graph, loader, false) .with_tasks(Some(Spanned::new(TaskName::from("special")))) .with_workspaces(vec![PackageName::from("app1"), PackageName::from("libA")]) .build() @@ -939,8 +867,8 @@ mod test { )] .into_iter() .collect(); - let engine = EngineBuilder::new(&repo_root, &package_graph, false) - .with_turbo_jsons(Some(turbo_jsons)) + let loader = TurboJsonLoader::noop(turbo_jsons); + let engine = EngineBuilder::new(&repo_root, &package_graph, loader, false) .with_tasks(vec![ Spanned::new(TaskName::from("build")), Spanned::new(TaskName::from("test")), @@ -992,8 +920,8 @@ mod test { )] .into_iter() .collect(); - let engine = EngineBuilder::new(&repo_root, &package_graph, false) - .with_turbo_jsons(Some(turbo_jsons)) + let loader = TurboJsonLoader::noop(turbo_jsons); + let engine = EngineBuilder::new(&repo_root, &package_graph, loader, false) .with_tasks(Some(Spanned::new(TaskName::from("build")))) .with_workspaces(vec![PackageName::from("app1")]) .with_root_tasks(vec![ @@ -1035,8 +963,8 @@ mod test { )] .into_iter() .collect(); - let engine = EngineBuilder::new(&repo_root, &package_graph, false) - .with_turbo_jsons(Some(turbo_jsons)) + let loader = TurboJsonLoader::noop(turbo_jsons); + let engine = EngineBuilder::new(&repo_root, &package_graph, loader, false) .with_tasks(Some(Spanned::new(TaskName::from("build")))) .with_workspaces(vec![PackageName::from("app1")]) .with_root_tasks(vec![TaskName::from("libA#build"), TaskName::from("build")]) @@ -1070,8 +998,8 @@ mod test { )] .into_iter() .collect(); - let engine = EngineBuilder::new(&repo_root, &package_graph, false) - .with_turbo_jsons(Some(turbo_jsons)) + let loader = TurboJsonLoader::noop(turbo_jsons); + let engine = EngineBuilder::new(&repo_root, &package_graph, loader, false) .with_tasks(Some(Spanned::new(TaskName::from("build")))) .with_workspaces(vec![PackageName::from("app1")]) .with_root_tasks(vec![ @@ -1115,8 +1043,8 @@ mod test { )] .into_iter() .collect(); - let engine = EngineBuilder::new(&repo_root, &package_graph, false) - .with_turbo_jsons(Some(turbo_jsons)) + let loader = TurboJsonLoader::noop(turbo_jsons); + let engine = EngineBuilder::new(&repo_root, &package_graph, loader, false) .with_tasks(Some(Spanned::new(TaskName::from("build")))) .with_workspaces(vec![PackageName::from("app1")]) .with_root_tasks(vec![ @@ -1154,8 +1082,8 @@ mod test { )] .into_iter() .collect(); - let engine = EngineBuilder::new(&repo_root, &package_graph, false) - .with_turbo_jsons(Some(turbo_jsons)) + let loader = TurboJsonLoader::noop(turbo_jsons); + let engine = EngineBuilder::new(&repo_root, &package_graph, loader, false) .with_tasks_only(true) .with_tasks(Some(Spanned::new(TaskName::from("test")))) .with_workspaces(vec![ @@ -1201,8 +1129,8 @@ mod test { )] .into_iter() .collect(); - let engine = EngineBuilder::new(&repo_root, &package_graph, false) - .with_turbo_jsons(Some(turbo_jsons)) + let loader = TurboJsonLoader::noop(turbo_jsons); + let engine = EngineBuilder::new(&repo_root, &package_graph, loader, false) .with_tasks_only(true) .with_tasks(Some(Spanned::new(TaskName::from("build")))) .with_workspaces(vec![PackageName::from("b")]) @@ -1240,8 +1168,8 @@ mod test { )] .into_iter() .collect(); - let engine = EngineBuilder::new(&repo_root, &package_graph, false) - .with_turbo_jsons(Some(turbo_jsons)) + let loader = TurboJsonLoader::noop(turbo_jsons); + let engine = EngineBuilder::new(&repo_root, &package_graph, loader, false) .with_tasks_only(true) .with_tasks(Some(Spanned::new(TaskName::from("build")))) .with_workspaces(vec![PackageName::from("b")]) diff --git a/crates/turborepo-lib/src/package_changes_watcher.rs b/crates/turborepo-lib/src/package_changes_watcher.rs index deb74acc5d199..d60ace1ffda93 100644 --- a/crates/turborepo-lib/src/package_changes_watcher.rs +++ b/crates/turborepo-lib/src/package_changes_watcher.rs @@ -21,7 +21,7 @@ use turborepo_repository::{ }; use turborepo_scm::package_deps::GitHashes; -use crate::turbo_json::{TurboJson, CONFIG_FILE}; +use crate::turbo_json::{TurboJson, TurboJsonLoader, CONFIG_FILE}; #[derive(Clone)] pub enum PackageChangeEvent { @@ -161,18 +161,6 @@ impl Subscriber { tracing::debug!("no package.json found, package watcher not available"); return None; }; - - let root_turbo_json = TurboJson::load( - &self.repo_root, - &self.repo_root.join_component(CONFIG_FILE), - &root_package_json, - false, - ) - .ok(); - - let gitignore_path = self.repo_root.join_component(".gitignore"); - let (root_gitignore, _) = Gitignore::new(&gitignore_path); - let Ok(pkg_dep_graph) = PackageGraphBuilder::new(&self.repo_root, root_package_json) .build() .await @@ -181,6 +169,18 @@ impl Subscriber { return None; }; + let root_turbo_json = TurboJsonLoader::workspace( + self.repo_root.clone(), + self.repo_root.join_component(CONFIG_FILE), + pkg_dep_graph.packages(), + ) + .load(&PackageName::Root) + .ok() + .cloned(); + + let gitignore_path = self.repo_root.join_component(".gitignore"); + let (root_gitignore, _) = Gitignore::new(&gitignore_path); + Some(( RepoState { root_turbo_json, diff --git a/crates/turborepo-lib/src/run/builder.rs b/crates/turborepo-lib/src/run/builder.rs index 04641f93df66c..aee20d56975b5 100644 --- a/crates/turborepo-lib/src/run/builder.rs +++ b/crates/turborepo-lib/src/run/builder.rs @@ -45,7 +45,7 @@ use crate::{ run::{scope, task_access::TaskAccess, task_id::TaskName, Error, Run, RunCache}, shim::TurboState, signal::{SignalHandler, SignalSubscriber}, - turbo_json::{TurboJson, UIMode}, + turbo_json::{TurboJson, TurboJsonLoader, UIMode}, DaemonConnector, }; @@ -65,6 +65,7 @@ pub struct RunBuilder { entrypoint_packages: Option>, should_print_prelude_override: Option, allow_missing_package_manager: bool, + allow_no_turbo_json: bool, } impl RunBuilder { @@ -85,6 +86,7 @@ impl RunBuilder { (!cfg!(windows) || matches!(opts.run_opts.ui_mode, UIMode::Tui)), ); let root_turbo_json_path = config.root_turbo_json_path(&base.repo_root); + let allow_no_turbo_json = config.allow_no_turbo_json(); let CommandBase { repo_root, @@ -105,6 +107,7 @@ impl RunBuilder { should_print_prelude_override: None, allow_missing_package_manager, root_turbo_json_path, + allow_no_turbo_json, }) } @@ -359,19 +362,32 @@ impl RunBuilder { let task_access = TaskAccess::new(self.repo_root.clone(), async_cache.clone(), &scm); task_access.restore_config().await; - let root_turbo_json = task_access - .load_turbo_json(&self.root_turbo_json_path) - .map_or_else( - || { - TurboJson::load( - &self.repo_root, - &self.root_turbo_json_path, - &root_package_json, - is_single_package, - ) - }, - Result::Ok, - )?; + let mut turbo_json_loader = if task_access.is_enabled() { + TurboJsonLoader::task_access( + self.repo_root.clone(), + self.root_turbo_json_path.clone(), + root_package_json.clone(), + ) + } else if is_single_package { + TurboJsonLoader::single_package( + self.repo_root.clone(), + self.root_turbo_json_path.clone(), + root_package_json.clone(), + ) + } else if self.allow_no_turbo_json && !self.root_turbo_json_path.exists() { + TurboJsonLoader::workspace_no_turbo_json( + self.repo_root.clone(), + pkg_dep_graph.packages(), + ) + } else { + TurboJsonLoader::workspace( + self.repo_root.clone(), + self.root_turbo_json_path.clone(), + pkg_dep_graph.packages(), + ) + }; + + let root_turbo_json = turbo_json_loader.load(&PackageName::Root)?.clone(); pkg_dep_graph.validate()?; @@ -384,11 +400,21 @@ impl RunBuilder { )?; let env_at_execution_start = EnvironmentVariableMap::infer(); - let mut engine = self.build_engine(&pkg_dep_graph, &root_turbo_json, &filtered_pkgs)?; + let mut engine = self.build_engine( + &pkg_dep_graph, + &root_turbo_json, + &filtered_pkgs, + turbo_json_loader.clone(), + )?; if self.opts.run_opts.parallel { pkg_dep_graph.remove_package_dependencies(); - engine = self.build_engine(&pkg_dep_graph, &root_turbo_json, &filtered_pkgs)?; + engine = self.build_engine( + &pkg_dep_graph, + &root_turbo_json, + &filtered_pkgs, + turbo_json_loader, + )?; } let color_selector = ColorSelector::default(); @@ -436,18 +462,15 @@ impl RunBuilder { pkg_dep_graph: &PackageGraph, root_turbo_json: &TurboJson, filtered_pkgs: &HashSet, + turbo_json_loader: TurboJsonLoader, ) -> Result { let mut engine = EngineBuilder::new( &self.repo_root, pkg_dep_graph, + turbo_json_loader, self.opts.run_opts.single_package, ) .with_root_tasks(root_turbo_json.tasks.keys().cloned()) - .with_turbo_jsons(Some( - Some((PackageName::Root, root_turbo_json.clone())) - .into_iter() - .collect(), - )) .with_tasks_only(self.opts.run_opts.only) .with_workspaces(filtered_pkgs.clone().into_iter().collect()) .with_tasks(self.opts.run_opts.tasks.iter().map(|task| { diff --git a/crates/turborepo-lib/src/run/summary/task.rs b/crates/turborepo-lib/src/run/summary/task.rs index 77aaa18c2689a..2f87ba15aef62 100644 --- a/crates/turborepo-lib/src/run/summary/task.rs +++ b/crates/turborepo-lib/src/run/summary/task.rs @@ -104,6 +104,8 @@ pub struct TaskSummaryTaskDefinition { env: Vec, pass_through_env: Option>, interactive: bool, + #[serde(skip_serializing_if = "Option::is_none")] + env_mode: Option, } #[derive(Debug, Serialize, Clone)] @@ -279,6 +281,7 @@ impl From for TaskSummaryTaskDefinition { output_logs, persistent, interactive, + env_mode, } = value; let mut outputs = inclusions; @@ -313,6 +316,7 @@ impl From for TaskSummaryTaskDefinition { interactive, env, pass_through_env, + env_mode, } } } diff --git a/crates/turborepo-lib/src/run/task_access.rs b/crates/turborepo-lib/src/run/task_access.rs index 65fa374b9c26b..c6fc7798e3275 100644 --- a/crates/turborepo-lib/src/run/task_access.rs +++ b/crates/turborepo-lib/src/run/task_access.rs @@ -7,13 +7,13 @@ use std::{ use serde::Deserialize; use tracing::{debug, error, warn}; -use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, PathRelation}; +use turbopath::{AbsoluteSystemPathBuf, PathRelation}; use turborepo_cache::AsyncCache; use turborepo_scm::SCM; use turborepo_unescape::UnescapedString; use super::ConfigCache; -use crate::{config::RawTurboJson, gitignore::ensure_turbo_is_gitignored, turbo_json::TurboJson}; +use crate::{config::RawTurboJson, gitignore::ensure_turbo_is_gitignored}; // Environment variable key that will be used to enable, and set the expected // trace location @@ -245,25 +245,6 @@ impl TaskAccess { } } - /// Attempt to load a task traced turbo.json - pub fn load_turbo_json(&self, root_turbo_json_path: &AbsoluteSystemPath) -> Option { - if !self.enabled { - return None; - } - let trace_json_path = self.repo_root.join_components(&TASK_ACCESS_CONFIG_PATH); - let turbo_from_trace = TurboJson::read(&self.repo_root, &trace_json_path); - - // check the zero config case (turbo trace file, but no turbo.json file) - if let Ok(turbo_from_trace) = turbo_from_trace { - if !root_turbo_json_path.exists() { - debug!("Using turbo.json synthesized from trace file"); - return Some(turbo_from_trace); - } - } - - None - } - async fn to_file(&self) -> Result<(), ToFileError> { // if task access tracing is not enabled, we don't need to do anything if !self.is_enabled() { diff --git a/crates/turborepo-lib/src/task_graph/mod.rs b/crates/turborepo-lib/src/task_graph/mod.rs index e1042db9d87f9..43cd452b296c6 100644 --- a/crates/turborepo-lib/src/task_graph/mod.rs +++ b/crates/turborepo-lib/src/task_graph/mod.rs @@ -9,7 +9,7 @@ use turborepo_errors::Spanned; pub use visitor::{Error as VisitorError, Visitor}; use crate::{ - cli::OutputLogsMode, + cli::{EnvMode, OutputLogsMode}, run::task_id::{TaskId, TaskName}, turbo_json::RawTaskDefinition, }; @@ -76,6 +76,9 @@ pub struct TaskDefinition { // Tasks that take stdin input cannot be cached as their outputs may depend on the // input. pub interactive: bool, + + // Override for global env mode setting + pub env_mode: Option, } impl Default for TaskDefinition { @@ -91,6 +94,7 @@ impl Default for TaskDefinition { output_logs: Default::default(), persistent: Default::default(), interactive: Default::default(), + env_mode: Default::default(), } } } diff --git a/crates/turborepo-lib/src/task_graph/visitor.rs b/crates/turborepo-lib/src/task_graph/visitor.rs index 4c37eaa77d943..f2313e4ad9584 100644 --- a/crates/turborepo-lib/src/task_graph/visitor.rs +++ b/crates/turborepo-lib/src/task_graph/visitor.rs @@ -219,7 +219,7 @@ impl<'a> Visitor<'a> { .task_definition(&info) .ok_or(Error::MissingDefinition)?; - let task_env_mode = self.global_env_mode; + let task_env_mode = task_definition.env_mode.unwrap_or(self.global_env_mode); package_task_event.track_env_mode(&task_env_mode.to_string()); let dependency_set = engine.dependencies(&info).ok_or(Error::MissingDefinition)?; diff --git a/crates/turborepo-lib/src/turbo_json/loader.rs b/crates/turborepo-lib/src/turbo_json/loader.rs new file mode 100644 index 0000000000000..b01fb054babfc --- /dev/null +++ b/crates/turborepo-lib/src/turbo_json/loader.rs @@ -0,0 +1,679 @@ +use std::collections::HashMap; + +use tracing::debug; +use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf}; +use turborepo_errors::Spanned; +use turborepo_repository::{ + package_graph::{PackageInfo, PackageName}, + package_json::PackageJson, +}; + +use super::{Pipeline, RawTaskDefinition, TurboJson, CONFIG_FILE}; +use crate::{ + cli::EnvMode, + config::Error, + run::{task_access::TASK_ACCESS_CONFIG_PATH, task_id::TaskName}, +}; + +/// Structure for loading TurboJson structures. +/// Depending on the strategy used, TurboJson might not correspond to +/// `turbo.json` file. +#[derive(Debug, Clone)] +pub struct TurboJsonLoader { + repo_root: AbsoluteSystemPathBuf, + cache: HashMap, + strategy: Strategy, +} + +#[derive(Debug, Clone)] +enum Strategy { + SinglePackage { + root_turbo_json: AbsoluteSystemPathBuf, + package_json: PackageJson, + }, + Workspace { + // Map of package names to their package specific turbo.json + packages: HashMap, + }, + WorkspaceNoTurboJson { + // Map of package names to their scripts + packages: HashMap>, + }, + TaskAccess { + root_turbo_json: AbsoluteSystemPathBuf, + package_json: PackageJson, + }, + Noop, +} + +impl TurboJsonLoader { + /// Create a loader that will load turbo.json files throughout the workspace + pub fn workspace<'a>( + repo_root: AbsoluteSystemPathBuf, + root_turbo_json_path: AbsoluteSystemPathBuf, + packages: impl Iterator, + ) -> Self { + let packages = package_turbo_jsons(&repo_root, root_turbo_json_path, packages); + Self { + repo_root, + cache: HashMap::new(), + strategy: Strategy::Workspace { packages }, + } + } + + /// Create a loader that will construct turbo.json structures based on + /// workspace `package.json`s. + pub fn workspace_no_turbo_json<'a>( + repo_root: AbsoluteSystemPathBuf, + packages: impl Iterator, + ) -> Self { + let packages = workspace_package_scripts(packages); + Self { + repo_root, + cache: HashMap::new(), + strategy: Strategy::WorkspaceNoTurboJson { packages }, + } + } + + /// Create a loader that will load a root turbo.json or synthesize one if + /// the file doesn't exist + pub fn single_package( + repo_root: AbsoluteSystemPathBuf, + root_turbo_json: AbsoluteSystemPathBuf, + package_json: PackageJson, + ) -> Self { + Self { + repo_root, + cache: HashMap::new(), + strategy: Strategy::SinglePackage { + root_turbo_json, + package_json, + }, + } + } + + /// Create a loader that will load a root turbo.json or synthesize one if + /// the file doesn't exist + pub fn task_access( + repo_root: AbsoluteSystemPathBuf, + root_turbo_json: AbsoluteSystemPathBuf, + package_json: PackageJson, + ) -> Self { + Self { + repo_root, + cache: HashMap::new(), + strategy: Strategy::TaskAccess { + root_turbo_json, + package_json, + }, + } + } + + /// Create a loader that will only return provided turbo.jsons and will + /// never hit the file system. + /// Primarily intended for testing + pub fn noop(turbo_jsons: HashMap) -> Self { + Self { + // This never gets read from so we populate it with + repo_root: AbsoluteSystemPath::new(if cfg!(windows) { "C:\\" } else { "/" }) + .expect("wasn't able to create absolute system path") + .to_owned(), + cache: turbo_jsons, + strategy: Strategy::Noop, + } + } + + /// Load a turbo.json for a given package + pub fn load<'a>(&'a mut self, package: &PackageName) -> Result<&'a TurboJson, Error> { + if !self.cache.contains_key(package) { + let turbo_json = self.uncached_load(package)?; + self.cache.insert(package.clone(), turbo_json); + } + Ok(self + .cache + .get(package) + .expect("just inserted value for this key")) + } + + fn uncached_load(&self, package: &PackageName) -> Result { + match &self.strategy { + Strategy::SinglePackage { + package_json, + root_turbo_json, + } => { + if !matches!(package, PackageName::Root) { + Err(Error::InvalidTurboJsonLoad(package.clone())) + } else { + load_from_root_package_json(&self.repo_root, root_turbo_json, package_json) + } + } + Strategy::Workspace { packages } => { + let path = packages.get(package).ok_or_else(|| Error::NoTurboJSON)?; + load_from_file(&self.repo_root, path) + } + Strategy::WorkspaceNoTurboJson { packages } => { + let script_names = packages.get(package).ok_or(Error::NoTurboJSON)?; + if matches!(package, PackageName::Root) { + root_turbo_json_from_scripts(script_names) + } else { + workspace_turbo_json_from_scripts(script_names) + } + } + Strategy::TaskAccess { + package_json, + root_turbo_json, + } => { + if !matches!(package, PackageName::Root) { + Err(Error::InvalidTurboJsonLoad(package.clone())) + } else { + load_task_access_trace_turbo_json( + &self.repo_root, + root_turbo_json, + package_json, + ) + } + } + Strategy::Noop => Err(Error::NoTurboJSON), + } + } +} + +/// Map all packages in the package graph to their turbo.json path +fn package_turbo_jsons<'a>( + repo_root: &AbsoluteSystemPath, + root_turbo_json_path: AbsoluteSystemPathBuf, + packages: impl Iterator, +) -> HashMap { + let mut package_turbo_jsons = HashMap::new(); + package_turbo_jsons.insert(PackageName::Root, root_turbo_json_path); + package_turbo_jsons.extend(packages.filter_map(|(pkg, info)| { + if pkg == &PackageName::Root { + None + } else { + Some(( + pkg.clone(), + repo_root + .resolve(info.package_path()) + .join_component(CONFIG_FILE), + )) + } + })); + package_turbo_jsons +} + +/// Map all packages in the package graph to their scripts +fn workspace_package_scripts<'a>( + packages: impl Iterator, +) -> HashMap> { + packages + .map(|(pkg, info)| { + ( + pkg.clone(), + info.package_json.scripts.keys().cloned().collect(), + ) + }) + .collect() +} + +fn load_from_file( + repo_root: &AbsoluteSystemPath, + turbo_json_path: &AbsoluteSystemPath, +) -> Result { + match TurboJson::read(repo_root, turbo_json_path) { + // If the file didn't exist, throw a custom error here instead of propagating + Err(Error::Io(_)) => Err(Error::NoTurboJSON), + // There was an error, and we don't have any chance of recovering + // because we aren't synthesizing anything + Err(e) => Err(e), + // We're not synthesizing anything and there was no error, we're done + Ok(turbo) => Ok(turbo), + } +} + +fn load_from_root_package_json( + repo_root: &AbsoluteSystemPath, + turbo_json_path: &AbsoluteSystemPath, + root_package_json: &PackageJson, +) -> Result { + let mut turbo_json = match TurboJson::read(repo_root, turbo_json_path) { + // we're synthesizing, but we have a starting point + // Note: this will have to change to support task inference in a monorepo + // for now, we're going to error on any "root" tasks and turn non-root tasks into root + // tasks + Ok(mut turbo_json) => { + let mut pipeline = Pipeline::default(); + for (task_name, task_definition) in turbo_json.tasks { + if task_name.is_package_task() { + let (span, text) = task_definition.span_and_text("turbo.json"); + + return Err(Error::PackageTaskInSinglePackageMode { + task_id: task_name.to_string(), + span, + text, + }); + } + + pipeline.insert(task_name.into_root_task(), task_definition); + } + + turbo_json.tasks = pipeline; + + turbo_json + } + // turbo.json doesn't exist, but we're going try to synthesize something + Err(Error::Io(_)) => TurboJson::default(), + // some other happened, we can't recover + Err(e) => { + return Err(e); + } + }; + + // TODO: Add location info from package.json + for script_name in root_package_json.scripts.keys() { + let task_name = TaskName::from(script_name.as_str()); + if !turbo_json.has_task(&task_name) { + let task_name = task_name.into_root_task(); + // Explicitly set cache to Some(false) in this definition + // so we can pretend it was set on purpose. That way it + // won't get clobbered by the merge function. + turbo_json.tasks.insert( + task_name, + Spanned::new(RawTaskDefinition { + cache: Some(Spanned::new(false)), + ..RawTaskDefinition::default() + }), + ); + } + } + + Ok(turbo_json) +} + +fn root_turbo_json_from_scripts(scripts: &[String]) -> Result { + let mut turbo_json = TurboJson { + ..Default::default() + }; + for script in scripts { + let task_name = TaskName::from(script.as_str()).into_root_task(); + turbo_json.tasks.insert( + task_name, + Spanned::new(RawTaskDefinition { + cache: Some(Spanned::new(false)), + env_mode: Some(EnvMode::Loose), + ..Default::default() + }), + ); + } + Ok(turbo_json) +} + +fn workspace_turbo_json_from_scripts(scripts: &[String]) -> Result { + let mut turbo_json = TurboJson { + extends: Spanned::new(vec!["//".to_owned()]), + ..Default::default() + }; + for script in scripts { + let task_name = TaskName::from(script.clone()); + turbo_json.tasks.insert( + task_name, + Spanned::new(RawTaskDefinition { + cache: Some(Spanned::new(false)), + env_mode: Some(EnvMode::Loose), + ..Default::default() + }), + ); + } + Ok(turbo_json) +} + +fn load_task_access_trace_turbo_json( + repo_root: &AbsoluteSystemPath, + turbo_json_path: &AbsoluteSystemPath, + root_package_json: &PackageJson, +) -> Result { + let trace_json_path = repo_root.join_components(&TASK_ACCESS_CONFIG_PATH); + let turbo_from_trace = TurboJson::read(repo_root, &trace_json_path); + + // check the zero config case (turbo trace file, but no turbo.json file) + if let Ok(turbo_from_trace) = turbo_from_trace { + if !turbo_json_path.exists() { + debug!("Using turbo.json synthesized from trace file"); + return Ok(turbo_from_trace); + } + } + load_from_root_package_json(repo_root, turbo_json_path, root_package_json) +} + +#[cfg(test)] +mod test { + use std::{collections::BTreeMap, fs}; + + use anyhow::Result; + use tempfile::tempdir; + use test_case::test_case; + + use super::*; + use crate::{task_graph::TaskDefinition, turbo_json::CONFIG_FILE}; + + #[test_case(r"{}", TurboJson::default() ; "empty")] + #[test_case(r#"{ "globalDependencies": ["tsconfig.json", "jest.config.js"] }"#, + TurboJson { + global_deps: vec!["jest.config.js".to_string(), "tsconfig.json".to_string()], + ..TurboJson::default() + } + ; "global dependencies (sorted)")] + #[test_case(r#"{ "globalPassThroughEnv": ["GITHUB_TOKEN", "AWS_SECRET_KEY"] }"#, + TurboJson { + global_pass_through_env: Some(vec!["AWS_SECRET_KEY".to_string(), "GITHUB_TOKEN".to_string()]), + ..TurboJson::default() + } + )] + #[test_case(r#"{ "//": "A comment"}"#, TurboJson::default() ; "faux comment")] + #[test_case(r#"{ "//": "A comment", "//": "Another comment" }"#, TurboJson::default() ; "two faux comments")] + fn test_get_root_turbo_no_synthesizing( + turbo_json_content: &str, + expected_turbo_json: TurboJson, + ) -> Result<()> { + let root_dir = tempdir()?; + let repo_root = AbsoluteSystemPath::from_std_path(root_dir.path())?; + let root_turbo_json = repo_root.join_component("turbo.json"); + fs::write(&root_turbo_json, turbo_json_content)?; + let mut loader = TurboJsonLoader { + repo_root: repo_root.to_owned(), + cache: HashMap::new(), + strategy: Strategy::Workspace { + packages: vec![(PackageName::Root, root_turbo_json)] + .into_iter() + .collect(), + }, + }; + + let mut turbo_json = loader.load(&PackageName::Root)?.clone(); + + turbo_json.text = None; + turbo_json.path = None; + assert_eq!(turbo_json, expected_turbo_json); + + Ok(()) + } + + #[test_case( + None, + PackageJson { + scripts: [("build".to_string(), Spanned::new("echo build".to_string()))].into_iter().collect(), + ..PackageJson::default() + }, + TurboJson { + tasks: Pipeline([( + "//#build".into(), + Spanned::new(RawTaskDefinition { + cache: Some(Spanned::new(false)), + ..RawTaskDefinition::default() + }) + )].into_iter().collect() + ), + ..TurboJson::default() + } + )] + #[test_case( + Some(r#"{ + "tasks": { + "build": { + "cache": true + } + } + }"#), + PackageJson { + scripts: [("test".to_string(), Spanned::new("echo test".to_string()))].into_iter().collect(), + ..PackageJson::default() + }, + TurboJson { + tasks: Pipeline([( + "//#build".into(), + Spanned::new(RawTaskDefinition { + cache: Some(Spanned::new(true).with_range(81..85)), + ..RawTaskDefinition::default() + }).with_range(50..103) + ), + ( + "//#test".into(), + Spanned::new(RawTaskDefinition { + cache: Some(Spanned::new(false)), + ..RawTaskDefinition::default() + }) + )].into_iter().collect()), + ..TurboJson::default() + } + )] + fn test_get_root_turbo_with_synthesizing( + turbo_json_content: Option<&str>, + root_package_json: PackageJson, + expected_turbo_json: TurboJson, + ) -> Result<()> { + let root_dir = tempdir()?; + let repo_root = AbsoluteSystemPath::from_std_path(root_dir.path())?; + let root_turbo_json = repo_root.join_component(CONFIG_FILE); + + if let Some(content) = turbo_json_content { + fs::write(&root_turbo_json, content)?; + } + + let mut loader = TurboJsonLoader::single_package( + repo_root.to_owned(), + root_turbo_json, + root_package_json, + ); + let mut turbo_json = loader.load(&PackageName::Root)?.clone(); + turbo_json.text = None; + turbo_json.path = None; + for (_, task_definition) in turbo_json.tasks.iter_mut() { + task_definition.path = None; + task_definition.text = None; + } + assert_eq!(turbo_json, expected_turbo_json); + + Ok(()) + } + + #[test_case( + Some(r#"{ "tasks": {"//#build": {"env": ["SPECIAL_VAR"]}} }"#), + Some(r#"{ "tasks": {"build": {"env": ["EXPLICIT_VAR"]}} }"#), + TaskDefinition { env: vec!["EXPLICIT_VAR".to_string()], .. Default::default() } + ; "both present")] + #[test_case( + None, + Some(r#"{ "tasks": {"build": {"env": ["EXPLICIT_VAR"]}} }"#), + TaskDefinition { env: vec!["EXPLICIT_VAR".to_string()], .. Default::default() } + ; "no trace")] + #[test_case( + Some(r#"{ "tasks": {"//#build": {"env": ["SPECIAL_VAR"]}} }"#), + None, + TaskDefinition { env: vec!["SPECIAL_VAR".to_string()], .. Default::default() } + ; "no turbo.json")] + #[test_case( + None, + None, + TaskDefinition { cache: false, .. Default::default() } + ; "both missing")] + fn test_task_access_loading( + trace_contents: Option<&str>, + turbo_json_content: Option<&str>, + expected_root_build: TaskDefinition, + ) -> Result<()> { + let root_dir = tempdir()?; + let repo_root = AbsoluteSystemPath::from_std_path(root_dir.path())?; + let root_turbo_json = repo_root.join_component(CONFIG_FILE); + + if let Some(content) = turbo_json_content { + root_turbo_json.create_with_contents(content.as_bytes())?; + } + if let Some(content) = trace_contents { + let trace_path = repo_root.join_components(&TASK_ACCESS_CONFIG_PATH); + trace_path.ensure_dir()?; + trace_path.create_with_contents(content.as_bytes())?; + } + + let mut scripts = BTreeMap::new(); + scripts.insert("build".into(), Spanned::new("echo building".into())); + let root_package_json = PackageJson { + scripts, + ..Default::default() + }; + + let mut loader = + TurboJsonLoader::task_access(repo_root.to_owned(), root_turbo_json, root_package_json); + let turbo_json = loader.load(&PackageName::Root)?; + let root_build = turbo_json + .tasks + .get(&TaskName::from("//#build")) + .expect("root build should always exist") + .as_inner(); + + assert_eq!( + expected_root_build, + TaskDefinition::try_from(root_build.clone())? + ); + + Ok(()) + } + + #[test] + fn test_single_package_loading_non_root() { + let junk_path = AbsoluteSystemPath::new(if cfg!(windows) { + "C:\\never\\loaded" + } else { + "/never/loaded" + }) + .unwrap(); + let non_root = PackageName::from("some-pkg"); + let single_loader = TurboJsonLoader::single_package( + junk_path.to_owned(), + junk_path.to_owned(), + PackageJson::default(), + ); + let task_access_loader = TurboJsonLoader::task_access( + junk_path.to_owned(), + junk_path.to_owned(), + PackageJson::default(), + ); + + for mut loader in [single_loader, task_access_loader] { + let result = loader.load(&non_root); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + matches!(err, Error::InvalidTurboJsonLoad(_)), + "expected {err} to be no turbo json" + ); + } + } + + #[test] + fn test_workspace_turbo_json_loading() { + let root_dir = tempdir().unwrap(); + let repo_root = AbsoluteSystemPath::from_std_path(root_dir.path()).unwrap(); + let a_turbo_json = repo_root.join_components(&["packages", "a", "turbo.json"]); + a_turbo_json.ensure_dir().unwrap(); + let packages = vec![(PackageName::from("a"), a_turbo_json.clone())] + .into_iter() + .collect(); + + let mut loader = TurboJsonLoader { + repo_root: repo_root.to_owned(), + cache: HashMap::new(), + strategy: Strategy::Workspace { packages }, + }; + let result = loader.load(&PackageName::from("a")); + assert!( + matches!(result.unwrap_err(), Error::NoTurboJSON), + "expected parsing to fail with missing turbo.json" + ); + + a_turbo_json + .create_with_contents(r#"{"tasks": {"build": {}}}"#) + .unwrap(); + + let turbo_json = loader.load(&PackageName::from("a")).unwrap(); + assert_eq!(turbo_json.tasks.len(), 1); + } + + #[test] + fn test_turbo_json_caching() { + let root_dir = tempdir().unwrap(); + let repo_root = AbsoluteSystemPath::from_std_path(root_dir.path()).unwrap(); + let a_turbo_json = repo_root.join_components(&["packages", "a", "turbo.json"]); + a_turbo_json.ensure_dir().unwrap(); + let packages = vec![(PackageName::from("a"), a_turbo_json.clone())] + .into_iter() + .collect(); + + let mut loader = TurboJsonLoader { + repo_root: repo_root.to_owned(), + cache: HashMap::new(), + strategy: Strategy::Workspace { packages }, + }; + a_turbo_json + .create_with_contents(r#"{"tasks": {"build": {}}}"#) + .unwrap(); + + let turbo_json = loader.load(&PackageName::from("a")).unwrap(); + assert_eq!(turbo_json.tasks.len(), 1); + a_turbo_json.remove().unwrap(); + assert!(loader.load(&PackageName::from("a")).is_ok()); + } + + #[test] + fn test_no_turbo_json() { + let root_dir = tempdir().unwrap(); + let repo_root = AbsoluteSystemPath::from_std_path(root_dir.path()).unwrap(); + let packages = vec![ + ( + PackageName::Root, + vec!["build".to_owned(), "lint".to_owned(), "test".to_owned()], + ), + ( + PackageName::from("pkg-a"), + vec!["build".to_owned(), "lint".to_owned(), "special".to_owned()], + ), + ] + .into_iter() + .collect(); + + let mut loader = TurboJsonLoader { + repo_root: repo_root.to_owned(), + cache: HashMap::new(), + strategy: Strategy::WorkspaceNoTurboJson { packages }, + }; + + { + let root_json = loader.load(&PackageName::Root).unwrap(); + for task_name in ["//#build", "//#lint", "//#test"] { + if let Some(def) = root_json.tasks.get(&TaskName::from(task_name)) { + assert_eq!( + def.cache.as_ref().map(|cache| *cache.as_inner()), + Some(false) + ); + } else { + panic!("didn't find {task_name}"); + } + } + } + + { + let pkg_a_json = loader.load(&PackageName::from("pkg-a")).unwrap(); + for task_name in ["build", "lint", "special"] { + if let Some(def) = pkg_a_json.tasks.get(&TaskName::from(task_name)) { + assert_eq!( + def.cache.as_ref().map(|cache| *cache.as_inner()), + Some(false) + ); + } else { + panic!("didn't find {task_name}"); + } + } + } + // Should get no turbo.json error if package wasn't declared + let goose_err = loader.load(&PackageName::from("goose")).unwrap_err(); + assert!(matches!(goose_err, Error::NoTurboJSON)); + } +} diff --git a/crates/turborepo-lib/src/turbo_json/mod.rs b/crates/turborepo-lib/src/turbo_json/mod.rs index e84b0cf6073c1..471990a7ec066 100644 --- a/crates/turborepo-lib/src/turbo_json/mod.rs +++ b/crates/turborepo-lib/src/turbo_json/mod.rs @@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize}; use struct_iterable::Iterable; use turbopath::AbsoluteSystemPath; use turborepo_errors::Spanned; -use turborepo_repository::{package_graph::ROOT_PKG_NAME, package_json::PackageJson}; +use turborepo_repository::package_graph::ROOT_PKG_NAME; use turborepo_unescape::UnescapedString; use crate::{ @@ -25,8 +25,11 @@ use crate::{ task_graph::{TaskDefinition, TaskOutputs}, }; +mod loader; pub mod parser; +pub use loader::TurboJsonLoader; + #[derive(Serialize, Deserialize, Debug, Default, PartialEq, Clone, Deserializable)] #[serde(rename_all = "camelCase")] pub struct SpacesJson { @@ -221,6 +224,10 @@ pub struct RawTaskDefinition { output_logs: Option>, #[serde(skip_serializing_if = "Option::is_none")] interactive: Option>, + // TODO: Remove this once we have the ability to load task definitions directly + // instead of deriving them from a TurboJson + #[serde(skip)] + env_mode: Option, } macro_rules! set_field { @@ -253,6 +260,7 @@ impl RawTaskDefinition { set_field!(self, other, env); set_field!(self, other, pass_through_env); set_field!(self, other, interactive); + set_field!(self, other, env_mode); } } @@ -401,6 +409,7 @@ impl TryFrom for TaskDefinition { output_logs: *raw_task.output_logs.unwrap_or_default(), persistent: *raw_task.persistent.unwrap_or_default(), interactive, + env_mode: raw_task.env_mode, }) } } @@ -548,75 +557,6 @@ impl TryFrom for TurboJson { } impl TurboJson { - /// Loads turbo.json by reading the file at `dir` and optionally combining - /// with synthesized information from the provided package.json - pub fn load( - repo_root: &AbsoluteSystemPath, - turbo_json_path: &AbsoluteSystemPath, - root_package_json: &PackageJson, - include_synthesized_from_root_package_json: bool, - ) -> Result { - let turbo_from_files = Self::read(repo_root, turbo_json_path); - - let mut turbo_json = match (include_synthesized_from_root_package_json, turbo_from_files) { - // If the file didn't exist, throw a custom error here instead of propagating - (false, Err(Error::Io(_))) => return Err(Error::NoTurboJSON), - // There was an error, and we don't have any chance of recovering - // because we aren't synthesizing anything - (false, Err(e)) => return Err(e), - // We're not synthesizing anything and there was no error, we're done - (false, Ok(turbo)) => return Ok(turbo), - // turbo.json doesn't exist, but we're going try to synthesize something - (true, Err(Error::Io(_))) => TurboJson::default(), - // some other happened, we can't recover - (true, Err(e)) => return Err(e), - // we're synthesizing, but we have a starting point - // Note: this will have to change to support task inference in a monorepo - // for now, we're going to error on any "root" tasks and turn non-root tasks into root - // tasks - (true, Ok(mut turbo_from_files)) => { - let mut pipeline = Pipeline::default(); - for (task_name, task_definition) in turbo_from_files.tasks { - if task_name.is_package_task() { - let (span, text) = task_definition.span_and_text("turbo.json"); - - return Err(Error::PackageTaskInSinglePackageMode { - task_id: task_name.to_string(), - span, - text, - }); - } - - pipeline.insert(task_name.into_root_task(), task_definition); - } - - turbo_from_files.tasks = pipeline; - - turbo_from_files - } - }; - - // TODO: Add location info from package.json - for script_name in root_package_json.scripts.keys() { - let task_name = TaskName::from(script_name.as_str()); - if !turbo_json.has_task(&task_name) { - let task_name = task_name.into_root_task(); - // Explicitly set cache to Some(false) in this definition - // so we can pretend it was set on purpose. That way it - // won't get clobbered by the merge function. - turbo_json.tasks.insert( - task_name, - Spanned::new(RawTaskDefinition { - cache: Some(Spanned::new(false)), - ..RawTaskDefinition::default() - }), - ); - } - } - - Ok(turbo_json) - } - fn has_task(&self, task_name: &TaskName) -> bool { for key in self.tasks.keys() { if key == task_name || (key.task() == task_name.task() && !task_name.is_package_task()) @@ -740,142 +680,22 @@ fn gather_env_vars( #[cfg(test)] mod tests { - use std::fs; - use anyhow::Result; use biome_deserialize::json::deserialize_from_json_str; use biome_json_parser::JsonParserOptions; use pretty_assertions::assert_eq; use serde_json::json; - use tempfile::tempdir; use test_case::test_case; - use turbopath::AbsoluteSystemPath; - use turborepo_repository::package_json::PackageJson; use turborepo_unescape::UnescapedString; - use super::{Pipeline, RawTurboJson, Spanned, UIMode}; + use super::{RawTurboJson, Spanned, UIMode}; use crate::{ cli::OutputLogsMode, run::task_id::TaskName, task_graph::{TaskDefinition, TaskOutputs}, - turbo_json::{RawTaskDefinition, TurboJson, CONFIG_FILE}, + turbo_json::RawTaskDefinition, }; - #[test_case(r"{}", TurboJson::default() ; "empty")] - #[test_case(r#"{ "globalDependencies": ["tsconfig.json", "jest.config.js"] }"#, - TurboJson { - global_deps: vec!["jest.config.js".to_string(), "tsconfig.json".to_string()], - ..TurboJson::default() - } - ; "global dependencies (sorted)")] - #[test_case(r#"{ "globalPassThroughEnv": ["GITHUB_TOKEN", "AWS_SECRET_KEY"] }"#, - TurboJson { - global_pass_through_env: Some(vec!["AWS_SECRET_KEY".to_string(), "GITHUB_TOKEN".to_string()]), - ..TurboJson::default() - } - )] - #[test_case(r#"{ "//": "A comment"}"#, TurboJson::default() ; "faux comment")] - #[test_case(r#"{ "//": "A comment", "//": "Another comment" }"#, TurboJson::default() ; "two faux comments")] - fn test_get_root_turbo_no_synthesizing( - turbo_json_content: &str, - expected_turbo_json: TurboJson, - ) -> Result<()> { - let root_dir = tempdir()?; - let root_package_json = PackageJson::default(); - let repo_root = AbsoluteSystemPath::from_std_path(root_dir.path())?; - fs::write(repo_root.join_component("turbo.json"), turbo_json_content)?; - - let mut turbo_json = TurboJson::load( - repo_root, - &repo_root.join_component(CONFIG_FILE), - &root_package_json, - false, - )?; - - turbo_json.text = None; - turbo_json.path = None; - assert_eq!(turbo_json, expected_turbo_json); - - Ok(()) - } - - #[test_case( - None, - PackageJson { - scripts: [("build".to_string(), Spanned::new("echo build".to_string()))].into_iter().collect(), - ..PackageJson::default() - }, - TurboJson { - tasks: Pipeline([( - "//#build".into(), - Spanned::new(RawTaskDefinition { - cache: Some(Spanned::new(false)), - ..RawTaskDefinition::default() - }) - )].into_iter().collect() - ), - ..TurboJson::default() - } - )] - #[test_case( - Some(r#"{ - "tasks": { - "build": { - "cache": true - } - } - }"#), - PackageJson { - scripts: [("test".to_string(), Spanned::new("echo test".to_string()))].into_iter().collect(), - ..PackageJson::default() - }, - TurboJson { - tasks: Pipeline([( - "//#build".into(), - Spanned::new(RawTaskDefinition { - cache: Some(Spanned::new(true).with_range(81..85)), - ..RawTaskDefinition::default() - }).with_range(50..103) - ), - ( - "//#test".into(), - Spanned::new(RawTaskDefinition { - cache: Some(Spanned::new(false)), - ..RawTaskDefinition::default() - }) - )].into_iter().collect()), - ..TurboJson::default() - } - )] - fn test_get_root_turbo_with_synthesizing( - turbo_json_content: Option<&str>, - root_package_json: PackageJson, - expected_turbo_json: TurboJson, - ) -> Result<()> { - let root_dir = tempdir()?; - let repo_root = AbsoluteSystemPath::from_std_path(root_dir.path())?; - - if let Some(content) = turbo_json_content { - fs::write(repo_root.join_component("turbo.json"), content)?; - } - - let mut turbo_json = TurboJson::load( - repo_root, - &repo_root.join_component(CONFIG_FILE), - &root_package_json, - true, - )?; - turbo_json.text = None; - turbo_json.path = None; - for (_, task_definition) in turbo_json.tasks.iter_mut() { - task_definition.path = None; - task_definition.text = None; - } - assert_eq!(turbo_json, expected_turbo_json); - - Ok(()) - } - #[test_case( "{}", RawTaskDefinition::default(), @@ -912,6 +732,7 @@ mod tests { output_logs: Some(Spanned::new(OutputLogsMode::Full).with_range(246..252)), persistent: Some(Spanned::new(true).with_range(278..282)), interactive: Some(Spanned::new(true).with_range(309..313)), + env_mode: None, }, TaskDefinition { env: vec!["OS".to_string()], @@ -927,6 +748,7 @@ mod tests { topological_dependencies: vec![], persistent: true, interactive: true, + env_mode: None, } ; "full" )] @@ -951,6 +773,7 @@ mod tests { output_logs: Some(Spanned::new(OutputLogsMode::Full).with_range(279..285)), persistent: Some(Spanned::new(true).with_range(315..319)), interactive: None, + env_mode: None, }, TaskDefinition { env: vec!["OS".to_string()], @@ -966,6 +789,7 @@ mod tests { topological_dependencies: vec![], persistent: true, interactive: false, + env_mode: None, } ; "full (windows)" )] diff --git a/turborepo-tests/integration/fixtures/monorepo_no_turbo_json/.gitignore b/turborepo-tests/integration/fixtures/monorepo_no_turbo_json/.gitignore new file mode 100644 index 0000000000000..77af9fc60321d --- /dev/null +++ b/turborepo-tests/integration/fixtures/monorepo_no_turbo_json/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +.turbo +.npmrc diff --git a/turborepo-tests/integration/fixtures/monorepo_no_turbo_json/apps/my-app/.env.local b/turborepo-tests/integration/fixtures/monorepo_no_turbo_json/apps/my-app/.env.local new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/turborepo-tests/integration/fixtures/monorepo_no_turbo_json/apps/my-app/package.json b/turborepo-tests/integration/fixtures/monorepo_no_turbo_json/apps/my-app/package.json new file mode 100644 index 0000000000000..37523c1c8e838 --- /dev/null +++ b/turborepo-tests/integration/fixtures/monorepo_no_turbo_json/apps/my-app/package.json @@ -0,0 +1,10 @@ +{ + "name": "my-app", + "scripts": { + "build": "echo building", + "test": "echo $MY_VAR" + }, + "dependencies": { + "util": "*" + } +} diff --git a/turborepo-tests/integration/fixtures/monorepo_no_turbo_json/bar.txt b/turborepo-tests/integration/fixtures/monorepo_no_turbo_json/bar.txt new file mode 100644 index 0000000000000..5e849f85df5d1 --- /dev/null +++ b/turborepo-tests/integration/fixtures/monorepo_no_turbo_json/bar.txt @@ -0,0 +1 @@ +other file, not a global dependency diff --git a/turborepo-tests/integration/fixtures/monorepo_no_turbo_json/foo.txt b/turborepo-tests/integration/fixtures/monorepo_no_turbo_json/foo.txt new file mode 100644 index 0000000000000..eebae5f3ca7b5 --- /dev/null +++ b/turborepo-tests/integration/fixtures/monorepo_no_turbo_json/foo.txt @@ -0,0 +1 @@ +global dep! all tasks depend on this content! diff --git a/turborepo-tests/integration/fixtures/monorepo_no_turbo_json/package.json b/turborepo-tests/integration/fixtures/monorepo_no_turbo_json/package.json new file mode 100644 index 0000000000000..e86a3bf3c329b --- /dev/null +++ b/turborepo-tests/integration/fixtures/monorepo_no_turbo_json/package.json @@ -0,0 +1,11 @@ +{ + "name": "monorepo", + "scripts": { + "something": "turbo run build" + }, + "packageManager": "bower", + "workspaces": [ + "apps/**", + "packages/**" + ] +} diff --git a/turborepo-tests/integration/fixtures/monorepo_no_turbo_json/packages/another/package.json b/turborepo-tests/integration/fixtures/monorepo_no_turbo_json/packages/another/package.json new file mode 100644 index 0000000000000..e9e34ea52c154 --- /dev/null +++ b/turborepo-tests/integration/fixtures/monorepo_no_turbo_json/packages/another/package.json @@ -0,0 +1,4 @@ +{ + "name": "another", + "scripts": {} +} diff --git a/turborepo-tests/integration/fixtures/monorepo_no_turbo_json/packages/util/package.json b/turborepo-tests/integration/fixtures/monorepo_no_turbo_json/packages/util/package.json new file mode 100644 index 0000000000000..7309726a1df4e --- /dev/null +++ b/turborepo-tests/integration/fixtures/monorepo_no_turbo_json/packages/util/package.json @@ -0,0 +1,6 @@ +{ + "name": "util", + "scripts": { + "build": "echo building" + } +} diff --git a/turborepo-tests/integration/tests/run/allow-no-root-turbo.t b/turborepo-tests/integration/tests/run/allow-no-root-turbo.t new file mode 100644 index 0000000000000..cb0ae8d7b1174 --- /dev/null +++ b/turborepo-tests/integration/tests/run/allow-no-root-turbo.t @@ -0,0 +1,50 @@ +Setup + $ . ${TESTDIR}/../../../helpers/setup_integration_test.sh monorepo_no_turbo_json + +Run fails if not configured to allow missing turbo.json + $ ${TURBO} test + x Could not find turbo.json. + | Follow directions at https://turbo.build/repo/docs to create one + + [1] +Runs test tasks + $ MY_VAR=foo ${TURBO} test --experimental-allow-no-turbo-json + \xe2\x80\xa2 Packages in scope: another, my-app, util (esc) + \xe2\x80\xa2 Running test in 3 packages (esc) + \xe2\x80\xa2 Remote caching disabled (esc) + my-app:test: cache bypass, force executing d80016a1a60c4c0a + my-app:test: + my-app:test: > test + my-app:test: > echo $MY_VAR + my-app:test: + my-app:test: foo + + Tasks: 1 successful, 1 total + Cached: 0 cached, 1 total + Time:\s*[\.0-9]+m?s (re) + + + +Ensure caching is disabled + $ MY_VAR=foo ${TURBO} test --experimental-allow-no-turbo-json + \xe2\x80\xa2 Packages in scope: another, my-app, util (esc) + \xe2\x80\xa2 Running test in 3 packages (esc) + \xe2\x80\xa2 Remote caching disabled (esc) + my-app:test: cache bypass, force executing d80016a1a60c4c0a + my-app:test: + my-app:test: > test + my-app:test: > echo $MY_VAR + my-app:test: + my-app:test: foo + + Tasks: 1 successful, 1 total + Cached: 0 cached, 1 total + Time:\s*[\.0-9]+m?s (re) + +Finds all tasks based on scripts + $ TURBO_ALLOW_NO_TURBO_JSON=true ${TURBO} build test --dry=json | jq '.tasks | map(.taskId)| sort' + [ + "my-app#build", + "my-app#test", + "util#build" + ]