feat: turbo-trace (#9134)

### Description

A long long time ago we had a
[node-file-trace](https://github.com/vercel/nft) implementation in the
repository. Unfortunately this was tied to turbopack and the turbo-tasks
code. Since that's now in a different repository, we're gonna implement
our own version, because that's easier than coordinating multi-repo
dependencies.

This implementation uses `oxc_resolver` and `swc` to resolve
dependencies and parse files.

This PR hooks it up to `turbo query` but we'll probably have other uses
later.

### Testing Instructions

Adds some tests in `turbo-trace.t` using `query`

---------

Co-authored-by: Turbobot <[email protected]>

Cargo.toml

@@ -3,6 +3,7 @@ resolver = "2"
 
 members = [
   "crates/tower-uds",
+  "crates/turbo-trace",
   "crates/turborepo*",
   "packages/turbo-repository/rust",
 ]
@@ -43,6 +44,7 @@ async-recursion = "1.0.2"
 miette = { version = "5.10.0", features = ["fancy"] }
 markdown = "1.0.0-alpha.18"
 
+turbo-trace = { path = "crates/turbo-trace" }
 turbo-updater = { path = "crates/turborepo-updater" }
 turbopath = { path = "crates/turborepo-paths" }
 turborepo = { path = "crates/turborepo" }

crates/turbo-trace/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "turbo-trace"
+version = "0.1.0"
+edition = "2021"
+license = "MIT"
+
+[dependencies]
+camino.workspace = true
+clap = { version = "4.5.17", features = ["derive"] }
+miette = { workspace = true, features = ["fancy"] }
+oxc_resolver = "1.11.0"
+swc_common = "0.37.5"
+swc_ecma_ast = "0.118.2"
+swc_ecma_parser = "0.149.1"
+swc_ecma_visit = "0.104.8"
+thiserror = { workspace = true }
+turbopath = { workspace = true }
+
+[lints]
+workspace = true

crates/turbo-trace/src/import_finder.rs

@@ -0,0 +1,50 @@
+use swc_common::{Span, Spanned};
+use swc_ecma_ast::{Decl, ModuleDecl, Stmt};
+use swc_ecma_visit::{Visit, VisitWith};
+
+#[derive(Default)]
+pub struct ImportFinder {
+    imports: Vec<(String, Span)>,
+}
+
+impl ImportFinder {
+    pub fn imports(&self) -> &[(String, Span)] {
+        &self.imports
+    }
+}
+
+impl Visit for ImportFinder {
+    fn visit_module_decl(&mut self, decl: &ModuleDecl) {
+        if let ModuleDecl::Import(import) = decl {
+            self.imports
+                .push((import.src.value.to_string(), import.span));
+        }
+    }
+
+    fn visit_stmt(&mut self, stmt: &Stmt) {
+        if let Stmt::Decl(Decl::Var(var_decl)) = stmt {
+            for decl in &var_decl.decls {
+                if let Some(init) = &decl.init {
+                    if let swc_ecma_ast::Expr::Call(call_expr) = &**init {
+                        if let swc_ecma_ast::Callee::Expr(expr) = &call_expr.callee {
+                            if let swc_ecma_ast::Expr::Ident(ident) = &**expr {
+                                if ident.sym == *"require" {
+                                    if let Some(arg) = call_expr.args.first() {
+                                        if let swc_ecma_ast::Expr::Lit(swc_ecma_ast::Lit::Str(
+                                            lit_str,
+                                        )) = &*arg.expr
+                                        {
+                                            self.imports
+                                                .push((lit_str.value.to_string(), expr.span()));
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        stmt.visit_children_with(self);
+    }
+}

crates/turbo-trace/src/lib.rs

@@ -0,0 +1,5 @@
+#![deny(clippy::all)]
+mod import_finder;
+mod tracer;
+
+pub use tracer::{TraceError, Tracer};

crates/turbo-trace/src/main.rs

@@ -0,0 +1,49 @@
+mod import_finder;
+mod tracer;
+
+use camino::Utf8PathBuf;
+use clap::Parser;
+use tracer::Tracer;
+use turbopath::{AbsoluteSystemPathBuf, PathError};
+
+#[derive(Parser, Debug)]
+struct Args {
+    #[clap(long, value_parser)]
+    cwd: Option<Utf8PathBuf>,
+    #[clap(long)]
+    ts_config: Option<Utf8PathBuf>,
+    files: Vec<Utf8PathBuf>,
+}
+
+fn main() -> Result<(), PathError> {
+    let args = Args::parse();
+
+    let abs_cwd = if let Some(cwd) = args.cwd {
+        AbsoluteSystemPathBuf::from_cwd(cwd)?
+    } else {
+        AbsoluteSystemPathBuf::cwd()?
+    };
+
+    let files = args
+        .files
+        .into_iter()
+        .map(|f| AbsoluteSystemPathBuf::from_unknown(&abs_cwd, f))
+        .collect();
+
+    let tracer = Tracer::new(abs_cwd, files, args.ts_config)?;
+
+    let result = tracer.trace();
+
+    if !result.errors.is_empty() {
+        for error in &result.errors {
+            eprintln!("error: {}", error);
+        }
+        std::process::exit(1);
+    } else {
+        for file in &result.files {
+            println!("{}", file);
+        }
+    }
+
+    Ok(())
+}

crates/turbo-trace/src/tracer.rs

@@ -0,0 +1,170 @@
+use std::{collections::HashSet, fs, rc::Rc};
+
+use camino::Utf8PathBuf;
+use miette::{Diagnostic, NamedSource, SourceSpan};
+use oxc_resolver::{
+    EnforceExtension, ResolveError, ResolveOptions, Resolver, TsconfigOptions, TsconfigReferences,
+};
+use swc_common::{comments::SingleThreadedComments, input::StringInput, FileName, SourceMap};
+use swc_ecma_ast::EsVersion;
+use swc_ecma_parser::{lexer::Lexer, Capturing, EsSyntax, Parser, Syntax, TsSyntax};
+use swc_ecma_visit::VisitWith;
+use thiserror::Error;
+use turbopath::{AbsoluteSystemPathBuf, PathError};
+
+use crate::import_finder::ImportFinder;
+
+pub struct Tracer {
+    files: Vec<AbsoluteSystemPathBuf>,
+    seen: HashSet<AbsoluteSystemPathBuf>,
+    ts_config: Option<AbsoluteSystemPathBuf>,
+    source_map: Rc<SourceMap>,
+}
+
+#[derive(Debug, Error, Diagnostic)]
+pub enum TraceError {
+    #[error("failed to read file: {0}")]
+    FileNotFound(AbsoluteSystemPathBuf),
+    #[error(transparent)]
+    PathEncoding(PathError),
+    #[error("tracing a root file `{0}`, no parent found")]
+    RootFile(AbsoluteSystemPathBuf),
+    #[error("failed to resolve import")]
+    Resolve {
+        #[label("import here")]
+        span: SourceSpan,
+        #[source_code]
+        text: NamedSource,
+    },
+}
+
+pub struct TraceResult {
+    pub errors: Vec<TraceError>,
+    pub files: HashSet<AbsoluteSystemPathBuf>,
+}
+
+impl Tracer {
+    pub fn new(
+        cwd: AbsoluteSystemPathBuf,
+        files: Vec<AbsoluteSystemPathBuf>,
+        ts_config: Option<Utf8PathBuf>,
+    ) -> Result<Self, PathError> {
+        let ts_config =
+            ts_config.map(|ts_config| AbsoluteSystemPathBuf::from_unknown(&cwd, ts_config));
+
+        let seen = HashSet::new();
+
+        Ok(Self {
+            files,
+            seen,
+            ts_config,
+            source_map: Rc::new(SourceMap::default()),
+        })
+    }
+
+    pub fn trace(mut self) -> TraceResult {
+        let mut options = ResolveOptions::default()
+            .with_builtin_modules(true)
+            .with_force_extension(EnforceExtension::Disabled)
+            .with_extension(".ts")
+            .with_extension(".tsx");
+        if let Some(ts_config) = self.ts_config.take() {
+            options.tsconfig = Some(TsconfigOptions {
+                config_file: ts_config.into(),
+                references: TsconfigReferences::Auto,
+            });
+        }
+
+        let resolver = Resolver::new(options);
+        let mut errors = vec![];
+
+        while let Some(file_path) = self.files.pop() {
+            if matches!(file_path.extension(), Some("json") | Some("css")) {
+                continue;
+            }
+
+            if self.seen.contains(&file_path) {
+                continue;
+            }
+
+            self.seen.insert(file_path.clone());
+
+            // Read the file content
+            let Ok(file_content) = fs::read_to_string(&file_path) else {
+                errors.push(TraceError::FileNotFound(file_path.clone()));
+                continue;
+            };
+
+            let comments = SingleThreadedComments::default();
+
+            let source_file = self.source_map.new_source_file(
+                FileName::Custom(file_path.to_string()).into(),
+                file_content.clone(),
+            );
+
+            let syntax =
+                if file_path.extension() == Some("ts") || file_path.extension() == Some("tsx") {
+                    Syntax::Typescript(TsSyntax {
+                        tsx: file_path.extension() == Some("tsx"),
+                        decorators: true,
+                        ..Default::default()
+                    })
+                } else {
+                    Syntax::Es(EsSyntax {
+                        jsx: file_path.ends_with(".jsx"),
+                        ..Default::default()
+                    })
+                };
+
+            let lexer = Lexer::new(
+                syntax,
+                EsVersion::EsNext,
+                StringInput::from(&*source_file),
+                Some(&comments),
+            );
+
+            let mut parser = Parser::new_from(Capturing::new(lexer));
+
+            // Parse the file as a module
+            let Ok(module) = parser.parse_module() else {
+                errors.push(TraceError::FileNotFound(file_path.to_owned()));
+                continue;
+            };
+
+            // Visit the AST and find imports
+            let mut finder = ImportFinder::default();
+            module.visit_with(&mut finder);
+
+            // Convert found imports/requires to absolute paths and add them to files to
+            // visit
+            for (import, span) in finder.imports() {
+                let Some(file_dir) = file_path.parent() else {
+                    errors.push(TraceError::RootFile(file_path.to_owned()));
+                    continue;
+                };
+                match resolver.resolve(file_dir, import) {
+                    Ok(resolved) => match resolved.into_path_buf().try_into() {
+                        Ok(path) => self.files.push(path),
+                        Err(err) => {
+                            errors.push(TraceError::PathEncoding(err));
+                        }
+                    },
+                    Err(ResolveError::Builtin(_)) => {}
+                    Err(_) => {
+                        let (start, end) = self.source_map.span_to_char_offset(&source_file, *span);
+
+                        errors.push(TraceError::Resolve {
+                            span: (start as usize, end as usize).into(),
+                            text: NamedSource::new(file_path.to_string(), file_content.clone()),
+                        });
+                    }
+                }
+            }
+        }
+
+        TraceResult {
+            files: self.seen,
+            errors,
+        }
+    }
+}

crates/turborepo-lib/Cargo.toml

@@ -113,6 +113,7 @@ tracing-appender = "0.2.2"
 tracing-chrome = "0.7.1"
 tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
 tracing.workspace = true
+turbo-trace = { workspace = true }
 turbo-updater = { workspace = true }
 turbopath = { workspace = true }
 turborepo-analytics = { path = "../turborepo-analytics" }

crates/turborepo-lib/src/query/file.rs

@@ -0,0 +1,61 @@
+use std::sync::Arc;
+
+use async_graphql::Object;
+use itertools::Itertools;
+use turbo_trace::Tracer;
+use turbopath::AbsoluteSystemPathBuf;
+
+use crate::{query::Error, run::Run};
+
+pub struct File {
+    run: Arc<Run>,
+    path: AbsoluteSystemPathBuf,
+}
+
+impl File {
+    pub fn new(run: Arc<Run>, path: AbsoluteSystemPathBuf) -> Self {
+        Self { run, path }
+    }
+}
+
+#[Object]
+impl File {
+    async fn contents(&self) -> Result<String, Error> {
+        let contents = self.path.read_to_string()?;
+        Ok(contents)
+    }
+
+    async fn path(&self) -> Result<String, Error> {
+        Ok(self
+            .run
+            .repo_root()
+            .anchor(&self.path)
+            .map(|path| path.to_string())?)
+    }
+
+    async fn absolute_path(&self) -> Result<String, Error> {
+        Ok(self.path.to_string())
+    }
+
+    async fn dependencies(&self) -> Result<Vec<File>, Error> {
+        let tracer = Tracer::new(
+            self.run.repo_root().to_owned(),
+            vec![self.path.clone()],
+            None,
+        )?;
+
+        let result = tracer.trace();
+        if !result.errors.is_empty() {
+            return Err(Error::Trace(result.errors));
+        }
+
+        Ok(result
+            .files
+            .into_iter()
+            // Filter out the file we're looking at
+            .filter(|file| file != &self.path)
+            .map(|path| File::new(self.run.clone(), path))
+            .sorted_by(|a, b| a.path.cmp(&b.path))
+            .collect())
+    }
+}