From 481212e4adb357785d34884897edf8253013264c Mon Sep 17 00:00:00 2001 From: Nicholas Yang Date: Thu, 19 Sep 2024 17:55:46 -0400 Subject: [PATCH] feat: turbo-trace (#9134) ### Description A long long time ago we had a [node-file-trace](https://github.com/vercel/nft) implementation in the repository. Unfortunately this was tied to turbopack and the turbo-tasks code. Since that's now in a different repository, we're gonna implement our own version, because that's easier than coordinating multi-repo dependencies. This implementation uses `oxc_resolver` and `swc` to resolve dependencies and parse files. This PR hooks it up to `turbo query` but we'll probably have other uses later. ### Testing Instructions Adds some tests in `turbo-trace.t` using `query` --------- Co-authored-by: Turbobot --- Cargo.lock | 466 ++++++++++++++++-- Cargo.toml | 2 + crates/turbo-trace/Cargo.toml | 20 + crates/turbo-trace/src/import_finder.rs | 50 ++ crates/turbo-trace/src/lib.rs | 5 + crates/turbo-trace/src/main.rs | 49 ++ crates/turbo-trace/src/tracer.rs | 170 +++++++ crates/turborepo-lib/Cargo.toml | 1 + crates/turborepo-lib/src/query/file.rs | 61 +++ .../src/{query.rs => query/mod.rs} | 206 +------- crates/turborepo-lib/src/query/package.rs | 186 +++++++ .../fixtures/turbo_trace/.gitignore | 38 ++ .../fixtures/turbo_trace/README.md | 81 +++ .../fixtures/turbo_trace/button.tsx | 3 + .../fixtures/turbo_trace/circular.ts | 1 + .../fixtures/turbo_trace/circular2.ts | 1 + .../integration/fixtures/turbo_trace/foo.js | 6 + .../integration/fixtures/turbo_trace/main.ts | 9 + .../fixtures/turbo_trace/package.json | 14 + .../fixtures/turbo_trace/turbo.json | 18 + .../integration/tests/turbo-trace.t | 60 +++ 21 files changed, 1238 insertions(+), 209 deletions(-) create mode 100644 crates/turbo-trace/Cargo.toml create mode 100644 crates/turbo-trace/src/import_finder.rs create mode 100644 crates/turbo-trace/src/lib.rs create mode 100644 crates/turbo-trace/src/main.rs create mode 100644 crates/turbo-trace/src/tracer.rs create mode 100644 crates/turborepo-lib/src/query/file.rs rename crates/turborepo-lib/src/{query.rs => query/mod.rs} (65%) create mode 100644 crates/turborepo-lib/src/query/package.rs create mode 100644 turborepo-tests/integration/fixtures/turbo_trace/.gitignore create mode 100644 turborepo-tests/integration/fixtures/turbo_trace/README.md create mode 100644 turborepo-tests/integration/fixtures/turbo_trace/button.tsx create mode 100644 turborepo-tests/integration/fixtures/turbo_trace/circular.ts create mode 100644 turborepo-tests/integration/fixtures/turbo_trace/circular2.ts create mode 100644 turborepo-tests/integration/fixtures/turbo_trace/foo.js create mode 100644 turborepo-tests/integration/fixtures/turbo_trace/main.ts create mode 100644 turborepo-tests/integration/fixtures/turbo_trace/package.json create mode 100644 turborepo-tests/integration/fixtures/turbo_trace/turbo.json create mode 100644 turborepo-tests/integration/tests/turbo-trace.t diff --git a/Cargo.lock b/Cargo.lock index 4949c6e3246c1..234d5d4d63dc4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -163,6 +163,18 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "ast_node" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9184f2b369b3e8625712493c89b785881f27eedc6cde480a81883cef78868b2" +dependencies = [ + "proc-macro2", + "quote", + "swc_macros_common", + "syn 2.0.58", +] + [[package]] name = "async-channel" version = "1.8.0" @@ -636,6 +648,15 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "better_scoped_tls" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "794edcc9b3fb07bb4aecaa11f093fd45663b4feadb782d68303a2268bc2701de" +dependencies = [ + "scoped-tls", +] + [[package]] name = "biome_console" version = "0.5.7" @@ -797,7 +818,7 @@ dependencies = [ "countme", "hashbrown 0.12.3", "memoffset 0.8.0", - "rustc-hash", + "rustc-hash 1.1.0", "tracing", ] @@ -931,9 +952,12 @@ checksum = "832133bbabbbaa9fbdba793456a2827627a7d2b8fb96032fa1e7666d7895832b" [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +dependencies = [ + "allocator-api2", +] [[package]] name = "bytecount" @@ -1055,9 +1079,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.16" +version = "4.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed6719fffa43d0d87e5fd8caeab59be1554fb028cd30edc88fc4369b17971019" +checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" dependencies = [ "clap_builder", "clap_derive", @@ -1065,9 +1089,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.15" +version = "4.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216aec2b177652e3846684cbfe25c9964d18ec45234f0f5da5157b207ed1aab6" +checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" dependencies = [ "anstream", "anstyle", @@ -1490,7 +1514,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.3", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", "lock_api", "once_cell", "parking_lot_core", @@ -1867,6 +1905,17 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "from_variant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32016f1242eb82af5474752d00fd8ebcd9004bd69b462b1c91de833972d08ed4" +dependencies = [ + "proc-macro2", + "swc_macros_common", + "syn 2.0.58", +] + [[package]] name = "fs-err" version = "2.9.0" @@ -2165,9 +2214,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash", "allocator-api2", @@ -2226,6 +2275,20 @@ dependencies = [ "winapi", ] +[[package]] +name = "hstr" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae404c0c5d4e95d4858876ab02eecd6a196bb8caa42050dfa809938833fc412" +dependencies = [ + "hashbrown 0.14.5", + "new_debug_unreachable", + "once_cell", + "phf", + "rustc-hash 1.1.0", + "triomphe", +] + [[package]] name = "http" version = "0.2.11" @@ -2518,7 +2581,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "serde", ] @@ -2609,6 +2672,18 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" +[[package]] +name = "is-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2069faacbe981460232f880d26bf3c7634e322d49053aa48c27e3ae642f728f1" +dependencies = [ + "Inflector", + "proc-macro2", + "quote", + "syn 2.0.58", +] + [[package]] name = "is-terminal" version = "0.4.9" @@ -2810,9 +2885,9 @@ checksum = "db13adb97ab515a3691f56e4dbab09283d0b86cb45abd991d8634a9d6f501760" [[package]] name = "libc" -version = "0.2.155" +version = "0.2.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" [[package]] name = "libgit2-sys" @@ -2907,7 +2982,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db2c024b41519440580066ba82aab04092b333e09066a5eb86c7c4890df31f22" dependencies = [ - "hashbrown 0.14.3", + "hashbrown 0.14.5", ] [[package]] @@ -3218,6 +3293,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + [[package]] name = "nibble_vec" version = "0.1.0" @@ -3328,12 +3409,32 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", + "serde", +] + [[package]] name = "num-conv" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -3441,7 +3542,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4d6a8c22fc714f0c2373e6091bf6f5e9b37b1bc0b1184874b7e0a4e303d318f" dependencies = [ "dlv-list", - "hashbrown 0.14.3", + "hashbrown 0.14.5", ] [[package]] @@ -3478,18 +3579,20 @@ checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" [[package]] name = "oxc_resolver" -version = "1.5.4" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2033cc3b0e72446d3321866db0954804b9ca559ad692480205053f6aea4bfc15" +checksum = "e7fe4d07afdfcf6b1d7fb952e6691d82692a54b71964a377cf49f3e47dac283d" dependencies = [ - "dashmap", + "cfg-if", + "dashmap 6.1.0", "dunce", "indexmap 2.2.6", "json-strip-comments", "once_cell", - "rustc-hash", + "rustc-hash 2.0.0", "serde", "serde_json", + "simdutf8", "thiserror", "tracing", ] @@ -3608,6 +3711,48 @@ dependencies = [ "indexmap 1.9.3", ] +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_macros" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.58", +] + +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher", +] + [[package]] name = "pidlock" version = "0.1.4" @@ -3925,6 +4070,35 @@ dependencies = [ "prost 0.11.8", ] +[[package]] +name = "psm" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa37f80ca58604976033fae9515a8a2989fc13797d953f7c04fb8fa36a11f205" +dependencies = [ + "cc", +] + +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "quickcheck" version = "1.0.3" @@ -4200,6 +4374,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc-hash" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" + [[package]] name = "rustc_version" version = "0.2.3" @@ -4364,6 +4544,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -4426,18 +4612,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.201" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "780f1cebed1629e4753a1a38a3c72d30b97ec044f0aef68cb26650a3c5cf363c" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.201" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e405930b9796f1c00bee880d03fc7e0bb4b9a11afc776885ffe84320da2865" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", @@ -4661,6 +4847,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simdutf8" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" + [[package]] name = "similar" version = "2.5.0" @@ -4671,6 +4863,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + [[package]] name = "slab" version = "0.4.9" @@ -4697,6 +4895,17 @@ version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" +[[package]] +name = "smartstring" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" +dependencies = [ + "autocfg", + "static_assertions", + "version_check", +] + [[package]] name = "smawk" version = "0.3.1" @@ -4751,6 +4960,19 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "stacker" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.59.0", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -4781,6 +5003,18 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9557cb6521e8d009c51a8666f09356f4b817ba9ba0981a305bd86aee47bd35c" +[[package]] +name = "string_enum" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05e383308aebc257e7d7920224fa055c632478d92744eca77f99be8fa1545b90" +dependencies = [ + "proc-macro2", + "quote", + "swc_macros_common", + "syn 2.0.58", +] + [[package]] name = "strsim" version = "0.11.1" @@ -4885,6 +5119,143 @@ dependencies = [ "time", ] +[[package]] +name = "swc_allocator" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adc8bd3075d1c6964010333fae9ddcd91ad422a4f8eb8b3206a9b2b6afb4209e" +dependencies = [ + "bumpalo", + "hashbrown 0.14.5", + "ptr_meta", + "rustc-hash 1.1.0", + "triomphe", +] + +[[package]] +name = "swc_atoms" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb6567e4e67485b3e7662b486f1565bdae54bd5b9d6b16b2ba1a9babb1e42125" +dependencies = [ + "hstr", + "once_cell", + "rustc-hash 1.1.0", + "serde", +] + +[[package]] +name = "swc_common" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12d0a8eaaf1606c9207077d75828008cb2dfb51b095a766bd2b72ef893576e31" +dependencies = [ + "ast_node", + "better_scoped_tls", + "cfg-if", + "either", + "from_variant", + "new_debug_unreachable", + "num-bigint", + "once_cell", + "rustc-hash 1.1.0", + "serde", + "siphasher", + "swc_allocator", + "swc_atoms", + "swc_eq_ignore_macros", + "swc_visit", + "tracing", + "unicode-width", + "url", +] + +[[package]] +name = "swc_ecma_ast" +version = "0.118.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6f866d12e4d519052b92a0a86d1ac7ff17570da1272ca0c89b3d6f802cd79df" +dependencies = [ + "bitflags 2.5.0", + "is-macro", + "num-bigint", + "phf", + "scoped-tls", + "string_enum", + "swc_atoms", + "swc_common", + "unicode-id-start", +] + +[[package]] +name = "swc_ecma_parser" +version = "0.149.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683dada14722714588b56481399c699378b35b2ba4deb5c4db2fb627a97fb54b" +dependencies = [ + "either", + "new_debug_unreachable", + "num-bigint", + "num-traits", + "phf", + "serde", + "smallvec", + "smartstring", + "stacker", + "swc_atoms", + "swc_common", + "swc_ecma_ast", + "tracing", + "typed-arena", +] + +[[package]] +name = "swc_ecma_visit" +version = "0.104.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b1c6802e68e51f336e8bc9644e9ff9da75d7da9c1a6247d532f2e908aa33e81" +dependencies = [ + "new_debug_unreachable", + "num-bigint", + "swc_atoms", + "swc_common", + "swc_ecma_ast", + "swc_visit", + "tracing", +] + +[[package]] +name = "swc_eq_ignore_macros" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63db0adcff29d220c3d151c5b25c0eabe7e32dd936212b84cdaa1392e3130497" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.58", +] + +[[package]] +name = "swc_macros_common" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f486687bfb7b5c560868f69ed2d458b880cebc9babebcb67e49f31b55c5bf847" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.58", +] + +[[package]] +name = "swc_visit" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ceb044142ba2719ef9eb3b6b454fce61ab849eb696c34d190f04651955c613d" +dependencies = [ + "either", + "new_debug_unreachable", +] + [[package]] name = "symbolic-common" version = "12.4.1" @@ -5098,18 +5469,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.58" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" +checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.58" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" +checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", @@ -5444,7 +5815,7 @@ dependencies = [ "async-trait", "auto_impl", "bytes", - "dashmap", + "dashmap 5.5.3", "futures", "httparse", "lsp-types", @@ -5603,6 +5974,16 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "triomphe" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6631e42e10b40c0690bf92f404ebcfe6e1fdb480391d15f17cc8e96eeed5369" +dependencies = [ + "serde", + "stable_deref_trait", +] + [[package]] name = "try-lock" version = "0.2.4" @@ -5651,6 +6032,22 @@ dependencies = [ "winapi", ] +[[package]] +name = "turbo-trace" +version = "0.1.0" +dependencies = [ + "camino", + "clap", + "miette", + "oxc_resolver", + "swc_common", + "swc_ecma_ast", + "swc_ecma_parser", + "swc_ecma_visit", + "thiserror", + "turbopath", +] + [[package]] name = "turbo-updater" version = "0.1.0" @@ -5979,6 +6376,7 @@ dependencies = [ "tracing-chrome", "tracing-subscriber", "tracing-test", + "turbo-trace", "turbo-updater", "turbopath", "turborepo-analytics", @@ -6250,6 +6648,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "typed-arena" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" + [[package]] name = "typenum" version = "1.16.0" @@ -6334,6 +6738,12 @@ version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7eec5d1121208364f6793f7d2e222bf75a915c19557537745b195b253dd64217" +[[package]] +name = "unicode-id-start" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc3882f69607a2ac8cc4de3ee7993d8f68bb06f2974271195065b3bd07f2edea" + [[package]] name = "unicode-ident" version = "1.0.11" diff --git a/Cargo.toml b/Cargo.toml index d655030df1620..80484ff05f4b2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ resolver = "2" members = [ "crates/tower-uds", + "crates/turbo-trace", "crates/turborepo*", "packages/turbo-repository/rust", ] @@ -43,6 +44,7 @@ async-recursion = "1.0.2" miette = { version = "5.10.0", features = ["fancy"] } markdown = "1.0.0-alpha.18" +turbo-trace = { path = "crates/turbo-trace" } turbo-updater = { path = "crates/turborepo-updater" } turbopath = { path = "crates/turborepo-paths" } turborepo = { path = "crates/turborepo" } diff --git a/crates/turbo-trace/Cargo.toml b/crates/turbo-trace/Cargo.toml new file mode 100644 index 0000000000000..5eeb45bfc5c34 --- /dev/null +++ b/crates/turbo-trace/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "turbo-trace" +version = "0.1.0" +edition = "2021" +license = "MIT" + +[dependencies] +camino.workspace = true +clap = { version = "4.5.17", features = ["derive"] } +miette = { workspace = true, features = ["fancy"] } +oxc_resolver = "1.11.0" +swc_common = "0.37.5" +swc_ecma_ast = "0.118.2" +swc_ecma_parser = "0.149.1" +swc_ecma_visit = "0.104.8" +thiserror = { workspace = true } +turbopath = { workspace = true } + +[lints] +workspace = true diff --git a/crates/turbo-trace/src/import_finder.rs b/crates/turbo-trace/src/import_finder.rs new file mode 100644 index 0000000000000..ddfbf83875418 --- /dev/null +++ b/crates/turbo-trace/src/import_finder.rs @@ -0,0 +1,50 @@ +use swc_common::{Span, Spanned}; +use swc_ecma_ast::{Decl, ModuleDecl, Stmt}; +use swc_ecma_visit::{Visit, VisitWith}; + +#[derive(Default)] +pub struct ImportFinder { + imports: Vec<(String, Span)>, +} + +impl ImportFinder { + pub fn imports(&self) -> &[(String, Span)] { + &self.imports + } +} + +impl Visit for ImportFinder { + fn visit_module_decl(&mut self, decl: &ModuleDecl) { + if let ModuleDecl::Import(import) = decl { + self.imports + .push((import.src.value.to_string(), import.span)); + } + } + + fn visit_stmt(&mut self, stmt: &Stmt) { + if let Stmt::Decl(Decl::Var(var_decl)) = stmt { + for decl in &var_decl.decls { + if let Some(init) = &decl.init { + if let swc_ecma_ast::Expr::Call(call_expr) = &**init { + if let swc_ecma_ast::Callee::Expr(expr) = &call_expr.callee { + if let swc_ecma_ast::Expr::Ident(ident) = &**expr { + if ident.sym == *"require" { + if let Some(arg) = call_expr.args.first() { + if let swc_ecma_ast::Expr::Lit(swc_ecma_ast::Lit::Str( + lit_str, + )) = &*arg.expr + { + self.imports + .push((lit_str.value.to_string(), expr.span())); + } + } + } + } + } + } + } + } + } + stmt.visit_children_with(self); + } +} diff --git a/crates/turbo-trace/src/lib.rs b/crates/turbo-trace/src/lib.rs new file mode 100644 index 0000000000000..b8f162159e1ba --- /dev/null +++ b/crates/turbo-trace/src/lib.rs @@ -0,0 +1,5 @@ +#![deny(clippy::all)] +mod import_finder; +mod tracer; + +pub use tracer::{TraceError, Tracer}; diff --git a/crates/turbo-trace/src/main.rs b/crates/turbo-trace/src/main.rs new file mode 100644 index 0000000000000..be8c5cd79857c --- /dev/null +++ b/crates/turbo-trace/src/main.rs @@ -0,0 +1,49 @@ +mod import_finder; +mod tracer; + +use camino::Utf8PathBuf; +use clap::Parser; +use tracer::Tracer; +use turbopath::{AbsoluteSystemPathBuf, PathError}; + +#[derive(Parser, Debug)] +struct Args { + #[clap(long, value_parser)] + cwd: Option, + #[clap(long)] + ts_config: Option, + files: Vec, +} + +fn main() -> Result<(), PathError> { + let args = Args::parse(); + + let abs_cwd = if let Some(cwd) = args.cwd { + AbsoluteSystemPathBuf::from_cwd(cwd)? + } else { + AbsoluteSystemPathBuf::cwd()? + }; + + let files = args + .files + .into_iter() + .map(|f| AbsoluteSystemPathBuf::from_unknown(&abs_cwd, f)) + .collect(); + + let tracer = Tracer::new(abs_cwd, files, args.ts_config)?; + + let result = tracer.trace(); + + if !result.errors.is_empty() { + for error in &result.errors { + eprintln!("error: {}", error); + } + std::process::exit(1); + } else { + for file in &result.files { + println!("{}", file); + } + } + + Ok(()) +} diff --git a/crates/turbo-trace/src/tracer.rs b/crates/turbo-trace/src/tracer.rs new file mode 100644 index 0000000000000..2d236fc9ef482 --- /dev/null +++ b/crates/turbo-trace/src/tracer.rs @@ -0,0 +1,170 @@ +use std::{collections::HashSet, fs, rc::Rc}; + +use camino::Utf8PathBuf; +use miette::{Diagnostic, NamedSource, SourceSpan}; +use oxc_resolver::{ + EnforceExtension, ResolveError, ResolveOptions, Resolver, TsconfigOptions, TsconfigReferences, +}; +use swc_common::{comments::SingleThreadedComments, input::StringInput, FileName, SourceMap}; +use swc_ecma_ast::EsVersion; +use swc_ecma_parser::{lexer::Lexer, Capturing, EsSyntax, Parser, Syntax, TsSyntax}; +use swc_ecma_visit::VisitWith; +use thiserror::Error; +use turbopath::{AbsoluteSystemPathBuf, PathError}; + +use crate::import_finder::ImportFinder; + +pub struct Tracer { + files: Vec, + seen: HashSet, + ts_config: Option, + source_map: Rc, +} + +#[derive(Debug, Error, Diagnostic)] +pub enum TraceError { + #[error("failed to read file: {0}")] + FileNotFound(AbsoluteSystemPathBuf), + #[error(transparent)] + PathEncoding(PathError), + #[error("tracing a root file `{0}`, no parent found")] + RootFile(AbsoluteSystemPathBuf), + #[error("failed to resolve import")] + Resolve { + #[label("import here")] + span: SourceSpan, + #[source_code] + text: NamedSource, + }, +} + +pub struct TraceResult { + pub errors: Vec, + pub files: HashSet, +} + +impl Tracer { + pub fn new( + cwd: AbsoluteSystemPathBuf, + files: Vec, + ts_config: Option, + ) -> Result { + let ts_config = + ts_config.map(|ts_config| AbsoluteSystemPathBuf::from_unknown(&cwd, ts_config)); + + let seen = HashSet::new(); + + Ok(Self { + files, + seen, + ts_config, + source_map: Rc::new(SourceMap::default()), + }) + } + + pub fn trace(mut self) -> TraceResult { + let mut options = ResolveOptions::default() + .with_builtin_modules(true) + .with_force_extension(EnforceExtension::Disabled) + .with_extension(".ts") + .with_extension(".tsx"); + if let Some(ts_config) = self.ts_config.take() { + options.tsconfig = Some(TsconfigOptions { + config_file: ts_config.into(), + references: TsconfigReferences::Auto, + }); + } + + let resolver = Resolver::new(options); + let mut errors = vec![]; + + while let Some(file_path) = self.files.pop() { + if matches!(file_path.extension(), Some("json") | Some("css")) { + continue; + } + + if self.seen.contains(&file_path) { + continue; + } + + self.seen.insert(file_path.clone()); + + // Read the file content + let Ok(file_content) = fs::read_to_string(&file_path) else { + errors.push(TraceError::FileNotFound(file_path.clone())); + continue; + }; + + let comments = SingleThreadedComments::default(); + + let source_file = self.source_map.new_source_file( + FileName::Custom(file_path.to_string()).into(), + file_content.clone(), + ); + + let syntax = + if file_path.extension() == Some("ts") || file_path.extension() == Some("tsx") { + Syntax::Typescript(TsSyntax { + tsx: file_path.extension() == Some("tsx"), + decorators: true, + ..Default::default() + }) + } else { + Syntax::Es(EsSyntax { + jsx: file_path.ends_with(".jsx"), + ..Default::default() + }) + }; + + let lexer = Lexer::new( + syntax, + EsVersion::EsNext, + StringInput::from(&*source_file), + Some(&comments), + ); + + let mut parser = Parser::new_from(Capturing::new(lexer)); + + // Parse the file as a module + let Ok(module) = parser.parse_module() else { + errors.push(TraceError::FileNotFound(file_path.to_owned())); + continue; + }; + + // Visit the AST and find imports + let mut finder = ImportFinder::default(); + module.visit_with(&mut finder); + + // Convert found imports/requires to absolute paths and add them to files to + // visit + for (import, span) in finder.imports() { + let Some(file_dir) = file_path.parent() else { + errors.push(TraceError::RootFile(file_path.to_owned())); + continue; + }; + match resolver.resolve(file_dir, import) { + Ok(resolved) => match resolved.into_path_buf().try_into() { + Ok(path) => self.files.push(path), + Err(err) => { + errors.push(TraceError::PathEncoding(err)); + } + }, + Err(ResolveError::Builtin(_)) => {} + Err(_) => { + let (start, end) = self.source_map.span_to_char_offset(&source_file, *span); + + errors.push(TraceError::Resolve { + span: (start as usize, end as usize).into(), + text: NamedSource::new(file_path.to_string(), file_content.clone()), + }); + } + } + } + } + + TraceResult { + files: self.seen, + errors, + } + } +} diff --git a/crates/turborepo-lib/Cargo.toml b/crates/turborepo-lib/Cargo.toml index 92a2976c76d54..701d3e2084aa1 100644 --- a/crates/turborepo-lib/Cargo.toml +++ b/crates/turborepo-lib/Cargo.toml @@ -113,6 +113,7 @@ tracing-appender = "0.2.2" tracing-chrome = "0.7.1" tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } tracing.workspace = true +turbo-trace = { workspace = true } turbo-updater = { workspace = true } turbopath = { workspace = true } turborepo-analytics = { path = "../turborepo-analytics" } diff --git a/crates/turborepo-lib/src/query/file.rs b/crates/turborepo-lib/src/query/file.rs new file mode 100644 index 0000000000000..717f4d136fd94 --- /dev/null +++ b/crates/turborepo-lib/src/query/file.rs @@ -0,0 +1,61 @@ +use std::sync::Arc; + +use async_graphql::Object; +use itertools::Itertools; +use turbo_trace::Tracer; +use turbopath::AbsoluteSystemPathBuf; + +use crate::{query::Error, run::Run}; + +pub struct File { + run: Arc, + path: AbsoluteSystemPathBuf, +} + +impl File { + pub fn new(run: Arc, path: AbsoluteSystemPathBuf) -> Self { + Self { run, path } + } +} + +#[Object] +impl File { + async fn contents(&self) -> Result { + let contents = self.path.read_to_string()?; + Ok(contents) + } + + async fn path(&self) -> Result { + Ok(self + .run + .repo_root() + .anchor(&self.path) + .map(|path| path.to_string())?) + } + + async fn absolute_path(&self) -> Result { + Ok(self.path.to_string()) + } + + async fn dependencies(&self) -> Result, Error> { + let tracer = Tracer::new( + self.run.repo_root().to_owned(), + vec![self.path.clone()], + None, + )?; + + let result = tracer.trace(); + if !result.errors.is_empty() { + return Err(Error::Trace(result.errors)); + } + + Ok(result + .files + .into_iter() + // Filter out the file we're looking at + .filter(|file| file != &self.path) + .map(|path| File::new(self.run.clone(), path)) + .sorted_by(|a, b| a.path.cmp(&b.path)) + .collect()) + } +} diff --git a/crates/turborepo-lib/src/query.rs b/crates/turborepo-lib/src/query/mod.rs similarity index 65% rename from crates/turborepo-lib/src/query.rs rename to crates/turborepo-lib/src/query/mod.rs index c1ce6a1314399..d26594ff2017e 100644 --- a/crates/turborepo-lib/src/query.rs +++ b/crates/turborepo-lib/src/query/mod.rs @@ -1,3 +1,6 @@ +mod file; +mod package; + use std::{io, sync::Arc}; use async_graphql::{http::GraphiQLSource, *}; @@ -5,19 +8,27 @@ use async_graphql_axum::GraphQL; use axum::{response, response::IntoResponse, routing::get, Router}; use itertools::Itertools; use miette::Diagnostic; +use package::Package; use thiserror::Error; use tokio::{net::TcpListener, select}; -use turborepo_repository::package_graph::{PackageName, PackageNode}; +use turbo_trace::TraceError; +use turbopath::AbsoluteSystemPathBuf; +use turborepo_repository::package_graph::PackageName; use crate::{ + query::file::File, run::{builder::RunBuilder, Run}, signal::SignalHandler, }; #[derive(Error, Debug, Diagnostic)] pub enum Error { + #[error("failed to get file dependencies")] + Trace(#[related] Vec), #[error("no signal handler")] NoSignalHandler, + #[error("file `{0}` not found")] + FileNotFound(String), #[error("failed to start GraphQL server")] Server(#[from] io::Error), #[error("package not found: {0}")] @@ -26,6 +37,8 @@ pub enum Error { Serde(#[from] serde_json::Error), #[error(transparent)] Run(#[from] crate::run::Error), + #[error(transparent)] + Path(#[from] turbopath::PathError), } pub struct Query { @@ -39,7 +52,7 @@ impl Query { } #[derive(Debug, SimpleObject)] -struct Array { +pub struct Array { items: Vec, length: usize, } @@ -51,54 +64,6 @@ impl FromIterator for Array { Self { items, length } } } - -struct Package { - run: Arc, - name: PackageName, -} - -impl Package { - fn direct_dependents_count(&self) -> usize { - self.run - .pkg_dep_graph() - .immediate_ancestors(&PackageNode::Workspace(self.name.clone())) - .map_or(0, |pkgs| pkgs.len()) - } - - fn direct_dependencies_count(&self) -> usize { - self.run - .pkg_dep_graph() - .immediate_dependencies(&PackageNode::Workspace(self.name.clone())) - .map_or(0, |pkgs| pkgs.len()) - } - - fn indirect_dependents_count(&self) -> usize { - let node: PackageNode = PackageNode::Workspace(self.name.clone()); - - self.run.pkg_dep_graph().ancestors(&node).len() - self.direct_dependents_count() - } - - fn indirect_dependencies_count(&self) -> usize { - let node: PackageNode = PackageNode::Workspace(self.name.clone()); - - self.run.pkg_dep_graph().dependencies(&node).len() - self.direct_dependencies_count() - } - - fn all_dependents_count(&self) -> usize { - self.run - .pkg_dep_graph() - .ancestors(&PackageNode::Workspace(self.name.clone())) - .len() - } - - fn all_dependencies_count(&self) -> usize { - self.run - .pkg_dep_graph() - .dependencies(&PackageNode::Workspace(self.name.clone())) - .len() - } -} - #[derive(Enum, Copy, Clone, Eq, PartialEq, Debug)] enum PackageFields { Name, @@ -335,6 +300,16 @@ impl Query { }) } + async fn file(&self, path: String) -> Result { + let abs_path = AbsoluteSystemPathBuf::from_unknown(self.run.repo_root(), path); + + if !abs_path.exists() { + return Err(Error::FileNotFound(abs_path.to_string())); + } + + Ok(File::new(self.run.clone(), abs_path)) + } + /// Gets a list of packages that match the given filter async fn packages(&self, filter: Option) -> Result, Error> { let Some(filter) = filter else { @@ -364,137 +339,6 @@ impl Query { } } -#[Object] -impl Package { - /// The name of the package - async fn name(&self) -> String { - self.name.to_string() - } - - /// The path to the package, relative to the repository root - async fn path(&self) -> Result { - Ok(self - .run - .pkg_dep_graph() - .package_info(&self.name) - .ok_or_else(|| Error::PackageNotFound(self.name.clone()))? - .package_path() - .to_string()) - } - - /// The upstream packages that have this package as a direct dependency - async fn direct_dependents(&self) -> Result, Error> { - let node: PackageNode = PackageNode::Workspace(self.name.clone()); - Ok(self - .run - .pkg_dep_graph() - .immediate_ancestors(&node) - .iter() - .flatten() - .map(|package| Package { - run: self.run.clone(), - name: package.as_package_name().clone(), - }) - .sorted_by(|a, b| a.name.cmp(&b.name)) - .collect()) - } - - /// The downstream packages that directly depend on this package - async fn direct_dependencies(&self) -> Result, Error> { - let node: PackageNode = PackageNode::Workspace(self.name.clone()); - Ok(self - .run - .pkg_dep_graph() - .immediate_dependencies(&node) - .iter() - .flatten() - .map(|package| Package { - run: self.run.clone(), - name: package.as_package_name().clone(), - }) - .sorted_by(|a, b| a.name.cmp(&b.name)) - .collect()) - } - - /// The downstream packages that depend on this package, transitively - async fn all_dependents(&self) -> Result, Error> { - let node: PackageNode = PackageNode::Workspace(self.name.clone()); - Ok(self - .run - .pkg_dep_graph() - .ancestors(&node) - .iter() - .map(|package| Package { - run: self.run.clone(), - name: package.as_package_name().clone(), - }) - .sorted_by(|a, b| a.name.cmp(&b.name)) - .collect()) - } - - /// The upstream packages that this package depends on, transitively - async fn all_dependencies(&self) -> Result, Error> { - let node: PackageNode = PackageNode::Workspace(self.name.clone()); - Ok(self - .run - .pkg_dep_graph() - .dependencies(&node) - .iter() - .map(|package| Package { - run: self.run.clone(), - name: package.as_package_name().clone(), - }) - .sorted_by(|a, b| a.name.cmp(&b.name)) - .collect()) - } - - /// The downstream packages that depend on this package, indirectly - async fn indirect_dependents(&self) -> Result, Error> { - let node: PackageNode = PackageNode::Workspace(self.name.clone()); - let immediate_dependents = self - .run - .pkg_dep_graph() - .immediate_ancestors(&node) - .ok_or_else(|| Error::PackageNotFound(self.name.clone()))?; - - Ok(self - .run - .pkg_dep_graph() - .ancestors(&node) - .iter() - .filter(|package| !immediate_dependents.contains(*package)) - .map(|package| Package { - run: self.run.clone(), - name: package.as_package_name().clone(), - }) - .sorted_by(|a, b| a.name.cmp(&b.name)) - .collect()) - } - - /// The upstream packages that this package depends on, indirectly - async fn indirect_dependencies(&self) -> Result, Error> { - let node: PackageNode = PackageNode::Workspace(self.name.clone()); - let immediate_dependencies = self - .run - .pkg_dep_graph() - .immediate_dependencies(&node) - .ok_or_else(|| Error::PackageNotFound(self.name.clone()))?; - - Ok(self - .run - .pkg_dep_graph() - .dependencies(&node) - .iter() - .filter(|package| !immediate_dependencies.contains(*package)) - .map(|package| Package { - run: self.run.clone(), - name: package.as_package_name().clone(), - }) - .sorted_by(|a, b| a.name.cmp(&b.name)) - .collect()) - } -} - async fn graphiql() -> impl IntoResponse { response::Html(GraphiQLSource::build().endpoint("/").finish()) } diff --git a/crates/turborepo-lib/src/query/package.rs b/crates/turborepo-lib/src/query/package.rs new file mode 100644 index 0000000000000..789f0c5135ebc --- /dev/null +++ b/crates/turborepo-lib/src/query/package.rs @@ -0,0 +1,186 @@ +use std::sync::Arc; + +use async_graphql::Object; +use itertools::Itertools; +use turborepo_repository::package_graph::{PackageName, PackageNode}; + +use crate::{ + query::{Array, Error}, + run::Run, +}; + +pub struct Package { + pub run: Arc, + pub name: PackageName, +} + +impl Package { + pub fn direct_dependents_count(&self) -> usize { + self.run + .pkg_dep_graph() + .immediate_ancestors(&PackageNode::Workspace(self.name.clone())) + .map_or(0, |pkgs| pkgs.len()) + } + + pub fn direct_dependencies_count(&self) -> usize { + self.run + .pkg_dep_graph() + .immediate_dependencies(&PackageNode::Workspace(self.name.clone())) + .map_or(0, |pkgs| pkgs.len()) + } + + pub fn indirect_dependents_count(&self) -> usize { + let node: PackageNode = PackageNode::Workspace(self.name.clone()); + + self.run.pkg_dep_graph().ancestors(&node).len() - self.direct_dependents_count() + } + + pub fn indirect_dependencies_count(&self) -> usize { + let node: PackageNode = PackageNode::Workspace(self.name.clone()); + + self.run.pkg_dep_graph().dependencies(&node).len() - self.direct_dependencies_count() + } + + pub fn all_dependents_count(&self) -> usize { + self.run + .pkg_dep_graph() + .ancestors(&PackageNode::Workspace(self.name.clone())) + .len() + } + + pub fn all_dependencies_count(&self) -> usize { + self.run + .pkg_dep_graph() + .dependencies(&PackageNode::Workspace(self.name.clone())) + .len() + } +} + +#[Object] +impl Package { + /// The name of the package + async fn name(&self) -> String { + self.name.to_string() + } + + /// The path to the package, relative to the repository root + async fn path(&self) -> Result { + Ok(self + .run + .pkg_dep_graph() + .package_info(&self.name) + .ok_or_else(|| Error::PackageNotFound(self.name.clone()))? + .package_path() + .to_string()) + } + + /// The upstream packages that have this package as a direct dependency + async fn direct_dependents(&self) -> Result, Error> { + let node: PackageNode = PackageNode::Workspace(self.name.clone()); + Ok(self + .run + .pkg_dep_graph() + .immediate_ancestors(&node) + .iter() + .flatten() + .map(|package| Package { + run: self.run.clone(), + name: package.as_package_name().clone(), + }) + .sorted_by(|a, b| a.name.cmp(&b.name)) + .collect()) + } + + /// The downstream packages that directly depend on this package + async fn direct_dependencies(&self) -> Result, Error> { + let node: PackageNode = PackageNode::Workspace(self.name.clone()); + Ok(self + .run + .pkg_dep_graph() + .immediate_dependencies(&node) + .iter() + .flatten() + .map(|package| Package { + run: self.run.clone(), + name: package.as_package_name().clone(), + }) + .sorted_by(|a, b| a.name.cmp(&b.name)) + .collect()) + } + + async fn all_dependents(&self) -> Result, Error> { + let node: PackageNode = PackageNode::Workspace(self.name.clone()); + Ok(self + .run + .pkg_dep_graph() + .ancestors(&node) + .iter() + .map(|package| Package { + run: self.run.clone(), + name: package.as_package_name().clone(), + }) + .sorted_by(|a, b| a.name.cmp(&b.name)) + .collect()) + } + + async fn all_dependencies(&self) -> Result, Error> { + let node: PackageNode = PackageNode::Workspace(self.name.clone()); + Ok(self + .run + .pkg_dep_graph() + .dependencies(&node) + .iter() + .map(|package| Package { + run: self.run.clone(), + name: package.as_package_name().clone(), + }) + .sorted_by(|a, b| a.name.cmp(&b.name)) + .collect()) + } + + /// The downstream packages that depend on this package, indirectly + async fn indirect_dependents(&self) -> Result, Error> { + let node: PackageNode = PackageNode::Workspace(self.name.clone()); + let immediate_dependents = self + .run + .pkg_dep_graph() + .immediate_ancestors(&node) + .ok_or_else(|| Error::PackageNotFound(self.name.clone()))?; + + Ok(self + .run + .pkg_dep_graph() + .ancestors(&node) + .iter() + .filter(|package| !immediate_dependents.contains(*package)) + .map(|package| Package { + run: self.run.clone(), + name: package.as_package_name().clone(), + }) + .sorted_by(|a, b| a.name.cmp(&b.name)) + .collect()) + } + + /// The upstream packages that this package depends on, indirectly + async fn indirect_dependencies(&self) -> Result, Error> { + let node: PackageNode = PackageNode::Workspace(self.name.clone()); + let immediate_dependencies = self + .run + .pkg_dep_graph() + .immediate_dependencies(&node) + .ok_or_else(|| Error::PackageNotFound(self.name.clone()))?; + + Ok(self + .run + .pkg_dep_graph() + .dependencies(&node) + .iter() + .filter(|package| !immediate_dependencies.contains(*package)) + .map(|package| Package { + run: self.run.clone(), + name: package.as_package_name().clone(), + }) + .sorted_by(|a, b| a.name.cmp(&b.name)) + .collect()) + } +} diff --git a/turborepo-tests/integration/fixtures/turbo_trace/.gitignore b/turborepo-tests/integration/fixtures/turbo_trace/.gitignore new file mode 100644 index 0000000000000..96fab4fed3424 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace/.gitignore @@ -0,0 +1,38 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# Dependencies +node_modules +.pnp +.pnp.js + +# Local env files +.env +.env.local +.env.development.local +.env.test.local +.env.production.local + +# Testing +coverage + +# Turbo +.turbo + +# Vercel +.vercel + +# Build Outputs +.next/ +out/ +build +dist + + +# Debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Misc +.DS_Store +*.pem diff --git a/turborepo-tests/integration/fixtures/turbo_trace/README.md b/turborepo-tests/integration/fixtures/turbo_trace/README.md new file mode 100644 index 0000000000000..7a4658aa87a77 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace/README.md @@ -0,0 +1,81 @@ +# Turborepo starter + +This is an official starter Turborepo. + +## Using this example + +Run the following command: + +```sh +npx create-turbo@latest +``` + +## What's inside? + +This Turborepo includes the following packages/apps: + +### Apps and Packages + +- `docs`: a [Next.js](https://nextjs.org/) app +- `web`: another [Next.js](https://nextjs.org/) app +- `@repo/ui`: a stub React component library shared by both `web` and `docs` applications +- `@repo/eslint-config`: `eslint` configurations (includes `eslint-config-next` and `eslint-config-prettier`) +- `@repo/typescript-config`: `tsconfig.json`s used throughout the monorepo + +Each package/app is 100% [TypeScript](https://www.typescriptlang.org/). + +### Utilities + +This Turborepo has some additional tools already setup for you: + +- [TypeScript](https://www.typescriptlang.org/) for static type checking +- [ESLint](https://eslint.org/) for code linting +- [Prettier](https://prettier.io) for code formatting + +### Build + +To build all apps and packages, run the following command: + +``` +cd my-turborepo +pnpm build +``` + +### Develop + +To develop all apps and packages, run the following command: + +``` +cd my-turborepo +pnpm dev +``` + +### Remote Caching + +Turborepo can use a technique known as [Remote Caching](https://turbo.build/repo/docs/core-concepts/remote-caching) to share cache artifacts across machines, enabling you to share build caches with your team and CI/CD pipelines. + +By default, Turborepo will cache locally. To enable Remote Caching you will need an account with Vercel. If you don't have an account you can [create one](https://vercel.com/signup), then enter the following commands: + +``` +cd my-turborepo +npx turbo login +``` + +This will authenticate the Turborepo CLI with your [Vercel account](https://vercel.com/docs/concepts/personal-accounts/overview). + +Next, you can link your Turborepo to your Remote Cache by running the following command from the root of your Turborepo: + +``` +npx turbo link +``` + +## Useful Links + +Learn more about the power of Turborepo: + +- [Tasks](https://turbo.build/repo/docs/core-concepts/monorepos/running-tasks) +- [Caching](https://turbo.build/repo/docs/core-concepts/caching) +- [Remote Caching](https://turbo.build/repo/docs/core-concepts/remote-caching) +- [Filtering](https://turbo.build/repo/docs/core-concepts/monorepos/filtering) +- [Configuration Options](https://turbo.build/repo/docs/reference/configuration) +- [CLI Usage](https://turbo.build/repo/docs/reference/command-line-reference) diff --git a/turborepo-tests/integration/fixtures/turbo_trace/button.tsx b/turborepo-tests/integration/fixtures/turbo_trace/button.tsx new file mode 100644 index 0000000000000..0d23cb038dfae --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace/button.tsx @@ -0,0 +1,3 @@ +export const Button = ({ children }: { children: React.ReactNode }) => { + return ; +}; diff --git a/turborepo-tests/integration/fixtures/turbo_trace/circular.ts b/turborepo-tests/integration/fixtures/turbo_trace/circular.ts new file mode 100644 index 0000000000000..079562059f5a1 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace/circular.ts @@ -0,0 +1 @@ +import circular2 from "./circular2"; diff --git a/turborepo-tests/integration/fixtures/turbo_trace/circular2.ts b/turborepo-tests/integration/fixtures/turbo_trace/circular2.ts new file mode 100644 index 0000000000000..ba85bbe368f26 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace/circular2.ts @@ -0,0 +1 @@ +import circular from "./circular"; diff --git a/turborepo-tests/integration/fixtures/turbo_trace/foo.js b/turborepo-tests/integration/fixtures/turbo_trace/foo.js new file mode 100644 index 0000000000000..de9fc706c8115 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace/foo.js @@ -0,0 +1,6 @@ +export default function foo() { + if (!process.env.IS_CI) { + return "bar"; + } + return "foo"; +} diff --git a/turborepo-tests/integration/fixtures/turbo_trace/main.ts b/turborepo-tests/integration/fixtures/turbo_trace/main.ts new file mode 100644 index 0000000000000..815521f02dd8d --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace/main.ts @@ -0,0 +1,9 @@ +import { Button } from "./button"; +import foo from "./foo"; +import repeat from "repeat-string"; + +const button = new Button(); + +button.render(); +repeat("foo", 5); +foo(); diff --git a/turborepo-tests/integration/fixtures/turbo_trace/package.json b/turborepo-tests/integration/fixtures/turbo_trace/package.json new file mode 100644 index 0000000000000..3fb9d50ee0b59 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace/package.json @@ -0,0 +1,14 @@ +{ + "name": "create_turbo", + "private": true, + "engines": { + "node": ">=18" + }, + "dependencies": { + "repeat-string": "^1.6.1" + }, + "workspaces": [ + "apps/*", + "packages/*" + ] +} diff --git a/turborepo-tests/integration/fixtures/turbo_trace/turbo.json b/turborepo-tests/integration/fixtures/turbo_trace/turbo.json new file mode 100644 index 0000000000000..807e324753413 --- /dev/null +++ b/turborepo-tests/integration/fixtures/turbo_trace/turbo.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://turbo.build/schema.json", + "ui": "tui", + "tasks": { + "build": { + "dependsOn": ["^build"], + "inputs": ["$TURBO_DEFAULT$", ".env*"], + "outputs": [".next/**", "!.next/cache/**"] + }, + "lint": { + "dependsOn": ["^lint"] + }, + "dev": { + "cache": false, + "persistent": true + } + } +} diff --git a/turborepo-tests/integration/tests/turbo-trace.t b/turborepo-tests/integration/tests/turbo-trace.t new file mode 100644 index 0000000000000..c40b5a913eece --- /dev/null +++ b/turborepo-tests/integration/tests/turbo-trace.t @@ -0,0 +1,60 @@ +Setup + $ . ${TESTDIR}/../../helpers/setup_integration_test.sh turbo_trace + + $ ${TURBO} query "query { file(path: \"main.ts\") { path } }" + WARNING query command is experimental and may change in the future + { + "data": { + "file": { + "path": "main.ts" + } + } + } + + $ ${TURBO} query "query { file(path: \"main.ts\") { path, dependencies { path } } }" + WARNING query command is experimental and may change in the future + { + "data": { + "file": { + "path": "main.ts", + "dependencies": [ + { + "path": "button.tsx" + }, + { + "path": "foo.js" + }, + { + "path": "node_modules(\/|\\\\)repeat-string(\/|\\\\)index.js" (re) + } + ] + } + } + } + + $ ${TURBO} query "query { file(path: \"button.tsx\") { path, dependencies { path } } }" + WARNING query command is experimental and may change in the future + { + "data": { + "file": { + "path": "button.tsx", + "dependencies": [] + } + } + } + + $ ${TURBO} query "query { file(path: \"circular.ts\") { path, dependencies { path } } }" + WARNING query command is experimental and may change in the future + { + "data": { + "file": { + "path": "circular.ts", + "dependencies": [ + { + "path": "circular2.ts" + } + ] + } + } + } +