From 4b89e21b2c0e9b24207e3666cbae490934fe0d13 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sun, 28 Jan 2024 10:14:19 -0500 Subject: [PATCH] Decode sdists async --- Cargo.lock | 395 ++++++++++++++++++- crates/puffin-distribution/src/source/mod.rs | 64 +-- crates/puffin-extract/Cargo.toml | 2 + crates/puffin-extract/src/lib.rs | 23 +- requirements.in | 3 + 5 files changed, 450 insertions(+), 37 deletions(-) create mode 100644 requirements.in diff --git a/Cargo.lock b/Cargo.lock index d9c52f8c23d1..30b39b2f93c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -178,6 +178,30 @@ dependencies = [ "tempfile", ] +[[package]] +name = "async-channel" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener 2.5.3", + "futures-core", +] + +[[package]] +name = "async-channel" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ca33f4bc4ed1babef42cad36cc1f51fa88be00420404e5b1e80ab1b18f7678c" +dependencies = [ + "concurrent-queue", + "event-listener 4.0.3", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-compression" version = "0.4.6" @@ -193,6 +217,140 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-executor" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17ae5ebefcc48e7452b4987947920dac9450be1110cadf34d1b8c116bdbaf97c" +dependencies = [ + "async-lock 3.3.0", + "async-task", + "concurrent-queue", + "fastrand 2.0.1", + "futures-lite 2.2.0", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" +dependencies = [ + "async-channel 2.1.1", + "async-executor", + "async-io 2.3.0", + "async-lock 3.3.0", + "blocking", + "futures-lite 2.2.0", + "once_cell", +] + +[[package]] +name = "async-io" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" +dependencies = [ + "async-lock 2.8.0", + "autocfg", + "cfg-if 1.0.0", + "concurrent-queue", + "futures-lite 1.13.0", + "log", + "parking", + "polling 2.8.0", + "rustix 0.37.27", + "slab", + "socket2 0.4.10", + "waker-fn", +] + +[[package]] +name = "async-io" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb41eb19024a91746eba0773aa5e16036045bbf45733766661099e182ea6a744" +dependencies = [ + "async-lock 3.3.0", + "cfg-if 1.0.0", + "concurrent-queue", + "futures-io", + "futures-lite 2.2.0", + "parking", + "polling 3.3.2", + "rustix 0.38.30", + "slab", + "tracing", + "windows-sys 0.52.0", +] + +[[package]] +name = "async-lock" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" +dependencies = [ + "event-listener 2.5.3", +] + +[[package]] +name = "async-lock" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" +dependencies = [ + "event-listener 4.0.3", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-std" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" +dependencies = [ + "async-channel 1.9.0", + "async-global-executor", + "async-io 1.13.0", + "async-lock 2.8.0", + "crossbeam-utils", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite 1.13.0", + "gloo-timers", + "kv-log-macro", + "log", + "memchr", + "once_cell", + "pin-project-lite", + "pin-utils", + "slab", + "wasm-bindgen-futures", +] + +[[package]] +name = "async-tar" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c49359998a76e32ef6e870dbc079ebad8f1e53e8441c5dd39d27b44493fe331" +dependencies = [ + "async-std", + "filetime", + "libc", + "pin-project", + "redox_syscall 0.2.16", + "xattr 0.2.3", +] + +[[package]] +name = "async-task" +version = "4.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbb36e985947064623dbd357f727af08ffd077f93d696782f3c56365fa2e2799" + [[package]] name = "async-trait" version = "0.1.77" @@ -229,13 +387,19 @@ source = "git+https://github.com/charliermarsh/rs-async-zip?rev=d76801da0943de98 dependencies = [ "async-compression", "crc32fast", - "futures-lite", + "futures-lite 2.2.0", "pin-project", "thiserror", "tokio", "tokio-util", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.1.0" @@ -326,6 +490,22 @@ dependencies = [ "generic-array", ] +[[package]] +name = "blocking" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" +dependencies = [ + "async-channel 2.1.1", + "async-lock 3.3.0", + "async-task", + "fastrand 2.0.1", + "futures-io", + "futures-lite 2.2.0", + "piper", + "tracing", +] + [[package]] name = "brotli" version = "3.4.0" @@ -582,6 +762,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "concurrent-queue" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "configparser" version = "3.0.4" @@ -928,6 +1117,42 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "event-listener" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" +dependencies = [ + "event-listener 4.0.3", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + [[package]] name = "fastrand" version = "2.0.1" @@ -1061,13 +1286,28 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +[[package]] +name = "futures-lite" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +dependencies = [ + "fastrand 1.9.0", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + [[package]] name = "futures-lite" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "445ba825b27408685aaecefd65178908c36c6e96aaf6d8599419d46e624192ba" dependencies = [ - "fastrand", + "fastrand 2.0.1", "futures-core", "futures-io", "parking", @@ -1189,6 +1429,18 @@ dependencies = [ "walkdir", ] +[[package]] +name = "gloo-timers" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "goblin" version = "0.8.0" @@ -1385,7 +1637,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.5.5", "tokio", "tower-service", "tracing", @@ -1574,6 +1826,17 @@ dependencies = [ "web-sys", ] +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "ipnet" version = "2.9.0" @@ -1587,7 +1850,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" dependencies = [ "hermit-abi", - "rustix", + "rustix 0.38.30", "windows-sys 0.52.0", ] @@ -1658,6 +1921,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -1756,6 +2028,12 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + [[package]] name = "linux-raw-sys" version = "0.4.13" @@ -1777,6 +2055,9 @@ name = "log" version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +dependencies = [ + "value-bag", +] [[package]] name = "mailparse" @@ -2199,6 +2480,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "piper" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" +dependencies = [ + "atomic-waker", + "fastrand 2.0.1", + "futures-io", +] + [[package]] name = "pkg-config" version = "0.3.29" @@ -2260,6 +2552,36 @@ dependencies = [ "time", ] +[[package]] +name = "polling" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" +dependencies = [ + "autocfg", + "bitflags 1.3.2", + "cfg-if 1.0.0", + "concurrent-queue", + "libc", + "log", + "pin-project-lite", + "windows-sys 0.48.0", +] + +[[package]] +name = "polling" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "545c980a3880efd47b2e262f6a4bb6daad6555cf3367aa9c4e52895f69537a41" +dependencies = [ + "cfg-if 1.0.0", + "concurrent-queue", + "pin-project-lite", + "rustix 0.38.30", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "portable-atomic" version = "1.6.0" @@ -2632,6 +2954,8 @@ dependencies = [ name = "puffin-extract" version = "0.0.1" dependencies = [ + "async-compression", + "async-tar", "async_zip", "flate2", "fs-err", @@ -3069,7 +3393,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "767be24c0da52e7448d495b8d162506a9aa125426651d547d545d6c2b4b65b62" dependencies = [ "cfg-if 1.0.0", - "rustix", + "rustix 0.38.30", "windows", ] @@ -3322,6 +3646,20 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustix" +version = "0.37.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys 0.3.8", + "windows-sys 0.48.0", +] + [[package]] name = "rustix" version = "0.38.30" @@ -3331,7 +3669,7 @@ dependencies = [ "bitflags 2.4.2", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.4.13", "windows-sys 0.52.0", ] @@ -3560,6 +3898,16 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" +[[package]] +name = "socket2" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "socket2" version = "0.5.5" @@ -3679,7 +4027,7 @@ checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" dependencies = [ "filetime", "libc", - "xattr", + "xattr 1.3.1", ] [[package]] @@ -3704,9 +4052,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" dependencies = [ "cfg-if 1.0.0", - "fastrand", + "fastrand 2.0.1", "redox_syscall 0.4.1", - "rustix", + "rustix 0.38.30", "windows-sys 0.52.0", ] @@ -3911,7 +4259,7 @@ dependencies = [ "num_cpus", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.5.5", "tokio-macros", "windows-sys 0.48.0", ] @@ -4216,6 +4564,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "value-bag" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cdbaf5e132e593e9fc1de6a15bbec912395b11fb9719e061cf64f804524c503" + [[package]] name = "vcpkg" version = "0.2.15" @@ -4280,6 +4634,12 @@ dependencies = [ "smallvec", ] +[[package]] +name = "waker-fn" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" + [[package]] name = "walkdir" version = "2.4.0" @@ -4424,7 +4784,7 @@ dependencies = [ "either", "home", "once_cell", - "rustix", + "rustix 0.38.30", "windows-sys 0.48.0", ] @@ -4638,6 +4998,15 @@ dependencies = [ "tap", ] +[[package]] +name = "xattr" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc" +dependencies = [ + "libc", +] + [[package]] name = "xattr" version = "1.3.1" @@ -4645,8 +5014,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" dependencies = [ "libc", - "linux-raw-sys", - "rustix", + "linux-raw-sys 0.4.13", + "rustix 0.38.30", ] [[package]] diff --git a/crates/puffin-distribution/src/source/mod.rs b/crates/puffin-distribution/src/source/mod.rs index d427db914069..8baf2b68d729 100644 --- a/crates/puffin-distribution/src/source/mod.rs +++ b/crates/puffin-distribution/src/source/mod.rs @@ -750,29 +750,47 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> { return Ok(cache_path); } - // Download the source distribution to a temporary file. - // TODO(charlie): Unzip as we download, as with wheels. - let span = - info_span!("download_source_dist", filename = filename, source_dist = %source_dist); - let download_dir = self.download_source_dist_url(response, filename).await?; - drop(span); - - // Unzip the source distribution to a temporary directory. - let span = - info_span!("extract_source_dist", filename = filename, source_dist = %source_dist); - let source_dist_dir = puffin_extract::extract_source( - download_dir.path().join(filename), - download_dir.path().join("extracted"), - )?; - drop(span); - - // Persist the unzipped distribution to the cache. - fs_err::tokio::create_dir_all(cache_path.parent().expect("Cache entry to have parent")) - .await - .map_err(Error::CacheWrite)?; - fs_err::tokio::rename(&source_dist_dir, &cache_path) - .await - .map_err(Error::CacheWrite)?; + // Download the source distribution to a temporary file. If it's a zip archive, we can unzip + // it directly into the cache. + if filename.ends_with(".zip") { + // Unzip the source distribution to a temporary directory. + let span = info_span!("download_unzip_source_dist", filename = filename, source_dist = %source_dist); + let temp_dir = tempfile::tempdir_in(self.build_context.cache().root()) + .map_err(Error::CacheWrite)?; + let reader = response + .bytes_stream() + .map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err)) + .into_async_read(); + puffin_extract::unzip_no_seek(reader.compat(), temp_dir.path()).await?; + drop(span); + + // Persist the unzipped distribution to the cache. + fs_err::tokio::create_dir_all(cache_path.parent().expect("Cache entry to have parent")) + .await + .map_err(Error::CacheWrite)?; + fs_err::tokio::rename(temp_dir.path(), &cache_path) + .await + .map_err(Error::CacheWrite)?; + } else { + // Unzip the source distribution to a temporary directory. + let span = info_span!("download_unzip_source_dist", filename = filename, source_dist = %source_dist); + let temp_dir = tempfile::tempdir_in(self.build_context.cache().root()) + .map_err(Error::CacheWrite)?; + let reader = response + .bytes_stream() + .map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err)) + .into_async_read(); + let extracted = puffin_extract::untar_no_seek(reader, temp_dir.path()).await?; + drop(span); + + // Persist the unzipped distribution to the cache. + fs_err::tokio::create_dir_all(cache_path.parent().expect("Cache entry to have parent")) + .await + .map_err(Error::CacheWrite)?; + fs_err::tokio::rename(extracted, &cache_path) + .await + .map_err(Error::CacheWrite)?; + } Ok(cache_path) } diff --git a/crates/puffin-extract/Cargo.toml b/crates/puffin-extract/Cargo.toml index d034f102b279..4ad46713d814 100644 --- a/crates/puffin-extract/Cargo.toml +++ b/crates/puffin-extract/Cargo.toml @@ -13,6 +13,8 @@ license = { workspace = true } workspace = true [dependencies] +async-compression = { version = "0.4.6", features = ["gzip"] } +async-tar = "0.4.2" async_zip = { workspace = true, features = ["tokio"] } flate2 = { workspace = true } fs-err = { workspace = true, features = ["tokio"] } diff --git a/crates/puffin-extract/src/lib.rs b/crates/puffin-extract/src/lib.rs index b5ecaff2373c..9e6e54dc67b6 100644 --- a/crates/puffin-extract/src/lib.rs +++ b/crates/puffin-extract/src/lib.rs @@ -23,7 +23,7 @@ pub enum Error { #[error("Unsupported archive type: {0}")] UnsupportedArchive(PathBuf), #[error( - "The top level of the archive must only contain a list directory, but it contains: {0:?}" + "The top level of the archive must only contain a list directory, but it contains: {0:?}" )] InvalidArchive(Vec), } @@ -221,3 +221,24 @@ pub fn extract_source( Ok(root.path()) } + + +pub async fn untar_no_seek( + reader: R, + target: &Path, +) -> Result { + let decompressed_bytes = async_compression::futures::bufread::GzipDecoder::new(reader); + let archive = async_tar::Archive::new(decompressed_bytes); + archive.unpack(target).await?; + + // > A .tar.gz source distribution (sdist) contains a single top-level directory called + // > `{name}-{version}` (e.g. foo-1.0), containing the source files of the package. + // TODO(konstin): Verify the name of the directory. + let top_level = + fs_err::read_dir(target)?.collect::>>()?; + let [root] = top_level.as_slice() else { + return Err(Error::InvalidArchive(top_level)); + }; + + Ok(root.path()) +} \ No newline at end of file diff --git a/requirements.in b/requirements.in new file mode 100644 index 000000000000..e6289cf6276d --- /dev/null +++ b/requirements.in @@ -0,0 +1,3 @@ +django @ https://files.pythonhosted.org/packages/53/82/c8e8ed137da1c72fa110e3be9ab0f26bcfcf6f3d2994601d164dfac86269/Django-5.0.1.tar.gz +flask @ https://files.pythonhosted.org/packages/b2/14/97b9137a02f57d2287f3a9731b3a339fda716d2d3a157d7d1d89c2bebf7b/flask-3.0.1.tar.gz +markupsafe @ https://files.pythonhosted.org/packages/fb/5a/fb1326fe32913e663c8e2d6bdf7cde6f472e51f9c21f0768d9b9080fe7c5/MarkupSafe-2.1.4.tar.gz