From 42507ebbc80a68674b01e62c3daafb7b1f92a106 Mon Sep 17 00:00:00 2001
From: Rain <rain@oxide.computer>
Date: Mon, 29 Jul 2024 16:15:50 -0700
Subject: [PATCH] [xtask] move downloader to a separate binary (#6171)

While iterating on xtask I noticed that it could be pretty slow to
compile. Turned out that a lot of the dependencies were actually coming from the
download tool.

Move the downloader into a separate binary.

On Linux, this results in the number of build units for xtask dropping
from 254 to 122. That's a pretty nice improvement.
---
 Cargo.lock                                    | 25 +++++++++++++------
 Cargo.toml                                    |  2 ++
 dev-tools/downloader/Cargo.toml               | 24 ++++++++++++++++++
 .../src/download.rs => downloader/src/lib.rs} |  3 +++
 dev-tools/downloader/src/main.rs              | 12 +++++++++
 dev-tools/xtask/Cargo.toml                    | 25 +++++++++++--------
 dev-tools/xtask/src/main.rs                   |  8 +++---
 7 files changed, 75 insertions(+), 24 deletions(-)
 create mode 100644 dev-tools/downloader/Cargo.toml
 rename dev-tools/{xtask/src/download.rs => downloader/src/lib.rs} (99%)
 create mode 100644 dev-tools/downloader/src/main.rs

diff --git a/Cargo.lock b/Cargo.lock
index 75952fd71c..c233ab94b9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -11626,24 +11626,33 @@ dependencies = [
  "cargo_metadata",
  "cargo_toml",
  "clap",
- "flate2",
  "fs-err",
- "futures",
  "macaddr",
- "reqwest",
  "serde",
+ "swrite",
+ "tabled",
+ "toml 0.8.15",
+ "usdt",
+]
+
+[[package]]
+name = "xtask-downloader"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "camino",
+ "clap",
+ "flate2",
+ "futures",
+ "omicron-workspace-hack",
+ "reqwest",
  "sha2",
  "slog",
  "slog-async",
- "slog-bunyan",
  "slog-term",
  "strum",
- "swrite",
- "tabled",
  "tar",
  "tokio",
- "toml 0.8.15",
- "usdt",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 12bea694f6..49317fd24a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -20,6 +20,7 @@ members = [
     "cockroach-admin/types",
     "common",
     "dev-tools/crdb-seed",
+    "dev-tools/downloader",
     "dev-tools/omdb",
     "dev-tools/omicron-dev",
     "dev-tools/openapi-manager",
@@ -122,6 +123,7 @@ default-members = [
     "cockroach-admin/types",
     "common",
     "dev-tools/crdb-seed",
+    "dev-tools/downloader",
     "dev-tools/omdb",
     "dev-tools/omicron-dev",
     "dev-tools/openapi-manager",
diff --git a/dev-tools/downloader/Cargo.toml b/dev-tools/downloader/Cargo.toml
new file mode 100644
index 0000000000..9f64a2a6e0
--- /dev/null
+++ b/dev-tools/downloader/Cargo.toml
@@ -0,0 +1,24 @@
+[package]
+name = "xtask-downloader"
+version = "0.1.0"
+edition = "2021"
+license = "MPL-2.0"
+
+[lints]
+workspace = true
+
+[dependencies]
+anyhow.workspace = true
+camino.workspace = true
+clap.workspace = true
+flate2.workspace = true
+futures.workspace = true
+omicron-workspace-hack.workspace = true
+reqwest.workspace = true
+sha2.workspace = true
+slog.workspace = true
+slog-async.workspace = true
+slog-term.workspace = true
+strum.workspace = true
+tar.workspace = true
+tokio.workspace = true
diff --git a/dev-tools/xtask/src/download.rs b/dev-tools/downloader/src/lib.rs
similarity index 99%
rename from dev-tools/xtask/src/download.rs
rename to dev-tools/downloader/src/lib.rs
index b5910e3915..d5b436244c 100644
--- a/dev-tools/xtask/src/download.rs
+++ b/dev-tools/downloader/src/lib.rs
@@ -3,6 +3,9 @@
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 //! Subcommand: cargo xtask download
+//!
+//! This is a separate binary because it requires many dependencies that other
+//! parts of `cargo xtask` do not.
 
 use anyhow::{bail, Context, Result};
 use camino::{Utf8Path, Utf8PathBuf};
diff --git a/dev-tools/downloader/src/main.rs b/dev-tools/downloader/src/main.rs
new file mode 100644
index 0000000000..eacddd476e
--- /dev/null
+++ b/dev-tools/downloader/src/main.rs
@@ -0,0 +1,12 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use clap::Parser;
+use xtask_downloader::{run_cmd, DownloadArgs};
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    let args = DownloadArgs::parse();
+    run_cmd(args).await
+}
diff --git a/dev-tools/xtask/Cargo.toml b/dev-tools/xtask/Cargo.toml
index acf225cd9d..ec1b7825c6 100644
--- a/dev-tools/xtask/Cargo.toml
+++ b/dev-tools/xtask/Cargo.toml
@@ -8,26 +8,29 @@ license = "MPL-2.0"
 workspace = true
 
 [dependencies]
+# Please read before adding a new dependency!
+#
+# cargo xtask is used for many developer flows, and it is important for devx
+# that it be quick to build. If a particular task needs many dependencies,
+# consider moving it out into a binary crate and building that separately.
+# Additional binaries generally mean that you only pay for what you need.
+#
+# For example, xtask-downloader is a separate binary because it requires many
+# additional dependencies, like `reqwest`, `tar`, and `tokio`.
+#
+# The combinatorial explosion of features that would ordinarily happen is not
+# an issue for omicron due to the workspace-hack crate. (For build speed
+# reasons, `xtask` does *not* depend on `omicron-workspace-hack`. But the
+# downstream binaries do depend on it.)
 anyhow.workspace = true
 camino.workspace = true
 cargo_toml = "0.20"
 cargo_metadata.workspace = true
 clap.workspace = true
-flate2.workspace = true
-futures.workspace = true
 fs-err.workspace = true
 macaddr.workspace = true
-reqwest = { workspace = true, features = [ "default-tls" ] }
 serde.workspace = true
-sha2.workspace = true
-slog.workspace = true
-slog-async.workspace = true
-slog-bunyan.workspace = true
-slog-term.workspace = true
-strum.workspace = true
 swrite.workspace = true
 tabled.workspace = true
-tar.workspace = true
-tokio = { workspace = true, features = ["full"] }
 toml.workspace = true
 usdt.workspace = true
diff --git a/dev-tools/xtask/src/main.rs b/dev-tools/xtask/src/main.rs
index 0ea2332c31..1d9dfd0b6c 100644
--- a/dev-tools/xtask/src/main.rs
+++ b/dev-tools/xtask/src/main.rs
@@ -13,7 +13,6 @@ use clap::{Parser, Subcommand};
 mod check_features;
 mod check_workspace_deps;
 mod clippy;
-mod download;
 #[cfg_attr(not(target_os = "illumos"), allow(dead_code))]
 mod external;
 mod usdt;
@@ -47,7 +46,7 @@ enum Cmds {
     /// Run configured clippy checks
     Clippy(clippy::ClippyArgs),
     /// Download binaries, OpenAPI specs, and other out-of-repo utilities.
-    Download(download::DownloadArgs),
+    Download(external::External),
 
     /// Manage OpenAPI specifications.
     ///
@@ -86,8 +85,7 @@ enum Cmds {
     },
 }
 
-#[tokio::main]
-async fn main() -> Result<()> {
+fn main() -> Result<()> {
     let args = Args::parse();
     match args.cmd {
         Cmds::Argon2(external) => {
@@ -96,7 +94,7 @@ async fn main() -> Result<()> {
         Cmds::Clippy(args) => clippy::run_cmd(args),
         Cmds::CheckFeatures(args) => check_features::run_cmd(args),
         Cmds::CheckWorkspaceDeps => check_workspace_deps::run_cmd(),
-        Cmds::Download(args) => download::run_cmd(args).await,
+        Cmds::Download(external) => external.exec_bin("xtask-downloader"),
         Cmds::Openapi(external) => external.exec_bin("openapi-manager"),
         #[cfg(target_os = "illumos")]
         Cmds::Releng(external) => {