[xtask] move downloader to a separate binary (#6171)

While iterating on xtask I noticed that it could be pretty slow to
compile. Turned out that a lot of the dependencies were actually coming from the
download tool.

Move the downloader into a separate binary.

On Linux, this results in the number of build units for xtask dropping
from 254 to 122. That's a pretty nice improvement.

Cargo.toml

@@ -20,6 +20,7 @@ members = [
     "cockroach-admin/types",
     "common",
     "dev-tools/crdb-seed",
+    "dev-tools/downloader",
     "dev-tools/omdb",
     "dev-tools/omicron-dev",
     "dev-tools/openapi-manager",
@@ -122,6 +123,7 @@ default-members = [
     "cockroach-admin/types",
     "common",
     "dev-tools/crdb-seed",
+    "dev-tools/downloader",
     "dev-tools/omdb",
     "dev-tools/omicron-dev",
     "dev-tools/openapi-manager",

dev-tools/downloader/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "xtask-downloader"
+version = "0.1.0"
+edition = "2021"
+license = "MPL-2.0"
+
+[lints]
+workspace = true
+
+[dependencies]
+anyhow.workspace = true
+camino.workspace = true
+clap.workspace = true
+flate2.workspace = true
+futures.workspace = true
+omicron-workspace-hack.workspace = true
+reqwest.workspace = true
+sha2.workspace = true
+slog.workspace = true
+slog-async.workspace = true
+slog-term.workspace = true
+strum.workspace = true
+tar.workspace = true
+tokio.workspace = true

dev-tools/xtask/src/download.rs → dev-tools/downloader/src/lib.rs

@@ -3,6 +3,9 @@
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 //! Subcommand: cargo xtask download
+//!
+//! This is a separate binary because it requires many dependencies that other
+//! parts of `cargo xtask` do not.
 
 use anyhow::{bail, Context, Result};
 use camino::{Utf8Path, Utf8PathBuf};

dev-tools/downloader/src/main.rs

@@ -0,0 +1,12 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use clap::Parser;
+use xtask_downloader::{run_cmd, DownloadArgs};
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    let args = DownloadArgs::parse();
+    run_cmd(args).await
+}

dev-tools/xtask/Cargo.toml

@@ -8,26 +8,29 @@ license = "MPL-2.0"
 workspace = true
 
 [dependencies]
+# Please read before adding a new dependency!
+#
+# cargo xtask is used for many developer flows, and it is important for devx
+# that it be quick to build. If a particular task needs many dependencies,
+# consider moving it out into a binary crate and building that separately.
+# Additional binaries generally mean that you only pay for what you need.
+#
+# For example, xtask-downloader is a separate binary because it requires many
+# additional dependencies, like `reqwest`, `tar`, and `tokio`.
+#
+# The combinatorial explosion of features that would ordinarily happen is not
+# an issue for omicron due to the workspace-hack crate. (For build speed
+# reasons, `xtask` does *not* depend on `omicron-workspace-hack`. But the
+# downstream binaries do depend on it.)
 anyhow.workspace = true
 camino.workspace = true
 cargo_toml = "0.20"
 cargo_metadata.workspace = true
 clap.workspace = true
-flate2.workspace = true
-futures.workspace = true
 fs-err.workspace = true
 macaddr.workspace = true
-reqwest = { workspace = true, features = [ "default-tls" ] }
 serde.workspace = true
-sha2.workspace = true
-slog.workspace = true
-slog-async.workspace = true
-slog-bunyan.workspace = true
-slog-term.workspace = true
-strum.workspace = true
 swrite.workspace = true
 tabled.workspace = true
-tar.workspace = true
-tokio = { workspace = true, features = ["full"] }
 toml.workspace = true
 usdt.workspace = true

dev-tools/xtask/src/main.rs

@@ -13,7 +13,6 @@ use clap::{Parser, Subcommand};
 mod check_features;
 mod check_workspace_deps;
 mod clippy;
-mod download;
 #[cfg_attr(not(target_os = "illumos"), allow(dead_code))]
 mod external;
 mod usdt;
@@ -47,7 +46,7 @@ enum Cmds {
     /// Run configured clippy checks
     Clippy(clippy::ClippyArgs),
     /// Download binaries, OpenAPI specs, and other out-of-repo utilities.
-    Download(download::DownloadArgs),
+    Download(external::External),
 
     /// Manage OpenAPI specifications.
     ///
@@ -86,8 +85,7 @@ enum Cmds {
     },
 }
 
-#[tokio::main]
-async fn main() -> Result<()> {
+fn main() -> Result<()> {
     let args = Args::parse();
     match args.cmd {
         Cmds::Argon2(external) => {
@@ -96,7 +94,7 @@ async fn main() -> Result<()> {
         Cmds::Clippy(args) => clippy::run_cmd(args),
         Cmds::CheckFeatures(args) => check_features::run_cmd(args),
         Cmds::CheckWorkspaceDeps => check_workspace_deps::run_cmd(),
-        Cmds::Download(args) => download::run_cmd(args).await,
+        Cmds::Download(external) => external.exec_bin("xtask-downloader"),
         Cmds::Openapi(external) => external.exec_bin("openapi-manager"),
         #[cfg(target_os = "illumos")]
         Cmds::Releng(external) => {