feat: install native packages in CI

This injects a new CI step, prior to package build, which installs packages specified
via a new field in Cargo.toml.

cargo-dist-schema/src/lib.rs

@@ -115,6 +115,9 @@ pub struct GithubMatrixEntry {
     /// Arguments to pass to cargo-dist
     #[serde(skip_serializing_if = "Option::is_none")]
     pub dist_args: Option<String>,
+    /// Command to run to install dependencies
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub packages_install: Option<String>,
 }
 
 /// Type of job to run on pull request

cargo-dist-schema/src/snapshots/cargo_dist_schema__emit.snap

@@ -403,6 +403,13 @@ expression: json_schema
             "null"
           ]
         },
+        "packages_install": {
+          "description": "Command to run to install dependencies",
+          "type": [
+            "string",
+            "null"
+          ]
+        },
         "runner": {
           "description": "Github Runner to user",
           "type": [

cargo-dist/src/backend/ci/github.rs

@@ -9,7 +9,7 @@ use tracing::warn;
 
 use crate::{
     backend::{diff_files, templates::TEMPLATE_CI_GITHUB},
-    config::ProductionMode,
+    config::{ProductionMode, SystemDependencies},
     errors::DistResult,
     DistGraph, SortedMap, SortedSet, TargetTriple,
 };
@@ -85,6 +85,7 @@ impl GithubCiInfo {
                 runner: Some(GITHUB_LINUX_RUNNER.into()),
                 dist_args: Some("--artifacts=global".into()),
                 install_dist: Some(install_dist_sh.clone()),
+                packages_install: None,
             })
         } else {
             None
@@ -106,13 +107,14 @@ impl GithubCiInfo {
             let install_dist =
                 install_dist_for_github_runner(runner, &install_dist_sh, &install_dist_ps1);
             let mut dist_args = String::from("--artifacts=local");
-            for target in targets {
+            for target in &targets {
                 write!(dist_args, " --target={target}").unwrap();
             }
             tasks.push(GithubMatrixEntry {
                 runner: Some(runner.to_owned()),
                 dist_args: Some(dist_args),
                 install_dist: Some(install_dist.to_owned()),
+                packages_install: package_install_for_targets(&targets, &dist.system_dependencies),
             });
         }
 
@@ -252,3 +254,60 @@ fn install_dist_for_github_runner<'a>(
         unreachable!("internal error: unknown github runner!?")
     }
 }
+
+fn brewfile_from(packages: &[String]) -> String {
+    let brewfile_lines: Vec<String> = packages
+        .iter()
+        .map(|p| format!(r#"brew "{p}""#).to_owned())
+        .collect();
+
+    brewfile_lines.join("\n")
+}
+
+fn brew_bundle_command(packages: &[String]) -> String {
+    format!(
+        r#"cat << EOF >Brewfile
+{}
+EOF
+
+brew bundle install"#,
+        brewfile_from(packages)
+    )
+}
+
+fn package_install_for_targets(
+    targets: &Vec<&TargetTriple>,
+    packages: &SystemDependencies,
+) -> Option<String> {
+    // TODO handle mixed-OS targets
+    for target in targets {
+        match target.as_str() {
+            "i686-apple-darwin" | "x86_64-apple-darwin" | "aarch64-apple-darwin" => {
+                if packages.homebrew.is_empty() {
+                    return None;
+                }
+
+                return Some(brew_bundle_command(&packages.homebrew));
+            }
+            "i686-unknown-linux-gnu" | "x86_64-unknown-linux-gnu" | "aarch64-unknown-linux-gnu" => {
+                if packages.apt.is_empty() {
+                    return None;
+                }
+
+                let apts = packages.apt.join(" ");
+                return Some(format!("sudo apt-get install {apts}").to_owned());
+            }
+            "i686-pc-windows-msvc" | "x86_64-pc-windows-msvc" | "aarch64-pc-windows-msvc" => {
+                if packages.chocolatey.is_empty() {
+                    return None;
+                }
+
+                let chocos = packages.chocolatey.join(" ");
+                return Some(format!("choco install {chocos}").to_owned());
+            }
+            _ => {}
+        }
+    }
+
+    None
+}

cargo-dist/src/config.rs

@@ -89,6 +89,11 @@ pub struct DistMetadata {
     /// A Homebrew tap to push the Homebrew formula to, if built
     pub tap: Option<String>,
 
+    /// A set of packages to install before building
+    #[serde(rename = "system-dependencies")]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub system_dependencies: Option<SystemDependencies>,
+
     /// The full set of target triples to build for.
     ///
     /// When generating full task graphs (such as CI scripts) we will to try to generate these.
@@ -279,6 +284,7 @@ impl DistMetadata {
             ci: _,
             installers: _,
             tap: _,
+            system_dependencies: _,
             targets: _,
             include,
             auto_includes: _,
@@ -321,6 +327,7 @@ impl DistMetadata {
             ci,
             installers,
             tap,
+            system_dependencies,
             targets,
             include,
             auto_includes,
@@ -420,6 +427,9 @@ impl DistMetadata {
         if tap.is_none() {
             *tap = workspace_config.tap.clone();
         }
+        if system_dependencies.is_none() {
+            *system_dependencies = workspace_config.system_dependencies.clone();
+        }
         if publish_jobs.is_none() {
             *publish_jobs = workspace_config.publish_jobs.clone();
         }
@@ -786,6 +796,23 @@ impl std::fmt::Display for GenerateMode {
     }
 }
 
+/// Packages to install before build from the system package manager
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
+pub struct SystemDependencies {
+    /// Packages to install in Homebrew
+    #[serde(default = "Vec::new")]
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub homebrew: Vec<String>,
+    /// Packages to install in apt
+    #[serde(default = "Vec::new")]
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub apt: Vec<String>,
+    /// Package to install in Chocolatey
+    #[serde(default = "Vec::new")]
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub chocolatey: Vec<String>,
+}
+
 /// Settings for which Generate targets can be dirty
 #[derive(Debug, Clone)]
 pub enum DirtyMode {

cargo-dist/src/init.rs

@@ -205,6 +205,7 @@ fn get_new_dist_metadata(
             ci: None,
             installers: None,
             tap: None,
+            system_dependencies: None,
             targets: cfg.targets.is_empty().not().then(|| cfg.targets.clone()),
             dist: None,
             include: None,
@@ -666,6 +667,7 @@ fn apply_dist_to_metadata(metadata: &mut toml_edit::Item, meta: &DistMetadata) {
         ci,
         installers,
         tap,
+        system_dependencies: _,
         targets,
         include,
         auto_includes,

cargo-dist/src/tasks.rs

@@ -62,7 +62,7 @@ use tracing::{info, warn};
 
 use crate::backend::ci::github::GithubCiInfo;
 use crate::backend::ci::CiInfo;
-use crate::config::{DirtyMode, ProductionMode};
+use crate::config::{DirtyMode, ProductionMode, SystemDependencies};
 use crate::{
     backend::{
         installer::{
@@ -205,6 +205,8 @@ pub struct DistGraph {
     pub publish_prereleases: bool,
     /// A GitHub repo to publish the Homebrew formula to
     pub tap: Option<String>,
+    /// A list of packages to install as dependencies
+    pub system_dependencies: SystemDependencies,
 }
 
 /// Various tools we have found installed on the system
@@ -638,6 +640,8 @@ impl<'pkg_graph> DistGraphBuilder<'pkg_graph> {
             // Only the final value merged into a package_config matters
             tap: _,
             // Only the final value merged into a package_config matters
+            system_dependencies: _,
+            // Only the final value merged into a package_config matters
             windows_archive: _,
             // Only the final value merged into a package_config matters
             unix_archive: _,
@@ -714,6 +718,16 @@ impl<'pkg_graph> DistGraphBuilder<'pkg_graph> {
             DirtyMode::AllowList(allow_dirty.clone().unwrap_or(vec![]))
         };
 
+        let system_dependencies =
+            workspace_metadata
+                .system_dependencies
+                .clone()
+                .unwrap_or(SystemDependencies {
+                    homebrew: vec![],
+                    apt: vec![],
+                    chocolatey: vec![],
+                });
+
         Ok(Self {
             inner: DistGraph {
                 is_init: dist_profile.is_some(),
@@ -744,6 +758,7 @@ impl<'pkg_graph> DistGraphBuilder<'pkg_graph> {
                 ci: CiInfo::default(),
                 pr_run_mode: workspace_metadata.pr_run_mode.unwrap_or_default(),
                 tap: workspace_metadata.tap.clone(),
+                system_dependencies,
                 publish_jobs,
                 publish_prereleases,
                 allow_dirty,

cargo-dist/templates/ci/github_ci.yml.j2

@@ -115,15 +115,24 @@ jobs:
       - uses: swatinem/rust-cache@v2
       - name: Install cargo-dist
         run: ${{ matrix.install_dist }}
+      - name: Install dependencies
+        run: |
+          ${{ matrix.packages_install }}
       - id: cargo-dist
+        name: Build
         # We force bash here just because github makes it really hard to get values up
         # to "real" actions without writing to env-vars, and writing to env-vars has
         # inconsistent syntax between shell and powershell. cargo-dist and jq work fine
         # in powershell.
         shell: bash
         run: |
+          CARGO="cargo"
+          if [[ -f "Brewfile" ]]; then
+            CARGO="brew bundle exec -- cargo"
+          fi
+
           # Actually do builds and make zips and whatnot
-          cargo dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
+          $CARGO dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
           echo "cargo dist ran successfully"
 
           # Parse out what we just built and upload it to the Github Release™

cargo-dist/tests/snapshots/akaikatana_basic.snap

@@ -1374,15 +1374,24 @@ jobs:
       - uses: swatinem/rust-cache@v2
       - name: Install cargo-dist
         run: ${{ matrix.install_dist }}
+      - name: Install dependencies
+        run: |
+          ${{ matrix.packages_install }}
       - id: cargo-dist
+        name: Build
         # We force bash here just because github makes it really hard to get values up
         # to "real" actions without writing to env-vars, and writing to env-vars has
         # inconsistent syntax between shell and powershell. cargo-dist and jq work fine
         # in powershell.
         shell: bash
         run: |
+          CARGO="cargo"
+          if [[ -f "Brewfile" ]]; then
+            CARGO="brew bundle exec -- cargo"
+          fi
+
           # Actually do builds and make zips and whatnot
-          cargo dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
+          $CARGO dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
           echo "cargo dist ran successfully"
 
           # Parse out what we just built and upload it to the Github Release™

cargo-dist/tests/snapshots/akaikatana_repo_with_dot_git.snap

@@ -1374,15 +1374,24 @@ jobs:
       - uses: swatinem/rust-cache@v2
       - name: Install cargo-dist
         run: ${{ matrix.install_dist }}
+      - name: Install dependencies
+        run: |
+          ${{ matrix.packages_install }}
       - id: cargo-dist
+        name: Build
         # We force bash here just because github makes it really hard to get values up
         # to "real" actions without writing to env-vars, and writing to env-vars has
         # inconsistent syntax between shell and powershell. cargo-dist and jq work fine
         # in powershell.
         shell: bash
         run: |
+          CARGO="cargo"
+          if [[ -f "Brewfile" ]]; then
+            CARGO="brew bundle exec -- cargo"
+          fi
+
           # Actually do builds and make zips and whatnot
-          cargo dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
+          $CARGO dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
           echo "cargo dist ran successfully"
 
           # Parse out what we just built and upload it to the Github Release™

cargo-dist/tests/snapshots/axolotlsay_basic.snap

@@ -2269,15 +2269,24 @@ jobs:
       - uses: swatinem/rust-cache@v2
       - name: Install cargo-dist
         run: ${{ matrix.install_dist }}
+      - name: Install dependencies
+        run: |
+          ${{ matrix.packages_install }}
       - id: cargo-dist
+        name: Build
         # We force bash here just because github makes it really hard to get values up
         # to "real" actions without writing to env-vars, and writing to env-vars has
         # inconsistent syntax between shell and powershell. cargo-dist and jq work fine
         # in powershell.
         shell: bash
         run: |
+          CARGO="cargo"
+          if [[ -f "Brewfile" ]]; then
+            CARGO="brew bundle exec -- cargo"
+          fi
+
           # Actually do builds and make zips and whatnot
-          cargo dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
+          $CARGO dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
           echo "cargo dist ran successfully"
 
           # Parse out what we just built and upload it to the Github Release™

cargo-dist/tests/snapshots/axolotlsay_edit_existing.snap

@@ -2244,15 +2244,24 @@ jobs:
       - uses: swatinem/rust-cache@v2
       - name: Install cargo-dist
         run: ${{ matrix.install_dist }}
+      - name: Install dependencies
+        run: |
+          ${{ matrix.packages_install }}
       - id: cargo-dist
+        name: Build
         # We force bash here just because github makes it really hard to get values up
         # to "real" actions without writing to env-vars, and writing to env-vars has
         # inconsistent syntax between shell and powershell. cargo-dist and jq work fine
         # in powershell.
         shell: bash
         run: |
+          CARGO="cargo"
+          if [[ -f "Brewfile" ]]; then
+            CARGO="brew bundle exec -- cargo"
+          fi
+
           # Actually do builds and make zips and whatnot
-          cargo dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
+          $CARGO dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
           echo "cargo dist ran successfully"
 
           # Parse out what we just built and upload it to the Github Release™