From d97bb5edbec9cd248160a3d8974f5deb4866c82d Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Tue, 29 Oct 2024 19:36:04 +0100 Subject: [PATCH] Add a cargo-dist-url-override key --- Cargo.lock | 1 + cargo-dist-schema/src/lib.rs | 38 ++++++++- cargo-dist/Cargo.toml | 1 + cargo-dist/src/backend/ci/github.rs | 51 +++++++----- cargo-dist/src/backend/ci/mod.rs | 123 ++++++++++++++++++---------- cargo-dist/src/config/v0.rs | 19 ++++- cargo-dist/src/config/v0_to_v1.rs | 2 + cargo-dist/src/config/v1/mod.rs | 15 ++++ cargo-dist/src/init.rs | 9 ++ 9 files changed, 194 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b37bb53be..276e4705f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -415,6 +415,7 @@ dependencies = [ "miette 7.2.0", "minijinja", "newline-converter", + "schemars", "semver", "serde", "serde_json", diff --git a/cargo-dist-schema/src/lib.rs b/cargo-dist-schema/src/lib.rs index 6cc355099..fbcc068df 100644 --- a/cargo-dist-schema/src/lib.rs +++ b/cargo-dist-schema/src/lib.rs @@ -302,8 +302,7 @@ pub struct GithubMatrixEntry { #[serde(skip_serializing_if = "Option::is_none")] pub runner: Option, /// Expression to execute to install dist - #[serde(skip_serializing_if = "Option::is_none")] - pub install_dist: Option, + pub install_dist: GhaRunStep, /// Arguments to pass to dist #[serde(skip_serializing_if = "Option::is_none")] pub dist_args: Option, @@ -315,6 +314,41 @@ pub struct GithubMatrixEntry { pub cache_provider: Option, } +/// A GitHub Actions "run" step, either bash or powershell +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +// this mirrors GHA's structure, see +// * +// * +#[serde(tag = "shell", content = "run")] +pub enum GhaRunStep { + /// see [`DashScript`] + #[serde(rename = "sh")] + Dash(DashScript), + /// see [`PowershellScript`] + #[serde(rename = "pwsh")] + Powershell(PowershellScript), +} + +impl From for GhaRunStep { + fn from(bash: DashScript) -> Self { + Self::Dash(bash) + } +} + +impl From for GhaRunStep { + fn from(powershell: PowershellScript) -> Self { + Self::Powershell(powershell) + } +} + +declare_strongly_typed_string! { + /// A bit of shell script (that can run with `/bin/sh`), ran on CI runners. Can be multi-line. + pub struct DashScript => &DashScriptRef; + + /// A bit of powershell script, ran on CI runners. Can be multi-line. + pub struct PowershellScript => &PowershellScriptRef; +} + /// Type of job to run on pull request #[derive( Debug, Copy, Clone, Serialize, Deserialize, JsonSchema, Default, PartialEq, Eq, PartialOrd, Ord, diff --git a/cargo-dist/Cargo.toml b/cargo-dist/Cargo.toml index f42f52b9a..9ce65e7ce 100644 --- a/cargo-dist/Cargo.toml +++ b/cargo-dist/Cargo.toml @@ -70,6 +70,7 @@ spdx.workspace = true base64.workspace = true lazy_static.workspace = true current_platform.workspace = true +schemars.workspace = true [dev-dependencies] insta.workspace = true diff --git a/cargo-dist/src/backend/ci/github.rs b/cargo-dist/src/backend/ci/github.rs index 0d27ebdab..ddafc3e53 100644 --- a/cargo-dist/src/backend/ci/github.rs +++ b/cargo-dist/src/backend/ci/github.rs @@ -8,7 +8,8 @@ use axoasset::{LocalAsset, SourceFile}; use axoprocess::Cmd; use camino::{Utf8Path, Utf8PathBuf}; use cargo_dist_schema::{ - GithubMatrix, GithubMatrixEntry, GithubRunner, GithubRunnerRef, TargetTriple, TargetTripleRef, + GhaRunStep, GithubMatrix, GithubMatrixEntry, GithubRunner, GithubRunnerRef, TargetTriple, + TargetTripleRef, }; use serde::{Deserialize, Serialize}; use tracing::warn; @@ -24,6 +25,8 @@ use crate::{ DistError, DistGraph, SortedMap, SortedSet, }; +use super::{DistInstallSettings, DistInstallStrategy}; + #[cfg(not(windows))] const GITHUB_CI_DIR: &str = ".github/workflows/"; #[cfg(windows)] @@ -31,6 +34,8 @@ const GITHUB_CI_DIR: &str = r".github\workflows\"; const GITHUB_CI_FILE: &str = "release.yml"; /// Info about running dist in Github CI +/// +/// THESE FIELDS ARE LOAD-BEARING because they're used in the templates. #[derive(Debug, Serialize)] pub struct GithubCiInfo { /// Cached path to github CI workflows dir @@ -38,10 +43,12 @@ pub struct GithubCiInfo { pub github_ci_workflow_dir: Utf8PathBuf, /// Version of rust toolchain to install (deprecated) pub rust_version: Option, - /// expression to use for installing dist via shell script - pub install_dist_sh: String, - /// expression to use for installing dist via powershell script - pub install_dist_ps1: String, + + // TODO: do something so the templates can actually use this to generate + // shell/powershell — or just pregenerate it. + // + /// Our install strategy for dist itself + pub dist_install_strategy: DistInstallStrategy, /// Whether to fail-fast pub fail_fast: bool, /// Whether to cache builds @@ -216,8 +223,12 @@ impl GithubCiInfo { } // Get the platform-specific installation methods - let install_dist_sh = super::install_dist_sh_for_version(dist_version); - let install_dist_ps1 = super::install_dist_ps1_for_version(dist_version); + let dist_install_strategy = (DistInstallSettings { + version: dist_version, + url_override: dist.config.dist_url_override.as_deref(), + }) + .install_strategy(); + let hosting_providers = dist .hosting .as_ref() @@ -245,7 +256,7 @@ impl GithubCiInfo { cache_provider: cache_provider_for_runner(global_runner), runner: Some(global_runner.to_owned()), dist_args: Some("--artifacts=global".into()), - install_dist: Some(install_dist_sh.clone()), + install_dist: dist_install_strategy.bash().into(), packages_install: None, }; @@ -300,8 +311,7 @@ impl GithubCiInfo { }; for (runner, targets) in local_runs { use std::fmt::Write; - let install_dist = - install_dist_for_targets(&targets, &install_dist_sh, &install_dist_ps1); + let install_dist = install_dist_for_targets(&targets, &dist_install_strategy); let mut dist_args = String::from("--artifacts=local"); for target in &targets { write!(dist_args, " --target={target}").unwrap(); @@ -311,7 +321,7 @@ impl GithubCiInfo { cache_provider: cache_provider_for_runner(&runner), runner: Some(runner), dist_args: Some(dist_args), - install_dist: Some(install_dist.to_owned()), + install_dist: install_dist.to_owned(), packages_install: package_install_for_targets(&targets, &dependencies), }); } @@ -334,8 +344,7 @@ impl GithubCiInfo { github_ci_workflow_dir, tag_namespace, rust_version, - install_dist_sh, - install_dist_ps1, + dist_install_strategy, fail_fast, cache_builds, build_local_artifacts, @@ -621,18 +630,20 @@ fn github_runner_for_target( /// Select the dist installer approach for a given Github Runner fn install_dist_for_targets<'a>( targets: &'a [&'a TargetTripleRef], - install_sh: &'a str, - install_ps1: &'a str, -) -> &'a str { + install_strategy: &'a DistInstallStrategy, +) -> GhaRunStep { + // FIXME: when doing cross-compilation, `host != runner`, and we should + // pick something that runs on the host. We need knowledge about "which + // platform is this GitHub runner" ahead of time to do that, though. + for target in targets { if target.is_linux() || target.is_apple() { - return install_sh; + return install_strategy.bash().into(); } else if target.is_windows() { - return install_ps1; + return install_strategy.powershell().into(); } } - - unreachable!("internal error: unknown target triple!?") + panic!("unsupported target triple(s) {targets:?}"); } fn brewfile_from(packages: &[String]) -> String { diff --git a/cargo-dist/src/backend/ci/mod.rs b/cargo-dist/src/backend/ci/mod.rs index ecc93c22d..9535fba7f 100644 --- a/cargo-dist/src/backend/ci/mod.rs +++ b/cargo-dist/src/backend/ci/mod.rs @@ -1,6 +1,10 @@ //! Support for generating CI scripts for running dist +use cargo_dist_schema::{DashScript, PowershellScript}; use semver::Version; +use serde::Serialize; + +use crate::config::v0::CargoDistUrlOverrideRef; use self::github::GithubCiInfo; @@ -17,52 +21,87 @@ pub struct CiInfo { pub github: Option, } -/// Get the command to invoke to install dist via sh script -fn install_dist_sh_for_version(version: &Version) -> String { - if let Some(git) = install_dist_git(version) { - return git; - } - let format = cargo_dist_schema::format_of_version(version); - let installer_name = if format.unsupported() { - // FIXME: we should probably do this check way higher up and produce a proper err... - panic!("requested dist v{version}, which is not supported by the this copy of dist ({SELF_DIST_VERSION})"); - } else if format.artifact_names_contain_versions() { - format!("cargo-dist-v{version}-installer.sh") - } else { - "cargo-dist-installer.sh".to_owned() - }; +/// Gives us the full information re: the version of dist we're supposed +/// to install/run in CI +struct DistInstallSettings<'a> { + version: &'a Version, + url_override: Option<&'a CargoDistUrlOverrideRef>, +} - // FIXME: it would be nice if these values were somehow using all the machinery - // to compute these values for packages we build *BUT* it's messy and not that important - let installer_url = format!("{BASE_DIST_FETCH_URL}/v{version}/{installer_name}"); - format!("curl --proto '=https' --tlsv1.2 -LsSf {installer_url} | sh") +/// The strategy used to install dist in CI +#[derive(Debug, Clone, Serialize)] +pub enum DistInstallStrategy { + /// Download an installer and run it + Installer { + /// the prefix of the installer url, e.g. + installer_url: String, + /// the installer name, without `.sh` or `.ps1` + installer_name: String, + }, + /// Run `cargo install --git` (slow!) + GitBranch { + /// the branch to install from — from + branch: String, + }, } -/// Get the command to invoke to install dist via ps1 script -fn install_dist_ps1_for_version(version: &Version) -> String { - if let Some(git) = install_dist_git(version) { - return git; - } - let format = cargo_dist_schema::format_of_version(version); - let installer_name = if format.unsupported() { - // FIXME: we should probably do this check way higher up and produce a proper err... - panic!("requested dist v{version}, which is not supported by the this copy of dist ({SELF_DIST_VERSION})"); - } else if format.artifact_names_contain_versions() { - format!("cargo-dist-v{version}-installer.ps1") - } else { - "cargo-dist-installer.ps1".to_owned() - }; +impl DistInstallSettings<'_> { + fn install_strategy(&self) -> DistInstallStrategy { + if let Some(branch) = self.version.pre.strip_prefix("github-") { + return DistInstallStrategy::GitBranch { + branch: branch.to_owned(), + }; + } + + if let Some(url) = self.url_override.as_ref() { + return DistInstallStrategy::Installer { + installer_url: url.as_str().to_owned(), + installer_name: "cargo-dist-installer".to_owned(), + }; + } - // FIXME: it would be nice if these values were somehow using all the machinery - // to compute these values for packages we build *BUT* it's messy and not that important - let installer_url = format!("{BASE_DIST_FETCH_URL}/v{version}/{installer_name}"); - format!(r#"powershell -c "irm {installer_url} | iex""#) + let version = self.version; + let format = cargo_dist_schema::format_of_version(version); + let installer_name = if format.unsupported() { + // FIXME: we should probably do this check way higher up and produce a proper err... + panic!("requested dist v{version}, which is not supported by the this copy of dist ({SELF_DIST_VERSION})"); + } else if format.artifact_names_contain_versions() { + format!("cargo-dist-v{version}-installer") + } else { + "cargo-dist-installer".to_owned() + }; + + DistInstallStrategy::Installer { + // FIXME: it would be nice if these values were somehow using all the machinery + // to compute these values for packages we build *BUT* it's messy and not that important + installer_url: format!("{BASE_DIST_FETCH_URL}/v{version}"), + installer_name, + } + } } -/// Cute little hack for developing dist itself: if we see a version like "0.0.3-github-config" -/// then install from the main github repo with branch=config! -fn install_dist_git(version: &Version) -> Option { - version.pre.strip_prefix("github-").map(|branch| { - format!("cargo install --git https://github.com/axodotdev/cargo-dist/ --branch={branch} cargo-dist") - }) +impl DistInstallStrategy { + /// Returns a bit of powershell to install the requested version of dist + fn powershell(&self) -> PowershellScript { + PowershellScript::new(match self { + DistInstallStrategy::Installer { installer_url, installer_name } => format!( + "irm {installer_url}/{installer_name}.ps1 | iex" + ), + DistInstallStrategy::GitBranch { branch } => format!( + "cargo install --git https://github.com/axodotdev/cargo-dist/ --branch={branch} cargo-dist" + ), + }) + } + + /// Returns a bit of bash to install the requested version of dist + pub fn bash(&self) -> DashScript { + DashScript::new(match self { + DistInstallStrategy::Installer { installer_url, installer_name } => format!( + "curl --proto '=https' --tlsv1.2 -LsSf {installer_url}/{installer_name}.sh | sh" + ), + DistInstallStrategy::GitBranch { branch } => format!( + "cargo install --git https://github.com/axodotdev/cargo-dist/ --branch={branch} cargo-dist" + ), + }) + } } diff --git a/cargo-dist/src/config/v0.rs b/cargo-dist/src/config/v0.rs index a853c6428..57c9c3ca8 100644 --- a/cargo-dist/src/config/v0.rs +++ b/cargo-dist/src/config/v0.rs @@ -1,7 +1,7 @@ //! v0 config use camino::{Utf8Path, Utf8PathBuf}; -use cargo_dist_schema::GithubRunner; +use cargo_dist_schema::{declare_strongly_typed_string, GithubRunner}; use semver::Version; use serde::{Deserialize, Serialize}; use tracing::log::warn; @@ -17,6 +17,14 @@ pub struct GenericConfig { pub dist: Option, } +declare_strongly_typed_string! { + /// A URL to use to install `cargo-dist` (with the installer script). + /// This overwrites `cargo_dist_version` and expects the URL to have + /// a similar structure to `./target/distrib` after running `dist build` + /// on itself. + pub struct CargoDistUrlOverride => &CargoDistUrlOverrideRef; +} + /// Contents of METADATA_DIST in Cargo.toml files #[derive(Serialize, Deserialize, Debug, Default, Clone)] #[serde(rename_all = "kebab-case")] @@ -32,6 +40,10 @@ pub struct DistMetadata { #[serde(skip_serializing_if = "Option::is_none")] pub cargo_dist_version: Option, + /// See [`CargoDistUrlOverride`] + #[serde(skip_serializing_if = "Option::is_none")] + pub cargo_dist_url_override: Option, + /// (deprecated) The intended version of Rust/Cargo to build with (rustup toolchain syntax) /// /// When generating full tasks graphs (such as CI scripts) we will pick this version. @@ -453,6 +465,7 @@ impl DistMetadata { extra_artifacts, // The rest of these don't include relative paths cargo_dist_version: _, + cargo_dist_url_override: _, rust_toolchain_version: _, dist: _, ci: _, @@ -548,6 +561,7 @@ impl DistMetadata { // This is intentionally written awkwardly to make you update it let DistMetadata { cargo_dist_version, + cargo_dist_url_override, rust_toolchain_version, dist, ci, @@ -614,6 +628,9 @@ impl DistMetadata { if cargo_dist_version.is_some() { warn!("package.metadata.dist.cargo-dist-version is set, but this is only accepted in workspace.metadata (value is being ignored): {}", package_manifest_path); } + if cargo_dist_url_override.is_some() { + warn!("package.metadata.dist.cargo-dist-url-override is set, but this is only accepted in workspace.metadata (value is being ignored): {}", package_manifest_path); + } if rust_toolchain_version.is_some() { warn!("package.metadata.dist.rust-toolchain-version is set, but this is only accepted in workspace.metadata (value is being ignored): {}", package_manifest_path); } diff --git a/cargo-dist/src/config/v0_to_v1.rs b/cargo-dist/src/config/v0_to_v1.rs index ba590957f..c19aceb81 100644 --- a/cargo-dist/src/config/v0_to_v1.rs +++ b/cargo-dist/src/config/v0_to_v1.rs @@ -23,6 +23,7 @@ impl DistMetadata { pub fn to_toml_layer(&self, is_global: bool) -> TomlLayer { let DistMetadata { cargo_dist_version, + cargo_dist_url_override, rust_toolchain_version, dist, ci, @@ -341,6 +342,7 @@ impl DistMetadata { TomlLayer { dist_version: cargo_dist_version, + dist_url_override: cargo_dist_url_override, dist, allow_dirty, targets, diff --git a/cargo-dist/src/config/v1/mod.rs b/cargo-dist/src/config/v1/mod.rs index 7158aaeff..9c5a8b968 100644 --- a/cargo-dist/src/config/v1/mod.rs +++ b/cargo-dist/src/config/v1/mod.rs @@ -114,6 +114,7 @@ pub mod publishers; use axoproject::{PackageIdx, WorkspaceGraph}; use semver::Version; +use v0::CargoDistUrlOverride; use super::*; use layer::*; @@ -161,6 +162,8 @@ pub fn app_config( pub struct WorkspaceConfig { /// The intended version of dist to build with. (normal Cargo SemVer syntax) pub dist_version: Option, + /// See [`CargoDistUrlOverride`] + pub dist_url_override: Option, /// Generate targets whose dist should avoid checking for up-to-dateness. pub allow_dirty: Vec, /// ci config @@ -181,6 +184,8 @@ pub struct WorkspaceConfig { pub struct WorkspaceConfigInheritable { /// The intended version of dist to build with. (normal Cargo SemVer syntax) pub dist_version: Option, + /// See [`CargoDistUrlOverride`] + pub dist_url_override: Option, /// Generate targets whose dist should avoid checking for up-to-dateness. pub allow_dirty: Vec, /// artifact config @@ -204,6 +209,7 @@ impl WorkspaceConfigInheritable { builds: BuildConfigInheritable::defaults_for_workspace(workspaces), installers: InstallerConfigInheritable::defaults_for_workspace(workspaces), dist_version: None, + dist_url_override: None, allow_dirty: vec![], } } @@ -216,6 +222,7 @@ impl WorkspaceConfigInheritable { builds, installers, dist_version, + dist_url_override, allow_dirty, } = self; WorkspaceConfig { @@ -225,6 +232,7 @@ impl WorkspaceConfigInheritable { builds: builds.apply_inheritance_for_workspace(workspaces), installers: installers.apply_inheritance_for_workspace(workspaces), dist_version, + dist_url_override, allow_dirty, } } @@ -241,6 +249,7 @@ impl ApplyLayer for WorkspaceConfigInheritable { ci, allow_dirty, dist_version, + dist_url_override, // app-scope only dist: _, targets: _, @@ -253,6 +262,7 @@ impl ApplyLayer for WorkspaceConfigInheritable { self.installers.apply_val_layer(installers); self.ci.apply_val_layer(ci); self.dist_version.apply_opt(dist_version); + self.dist_url_override.apply_opt(dist_url_override); self.allow_dirty.apply_val(allow_dirty); } } @@ -350,6 +360,7 @@ impl ApplyLayer for AppConfigInheritable { ci: _, allow_dirty: _, dist_version: _, + dist_url_override: _, }: Self::Layer, ) { self.artifacts.apply_val_layer(artifacts); @@ -377,6 +388,10 @@ pub struct TomlLayer { #[serde(skip_serializing_if = "Option::is_none")] pub dist_version: Option, + /// see [`CargoDistUrlOverride`] + #[serde(skip_serializing_if = "Option::is_none")] + pub dist_url_override: Option, + /// Whether the package should be distributed/built by dist /// /// This mainly exists to be set to `false` to make dist ignore the existence of this diff --git a/cargo-dist/src/init.rs b/cargo-dist/src/init.rs index d0a595f86..3da51eabb 100644 --- a/cargo-dist/src/init.rs +++ b/cargo-dist/src/init.rs @@ -359,6 +359,7 @@ fn get_new_dist_metadata( DistMetadata { // If they init with this version we're gonna try to stick to it! cargo_dist_version: Some(std::env!("CARGO_PKG_VERSION").parse().unwrap()), + cargo_dist_url_override: None, // deprecated, default to not emitting it rust_toolchain_version: None, ci: None, @@ -840,6 +841,7 @@ fn apply_dist_to_metadata(metadata: &mut toml_edit::Item, meta: &DistMetadata) { // This is intentionally written awkwardly to make you update this let DistMetadata { cargo_dist_version, + cargo_dist_url_override, rust_toolchain_version, dist, ci, @@ -926,6 +928,13 @@ fn apply_dist_to_metadata(metadata: &mut toml_edit::Item, meta: &DistMetadata) { cargo_dist_version.as_ref().map(|v| v.to_string()), ); + apply_optional_value( + table, + "cargo-dist-url-override", + "# A URL to use to install `cargo-dist` (with the installer script)\n", + cargo_dist_url_override.as_ref().map(|v| v.to_string()), + ); + apply_optional_value( table, "rust-toolchain-version",