diff --git a/.gitignore b/.gitignore index 0871431f4..de1bcf0a2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.snap.new /book/book/ cargo-dist/wix +/dist-manifest-schema.json # Oranda oranda-debug.log diff --git a/Cargo.toml b/Cargo.toml index ef27039ee..fdfd078ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,10 @@ targets = ["x86_64-unknown-linux-gnu", "aarch64-apple-darwin", "x86_64-apple-dar # Publish jobs to run in CI pr-run-mode = "plan" +[[workspace.metadata.dist.extra-artifacts]] +artifacts = ["dist-manifest-schema.json"] +build = ["cargo", "run", "--", "dist", "manifest-schema", "--output=dist-manifest-schema.json"] + # The profile that 'cargo dist' will build with [profile.dist] inherits = "release" diff --git a/cargo-dist-schema/src/lib.rs b/cargo-dist-schema/src/lib.rs index 5ab3c3ed7..44642d75b 100644 --- a/cargo-dist-schema/src/lib.rs +++ b/cargo-dist-schema/src/lib.rs @@ -302,6 +302,9 @@ pub enum ArtifactKind { /// A tarball containing the source code #[serde(rename = "source-tarball")] SourceTarball, + /// Some form of extra artifact produced by a sidecar build + #[serde(rename = "extra-artifact")] + ExtraArtifact, /// Unknown to this version of cargo-dist-schema /// /// This is a fallback for forward/backward-compat diff --git a/cargo-dist-schema/src/snapshots/cargo_dist_schema__emit.snap b/cargo-dist-schema/src/snapshots/cargo_dist_schema__emit.snap index 19fccbdba..41151b7bf 100644 --- a/cargo-dist-schema/src/snapshots/cargo_dist_schema__emit.snap +++ b/cargo-dist-schema/src/snapshots/cargo_dist_schema__emit.snap @@ -180,6 +180,21 @@ expression: json_schema } } }, + { + "description": "Some form of extra artifact produced by a sidecar build", + "type": "object", + "required": [ + "kind" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "extra-artifact" + ] + } + } + }, { "description": "Unknown to this version of cargo-dist-schema\n\nThis is a fallback for forward/backward-compat", "type": "object", diff --git a/cargo-dist/src/cli.rs b/cargo-dist/src/cli.rs index 176dd838e..fba8685e7 100644 --- a/cargo-dist/src/cli.rs +++ b/cargo-dist/src/cli.rs @@ -374,7 +374,11 @@ pub enum OutputFormat { } #[derive(Args, Clone, Debug)] -pub struct ManifestSchemaArgs {} +pub struct ManifestSchemaArgs { + /// Write the manifest schema to the named file instead of stdout + #[clap(long)] + pub output: Option, +} #[derive(Args, Clone, Debug)] pub struct HostArgs { diff --git a/cargo-dist/src/config.rs b/cargo-dist/src/config.rs index b950be6f8..61c3de361 100644 --- a/cargo-dist/src/config.rs +++ b/cargo-dist/src/config.rs @@ -272,6 +272,10 @@ pub struct DistMetadata { /// Hosting provider #[serde(skip_serializing_if = "Option::is_none")] pub hosting: Option>, + + /// Any extra artifacts and their buildscripts + #[serde(skip_serializing_if = "Option::is_none")] + pub extra_artifacts: Option>, } impl DistMetadata { @@ -308,6 +312,7 @@ impl DistMetadata { ssldotcom_windows_sign: _, msvc_crt_static: _, hosting: _, + extra_artifacts: _, } = self; if let Some(include) = include { for include in include { @@ -353,6 +358,7 @@ impl DistMetadata { ssldotcom_windows_sign, msvc_crt_static, hosting, + extra_artifacts, } = self; // Check for global settings on local packages @@ -444,6 +450,9 @@ impl DistMetadata { if publish_jobs.is_none() { *publish_jobs = workspace_config.publish_jobs.clone(); } + if extra_artifacts.is_none() { + *extra_artifacts = workspace_config.extra_artifacts.clone(); + } // This was historically implemented as extend, but I'm not convinced the // inconsistency is worth the inconvenience... @@ -1034,6 +1043,16 @@ pub enum ProductionMode { Prod, } +/// An extra artifact to upload alongside the release tarballs, +/// and the build command which produces it. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ExtraArtifact { + /// The build command to invoke + pub build: Vec, + /// The artifact(s) produced via this build script + pub artifacts: Vec, +} + impl std::fmt::Display for ProductionMode { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { diff --git a/cargo-dist/src/generic_build.rs b/cargo-dist/src/generic_build.rs index a4814a416..ce87d7d74 100644 --- a/cargo-dist/src/generic_build.rs +++ b/cargo-dist/src/generic_build.rs @@ -3,7 +3,7 @@ use std::{ env, io::{stderr, Write}, - process::Command, + process::{Command, Output}, }; use camino::Utf8Path; @@ -14,7 +14,8 @@ use crate::{ copy_file, env::{calculate_cflags, calculate_ldflags, fetch_brew_env, parse_env, select_brew_env}, errors::Result, - BinaryIdx, BuildStep, DistGraph, DistGraphBuilder, GenericBuildStep, SortedMap, TargetTriple, + BinaryIdx, BuildStep, DistGraph, DistGraphBuilder, ExtraBuildStep, GenericBuildStep, SortedMap, + TargetTriple, }; impl<'a> DistGraphBuilder<'a> { @@ -72,14 +73,12 @@ fn platform_appropriate_cxx(target: &str) -> &str { } } -/// Build a generic target -pub fn build_generic_target(dist_graph: &DistGraph, target: &GenericBuildStep) -> Result<()> { - let mut command_string = target.build_command.clone(); - eprintln!( - "building generic target ({} via {})", - target.target_triple, - command_string.join(" ") - ); +fn run_build( + dist_graph: &DistGraph, + command_string: &[String], + target: Option<&str>, +) -> Result { + let mut command_string = command_string.to_owned(); let mut desired_extra_env = vec![]; let mut cflags = None; @@ -109,16 +108,16 @@ pub fn build_generic_target(dist_graph: &DistGraph, target: &GenericBuildStep) - // inject into the environment, apply them now. command.envs(desired_extra_env); - // Ensure we inform the build what architecture and platform - // it's building for. - command.env("CARGO_DIST_TARGET", &target.target_triple); + if let Some(target) = target { + // Ensure we inform the build what architecture and platform + // it's building for. + command.env("CARGO_DIST_TARGET", target); - let cc = - std::env::var("CC").unwrap_or(platform_appropriate_cc(&target.target_triple).to_owned()); - command.env("CC", cc); - let cxx = - std::env::var("CXX").unwrap_or(platform_appropriate_cxx(&target.target_triple).to_owned()); - command.env("CXX", cxx); + let cc = std::env::var("CC").unwrap_or(platform_appropriate_cc(target).to_owned()); + command.env("CC", cc); + let cxx = std::env::var("CXX").unwrap_or(platform_appropriate_cxx(target).to_owned()); + command.env("CXX", cxx); + } // Pass CFLAGS/LDFLAGS for C builds if let Some(cflags) = cflags { @@ -133,10 +132,25 @@ pub fn build_generic_target(dist_graph: &DistGraph, target: &GenericBuildStep) - } info!("exec: {:?}", command); - let result = command + command .output() .into_diagnostic() - .wrap_err_with(|| format!("failed to exec generic build: {command:?}"))?; + .wrap_err_with(|| format!("failed to exec generic build: {command:?}")) +} + +/// Build a generic target +pub fn build_generic_target(dist_graph: &DistGraph, target: &GenericBuildStep) -> Result<()> { + eprintln!( + "building generic target ({} via {})", + target.target_triple, + target.build_command.join(" ") + ); + + let result = run_build( + dist_graph, + &target.build_command, + Some(&target.target_triple), + )?; if !result.status.success() { println!("Build exited non-zero: {}", result.status); @@ -163,3 +177,37 @@ pub fn build_generic_target(dist_graph: &DistGraph, target: &GenericBuildStep) - Ok(()) } + +/// Similar to the above, but with slightly different signatures since +/// it's not based around axoproject-identified binaries +pub fn run_extra_artifacts_build(dist_graph: &DistGraph, target: &ExtraBuildStep) -> Result<()> { + eprintln!( + "building extra artifacts target (via {})", + target.build_command.join(" ") + ); + + let result = run_build(dist_graph, &target.build_command, None)?; + let dest = dist_graph.dist_dir.to_owned(); + + if !result.status.success() { + println!("Build exited non-zero: {}", result.status); + } + eprintln!(); + eprintln!("stdout:"); + stderr().write_all(&result.stdout).into_diagnostic()?; + + // Check that we got everything we expected, and copy into the distribution path + for artifact in &target.expected_artifacts { + let binary_path = Utf8Path::new(artifact); + if binary_path.exists() { + copy_file(binary_path, &dest.join(artifact))?; + } else { + return Err(miette!( + "failed to find bin {} -- did the build above have errors?", + binary_path + )); + } + } + + Ok(()) +} diff --git a/cargo-dist/src/init.rs b/cargo-dist/src/init.rs index ca06c2ed7..b79cc97fb 100644 --- a/cargo-dist/src/init.rs +++ b/cargo-dist/src/init.rs @@ -251,6 +251,7 @@ fn get_new_dist_metadata( ssldotcom_windows_sign: None, msvc_crt_static: None, hosting: None, + extra_artifacts: None, } }; @@ -770,6 +771,7 @@ fn apply_dist_to_metadata(metadata: &mut toml_edit::Item, meta: &DistMetadata) { ssldotcom_windows_sign, msvc_crt_static, hosting, + extra_artifacts: _, } = &meta; apply_optional_value( diff --git a/cargo-dist/src/lib.rs b/cargo-dist/src/lib.rs index 1db3a09ee..77a3e5af7 100644 --- a/cargo-dist/src/lib.rs +++ b/cargo-dist/src/lib.rs @@ -25,7 +25,7 @@ use config::{ ArtifactMode, ChecksumStyle, CompressionImpl, Config, DirtyMode, GenerateMode, ZipStyle, }; use console::Term; -use generic_build::build_generic_target; +use generic_build::{build_generic_target, run_extra_artifacts_build}; use semver::Version; use tracing::info; @@ -138,6 +138,7 @@ fn run_build_step( prefix, target, }) => Ok(generate_source_tarball(committish, prefix, target)?), + BuildStep::Extra(target) => run_extra_artifacts_build(dist_graph, target), } } diff --git a/cargo-dist/src/main.rs b/cargo-dist/src/main.rs index 43cd203a4..46f8395c3 100644 --- a/cargo-dist/src/main.rs +++ b/cargo-dist/src/main.rs @@ -4,6 +4,7 @@ use std::io::Write; +use axoasset::LocalAsset; use camino::Utf8PathBuf; // Import everything from the lib version of ourselves use cargo_dist::{linkage::Linkage, *}; @@ -492,10 +493,16 @@ fn print_help_markdown(out: &mut dyn Write) -> std::io::Result<()> { fn cmd_manifest_schema( _config: &Cli, - _args: &cli::ManifestSchemaArgs, + args: &cli::ManifestSchemaArgs, ) -> Result<(), miette::ErrReport> { let schema = cargo_dist_schema::DistManifest::json_schema(); let json_schema = serde_json::to_string_pretty(&schema).expect("failed to stringify schema!?"); - println!("{json_schema}"); + + if let Some(destination) = args.output.to_owned() { + let contents = json_schema + "\n"; + LocalAsset::write_new(&contents, destination)?; + } else { + println!("{json_schema}"); + } Ok(()) } diff --git a/cargo-dist/src/manifest.rs b/cargo-dist/src/manifest.rs index 1650aa0fd..0df8533c6 100644 --- a/cargo-dist/src/manifest.rs +++ b/cargo-dist/src/manifest.rs @@ -274,6 +274,11 @@ fn manifest_artifact( description = None; kind = cargo_dist_schema::ArtifactKind::SourceTarball; } + ArtifactKind::ExtraArtifact(_) => { + install_hint = None; + description = None; + kind = cargo_dist_schema::ArtifactKind::ExtraArtifact; + } }; let checksum = artifact.checksum.map(|idx| dist.artifact(idx).id.clone()); diff --git a/cargo-dist/src/tasks.rs b/cargo-dist/src/tasks.rs index cdf2fa5e1..269102a26 100644 --- a/cargo-dist/src/tasks.rs +++ b/cargo-dist/src/tasks.rs @@ -60,7 +60,7 @@ use tracing::{info, warn}; use crate::announce::{self, AnnouncementTag}; use crate::backend::ci::github::GithubCiInfo; use crate::backend::ci::CiInfo; -use crate::config::{DependencyKind, DirtyMode, ProductionMode, SystemDependencies}; +use crate::config::{DependencyKind, DirtyMode, ExtraArtifact, ProductionMode, SystemDependencies}; use crate::{ backend::{ installer::{ @@ -196,6 +196,8 @@ pub struct DistGraph { pub msvc_crt_static: bool, /// List of hosting providers to use pub hosting: Option, + /// Additional artifacts to build and upload + pub extra_artifacts: Vec, } /// Info about artifacts should be hosted @@ -286,6 +288,8 @@ pub enum BuildStep { Generic(GenericBuildStep), /// Do a cargo build (and copy the outputs to various locations) Cargo(CargoBuildStep), + /// Do an extra artifact build (and copy the outputs to various locations) + Extra(ExtraBuildStep), /// Run rustup to get a toolchain Rustup(RustupStep), /// Copy a file @@ -334,6 +338,15 @@ pub struct GenericBuildStep { pub build_command: Vec, } +/// An "extra" build step, producing new sidecar artifacts +#[derive(Debug)] +pub struct ExtraBuildStep { + /// Binaries we expect from this build + pub expected_artifacts: Vec, + /// The command to run to produce the expected binaries + pub build_command: Vec, +} + /// A cargo build (and copy the outputs to various locations) #[derive(Debug)] pub struct RustupStep { @@ -474,6 +487,8 @@ pub enum ArtifactKind { Checksum(ChecksumImpl), /// A source tarball SourceTarball(SourceTarball), + /// An extra artifact specified via config + ExtraArtifact(ExtraArtifactImpl), } /// An Archive containing binaries (aka ExecutableZip) @@ -501,6 +516,15 @@ pub struct SourceTarball { pub target: Utf8PathBuf, } +/// An extra artifact of some kind +#[derive(Clone, Debug)] +pub struct ExtraArtifactImpl { + /// The build command to run to produce this artifact + pub build: Vec, + /// The artifact this build should produce + pub artifact: Utf8PathBuf, +} + /// A logical release of an application that artifacts are grouped under #[derive(Clone, Debug)] pub struct Release { @@ -709,6 +733,7 @@ impl<'pkg_graph> DistGraphBuilder<'pkg_graph> { allow_dirty, msvc_crt_static, hosting, + extra_artifacts, } = &workspace_metadata; let desired_cargo_dist_version = cargo_dist_version.clone(); @@ -820,6 +845,7 @@ impl<'pkg_graph> DistGraphBuilder<'pkg_graph> { allow_dirty, msvc_crt_static, hosting, + extra_artifacts: extra_artifacts.clone().unwrap_or_default(), }, manifest: DistManifest { dist_version: Some(env!("CARGO_PKG_VERSION").to_owned()), @@ -1053,11 +1079,40 @@ impl<'pkg_graph> DistGraphBuilder<'pkg_graph> { } } - fn add_source_tarball(&mut self, _tag: &str, to_release: ReleaseIdx) { + fn add_extra_artifacts(&mut self, dist_metadata: &DistMetadata, to_release: ReleaseIdx) { if !self.global_artifacts_enabled() { return; } + let dist_dir = &self.inner.dist_dir.to_owned(); + let artifacts = dist_metadata.extra_artifacts.to_owned().unwrap_or_default(); + + for extra in artifacts { + for filename in extra.artifacts.clone() { + let target_path = dist_dir.join(&filename); + let artifact = Artifact { + id: filename.to_owned(), + target_triples: vec![], + file_path: target_path.to_owned(), + required_binaries: FastMap::new(), + archive: None, + kind: ArtifactKind::ExtraArtifact(ExtraArtifactImpl { + build: extra.build.to_owned(), + artifact: target_path.to_owned(), + }), + checksum: None, + is_global: true, + }; + + self.add_global_artifact(to_release, artifact); + } + } + } + + fn add_source_tarball(&mut self, _tag: &str, to_release: ReleaseIdx) { + if !self.global_artifacts_enabled() { + return; + } let release = self.release(to_release); let checksum = release.checksum; info!("adding source tarball to release {}", release.id); @@ -2073,6 +2128,19 @@ impl<'pkg_graph> DistGraphBuilder<'pkg_graph> { idx } + fn compute_extra_builds(&mut self) -> Vec { + self.inner + .extra_artifacts + .iter() + .map(|extra| { + BuildStep::Extra(ExtraBuildStep { + expected_artifacts: extra.artifacts.clone(), + build_command: extra.build.clone(), + }) + }) + .collect() + } + fn compute_build_steps(&mut self) { // FIXME: more intelligently schedule these in a proper graph? @@ -2083,6 +2151,7 @@ impl<'pkg_graph> DistGraphBuilder<'pkg_graph> { axoproject::WorkspaceKind::Rust => self.compute_cargo_builds(), }; local_build_steps.extend(builds); + local_build_steps.extend(self.compute_extra_builds()); Self::add_build_steps_for_artifacts( &self @@ -2140,6 +2209,9 @@ impl<'pkg_graph> DistGraphBuilder<'pkg_graph> { target: tarball.target.to_owned(), })); } + ArtifactKind::ExtraArtifact(_) => { + // compute_extra_builds handles this + } } if let Some(archive) = &artifact.archive { @@ -2218,6 +2290,9 @@ impl<'pkg_graph> DistGraphBuilder<'pkg_graph> { // Always add the source tarball self.add_source_tarball(&announcing.tag, release); + // Add any extra artifacts defined in the config + self.add_extra_artifacts(&package_config, release); + // Add installers to the Release // Prefer the CLI's choices (`cfg`) if they're non-empty let installers = if cfg.installers.is_empty() { diff --git a/cargo-dist/tests/snapshots/manifest.snap b/cargo-dist/tests/snapshots/manifest.snap index e2c6f8e81..5a7688f78 100644 --- a/cargo-dist/tests/snapshots/manifest.snap +++ b/cargo-dist/tests/snapshots/manifest.snap @@ -20,6 +20,7 @@ stdout: "artifacts": [ "source.tar.gz", "source.tar.gz.sha256", + "dist-manifest-schema.json", "cargo-dist-installer.sh", "cargo-dist-installer.ps1", "cargo-dist.rb", @@ -285,6 +286,10 @@ stdout: "install_hint": "brew install axodotdev/homebrew-tap/cargo-dist", "description": "Install prebuilt binaries via Homebrew" }, + "dist-manifest-schema.json": { + "name": "dist-manifest-schema.json", + "kind": "extra-artifact" + }, "source.tar.gz": { "name": "source.tar.gz", "kind": "source-tarball",