From 41e79b10533c10c85a0c0bc5fb556e096c2c6357 Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Mon, 4 Dec 2023 15:25:06 -0800 Subject: [PATCH] feat: support custom GitHub runners (#614) --- cargo-dist/src/backend/ci/github.rs | 65 ++- cargo-dist/src/config.rs | 11 +- cargo-dist/src/init.rs | 10 + cargo-dist/src/tasks.rs | 8 + cargo-dist/tests/integration-tests.rs | 32 ++ .../axolotlsay_custom_github_runners.snap | 526 ++++++++++++++++++ 6 files changed, 625 insertions(+), 27 deletions(-) create mode 100644 cargo-dist/tests/snapshots/axolotlsay_custom_github_runners.snap diff --git a/cargo-dist/src/backend/ci/github.rs b/cargo-dist/src/backend/ci/github.rs index 7359df6c1..1473f27e0 100644 --- a/cargo-dist/src/backend/ci/github.rs +++ b/cargo-dist/src/backend/ci/github.rs @@ -2,6 +2,8 @@ //! //! In the future this may get split up into submodules. +use std::collections::HashMap; + use axoasset::LocalAsset; use cargo_dist_schema::{GithubMatrix, GithubMatrixEntry}; use serde::Serialize; @@ -108,14 +110,14 @@ impl GithubCiInfo { // Figure out what Local Artifact tasks we need let local_runs = if dist.merge_tasks { - distribute_targets_to_runners_merged(local_targets) + distribute_targets_to_runners_merged(local_targets, &dist.github_custom_runners) } else { - distribute_targets_to_runners_split(local_targets) + distribute_targets_to_runners_split(local_targets, &dist.github_custom_runners) }; for (runner, targets) in local_runs { use std::fmt::Write; let install_dist = - install_dist_for_github_runner(runner, &install_dist_sh, &install_dist_ps1); + install_dist_for_targets(&targets, &install_dist_sh, &install_dist_ps1); let mut dist_args = String::from("--artifacts=local"); for target in &targets { write!(dist_args, " --target={target}").unwrap(); @@ -192,16 +194,17 @@ impl GithubCiInfo { /// succeed (uploading itself to the draft release). /// /// In priniciple it does remove some duplicated setup work, so this is ostensibly "cheaper". -fn distribute_targets_to_runners_merged( - targets: SortedSet<&TargetTriple>, -) -> std::vec::IntoIter<(GithubRunner, Vec<&TargetTriple>)> { +fn distribute_targets_to_runners_merged<'a>( + targets: SortedSet<&'a TargetTriple>, + custom_runners: &HashMap, +) -> std::vec::IntoIter<(GithubRunner, Vec<&'a TargetTriple>)> { let mut groups = SortedMap::>::new(); for target in targets { - let runner = github_runner_for_target(target); + let runner = github_runner_for_target(target, custom_runners); let runner = runner.unwrap_or_else(|| { let default = GITHUB_LINUX_RUNNER; warn!("not sure which github runner should be used for {target}, assuming {default}"); - default + default.to_owned() }); groups.entry(runner).or_default().push(target); } @@ -212,16 +215,17 @@ fn distribute_targets_to_runners_merged( /// Given a set of targets we want to build local artifacts for, map them to Github Runners /// while preferring each target gets its own runner for latency and fault-isolation. -fn distribute_targets_to_runners_split( - targets: SortedSet<&TargetTriple>, -) -> std::vec::IntoIter<(GithubRunner, Vec<&TargetTriple>)> { +fn distribute_targets_to_runners_split<'a>( + targets: SortedSet<&'a TargetTriple>, + custom_runners: &HashMap, +) -> std::vec::IntoIter<(GithubRunner, Vec<&'a TargetTriple>)> { let mut groups = vec![]; for target in targets { - let runner = github_runner_for_target(target); + let runner = github_runner_for_target(target, custom_runners); let runner = runner.unwrap_or_else(|| { let default = GITHUB_LINUX_RUNNER; warn!("not sure which github runner should be used for {target}, assuming {default}"); - default + default.to_owned() }); groups.push((runner, vec![target])); } @@ -229,7 +233,7 @@ fn distribute_targets_to_runners_split( } /// A string representing a Github Runner -type GithubRunner = &'static str; +type GithubRunner = String; /// The Github Runner to use for Linux const GITHUB_LINUX_RUNNER: &str = "ubuntu-20.04"; /// The Github Runner to use for macos @@ -238,34 +242,43 @@ const GITHUB_MACOS_RUNNER: &str = "macos-11"; const GITHUB_WINDOWS_RUNNER: &str = "windows-2019"; /// Get the appropriate Github Runner for building a target -fn github_runner_for_target(target: &TargetTriple) -> Option { +fn github_runner_for_target( + target: &TargetTriple, + custom_runners: &HashMap, +) -> Option { + if let Some(runner) = custom_runners.get(target) { + return Some(runner.to_owned()); + } + // We want to default to older runners to minimize the places // where random system dependencies can creep in and be very // recent. This helps with portability! if target.contains("linux") { - Some(GITHUB_LINUX_RUNNER) + Some(GITHUB_LINUX_RUNNER.to_owned()) } else if target.contains("apple") { - Some(GITHUB_MACOS_RUNNER) + Some(GITHUB_MACOS_RUNNER.to_owned()) } else if target.contains("windows") { - Some(GITHUB_WINDOWS_RUNNER) + Some(GITHUB_WINDOWS_RUNNER.to_owned()) } else { None } } /// Select the cargo-dist installer approach for a given Github Runner -fn install_dist_for_github_runner<'a>( - runner: GithubRunner, +fn install_dist_for_targets<'a>( + targets: &'a [&'a TargetTriple], install_sh: &'a str, install_ps1: &'a str, ) -> &'a str { - if runner == GITHUB_LINUX_RUNNER || runner == GITHUB_MACOS_RUNNER { - install_sh - } else if runner == GITHUB_WINDOWS_RUNNER { - install_ps1 - } else { - unreachable!("internal error: unknown github runner!?") + for target in targets { + if target.contains("linux") || target.contains("apple") { + return install_sh; + } else if target.contains("windows") { + return install_ps1; + } } + + unreachable!("internal error: unknown target triple!?") } fn brewfile_from(packages: &[String]) -> String { diff --git a/cargo-dist/src/config.rs b/cargo-dist/src/config.rs index 61c3de361..8553acd2c 100644 --- a/cargo-dist/src/config.rs +++ b/cargo-dist/src/config.rs @@ -1,6 +1,6 @@ //! Config types (for workspace.metadata.dist) -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use axoasset::{toml_edit, SourceFile}; use axoproject::{WorkspaceKind, WorkspaceSearch}; @@ -276,6 +276,10 @@ pub struct DistMetadata { /// Any extra artifacts and their buildscripts #[serde(skip_serializing_if = "Option::is_none")] pub extra_artifacts: Option>, + + /// Custom GitHub runners, mapped by triple target + #[serde(skip_serializing_if = "Option::is_none")] + pub github_custom_runners: Option>, } impl DistMetadata { @@ -313,6 +317,7 @@ impl DistMetadata { msvc_crt_static: _, hosting: _, extra_artifacts: _, + github_custom_runners: _, } = self; if let Some(include) = include { for include in include { @@ -359,6 +364,7 @@ impl DistMetadata { msvc_crt_static, hosting, extra_artifacts, + github_custom_runners, } = self; // Check for global settings on local packages @@ -453,6 +459,9 @@ impl DistMetadata { if extra_artifacts.is_none() { *extra_artifacts = workspace_config.extra_artifacts.clone(); } + if github_custom_runners.is_none() { + *github_custom_runners = workspace_config.github_custom_runners.clone(); + } // This was historically implemented as extend, but I'm not convinced the // inconsistency is worth the inconvenience... diff --git a/cargo-dist/src/init.rs b/cargo-dist/src/init.rs index b79cc97fb..55cbf2b8c 100644 --- a/cargo-dist/src/init.rs +++ b/cargo-dist/src/init.rs @@ -252,6 +252,7 @@ fn get_new_dist_metadata( msvc_crt_static: None, hosting: None, extra_artifacts: None, + github_custom_runners: None, } }; @@ -772,6 +773,7 @@ fn apply_dist_to_metadata(metadata: &mut toml_edit::Item, meta: &DistMetadata) { msvc_crt_static, hosting, extra_artifacts: _, + github_custom_runners: _, } = &meta; apply_optional_value( @@ -965,6 +967,14 @@ fn apply_dist_to_metadata(metadata: &mut toml_edit::Item, meta: &DistMetadata) { hosting.as_ref(), ); + // NOTE: HashMap not supported by axoasset + // apply_optional_value( + // table, + // "github-custom-runners", + // "# Custom GitHub runners to use for builds, mapped by triple target\n", + // github_custom_runners.as_ref(), + // ); + // Finalize the table table .decor_mut() diff --git a/cargo-dist/src/tasks.rs b/cargo-dist/src/tasks.rs index 269102a26..a620eace0 100644 --- a/cargo-dist/src/tasks.rs +++ b/cargo-dist/src/tasks.rs @@ -48,6 +48,7 @@ //! Also note that the BuildSteps for installers are basically monolithic "build that installer" //! steps to give them the freedom to do whatever they need to do. +use std::collections::HashMap; use std::process::Command; use axoproject::{PackageId, PackageIdx, WorkspaceInfo}; @@ -198,6 +199,8 @@ pub struct DistGraph { pub hosting: Option, /// Additional artifacts to build and upload pub extra_artifacts: Vec, + /// Custom GitHub runners, mapped by triple target + pub github_custom_runners: HashMap, } /// Info about artifacts should be hosted @@ -734,6 +737,7 @@ impl<'pkg_graph> DistGraphBuilder<'pkg_graph> { msvc_crt_static, hosting, extra_artifacts, + github_custom_runners: _, } = &workspace_metadata; let desired_cargo_dist_version = cargo_dist_version.clone(); @@ -846,6 +850,10 @@ impl<'pkg_graph> DistGraphBuilder<'pkg_graph> { msvc_crt_static, hosting, extra_artifacts: extra_artifacts.clone().unwrap_or_default(), + github_custom_runners: workspace_metadata + .github_custom_runners + .clone() + .unwrap_or_default(), }, manifest: DistManifest { dist_version: Some(env!("CARGO_PKG_VERSION").to_owned()), diff --git a/cargo-dist/tests/integration-tests.rs b/cargo-dist/tests/integration-tests.rs index 07e4bf530..e5baa5b7a 100644 --- a/cargo-dist/tests/integration-tests.rs +++ b/cargo-dist/tests/integration-tests.rs @@ -359,6 +359,38 @@ scope = "@axodotdev" }) } +#[test] +fn axolotlsay_custom_github_runners() -> Result<(), miette::Report> { + let test_name = _function_name!(); + AXOLOTLSAY.run_test(|ctx| { + let dist_version = ctx.tools.cargo_dist.version().unwrap(); + ctx.patch_cargo_toml(format!(r#" +[workspace.metadata.dist] +cargo-dist-version = "{dist_version}" +installers = [] +targets = ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu", "x86_64-unknown-linux-musl", "aarch64-unknown-linux-musl"] +ci = ["github"] + +[workspace.metadata.dist.github-custom-runners] +x86_64-unknown-linux-gnu = "buildjet-8vcpu-ubuntu-2204" +x86_64-unknown-linux-musl = "buildjet-8vcpu-ubuntu-2204" +aarch64-unknown-linux-gnu = "buildjet-8vcpu-ubuntu-2204-arm" +aarch64-unknown-linux-musl = "buildjet-8vcpu-ubuntu-2204-arm" +"# + ))?; + + // Run generate to make sure stuff is up to date before running other commands + let ci_result = ctx.cargo_dist_generate(test_name)?; + let ci_snap = ci_result.check_all()?; + // Do usual build+plan checks + let main_result = ctx.cargo_dist_build_and_plan(test_name)?; + let main_snap = main_result.check_all(ctx, ".cargo/bin/")?; + // snapshot all + main_snap.join(ci_snap).snap(); + Ok(()) + }) +} + #[test] fn akaikatana_basic() -> Result<(), miette::Report> { let test_name = _function_name!(); diff --git a/cargo-dist/tests/snapshots/axolotlsay_custom_github_runners.snap b/cargo-dist/tests/snapshots/axolotlsay_custom_github_runners.snap new file mode 100644 index 000000000..74c0af632 --- /dev/null +++ b/cargo-dist/tests/snapshots/axolotlsay_custom_github_runners.snap @@ -0,0 +1,526 @@ +--- +source: cargo-dist/tests/gallery/dist.rs +expression: self.payload +--- +================ dist-manifest.json ================ +{ + "dist_version": "CENSORED", + "announcement_tag": "v0.2.1", + "announcement_is_prerelease": false, + "announcement_title": "Version 0.2.1", + "announcement_changelog": "```text\n +--------------------------------------+\n | now with linux static musl binary!!! |\n +--------------------------------------+\n /\n≽(◕ ᴗ ◕)≼\n```", + "announcement_github_body": "## Release Notes\n\n```text\n +--------------------------------------+\n | now with linux static musl binary!!! |\n +--------------------------------------+\n /\n≽(◕ ᴗ ◕)≼\n```\n\n## Download axolotlsay 0.2.1\n\n| File | Platform | Checksum |\n|--------|----------|----------|\n| [axolotlsay-aarch64-unknown-linux-gnu.tar.xz](https://github.com/axodotdev/axolotlsay/releases/download/v0.2.1/axolotlsay-aarch64-unknown-linux-gnu.tar.xz) | Linux arm64 | [checksum](https://github.com/axodotdev/axolotlsay/releases/download/v0.2.1/axolotlsay-aarch64-unknown-linux-gnu.tar.xz.sha256) |\n| [axolotlsay-aarch64-unknown-linux-musl.tar.xz](https://github.com/axodotdev/axolotlsay/releases/download/v0.2.1/axolotlsay-aarch64-unknown-linux-musl.tar.xz) | musl Linux arm64 | [checksum](https://github.com/axodotdev/axolotlsay/releases/download/v0.2.1/axolotlsay-aarch64-unknown-linux-musl.tar.xz.sha256) |\n| [axolotlsay-x86_64-unknown-linux-gnu.tar.xz](https://github.com/axodotdev/axolotlsay/releases/download/v0.2.1/axolotlsay-x86_64-unknown-linux-gnu.tar.xz) | Linux x64 | [checksum](https://github.com/axodotdev/axolotlsay/releases/download/v0.2.1/axolotlsay-x86_64-unknown-linux-gnu.tar.xz.sha256) |\n| [axolotlsay-x86_64-unknown-linux-musl.tar.xz](https://github.com/axodotdev/axolotlsay/releases/download/v0.2.1/axolotlsay-x86_64-unknown-linux-musl.tar.xz) | musl Linux x64 | [checksum](https://github.com/axodotdev/axolotlsay/releases/download/v0.2.1/axolotlsay-x86_64-unknown-linux-musl.tar.xz.sha256) |\n\n", + "system_info": { + "cargo_version_line": "CENSORED" + }, + "releases": [ + { + "app_name": "axolotlsay", + "app_version": "0.2.1", + "artifacts": [ + "source.tar.gz", + "source.tar.gz.sha256", + "axolotlsay-aarch64-unknown-linux-gnu.tar.xz", + "axolotlsay-aarch64-unknown-linux-gnu.tar.xz.sha256", + "axolotlsay-aarch64-unknown-linux-musl.tar.xz", + "axolotlsay-aarch64-unknown-linux-musl.tar.xz.sha256", + "axolotlsay-x86_64-unknown-linux-gnu.tar.xz", + "axolotlsay-x86_64-unknown-linux-gnu.tar.xz.sha256", + "axolotlsay-x86_64-unknown-linux-musl.tar.xz", + "axolotlsay-x86_64-unknown-linux-musl.tar.xz.sha256" + ], + "hosting": { + "github": { + "artifact_download_url": "https://github.com/axodotdev/axolotlsay/releases/download/v0.2.1" + } + } + } + ], + "artifacts": { + "axolotlsay-aarch64-unknown-linux-gnu.tar.xz": { + "name": "axolotlsay-aarch64-unknown-linux-gnu.tar.xz", + "kind": "executable-zip", + "target_triples": [ + "aarch64-unknown-linux-gnu" + ], + "assets": [ + { + "name": "CHANGELOG.md", + "path": "CHANGELOG.md", + "kind": "changelog" + }, + { + "name": "LICENSE-APACHE", + "path": "LICENSE-APACHE", + "kind": "license" + }, + { + "name": "LICENSE-MIT", + "path": "LICENSE-MIT", + "kind": "license" + }, + { + "name": "README.md", + "path": "README.md", + "kind": "readme" + }, + { + "name": "axolotlsay", + "path": "axolotlsay", + "kind": "executable" + } + ], + "checksum": "axolotlsay-aarch64-unknown-linux-gnu.tar.xz.sha256" + }, + "axolotlsay-aarch64-unknown-linux-gnu.tar.xz.sha256": { + "name": "axolotlsay-aarch64-unknown-linux-gnu.tar.xz.sha256", + "kind": "checksum", + "target_triples": [ + "aarch64-unknown-linux-gnu" + ] + }, + "axolotlsay-aarch64-unknown-linux-musl.tar.xz": { + "name": "axolotlsay-aarch64-unknown-linux-musl.tar.xz", + "kind": "executable-zip", + "target_triples": [ + "aarch64-unknown-linux-musl" + ], + "assets": [ + { + "name": "CHANGELOG.md", + "path": "CHANGELOG.md", + "kind": "changelog" + }, + { + "name": "LICENSE-APACHE", + "path": "LICENSE-APACHE", + "kind": "license" + }, + { + "name": "LICENSE-MIT", + "path": "LICENSE-MIT", + "kind": "license" + }, + { + "name": "README.md", + "path": "README.md", + "kind": "readme" + }, + { + "name": "axolotlsay", + "path": "axolotlsay", + "kind": "executable" + } + ], + "checksum": "axolotlsay-aarch64-unknown-linux-musl.tar.xz.sha256" + }, + "axolotlsay-aarch64-unknown-linux-musl.tar.xz.sha256": { + "name": "axolotlsay-aarch64-unknown-linux-musl.tar.xz.sha256", + "kind": "checksum", + "target_triples": [ + "aarch64-unknown-linux-musl" + ] + }, + "axolotlsay-x86_64-unknown-linux-gnu.tar.xz": { + "name": "axolotlsay-x86_64-unknown-linux-gnu.tar.xz", + "kind": "executable-zip", + "target_triples": [ + "x86_64-unknown-linux-gnu" + ], + "assets": [ + { + "name": "CHANGELOG.md", + "path": "CHANGELOG.md", + "kind": "changelog" + }, + { + "name": "LICENSE-APACHE", + "path": "LICENSE-APACHE", + "kind": "license" + }, + { + "name": "LICENSE-MIT", + "path": "LICENSE-MIT", + "kind": "license" + }, + { + "name": "README.md", + "path": "README.md", + "kind": "readme" + }, + { + "name": "axolotlsay", + "path": "axolotlsay", + "kind": "executable" + } + ], + "checksum": "axolotlsay-x86_64-unknown-linux-gnu.tar.xz.sha256" + }, + "axolotlsay-x86_64-unknown-linux-gnu.tar.xz.sha256": { + "name": "axolotlsay-x86_64-unknown-linux-gnu.tar.xz.sha256", + "kind": "checksum", + "target_triples": [ + "x86_64-unknown-linux-gnu" + ] + }, + "axolotlsay-x86_64-unknown-linux-musl.tar.xz": { + "name": "axolotlsay-x86_64-unknown-linux-musl.tar.xz", + "kind": "executable-zip", + "target_triples": [ + "x86_64-unknown-linux-musl" + ], + "assets": [ + { + "name": "CHANGELOG.md", + "path": "CHANGELOG.md", + "kind": "changelog" + }, + { + "name": "LICENSE-APACHE", + "path": "LICENSE-APACHE", + "kind": "license" + }, + { + "name": "LICENSE-MIT", + "path": "LICENSE-MIT", + "kind": "license" + }, + { + "name": "README.md", + "path": "README.md", + "kind": "readme" + }, + { + "name": "axolotlsay", + "path": "axolotlsay", + "kind": "executable" + } + ], + "checksum": "axolotlsay-x86_64-unknown-linux-musl.tar.xz.sha256" + }, + "axolotlsay-x86_64-unknown-linux-musl.tar.xz.sha256": { + "name": "axolotlsay-x86_64-unknown-linux-musl.tar.xz.sha256", + "kind": "checksum", + "target_triples": [ + "x86_64-unknown-linux-musl" + ] + }, + "source.tar.gz": { + "name": "source.tar.gz", + "kind": "source-tarball", + "checksum": "source.tar.gz.sha256" + }, + "source.tar.gz.sha256": { + "name": "source.tar.gz.sha256", + "kind": "checksum" + } + }, + "publish_prereleases": false, + "ci": { + "github": { + "artifacts_matrix": { + "include": [ + { + "targets": [ + "aarch64-unknown-linux-gnu" + ], + "runner": "buildjet-8vcpu-ubuntu-2204-arm", + "install_dist": "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/vSOME_VERSION/cargo-dist-installer.sh | sh", + "dist_args": "--artifacts=local --target=aarch64-unknown-linux-gnu" + }, + { + "targets": [ + "aarch64-unknown-linux-musl" + ], + "runner": "buildjet-8vcpu-ubuntu-2204-arm", + "install_dist": "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/vSOME_VERSION/cargo-dist-installer.sh | sh", + "dist_args": "--artifacts=local --target=aarch64-unknown-linux-musl", + "packages_install": "sudo apt-get install musl-tools" + }, + { + "targets": [ + "x86_64-unknown-linux-gnu" + ], + "runner": "buildjet-8vcpu-ubuntu-2204", + "install_dist": "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/vSOME_VERSION/cargo-dist-installer.sh | sh", + "dist_args": "--artifacts=local --target=x86_64-unknown-linux-gnu" + }, + { + "targets": [ + "x86_64-unknown-linux-musl" + ], + "runner": "buildjet-8vcpu-ubuntu-2204", + "install_dist": "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/vSOME_VERSION/cargo-dist-installer.sh | sh", + "dist_args": "--artifacts=local --target=x86_64-unknown-linux-musl", + "packages_install": "sudo apt-get install musl-tools" + } + ] + }, + "pr_run_mode": "plan" + } + }, + "linkage": [] +} + +================ github-ci.yml ================ +# Copyright 2022-2023, axodotdev +# SPDX-License-Identifier: MIT or Apache-2.0 +# +# CI that: +# +# * checks for a Git Tag that looks like a release +# * builds artifacts with cargo-dist (archives, installers, hashes) +# * uploads those artifacts to temporary workflow zip +# * on success, uploads the artifacts to a Github Release +# +# Note that the Github Release will be created with a generated +# title/body based on your changelogs. + +name: Release + +permissions: + contents: write + +# This task will run whenever you push a git tag that looks like a version +# like "1.0.0", "v0.1.0-prerelease.1", "my-app/0.1.0", "releases/v1.0.0", etc. +# Various formats will be parsed into a VERSION and an optional PACKAGE_NAME, where +# PACKAGE_NAME must be the name of a Cargo package in your workspace, and VERSION +# must be a Cargo-style SemVer Version (must have at least major.minor.patch). +# +# If PACKAGE_NAME is specified, then the announcement will be for that +# package (erroring out if it doesn't have the given version or isn't cargo-dist-able). +# +# If PACKAGE_NAME isn't specified, then the announcement will be for all +# (cargo-dist-able) packages in the workspace with that version (this mode is +# intended for workspaces with only one dist-able package, or with all dist-able +# packages versioned/released in lockstep). +# +# If you push multiple tags at once, separate instances of this workflow will +# spin up, creating an independent announcement for each one. However Github +# will hard limit this to 3 tags per commit, as it will assume more tags is a +# mistake. +# +# If there's a prerelease-style suffix to the version, then the release(s) +# will be marked as a prerelease. +on: + push: + tags: + - '**[0-9]+.[0-9]+.[0-9]+*' + pull_request: + +jobs: + # Run 'cargo dist plan' (or host) to determine what tasks we need to do + plan: + runs-on: ubuntu-latest + outputs: + val: ${{ steps.plan.outputs.manifest }} + tag: ${{ !github.event.pull_request && github.ref_name || '' }} + tag-flag: ${{ !github.event.pull_request && format('--tag={0}', github.ref_name) || '' }} + publishing: ${{ !github.event.pull_request }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install cargo-dist + # we specify bash to get pipefail; it guards against the `curl` command + # failing. otherwise `sh` won't catch that `curl` returned non-0 + shell: bash + run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/vSOME_VERSION/cargo-dist-installer.sh | sh" + # sure would be cool if github gave us proper conditionals... + # so here's a doubly-nested ternary-via-truthiness to try to provide the best possible + # functionality based on whether this is a pull_request, and whether it's from a fork. + # (PRs run on the *source* but secrets are usually on the *target* -- that's *good* + # but also really annoying to build CI around when it needs secrets to work right.) + - id: plan + run: | + cargo dist ${{ !github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name) || (github.event.pull_request.head.repo.fork && 'plan' || 'host --steps=check') }} --output-format=json > dist-manifest.json + echo "cargo dist ran successfully" + cat dist-manifest.json + echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT" + - name: "Upload dist-manifest.json" + uses: actions/upload-artifact@v3 + with: + name: artifacts + path: dist-manifest.json + + # Build and packages all the platform-specific things + build-local-artifacts: + name: build-local-artifacts (${{ join(matrix.targets, ', ') }}) + # Let the initial task tell us to not run (currently very blunt) + needs: plan + if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }} + strategy: + fail-fast: false + # Target platforms/runners are computed by cargo-dist in create-release. + # Each member of the matrix has the following arguments: + # + # - runner: the github runner + # - dist-args: cli flags to pass to cargo dist + # - install-dist: expression to run to install cargo-dist on the runner + # + # Typically there will be: + # - 1 "global" task that builds universal installers + # - N "local" tasks that build each platform's binaries and platform-specific installers + matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }} + runs-on: ${{ matrix.runner }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: swatinem/rust-cache@v2 + - name: Install cargo-dist + run: ${{ matrix.install_dist }} + # Get the dist-manifest + - name: Fetch local artifacts + uses: actions/download-artifact@v3 + with: + name: artifacts + path: target/distrib/ + - name: Install dependencies + run: | + ${{ matrix.packages_install }} + - name: Build artifacts + run: | + # Actually do builds and make zips and whatnot + cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json + echo "cargo dist ran successfully" + - id: cargo-dist + name: Post-build + # We force bash here just because github makes it really hard to get values up + # to "real" actions without writing to env-vars, and writing to env-vars has + # inconsistent syntax between shell and powershell. + shell: bash + run: | + # Parse out what we just built and upload it to scratch storage + echo "paths<> "$GITHUB_OUTPUT" + jq --raw-output ".artifacts[]?.path | select( . != null )" dist-manifest.json >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + cp dist-manifest.json "$BUILD_MANIFEST_NAME" + - name: "Upload artifacts" + uses: actions/upload-artifact@v3 + with: + name: artifacts + path: | + ${{ steps.cargo-dist.outputs.paths }} + ${{ env.BUILD_MANIFEST_NAME }} + + # Build and package all the platform-agnostic(ish) things + build-global-artifacts: + needs: + - plan + - build-local-artifacts + runs-on: "ubuntu-20.04" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BUILD_MANIFEST_NAME: target/distrib/dist-manifest.json + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install cargo-dist + run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/vSOME_VERSION/cargo-dist-installer.sh | sh" + # Get all the local artifacts for the global tasks to use (for e.g. checksums) + - name: Fetch local artifacts + uses: actions/download-artifact@v3 + with: + name: artifacts + path: target/distrib/ + - id: cargo-dist + shell: bash + run: | + cargo dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json + echo "cargo dist ran successfully" + + # Parse out what we just built and upload it to scratch storage + echo "paths<> "$GITHUB_OUTPUT" + jq --raw-output ".artifacts[]?.path | select( . != null )" dist-manifest.json >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + cp dist-manifest.json "$BUILD_MANIFEST_NAME" + - name: "Upload artifacts" + uses: actions/upload-artifact@v3 + with: + name: artifacts + path: | + ${{ steps.cargo-dist.outputs.paths }} + ${{ env.BUILD_MANIFEST_NAME }} + # Determines if we should publish/announce + host: + needs: + - plan + - build-local-artifacts + - build-global-artifacts + # Only run if we're "publishing", and only if local and global didn't fail (skipped is fine) + if: ${{ always() && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + runs-on: "ubuntu-20.04" + outputs: + val: ${{ steps.host.outputs.manifest }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install cargo-dist + run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/vSOME_VERSION/cargo-dist-installer.sh | sh" + # Fetch artifacts from scratch-storage + - name: Fetch artifacts + uses: actions/download-artifact@v3 + with: + name: artifacts + path: target/distrib/ + # This is a harmless no-op for Github Releases, hosting for that happens in "announce" + - id: host + shell: bash + run: | + cargo dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json + echo "artifacts uploaded and released successfully" + cat dist-manifest.json + echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT" + - name: "Upload dist-manifest.json" + uses: actions/upload-artifact@v3 + with: + name: artifacts + path: dist-manifest.json + + # Create a Github Release while uploading all files to it + announce: + needs: + - plan + - host + # use "always() && ..." to allow us to wait for all publish jobs while + # still allowing individual publish jobs to skip themselves (for prereleases). + # "host" however must run to completion, no skipping allowed! + if: ${{ always() && needs.host.result == 'success' }} + runs-on: "ubuntu-20.04" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: "Download Github Artifacts" + uses: actions/download-artifact@v3 + with: + name: artifacts + path: artifacts + - name: Cleanup + run: | + # Remove the granular manifests + rm -f artifacts/*-dist-manifest.json + - name: Create Github Release + uses: ncipollo/release-action@v1 + with: + tag: ${{ needs.plan.outputs.tag }} + name: ${{ fromJson(needs.host.outputs.val).announcement_title }} + body: ${{ fromJson(needs.host.outputs.val).announcement_github_body }} + prerelease: ${{ fromJson(needs.host.outputs.val).announcement_is_prerelease }} + artifacts: "artifacts/*" + +