From eb9567beea749c8a21624f6772c4390ecff80c8b Mon Sep 17 00:00:00 2001 From: Schneems Date: Tue, 23 Jul 2024 17:20:39 -0500 Subject: [PATCH 01/18] Generate Inventory & Sha7 appended filenames for JRuby Currently the Ruby buildapack is tightly coupled to the URLs that are generated by this repo. I am in the process of introducing an inventory file that the buildpack can use as a lookup. To start that process this commit introduces generating tgz files with the first 7 SHA256 characters appended. This allows us to update the same version number in the future without worrying that it will break future digest checks of the URL. With this change the builder will continue to generate the original URLs, but it will also generate "-.tgz" files as well and append this information to a `jruby_inventory.toml` file. --- .github/workflows/build_jruby.yml | 10 + Cargo.lock | 260 +++++++++++++++++++++++- Cargo.toml | 8 + jruby_executable/Cargo.toml | 2 + jruby_executable/src/bin/jruby_build.rs | 48 ++++- jruby_inventory.toml | 1 + shared/Cargo.toml | 6 + shared/src/base_image.rs | 21 ++ shared/src/lib.rs | 124 +++++++++++ 9 files changed, 473 insertions(+), 7 deletions(-) create mode 100644 jruby_inventory.toml diff --git a/.github/workflows/build_jruby.yml b/.github/workflows/build_jruby.yml index d3f7e4a..2b79100 100644 --- a/.github/workflows/build_jruby.yml +++ b/.github/workflows/build_jruby.yml @@ -51,3 +51,13 @@ jobs: - name: Upload Ruby runtime archive to S3 production if: (!inputs.dry_run) run: aws s3 sync ./output "s3://${S3_BUCKET}" + + after-build-and-upload: + needs: build-and-upload + runs-on: pub-hk-ubuntu-24.04-xlarge + steps: + - name: Update Jruby inventory file locally + uses: peter-evans/create-pull-request@v6 + with: + path: jruby_inventory.toml + title: "Add JRuby ${{inputs.jruby_version}} to inventory" diff --git a/Cargo.lock b/Cargo.lock index 858d91c..62f1b26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "0.6.14" @@ -126,6 +141,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bullet_stream" version = "0.2.0" @@ -156,6 +180,21 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets 0.52.5", +] + [[package]] name = "chunked_transfer" version = "1.5.0" @@ -224,6 +263,15 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -233,6 +281,26 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "encoding_rs" version = "0.8.34" @@ -325,6 +393,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "fun_run" version = "0.2.0" @@ -385,6 +463,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "gimli" version = "0.29.0" @@ -434,6 +522,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "http" version = "1.1.0" @@ -536,6 +630,29 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "idna" version = "0.5.0" @@ -562,6 +679,18 @@ version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" +[[package]] +name = "inventory" +version = "0.1.0" +source = "git+https://github.com/Malax/inventory#e4eed7de95e49d441cdf13f9c001644ed9159393" +dependencies = [ + "hex", + "serde", + "sha2", + "thiserror", + "toml", +] + [[package]] name = "ipnet" version = "2.9.0" @@ -596,12 +725,14 @@ name = "jruby_executable" version = "0.1.0" dependencies = [ "bullet_stream", + "chrono", "clap", "flate2", "fs-err", "fun_run", "glob", "indoc", + "inventory", "java-properties", "lazy_static", "nom", @@ -711,6 +842,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.16.0" @@ -1017,18 +1157,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", @@ -1046,6 +1186,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -1058,24 +1207,41 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shared" version = "0.1.0" dependencies = [ "bullet_stream", + "chrono", "clap", "flate2", "fs-err", + "fs2", "fun_run", "glob", "indoc", + "inventory", "nom", "regex", "reqwest", + "serde", + "sha2", "tar", "tempfile", "thiserror", "tiny_http", + "toml", ] [[package]] @@ -1256,6 +1422,40 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2caab0bf757388c6c0ae23b3293fdb463fee59434529014f85e3263b995c28" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "278f3d518e152219c994ce877758516bca5e118eaed6996192a774fb9fbf0788" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.4.13" @@ -1308,6 +1508,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "unicode-bidi" version = "0.3.15" @@ -1352,6 +1558,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + [[package]] name = "want" version = "0.3.1" @@ -1443,6 +1655,37 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.5", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -1582,6 +1825,15 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +[[package]] +name = "winnow" +version = "0.6.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "557404e450152cd6795bb558bca69e43c585055f4606e3bcae5894fc6dac9ba0" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.52.0" diff --git a/Cargo.toml b/Cargo.toml index 1d08439..f9a3104 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,3 +22,11 @@ tar = "0.4.40" tempfile = "3.10.1" thiserror = "1.0.61" bullet_stream = "0.2.0" +sha2 = "0.10" +toml = "0.8" +chrono = {version = "0.4", features = ["serde"] } +serde = {version = "1.0", features = ["derive"] } +inventory = { git = "https://github.com/Malax/inventory", features = ["sha2"] } + +# File locking (FLOCK) +fs2 = "0.4" diff --git a/jruby_executable/Cargo.toml b/jruby_executable/Cargo.toml index c10b017..ca81df1 100644 --- a/jruby_executable/Cargo.toml +++ b/jruby_executable/Cargo.toml @@ -32,3 +32,5 @@ tar = { workspace = true} tempfile = { workspace = true} thiserror = { workspace = true} bullet_stream = { workspace = true } +inventory = { workspace = true } +chrono = { workspace = true } diff --git a/jruby_executable/src/bin/jruby_build.rs b/jruby_executable/src/bin/jruby_build.rs index 29e60d4..92e59d6 100644 --- a/jruby_executable/src/bin/jruby_build.rs +++ b/jruby_executable/src/bin/jruby_build.rs @@ -2,10 +2,18 @@ use bullet_stream::{style, Print}; use clap::Parser; use fs_err::PathExt; use indoc::formatdoc; +use inventory::artifact::Artifact; use jruby_executable::jruby_build_properties; -use shared::{download_tar, source_dir, tar_dir_to_file, untar_to_dir, BaseImage, TarDownloadPath}; +use shared::{ + append_filename_with, append_manifest, download_tar, sha256_from_path, source_dir, + tar_dir_to_file, untar_to_dir, ArtifactMetadata, BaseImage, CpuArch, ManifestVersion, + TarDownloadPath, +}; +use std::convert::From; use std::error::Error; +static S3_BASE_URL: &str = "https://heroku-buildpack-ruby.s3.us-east-1.amazonaws.com"; + #[derive(Parser, Debug)] struct Args { #[arg(long)] @@ -25,6 +33,8 @@ fn jruby_build(args: &Args) -> Result<(), Box> { let volume_cache_dir = source_dir().join("cache"); let volume_output_dir = source_dir().join("output"); + let inventory = source_dir().join("jruby_inventory.toml"); + fs_err::create_dir_all(&volume_cache_dir)?; let temp_dir = tempfile::tempdir()?; @@ -89,6 +99,10 @@ fn jruby_build(args: &Args) -> Result<(), Box> { log = { let mut bullet = log.bullet("Creating tgz archives"); + bullet = bullet.sub_bullet(format!( + "Inventory file {}", + style::value(inventory.to_string_lossy()) + )); let tar_dir = volume_output_dir.join(base_image.to_string()); fs_err::create_dir_all(&tar_dir)?; @@ -99,17 +113,45 @@ fn jruby_build(args: &Args) -> Result<(), Box> { tar_dir_to_file(&jruby_dir, &tar_file)?; bullet = timer.done(); - for arch in &["amd64", "arm64"] { - let dir = volume_output_dir.join(base_image.to_string()).join(arch); + for cpu_arch in &[CpuArch::new("amd64")?, CpuArch::new("arm64")?] { + let dir = volume_output_dir + .join(base_image.to_string()) + .join(cpu_arch.to_string()); fs_err::create_dir_all(&dir)?; let path = dir.join(&tgz_name); bullet = bullet.sub_bullet(format!("Write {}", path.display())); fs_err::copy(tar_file.path(), &path)?; + + let sha = sha256_from_path(&path)?; + let sha_seven = sha.chars().take(7).collect::(); + let sha_seven_path = append_filename_with(&path, &format!("-{sha_seven}"), ".tgz")?; + + bullet = bullet.sub_bullet(format!("Write {}", sha_seven_path.display(),)); + fs_err::copy(tar_file.path(), &sha_seven_path)?; + + append_manifest( + &inventory, + Artifact { + version: ManifestVersion::new(version), + os: inventory::artifact::Os::Linux, + arch: cpu_arch.into(), + url: format!( + "{S3_BASE_URL}/{}", + sha_seven_path.strip_prefix(&volume_output_dir)?.display() + ), + checksum: format!("sha256:{sha}").parse()?, + metadata: ArtifactMetadata { + distro_version: base_image.distro_version(), + timestamp: chrono::Utc::now(), + }, + }, + )?; } bullet.done() }; + log.done(); Ok(()) diff --git a/jruby_inventory.toml b/jruby_inventory.toml new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/jruby_inventory.toml @@ -0,0 +1 @@ + diff --git a/shared/Cargo.toml b/shared/Cargo.toml index 1072bf5..09cfc82 100644 --- a/shared/Cargo.toml +++ b/shared/Cargo.toml @@ -17,6 +17,12 @@ tar = { workspace = true} tempfile = { workspace = true} thiserror = { workspace = true} bullet_stream = { workspace = true } +sha2 = { workspace = true } +chrono = { workspace = true } +serde = { workspace = true } +toml = { workspace = true } +inventory = { workspace = true } +fs2 = { workspace = true } [dev-dependencies] tiny_http = "0.12.0" diff --git a/shared/src/base_image.rs b/shared/src/base_image.rs index 4e95870..f3c0563 100644 --- a/shared/src/base_image.rs +++ b/shared/src/base_image.rs @@ -1,6 +1,8 @@ use std::fmt::Display; use std::str::FromStr; +use serde::{Deserialize, Serialize}; + static KNOWN_ARCHITECTURES: [&str; 2] = ["amd64", "arm64"]; static KNOWN_BASE_IMAGES: &[(&str, &str)] = &[ ("heroku-20", "20"), @@ -19,6 +21,9 @@ pub struct BaseImage { distro_number: String, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct DistroVersion(String); + impl BaseImage { pub fn new(s: &str) -> Result { KNOWN_BASE_IMAGES @@ -31,6 +36,10 @@ impl BaseImage { .ok_or_else(|| BaseImageError(s.to_owned())) } + pub fn distro_version(&self) -> DistroVersion { + DistroVersion(format!("{}.04", self.distro_number)) + } + pub fn is_arch_aware(&self) -> bool { MULTI_ARCH_BASE_IMAGES.contains(&self.name.as_str()) } @@ -83,6 +92,18 @@ impl FromStr for CpuArch { #[error("Invalid CPU architecture {0} must be one of {}", KNOWN_ARCHITECTURES.join(", "))] pub struct CpuArchError(String); +impl From<&CpuArch> for inventory::artifact::Arch { + fn from(value: &CpuArch) -> Self { + if &value.name == "amd64" { + inventory::artifact::Arch::Amd64 + } else if value.name == "arm64" { + inventory::artifact::Arch::Arm64 + } else { + unreachable!(); + } + } +} + impl CpuArch { pub fn new(s: &str) -> Result { KNOWN_ARCHITECTURES diff --git a/shared/src/lib.rs b/shared/src/lib.rs index 5391370..efa5961 100644 --- a/shared/src/lib.rs +++ b/shared/src/lib.rs @@ -1,7 +1,14 @@ +use base_image::DistroVersion; use bullet_stream::state::SubBullet; use bullet_stream::Print; +use chrono::{DateTime, Utc}; +use fs2::FileExt; use fs_err::{File, PathExt}; use fun_run::CommandWithName; +use inventory::artifact::Artifact; +use inventory::inventory::Inventory; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use std::io::{Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; use std::process::Command; @@ -12,6 +19,120 @@ mod download_ruby_version; pub use base_image::{BaseImage, CpuArch, CpuArchError}; pub use download_ruby_version::RubyDownloadVersion; +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct ArtifactMetadata { + pub timestamp: DateTime, + pub distro_version: DistroVersion, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct ManifestVersion(pub String); + +impl ManifestVersion { + pub fn new(version: &str) -> Self { + Self(version.to_string()) + } +} + +/// Appends the given artifact to the inventory file at the given path +/// +/// If the file doesn't exist, it will be created. +/// Uses file locking to ensure atomic updating. +pub fn append_manifest( + path: &Path, + artifact: Artifact, +) -> Result<(), Error> { + fs_err::create_dir_all( + path.parent().ok_or_else(|| { + Error::Other(format!("Cannot determine parent from {}", path.display())) + })?, + ) + .map_err(Error::FsError)?; + + let mut file: std::fs::File = fs_err::OpenOptions::new() + .append(true) + .create(true) + .open(path) + .map_err(Error::FsError)? + .into(); + + file.lock_exclusive().map_err(|e| { + Error::Other(format!( + "Error {e} obtaining file lock on {}", + path.display() + )) + })?; + + let inventory_string = fs_err::read_to_string(path).map_err(Error::FsError)?; + + let mut inventory = if inventory_string.trim().is_empty() { + Inventory { + artifacts: Vec::new(), + } + } else { + inventory_string + .parse::>() + .map_err(|e| Error::Other(format!("Error {e} parsing inventory: {}", path.display())))? + }; + + inventory.push(artifact); + writeln!(file, "{inventory}").expect("Writeable file"); + + file.unlock().map_err(|e| { + Error::Other(format!( + "Error {e} releasing file lock on {}", + path.display() + )) + })?; + + Ok(()) +} + +/// Appends the given string after the filename and before the `ends_with` +/// +/// ``` +/// use std::path::Path; +/// use shared::append_filename_with; +/// +/// let path = Path::new("/tmp/file.txt"); +/// let out = append_filename_with(path, "-lol", ".txt").unwrap(); +/// assert_eq!(Path::new("/tmp/file-lol.txt"), out); +/// ``` +/// +/// Raises an error if the files doesn't exist or if the file name doesn't end with `ends_with` +pub fn append_filename_with(path: &Path, append: &str, ends_with: &str) -> Result { + let parent = path + .parent() + .ok_or_else(|| Error::Other(format!("Cannot determine parent from {}", path.display())))?; + let file_name = path + .file_name() + .ok_or_else(|| { + Error::Other(format!( + "Cannot determine file name from {}", + path.display() + )) + })? + .to_string_lossy(); + + if !file_name.ends_with(ends_with) { + Err(Error::Other(format!( + "File name {} does not end with {}", + file_name, ends_with + )))?; + } + let file_base = file_name.trim_end_matches(ends_with); + + Ok(parent.join(format!("{file_base}{append}{ends_with}"))) +} + +/// Returns the sha256 hash of the file at the given path +pub fn sha256_from_path(path: &Path) -> Result { + let mut hasher = Sha256::new(); + hasher.update(fs_err::read(path).map_err(Error::FsError)?); + + Ok(format!("{:x}", hasher.finalize())) +} + #[derive(Debug, Clone)] pub struct TarDownloadPath(pub PathBuf); @@ -67,6 +188,9 @@ pub enum Error { #[source] source: std::io::Error, }, + + #[error("Error {0}")] + Other(String), } pub fn source_dir() -> PathBuf { From 4b53490d4dfa63a35f932350f5d07a7290a8c297 Mon Sep 17 00:00:00 2001 From: Schneems Date: Wed, 24 Jul 2024 10:23:24 -0500 Subject: [PATCH 02/18] Use FromStr for converting between arch structs --- shared/src/base_image.rs | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/shared/src/base_image.rs b/shared/src/base_image.rs index f3c0563..6c5b6b0 100644 --- a/shared/src/base_image.rs +++ b/shared/src/base_image.rs @@ -89,18 +89,19 @@ impl FromStr for CpuArch { } #[derive(Debug, thiserror::Error)] -#[error("Invalid CPU architecture {0} must be one of {}", KNOWN_ARCHITECTURES.join(", "))] -pub struct CpuArchError(String); - -impl From<&CpuArch> for inventory::artifact::Arch { - fn from(value: &CpuArch) -> Self { - if &value.name == "amd64" { - inventory::artifact::Arch::Amd64 - } else if value.name == "arm64" { - inventory::artifact::Arch::Arm64 - } else { - unreachable!(); - } +pub enum CpuArchError { + #[error("Invalid CPU architecture {0} must be one of {}", KNOWN_ARCHITECTURES.join(", "))] + Unknown(String), + + #[error("CPU architecture {0} cannot be converted {1}")] + CannotConvert(String, inventory::artifact::UnsupportedArchError), +} + +impl TryFrom<&CpuArch> for inventory::artifact::Arch { + type Error = CpuArchError; + + fn try_from(value: &CpuArch) -> Result { + Self::from_str(&value.name).map_err(|e| CpuArchError::CannotConvert(value.name.clone(), e)) } } @@ -110,7 +111,7 @@ impl CpuArch { .iter() .find(|&&name| name == s) .map(|_| Self { name: s.to_owned() }) - .ok_or_else(|| CpuArchError(s.to_owned())) + .ok_or_else(|| CpuArchError::Unknown(s.to_owned())) } pub fn from_system() -> Result { From 0883a80e8abb8eef4127fbd93794bd8b55c7db17 Mon Sep 17 00:00:00 2001 From: Schneems Date: Wed, 24 Jul 2024 12:09:18 -0500 Subject: [PATCH 03/18] Refactor and add inventory_check - We now create only one sha named archive for jruby builds since both ARM and AMD point to the same source. - Binary `inventory_check` takes in a file and validates that all checksums are valid. --- .github/workflows/ci.yml | 19 ++ Cargo.lock | 54 ++++++ Cargo.toml | 3 + jruby_executable/Cargo.toml | 1 + jruby_executable/src/bin/jruby_build.rs | 46 ++--- jruby_inventory.toml | 21 +++ shared/Cargo.toml | 7 + shared/src/bin/inventory_check.rs | 29 +++ shared/src/inventory_help.rs | 226 ++++++++++++++++++++++++ shared/src/lib.rs | 88 +-------- 10 files changed, 390 insertions(+), 104 deletions(-) create mode 100644 shared/src/bin/inventory_check.rs create mode 100644 shared/src/inventory_help.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 037c7ef..37d07fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,3 +102,22 @@ jobs: run: cargo run --bin jruby_build -- --version ${{matrix.version}} --base-image ${{matrix.base_image}} - name: Check JRuby run: cargo run --bin jruby_check -- --version ${{matrix.version}} --base-image ${{matrix.base_image}} --arch ${{matrix.arch}} + + check_inventory_urls: + runs-on: ubuntu-24.04 + if: (!contains(github.event.pull_request.labels.*.name, 'skip inventory check')) + strategy: + matrix: + inventory: ["jruby_inventory.toml", "ruby_inventory.toml"] + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Grab prior commits + run: | + set -eu + set pipefail + + git fetch origin ${{ github.base_ref }} --depth 1 && \ + git diff --unified=0 remotes/origin/${{ github.base_ref }} ${{matrix.inventory}} | grep '^+' | grep -v '^+++' | cut -c2- > check_inventory.toml + - name: Check manifest URLs + run: cargo run --bin inventory_check -- check_inventory.toml diff --git a/Cargo.lock b/Cargo.lock index 62f1b26..8746d87 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -281,6 +281,31 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + [[package]] name = "crypto-common" version = "0.1.6" @@ -301,6 +326,12 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + [[package]] name = "encoding_rs" version = "0.8.34" @@ -738,6 +769,7 @@ dependencies = [ "nom", "regex", "reqwest", + "sha2", "shared", "tar", "tempfile", @@ -982,6 +1014,26 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -1230,9 +1282,11 @@ dependencies = [ "fs2", "fun_run", "glob", + "hex", "indoc", "inventory", "nom", + "rayon", "regex", "reqwest", "serde", diff --git a/Cargo.toml b/Cargo.toml index f9a3104..e7ca917 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,9 @@ toml = "0.8" chrono = {version = "0.4", features = ["serde"] } serde = {version = "1.0", features = ["derive"] } inventory = { git = "https://github.com/Malax/inventory", features = ["sha2"] } +rayon = "1.10" +hex = "0.4.3" # File locking (FLOCK) fs2 = "0.4" + diff --git a/jruby_executable/Cargo.toml b/jruby_executable/Cargo.toml index ca81df1..43dc730 100644 --- a/jruby_executable/Cargo.toml +++ b/jruby_executable/Cargo.toml @@ -34,3 +34,4 @@ thiserror = { workspace = true} bullet_stream = { workspace = true } inventory = { workspace = true } chrono = { workspace = true } +sha2 = { workspace = true } diff --git a/jruby_executable/src/bin/jruby_build.rs b/jruby_executable/src/bin/jruby_build.rs index 92e59d6..3f94bc6 100644 --- a/jruby_executable/src/bin/jruby_build.rs +++ b/jruby_executable/src/bin/jruby_build.rs @@ -5,7 +5,7 @@ use indoc::formatdoc; use inventory::artifact::Artifact; use jruby_executable::jruby_build_properties; use shared::{ - append_filename_with, append_manifest, download_tar, sha256_from_path, source_dir, + append_filename_with, append_inventory, download_tar, sha256_from_path, source_dir, tar_dir_to_file, untar_to_dir, ArtifactMetadata, BaseImage, CpuArch, ManifestVersion, TarDownloadPath, }; @@ -113,42 +113,48 @@ fn jruby_build(args: &Args) -> Result<(), Box> { tar_dir_to_file(&jruby_dir, &tar_file)?; bullet = timer.done(); - for cpu_arch in &[CpuArch::new("amd64")?, CpuArch::new("arm64")?] { - let dir = volume_output_dir - .join(base_image.to_string()) - .join(cpu_arch.to_string()); - fs_err::create_dir_all(&dir)?; + let tar_path = tar_file.path(); + let sha = sha256_from_path(tar_path)?; + let sha_seven = sha.chars().take(7).collect::(); + let sha_seven_path = append_filename_with(tar_path, &format!("-{sha_seven}"), ".tgz")?; - let path = dir.join(&tgz_name); - bullet = bullet.sub_bullet(format!("Write {}", path.display())); - fs_err::copy(tar_file.path(), &path)?; - - let sha = sha256_from_path(&path)?; - let sha_seven = sha.chars().take(7).collect::(); - let sha_seven_path = append_filename_with(&path, &format!("-{sha_seven}"), ".tgz")?; + bullet = bullet.sub_bullet(format!("Write {}", sha_seven_path.display(),)); + fs_err::copy(tar_file.path(), &sha_seven_path)?; - bullet = bullet.sub_bullet(format!("Write {}", sha_seven_path.display(),)); - fs_err::copy(tar_file.path(), &sha_seven_path)?; - - append_manifest( + let timestamp = chrono::Utc::now(); + for cpu_arch in &[CpuArch::new("amd64")?, CpuArch::new("arm64")?] { + let distro_version = base_image.distro_version(); + append_inventory( &inventory, Artifact { version: ManifestVersion::new(version), os: inventory::artifact::Os::Linux, - arch: cpu_arch.into(), + arch: cpu_arch.try_into()?, url: format!( "{S3_BASE_URL}/{}", sha_seven_path.strip_prefix(&volume_output_dir)?.display() ), checksum: format!("sha256:{sha}").parse()?, metadata: ArtifactMetadata { - distro_version: base_image.distro_version(), - timestamp: chrono::Utc::now(), + distro_version, + timestamp, }, }, )?; } + // Can be removed once manifest file support is fully rolled out + for cpu_arch in &[CpuArch::new("amd64")?, CpuArch::new("arm64")?] { + let dir = volume_output_dir + .join(base_image.to_string()) + .join(cpu_arch.to_string()); + fs_err::create_dir_all(&dir)?; + + let path = dir.join(&tgz_name); + bullet = bullet.sub_bullet(format!("Write {}", path.display())); + fs_err::copy(tar_file.path(), &path)?; + } + bullet.done() }; diff --git a/jruby_inventory.toml b/jruby_inventory.toml index 8b13789..96caf6b 100644 --- a/jruby_inventory.toml +++ b/jruby_inventory.toml @@ -1 +1,22 @@ +[[artifacts]] +version = "9.4.8.0" +os = "linux" +arch = "amd64" +url = "https://heroku-buildpack-ruby.s3.us-east-1.amazonaws.com/heroku-24/ruby-3.1.4-jruby-9.4.8.0-dd073bd.tgz" +checksum = "sha256:dd073bda5665e758c3e6f861a6df435175c8e8faf5ec75bc2afaab1e3eebb2c7" + +[artifacts.metadata] +timestamp = "2024-07-24T16:17:35.341413Z" +distro_version = "24.04" + +[[artifacts]] +version = "9.4.8.0" +os = "linux" +arch = "arm64" +url = "https://heroku-buildpack-ruby.s3.us-east-1.amazonaws.com/heroku-24/ruby-3.1.4-jruby-9.4.8.0-dd073bd.tgz" +checksum = "sha256:dd073bda5665e758c3e6f861a6df435175c8e8faf5ec75bc2afaab1e3eebb2c7" + +[artifacts.metadata] +timestamp = "2024-07-24T16:17:35.341917Z" +distro_version = "24.04" diff --git a/shared/Cargo.toml b/shared/Cargo.toml index 09cfc82..7aa813c 100644 --- a/shared/Cargo.toml +++ b/shared/Cargo.toml @@ -3,6 +3,10 @@ name = "shared" version = "0.1.0" edition = "2021" +[[bin]] +name = "inventory_check" +path = "src/bin/inventory_check.rs" + [dependencies] glob = { workspace = true} clap = { workspace = true} @@ -23,6 +27,9 @@ serde = { workspace = true } toml = { workspace = true } inventory = { workspace = true } fs2 = { workspace = true } +rayon = { workspace = true } +hex = { workspace = true } + [dev-dependencies] tiny_http = "0.12.0" diff --git a/shared/src/bin/inventory_check.rs b/shared/src/bin/inventory_check.rs new file mode 100644 index 0000000..e42951b --- /dev/null +++ b/shared/src/bin/inventory_check.rs @@ -0,0 +1,29 @@ +use clap::Parser; +use indoc::formatdoc; +use shared::inventory_check; +use std::path::{Path, PathBuf}; + +#[derive(Parser, Debug)] +struct Args { + path: PathBuf, +} + +fn check(path: &Path) -> Result<(), Box> { + let contents = fs_err::read_to_string(path)?; + inventory_check(&contents)?; + Ok(()) +} + +fn main() { + let args = Args::parse(); + if let Err(error) = check(&args.path) { + bullet_stream::Print::new(std::io::stderr()) + .without_header() + .error(formatdoc! {" + ❌ Command failed ❌ + + {error} + "}); + std::process::exit(1); + } +} diff --git a/shared/src/inventory_help.rs b/shared/src/inventory_help.rs new file mode 100644 index 0000000..45de471 --- /dev/null +++ b/shared/src/inventory_help.rs @@ -0,0 +1,226 @@ +use crate::base_image::DistroVersion; +use crate::{download_tar, Error, TarDownloadPath}; +use chrono::{DateTime, Utc}; +use fs2::FileExt; +use inventory::artifact::Artifact; +use inventory::checksum::Checksum; +use inventory::inventory::Inventory; +use rayon::prelude::*; +use serde::{Deserialize, Serialize}; +use sha2::Sha256; +use std::borrow::Borrow; +use std::io::{Read, Write}; +use std::path::Path; + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct ArtifactMetadata { + pub timestamp: DateTime, + pub distro_version: DistroVersion, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct ManifestVersion(pub String); + +impl ManifestVersion { + pub fn new(version: &str) -> Self { + Self(version.to_string()) + } +} + +/// ``` +/// use shared::inventory_check; +/// +/// let contents = r#" +/// [[artifacts]] +/// version = "9.4.8.0" +/// os = "linux" +/// arch = "amd64" +/// url = "https://heroku-buildpack-ruby.s3.us-east-1.amazonaws.com/heroku-24/ruby-3.1.4-jruby-9.4.8.0.tgz" +/// checksum = "sha256:815b31d2b204a524bf74aabae341bf85353add4d1128d5d276d08fa5e8ff3c39" +/// +/// [artifacts.metadata] +/// timestamp = "2024-07-24T16:17:35.341413Z" +/// distro_version = "24.04" +/// "#; +/// inventory_check(contents).unwrap(); +/// ``` +pub fn inventory_check(contents: &str) -> Result<(), Error> { + if contents.trim().is_empty() { + return Ok(()); + } + + let inventory = contents + .parse::>() + .map_err(|e| Error::Other(format!("Could not parse inventory. Error: {e}")))?; + + let results = inventory + .artifacts + .par_iter() + .map(|artifact| { + let temp = tempfile::tempdir().expect("Tempdir"); + let dir = temp.path(); + let path = dir.join("file.tar"); + + download_tar(&artifact.url, &TarDownloadPath(path.clone())) + .map_err(|e| format!("Error {e}")) + .and_then(|_| { + sha256_from_path(&path) + .map_err(|e| format!("Error {e}")) + .and_then(|checksum_string| { + format!("sha256:{checksum_string}") + .parse() + .map_err(|e| format!("Error {e}")) + }) + }) + .and_then(|checksum: Checksum| { + if checksum == artifact.checksum { + Ok(()) + } else { + Err(format!( + "Checksum mismatch for {url} expected {expected} got {actual}", + url = artifact.url, + expected = hex::encode(&artifact.checksum.value), + actual = hex::encode(&checksum.value) + )) + } + }) + }) + .collect::>>(); + + if results.iter().any(|r| r.is_err()) { + Err(Error::Other( + results + .iter() + .map(|r| r.as_ref().unwrap_err().borrow()) + .collect::>() + .join("\n"), + )) + } else { + Ok(()) + } +} + +/// Appends the given artifact to the inventory file at the given path +/// +/// If the file doesn't exist, it will be created. +/// Uses file locking to ensure atomic updating. +pub fn append_inventory( + path: &Path, + artifact: Artifact, +) -> Result<(), Error> { + fs_err::create_dir_all( + path.parent().ok_or_else(|| { + Error::Other(format!("Cannot determine parent from {}", path.display())) + })?, + ) + .map_err(Error::FsError)?; + + let mut file: std::fs::File = fs_err::OpenOptions::new() + .write(true) + .create(true) + .open(path) + .map_err(Error::FsError)? + .into(); + + file.lock_exclusive().map_err(|e| { + Error::Other(format!( + "Error {e} obtaining file lock on {}", + path.display() + )) + })?; + + let inventory_string = fs_err::read_to_string(path).map_err(Error::FsError)?; + let mut inventory = parse_contents(&inventory_string)?; + inventory.push(artifact); + + writeln!(file, "{inventory}").expect("Writeable file"); + + file.unlock().map_err(|e| { + Error::Other(format!( + "Error {e} releasing file lock on {}", + path.display() + )) + })?; + + Ok(()) +} + +fn parse_contents( + contents: &str, +) -> Result, Error> { + if contents.trim().is_empty() { + Ok(Inventory { + artifacts: Vec::new(), + }) + } else { + contents + .parse::>() + .map_err(|e| Error::Other(format!("Error {e} parsing inventory:\n{contents}"))) + } +} + +/// Returns the sha256 hash of the file at the given path +pub fn sha256_from_path(path: &Path) -> Result { + digest::(fs_err::File::open(path).map_err(Error::FsError)?) + .map(|digest| format!("{:x}", digest)) + .map_err(|e| { + Error::Other(format!( + "Error {e} calculating sha256 for {path}", + path = path.display() + )) + }) +} + +pub fn digest(mut input: impl Read) -> Result, std::io::Error> +where + D: Default + sha2::digest::Update + sha2::digest::FixedOutput, +{ + let mut digest = D::default(); + + let mut buffer = [0x00; 1024]; + loop { + let bytes_read = input.read(&mut buffer)?; + + if bytes_read > 0 { + digest.update(&buffer[..bytes_read]); + } else { + break; + } + } + + Ok(digest.finalize_fixed()) +} + +#[cfg(test)] +mod test { + use crate::BaseImage; + + use super::*; + + #[test] + fn test_append_inventory() { + let temp = tempfile::tempdir().expect("Tempdir"); + let path = temp.path().join("inventory.toml"); + let artifact = Artifact { + os: inventory::artifact::Os::Linux, + arch: inventory::artifact::Arch::Amd64, + version: ManifestVersion("1.0.0".to_string()), + checksum: "sha256:dd073bda5665e758c3e6f861a6df435175c8e8faf5ec75bc2afaab1e3eebb2c7" + .parse() + .unwrap(), + metadata: ArtifactMetadata { + timestamp: Utc::now(), + distro_version: BaseImage::new("heroku-24").unwrap().distro_version(), + }, + url: "https://example.com".to_string(), + }; + append_inventory(&path, artifact.clone()).unwrap(); + + let inventory = parse_contents(&fs_err::read_to_string(&path).unwrap()).unwrap(); + assert_eq!(1, inventory.artifacts.len()); + + append_inventory(&path, artifact.clone()).unwrap(); + let inventory = parse_contents(&fs_err::read_to_string(&path).unwrap()).unwrap(); + assert_eq!(2, inventory.artifacts.len()); + } +} diff --git a/shared/src/lib.rs b/shared/src/lib.rs index efa5961..cc201cb 100644 --- a/shared/src/lib.rs +++ b/shared/src/lib.rs @@ -1,92 +1,20 @@ -use base_image::DistroVersion; use bullet_stream::state::SubBullet; use bullet_stream::Print; -use chrono::{DateTime, Utc}; -use fs2::FileExt; use fs_err::{File, PathExt}; use fun_run::CommandWithName; -use inventory::artifact::Artifact; -use inventory::inventory::Inventory; -use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; use std::io::{Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; use std::process::Command; mod base_image; mod download_ruby_version; +mod inventory_help; pub use base_image::{BaseImage, CpuArch, CpuArchError}; pub use download_ruby_version::RubyDownloadVersion; - -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct ArtifactMetadata { - pub timestamp: DateTime, - pub distro_version: DistroVersion, -} - -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct ManifestVersion(pub String); - -impl ManifestVersion { - pub fn new(version: &str) -> Self { - Self(version.to_string()) - } -} - -/// Appends the given artifact to the inventory file at the given path -/// -/// If the file doesn't exist, it will be created. -/// Uses file locking to ensure atomic updating. -pub fn append_manifest( - path: &Path, - artifact: Artifact, -) -> Result<(), Error> { - fs_err::create_dir_all( - path.parent().ok_or_else(|| { - Error::Other(format!("Cannot determine parent from {}", path.display())) - })?, - ) - .map_err(Error::FsError)?; - - let mut file: std::fs::File = fs_err::OpenOptions::new() - .append(true) - .create(true) - .open(path) - .map_err(Error::FsError)? - .into(); - - file.lock_exclusive().map_err(|e| { - Error::Other(format!( - "Error {e} obtaining file lock on {}", - path.display() - )) - })?; - - let inventory_string = fs_err::read_to_string(path).map_err(Error::FsError)?; - - let mut inventory = if inventory_string.trim().is_empty() { - Inventory { - artifacts: Vec::new(), - } - } else { - inventory_string - .parse::>() - .map_err(|e| Error::Other(format!("Error {e} parsing inventory: {}", path.display())))? - }; - - inventory.push(artifact); - writeln!(file, "{inventory}").expect("Writeable file"); - - file.unlock().map_err(|e| { - Error::Other(format!( - "Error {e} releasing file lock on {}", - path.display() - )) - })?; - - Ok(()) -} +pub use inventory_help::{ + append_inventory, inventory_check, sha256_from_path, ArtifactMetadata, ManifestVersion, +}; /// Appends the given string after the filename and before the `ends_with` /// @@ -125,14 +53,6 @@ pub fn append_filename_with(path: &Path, append: &str, ends_with: &str) -> Resul Ok(parent.join(format!("{file_base}{append}{ends_with}"))) } -/// Returns the sha256 hash of the file at the given path -pub fn sha256_from_path(path: &Path) -> Result { - let mut hasher = Sha256::new(); - hasher.update(fs_err::read(path).map_err(Error::FsError)?); - - Ok(format!("{:x}", hasher.finalize())) -} - #[derive(Debug, Clone)] pub struct TarDownloadPath(pub PathBuf); From c486de7aeb0f9467f4d56eeb897014e43eedd462 Mon Sep 17 00:00:00 2001 From: Schneems Date: Wed, 24 Jul 2024 15:59:19 -0500 Subject: [PATCH 04/18] Use GemVersion --- Cargo.lock | 39 +++++++++++++++++++++++++ Cargo.toml | 1 + jruby_executable/Cargo.toml | 1 + jruby_executable/src/bin/jruby_build.rs | 7 +++-- shared/Cargo.toml | 1 + shared/src/inventory_help.rs | 22 +++++--------- shared/src/lib.rs | 4 +-- 7 files changed, 55 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8746d87..d2ce5ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -129,6 +129,21 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bitflags" version = "1.3.2" @@ -357,6 +372,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "fancy-regex" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.1.0" @@ -494,6 +520,17 @@ dependencies = [ "slab", ] +[[package]] +name = "gem_version" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804d260cc5d12cb2ed81b4e9228eb48c989a36e86b5b7e5c6113509f88d4a69a" +dependencies = [ + "fancy-regex", + "regex", + "serde", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -761,6 +798,7 @@ dependencies = [ "flate2", "fs-err", "fun_run", + "gem_version", "glob", "indoc", "inventory", @@ -1281,6 +1319,7 @@ dependencies = [ "fs-err", "fs2", "fun_run", + "gem_version", "glob", "hex", "indoc", diff --git a/Cargo.toml b/Cargo.toml index e7ca917..e6bb04a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ serde = {version = "1.0", features = ["derive"] } inventory = { git = "https://github.com/Malax/inventory", features = ["sha2"] } rayon = "1.10" hex = "0.4.3" +gem_version = "0.3" # File locking (FLOCK) fs2 = "0.4" diff --git a/jruby_executable/Cargo.toml b/jruby_executable/Cargo.toml index 43dc730..9a2a6cd 100644 --- a/jruby_executable/Cargo.toml +++ b/jruby_executable/Cargo.toml @@ -35,3 +35,4 @@ bullet_stream = { workspace = true } inventory = { workspace = true } chrono = { workspace = true } sha2 = { workspace = true } +gem_version = { workspace = true } diff --git a/jruby_executable/src/bin/jruby_build.rs b/jruby_executable/src/bin/jruby_build.rs index 3f94bc6..02a48fe 100644 --- a/jruby_executable/src/bin/jruby_build.rs +++ b/jruby_executable/src/bin/jruby_build.rs @@ -1,16 +1,17 @@ use bullet_stream::{style, Print}; use clap::Parser; use fs_err::PathExt; +use gem_version::GemVersion; use indoc::formatdoc; use inventory::artifact::Artifact; use jruby_executable::jruby_build_properties; use shared::{ append_filename_with, append_inventory, download_tar, sha256_from_path, source_dir, - tar_dir_to_file, untar_to_dir, ArtifactMetadata, BaseImage, CpuArch, ManifestVersion, - TarDownloadPath, + tar_dir_to_file, untar_to_dir, ArtifactMetadata, BaseImage, CpuArch, TarDownloadPath, }; use std::convert::From; use std::error::Error; +use std::str::FromStr; static S3_BASE_URL: &str = "https://heroku-buildpack-ruby.s3.us-east-1.amazonaws.com"; @@ -127,7 +128,7 @@ fn jruby_build(args: &Args) -> Result<(), Box> { append_inventory( &inventory, Artifact { - version: ManifestVersion::new(version), + version: GemVersion::from_str(version)?, os: inventory::artifact::Os::Linux, arch: cpu_arch.try_into()?, url: format!( diff --git a/shared/Cargo.toml b/shared/Cargo.toml index 7aa813c..3a624eb 100644 --- a/shared/Cargo.toml +++ b/shared/Cargo.toml @@ -29,6 +29,7 @@ inventory = { workspace = true } fs2 = { workspace = true } rayon = { workspace = true } hex = { workspace = true } +gem_version = { workspace = true } [dev-dependencies] diff --git a/shared/src/inventory_help.rs b/shared/src/inventory_help.rs index 45de471..98c75bc 100644 --- a/shared/src/inventory_help.rs +++ b/shared/src/inventory_help.rs @@ -2,6 +2,7 @@ use crate::base_image::DistroVersion; use crate::{download_tar, Error, TarDownloadPath}; use chrono::{DateTime, Utc}; use fs2::FileExt; +use gem_version::GemVersion; use inventory::artifact::Artifact; use inventory::checksum::Checksum; use inventory::inventory::Inventory; @@ -18,15 +19,6 @@ pub struct ArtifactMetadata { pub distro_version: DistroVersion, } -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct ManifestVersion(pub String); - -impl ManifestVersion { - pub fn new(version: &str) -> Self { - Self(version.to_string()) - } -} - /// ``` /// use shared::inventory_check; /// @@ -50,7 +42,7 @@ pub fn inventory_check(contents: &str) -> Result<(), Error> { } let inventory = contents - .parse::>() + .parse::>() .map_err(|e| Error::Other(format!("Could not parse inventory. Error: {e}")))?; let results = inventory @@ -106,7 +98,7 @@ pub fn inventory_check(contents: &str) -> Result<(), Error> { /// Uses file locking to ensure atomic updating. pub fn append_inventory( path: &Path, - artifact: Artifact, + artifact: Artifact, ) -> Result<(), Error> { fs_err::create_dir_all( path.parent().ok_or_else(|| { @@ -147,14 +139,14 @@ pub fn append_inventory( fn parse_contents( contents: &str, -) -> Result, Error> { +) -> Result, Error> { if contents.trim().is_empty() { Ok(Inventory { artifacts: Vec::new(), }) } else { contents - .parse::>() + .parse::>() .map_err(|e| Error::Other(format!("Error {e} parsing inventory:\n{contents}"))) } } @@ -193,6 +185,8 @@ where #[cfg(test)] mod test { + use std::str::FromStr; + use crate::BaseImage; use super::*; @@ -204,7 +198,7 @@ mod test { let artifact = Artifact { os: inventory::artifact::Os::Linux, arch: inventory::artifact::Arch::Amd64, - version: ManifestVersion("1.0.0".to_string()), + version: GemVersion::from_str("1.0.0").unwrap(), checksum: "sha256:dd073bda5665e758c3e6f861a6df435175c8e8faf5ec75bc2afaab1e3eebb2c7" .parse() .unwrap(), diff --git a/shared/src/lib.rs b/shared/src/lib.rs index cc201cb..2628f3a 100644 --- a/shared/src/lib.rs +++ b/shared/src/lib.rs @@ -12,9 +12,7 @@ mod inventory_help; pub use base_image::{BaseImage, CpuArch, CpuArchError}; pub use download_ruby_version::RubyDownloadVersion; -pub use inventory_help::{ - append_inventory, inventory_check, sha256_from_path, ArtifactMetadata, ManifestVersion, -}; +pub use inventory_help::{append_inventory, inventory_check, sha256_from_path, ArtifactMetadata}; /// Appends the given string after the filename and before the `ends_with` /// From 3980900d8f6ebb6a93d8087819ef3041ce386007 Mon Sep 17 00:00:00 2001 From: Schneems Date: Wed, 24 Jul 2024 16:26:50 -0500 Subject: [PATCH 05/18] Refactor atomic calls into their own function --- shared/src/inventory_help.rs | 56 ++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/shared/src/inventory_help.rs b/shared/src/inventory_help.rs index 98c75bc..1a72d4c 100644 --- a/shared/src/inventory_help.rs +++ b/shared/src/inventory_help.rs @@ -2,6 +2,7 @@ use crate::base_image::DistroVersion; use crate::{download_tar, Error, TarDownloadPath}; use chrono::{DateTime, Utc}; use fs2::FileExt; +use fs_err::os::unix::fs; use gem_version::GemVersion; use inventory::artifact::Artifact; use inventory::checksum::Checksum; @@ -10,7 +11,7 @@ use rayon::prelude::*; use serde::{Deserialize, Serialize}; use sha2::Sha256; use std::borrow::Borrow; -use std::io::{Read, Write}; +use std::io::{Read, Seek, Write}; use std::path::Path; #[derive(Debug, Serialize, Deserialize, Clone)] @@ -100,6 +101,24 @@ pub fn append_inventory( path: &Path, artifact: Artifact, ) -> Result<(), Error> { + atomic_file_access(path, |file, contents| { + let mut inventory = parse_contents(contents)?; + + inventory.push(artifact); + + writeln!(file, "{inventory}").expect("Writeable file"); + + Ok(()) + }) + .map_err(|e| Error::Other(e.to_string()))?; + + Ok(()) +} + +fn atomic_file_access(path: &Path, f: F) -> Result> +where + F: FnOnce(&mut std::fs::File, &str) -> Result>, +{ fs_err::create_dir_all( path.parent().ok_or_else(|| { Error::Other(format!("Cannot determine parent from {}", path.display())) @@ -108,33 +127,20 @@ pub fn append_inventory( .map_err(Error::FsError)?; let mut file: std::fs::File = fs_err::OpenOptions::new() + .read(true) .write(true) .create(true) - .open(path) - .map_err(Error::FsError)? + .open(path)? .into(); - - file.lock_exclusive().map_err(|e| { - Error::Other(format!( - "Error {e} obtaining file lock on {}", - path.display() - )) - })?; - - let inventory_string = fs_err::read_to_string(path).map_err(Error::FsError)?; - let mut inventory = parse_contents(&inventory_string)?; - inventory.push(artifact); - - writeln!(file, "{inventory}").expect("Writeable file"); - - file.unlock().map_err(|e| { - Error::Other(format!( - "Error {e} releasing file lock on {}", - path.display() - )) - })?; - - Ok(()) + file.lock_exclusive()?; + use std::io::Seek; + + let mut contents = String::new(); + file.read_to_string(&mut contents).map_err(Error::FsError)?; + file.rewind()?; + let result: Result> = f(&mut file, &contents); + file.unlock()?; + result } fn parse_contents( From 77a4f1ec4e2b7ac10eb338b249a0b7e54271de6c Mon Sep 17 00:00:00 2001 From: Schneems Date: Wed, 24 Jul 2024 16:45:17 -0500 Subject: [PATCH 06/18] Refactor: Prefer composable methods --- jruby_executable/src/bin/jruby_build.rs | 21 +++++++---- shared/src/inventory_help.rs | 47 ++++++++++--------------- shared/src/lib.rs | 4 ++- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/jruby_executable/src/bin/jruby_build.rs b/jruby_executable/src/bin/jruby_build.rs index 02a48fe..34bda4e 100644 --- a/jruby_executable/src/bin/jruby_build.rs +++ b/jruby_executable/src/bin/jruby_build.rs @@ -6,11 +6,13 @@ use indoc::formatdoc; use inventory::artifact::Artifact; use jruby_executable::jruby_build_properties; use shared::{ - append_filename_with, append_inventory, download_tar, sha256_from_path, source_dir, - tar_dir_to_file, untar_to_dir, ArtifactMetadata, BaseImage, CpuArch, TarDownloadPath, + append_filename_with, atomic_file_contents, download_tar, parse_inventory, sha256_from_path, + source_dir, tar_dir_to_file, untar_to_dir, ArtifactMetadata, BaseImage, CpuArch, + TarDownloadPath, }; use std::convert::From; use std::error::Error; +use std::io::Write; use std::str::FromStr; static S3_BASE_URL: &str = "https://heroku-buildpack-ruby.s3.us-east-1.amazonaws.com"; @@ -125,9 +127,10 @@ fn jruby_build(args: &Args) -> Result<(), Box> { let timestamp = chrono::Utc::now(); for cpu_arch in &[CpuArch::new("amd64")?, CpuArch::new("arm64")?] { let distro_version = base_image.distro_version(); - append_inventory( - &inventory, - Artifact { + + atomic_file_contents(&inventory, |file, contents| { + let mut inventory = parse_inventory(contents)?; + inventory.push(Artifact { version: GemVersion::from_str(version)?, os: inventory::artifact::Os::Linux, arch: cpu_arch.try_into()?, @@ -140,8 +143,12 @@ fn jruby_build(args: &Args) -> Result<(), Box> { distro_version, timestamp, }, - }, - )?; + }); + + writeln!(file, "{inventory}").expect("Writeable file"); + + Ok(()) + })? } // Can be removed once manifest file support is fully rolled out diff --git a/shared/src/inventory_help.rs b/shared/src/inventory_help.rs index 1a72d4c..f5b9cd5 100644 --- a/shared/src/inventory_help.rs +++ b/shared/src/inventory_help.rs @@ -93,29 +93,7 @@ pub fn inventory_check(contents: &str) -> Result<(), Error> { } } -/// Appends the given artifact to the inventory file at the given path -/// -/// If the file doesn't exist, it will be created. -/// Uses file locking to ensure atomic updating. -pub fn append_inventory( - path: &Path, - artifact: Artifact, -) -> Result<(), Error> { - atomic_file_access(path, |file, contents| { - let mut inventory = parse_contents(contents)?; - - inventory.push(artifact); - - writeln!(file, "{inventory}").expect("Writeable file"); - - Ok(()) - }) - .map_err(|e| Error::Other(e.to_string()))?; - - Ok(()) -} - -fn atomic_file_access(path: &Path, f: F) -> Result> +pub fn atomic_file_contents(path: &Path, f: F) -> Result> where F: FnOnce(&mut std::fs::File, &str) -> Result>, { @@ -143,7 +121,7 @@ where result } -fn parse_contents( +pub fn parse_inventory( contents: &str, ) -> Result, Error> { if contents.trim().is_empty() { @@ -214,13 +192,26 @@ mod test { }, url: "https://example.com".to_string(), }; - append_inventory(&path, artifact.clone()).unwrap(); - let inventory = parse_contents(&fs_err::read_to_string(&path).unwrap()).unwrap(); + atomic_file_contents(&path, |file, contents| { + let mut inventory = parse_inventory(contents)?; + inventory.push(artifact.clone()); + write!(file, "{inventory}").expect("Writeable file"); + Ok(()) + }) + .unwrap(); + + let inventory = parse_inventory(&fs_err::read_to_string(&path).unwrap()).unwrap(); assert_eq!(1, inventory.artifacts.len()); - append_inventory(&path, artifact.clone()).unwrap(); - let inventory = parse_contents(&fs_err::read_to_string(&path).unwrap()).unwrap(); + atomic_file_contents(&path, |file, contents| { + let mut inventory = parse_inventory(contents)?; + inventory.push(artifact.clone()); + write!(file, "{inventory}").expect("Writeable file"); + Ok(()) + }) + .unwrap(); + let inventory = parse_inventory(&fs_err::read_to_string(&path).unwrap()).unwrap(); assert_eq!(2, inventory.artifacts.len()); } } diff --git a/shared/src/lib.rs b/shared/src/lib.rs index 2628f3a..04f9561 100644 --- a/shared/src/lib.rs +++ b/shared/src/lib.rs @@ -12,7 +12,9 @@ mod inventory_help; pub use base_image::{BaseImage, CpuArch, CpuArchError}; pub use download_ruby_version::RubyDownloadVersion; -pub use inventory_help::{append_inventory, inventory_check, sha256_from_path, ArtifactMetadata}; +pub use inventory_help::{ + atomic_file_contents, inventory_check, parse_inventory, sha256_from_path, ArtifactMetadata, +}; /// Appends the given string after the filename and before the `ends_with` /// From 5d39fdcd4135436685216ab50465435224c00a82 Mon Sep 17 00:00:00 2001 From: Schneems Date: Wed, 24 Jul 2024 17:04:24 -0500 Subject: [PATCH 07/18] I did not mean to commit that --- jruby_inventory.toml | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/jruby_inventory.toml b/jruby_inventory.toml index 96caf6b..e69de29 100644 --- a/jruby_inventory.toml +++ b/jruby_inventory.toml @@ -1,22 +0,0 @@ -[[artifacts]] -version = "9.4.8.0" -os = "linux" -arch = "amd64" -url = "https://heroku-buildpack-ruby.s3.us-east-1.amazonaws.com/heroku-24/ruby-3.1.4-jruby-9.4.8.0-dd073bd.tgz" -checksum = "sha256:dd073bda5665e758c3e6f861a6df435175c8e8faf5ec75bc2afaab1e3eebb2c7" - -[artifacts.metadata] -timestamp = "2024-07-24T16:17:35.341413Z" -distro_version = "24.04" - -[[artifacts]] -version = "9.4.8.0" -os = "linux" -arch = "arm64" -url = "https://heroku-buildpack-ruby.s3.us-east-1.amazonaws.com/heroku-24/ruby-3.1.4-jruby-9.4.8.0-dd073bd.tgz" -checksum = "sha256:dd073bda5665e758c3e6f861a6df435175c8e8faf5ec75bc2afaab1e3eebb2c7" - -[artifacts.metadata] -timestamp = "2024-07-24T16:17:35.341917Z" -distro_version = "24.04" - From 2c4b20d0858b1e8fb155bd56da87561bb6954313 Mon Sep 17 00:00:00 2001 From: Schneems Date: Wed, 24 Jul 2024 17:05:48 -0500 Subject: [PATCH 08/18] Replace older entries by default --- jruby_executable/src/bin/jruby_build.rs | 34 +++++++++++++++---------- shared/src/inventory_help.rs | 9 +++---- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/jruby_executable/src/bin/jruby_build.rs b/jruby_executable/src/bin/jruby_build.rs index 34bda4e..f8ea97a 100644 --- a/jruby_executable/src/bin/jruby_build.rs +++ b/jruby_executable/src/bin/jruby_build.rs @@ -127,24 +127,30 @@ fn jruby_build(args: &Args) -> Result<(), Box> { let timestamp = chrono::Utc::now(); for cpu_arch in &[CpuArch::new("amd64")?, CpuArch::new("arm64")?] { let distro_version = base_image.distro_version(); - + let artifact = Artifact { + version: GemVersion::from_str(version)?, + os: inventory::artifact::Os::Linux, + arch: cpu_arch.try_into()?, + url: format!( + "{S3_BASE_URL}/{}", + sha_seven_path.strip_prefix(&volume_output_dir)?.display() + ), + checksum: format!("sha256:{sha}").parse()?, + metadata: ArtifactMetadata { + distro_version, + timestamp, + }, + }; atomic_file_contents(&inventory, |file, contents| { let mut inventory = parse_inventory(contents)?; - inventory.push(Artifact { - version: GemVersion::from_str(version)?, - os: inventory::artifact::Os::Linux, - arch: cpu_arch.try_into()?, - url: format!( - "{S3_BASE_URL}/{}", - sha_seven_path.strip_prefix(&volume_output_dir)?.display() - ), - checksum: format!("sha256:{sha}").parse()?, - metadata: ArtifactMetadata { - distro_version, - timestamp, - }, + inventory.artifacts.retain(|a| { + a.version != artifact.version + || a.arch != artifact.arch + || a.metadata.distro_version != artifact.metadata.distro_version }); + inventory.push(artifact); + writeln!(file, "{inventory}").expect("Writeable file"); Ok(()) diff --git a/shared/src/inventory_help.rs b/shared/src/inventory_help.rs index f5b9cd5..ef57d69 100644 --- a/shared/src/inventory_help.rs +++ b/shared/src/inventory_help.rs @@ -2,16 +2,14 @@ use crate::base_image::DistroVersion; use crate::{download_tar, Error, TarDownloadPath}; use chrono::{DateTime, Utc}; use fs2::FileExt; -use fs_err::os::unix::fs; use gem_version::GemVersion; -use inventory::artifact::Artifact; use inventory::checksum::Checksum; use inventory::inventory::Inventory; use rayon::prelude::*; use serde::{Deserialize, Serialize}; use sha2::Sha256; use std::borrow::Borrow; -use std::io::{Read, Seek, Write}; +use std::io::Read; use std::path::Path; #[derive(Debug, Serialize, Deserialize, Clone)] @@ -169,9 +167,10 @@ where #[cfg(test)] mod test { - use std::str::FromStr; - use crate::BaseImage; + use inventory::artifact::Artifact; + use std::io::Write; + use std::str::FromStr; use super::*; From ebe5e06fcaecc662cb5ab3c19b79b4f1d9acbaa9 Mon Sep 17 00:00:00 2001 From: Schneems Date: Thu, 25 Jul 2024 15:32:32 -0500 Subject: [PATCH 09/18] Refactor: API to mutate the inventory, not the file --- jruby_executable/src/bin/jruby_build.rs | 12 +++--------- shared/src/inventory_help.rs | 19 +++++++++++++++++-- shared/src/lib.rs | 2 +- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/jruby_executable/src/bin/jruby_build.rs b/jruby_executable/src/bin/jruby_build.rs index f8ea97a..ce47c71 100644 --- a/jruby_executable/src/bin/jruby_build.rs +++ b/jruby_executable/src/bin/jruby_build.rs @@ -6,13 +6,11 @@ use indoc::formatdoc; use inventory::artifact::Artifact; use jruby_executable::jruby_build_properties; use shared::{ - append_filename_with, atomic_file_contents, download_tar, parse_inventory, sha256_from_path, - source_dir, tar_dir_to_file, untar_to_dir, ArtifactMetadata, BaseImage, CpuArch, - TarDownloadPath, + append_filename_with, atomic_inventory_update, download_tar, sha256_from_path, source_dir, + tar_dir_to_file, untar_to_dir, ArtifactMetadata, BaseImage, CpuArch, TarDownloadPath, }; use std::convert::From; use std::error::Error; -use std::io::Write; use std::str::FromStr; static S3_BASE_URL: &str = "https://heroku-buildpack-ruby.s3.us-east-1.amazonaws.com"; @@ -141,8 +139,7 @@ fn jruby_build(args: &Args) -> Result<(), Box> { timestamp, }, }; - atomic_file_contents(&inventory, |file, contents| { - let mut inventory = parse_inventory(contents)?; + atomic_inventory_update(&inventory, |inventory| { inventory.artifacts.retain(|a| { a.version != artifact.version || a.arch != artifact.arch @@ -150,9 +147,6 @@ fn jruby_build(args: &Args) -> Result<(), Box> { }); inventory.push(artifact); - - writeln!(file, "{inventory}").expect("Writeable file"); - Ok(()) })? } diff --git a/shared/src/inventory_help.rs b/shared/src/inventory_help.rs index ef57d69..feddf12 100644 --- a/shared/src/inventory_help.rs +++ b/shared/src/inventory_help.rs @@ -10,6 +10,7 @@ use serde::{Deserialize, Serialize}; use sha2::Sha256; use std::borrow::Borrow; use std::io::Read; +use std::io::Write; use std::path::Path; #[derive(Debug, Serialize, Deserialize, Clone)] @@ -91,7 +92,7 @@ pub fn inventory_check(contents: &str) -> Result<(), Error> { } } -pub fn atomic_file_contents(path: &Path, f: F) -> Result> +fn atomic_file_contents(path: &Path, f: F) -> Result> where F: FnOnce(&mut std::fs::File, &str) -> Result>, { @@ -119,7 +120,21 @@ where result } -pub fn parse_inventory( +pub fn atomic_inventory_update(path: &Path, f: F) -> Result<(), Box> +where + F: FnOnce( + &mut Inventory, + ) -> Result<(), Box>, +{ + atomic_file_contents(path, |file, contents| { + let mut inventory = parse_inventory(contents)?; + f(&mut inventory)?; + write!(file, "{inventory}").map_err(Error::FsError)?; + Ok(()) + }) +} + +fn parse_inventory( contents: &str, ) -> Result, Error> { if contents.trim().is_empty() { diff --git a/shared/src/lib.rs b/shared/src/lib.rs index 04f9561..f5ceb48 100644 --- a/shared/src/lib.rs +++ b/shared/src/lib.rs @@ -13,7 +13,7 @@ mod inventory_help; pub use base_image::{BaseImage, CpuArch, CpuArchError}; pub use download_ruby_version::RubyDownloadVersion; pub use inventory_help::{ - atomic_file_contents, inventory_check, parse_inventory, sha256_from_path, ArtifactMetadata, + atomic_inventory_update, inventory_check, sha256_from_path, ArtifactMetadata, }; /// Appends the given string after the filename and before the `ends_with` From e2cb47dca9f37aef2ae63615fb705f443d71fbf9 Mon Sep 17 00:00:00 2001 From: Schneems Date: Thu, 25 Jul 2024 15:43:41 -0500 Subject: [PATCH 10/18] Test logic to preserve/remove artifacts from inventory --- jruby_executable/src/bin/jruby_build.rs | 13 ++++---- shared/src/inventory_help.rs | 41 +++++++++++++++++++++++++ shared/src/lib.rs | 3 +- 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/jruby_executable/src/bin/jruby_build.rs b/jruby_executable/src/bin/jruby_build.rs index ce47c71..326c1f7 100644 --- a/jruby_executable/src/bin/jruby_build.rs +++ b/jruby_executable/src/bin/jruby_build.rs @@ -6,8 +6,9 @@ use indoc::formatdoc; use inventory::artifact::Artifact; use jruby_executable::jruby_build_properties; use shared::{ - append_filename_with, atomic_inventory_update, download_tar, sha256_from_path, source_dir, - tar_dir_to_file, untar_to_dir, ArtifactMetadata, BaseImage, CpuArch, TarDownloadPath, + append_filename_with, artifact_is_different, atomic_inventory_update, download_tar, + sha256_from_path, source_dir, tar_dir_to_file, untar_to_dir, ArtifactMetadata, BaseImage, + CpuArch, TarDownloadPath, }; use std::convert::From; use std::error::Error; @@ -140,11 +141,9 @@ fn jruby_build(args: &Args) -> Result<(), Box> { }, }; atomic_inventory_update(&inventory, |inventory| { - inventory.artifacts.retain(|a| { - a.version != artifact.version - || a.arch != artifact.arch - || a.metadata.distro_version != artifact.metadata.distro_version - }); + inventory + .artifacts + .retain(|a| artifact_is_different(a, &artifact)); inventory.push(artifact); Ok(()) diff --git a/shared/src/inventory_help.rs b/shared/src/inventory_help.rs index feddf12..d4fcea9 100644 --- a/shared/src/inventory_help.rs +++ b/shared/src/inventory_help.rs @@ -180,6 +180,15 @@ where Ok(digest.finalize_fixed()) } +pub fn artifact_is_different( + a: &inventory::artifact::Artifact, + b: &inventory::artifact::Artifact, +) -> bool { + a.version != b.version + || a.arch != b.arch + || a.metadata.distro_version != b.metadata.distro_version +} + #[cfg(test)] mod test { use crate::BaseImage; @@ -189,6 +198,38 @@ mod test { use super::*; + #[test] + fn test_is_not_version_match() { + let a = Artifact { + os: inventory::artifact::Os::Linux, + arch: inventory::artifact::Arch::Amd64, + version: GemVersion::from_str("1.0.0").unwrap(), + checksum: "sha256:dd073bda5665e758c3e6f861a6df435175c8e8faf5ec75bc2afaab1e3eebb2c7" + .parse() + .unwrap(), + metadata: ArtifactMetadata { + timestamp: Utc::now(), + distro_version: BaseImage::new("heroku-24").unwrap().distro_version(), + }, + url: "https://example.com".to_string(), + }; + + let b = a.clone(); + assert!(!artifact_is_different(&a, &b)); + + let mut b = a.clone(); + b.version = GemVersion::from_str("1.0.1").unwrap(); + assert!(artifact_is_different(&a, &b)); + + let mut b = a.clone(); + b.arch = inventory::artifact::Arch::Arm64; + assert!(artifact_is_different(&a, &b)); + + let mut b = a.clone(); + b.metadata.distro_version = BaseImage::new("heroku-22").unwrap().distro_version(); + assert!(artifact_is_different(&a, &b)); + } + #[test] fn test_append_inventory() { let temp = tempfile::tempdir().expect("Tempdir"); diff --git a/shared/src/lib.rs b/shared/src/lib.rs index f5ceb48..ae73018 100644 --- a/shared/src/lib.rs +++ b/shared/src/lib.rs @@ -13,7 +13,8 @@ mod inventory_help; pub use base_image::{BaseImage, CpuArch, CpuArchError}; pub use download_ruby_version::RubyDownloadVersion; pub use inventory_help::{ - atomic_inventory_update, inventory_check, sha256_from_path, ArtifactMetadata, + artifact_is_different, atomic_inventory_update, inventory_check, sha256_from_path, + ArtifactMetadata, }; /// Appends the given string after the filename and before the `ends_with` From 928adf12b2e0d37e7e6862799f5c3bd48b0f4fab Mon Sep 17 00:00:00 2001 From: Schneems Date: Thu, 25 Jul 2024 15:56:04 -0500 Subject: [PATCH 11/18] Introduce Ruby manifest file --- Cargo.lock | 4 ++ jruby_executable/src/bin/jruby_build.rs | 3 +- ruby_executable/Cargo.toml | 4 ++ ruby_executable/src/bin/ruby_build.rs | 55 ++++++++++++++++++++++--- ruby_inventory.toml | 1 + 5 files changed, 60 insertions(+), 7 deletions(-) create mode 100644 ruby_inventory.toml diff --git a/Cargo.lock b/Cargo.lock index d2ce5ed..4ffdedc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1158,14 +1158,18 @@ name = "ruby_executable" version = "0.1.0" dependencies = [ "bullet_stream", + "chrono", "clap", "flate2", "fs-err", "fun_run", + "gem_version", "indoc", + "inventory", "nom", "regex", "reqwest", + "sha2", "shared", "tar", "tempfile", diff --git a/jruby_executable/src/bin/jruby_build.rs b/jruby_executable/src/bin/jruby_build.rs index 326c1f7..9acd3ae 100644 --- a/jruby_executable/src/bin/jruby_build.rs +++ b/jruby_executable/src/bin/jruby_build.rs @@ -32,11 +32,10 @@ fn jruby_build(args: &Args) -> Result<(), Box> { } = args; let mut log = Print::new(std::io::stderr()).h1("Building JRuby"); + let inventory = source_dir().join("jruby_inventory.toml"); let volume_cache_dir = source_dir().join("cache"); let volume_output_dir = source_dir().join("output"); - let inventory = source_dir().join("jruby_inventory.toml"); - fs_err::create_dir_all(&volume_cache_dir)?; let temp_dir = tempfile::tempdir()?; diff --git a/ruby_executable/Cargo.toml b/ruby_executable/Cargo.toml index b0185e0..e7df66b 100644 --- a/ruby_executable/Cargo.toml +++ b/ruby_executable/Cargo.toml @@ -17,3 +17,7 @@ tar = { workspace = true} tempfile = { workspace = true} thiserror = { workspace = true} bullet_stream = { workspace = true } +inventory = { workspace = true } +chrono = { workspace = true } +sha2 = { workspace = true } +gem_version = { workspace = true } diff --git a/ruby_executable/src/bin/ruby_build.rs b/ruby_executable/src/bin/ruby_build.rs index 829df6f..5754357 100644 --- a/ruby_executable/src/bin/ruby_build.rs +++ b/ruby_executable/src/bin/ruby_build.rs @@ -2,19 +2,24 @@ use bullet_stream::{style, Print}; use clap::Parser; use fs_err::PathExt; use fun_run::CommandWithName; +use gem_version::GemVersion; use indoc::{formatdoc, indoc}; +use inventory::artifact::Artifact; use shared::{ - download_tar, output_tar_path, source_dir, validate_version_for_stack, BaseImage, CpuArch, - RubyDownloadVersion, TarDownloadPath, + append_filename_with, artifact_is_different, atomic_inventory_update, download_tar, + output_tar_path, sha256_from_path, source_dir, validate_version_for_stack, ArtifactMetadata, + BaseImage, CpuArch, RubyDownloadVersion, TarDownloadPath, }; use std::{ io::Write, path::{Path, PathBuf}, process::Command, + str::FromStr, }; static INNER_OUTPUT: &str = "/tmp/output"; static INNER_CACHE: &str = "/tmp/cache"; +static S3_BASE_URL: &str = "https://heroku-buildpack-ruby.s3.us-east-1.amazonaws.com"; #[derive(Parser, Debug)] struct RubyArgs { @@ -57,6 +62,7 @@ fn ruby_build(args: &RubyArgs) -> Result<(), Box> { } = args; let mut log = Print::new(std::io::stderr()).h1("Building Ruby"); + let inventory = source_dir().join("ruby_inventory.toml"); let volume_cache_dir = source_dir().join("cache"); let volume_output_dir = source_dir().join("output"); @@ -116,12 +122,10 @@ fn ruby_build(args: &RubyArgs) -> Result<(), Box> { log = { let mut bullet = log.bullet("Make Ruby"); - let input_tar = PathBuf::from(INNER_CACHE).join(format!("ruby-source-{version}.tgz")); let output_tar = output_tar_path(Path::new(INNER_OUTPUT), version, base_image, arch); - - let volume_output = volume_output_dir.display(); let volume_cache = volume_cache_dir.display(); + let volume_output = volume_output_dir.display(); let mut docker_run = Command::new("docker"); docker_run.arg("run"); @@ -146,6 +150,47 @@ fn ruby_build(args: &RubyArgs) -> Result<(), Box> { format!("Running {}", style::command(docker_run.name())), |stdout, stderr| docker_run.stream_output(stdout, stderr), )?; + bullet.done() + }; + + log = { + let mut bullet = log.bullet(format!( + "Updating manifest {}", + style::value(inventory.to_string_lossy()) + )); + + let output_tar = output_tar_path(&volume_output_dir, version, base_image, arch); + + let sha = sha256_from_path(&output_tar)?; + let sha_seven = sha.chars().take(7).collect::(); + let sha_seven_path = append_filename_with(&output_tar, &format!("-{sha_seven}"), ".tgz")?; + let url = format!( + "{S3_BASE_URL}/{}", + sha_seven_path.strip_prefix(&volume_output_dir)?.display() + ); + + bullet = bullet.sub_bullet(format!("Copying SHA tgz {}", sha_seven_path.display(),)); + fs_err::copy(output_tar, &sha_seven_path)?; + + let artifact = Artifact { + version: GemVersion::from_str(&version.bundler_format())?, + os: inventory::artifact::Os::Linux, + arch: arch.try_into()?, + url, + checksum: format!("sha256:{sha}").parse()?, + metadata: ArtifactMetadata { + distro_version: base_image.distro_version(), + timestamp: chrono::Utc::now(), + }, + }; + + atomic_inventory_update(&inventory, |inventory| { + inventory + .artifacts + .retain(|a| artifact_is_different(a, &artifact)); + inventory.push(artifact); + Ok(()) + })?; bullet.done() }; diff --git a/ruby_inventory.toml b/ruby_inventory.toml new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/ruby_inventory.toml @@ -0,0 +1 @@ + From d7b6ed53679731aebea4e5f8ea041875bc12624e Mon Sep 17 00:00:00 2001 From: Schneems Date: Thu, 25 Jul 2024 16:52:09 -0500 Subject: [PATCH 12/18] Guard against the case of same sha7 but different sha256 --- jruby_executable/src/bin/jruby_build.rs | 21 +++++++++-- ruby_executable/src/bin/ruby_build.rs | 23 ++++++++++-- ruby_inventory.toml | 9 +++++ shared/src/inventory_help.rs | 48 +++++++++++++++++++++++++ shared/src/lib.rs | 4 +-- 5 files changed, 97 insertions(+), 8 deletions(-) diff --git a/jruby_executable/src/bin/jruby_build.rs b/jruby_executable/src/bin/jruby_build.rs index 9acd3ae..f395a5d 100644 --- a/jruby_executable/src/bin/jruby_build.rs +++ b/jruby_executable/src/bin/jruby_build.rs @@ -6,9 +6,9 @@ use indoc::formatdoc; use inventory::artifact::Artifact; use jruby_executable::jruby_build_properties; use shared::{ - append_filename_with, artifact_is_different, atomic_inventory_update, download_tar, - sha256_from_path, source_dir, tar_dir_to_file, untar_to_dir, ArtifactMetadata, BaseImage, - CpuArch, TarDownloadPath, + append_filename_with, artifact_is_different, artifact_same_url_different_checksum, + atomic_inventory_update, download_tar, sha256_from_path, source_dir, tar_dir_to_file, + untar_to_dir, ArtifactMetadata, BaseImage, CpuArch, TarDownloadPath, }; use std::convert::From; use std::error::Error; @@ -140,6 +140,21 @@ fn jruby_build(args: &Args) -> Result<(), Box> { }, }; atomic_inventory_update(&inventory, |inventory| { + for prior in &inventory.artifacts { + if let Err(error) = artifact_same_url_different_checksum(prior, &artifact) { + // TODO: Investigate bullet stream ownership + println!( + "{}", + style::important(format!( + "!!!!!!!!!! Error updating inventory: {error}" + )) + ); + + fs_err::remove_file(&sha_seven_path)?; + return Err(error); + }; + } + inventory .artifacts .retain(|a| artifact_is_different(a, &artifact)); diff --git a/ruby_executable/src/bin/ruby_build.rs b/ruby_executable/src/bin/ruby_build.rs index 5754357..474edf1 100644 --- a/ruby_executable/src/bin/ruby_build.rs +++ b/ruby_executable/src/bin/ruby_build.rs @@ -5,10 +5,12 @@ use fun_run::CommandWithName; use gem_version::GemVersion; use indoc::{formatdoc, indoc}; use inventory::artifact::Artifact; +use nom::Err; use shared::{ - append_filename_with, artifact_is_different, atomic_inventory_update, download_tar, - output_tar_path, sha256_from_path, source_dir, validate_version_for_stack, ArtifactMetadata, - BaseImage, CpuArch, RubyDownloadVersion, TarDownloadPath, + append_filename_with, artifact_is_different, artifact_same_url_different_checksum, + atomic_inventory_update, download_tar, output_tar_path, sha256_from_path, source_dir, + validate_version_for_stack, ArtifactMetadata, BaseImage, CpuArch, RubyDownloadVersion, + TarDownloadPath, }; use std::{ io::Write, @@ -185,10 +187,25 @@ fn ruby_build(args: &RubyArgs) -> Result<(), Box> { }; atomic_inventory_update(&inventory, |inventory| { + for prior in &inventory.artifacts { + if let Err(error) = artifact_same_url_different_checksum(prior, &artifact) { + // TODO: Investigate bullet stream ownership + println!( + "{}", + style::important(format!("!!!!!!!!!! Error updating inventory: {error}")) + ); + + fs_err::remove_file(&sha_seven_path)?; + return Err(error); + }; + } + inventory .artifacts .retain(|a| artifact_is_different(a, &artifact)); + inventory.push(artifact); + Ok(()) })?; diff --git a/ruby_inventory.toml b/ruby_inventory.toml index 8b13789..05fdddb 100644 --- a/ruby_inventory.toml +++ b/ruby_inventory.toml @@ -1 +1,10 @@ +[[artifacts]] +version = "3.3.4" +os = "linux" +arch = "arm64" +url = "https://heroku-buildpack-ruby.s3.us-east-1.amazonaws.com/heroku-24/arm64/ruby-3.3.4-7bebeee.tgz" +checksum = "sha256:7bebeee1b9128bdbb290331b813fa01cf43e30cd0098286f7de011796cb8eee5" +[artifacts.metadata] +timestamp = "2024-07-25T21:13:38.723081Z" +distro_version = "24.04" diff --git a/shared/src/inventory_help.rs b/shared/src/inventory_help.rs index d4fcea9..9635afa 100644 --- a/shared/src/inventory_help.rs +++ b/shared/src/inventory_help.rs @@ -180,6 +180,28 @@ where Ok(digest.finalize_fixed()) } +/// Raises an error if the same URL has a different checksum +/// +/// This protects against the (reasonably) unlikely event that the same version generates a checksum with the same first 7 characters but a net different checksum. +/// While unlikely, it's still possible. If we didn't guard against this case, then it could break people's builds who are relying on the old checksum +/// no not change. +pub fn artifact_same_url_different_checksum( + a: &inventory::artifact::Artifact, + b: &inventory::artifact::Artifact, +) -> Result<(), Box> { + if a.url == b.url && a.checksum != b.checksum { + Err(format!( + "Duplicate url {url} has different checksums {a_checksum} != {b_checksum}", + url = a.url, + a_checksum = hex::encode(&a.checksum.value), + b_checksum = hex::encode(&b.checksum.value) + ) + .into()) + } else { + Ok(()) + } +} + pub fn artifact_is_different( a: &inventory::artifact::Artifact, b: &inventory::artifact::Artifact, @@ -198,6 +220,32 @@ mod test { use super::*; + #[test] + fn test_same_url_different_checksum_raises_error() { + let a = Artifact { + os: inventory::artifact::Os::Linux, + arch: inventory::artifact::Arch::Amd64, + version: GemVersion::from_str("1.0.0").unwrap(), + checksum: "sha256:dd073bda5665e758c3e6f861a6df435175c8e8faf5ec75bc2afaab1e3eebb2c7" + .parse() + .unwrap(), + metadata: ArtifactMetadata { + timestamp: Utc::now(), + distro_version: BaseImage::new("heroku-24").unwrap().distro_version(), + }, + url: "https://example.com".to_string(), + }; + + let b = a.clone(); + artifact_same_url_different_checksum(&a, &b).unwrap(); + + let mut b = a.clone(); + b.checksum = "sha256:7bebeee1b9128bdbb290331b813fa01cf43e30cd0098286f7de011796cb8eee5" + .parse() + .unwrap(); + assert!(artifact_same_url_different_checksum(&a, &b).is_err()); + } + #[test] fn test_is_not_version_match() { let a = Artifact { diff --git a/shared/src/lib.rs b/shared/src/lib.rs index ae73018..424b13d 100644 --- a/shared/src/lib.rs +++ b/shared/src/lib.rs @@ -13,8 +13,8 @@ mod inventory_help; pub use base_image::{BaseImage, CpuArch, CpuArchError}; pub use download_ruby_version::RubyDownloadVersion; pub use inventory_help::{ - artifact_is_different, atomic_inventory_update, inventory_check, sha256_from_path, - ArtifactMetadata, + artifact_is_different, artifact_same_url_different_checksum, atomic_inventory_update, + inventory_check, sha256_from_path, ArtifactMetadata, }; /// Appends the given string after the filename and before the `ends_with` From e52a87d05199c7104137cc87cb007f8f786facf9 Mon Sep 17 00:00:00 2001 From: Schneems Date: Thu, 25 Jul 2024 16:52:56 -0500 Subject: [PATCH 13/18] Fix clippy --- jruby_executable/src/bin/jruby_check.rs | 2 +- ruby_executable/src/bin/ruby_build.rs | 1 - ruby_executable/src/bin/ruby_check.rs | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/jruby_executable/src/bin/jruby_check.rs b/jruby_executable/src/bin/jruby_check.rs index f5a8347..e55e0ca 100644 --- a/jruby_executable/src/bin/jruby_check.rs +++ b/jruby_executable/src/bin/jruby_check.rs @@ -97,7 +97,7 @@ fn jruby_check(args: &RubyArgs) -> Result<(), Box> { cmd.arg(image_name); cmd.args(["bash", "-c"]); cmd.arg( - &[ + [ "mkdir /tmp/unzipped", &format!("tar xzf {} -C /tmp/unzipped", inner_jruby_path.display()), "export PATH=\"tmp/unzipped/bin:$PATH\"", diff --git a/ruby_executable/src/bin/ruby_build.rs b/ruby_executable/src/bin/ruby_build.rs index 474edf1..693411f 100644 --- a/ruby_executable/src/bin/ruby_build.rs +++ b/ruby_executable/src/bin/ruby_build.rs @@ -5,7 +5,6 @@ use fun_run::CommandWithName; use gem_version::GemVersion; use indoc::{formatdoc, indoc}; use inventory::artifact::Artifact; -use nom::Err; use shared::{ append_filename_with, artifact_is_different, artifact_same_url_different_checksum, atomic_inventory_update, download_tar, output_tar_path, sha256_from_path, source_dir, diff --git a/ruby_executable/src/bin/ruby_check.rs b/ruby_executable/src/bin/ruby_check.rs index ac5bc87..702043b 100644 --- a/ruby_executable/src/bin/ruby_check.rs +++ b/ruby_executable/src/bin/ruby_check.rs @@ -48,7 +48,7 @@ fn ruby_check(args: &RubyArgs) -> Result<(), Box> { cmd.arg(image_name); cmd.args(["bash", "-c"]); cmd.arg( - &[ + [ "mkdir /tmp/unzipped", &format!("tar xzf {} -C /tmp/unzipped", path.display()), "echo -n '- Rubygems version: '", From dd38965bde9bb4b876ea8e9eb0870b86866c4fe1 Mon Sep 17 00:00:00 2001 From: Schneems Date: Thu, 25 Jul 2024 17:07:42 -0500 Subject: [PATCH 14/18] Remove accidental inventory addition Sidenote, the check to validate that the URLs are valid works! https://github.com/heroku/docker-heroku-ruby-builder/actions/runs/10102146736/job/27937182426?pr=51 --- ruby_inventory.toml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/ruby_inventory.toml b/ruby_inventory.toml index 05fdddb..8b13789 100644 --- a/ruby_inventory.toml +++ b/ruby_inventory.toml @@ -1,10 +1 @@ -[[artifacts]] -version = "3.3.4" -os = "linux" -arch = "arm64" -url = "https://heroku-buildpack-ruby.s3.us-east-1.amazonaws.com/heroku-24/arm64/ruby-3.3.4-7bebeee.tgz" -checksum = "sha256:7bebeee1b9128bdbb290331b813fa01cf43e30cd0098286f7de011796cb8eee5" -[artifacts.metadata] -timestamp = "2024-07-25T21:13:38.723081Z" -distro_version = "24.04" From 74a8acf9b50736165ddaa8c88e1ec28170cbe600 Mon Sep 17 00:00:00 2001 From: Schneems Date: Thu, 25 Jul 2024 17:11:06 -0500 Subject: [PATCH 15/18] Auto PR for ruby_inventory.toml --- .github/workflows/build_ruby.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_ruby.yml b/.github/workflows/build_ruby.yml index ffe6580..ef26f63 100644 --- a/.github/workflows/build_ruby.yml +++ b/.github/workflows/build_ruby.yml @@ -24,7 +24,7 @@ env: S3_BUCKET: "heroku-buildpack-ruby" jobs: - build_ruby: + build-and-upload: runs-on: ${{ matrix.arch == 'arm64' && 'pub-hk-ubuntu-24.04-arm-xlarge' || 'pub-hk-ubuntu-24.04-xlarge' }} strategy: matrix: @@ -56,3 +56,13 @@ jobs: - name: Upload Ruby runtime archive to S3 production if: (!inputs.dry_run) run: aws s3 sync ./output "s3://${S3_BUCKET}" + + after-build-and-upload: + needs: build-and-upload + runs-on: pub-hk-ubuntu-24.04-xlarge + steps: + - name: Update Ruby inventory file locally + uses: peter-evans/create-pull-request@v6 + with: + path: ruby_inventory.toml + title: "Add Ruby ${{inputs.ruby_version}} to inventory" From 10506b38f0b82a4630b5c7f13c8e8906df7ca619 Mon Sep 17 00:00:00 2001 From: Schneems Date: Fri, 26 Jul 2024 21:49:34 -0500 Subject: [PATCH 16/18] Manual global write permissions Before: ``` drwxr-xr-x 2 root root 4096 Jul 25 22:34 . drwxr-xr-x 3 runner docker 4096 Jul 25 22:32 .. -rw-r--r-- 1 root root 24061018 Jul 25 22:34 ruby-3.2.3.tgz ``` With these permissions we were unable to write to the directory. After: After ``` drwxrwxrwx 2 root root 4096 Jul 27 02:53 . drwxr-xr-x 3 runner docker 4096 Jul 27 02:51 .. -rw-r--r-- 1 root root 24061325 Jul 27 02:53 ruby-3.2.3.tgz ``` Now we can copy the file to a new filename in the same directory. --- make_ruby.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/make_ruby.sh b/make_ruby.sh index b982bf4..6c924c0 100755 --- a/make_ruby.sh +++ b/make_ruby.sh @@ -23,6 +23,9 @@ PS4='>\e[33m${BASH_SOURCE}:${LINENO} $\e[0m ' set -o xtrace mkdir -p "$(dirname "$OUT_TAR")" + +# Docker issue with permissions +chmod a+w "$(dirname "$OUT_TAR")" mkdir -p /tmp/source mkdir -p /tmp/compiled From b8cfa558e671e1cd5620a8bfe759dee1c0cf76c5 Mon Sep 17 00:00:00 2001 From: Rune Soerensen Date: Wed, 4 Sep 2024 16:52:14 -0400 Subject: [PATCH 17/18] Prefer inventory::artifact::Arch --- jruby_executable/src/bin/jruby_build.rs | 10 ++-- jruby_executable/src/bin/jruby_check.rs | 5 +- ruby_executable/src/bin/ruby_build.rs | 9 ++-- ruby_executable/src/bin/ruby_check.rs | 5 +- shared/src/base_image.rs | 66 ------------------------- shared/src/lib.rs | 12 ++--- 6 files changed, 20 insertions(+), 87 deletions(-) diff --git a/jruby_executable/src/bin/jruby_build.rs b/jruby_executable/src/bin/jruby_build.rs index f395a5d..b6deb30 100644 --- a/jruby_executable/src/bin/jruby_build.rs +++ b/jruby_executable/src/bin/jruby_build.rs @@ -3,12 +3,12 @@ use clap::Parser; use fs_err::PathExt; use gem_version::GemVersion; use indoc::formatdoc; -use inventory::artifact::Artifact; +use inventory::artifact::{Arch, Artifact}; use jruby_executable::jruby_build_properties; use shared::{ append_filename_with, artifact_is_different, artifact_same_url_different_checksum, atomic_inventory_update, download_tar, sha256_from_path, source_dir, tar_dir_to_file, - untar_to_dir, ArtifactMetadata, BaseImage, CpuArch, TarDownloadPath, + untar_to_dir, ArtifactMetadata, BaseImage, TarDownloadPath, }; use std::convert::From; use std::error::Error; @@ -123,12 +123,12 @@ fn jruby_build(args: &Args) -> Result<(), Box> { fs_err::copy(tar_file.path(), &sha_seven_path)?; let timestamp = chrono::Utc::now(); - for cpu_arch in &[CpuArch::new("amd64")?, CpuArch::new("arm64")?] { + for cpu_arch in [Arch::Amd64, Arch::Arm64] { let distro_version = base_image.distro_version(); let artifact = Artifact { version: GemVersion::from_str(version)?, os: inventory::artifact::Os::Linux, - arch: cpu_arch.try_into()?, + arch: cpu_arch, url: format!( "{S3_BASE_URL}/{}", sha_seven_path.strip_prefix(&volume_output_dir)?.display() @@ -165,7 +165,7 @@ fn jruby_build(args: &Args) -> Result<(), Box> { } // Can be removed once manifest file support is fully rolled out - for cpu_arch in &[CpuArch::new("amd64")?, CpuArch::new("arm64")?] { + for cpu_arch in [Arch::Amd64, Arch::Arm64] { let dir = volume_output_dir .join(base_image.to_string()) .join(cpu_arch.to_string()); diff --git a/jruby_executable/src/bin/jruby_check.rs b/jruby_executable/src/bin/jruby_check.rs index e55e0ca..d276067 100644 --- a/jruby_executable/src/bin/jruby_check.rs +++ b/jruby_executable/src/bin/jruby_check.rs @@ -2,8 +2,9 @@ use bullet_stream::{style, Print}; use clap::Parser; use fun_run::CommandWithName; use indoc::formatdoc; +use inventory::artifact::Arch; use jruby_executable::jruby_build_properties; -use shared::{source_dir, BaseImage, CpuArch}; +use shared::{source_dir, BaseImage}; use std::error::Error; use std::io::Write; use std::{path::PathBuf, process::Command}; @@ -13,7 +14,7 @@ static INNER_OUTPUT: &str = "/tmp/output"; #[derive(Parser, Debug)] struct RubyArgs { #[arg(long)] - arch: CpuArch, + arch: Arch, #[arg(long)] version: String, diff --git a/ruby_executable/src/bin/ruby_build.rs b/ruby_executable/src/bin/ruby_build.rs index 693411f..043ce0c 100644 --- a/ruby_executable/src/bin/ruby_build.rs +++ b/ruby_executable/src/bin/ruby_build.rs @@ -4,12 +4,11 @@ use fs_err::PathExt; use fun_run::CommandWithName; use gem_version::GemVersion; use indoc::{formatdoc, indoc}; -use inventory::artifact::Artifact; +use inventory::artifact::{Arch, Artifact}; use shared::{ append_filename_with, artifact_is_different, artifact_same_url_different_checksum, atomic_inventory_update, download_tar, output_tar_path, sha256_from_path, source_dir, - validate_version_for_stack, ArtifactMetadata, BaseImage, CpuArch, RubyDownloadVersion, - TarDownloadPath, + validate_version_for_stack, ArtifactMetadata, BaseImage, RubyDownloadVersion, TarDownloadPath, }; use std::{ io::Write, @@ -25,7 +24,7 @@ static S3_BASE_URL: &str = "https://heroku-buildpack-ruby.s3.us-east-1.amazonaws #[derive(Parser, Debug)] struct RubyArgs { #[arg(long)] - arch: CpuArch, + arch: Arch, #[arg(long)] version: RubyDownloadVersion, @@ -176,7 +175,7 @@ fn ruby_build(args: &RubyArgs) -> Result<(), Box> { let artifact = Artifact { version: GemVersion::from_str(&version.bundler_format())?, os: inventory::artifact::Os::Linux, - arch: arch.try_into()?, + arch: *arch, url, checksum: format!("sha256:{sha}").parse()?, metadata: ArtifactMetadata { diff --git a/ruby_executable/src/bin/ruby_check.rs b/ruby_executable/src/bin/ruby_check.rs index 702043b..6ad12cf 100644 --- a/ruby_executable/src/bin/ruby_check.rs +++ b/ruby_executable/src/bin/ruby_check.rs @@ -2,7 +2,8 @@ use bullet_stream::{style, Print}; use clap::Parser; use fun_run::CommandWithName; use indoc::formatdoc; -use shared::{output_tar_path, source_dir, BaseImage, CpuArch, RubyDownloadVersion}; +use inventory::artifact::Arch; +use shared::{output_tar_path, source_dir, BaseImage, RubyDownloadVersion}; use std::{error::Error, path::PathBuf, process::Command}; static INNER_OUTPUT: &str = "/tmp/output"; @@ -10,7 +11,7 @@ static INNER_OUTPUT: &str = "/tmp/output"; #[derive(Parser, Debug)] struct RubyArgs { #[arg(long)] - arch: CpuArch, + arch: Arch, #[arg(long)] version: RubyDownloadVersion, diff --git a/shared/src/base_image.rs b/shared/src/base_image.rs index 6c5b6b0..e8477d6 100644 --- a/shared/src/base_image.rs +++ b/shared/src/base_image.rs @@ -3,7 +3,6 @@ use std::str::FromStr; use serde::{Deserialize, Serialize}; -static KNOWN_ARCHITECTURES: [&str; 2] = ["amd64", "arm64"]; static KNOWN_BASE_IMAGES: &[(&str, &str)] = &[ ("heroku-20", "20"), ("heroku-22", "22"), @@ -68,68 +67,3 @@ impl FromStr for BaseImage { BaseImage::new(s) } } - -#[derive(Debug, Clone)] -pub struct CpuArch { - name: String, -} - -impl Display for CpuArch { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.name) - } -} - -impl FromStr for CpuArch { - type Err = CpuArchError; - - fn from_str(s: &str) -> Result { - CpuArch::new(s) - } -} - -#[derive(Debug, thiserror::Error)] -pub enum CpuArchError { - #[error("Invalid CPU architecture {0} must be one of {}", KNOWN_ARCHITECTURES.join(", "))] - Unknown(String), - - #[error("CPU architecture {0} cannot be converted {1}")] - CannotConvert(String, inventory::artifact::UnsupportedArchError), -} - -impl TryFrom<&CpuArch> for inventory::artifact::Arch { - type Error = CpuArchError; - - fn try_from(value: &CpuArch) -> Result { - Self::from_str(&value.name).map_err(|e| CpuArchError::CannotConvert(value.name.clone(), e)) - } -} - -impl CpuArch { - pub fn new(s: &str) -> Result { - KNOWN_ARCHITECTURES - .iter() - .find(|&&name| name == s) - .map(|_| Self { name: s.to_owned() }) - .ok_or_else(|| CpuArchError::Unknown(s.to_owned())) - } - - pub fn from_system() -> Result { - let arch = if cfg!(target_arch = "aarch64") { - "arm64" - } else if cfg!(target_arch = "x86_64") { - "amd64" - } else { - "Unknown architecture" - }; - - Self::new(arch) - } - - #[cfg(test)] - pub(crate) fn from_test_str(name: &str) -> Self { - Self { - name: name.to_string(), - } - } -} diff --git a/shared/src/lib.rs b/shared/src/lib.rs index 424b13d..2a62978 100644 --- a/shared/src/lib.rs +++ b/shared/src/lib.rs @@ -2,6 +2,7 @@ use bullet_stream::state::SubBullet; use bullet_stream::Print; use fs_err::{File, PathExt}; use fun_run::CommandWithName; +use inventory::artifact::Arch; use std::io::{Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; use std::process::Command; @@ -10,7 +11,7 @@ mod base_image; mod download_ruby_version; mod inventory_help; -pub use base_image::{BaseImage, CpuArch, CpuArchError}; +pub use base_image::BaseImage; pub use download_ruby_version::RubyDownloadVersion; pub use inventory_help::{ artifact_is_different, artifact_same_url_different_checksum, atomic_inventory_update, @@ -81,9 +82,6 @@ pub fn untar_to_dir(tar_path: &TarDownloadPath, workspace: &Path) -> Result<(), #[derive(thiserror::Error, Debug)] pub enum Error { - #[error("Error {0}")] - UnknownArchitecture(CpuArchError), - #[error("Command failed {0}")] CmdError(fun_run::CmdError), @@ -155,7 +153,7 @@ pub fn output_tar_path( output: &Path, version: &RubyDownloadVersion, base_image: &BaseImage, - cpu_architecture: &CpuArch, + cpu_architecture: &Arch, ) -> PathBuf { let directory = if base_image.is_arch_aware() { PathBuf::from(base_image.to_string()).join(cpu_architecture.to_string()) @@ -307,7 +305,7 @@ mod test { let output = PathBuf::from("/tmp"); let version = RubyDownloadVersion::from_str("2.7.3").unwrap(); let base_image = BaseImage::new("heroku-20").unwrap(); - let cpu_architecture = CpuArch::from_test_str("amd64"); + let cpu_architecture = Arch::Amd64; let tar_path = output_tar_path(&output, &version, &base_image, &cpu_architecture); @@ -320,7 +318,7 @@ mod test { let output = PathBuf::from("/tmp"); let version = RubyDownloadVersion::from_str("2.7.3").unwrap(); let base_image = BaseImage::new("heroku-24").unwrap(); - let cpu_architecture = CpuArch::from_test_str("amd64"); + let cpu_architecture = Arch::Amd64; let tar_path = output_tar_path(&output, &version, &base_image, &cpu_architecture); From e3cc81607247c055116240fb71d52a7397ccc65c Mon Sep 17 00:00:00 2001 From: Schneems Date: Thu, 5 Sep 2024 12:33:34 -0500 Subject: [PATCH 18/18] Fix clippy and doc tests ``` error: the borrowed expression implements the required traits --> ruby_executable/src/bin/ruby_build.rs:143:24 | 143 | docker_run.arg(&format!( | ________________________^ 144 | | "./make_ruby.sh {} {}", 145 | | input_tar.display(), 146 | | output_tar.display() 147 | | )); | |_________^ | = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#needless_borrows_for_generic_args = note: `-D clippy::needless-borrows-for-generic-args` implied by `-D warnings` = help: to override `-D warnings` add `#[allow(clippy::needless_borrows_for_generic_args)]` help: change this to | 143 ~ docker_run.arg(format!( 144 + "./make_ruby.sh {} {}", 145 + input_tar.display(), 146 + output_tar.display() 147 ~ )); | error: could not compile `ruby_executable` (bin "ruby_build" test) due to 1 previous error warning: build failed, waiting for other jobs to finish... ``` --- jruby_executable/src/lib.rs | 4 ++-- ruby_executable/src/bin/ruby_build.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jruby_executable/src/lib.rs b/jruby_executable/src/lib.rs index 6f33d5d..ac597c8 100644 --- a/jruby_executable/src/lib.rs +++ b/jruby_executable/src/lib.rs @@ -9,14 +9,14 @@ /// implements Ruby 3.1.4 stdlib. When people use jruby they specify both the /// jruby version and the stdlib version, for example: /// -/// ```ignore +/// ```ruby /// # Gemfile /// ruby "3.1.4", engine: "jruby", engine_version: "9.4.3.0" /// ``` /// /// Example file for /// -/// ```ignore +/// ```ini /// # Defaults. To override, create a file called build.properties in /// # the same directory and put your changes in that. /// #src.dir=src diff --git a/ruby_executable/src/bin/ruby_build.rs b/ruby_executable/src/bin/ruby_build.rs index 043ce0c..eedabf0 100644 --- a/ruby_executable/src/bin/ruby_build.rs +++ b/ruby_executable/src/bin/ruby_build.rs @@ -140,7 +140,7 @@ fn ruby_build(args: &RubyArgs) -> Result<(), Box> { docker_run.arg(&image_name); docker_run.args(["bash", "-c"]); - docker_run.arg(&format!( + docker_run.arg(format!( "./make_ruby.sh {} {}", input_tar.display(), output_tar.display()