diff --git a/.github/workflows/build_jruby.yml b/.github/workflows/build_jruby.yml index d3f7e4a..2b79100 100644 --- a/.github/workflows/build_jruby.yml +++ b/.github/workflows/build_jruby.yml @@ -51,3 +51,13 @@ jobs: - name: Upload Ruby runtime archive to S3 production if: (!inputs.dry_run) run: aws s3 sync ./output "s3://${S3_BUCKET}" + + after-build-and-upload: + needs: build-and-upload + runs-on: pub-hk-ubuntu-24.04-xlarge + steps: + - name: Update Jruby inventory file locally + uses: peter-evans/create-pull-request@v6 + with: + path: jruby_inventory.toml + title: "Add JRuby ${{inputs.jruby_version}} to inventory" diff --git a/.github/workflows/build_ruby.yml b/.github/workflows/build_ruby.yml index ffe6580..ef26f63 100644 --- a/.github/workflows/build_ruby.yml +++ b/.github/workflows/build_ruby.yml @@ -24,7 +24,7 @@ env: S3_BUCKET: "heroku-buildpack-ruby" jobs: - build_ruby: + build-and-upload: runs-on: ${{ matrix.arch == 'arm64' && 'pub-hk-ubuntu-24.04-arm-xlarge' || 'pub-hk-ubuntu-24.04-xlarge' }} strategy: matrix: @@ -56,3 +56,13 @@ jobs: - name: Upload Ruby runtime archive to S3 production if: (!inputs.dry_run) run: aws s3 sync ./output "s3://${S3_BUCKET}" + + after-build-and-upload: + needs: build-and-upload + runs-on: pub-hk-ubuntu-24.04-xlarge + steps: + - name: Update Ruby inventory file locally + uses: peter-evans/create-pull-request@v6 + with: + path: ruby_inventory.toml + title: "Add Ruby ${{inputs.ruby_version}} to inventory" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 037c7ef..37d07fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,3 +102,22 @@ jobs: run: cargo run --bin jruby_build -- --version ${{matrix.version}} --base-image ${{matrix.base_image}} - name: Check JRuby run: cargo run --bin jruby_check -- --version ${{matrix.version}} --base-image ${{matrix.base_image}} --arch ${{matrix.arch}} + + check_inventory_urls: + runs-on: ubuntu-24.04 + if: (!contains(github.event.pull_request.labels.*.name, 'skip inventory check')) + strategy: + matrix: + inventory: ["jruby_inventory.toml", "ruby_inventory.toml"] + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Grab prior commits + run: | + set -eu + set pipefail + + git fetch origin ${{ github.base_ref }} --depth 1 && \ + git diff --unified=0 remotes/origin/${{ github.base_ref }} ${{matrix.inventory}} | grep '^+' | grep -v '^+++' | cut -c2- > check_inventory.toml + - name: Check manifest URLs + run: cargo run --bin inventory_check -- check_inventory.toml diff --git a/Cargo.lock b/Cargo.lock index 858d91c..4ffdedc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "0.6.14" @@ -114,6 +129,21 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bitflags" version = "1.3.2" @@ -126,6 +156,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bullet_stream" version = "0.2.0" @@ -156,6 +195,21 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets 0.52.5", +] + [[package]] name = "chunked_transfer" version = "1.5.0" @@ -224,6 +278,15 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -233,6 +296,57 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + [[package]] name = "encoding_rs" version = "0.8.34" @@ -258,6 +372,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "fancy-regex" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.1.0" @@ -325,6 +450,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "fun_run" version = "0.2.0" @@ -385,6 +520,27 @@ dependencies = [ "slab", ] +[[package]] +name = "gem_version" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804d260cc5d12cb2ed81b4e9228eb48c989a36e86b5b7e5c6113509f88d4a69a" +dependencies = [ + "fancy-regex", + "regex", + "serde", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "gimli" version = "0.29.0" @@ -434,6 +590,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "http" version = "1.1.0" @@ -536,6 +698,29 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "idna" version = "0.5.0" @@ -562,6 +747,18 @@ version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" +[[package]] +name = "inventory" +version = "0.1.0" +source = "git+https://github.com/Malax/inventory#e4eed7de95e49d441cdf13f9c001644ed9159393" +dependencies = [ + "hex", + "serde", + "sha2", + "thiserror", + "toml", +] + [[package]] name = "ipnet" version = "2.9.0" @@ -596,17 +793,21 @@ name = "jruby_executable" version = "0.1.0" dependencies = [ "bullet_stream", + "chrono", "clap", "flate2", "fs-err", "fun_run", + "gem_version", "glob", "indoc", + "inventory", "java-properties", "lazy_static", "nom", "regex", "reqwest", + "sha2", "shared", "tar", "tempfile", @@ -711,6 +912,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.16.0" @@ -842,6 +1052,26 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -928,14 +1158,18 @@ name = "ruby_executable" version = "0.1.0" dependencies = [ "bullet_stream", + "chrono", "clap", "flate2", "fs-err", "fun_run", + "gem_version", "indoc", + "inventory", "nom", "regex", "reqwest", + "sha2", "shared", "tar", "tempfile", @@ -1017,18 +1251,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", @@ -1046,6 +1280,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -1058,24 +1301,44 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shared" version = "0.1.0" dependencies = [ "bullet_stream", + "chrono", "clap", "flate2", "fs-err", + "fs2", "fun_run", + "gem_version", "glob", + "hex", "indoc", + "inventory", "nom", + "rayon", "regex", "reqwest", + "serde", + "sha2", "tar", "tempfile", "thiserror", "tiny_http", + "toml", ] [[package]] @@ -1256,6 +1519,40 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2caab0bf757388c6c0ae23b3293fdb463fee59434529014f85e3263b995c28" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "278f3d518e152219c994ce877758516bca5e118eaed6996192a774fb9fbf0788" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.4.13" @@ -1308,6 +1605,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "unicode-bidi" version = "0.3.15" @@ -1352,6 +1655,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + [[package]] name = "want" version = "0.3.1" @@ -1443,6 +1752,37 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.5", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -1582,6 +1922,15 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +[[package]] +name = "winnow" +version = "0.6.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "557404e450152cd6795bb558bca69e43c585055f4606e3bcae5894fc6dac9ba0" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.52.0" diff --git a/Cargo.toml b/Cargo.toml index 1d08439..e6bb04a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,3 +22,15 @@ tar = "0.4.40" tempfile = "3.10.1" thiserror = "1.0.61" bullet_stream = "0.2.0" +sha2 = "0.10" +toml = "0.8" +chrono = {version = "0.4", features = ["serde"] } +serde = {version = "1.0", features = ["derive"] } +inventory = { git = "https://github.com/Malax/inventory", features = ["sha2"] } +rayon = "1.10" +hex = "0.4.3" +gem_version = "0.3" + +# File locking (FLOCK) +fs2 = "0.4" + diff --git a/jruby_executable/Cargo.toml b/jruby_executable/Cargo.toml index c10b017..9a2a6cd 100644 --- a/jruby_executable/Cargo.toml +++ b/jruby_executable/Cargo.toml @@ -32,3 +32,7 @@ tar = { workspace = true} tempfile = { workspace = true} thiserror = { workspace = true} bullet_stream = { workspace = true } +inventory = { workspace = true } +chrono = { workspace = true } +sha2 = { workspace = true } +gem_version = { workspace = true } diff --git a/jruby_executable/src/bin/jruby_build.rs b/jruby_executable/src/bin/jruby_build.rs index 29e60d4..b6deb30 100644 --- a/jruby_executable/src/bin/jruby_build.rs +++ b/jruby_executable/src/bin/jruby_build.rs @@ -1,10 +1,20 @@ use bullet_stream::{style, Print}; use clap::Parser; use fs_err::PathExt; +use gem_version::GemVersion; use indoc::formatdoc; +use inventory::artifact::{Arch, Artifact}; use jruby_executable::jruby_build_properties; -use shared::{download_tar, source_dir, tar_dir_to_file, untar_to_dir, BaseImage, TarDownloadPath}; +use shared::{ + append_filename_with, artifact_is_different, artifact_same_url_different_checksum, + atomic_inventory_update, download_tar, sha256_from_path, source_dir, tar_dir_to_file, + untar_to_dir, ArtifactMetadata, BaseImage, TarDownloadPath, +}; +use std::convert::From; use std::error::Error; +use std::str::FromStr; + +static S3_BASE_URL: &str = "https://heroku-buildpack-ruby.s3.us-east-1.amazonaws.com"; #[derive(Parser, Debug)] struct Args { @@ -22,6 +32,7 @@ fn jruby_build(args: &Args) -> Result<(), Box> { } = args; let mut log = Print::new(std::io::stderr()).h1("Building JRuby"); + let inventory = source_dir().join("jruby_inventory.toml"); let volume_cache_dir = source_dir().join("cache"); let volume_output_dir = source_dir().join("output"); @@ -89,6 +100,10 @@ fn jruby_build(args: &Args) -> Result<(), Box> { log = { let mut bullet = log.bullet("Creating tgz archives"); + bullet = bullet.sub_bullet(format!( + "Inventory file {}", + style::value(inventory.to_string_lossy()) + )); let tar_dir = volume_output_dir.join(base_image.to_string()); fs_err::create_dir_all(&tar_dir)?; @@ -99,8 +114,61 @@ fn jruby_build(args: &Args) -> Result<(), Box> { tar_dir_to_file(&jruby_dir, &tar_file)?; bullet = timer.done(); - for arch in &["amd64", "arm64"] { - let dir = volume_output_dir.join(base_image.to_string()).join(arch); + let tar_path = tar_file.path(); + let sha = sha256_from_path(tar_path)?; + let sha_seven = sha.chars().take(7).collect::(); + let sha_seven_path = append_filename_with(tar_path, &format!("-{sha_seven}"), ".tgz")?; + + bullet = bullet.sub_bullet(format!("Write {}", sha_seven_path.display(),)); + fs_err::copy(tar_file.path(), &sha_seven_path)?; + + let timestamp = chrono::Utc::now(); + for cpu_arch in [Arch::Amd64, Arch::Arm64] { + let distro_version = base_image.distro_version(); + let artifact = Artifact { + version: GemVersion::from_str(version)?, + os: inventory::artifact::Os::Linux, + arch: cpu_arch, + url: format!( + "{S3_BASE_URL}/{}", + sha_seven_path.strip_prefix(&volume_output_dir)?.display() + ), + checksum: format!("sha256:{sha}").parse()?, + metadata: ArtifactMetadata { + distro_version, + timestamp, + }, + }; + atomic_inventory_update(&inventory, |inventory| { + for prior in &inventory.artifacts { + if let Err(error) = artifact_same_url_different_checksum(prior, &artifact) { + // TODO: Investigate bullet stream ownership + println!( + "{}", + style::important(format!( + "!!!!!!!!!! Error updating inventory: {error}" + )) + ); + + fs_err::remove_file(&sha_seven_path)?; + return Err(error); + }; + } + + inventory + .artifacts + .retain(|a| artifact_is_different(a, &artifact)); + + inventory.push(artifact); + Ok(()) + })? + } + + // Can be removed once manifest file support is fully rolled out + for cpu_arch in [Arch::Amd64, Arch::Arm64] { + let dir = volume_output_dir + .join(base_image.to_string()) + .join(cpu_arch.to_string()); fs_err::create_dir_all(&dir)?; let path = dir.join(&tgz_name); @@ -110,6 +178,7 @@ fn jruby_build(args: &Args) -> Result<(), Box> { bullet.done() }; + log.done(); Ok(()) diff --git a/jruby_executable/src/bin/jruby_check.rs b/jruby_executable/src/bin/jruby_check.rs index f5a8347..d276067 100644 --- a/jruby_executable/src/bin/jruby_check.rs +++ b/jruby_executable/src/bin/jruby_check.rs @@ -2,8 +2,9 @@ use bullet_stream::{style, Print}; use clap::Parser; use fun_run::CommandWithName; use indoc::formatdoc; +use inventory::artifact::Arch; use jruby_executable::jruby_build_properties; -use shared::{source_dir, BaseImage, CpuArch}; +use shared::{source_dir, BaseImage}; use std::error::Error; use std::io::Write; use std::{path::PathBuf, process::Command}; @@ -13,7 +14,7 @@ static INNER_OUTPUT: &str = "/tmp/output"; #[derive(Parser, Debug)] struct RubyArgs { #[arg(long)] - arch: CpuArch, + arch: Arch, #[arg(long)] version: String, @@ -97,7 +98,7 @@ fn jruby_check(args: &RubyArgs) -> Result<(), Box> { cmd.arg(image_name); cmd.args(["bash", "-c"]); cmd.arg( - &[ + [ "mkdir /tmp/unzipped", &format!("tar xzf {} -C /tmp/unzipped", inner_jruby_path.display()), "export PATH=\"tmp/unzipped/bin:$PATH\"", diff --git a/jruby_executable/src/lib.rs b/jruby_executable/src/lib.rs index 6f33d5d..ac597c8 100644 --- a/jruby_executable/src/lib.rs +++ b/jruby_executable/src/lib.rs @@ -9,14 +9,14 @@ /// implements Ruby 3.1.4 stdlib. When people use jruby they specify both the /// jruby version and the stdlib version, for example: /// -/// ```ignore +/// ```ruby /// # Gemfile /// ruby "3.1.4", engine: "jruby", engine_version: "9.4.3.0" /// ``` /// /// Example file for /// -/// ```ignore +/// ```ini /// # Defaults. To override, create a file called build.properties in /// # the same directory and put your changes in that. /// #src.dir=src diff --git a/jruby_inventory.toml b/jruby_inventory.toml new file mode 100644 index 0000000..e69de29 diff --git a/make_ruby.sh b/make_ruby.sh index b982bf4..6c924c0 100755 --- a/make_ruby.sh +++ b/make_ruby.sh @@ -23,6 +23,9 @@ PS4='>\e[33m${BASH_SOURCE}:${LINENO} $\e[0m ' set -o xtrace mkdir -p "$(dirname "$OUT_TAR")" + +# Docker issue with permissions +chmod a+w "$(dirname "$OUT_TAR")" mkdir -p /tmp/source mkdir -p /tmp/compiled diff --git a/ruby_executable/Cargo.toml b/ruby_executable/Cargo.toml index b0185e0..e7df66b 100644 --- a/ruby_executable/Cargo.toml +++ b/ruby_executable/Cargo.toml @@ -17,3 +17,7 @@ tar = { workspace = true} tempfile = { workspace = true} thiserror = { workspace = true} bullet_stream = { workspace = true } +inventory = { workspace = true } +chrono = { workspace = true } +sha2 = { workspace = true } +gem_version = { workspace = true } diff --git a/ruby_executable/src/bin/ruby_build.rs b/ruby_executable/src/bin/ruby_build.rs index 829df6f..eedabf0 100644 --- a/ruby_executable/src/bin/ruby_build.rs +++ b/ruby_executable/src/bin/ruby_build.rs @@ -2,24 +2,29 @@ use bullet_stream::{style, Print}; use clap::Parser; use fs_err::PathExt; use fun_run::CommandWithName; +use gem_version::GemVersion; use indoc::{formatdoc, indoc}; +use inventory::artifact::{Arch, Artifact}; use shared::{ - download_tar, output_tar_path, source_dir, validate_version_for_stack, BaseImage, CpuArch, - RubyDownloadVersion, TarDownloadPath, + append_filename_with, artifact_is_different, artifact_same_url_different_checksum, + atomic_inventory_update, download_tar, output_tar_path, sha256_from_path, source_dir, + validate_version_for_stack, ArtifactMetadata, BaseImage, RubyDownloadVersion, TarDownloadPath, }; use std::{ io::Write, path::{Path, PathBuf}, process::Command, + str::FromStr, }; static INNER_OUTPUT: &str = "/tmp/output"; static INNER_CACHE: &str = "/tmp/cache"; +static S3_BASE_URL: &str = "https://heroku-buildpack-ruby.s3.us-east-1.amazonaws.com"; #[derive(Parser, Debug)] struct RubyArgs { #[arg(long)] - arch: CpuArch, + arch: Arch, #[arg(long)] version: RubyDownloadVersion, @@ -57,6 +62,7 @@ fn ruby_build(args: &RubyArgs) -> Result<(), Box> { } = args; let mut log = Print::new(std::io::stderr()).h1("Building Ruby"); + let inventory = source_dir().join("ruby_inventory.toml"); let volume_cache_dir = source_dir().join("cache"); let volume_output_dir = source_dir().join("output"); @@ -116,12 +122,10 @@ fn ruby_build(args: &RubyArgs) -> Result<(), Box> { log = { let mut bullet = log.bullet("Make Ruby"); - let input_tar = PathBuf::from(INNER_CACHE).join(format!("ruby-source-{version}.tgz")); let output_tar = output_tar_path(Path::new(INNER_OUTPUT), version, base_image, arch); - - let volume_output = volume_output_dir.display(); let volume_cache = volume_cache_dir.display(); + let volume_output = volume_output_dir.display(); let mut docker_run = Command::new("docker"); docker_run.arg("run"); @@ -136,7 +140,7 @@ fn ruby_build(args: &RubyArgs) -> Result<(), Box> { docker_run.arg(&image_name); docker_run.args(["bash", "-c"]); - docker_run.arg(&format!( + docker_run.arg(format!( "./make_ruby.sh {} {}", input_tar.display(), output_tar.display() @@ -146,6 +150,62 @@ fn ruby_build(args: &RubyArgs) -> Result<(), Box> { format!("Running {}", style::command(docker_run.name())), |stdout, stderr| docker_run.stream_output(stdout, stderr), )?; + bullet.done() + }; + + log = { + let mut bullet = log.bullet(format!( + "Updating manifest {}", + style::value(inventory.to_string_lossy()) + )); + + let output_tar = output_tar_path(&volume_output_dir, version, base_image, arch); + + let sha = sha256_from_path(&output_tar)?; + let sha_seven = sha.chars().take(7).collect::(); + let sha_seven_path = append_filename_with(&output_tar, &format!("-{sha_seven}"), ".tgz")?; + let url = format!( + "{S3_BASE_URL}/{}", + sha_seven_path.strip_prefix(&volume_output_dir)?.display() + ); + + bullet = bullet.sub_bullet(format!("Copying SHA tgz {}", sha_seven_path.display(),)); + fs_err::copy(output_tar, &sha_seven_path)?; + + let artifact = Artifact { + version: GemVersion::from_str(&version.bundler_format())?, + os: inventory::artifact::Os::Linux, + arch: *arch, + url, + checksum: format!("sha256:{sha}").parse()?, + metadata: ArtifactMetadata { + distro_version: base_image.distro_version(), + timestamp: chrono::Utc::now(), + }, + }; + + atomic_inventory_update(&inventory, |inventory| { + for prior in &inventory.artifacts { + if let Err(error) = artifact_same_url_different_checksum(prior, &artifact) { + // TODO: Investigate bullet stream ownership + println!( + "{}", + style::important(format!("!!!!!!!!!! Error updating inventory: {error}")) + ); + + fs_err::remove_file(&sha_seven_path)?; + return Err(error); + }; + } + + inventory + .artifacts + .retain(|a| artifact_is_different(a, &artifact)); + + inventory.push(artifact); + + Ok(()) + })?; bullet.done() }; diff --git a/ruby_executable/src/bin/ruby_check.rs b/ruby_executable/src/bin/ruby_check.rs index ac5bc87..6ad12cf 100644 --- a/ruby_executable/src/bin/ruby_check.rs +++ b/ruby_executable/src/bin/ruby_check.rs @@ -2,7 +2,8 @@ use bullet_stream::{style, Print}; use clap::Parser; use fun_run::CommandWithName; use indoc::formatdoc; -use shared::{output_tar_path, source_dir, BaseImage, CpuArch, RubyDownloadVersion}; +use inventory::artifact::Arch; +use shared::{output_tar_path, source_dir, BaseImage, RubyDownloadVersion}; use std::{error::Error, path::PathBuf, process::Command}; static INNER_OUTPUT: &str = "/tmp/output"; @@ -10,7 +11,7 @@ static INNER_OUTPUT: &str = "/tmp/output"; #[derive(Parser, Debug)] struct RubyArgs { #[arg(long)] - arch: CpuArch, + arch: Arch, #[arg(long)] version: RubyDownloadVersion, @@ -48,7 +49,7 @@ fn ruby_check(args: &RubyArgs) -> Result<(), Box> { cmd.arg(image_name); cmd.args(["bash", "-c"]); cmd.arg( - &[ + [ "mkdir /tmp/unzipped", &format!("tar xzf {} -C /tmp/unzipped", path.display()), "echo -n '- Rubygems version: '", diff --git a/ruby_inventory.toml b/ruby_inventory.toml new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/ruby_inventory.toml @@ -0,0 +1 @@ + diff --git a/shared/Cargo.toml b/shared/Cargo.toml index 1072bf5..3a624eb 100644 --- a/shared/Cargo.toml +++ b/shared/Cargo.toml @@ -3,6 +3,10 @@ name = "shared" version = "0.1.0" edition = "2021" +[[bin]] +name = "inventory_check" +path = "src/bin/inventory_check.rs" + [dependencies] glob = { workspace = true} clap = { workspace = true} @@ -17,6 +21,16 @@ tar = { workspace = true} tempfile = { workspace = true} thiserror = { workspace = true} bullet_stream = { workspace = true } +sha2 = { workspace = true } +chrono = { workspace = true } +serde = { workspace = true } +toml = { workspace = true } +inventory = { workspace = true } +fs2 = { workspace = true } +rayon = { workspace = true } +hex = { workspace = true } +gem_version = { workspace = true } + [dev-dependencies] tiny_http = "0.12.0" diff --git a/shared/src/base_image.rs b/shared/src/base_image.rs index 4e95870..e8477d6 100644 --- a/shared/src/base_image.rs +++ b/shared/src/base_image.rs @@ -1,7 +1,8 @@ use std::fmt::Display; use std::str::FromStr; -static KNOWN_ARCHITECTURES: [&str; 2] = ["amd64", "arm64"]; +use serde::{Deserialize, Serialize}; + static KNOWN_BASE_IMAGES: &[(&str, &str)] = &[ ("heroku-20", "20"), ("heroku-22", "22"), @@ -19,6 +20,9 @@ pub struct BaseImage { distro_number: String, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct DistroVersion(String); + impl BaseImage { pub fn new(s: &str) -> Result { KNOWN_BASE_IMAGES @@ -31,6 +35,10 @@ impl BaseImage { .ok_or_else(|| BaseImageError(s.to_owned())) } + pub fn distro_version(&self) -> DistroVersion { + DistroVersion(format!("{}.04", self.distro_number)) + } + pub fn is_arch_aware(&self) -> bool { MULTI_ARCH_BASE_IMAGES.contains(&self.name.as_str()) } @@ -59,55 +67,3 @@ impl FromStr for BaseImage { BaseImage::new(s) } } - -#[derive(Debug, Clone)] -pub struct CpuArch { - name: String, -} - -impl Display for CpuArch { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.name) - } -} - -impl FromStr for CpuArch { - type Err = CpuArchError; - - fn from_str(s: &str) -> Result { - CpuArch::new(s) - } -} - -#[derive(Debug, thiserror::Error)] -#[error("Invalid CPU architecture {0} must be one of {}", KNOWN_ARCHITECTURES.join(", "))] -pub struct CpuArchError(String); - -impl CpuArch { - pub fn new(s: &str) -> Result { - KNOWN_ARCHITECTURES - .iter() - .find(|&&name| name == s) - .map(|_| Self { name: s.to_owned() }) - .ok_or_else(|| CpuArchError(s.to_owned())) - } - - pub fn from_system() -> Result { - let arch = if cfg!(target_arch = "aarch64") { - "arm64" - } else if cfg!(target_arch = "x86_64") { - "amd64" - } else { - "Unknown architecture" - }; - - Self::new(arch) - } - - #[cfg(test)] - pub(crate) fn from_test_str(name: &str) -> Self { - Self { - name: name.to_string(), - } - } -} diff --git a/shared/src/bin/inventory_check.rs b/shared/src/bin/inventory_check.rs new file mode 100644 index 0000000..e42951b --- /dev/null +++ b/shared/src/bin/inventory_check.rs @@ -0,0 +1,29 @@ +use clap::Parser; +use indoc::formatdoc; +use shared::inventory_check; +use std::path::{Path, PathBuf}; + +#[derive(Parser, Debug)] +struct Args { + path: PathBuf, +} + +fn check(path: &Path) -> Result<(), Box> { + let contents = fs_err::read_to_string(path)?; + inventory_check(&contents)?; + Ok(()) +} + +fn main() { + let args = Args::parse(); + if let Err(error) = check(&args.path) { + bullet_stream::Print::new(std::io::stderr()) + .without_header() + .error(formatdoc! {" + ❌ Command failed ❌ + + {error} + "}); + std::process::exit(1); + } +} diff --git a/shared/src/inventory_help.rs b/shared/src/inventory_help.rs new file mode 100644 index 0000000..9635afa --- /dev/null +++ b/shared/src/inventory_help.rs @@ -0,0 +1,320 @@ +use crate::base_image::DistroVersion; +use crate::{download_tar, Error, TarDownloadPath}; +use chrono::{DateTime, Utc}; +use fs2::FileExt; +use gem_version::GemVersion; +use inventory::checksum::Checksum; +use inventory::inventory::Inventory; +use rayon::prelude::*; +use serde::{Deserialize, Serialize}; +use sha2::Sha256; +use std::borrow::Borrow; +use std::io::Read; +use std::io::Write; +use std::path::Path; + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct ArtifactMetadata { + pub timestamp: DateTime, + pub distro_version: DistroVersion, +} + +/// ``` +/// use shared::inventory_check; +/// +/// let contents = r#" +/// [[artifacts]] +/// version = "9.4.8.0" +/// os = "linux" +/// arch = "amd64" +/// url = "https://heroku-buildpack-ruby.s3.us-east-1.amazonaws.com/heroku-24/ruby-3.1.4-jruby-9.4.8.0.tgz" +/// checksum = "sha256:815b31d2b204a524bf74aabae341bf85353add4d1128d5d276d08fa5e8ff3c39" +/// +/// [artifacts.metadata] +/// timestamp = "2024-07-24T16:17:35.341413Z" +/// distro_version = "24.04" +/// "#; +/// inventory_check(contents).unwrap(); +/// ``` +pub fn inventory_check(contents: &str) -> Result<(), Error> { + if contents.trim().is_empty() { + return Ok(()); + } + + let inventory = contents + .parse::>() + .map_err(|e| Error::Other(format!("Could not parse inventory. Error: {e}")))?; + + let results = inventory + .artifacts + .par_iter() + .map(|artifact| { + let temp = tempfile::tempdir().expect("Tempdir"); + let dir = temp.path(); + let path = dir.join("file.tar"); + + download_tar(&artifact.url, &TarDownloadPath(path.clone())) + .map_err(|e| format!("Error {e}")) + .and_then(|_| { + sha256_from_path(&path) + .map_err(|e| format!("Error {e}")) + .and_then(|checksum_string| { + format!("sha256:{checksum_string}") + .parse() + .map_err(|e| format!("Error {e}")) + }) + }) + .and_then(|checksum: Checksum| { + if checksum == artifact.checksum { + Ok(()) + } else { + Err(format!( + "Checksum mismatch for {url} expected {expected} got {actual}", + url = artifact.url, + expected = hex::encode(&artifact.checksum.value), + actual = hex::encode(&checksum.value) + )) + } + }) + }) + .collect::>>(); + + if results.iter().any(|r| r.is_err()) { + Err(Error::Other( + results + .iter() + .map(|r| r.as_ref().unwrap_err().borrow()) + .collect::>() + .join("\n"), + )) + } else { + Ok(()) + } +} + +fn atomic_file_contents(path: &Path, f: F) -> Result> +where + F: FnOnce(&mut std::fs::File, &str) -> Result>, +{ + fs_err::create_dir_all( + path.parent().ok_or_else(|| { + Error::Other(format!("Cannot determine parent from {}", path.display())) + })?, + ) + .map_err(Error::FsError)?; + + let mut file: std::fs::File = fs_err::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(path)? + .into(); + file.lock_exclusive()?; + use std::io::Seek; + + let mut contents = String::new(); + file.read_to_string(&mut contents).map_err(Error::FsError)?; + file.rewind()?; + let result: Result> = f(&mut file, &contents); + file.unlock()?; + result +} + +pub fn atomic_inventory_update(path: &Path, f: F) -> Result<(), Box> +where + F: FnOnce( + &mut Inventory, + ) -> Result<(), Box>, +{ + atomic_file_contents(path, |file, contents| { + let mut inventory = parse_inventory(contents)?; + f(&mut inventory)?; + write!(file, "{inventory}").map_err(Error::FsError)?; + Ok(()) + }) +} + +fn parse_inventory( + contents: &str, +) -> Result, Error> { + if contents.trim().is_empty() { + Ok(Inventory { + artifacts: Vec::new(), + }) + } else { + contents + .parse::>() + .map_err(|e| Error::Other(format!("Error {e} parsing inventory:\n{contents}"))) + } +} + +/// Returns the sha256 hash of the file at the given path +pub fn sha256_from_path(path: &Path) -> Result { + digest::(fs_err::File::open(path).map_err(Error::FsError)?) + .map(|digest| format!("{:x}", digest)) + .map_err(|e| { + Error::Other(format!( + "Error {e} calculating sha256 for {path}", + path = path.display() + )) + }) +} + +pub fn digest(mut input: impl Read) -> Result, std::io::Error> +where + D: Default + sha2::digest::Update + sha2::digest::FixedOutput, +{ + let mut digest = D::default(); + + let mut buffer = [0x00; 1024]; + loop { + let bytes_read = input.read(&mut buffer)?; + + if bytes_read > 0 { + digest.update(&buffer[..bytes_read]); + } else { + break; + } + } + + Ok(digest.finalize_fixed()) +} + +/// Raises an error if the same URL has a different checksum +/// +/// This protects against the (reasonably) unlikely event that the same version generates a checksum with the same first 7 characters but a net different checksum. +/// While unlikely, it's still possible. If we didn't guard against this case, then it could break people's builds who are relying on the old checksum +/// no not change. +pub fn artifact_same_url_different_checksum( + a: &inventory::artifact::Artifact, + b: &inventory::artifact::Artifact, +) -> Result<(), Box> { + if a.url == b.url && a.checksum != b.checksum { + Err(format!( + "Duplicate url {url} has different checksums {a_checksum} != {b_checksum}", + url = a.url, + a_checksum = hex::encode(&a.checksum.value), + b_checksum = hex::encode(&b.checksum.value) + ) + .into()) + } else { + Ok(()) + } +} + +pub fn artifact_is_different( + a: &inventory::artifact::Artifact, + b: &inventory::artifact::Artifact, +) -> bool { + a.version != b.version + || a.arch != b.arch + || a.metadata.distro_version != b.metadata.distro_version +} + +#[cfg(test)] +mod test { + use crate::BaseImage; + use inventory::artifact::Artifact; + use std::io::Write; + use std::str::FromStr; + + use super::*; + + #[test] + fn test_same_url_different_checksum_raises_error() { + let a = Artifact { + os: inventory::artifact::Os::Linux, + arch: inventory::artifact::Arch::Amd64, + version: GemVersion::from_str("1.0.0").unwrap(), + checksum: "sha256:dd073bda5665e758c3e6f861a6df435175c8e8faf5ec75bc2afaab1e3eebb2c7" + .parse() + .unwrap(), + metadata: ArtifactMetadata { + timestamp: Utc::now(), + distro_version: BaseImage::new("heroku-24").unwrap().distro_version(), + }, + url: "https://example.com".to_string(), + }; + + let b = a.clone(); + artifact_same_url_different_checksum(&a, &b).unwrap(); + + let mut b = a.clone(); + b.checksum = "sha256:7bebeee1b9128bdbb290331b813fa01cf43e30cd0098286f7de011796cb8eee5" + .parse() + .unwrap(); + assert!(artifact_same_url_different_checksum(&a, &b).is_err()); + } + + #[test] + fn test_is_not_version_match() { + let a = Artifact { + os: inventory::artifact::Os::Linux, + arch: inventory::artifact::Arch::Amd64, + version: GemVersion::from_str("1.0.0").unwrap(), + checksum: "sha256:dd073bda5665e758c3e6f861a6df435175c8e8faf5ec75bc2afaab1e3eebb2c7" + .parse() + .unwrap(), + metadata: ArtifactMetadata { + timestamp: Utc::now(), + distro_version: BaseImage::new("heroku-24").unwrap().distro_version(), + }, + url: "https://example.com".to_string(), + }; + + let b = a.clone(); + assert!(!artifact_is_different(&a, &b)); + + let mut b = a.clone(); + b.version = GemVersion::from_str("1.0.1").unwrap(); + assert!(artifact_is_different(&a, &b)); + + let mut b = a.clone(); + b.arch = inventory::artifact::Arch::Arm64; + assert!(artifact_is_different(&a, &b)); + + let mut b = a.clone(); + b.metadata.distro_version = BaseImage::new("heroku-22").unwrap().distro_version(); + assert!(artifact_is_different(&a, &b)); + } + + #[test] + fn test_append_inventory() { + let temp = tempfile::tempdir().expect("Tempdir"); + let path = temp.path().join("inventory.toml"); + let artifact = Artifact { + os: inventory::artifact::Os::Linux, + arch: inventory::artifact::Arch::Amd64, + version: GemVersion::from_str("1.0.0").unwrap(), + checksum: "sha256:dd073bda5665e758c3e6f861a6df435175c8e8faf5ec75bc2afaab1e3eebb2c7" + .parse() + .unwrap(), + metadata: ArtifactMetadata { + timestamp: Utc::now(), + distro_version: BaseImage::new("heroku-24").unwrap().distro_version(), + }, + url: "https://example.com".to_string(), + }; + + atomic_file_contents(&path, |file, contents| { + let mut inventory = parse_inventory(contents)?; + inventory.push(artifact.clone()); + write!(file, "{inventory}").expect("Writeable file"); + Ok(()) + }) + .unwrap(); + + let inventory = parse_inventory(&fs_err::read_to_string(&path).unwrap()).unwrap(); + assert_eq!(1, inventory.artifacts.len()); + + atomic_file_contents(&path, |file, contents| { + let mut inventory = parse_inventory(contents)?; + inventory.push(artifact.clone()); + write!(file, "{inventory}").expect("Writeable file"); + Ok(()) + }) + .unwrap(); + let inventory = parse_inventory(&fs_err::read_to_string(&path).unwrap()).unwrap(); + assert_eq!(2, inventory.artifacts.len()); + } +} diff --git a/shared/src/lib.rs b/shared/src/lib.rs index 5391370..2a62978 100644 --- a/shared/src/lib.rs +++ b/shared/src/lib.rs @@ -2,15 +2,58 @@ use bullet_stream::state::SubBullet; use bullet_stream::Print; use fs_err::{File, PathExt}; use fun_run::CommandWithName; +use inventory::artifact::Arch; use std::io::{Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; use std::process::Command; mod base_image; mod download_ruby_version; +mod inventory_help; -pub use base_image::{BaseImage, CpuArch, CpuArchError}; +pub use base_image::BaseImage; pub use download_ruby_version::RubyDownloadVersion; +pub use inventory_help::{ + artifact_is_different, artifact_same_url_different_checksum, atomic_inventory_update, + inventory_check, sha256_from_path, ArtifactMetadata, +}; + +/// Appends the given string after the filename and before the `ends_with` +/// +/// ``` +/// use std::path::Path; +/// use shared::append_filename_with; +/// +/// let path = Path::new("/tmp/file.txt"); +/// let out = append_filename_with(path, "-lol", ".txt").unwrap(); +/// assert_eq!(Path::new("/tmp/file-lol.txt"), out); +/// ``` +/// +/// Raises an error if the files doesn't exist or if the file name doesn't end with `ends_with` +pub fn append_filename_with(path: &Path, append: &str, ends_with: &str) -> Result { + let parent = path + .parent() + .ok_or_else(|| Error::Other(format!("Cannot determine parent from {}", path.display())))?; + let file_name = path + .file_name() + .ok_or_else(|| { + Error::Other(format!( + "Cannot determine file name from {}", + path.display() + )) + })? + .to_string_lossy(); + + if !file_name.ends_with(ends_with) { + Err(Error::Other(format!( + "File name {} does not end with {}", + file_name, ends_with + )))?; + } + let file_base = file_name.trim_end_matches(ends_with); + + Ok(parent.join(format!("{file_base}{append}{ends_with}"))) +} #[derive(Debug, Clone)] pub struct TarDownloadPath(pub PathBuf); @@ -39,9 +82,6 @@ pub fn untar_to_dir(tar_path: &TarDownloadPath, workspace: &Path) -> Result<(), #[derive(thiserror::Error, Debug)] pub enum Error { - #[error("Error {0}")] - UnknownArchitecture(CpuArchError), - #[error("Command failed {0}")] CmdError(fun_run::CmdError), @@ -67,6 +107,9 @@ pub enum Error { #[source] source: std::io::Error, }, + + #[error("Error {0}")] + Other(String), } pub fn source_dir() -> PathBuf { @@ -110,7 +153,7 @@ pub fn output_tar_path( output: &Path, version: &RubyDownloadVersion, base_image: &BaseImage, - cpu_architecture: &CpuArch, + cpu_architecture: &Arch, ) -> PathBuf { let directory = if base_image.is_arch_aware() { PathBuf::from(base_image.to_string()).join(cpu_architecture.to_string()) @@ -262,7 +305,7 @@ mod test { let output = PathBuf::from("/tmp"); let version = RubyDownloadVersion::from_str("2.7.3").unwrap(); let base_image = BaseImage::new("heroku-20").unwrap(); - let cpu_architecture = CpuArch::from_test_str("amd64"); + let cpu_architecture = Arch::Amd64; let tar_path = output_tar_path(&output, &version, &base_image, &cpu_architecture); @@ -275,7 +318,7 @@ mod test { let output = PathBuf::from("/tmp"); let version = RubyDownloadVersion::from_str("2.7.3").unwrap(); let base_image = BaseImage::new("heroku-24").unwrap(); - let cpu_architecture = CpuArch::from_test_str("amd64"); + let cpu_architecture = Arch::Amd64; let tar_path = output_tar_path(&output, &version, &base_image, &cpu_architecture);