From c5f2030754560bc8d830053a2935063ab14b491a Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Wed, 18 Sep 2024 14:03:13 -0500 Subject: [PATCH] Generate Inventory & Sha7 appended filenames for binaries (#51) * Generate Inventory & Sha7 appended filenames for JRuby Currently the Ruby buildapack is tightly coupled to the URLs that are generated by this repo. I am in the process of introducing an inventory file that the buildpack can use as a lookup. To start that process this commit introduces generating tgz files with the first 7 SHA256 characters appended. This allows us to update the same version number in the future without worrying that it will break future digest checks of the URL. With this change the builder will continue to generate the original URLs, but it will also generate "-.tgz" files as well and append this information to a `jruby_inventory.toml` file. * Use FromStr for converting between arch structs * Refactor and add inventory_check - We now create only one sha named archive for jruby builds since both ARM and AMD point to the same source. - Binary `inventory_check` takes in a file and validates that all checksums are valid. * Use GemVersion * Refactor atomic calls into their own function * Refactor: Prefer composable methods * I did not mean to commit that * Replace older entries by default * Refactor: API to mutate the inventory, not the file * Test logic to preserve/remove artifacts from inventory * Introduce Ruby manifest file * Guard against the case of same sha7 but different sha256 * Fix clippy * Remove accidental inventory addition Sidenote, the check to validate that the URLs are valid works! https://github.com/heroku/docker-heroku-ruby-builder/actions/runs/10102146736/job/27937182426?pr=51 * Auto PR for ruby_inventory.toml * Manual global write permissions Before: ``` drwxr-xr-x 2 root root 4096 Jul 25 22:34 . drwxr-xr-x 3 runner docker 4096 Jul 25 22:32 .. -rw-r--r-- 1 root root 24061018 Jul 25 22:34 ruby-3.2.3.tgz ``` With these permissions we were unable to write to the directory. After: After ``` drwxrwxrwx 2 root root 4096 Jul 27 02:53 . drwxr-xr-x 3 runner docker 4096 Jul 27 02:51 .. -rw-r--r-- 1 root root 24061325 Jul 27 02:53 ruby-3.2.3.tgz ``` Now we can copy the file to a new filename in the same directory. * Prefer inventory::artifact::Arch * Fix clippy and doc tests ``` error: the borrowed expression implements the required traits --> ruby_executable/src/bin/ruby_build.rs:143:24 | 143 | docker_run.arg(&format!( | ________________________^ 144 | | "./make_ruby.sh {} {}", 145 | | input_tar.display(), 146 | | output_tar.display() 147 | | )); | |_________^ | = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#needless_borrows_for_generic_args = note: `-D clippy::needless-borrows-for-generic-args` implied by `-D warnings` = help: to override `-D warnings` add `#[allow(clippy::needless_borrows_for_generic_args)]` help: change this to | 143 ~ docker_run.arg(format!( 144 + "./make_ruby.sh {} {}", 145 + input_tar.display(), 146 + output_tar.display() 147 ~ )); | error: could not compile `ruby_executable` (bin "ruby_build" test) due to 1 previous error warning: build failed, waiting for other jobs to finish... ``` --------- Co-authored-by: Rune Soerensen --- .github/workflows/build_jruby.yml | 10 + .github/workflows/build_ruby.yml | 12 +- .github/workflows/ci.yml | 19 ++ Cargo.lock | 357 +++++++++++++++++++++++- Cargo.toml | 12 + jruby_executable/Cargo.toml | 4 + jruby_executable/src/bin/jruby_build.rs | 75 ++++- jruby_executable/src/bin/jruby_check.rs | 7 +- jruby_executable/src/lib.rs | 4 +- jruby_inventory.toml | 0 make_ruby.sh | 3 + ruby_executable/Cargo.toml | 4 + ruby_executable/src/bin/ruby_build.rs | 74 ++++- ruby_executable/src/bin/ruby_check.rs | 7 +- ruby_inventory.toml | 1 + shared/Cargo.toml | 14 + shared/src/base_image.rs | 62 +--- shared/src/bin/inventory_check.rs | 29 ++ shared/src/inventory_help.rs | 320 +++++++++++++++++++++ shared/src/lib.rs | 57 +++- 20 files changed, 988 insertions(+), 83 deletions(-) create mode 100644 jruby_inventory.toml create mode 100644 ruby_inventory.toml create mode 100644 shared/src/bin/inventory_check.rs create mode 100644 shared/src/inventory_help.rs diff --git a/.github/workflows/build_jruby.yml b/.github/workflows/build_jruby.yml index d3f7e4a..2b79100 100644 --- a/.github/workflows/build_jruby.yml +++ b/.github/workflows/build_jruby.yml @@ -51,3 +51,13 @@ jobs: - name: Upload Ruby runtime archive to S3 production if: (!inputs.dry_run) run: aws s3 sync ./output "s3://${S3_BUCKET}" + + after-build-and-upload: + needs: build-and-upload + runs-on: pub-hk-ubuntu-24.04-xlarge + steps: + - name: Update Jruby inventory file locally + uses: peter-evans/create-pull-request@v6 + with: + path: jruby_inventory.toml + title: "Add JRuby ${{inputs.jruby_version}} to inventory" diff --git a/.github/workflows/build_ruby.yml b/.github/workflows/build_ruby.yml index ffe6580..ef26f63 100644 --- a/.github/workflows/build_ruby.yml +++ b/.github/workflows/build_ruby.yml @@ -24,7 +24,7 @@ env: S3_BUCKET: "heroku-buildpack-ruby" jobs: - build_ruby: + build-and-upload: runs-on: ${{ matrix.arch == 'arm64' && 'pub-hk-ubuntu-24.04-arm-xlarge' || 'pub-hk-ubuntu-24.04-xlarge' }} strategy: matrix: @@ -56,3 +56,13 @@ jobs: - name: Upload Ruby runtime archive to S3 production if: (!inputs.dry_run) run: aws s3 sync ./output "s3://${S3_BUCKET}" + + after-build-and-upload: + needs: build-and-upload + runs-on: pub-hk-ubuntu-24.04-xlarge + steps: + - name: Update Ruby inventory file locally + uses: peter-evans/create-pull-request@v6 + with: + path: ruby_inventory.toml + title: "Add Ruby ${{inputs.ruby_version}} to inventory" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 037c7ef..37d07fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,3 +102,22 @@ jobs: run: cargo run --bin jruby_build -- --version ${{matrix.version}} --base-image ${{matrix.base_image}} - name: Check JRuby run: cargo run --bin jruby_check -- --version ${{matrix.version}} --base-image ${{matrix.base_image}} --arch ${{matrix.arch}} + + check_inventory_urls: + runs-on: ubuntu-24.04 + if: (!contains(github.event.pull_request.labels.*.name, 'skip inventory check')) + strategy: + matrix: + inventory: ["jruby_inventory.toml", "ruby_inventory.toml"] + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Grab prior commits + run: | + set -eu + set pipefail + + git fetch origin ${{ github.base_ref }} --depth 1 && \ + git diff --unified=0 remotes/origin/${{ github.base_ref }} ${{matrix.inventory}} | grep '^+' | grep -v '^+++' | cut -c2- > check_inventory.toml + - name: Check manifest URLs + run: cargo run --bin inventory_check -- check_inventory.toml diff --git a/Cargo.lock b/Cargo.lock index 858d91c..4ffdedc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "0.6.14" @@ -114,6 +129,21 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bitflags" version = "1.3.2" @@ -126,6 +156,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bullet_stream" version = "0.2.0" @@ -156,6 +195,21 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets 0.52.5", +] + [[package]] name = "chunked_transfer" version = "1.5.0" @@ -224,6 +278,15 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -233,6 +296,57 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + [[package]] name = "encoding_rs" version = "0.8.34" @@ -258,6 +372,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "fancy-regex" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.1.0" @@ -325,6 +450,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "fun_run" version = "0.2.0" @@ -385,6 +520,27 @@ dependencies = [ "slab", ] +[[package]] +name = "gem_version" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804d260cc5d12cb2ed81b4e9228eb48c989a36e86b5b7e5c6113509f88d4a69a" +dependencies = [ + "fancy-regex", + "regex", + "serde", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "gimli" version = "0.29.0" @@ -434,6 +590,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "http" version = "1.1.0" @@ -536,6 +698,29 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "idna" version = "0.5.0" @@ -562,6 +747,18 @@ version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" +[[package]] +name = "inventory" +version = "0.1.0" +source = "git+https://github.com/Malax/inventory#e4eed7de95e49d441cdf13f9c001644ed9159393" +dependencies = [ + "hex", + "serde", + "sha2", + "thiserror", + "toml", +] + [[package]] name = "ipnet" version = "2.9.0" @@ -596,17 +793,21 @@ name = "jruby_executable" version = "0.1.0" dependencies = [ "bullet_stream", + "chrono", "clap", "flate2", "fs-err", "fun_run", + "gem_version", "glob", "indoc", + "inventory", "java-properties", "lazy_static", "nom", "regex", "reqwest", + "sha2", "shared", "tar", "tempfile", @@ -711,6 +912,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.16.0" @@ -842,6 +1052,26 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -928,14 +1158,18 @@ name = "ruby_executable" version = "0.1.0" dependencies = [ "bullet_stream", + "chrono", "clap", "flate2", "fs-err", "fun_run", + "gem_version", "indoc", + "inventory", "nom", "regex", "reqwest", + "sha2", "shared", "tar", "tempfile", @@ -1017,18 +1251,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", @@ -1046,6 +1280,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -1058,24 +1301,44 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shared" version = "0.1.0" dependencies = [ "bullet_stream", + "chrono", "clap", "flate2", "fs-err", + "fs2", "fun_run", + "gem_version", "glob", + "hex", "indoc", + "inventory", "nom", + "rayon", "regex", "reqwest", + "serde", + "sha2", "tar", "tempfile", "thiserror", "tiny_http", + "toml", ] [[package]] @@ -1256,6 +1519,40 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2caab0bf757388c6c0ae23b3293fdb463fee59434529014f85e3263b995c28" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "278f3d518e152219c994ce877758516bca5e118eaed6996192a774fb9fbf0788" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.4.13" @@ -1308,6 +1605,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "unicode-bidi" version = "0.3.15" @@ -1352,6 +1655,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + [[package]] name = "want" version = "0.3.1" @@ -1443,6 +1752,37 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.5", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -1582,6 +1922,15 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +[[package]] +name = "winnow" +version = "0.6.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "557404e450152cd6795bb558bca69e43c585055f4606e3bcae5894fc6dac9ba0" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.52.0" diff --git a/Cargo.toml b/Cargo.toml index 1d08439..e6bb04a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,3 +22,15 @@ tar = "0.4.40" tempfile = "3.10.1" thiserror = "1.0.61" bullet_stream = "0.2.0" +sha2 = "0.10" +toml = "0.8" +chrono = {version = "0.4", features = ["serde"] } +serde = {version = "1.0", features = ["derive"] } +inventory = { git = "https://github.com/Malax/inventory", features = ["sha2"] } +rayon = "1.10" +hex = "0.4.3" +gem_version = "0.3" + +# File locking (FLOCK) +fs2 = "0.4" + diff --git a/jruby_executable/Cargo.toml b/jruby_executable/Cargo.toml index c10b017..9a2a6cd 100644 --- a/jruby_executable/Cargo.toml +++ b/jruby_executable/Cargo.toml @@ -32,3 +32,7 @@ tar = { workspace = true} tempfile = { workspace = true} thiserror = { workspace = true} bullet_stream = { workspace = true } +inventory = { workspace = true } +chrono = { workspace = true } +sha2 = { workspace = true } +gem_version = { workspace = true } diff --git a/jruby_executable/src/bin/jruby_build.rs b/jruby_executable/src/bin/jruby_build.rs index 29e60d4..b6deb30 100644 --- a/jruby_executable/src/bin/jruby_build.rs +++ b/jruby_executable/src/bin/jruby_build.rs @@ -1,10 +1,20 @@ use bullet_stream::{style, Print}; use clap::Parser; use fs_err::PathExt; +use gem_version::GemVersion; use indoc::formatdoc; +use inventory::artifact::{Arch, Artifact}; use jruby_executable::jruby_build_properties; -use shared::{download_tar, source_dir, tar_dir_to_file, untar_to_dir, BaseImage, TarDownloadPath}; +use shared::{ + append_filename_with, artifact_is_different, artifact_same_url_different_checksum, + atomic_inventory_update, download_tar, sha256_from_path, source_dir, tar_dir_to_file, + untar_to_dir, ArtifactMetadata, BaseImage, TarDownloadPath, +}; +use std::convert::From; use std::error::Error; +use std::str::FromStr; + +static S3_BASE_URL: &str = "https://heroku-buildpack-ruby.s3.us-east-1.amazonaws.com"; #[derive(Parser, Debug)] struct Args { @@ -22,6 +32,7 @@ fn jruby_build(args: &Args) -> Result<(), Box> { } = args; let mut log = Print::new(std::io::stderr()).h1("Building JRuby"); + let inventory = source_dir().join("jruby_inventory.toml"); let volume_cache_dir = source_dir().join("cache"); let volume_output_dir = source_dir().join("output"); @@ -89,6 +100,10 @@ fn jruby_build(args: &Args) -> Result<(), Box> { log = { let mut bullet = log.bullet("Creating tgz archives"); + bullet = bullet.sub_bullet(format!( + "Inventory file {}", + style::value(inventory.to_string_lossy()) + )); let tar_dir = volume_output_dir.join(base_image.to_string()); fs_err::create_dir_all(&tar_dir)?; @@ -99,8 +114,61 @@ fn jruby_build(args: &Args) -> Result<(), Box> { tar_dir_to_file(&jruby_dir, &tar_file)?; bullet = timer.done(); - for arch in &["amd64", "arm64"] { - let dir = volume_output_dir.join(base_image.to_string()).join(arch); + let tar_path = tar_file.path(); + let sha = sha256_from_path(tar_path)?; + let sha_seven = sha.chars().take(7).collect::(); + let sha_seven_path = append_filename_with(tar_path, &format!("-{sha_seven}"), ".tgz")?; + + bullet = bullet.sub_bullet(format!("Write {}", sha_seven_path.display(),)); + fs_err::copy(tar_file.path(), &sha_seven_path)?; + + let timestamp = chrono::Utc::now(); + for cpu_arch in [Arch::Amd64, Arch::Arm64] { + let distro_version = base_image.distro_version(); + let artifact = Artifact { + version: GemVersion::from_str(version)?, + os: inventory::artifact::Os::Linux, + arch: cpu_arch, + url: format!( + "{S3_BASE_URL}/{}", + sha_seven_path.strip_prefix(&volume_output_dir)?.display() + ), + checksum: format!("sha256:{sha}").parse()?, + metadata: ArtifactMetadata { + distro_version, + timestamp, + }, + }; + atomic_inventory_update(&inventory, |inventory| { + for prior in &inventory.artifacts { + if let Err(error) = artifact_same_url_different_checksum(prior, &artifact) { + // TODO: Investigate bullet stream ownership + println!( + "{}", + style::important(format!( + "!!!!!!!!!! Error updating inventory: {error}" + )) + ); + + fs_err::remove_file(&sha_seven_path)?; + return Err(error); + }; + } + + inventory + .artifacts + .retain(|a| artifact_is_different(a, &artifact)); + + inventory.push(artifact); + Ok(()) + })? + } + + // Can be removed once manifest file support is fully rolled out + for cpu_arch in [Arch::Amd64, Arch::Arm64] { + let dir = volume_output_dir + .join(base_image.to_string()) + .join(cpu_arch.to_string()); fs_err::create_dir_all(&dir)?; let path = dir.join(&tgz_name); @@ -110,6 +178,7 @@ fn jruby_build(args: &Args) -> Result<(), Box> { bullet.done() }; + log.done(); Ok(()) diff --git a/jruby_executable/src/bin/jruby_check.rs b/jruby_executable/src/bin/jruby_check.rs index f5a8347..d276067 100644 --- a/jruby_executable/src/bin/jruby_check.rs +++ b/jruby_executable/src/bin/jruby_check.rs @@ -2,8 +2,9 @@ use bullet_stream::{style, Print}; use clap::Parser; use fun_run::CommandWithName; use indoc::formatdoc; +use inventory::artifact::Arch; use jruby_executable::jruby_build_properties; -use shared::{source_dir, BaseImage, CpuArch}; +use shared::{source_dir, BaseImage}; use std::error::Error; use std::io::Write; use std::{path::PathBuf, process::Command}; @@ -13,7 +14,7 @@ static INNER_OUTPUT: &str = "/tmp/output"; #[derive(Parser, Debug)] struct RubyArgs { #[arg(long)] - arch: CpuArch, + arch: Arch, #[arg(long)] version: String, @@ -97,7 +98,7 @@ fn jruby_check(args: &RubyArgs) -> Result<(), Box> { cmd.arg(image_name); cmd.args(["bash", "-c"]); cmd.arg( - &[ + [ "mkdir /tmp/unzipped", &format!("tar xzf {} -C /tmp/unzipped", inner_jruby_path.display()), "export PATH=\"tmp/unzipped/bin:$PATH\"", diff --git a/jruby_executable/src/lib.rs b/jruby_executable/src/lib.rs index 6f33d5d..ac597c8 100644 --- a/jruby_executable/src/lib.rs +++ b/jruby_executable/src/lib.rs @@ -9,14 +9,14 @@ /// implements Ruby 3.1.4 stdlib. When people use jruby they specify both the /// jruby version and the stdlib version, for example: /// -/// ```ignore +/// ```ruby /// # Gemfile /// ruby "3.1.4", engine: "jruby", engine_version: "9.4.3.0" /// ``` /// /// Example file for /// -/// ```ignore +/// ```ini /// # Defaults. To override, create a file called build.properties in /// # the same directory and put your changes in that. /// #src.dir=src diff --git a/jruby_inventory.toml b/jruby_inventory.toml new file mode 100644 index 0000000..e69de29 diff --git a/make_ruby.sh b/make_ruby.sh index b982bf4..6c924c0 100755 --- a/make_ruby.sh +++ b/make_ruby.sh @@ -23,6 +23,9 @@ PS4='>\e[33m${BASH_SOURCE}:${LINENO} $\e[0m ' set -o xtrace mkdir -p "$(dirname "$OUT_TAR")" + +# Docker issue with permissions +chmod a+w "$(dirname "$OUT_TAR")" mkdir -p /tmp/source mkdir -p /tmp/compiled diff --git a/ruby_executable/Cargo.toml b/ruby_executable/Cargo.toml index b0185e0..e7df66b 100644 --- a/ruby_executable/Cargo.toml +++ b/ruby_executable/Cargo.toml @@ -17,3 +17,7 @@ tar = { workspace = true} tempfile = { workspace = true} thiserror = { workspace = true} bullet_stream = { workspace = true } +inventory = { workspace = true } +chrono = { workspace = true } +sha2 = { workspace = true } +gem_version = { workspace = true } diff --git a/ruby_executable/src/bin/ruby_build.rs b/ruby_executable/src/bin/ruby_build.rs index 829df6f..eedabf0 100644 --- a/ruby_executable/src/bin/ruby_build.rs +++ b/ruby_executable/src/bin/ruby_build.rs @@ -2,24 +2,29 @@ use bullet_stream::{style, Print}; use clap::Parser; use fs_err::PathExt; use fun_run::CommandWithName; +use gem_version::GemVersion; use indoc::{formatdoc, indoc}; +use inventory::artifact::{Arch, Artifact}; use shared::{ - download_tar, output_tar_path, source_dir, validate_version_for_stack, BaseImage, CpuArch, - RubyDownloadVersion, TarDownloadPath, + append_filename_with, artifact_is_different, artifact_same_url_different_checksum, + atomic_inventory_update, download_tar, output_tar_path, sha256_from_path, source_dir, + validate_version_for_stack, ArtifactMetadata, BaseImage, RubyDownloadVersion, TarDownloadPath, }; use std::{ io::Write, path::{Path, PathBuf}, process::Command, + str::FromStr, }; static INNER_OUTPUT: &str = "/tmp/output"; static INNER_CACHE: &str = "/tmp/cache"; +static S3_BASE_URL: &str = "https://heroku-buildpack-ruby.s3.us-east-1.amazonaws.com"; #[derive(Parser, Debug)] struct RubyArgs { #[arg(long)] - arch: CpuArch, + arch: Arch, #[arg(long)] version: RubyDownloadVersion, @@ -57,6 +62,7 @@ fn ruby_build(args: &RubyArgs) -> Result<(), Box> { } = args; let mut log = Print::new(std::io::stderr()).h1("Building Ruby"); + let inventory = source_dir().join("ruby_inventory.toml"); let volume_cache_dir = source_dir().join("cache"); let volume_output_dir = source_dir().join("output"); @@ -116,12 +122,10 @@ fn ruby_build(args: &RubyArgs) -> Result<(), Box> { log = { let mut bullet = log.bullet("Make Ruby"); - let input_tar = PathBuf::from(INNER_CACHE).join(format!("ruby-source-{version}.tgz")); let output_tar = output_tar_path(Path::new(INNER_OUTPUT), version, base_image, arch); - - let volume_output = volume_output_dir.display(); let volume_cache = volume_cache_dir.display(); + let volume_output = volume_output_dir.display(); let mut docker_run = Command::new("docker"); docker_run.arg("run"); @@ -136,7 +140,7 @@ fn ruby_build(args: &RubyArgs) -> Result<(), Box> { docker_run.arg(&image_name); docker_run.args(["bash", "-c"]); - docker_run.arg(&format!( + docker_run.arg(format!( "./make_ruby.sh {} {}", input_tar.display(), output_tar.display() @@ -146,6 +150,62 @@ fn ruby_build(args: &RubyArgs) -> Result<(), Box> { format!("Running {}", style::command(docker_run.name())), |stdout, stderr| docker_run.stream_output(stdout, stderr), )?; + bullet.done() + }; + + log = { + let mut bullet = log.bullet(format!( + "Updating manifest {}", + style::value(inventory.to_string_lossy()) + )); + + let output_tar = output_tar_path(&volume_output_dir, version, base_image, arch); + + let sha = sha256_from_path(&output_tar)?; + let sha_seven = sha.chars().take(7).collect::(); + let sha_seven_path = append_filename_with(&output_tar, &format!("-{sha_seven}"), ".tgz")?; + let url = format!( + "{S3_BASE_URL}/{}", + sha_seven_path.strip_prefix(&volume_output_dir)?.display() + ); + + bullet = bullet.sub_bullet(format!("Copying SHA tgz {}", sha_seven_path.display(),)); + fs_err::copy(output_tar, &sha_seven_path)?; + + let artifact = Artifact { + version: GemVersion::from_str(&version.bundler_format())?, + os: inventory::artifact::Os::Linux, + arch: *arch, + url, + checksum: format!("sha256:{sha}").parse()?, + metadata: ArtifactMetadata { + distro_version: base_image.distro_version(), + timestamp: chrono::Utc::now(), + }, + }; + + atomic_inventory_update(&inventory, |inventory| { + for prior in &inventory.artifacts { + if let Err(error) = artifact_same_url_different_checksum(prior, &artifact) { + // TODO: Investigate bullet stream ownership + println!( + "{}", + style::important(format!("!!!!!!!!!! Error updating inventory: {error}")) + ); + + fs_err::remove_file(&sha_seven_path)?; + return Err(error); + }; + } + + inventory + .artifacts + .retain(|a| artifact_is_different(a, &artifact)); + + inventory.push(artifact); + + Ok(()) + })?; bullet.done() }; diff --git a/ruby_executable/src/bin/ruby_check.rs b/ruby_executable/src/bin/ruby_check.rs index ac5bc87..6ad12cf 100644 --- a/ruby_executable/src/bin/ruby_check.rs +++ b/ruby_executable/src/bin/ruby_check.rs @@ -2,7 +2,8 @@ use bullet_stream::{style, Print}; use clap::Parser; use fun_run::CommandWithName; use indoc::formatdoc; -use shared::{output_tar_path, source_dir, BaseImage, CpuArch, RubyDownloadVersion}; +use inventory::artifact::Arch; +use shared::{output_tar_path, source_dir, BaseImage, RubyDownloadVersion}; use std::{error::Error, path::PathBuf, process::Command}; static INNER_OUTPUT: &str = "/tmp/output"; @@ -10,7 +11,7 @@ static INNER_OUTPUT: &str = "/tmp/output"; #[derive(Parser, Debug)] struct RubyArgs { #[arg(long)] - arch: CpuArch, + arch: Arch, #[arg(long)] version: RubyDownloadVersion, @@ -48,7 +49,7 @@ fn ruby_check(args: &RubyArgs) -> Result<(), Box> { cmd.arg(image_name); cmd.args(["bash", "-c"]); cmd.arg( - &[ + [ "mkdir /tmp/unzipped", &format!("tar xzf {} -C /tmp/unzipped", path.display()), "echo -n '- Rubygems version: '", diff --git a/ruby_inventory.toml b/ruby_inventory.toml new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/ruby_inventory.toml @@ -0,0 +1 @@ + diff --git a/shared/Cargo.toml b/shared/Cargo.toml index 1072bf5..3a624eb 100644 --- a/shared/Cargo.toml +++ b/shared/Cargo.toml @@ -3,6 +3,10 @@ name = "shared" version = "0.1.0" edition = "2021" +[[bin]] +name = "inventory_check" +path = "src/bin/inventory_check.rs" + [dependencies] glob = { workspace = true} clap = { workspace = true} @@ -17,6 +21,16 @@ tar = { workspace = true} tempfile = { workspace = true} thiserror = { workspace = true} bullet_stream = { workspace = true } +sha2 = { workspace = true } +chrono = { workspace = true } +serde = { workspace = true } +toml = { workspace = true } +inventory = { workspace = true } +fs2 = { workspace = true } +rayon = { workspace = true } +hex = { workspace = true } +gem_version = { workspace = true } + [dev-dependencies] tiny_http = "0.12.0" diff --git a/shared/src/base_image.rs b/shared/src/base_image.rs index 4e95870..e8477d6 100644 --- a/shared/src/base_image.rs +++ b/shared/src/base_image.rs @@ -1,7 +1,8 @@ use std::fmt::Display; use std::str::FromStr; -static KNOWN_ARCHITECTURES: [&str; 2] = ["amd64", "arm64"]; +use serde::{Deserialize, Serialize}; + static KNOWN_BASE_IMAGES: &[(&str, &str)] = &[ ("heroku-20", "20"), ("heroku-22", "22"), @@ -19,6 +20,9 @@ pub struct BaseImage { distro_number: String, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct DistroVersion(String); + impl BaseImage { pub fn new(s: &str) -> Result { KNOWN_BASE_IMAGES @@ -31,6 +35,10 @@ impl BaseImage { .ok_or_else(|| BaseImageError(s.to_owned())) } + pub fn distro_version(&self) -> DistroVersion { + DistroVersion(format!("{}.04", self.distro_number)) + } + pub fn is_arch_aware(&self) -> bool { MULTI_ARCH_BASE_IMAGES.contains(&self.name.as_str()) } @@ -59,55 +67,3 @@ impl FromStr for BaseImage { BaseImage::new(s) } } - -#[derive(Debug, Clone)] -pub struct CpuArch { - name: String, -} - -impl Display for CpuArch { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.name) - } -} - -impl FromStr for CpuArch { - type Err = CpuArchError; - - fn from_str(s: &str) -> Result { - CpuArch::new(s) - } -} - -#[derive(Debug, thiserror::Error)] -#[error("Invalid CPU architecture {0} must be one of {}", KNOWN_ARCHITECTURES.join(", "))] -pub struct CpuArchError(String); - -impl CpuArch { - pub fn new(s: &str) -> Result { - KNOWN_ARCHITECTURES - .iter() - .find(|&&name| name == s) - .map(|_| Self { name: s.to_owned() }) - .ok_or_else(|| CpuArchError(s.to_owned())) - } - - pub fn from_system() -> Result { - let arch = if cfg!(target_arch = "aarch64") { - "arm64" - } else if cfg!(target_arch = "x86_64") { - "amd64" - } else { - "Unknown architecture" - }; - - Self::new(arch) - } - - #[cfg(test)] - pub(crate) fn from_test_str(name: &str) -> Self { - Self { - name: name.to_string(), - } - } -} diff --git a/shared/src/bin/inventory_check.rs b/shared/src/bin/inventory_check.rs new file mode 100644 index 0000000..e42951b --- /dev/null +++ b/shared/src/bin/inventory_check.rs @@ -0,0 +1,29 @@ +use clap::Parser; +use indoc::formatdoc; +use shared::inventory_check; +use std::path::{Path, PathBuf}; + +#[derive(Parser, Debug)] +struct Args { + path: PathBuf, +} + +fn check(path: &Path) -> Result<(), Box> { + let contents = fs_err::read_to_string(path)?; + inventory_check(&contents)?; + Ok(()) +} + +fn main() { + let args = Args::parse(); + if let Err(error) = check(&args.path) { + bullet_stream::Print::new(std::io::stderr()) + .without_header() + .error(formatdoc! {" + ❌ Command failed ❌ + + {error} + "}); + std::process::exit(1); + } +} diff --git a/shared/src/inventory_help.rs b/shared/src/inventory_help.rs new file mode 100644 index 0000000..9635afa --- /dev/null +++ b/shared/src/inventory_help.rs @@ -0,0 +1,320 @@ +use crate::base_image::DistroVersion; +use crate::{download_tar, Error, TarDownloadPath}; +use chrono::{DateTime, Utc}; +use fs2::FileExt; +use gem_version::GemVersion; +use inventory::checksum::Checksum; +use inventory::inventory::Inventory; +use rayon::prelude::*; +use serde::{Deserialize, Serialize}; +use sha2::Sha256; +use std::borrow::Borrow; +use std::io::Read; +use std::io::Write; +use std::path::Path; + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct ArtifactMetadata { + pub timestamp: DateTime, + pub distro_version: DistroVersion, +} + +/// ``` +/// use shared::inventory_check; +/// +/// let contents = r#" +/// [[artifacts]] +/// version = "9.4.8.0" +/// os = "linux" +/// arch = "amd64" +/// url = "https://heroku-buildpack-ruby.s3.us-east-1.amazonaws.com/heroku-24/ruby-3.1.4-jruby-9.4.8.0.tgz" +/// checksum = "sha256:815b31d2b204a524bf74aabae341bf85353add4d1128d5d276d08fa5e8ff3c39" +/// +/// [artifacts.metadata] +/// timestamp = "2024-07-24T16:17:35.341413Z" +/// distro_version = "24.04" +/// "#; +/// inventory_check(contents).unwrap(); +/// ``` +pub fn inventory_check(contents: &str) -> Result<(), Error> { + if contents.trim().is_empty() { + return Ok(()); + } + + let inventory = contents + .parse::>() + .map_err(|e| Error::Other(format!("Could not parse inventory. Error: {e}")))?; + + let results = inventory + .artifacts + .par_iter() + .map(|artifact| { + let temp = tempfile::tempdir().expect("Tempdir"); + let dir = temp.path(); + let path = dir.join("file.tar"); + + download_tar(&artifact.url, &TarDownloadPath(path.clone())) + .map_err(|e| format!("Error {e}")) + .and_then(|_| { + sha256_from_path(&path) + .map_err(|e| format!("Error {e}")) + .and_then(|checksum_string| { + format!("sha256:{checksum_string}") + .parse() + .map_err(|e| format!("Error {e}")) + }) + }) + .and_then(|checksum: Checksum| { + if checksum == artifact.checksum { + Ok(()) + } else { + Err(format!( + "Checksum mismatch for {url} expected {expected} got {actual}", + url = artifact.url, + expected = hex::encode(&artifact.checksum.value), + actual = hex::encode(&checksum.value) + )) + } + }) + }) + .collect::>>(); + + if results.iter().any(|r| r.is_err()) { + Err(Error::Other( + results + .iter() + .map(|r| r.as_ref().unwrap_err().borrow()) + .collect::>() + .join("\n"), + )) + } else { + Ok(()) + } +} + +fn atomic_file_contents(path: &Path, f: F) -> Result> +where + F: FnOnce(&mut std::fs::File, &str) -> Result>, +{ + fs_err::create_dir_all( + path.parent().ok_or_else(|| { + Error::Other(format!("Cannot determine parent from {}", path.display())) + })?, + ) + .map_err(Error::FsError)?; + + let mut file: std::fs::File = fs_err::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(path)? + .into(); + file.lock_exclusive()?; + use std::io::Seek; + + let mut contents = String::new(); + file.read_to_string(&mut contents).map_err(Error::FsError)?; + file.rewind()?; + let result: Result> = f(&mut file, &contents); + file.unlock()?; + result +} + +pub fn atomic_inventory_update(path: &Path, f: F) -> Result<(), Box> +where + F: FnOnce( + &mut Inventory, + ) -> Result<(), Box>, +{ + atomic_file_contents(path, |file, contents| { + let mut inventory = parse_inventory(contents)?; + f(&mut inventory)?; + write!(file, "{inventory}").map_err(Error::FsError)?; + Ok(()) + }) +} + +fn parse_inventory( + contents: &str, +) -> Result, Error> { + if contents.trim().is_empty() { + Ok(Inventory { + artifacts: Vec::new(), + }) + } else { + contents + .parse::>() + .map_err(|e| Error::Other(format!("Error {e} parsing inventory:\n{contents}"))) + } +} + +/// Returns the sha256 hash of the file at the given path +pub fn sha256_from_path(path: &Path) -> Result { + digest::(fs_err::File::open(path).map_err(Error::FsError)?) + .map(|digest| format!("{:x}", digest)) + .map_err(|e| { + Error::Other(format!( + "Error {e} calculating sha256 for {path}", + path = path.display() + )) + }) +} + +pub fn digest(mut input: impl Read) -> Result, std::io::Error> +where + D: Default + sha2::digest::Update + sha2::digest::FixedOutput, +{ + let mut digest = D::default(); + + let mut buffer = [0x00; 1024]; + loop { + let bytes_read = input.read(&mut buffer)?; + + if bytes_read > 0 { + digest.update(&buffer[..bytes_read]); + } else { + break; + } + } + + Ok(digest.finalize_fixed()) +} + +/// Raises an error if the same URL has a different checksum +/// +/// This protects against the (reasonably) unlikely event that the same version generates a checksum with the same first 7 characters but a net different checksum. +/// While unlikely, it's still possible. If we didn't guard against this case, then it could break people's builds who are relying on the old checksum +/// no not change. +pub fn artifact_same_url_different_checksum( + a: &inventory::artifact::Artifact, + b: &inventory::artifact::Artifact, +) -> Result<(), Box> { + if a.url == b.url && a.checksum != b.checksum { + Err(format!( + "Duplicate url {url} has different checksums {a_checksum} != {b_checksum}", + url = a.url, + a_checksum = hex::encode(&a.checksum.value), + b_checksum = hex::encode(&b.checksum.value) + ) + .into()) + } else { + Ok(()) + } +} + +pub fn artifact_is_different( + a: &inventory::artifact::Artifact, + b: &inventory::artifact::Artifact, +) -> bool { + a.version != b.version + || a.arch != b.arch + || a.metadata.distro_version != b.metadata.distro_version +} + +#[cfg(test)] +mod test { + use crate::BaseImage; + use inventory::artifact::Artifact; + use std::io::Write; + use std::str::FromStr; + + use super::*; + + #[test] + fn test_same_url_different_checksum_raises_error() { + let a = Artifact { + os: inventory::artifact::Os::Linux, + arch: inventory::artifact::Arch::Amd64, + version: GemVersion::from_str("1.0.0").unwrap(), + checksum: "sha256:dd073bda5665e758c3e6f861a6df435175c8e8faf5ec75bc2afaab1e3eebb2c7" + .parse() + .unwrap(), + metadata: ArtifactMetadata { + timestamp: Utc::now(), + distro_version: BaseImage::new("heroku-24").unwrap().distro_version(), + }, + url: "https://example.com".to_string(), + }; + + let b = a.clone(); + artifact_same_url_different_checksum(&a, &b).unwrap(); + + let mut b = a.clone(); + b.checksum = "sha256:7bebeee1b9128bdbb290331b813fa01cf43e30cd0098286f7de011796cb8eee5" + .parse() + .unwrap(); + assert!(artifact_same_url_different_checksum(&a, &b).is_err()); + } + + #[test] + fn test_is_not_version_match() { + let a = Artifact { + os: inventory::artifact::Os::Linux, + arch: inventory::artifact::Arch::Amd64, + version: GemVersion::from_str("1.0.0").unwrap(), + checksum: "sha256:dd073bda5665e758c3e6f861a6df435175c8e8faf5ec75bc2afaab1e3eebb2c7" + .parse() + .unwrap(), + metadata: ArtifactMetadata { + timestamp: Utc::now(), + distro_version: BaseImage::new("heroku-24").unwrap().distro_version(), + }, + url: "https://example.com".to_string(), + }; + + let b = a.clone(); + assert!(!artifact_is_different(&a, &b)); + + let mut b = a.clone(); + b.version = GemVersion::from_str("1.0.1").unwrap(); + assert!(artifact_is_different(&a, &b)); + + let mut b = a.clone(); + b.arch = inventory::artifact::Arch::Arm64; + assert!(artifact_is_different(&a, &b)); + + let mut b = a.clone(); + b.metadata.distro_version = BaseImage::new("heroku-22").unwrap().distro_version(); + assert!(artifact_is_different(&a, &b)); + } + + #[test] + fn test_append_inventory() { + let temp = tempfile::tempdir().expect("Tempdir"); + let path = temp.path().join("inventory.toml"); + let artifact = Artifact { + os: inventory::artifact::Os::Linux, + arch: inventory::artifact::Arch::Amd64, + version: GemVersion::from_str("1.0.0").unwrap(), + checksum: "sha256:dd073bda5665e758c3e6f861a6df435175c8e8faf5ec75bc2afaab1e3eebb2c7" + .parse() + .unwrap(), + metadata: ArtifactMetadata { + timestamp: Utc::now(), + distro_version: BaseImage::new("heroku-24").unwrap().distro_version(), + }, + url: "https://example.com".to_string(), + }; + + atomic_file_contents(&path, |file, contents| { + let mut inventory = parse_inventory(contents)?; + inventory.push(artifact.clone()); + write!(file, "{inventory}").expect("Writeable file"); + Ok(()) + }) + .unwrap(); + + let inventory = parse_inventory(&fs_err::read_to_string(&path).unwrap()).unwrap(); + assert_eq!(1, inventory.artifacts.len()); + + atomic_file_contents(&path, |file, contents| { + let mut inventory = parse_inventory(contents)?; + inventory.push(artifact.clone()); + write!(file, "{inventory}").expect("Writeable file"); + Ok(()) + }) + .unwrap(); + let inventory = parse_inventory(&fs_err::read_to_string(&path).unwrap()).unwrap(); + assert_eq!(2, inventory.artifacts.len()); + } +} diff --git a/shared/src/lib.rs b/shared/src/lib.rs index 5391370..2a62978 100644 --- a/shared/src/lib.rs +++ b/shared/src/lib.rs @@ -2,15 +2,58 @@ use bullet_stream::state::SubBullet; use bullet_stream::Print; use fs_err::{File, PathExt}; use fun_run::CommandWithName; +use inventory::artifact::Arch; use std::io::{Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; use std::process::Command; mod base_image; mod download_ruby_version; +mod inventory_help; -pub use base_image::{BaseImage, CpuArch, CpuArchError}; +pub use base_image::BaseImage; pub use download_ruby_version::RubyDownloadVersion; +pub use inventory_help::{ + artifact_is_different, artifact_same_url_different_checksum, atomic_inventory_update, + inventory_check, sha256_from_path, ArtifactMetadata, +}; + +/// Appends the given string after the filename and before the `ends_with` +/// +/// ``` +/// use std::path::Path; +/// use shared::append_filename_with; +/// +/// let path = Path::new("/tmp/file.txt"); +/// let out = append_filename_with(path, "-lol", ".txt").unwrap(); +/// assert_eq!(Path::new("/tmp/file-lol.txt"), out); +/// ``` +/// +/// Raises an error if the files doesn't exist or if the file name doesn't end with `ends_with` +pub fn append_filename_with(path: &Path, append: &str, ends_with: &str) -> Result { + let parent = path + .parent() + .ok_or_else(|| Error::Other(format!("Cannot determine parent from {}", path.display())))?; + let file_name = path + .file_name() + .ok_or_else(|| { + Error::Other(format!( + "Cannot determine file name from {}", + path.display() + )) + })? + .to_string_lossy(); + + if !file_name.ends_with(ends_with) { + Err(Error::Other(format!( + "File name {} does not end with {}", + file_name, ends_with + )))?; + } + let file_base = file_name.trim_end_matches(ends_with); + + Ok(parent.join(format!("{file_base}{append}{ends_with}"))) +} #[derive(Debug, Clone)] pub struct TarDownloadPath(pub PathBuf); @@ -39,9 +82,6 @@ pub fn untar_to_dir(tar_path: &TarDownloadPath, workspace: &Path) -> Result<(), #[derive(thiserror::Error, Debug)] pub enum Error { - #[error("Error {0}")] - UnknownArchitecture(CpuArchError), - #[error("Command failed {0}")] CmdError(fun_run::CmdError), @@ -67,6 +107,9 @@ pub enum Error { #[source] source: std::io::Error, }, + + #[error("Error {0}")] + Other(String), } pub fn source_dir() -> PathBuf { @@ -110,7 +153,7 @@ pub fn output_tar_path( output: &Path, version: &RubyDownloadVersion, base_image: &BaseImage, - cpu_architecture: &CpuArch, + cpu_architecture: &Arch, ) -> PathBuf { let directory = if base_image.is_arch_aware() { PathBuf::from(base_image.to_string()).join(cpu_architecture.to_string()) @@ -262,7 +305,7 @@ mod test { let output = PathBuf::from("/tmp"); let version = RubyDownloadVersion::from_str("2.7.3").unwrap(); let base_image = BaseImage::new("heroku-20").unwrap(); - let cpu_architecture = CpuArch::from_test_str("amd64"); + let cpu_architecture = Arch::Amd64; let tar_path = output_tar_path(&output, &version, &base_image, &cpu_architecture); @@ -275,7 +318,7 @@ mod test { let output = PathBuf::from("/tmp"); let version = RubyDownloadVersion::from_str("2.7.3").unwrap(); let base_image = BaseImage::new("heroku-24").unwrap(); - let cpu_architecture = CpuArch::from_test_str("amd64"); + let cpu_architecture = Arch::Amd64; let tar_path = output_tar_path(&output, &version, &base_image, &cpu_architecture);