From 860a1ea07467788473f0a83edc4d7a02d8ad3c1a Mon Sep 17 00:00:00 2001 From: Antoine Poinsot Date: Thu, 18 Apr 2024 16:58:04 +0200 Subject: [PATCH] guix: use Rust 1.70 for release builds We are being pulled over in two different directions when it comes to our reproducible builds. On the one hand we need to target reasonably old glibc versions in order to be compatible with older systems. On the other hand the immaturity of the Rust ecosystem makes us require bleeding edge versions of the compiler. With Guix to get the newer versions of the compiler we need to also bump the glibc version. This was not a sustainable situation. I was planning for a long time to cleanup our reproducible builds. To create a proper Guix package for both the daemon and the GUI using the build system they provide. I had envisioned this way i could rewrite the inputs of the Guix package to use an older glibc, while being able to bump the Guix time-machine. It would even have allowed us to perform Windows builds inside Guix! And who knows i could even have attempted to perform Apple ones too. Unfortunately it turned out to be more complicated than that. I couldn't manage to get my package to compile using an older glibc. Some details about some of my failed attempts can be found there: https://lists.gnu.org/archive/html/help-guix/2024-04/msg00056.html. Instead of wasting more time on this, backport the newer Rust declarations from up-to-date Guix to Guix-of-our-time-machine. --- contrib/reproducible/guix/guix-build.sh | 2 +- contrib/reproducible/guix/manifest.scm | 145 +++++++++++++++++- .../patches/rust-1.70-fix-rustix-build.patch | 21 +++ doc/BUILD.md | 12 +- 4 files changed, 165 insertions(+), 15 deletions(-) create mode 100644 contrib/reproducible/guix/patches/rust-1.70-fix-rustix-build.patch diff --git a/contrib/reproducible/guix/guix-build.sh b/contrib/reproducible/guix/guix-build.sh index 79fd1257c..a2e385102 100755 --- a/contrib/reproducible/guix/guix-build.sh +++ b/contrib/reproducible/guix/guix-build.sh @@ -60,7 +60,7 @@ for project_folder in "" "gui"; do PROJECT_ROOT="$PWD/$project_folder" PROJECT_VENDOR_DIR="$VENDOR_DIR/$project_folder" PROJECT_OUT_DIR="$OUT_DIR/$project_folder" - PROJECT_PATCHES_ROOT="$PWD/contrib/reproducible/guix/patches/$project_folder" + PROJECT_PATCHES_ROOT="$PWD/contrib/reproducible/guix/liana-patches/$project_folder" project_needs_patches() { test "$(ls -A1q "$PROJECT_PATCHES_ROOT" |grep patch)" diff --git a/contrib/reproducible/guix/manifest.scm b/contrib/reproducible/guix/manifest.scm index 304e63011..aac15df94 100644 --- a/contrib/reproducible/guix/manifest.scm +++ b/contrib/reproducible/guix/manifest.scm @@ -1,3 +1,135 @@ +(use-modules + (gnu packages llvm) + (gnu packages rust) + (guix packages)) + +;; Stolen from Bitcoin Core. Some patches are needed to bootstrap a newer rustc. +(define-syntax-rule (search-our-patches file-name ...) + "Return the list of absolute file names corresponding to each +FILE-NAME found in ./patches relative to the current file." + (parameterize + ((%patch-path (list (string-append (dirname (current-filename)) "/patches")))) + (list (search-patch file-name) ...))) + +;; What follows are the declaration of more recent rustc versions from the current Guix master +;; (https://git.savannah.gnu.org/cgit/guix.git/tree/gnu/packages/rust.scm), which we can't use +;; because it would require an unreasonable glibc version. + +(define rust-bootstrapped-package + (@@ (gnu packages rust) rust-bootstrapped-package)) + +(define %cargo-reference-hash + (@@ (gnu packages rust) %cargo-reference-hash)) + +(define rust-1.65 + (@@ (gnu packages rust) rust-1.65)) + +(define rust-1.66 + (rust-bootstrapped-package + rust-1.65 "1.66.1" "1fjr94gsicsxd2ypz4zm8aad1zdbiccr7qjfbmq8f8f7jhx96g2v")) + +(define rust-1.67 + (let ((base-rust + (rust-bootstrapped-package + rust-1.66 "1.67.1" "0vpzv6rm3w1wbni17ryvcw83k5klhghklylfdza3nnp8blz3sj26"))) + (package + (inherit base-rust) + (source + (origin + (inherit (package-source base-rust)) + (snippet + '(begin + (for-each delete-file-recursively + '("src/llvm-project" + "vendor/openssl-src/openssl" + "vendor/tikv-jemalloc-sys/jemalloc")) + ;; Adjust rustix to always build with cc. + (substitute* '("Cargo.lock" + "src/bootstrap/Cargo.lock") + (("\"errno\",") "\"cc\",\n \"errno\",")) + ;; Add a dependency on the the 'cc' feature of rustix. + (substitute* "vendor/fd-lock/Cargo.toml" + (("\"fs\"") "\"fs\", \"cc\"")) + (substitute* "vendor/is-terminal/Cargo.toml" + (("\"termios\"") "\"termios\", \"cc\"")) + ;; Remove vendored dynamically linked libraries. + ;; find . -not -type d -executable -exec file {} \+ | grep ELF + (delete-file "vendor/vte/vim10m_match") + (delete-file "vendor/vte/vim10m_table") + ;; Also remove the bundled (mostly Windows) libraries. + (for-each delete-file + (find-files "vendor" "\\.(a|dll|exe|lib)$")))))) + (inputs (modify-inputs (package-inputs base-rust) + (replace "llvm" llvm-15)))))) + +(define rust-1.68 + (rust-bootstrapped-package + rust-1.67 "1.68.2" "15ifyd5jj8rd979dkakp887hgmhndr68pqaqvd2hqkfdywirqcwk")) + +(define rust-1.69 + (let ((base-rust + (rust-bootstrapped-package + rust-1.68 "1.69.0" + "03zn7kx5bi5mdfsqfccj4h8gd6abm7spj0kjsfxwlv5dcwc9f1gv"))) + (package + (inherit base-rust) + (source + (origin + (inherit (package-source base-rust)) + (snippet + '(begin + (for-each delete-file-recursively + '("src/llvm-project" + "vendor/openssl-src/openssl" + "vendor/tikv-jemalloc-sys/jemalloc")) + ;; Adjust rustix to always build with cc. + (substitute* '("Cargo.lock" + "src/bootstrap/Cargo.lock") + (("\"errno\",") "\"cc\",\n \"errno\",")) + ;; Add a dependency on the the 'cc' feature of rustix. + (substitute* "vendor/fd-lock/Cargo.toml" + (("\"fs\"") "\"fs\", \"cc\"")) + (substitute* "vendor/is-terminal/Cargo.toml" + (("\"termios\"") "\"termios\", \"cc\"")) + ;; Also remove the bundled (mostly Windows) libraries. + (for-each delete-file + (find-files "vendor" "\\.(a|dll|exe|lib)$"))))))))) + +(define rust-1.70 + (let ((base-rust + (rust-bootstrapped-package + rust-1.69 "1.70.0" + "0z6j7d0ni0rmfznv0w3mrf882m11kyh51g2bxkj40l3s1c0axgxj"))) + (package + (inherit base-rust) + (source + (origin + (inherit (package-source base-rust)) + (snippet + '(begin + (for-each delete-file-recursively + '("src/llvm-project" + "vendor/openssl-src/openssl" + "vendor/tikv-jemalloc-sys/jemalloc")) + ;; Adjust rustix to always build with cc. + (substitute* "Cargo.lock" + (("\"errno\",") "\"cc\",\n \"errno\",")) + ;; Add a dependency on the the 'cc' feature of rustix. + (substitute* '("vendor/is-terminal/Cargo.toml" + "vendor/is-terminal-0.4.4/Cargo.toml") + (("\"termios\"") "\"termios\", \"cc\"")) + ;; Also remove the bundled (mostly Windows) libraries. + (for-each delete-file + (find-files "vendor" "\\.(a|dll|exe|lib)$")))) + ;; Rust 1.70 adds the rustix library which depends on the vendored + ;; fd-lock crate. The fd-lock crate uses Outline assembly which expects + ;; a precompiled static library. Enabling the "cc" feature tells the + ;; build.rs script to compile the assembly files instead of searching + ;; for a precompiled library. + (patches (search-our-patches "rust-1.70-fix-rustix-build.patch"))))))) + +;; END of the newer rustc versions copied over from the current Guix master. + (let ((is_gui (getenv "IS_GUI"))) (concatenate-manifests (list @@ -15,13 +147,10 @@ "eudev" "fontconfig") '()))) - ;; The GUI's MSRV is 1.65 and the daemon's 1.63. + ;; The GUI's MSRV is 1.70 and the daemon's 1.63. We just use the same rustc version for + ;; both. ;; FIXME: be able to compile against a specified glibc (or musl) instead of having to - ;; resort to not bumping the time machine and use at-the-time unpublished rustc. + ;; resort to backporting the newer rustc releases here. Also have proper Guix packages + ;; for the two projects. (packages->manifest - (if - (string=? is_gui "1") - (list - (@@ (gnu packages rust) rust-1.65)) - (list - (@@ (gnu packages rust) rust-1.63))))))) + `(,rust-1.70))))) diff --git a/contrib/reproducible/guix/patches/rust-1.70-fix-rustix-build.patch b/contrib/reproducible/guix/patches/rust-1.70-fix-rustix-build.patch new file mode 100644 index 000000000..d72a8fb07 --- /dev/null +++ b/contrib/reproducible/guix/patches/rust-1.70-fix-rustix-build.patch @@ -0,0 +1,21 @@ +@@ -0,0 +1,20 @@ +--- a/vendor/fd-lock/Cargo.toml 2023-05-31 14:44:48.000000000 -0700 ++++ b/vendor/fd-lock/Cargo.toml 2023-07-14 21:19:34.637702319 -0700 +@@ -45,7 +45,7 @@ + + [target."cfg(unix)".dependencies.rustix] + version = "0.37.0" +-features = ["fs"] ++features = ["fs", "cc"] + + [target."cfg(windows)".dependencies.windows-sys] + version = "0.45.0" +--- a/src/bootstrap/Cargo.lock 2023-07-11 20:32:40.000000000 -0700 ++++ b/src/bootstrap/Cargo.lock 2023-07-14 22:41:53.269284713 -0700 +@@ -618,6 +618,7 @@ + dependencies = [ + "bitflags", ++ "cc", + "errno", + "io-lifetimes", + "libc", diff --git a/doc/BUILD.md b/doc/BUILD.md index fceca1c7b..f36e73df3 100644 --- a/doc/BUILD.md +++ b/doc/BUILD.md @@ -12,7 +12,7 @@ community). See [`CONTRIBUTING.md`](../CONTRIBUTING.md) for the currently minim supported by `lianad`. To build the GUI too, you'll unfortunately need a more recent Rust version. The minimum version -supported by the GUI at the moment is `1.65`. You will most likely have to [manually download +supported by the GUI at the moment is `1.70`. You will most likely have to [manually download it](#by-manually-downloading-the-latest-stable-version) or [use `rustup`](#through-rustup) to install more recent compilers. @@ -50,11 +50,11 @@ And then you can download the archive corresponding to your system and CPU archi signature and use the `cargo` binary from this archive to build Liana. Here is an example for `amd64`: ``` -$ curl -O https://static.rust-lang.org/dist/rust-1.65.0-x86_64-unknown-linux-gnu.tar.gz -$ curl -O https://static.rust-lang.org/dist/rust-1.65.0-x86_64-unknown-linux-gnu.tar.gz.asc -$ gpg --verify rust-1.65.0-x86_64-unknown-linux-gnu.tar.gz.asc -$ tar -xzf rust-1.65.0-x86_64-unknown-linux-gnu.tar.gz -$ ./rust-1.65.0-x86_64-unknown-linux-gnu/cargo/bin/cargo build --release +$ curl -O https://static.rust-lang.org/dist/rust-1.70.0-x86_64-unknown-linux-gnu.tar.gz +$ curl -O https://static.rust-lang.org/dist/rust-1.70.0-x86_64-unknown-linux-gnu.tar.gz.asc +$ gpg --verify rust-1.70.0-x86_64-unknown-linux-gnu.tar.gz.asc +$ tar -xzf rust-1.70.0-x86_64-unknown-linux-gnu.tar.gz +$ ./rust-1.70.0-x86_64-unknown-linux-gnu/cargo/bin/cargo build --release ``` ### Through `rustup`