From 016d144395fb5f3f1d4364182b925fed7e2b8dd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Ci=C4=99=C5=BCarkiewicz?= Date: Thu, 17 Oct 2024 14:29:01 -0700 Subject: [PATCH 1/4] chore: replace our cargo wrappers with `cargo-deluxe` --- flake.lock | 319 ++++++++++++++++++++++++++++++-- flake.nix | 18 +- nix/cargo-wrapper/cargo | 61 ------ nix/flakebox.nix | 73 +++++--- scripts/dev/rustc-wrapper/rustc | 43 ----- scripts/tests/test-ci-all.sh | 2 +- 6 files changed, 368 insertions(+), 148 deletions(-) delete mode 100755 nix/cargo-wrapper/cargo delete mode 100755 scripts/dev/rustc-wrapper/rustc diff --git a/flake.lock b/flake.lock index d57fcea4623..7863248ad61 100644 --- a/flake.lock +++ b/flake.lock @@ -19,7 +19,32 @@ "android-nixpkgs": { "inputs": { "devshell": "devshell", - "flake-utils": "flake-utils_4", + "flake-utils": "flake-utils_3", + "nixpkgs": [ + "cargo-deluxe", + "flakebox", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1727381935, + "narHash": "sha256-G2fOYRZM7bXK5eBb+GK3k/WmO+q5JA/GtFwSPc3kdc8=", + "owner": "tadfisher", + "repo": "android-nixpkgs", + "rev": "522d86121cbd413aff922c54f38106ecf8740107", + "type": "github" + }, + "original": { + "owner": "tadfisher", + "repo": "android-nixpkgs", + "rev": "522d86121cbd413aff922c54f38106ecf8740107", + "type": "github" + } + }, + "android-nixpkgs_2": { + "inputs": { + "devshell": "devshell_2", + "flake-utils": "flake-utils_7", "nixpkgs": [ "flakebox", "nixpkgs" @@ -61,7 +86,53 @@ "type": "github" } }, + "cargo-deluxe": { + "inputs": { + "flake-utils": "flake-utils_2", + "flakebox": "flakebox", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1729202329, + "narHash": "sha256-lltxyfay1Vg1CmcAU/r3kOCQs7/WdK22YKORAgzmXkg=", + "owner": "rustshop", + "repo": "cargo-deluxe", + "rev": "4acc6488d02f032434a5a1341f21f20d328bba40", + "type": "github" + }, + "original": { + "owner": "rustshop", + "repo": "cargo-deluxe", + "rev": "4acc6488d02f032434a5a1341f21f20d328bba40", + "type": "github" + } + }, "crane": { + "inputs": { + "nixpkgs": [ + "cargo-deluxe", + "flakebox", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1717383740, + "narHash": "sha256-559HbY4uhNeoYvK3H6AMZAtVfmR3y8plXZ1x6ON/cWU=", + "owner": "ipetkov", + "repo": "crane", + "rev": "b65673fce97d277934488a451724be94cc62499a", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "rev": "b65673fce97d277934488a451724be94cc62499a", + "type": "github" + } + }, + "crane_2": { "inputs": { "nixpkgs": [ "flakebox", @@ -85,7 +156,31 @@ }, "devshell": { "inputs": { - "flake-utils": "flake-utils_3", + "nixpkgs": [ + "cargo-deluxe", + "flakebox", + "android-nixpkgs", + "nixpkgs" + ], + "systems": "systems_2" + }, + "locked": { + "lastModified": 1695195896, + "narHash": "sha256-pq9q7YsGXnQzJFkR5284TmxrLNFc0wo4NQ/a5E93CQU=", + "owner": "numtide", + "repo": "devshell", + "rev": "05d40d17bf3459606316e3e9ec683b784ff28f16", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "devshell", + "type": "github" + } + }, + "devshell_2": { + "inputs": { + "flake-utils": "flake-utils_6", "nixpkgs": [ "flakebox", "android-nixpkgs", @@ -109,10 +204,33 @@ "fenix": { "inputs": { "nixpkgs": [ + "cargo-deluxe", + "flakebox", "nixpkgs" ], "rust-analyzer-src": "rust-analyzer-src" }, + "locked": { + "lastModified": 1696918968, + "narHash": "sha256-18rAHsM9YsGp7aKKemO4gKIeWfrSyDsdJZ/mk4dQ3JI=", + "owner": "nix-community", + "repo": "fenix", + "rev": "638fc95a2a3d01b372c76f71cbb6d73c63909d6e", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "fenix", + "type": "github" + } + }, + "fenix_2": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "rust-analyzer-src": "rust-analyzer-src_2" + }, "locked": { "lastModified": 1727245890, "narHash": "sha256-B4gUhZxqdn24PqL7z7ZuvLOS84HVskhKRByWdgA4/RI=", @@ -146,6 +264,64 @@ "inputs": { "systems": "systems" }, + "locked": { + "lastModified": 1694529238, + "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_3": { + "inputs": { + "systems": "systems_3" + }, + "locked": { + "lastModified": 1694529238, + "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_4": { + "inputs": { + "systems": [ + "cargo-deluxe", + "flakebox", + "systems" + ] + }, + "locked": { + "lastModified": 1694529238, + "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_5": { + "inputs": { + "systems": "systems_5" + }, "locked": { "lastModified": 1726560853, "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=", @@ -160,9 +336,9 @@ "type": "github" } }, - "flake-utils_3": { + "flake-utils_6": { "inputs": { - "systems": "systems_2" + "systems": "systems_6" }, "locked": { "lastModified": 1701680307, @@ -178,9 +354,9 @@ "type": "github" } }, - "flake-utils_4": { + "flake-utils_7": { "inputs": { - "systems": "systems_3" + "systems": "systems_7" }, "locked": { "lastModified": 1710146030, @@ -196,7 +372,7 @@ "type": "github" } }, - "flake-utils_5": { + "flake-utils_8": { "inputs": { "systems": [ "flakebox", @@ -221,14 +397,37 @@ "inputs": { "android-nixpkgs": "android-nixpkgs", "crane": "crane", + "fenix": "fenix", + "flake-utils": "flake-utils_4", + "nixpkgs": "nixpkgs_4", + "systems": "systems_4" + }, + "locked": { + "lastModified": 1728662483, + "narHash": "sha256-phVyjjgVmHFUF6+DGFRA+1fCV0zs8ipwRn/M33LtIYY=", + "owner": "rustshop", + "repo": "flakebox", + "rev": "1a65534520d8a592fd7d75f7db18cfed40fa5c09", + "type": "github" + }, + "original": { + "owner": "rustshop", + "repo": "flakebox", + "type": "github" + } + }, + "flakebox_2": { + "inputs": { + "android-nixpkgs": "android-nixpkgs_2", + "crane": "crane_2", "fenix": [ "fenix" ], - "flake-utils": "flake-utils_5", + "flake-utils": "flake-utils_8", "nixpkgs": [ "nixpkgs" ], - "systems": "systems_4" + "systems": "systems_8" }, "locked": { "lastModified": 1727478500, @@ -327,6 +526,22 @@ } }, "nixpkgs_4": { + "locked": { + "lastModified": 1728492678, + "narHash": "sha256-9UTxR8eukdg+XZeHgxW5hQA9fIKHsKCdOIUycTryeVw=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "5633bcff0c6162b9e4b5f1264264611e950c8ec7", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_5": { "locked": { "lastModified": 1727129439, "narHash": "sha256-nPyrcFm6FSk7CxzVW4x2hu62aLDghNcv9dX6DF3dXw8=", @@ -346,13 +561,31 @@ "inputs": { "advisory-db": "advisory-db", "bundlers": "bundlers", - "fenix": "fenix", - "flake-utils": "flake-utils_2", - "flakebox": "flakebox", - "nixpkgs": "nixpkgs_4" + "cargo-deluxe": "cargo-deluxe", + "fenix": "fenix_2", + "flake-utils": "flake-utils_5", + "flakebox": "flakebox_2", + "nixpkgs": "nixpkgs_5" } }, "rust-analyzer-src": { + "flake": false, + "locked": { + "lastModified": 1696840854, + "narHash": "sha256-wphOvjDSDsUN5DMe3MOhdargANIab7YE3hkh3Qv7qso=", + "owner": "rust-lang", + "repo": "rust-analyzer", + "rev": "aaa1e8e1b82d742b876d164a30dda02f318ce809", + "type": "github" + }, + "original": { + "owner": "rust-lang", + "ref": "nightly", + "repo": "rust-analyzer", + "type": "github" + } + }, + "rust-analyzer-src_2": { "flake": false, "locked": { "lastModified": 1727104575, @@ -428,6 +661,66 @@ "repo": "default", "type": "github" } + }, + "systems_5": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_6": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_7": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_8": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index cc3f33a4ec0..212b257ee67 100644 --- a/flake.nix +++ b/flake.nix @@ -13,6 +13,10 @@ inputs.nixpkgs.follows = "nixpkgs"; inputs.fenix.follows = "fenix"; }; + cargo-deluxe = { + url = "github:rustshop/cargo-deluxe?rev=4acc6488d02f032434a5a1341f21f20d328bba40"; + inputs.nixpkgs.follows = "nixpkgs"; + }; bundlers = { # TODO: switch back to upstream after https://github.com/matthewbauer/nix-bundle/pull/103 is available url = "github:dpc/bundlers?branch=24-02-21-tar-deterministic&rev=e8aafe89a11ae0a5f3ce97d1d7d0fcfb354c79eb"; @@ -29,6 +33,7 @@ nixpkgs, flake-utils, flakebox, + cargo-deluxe, advisory-db, bundlers, ... @@ -69,7 +74,13 @@ let pkgs = import nixpkgs { inherit system; - overlays = [ overlayAll ]; + overlays = [ + overlayAll + + (final: prev: { + cargo-deluxe = cargo-deluxe.packages.${system}.default; + }) + ]; }; lib = pkgs.lib; @@ -386,15 +397,12 @@ # was moved into another shell. cross = flakeboxLib.mkDevShell ( commonShellArgs - // craneMultiBuild.commonEnvsShellRocksdbLinkCross + // craneMultiBuild.commonEnvsCrossShell // { toolchain = toolchainAll; shellHook = '' - # hijack cargo for our evil purposes - export CARGO_ORIG_BIN="$(${pkgs.which}/bin/which cargo)" export REPO_ROOT="$(git rev-parse --show-toplevel)" export PATH="$REPO_ROOT/bin:$PATH" - export PATH="''${REPO_ROOT}/nix/cargo-wrapper/:$PATH" ''; } ); diff --git a/nix/cargo-wrapper/cargo b/nix/cargo-wrapper/cargo deleted file mode 100755 index 7995b16ff8b..00000000000 --- a/nix/cargo-wrapper/cargo +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash -# -# Our wrapper over cargo to workaround some limitations: -# -# https://github.com/rust-rocksdb/rust-rocksdb/issues/823 - - -if [ -z "$CARGO_ORIG_BIN" ]; then - >&2 echo "CARGO_ORIG_BIN not set" - exit 1 -fi - -args=() - -while [[ $# -gt 0 ]]; do - case "$1" in - --target) - target="${2//-/_}" - args+=("$1") - args+=("$2") - shift - ;; - *) - args+=("$1") - ;; - esac - - # Move to the next argument - shift -done - -# Check if the target value is empty (not provided) -if [ -n "$target" ]; then - rocksdb_lib_dir="ROCKSDB_${target}_LIB_DIR" - snappy_lib_dir="SNAPPY_${target}_LIB_DIR" - rocksdb_static="ROCKSDB_${target}_STATIC" - snappy_static="SNAPPY_${target}_STATIC" - - if [ -n "${!rocksdb_lib_dir}" ]; then - export ROCKSDB_LIB_DIR="${!rocksdb_lib_dir}" - else - unset ROCKSDB_LIB_DIR - fi - if [ -n "${!snappy_lib_dir}" ]; then - export SNAPPY_LIB_DIR="${!snappy_lib_dir}" - else - unset SNAPPY_LIB_DIR - fi - if [ -n "${!rocksdb_static}" ]; then - export ROCKSDB_STATIC="${!rocksdb_static}" - else - unset ROCKSDB_STATIC - fi - if [ -n "${!snappy_static}" ]; then - export SNAPPY_STATIC="${!snappy_static}" - else - unset SNAPPY_STATIC - fi -fi - -exec "$CARGO_ORIG_BIN" "${args[@]}" diff --git a/nix/flakebox.nix b/nix/flakebox.nix index 47265e222d7..a3b608b0c07 100644 --- a/nix/flakebox.nix +++ b/nix/flakebox.nix @@ -103,30 +103,49 @@ let src: filterSrcWithRegexes (filterWorkspaceDepsBuildFilesRegex ++ [ "deny.toml" ]) src; # env vars for linking rocksdb - commonEnvsShellRocksdbLink = + commonEnvsCross = let - target_underscores = lib.strings.replaceStrings [ "-" ] [ "_" ] pkgs.stdenv.buildPlatform.config; + build_arch_underscores = lib.strings.replaceStrings [ "-" ] [ "_" ] pkgs.stdenv.buildPlatform.config; in { - ROCKSDB_STATIC = "true"; - ROCKSDB_LIB_DIR = "${pkgs.rocksdb}/lib/"; + # for cargo-deluxe + CARGO_TARGET_SPECIFIC_ENVS = builtins.concatStringsSep "," [ + "ROCKSDB_target_STATIC" + "ROCKSDB_target_LIB_DIR" + "SNAPPY_target_STATIC" + "SNAPPY_target_LIB_DIR" + "SNAPPY_target_COMPILE" + "SQLITE3_target_STATIC" + "SQLITE3_target_LIB_DIR" + "SQLCIPHER_target_STATIC" + "SQLCIPHER_target_LIB_DIR" + ]; + } // pkgs.lib.optionalAttrs (!pkgs.stdenv.isDarwin) { + "ROCKSDB_${build_arch_underscores}_STATIC" = "true"; + "ROCKSDB_${build_arch_underscores}_LIB_DIR" = "${pkgs.rocksdb}/lib/"; - "ROCKSDB_${target_underscores}_STATIC" = "true"; - "ROCKSDB_${target_underscores}_LIB_DIR" = "${pkgs.rocksdb}/lib/"; - } - // pkgs.lib.optionalAttrs (!(pkgs.stdenv.isDarwin && pkgs.stdenv.isx86_64)) { - # FIX: error: don't yet have a `targetPackages.darwin.LibsystemCross for x86_64-apple-darwin` - SNAPPY_LIB_DIR = "${pkgs.pkgsStatic.snappy}/lib/"; - "SNAPPY_${target_underscores}_LIB_DIR" = "${pkgs.pkgsStatic.snappy}/lib/"; - } - // pkgs.lib.optionalAttrs (!pkgs.stdenv.isDarwin) { - # macos can't static libraries - SNAPPY_STATIC = "true"; - "SNAPPY_${target_underscores}_STATIC" = "true"; + # does not produce static lib in most versions + "SNAPPY_${build_arch_underscores}_STATIC" = "true"; + "SNAPPY_${build_arch_underscores}_LIB_DIR" = "${pkgs.pkgsStatic.snappy}/lib/"; + # "SNAPPY_${build_arch_underscores}_COMPILE" = "true"; + + + "SQLITE3_${build_arch_underscores}_STATIC" = "true"; + "SQLITE3_${build_arch_underscores}_LIB_DIR" = "${pkgs.pkgsStatic.sqlite.out}/lib/"; + + "SQLCIPHER_${build_arch_underscores}_LIB_DIR" = "${pkgs.pkgsStatic.sqlcipher}/lib/"; + "SQLCIPHER_${build_arch_underscores}_STATIC" = "true"; + } // pkgs.lib.optionalAttrs pkgs.stdenv.isDarwin { + # tons of problems, just compile + # "SNAPPY_${build_arch_underscores}_LIB_DIR" = "${pkgs.snappy}/lib/"; + "SNAPPY_${build_arch_underscores}_COMPILE" = "true"; + + "SQLITE3_${build_arch_underscores}_LIB_DIR" = "${pkgs.sqlite.out}/lib/"; + "SQLCIPHER_${build_arch_underscores}_LIB_DIR" = "${pkgs.sqlcipher}/lib/"; }; - commonEnvsShellRocksdbLinkCross = - commonEnvsShellRocksdbLink + commonEnvsCrossShell = + commonEnvsCross // pkgs.lib.optionalAttrs (!pkgs.stdenv.isDarwin) { # TODO: could we used the android-nixpkgs toolchain instead of another one? # ROCKSDB_aarch64_linux_android_STATIC = "true"; @@ -161,7 +180,7 @@ let }; # env variables we want to set in all nix derivations & nix develop shell - commonEnvsShell = commonEnvsShellRocksdbLink // { + commonEnvsShell = commonEnvsCross // { PROTOC = "${pkgs.protobuf}/bin/protoc"; PROTOC_INCLUDE = "${pkgs.protobuf}/include"; CLIPPY_ARGS = "--deny warnings --allow deprecated"; @@ -176,6 +195,12 @@ let commonArgs = { pname = "fedimint"; + packages = [ + # flakebox adds toolchains via `packages`, which seems to always take precedence + # `nativeBuildInputs` in `mkShell`, so we need to add it here as well. + (lib.hiPrio pkgs.cargo-deluxe) + ]; + buildInputs = with pkgs; [ @@ -286,7 +311,7 @@ in "stable" "nightly" ]) - ) commonEnvsShellRocksdbLinkCross + ) commonEnvsCrossShell ); craneLibTests = craneLib.overrideArgs ( @@ -357,9 +382,7 @@ in in rec { inherit commonArgs; - inherit commonEnvsShell; - inherit commonEnvsShellRocksdbLink; - inherit commonEnvsShellRocksdbLinkCross; + inherit commonEnvsShell commonEnvsCrossShell; inherit gitHashPlaceholderValue; commonArgsBase = commonArgs; @@ -393,7 +416,7 @@ in cargoExtraArgs = "--workspace --all-targets --locked"; FM_DISCOVER_API_VERSION_TIMEOUT = "10"; - FM_CARGO_DENY_COMPILATION = "1"; + CARGO_DENY_COMPILATION = "1"; }; workspaceTestDoc = craneLib.cargoTest { @@ -583,7 +606,7 @@ in unset CARGO_BUILD_TARGET patchShebangs ./scripts - export FM_CARGO_DENY_COMPILATION=1 + export CARGO_DENY_COMPILATION=1 export FM_TEST_CI_ALL_TIMES=${builtins.toString times} export FM_TEST_CI_ALL_DISABLE_ETA=1 ./scripts/tests/test-ci-all.sh || exit 1 diff --git a/scripts/dev/rustc-wrapper/rustc b/scripts/dev/rustc-wrapper/rustc deleted file mode 100755 index 96f7efe9e8e..00000000000 --- a/scripts/dev/rustc-wrapper/rustc +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# -# Our own wrapper around 'rustc' -# -# In certain places spurious Rust rebuilds are big issues, -# causing out of disk space failures due to excessive rebuilds, -# that are usually silently failing and generally hard to debug. -# Cargo has no good way to detect/prevent it. -# -# That's why we wrap 'rustc' in this wrappper, so we get -# an additional way to detect and control what is going on. -# -# Note: when calling into original binary, this script must not -# have printed anything yet, even on stderr. - -set -euo pipefail - -orig_bin="${1:-}" -shift 1 - -if [ -z "$orig_bin" ]; then - >&2 echo "rustc-wrapper: Original binary missing" - exit 1 -fi - -if [ -n "${FM_CARGO_DENY_COMPILATION:-}" ]; then - # 'cargo' makes some invocations to rustc to get some - # configs, etc. - # We need to let them through, they are not causing - # rebuilds. - if [ "$1" = '-vV' ]; then - exec "$orig_bin" "$@" - fi - - if [ "$3" = "___" ]; then - exec "$orig_bin" "$@" - fi - - >&2 echo "rustc (wrapper) attempting recompilation while FM_CARGO_DENY_COMPILATION set: $orig_bin $*" - exit 1 -fi - -exec "$orig_bin" "$@" diff --git a/scripts/tests/test-ci-all.sh b/scripts/tests/test-ci-all.sh index 41a8346a31e..609098a697c 100755 --- a/scripts/tests/test-ci-all.sh +++ b/scripts/tests/test-ci-all.sh @@ -33,7 +33,7 @@ runLowPrio cargo nextest run --no-run ${CARGO_PROFILE:+--cargo-profile ${CARGO_P # let us enforce it, we need to go behind its back. We put a fake 'rustc' # in the PATH. # If you really need to break this rule, ping dpc -export FM_CARGO_DENY_COMPILATION=1 +export CARGO_DENY_COMPILATION=1 function rust_unit_tests() { # unit tests don't use binaries from old versions, so there's no need to run for backwards-compatibility tests From 25be817b6f2846b2b25f3b124f1b06dd52e59c88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Ci=C4=99=C5=BCarkiewicz?= Date: Thu, 17 Oct 2024 14:51:58 -0700 Subject: [PATCH 2/4] fix: wrong cargo metadata field used for crane --- Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 34ea5ebf32e..53b56e929c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,9 +67,6 @@ resolver = "2" [workspace.package] version = "0.5.0-alpha" -[workspace.package.crane] -name = "fedimint" - [workspace.metadata] name = "fedimint" authors = ["The Fedimint Developers"] @@ -82,6 +79,9 @@ repository = "https://github.com/fedimint/fedimint" license-file = "LICENSE" keywords = ["bitcoin", "lightning", "chaumian", "e-cash", "federated"] +[workspace.metadata.crane] +name = "fedimint" + [workspace.dependencies] anyhow = "1.0.89" aquamarine = "0.5.0" From 76b502d143c89d8998020dae998217188d65dbd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Ci=C4=99=C5=BCarkiewicz?= Date: Thu, 17 Oct 2024 14:53:03 -0700 Subject: [PATCH 3/4] fix(tests): accidental needless project rebuilds --- modules/fedimint-wallet-tests/Cargo.toml | 5 ---- modules/fedimint-wallet-tests/tests/tests.rs | 29 +++++++------------- scripts/tests/backend-test.sh | 4 --- 3 files changed, 10 insertions(+), 28 deletions(-) diff --git a/modules/fedimint-wallet-tests/Cargo.toml b/modules/fedimint-wallet-tests/Cargo.toml index 6e616e71b2d..f760bafecc4 100644 --- a/modules/fedimint-wallet-tests/Cargo.toml +++ b/modules/fedimint-wallet-tests/Cargo.toml @@ -7,11 +7,6 @@ description = "fedimint-wallet-tests contains integration tests for the lightnin license = "MIT" publish = false -[features] -bitcoind = [] -electrum = [] -esplora = [] - [[bin]] name = "circular-deposit-test" path = "src/bin/circular-deposit-test.rs" diff --git a/modules/fedimint-wallet-tests/tests/tests.rs b/modules/fedimint-wallet-tests/tests/tests.rs index bba7bcc2599..d222b9e3c90 100644 --- a/modules/fedimint-wallet-tests/tests/tests.rs +++ b/modules/fedimint-wallet-tests/tests/tests.rs @@ -1,4 +1,5 @@ use std::collections::HashSet; +use std::env; use std::time::Duration; use anyhow::{bail, Context}; @@ -18,6 +19,7 @@ use fedimint_dummy_client::DummyClientInit; use fedimint_dummy_common::config::DummyGenParams; use fedimint_dummy_server::DummyInit; use fedimint_testing::btc::BitcoinTest; +use fedimint_testing::envs::{FM_TEST_BACKEND_BITCOIN_RPC_KIND_ENV, FM_TEST_USE_REAL_DAEMONS_ENV}; use fedimint_testing::fixtures::Fixtures; use fedimint_wallet_client::api::WalletFederationApi; use fedimint_wallet_client::{DepositStateV2, WalletClientInit, WalletClientModule, WithdrawState}; @@ -1380,29 +1382,18 @@ mod fedimint_migration_tests { // Verify the correct Bitcoin RPC is used -#[cfg(feature = "bitcoind")] #[test] fn verify_bitcoind_backend() { let fixtures = fixtures(); let dyn_bitcoin_rpc = fixtures.dyn_bitcoin_rpc(); let bitcoin_rpc_kind = dyn_bitcoin_rpc.get_bitcoin_rpc_config().kind; - assert_eq!(bitcoin_rpc_kind, "bitcoind") -} - -#[cfg(feature = "esplora")] -#[test] -fn verify_esplora_backend() { - let fixtures = fixtures(); - let dyn_bitcoin_rpc = fixtures.dyn_bitcoin_rpc(); - let bitcoin_rpc_kind = dyn_bitcoin_rpc.get_bitcoin_rpc_config().kind; - assert_eq!(bitcoin_rpc_kind, "esplora") -} -#[cfg(feature = "electrum")] -#[test] -fn verify_electrum_backend() { - let fixtures = fixtures(); - let dyn_bitcoin_rpc = fixtures.dyn_bitcoin_rpc(); - let bitcoin_rpc_kind = dyn_bitcoin_rpc.get_bitcoin_rpc_config().kind; - assert_eq!(bitcoin_rpc_kind, "electrum") + assert_eq!( + bitcoin_rpc_kind, + if env::var(FM_TEST_USE_REAL_DAEMONS_ENV) == Ok("1".to_string()) { + env::var(FM_TEST_BACKEND_BITCOIN_RPC_KIND_ENV).unwrap_or_else(|_| "bitcoind".into()) + } else { + "mock_kind".into() + } + ) } diff --git a/scripts/tests/backend-test.sh b/scripts/tests/backend-test.sh index 8ce31d5619e..e4f458e38ff 100755 --- a/scripts/tests/backend-test.sh +++ b/scripts/tests/backend-test.sh @@ -9,7 +9,6 @@ build_workspace add_target_dir_to_path make_fm_test_marker - function run_tests() { set -euo pipefail @@ -41,7 +40,6 @@ function run_tests() { cargo nextest run --locked --workspace --all-targets \ ${CARGO_PROFILE:+--cargo-profile ${CARGO_PROFILE}} ${CARGO_PROFILE:+--profile ${CARGO_PROFILE}} \ ${TEST_ARGS_THREADED} \ - --features bitcoind \ -E 'package(fedimint-wallet-tests)' fi if [ -z "${FM_BITCOIND_TEST_ONLY:-}" ] || [ "${FM_BITCOIND_TEST_ONLY:-}" = "ln" ]; then @@ -91,7 +89,6 @@ function run_tests() { cargo nextest run --locked --workspace --all-targets \ ${CARGO_PROFILE:+--cargo-profile ${CARGO_PROFILE}} ${CARGO_PROFILE:+--profile ${CARGO_PROFILE}} \ ${TEST_ARGS_SERIALIZED} \ - --features electrum \ -E 'package(fedimint-wallet-tests)' >&2 echo "### Testing against electrs - complete" fi @@ -105,7 +102,6 @@ function run_tests() { cargo nextest run --locked --workspace --all-targets \ ${CARGO_PROFILE:+--cargo-profile ${CARGO_PROFILE}} ${CARGO_PROFILE:+--profile ${CARGO_PROFILE}} \ ${TEST_ARGS_SERIALIZED} \ - --features esplora \ -E 'package(fedimint-wallet-tests)' >&2 echo "### Testing against esplora - complete" fi From 3cda68d38377a2c41973a4374eb5032d3a1e9d05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Ci=C4=99=C5=BCarkiewicz?= Date: Thu, 17 Oct 2024 15:30:17 -0700 Subject: [PATCH 4/4] chore: switch to (now fixed) upstream nixos bundlers --- flake.lock | 111 ++++++++++++++++++++++++++++++++--------------------- flake.nix | 8 +--- 2 files changed, 69 insertions(+), 50 deletions(-) diff --git a/flake.lock b/flake.lock index 7863248ad61..af3f09aa44e 100644 --- a/flake.lock +++ b/flake.lock @@ -69,20 +69,22 @@ "inputs": { "nix-bundle": "nix-bundle", "nix-utils": "nix-utils", - "nixpkgs": "nixpkgs_3" + "nixpkgs": [ + "nixpkgs" + ] }, "locked": { - "lastModified": 1708548996, - "narHash": "sha256-U7DF6oLGaGm4SmfUnaK8X2Y36fprVd7EFr8sqV6Ocp8=", - "owner": "dpc", + "lastModified": 1729104746, + "narHash": "sha256-7K+dlB8JPT/HD8wcnbb+BV3kFllnBQ6jpXtJX6wmDPk=", + "owner": "NixOS", "repo": "bundlers", - "rev": "e8aafe89a11ae0a5f3ce97d1d7d0fcfb354c79eb", + "rev": "ea1e72ad1dbb0864fd55b3ba52ed166cd190afa2", "type": "github" }, "original": { - "owner": "dpc", + "owner": "NixOS", "repo": "bundlers", - "rev": "e8aafe89a11ae0a5f3ce97d1d7d0fcfb354c79eb", + "rev": "ea1e72ad1dbb0864fd55b3ba52ed166cd190afa2", "type": "github" } }, @@ -162,7 +164,7 @@ "android-nixpkgs", "nixpkgs" ], - "systems": "systems_2" + "systems": "systems_3" }, "locked": { "lastModified": 1695195896, @@ -262,7 +264,7 @@ }, "flake-utils_2": { "inputs": { - "systems": "systems" + "systems": "systems_2" }, "locked": { "lastModified": 1694529238, @@ -280,7 +282,7 @@ }, "flake-utils_3": { "inputs": { - "systems": "systems_3" + "systems": "systems_4" }, "locked": { "lastModified": 1694529238, @@ -320,7 +322,7 @@ }, "flake-utils_5": { "inputs": { - "systems": "systems_5" + "systems": "systems_6" }, "locked": { "lastModified": 1726560853, @@ -338,7 +340,7 @@ }, "flake-utils_6": { "inputs": { - "systems": "systems_6" + "systems": "systems_7" }, "locked": { "lastModified": 1701680307, @@ -356,7 +358,7 @@ }, "flake-utils_7": { "inputs": { - "systems": "systems_7" + "systems": "systems_8" }, "locked": { "lastModified": 1710146030, @@ -399,8 +401,8 @@ "crane": "crane", "fenix": "fenix", "flake-utils": "flake-utils_4", - "nixpkgs": "nixpkgs_4", - "systems": "systems_4" + "nixpkgs": "nixpkgs_3", + "systems": "systems_5" }, "locked": { "lastModified": 1728662483, @@ -427,7 +429,7 @@ "nixpkgs": [ "nixpkgs" ], - "systems": "systems_8" + "systems": "systems_9" }, "locked": { "lastModified": 1727478500, @@ -446,20 +448,20 @@ }, "nix-bundle": { "inputs": { - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "utils": "utils" }, "locked": { - "lastModified": 1708548961, - "narHash": "sha256-WVA2EMhl/jk5GS84y20ExnIW03UyH43wRuoiySxHGZE=", - "owner": "dpc", + "lastModified": 1729095141, + "narHash": "sha256-2AXV3I8D/I3UkQY797j/fSKrIwbUpTVZ82s2O/ErtVM=", + "owner": "matthewbauer", "repo": "nix-bundle", - "rev": "8ab9acfe0a31805de7899eddb8f3312d4e34a13d", + "rev": "4f6330b20767744a4c28788e3cdb05e02d096cd8", "type": "github" }, "original": { - "owner": "dpc", + "owner": "matthewbauer", "repo": "nix-bundle", - "rev": "8ab9acfe0a31805de7899eddb8f3312d4e34a13d", "type": "github" } }, @@ -484,17 +486,18 @@ }, "nixpkgs": { "locked": { - "lastModified": 1620055814, - "narHash": "sha256-8LEHoYSJiL901bTMVatq+rf8y7QtWuZhwwpKE2fyaRY=", + "lastModified": 1728888510, + "narHash": "sha256-nsNdSldaAyu6PE3YUA+YQLqUDJh+gRbBooMMekZJwvI=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "1db42b7fe3878f3f5f7a4f2dc210772fd080e205", + "rev": "a3c0b3b21515f74fd2665903d4ce6bc4dc81c77c", "type": "github" }, "original": { - "id": "nixpkgs", - "ref": "nixos-20.03-small", - "type": "indirect" + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" } }, "nixpkgs_2": { @@ -513,19 +516,6 @@ } }, "nixpkgs_3": { - "locked": { - "lastModified": 1634782485, - "narHash": "sha256-psfh4OQSokGXG0lpq3zKFbhOo3QfoeudRcaUnwMRkQo=", - "path": "/nix/store/p53cz6rh27q40g9i0q98k3vfrz6lm12w-source", - "rev": "34ad3ffe08adfca17fcb4e4a47bb5f3b113687be", - "type": "path" - }, - "original": { - "id": "nixpkgs", - "type": "indirect" - } - }, - "nixpkgs_4": { "locked": { "lastModified": 1728492678, "narHash": "sha256-9UTxR8eukdg+XZeHgxW5hQA9fIKHsKCdOIUycTryeVw=", @@ -541,7 +531,7 @@ "type": "github" } }, - "nixpkgs_5": { + "nixpkgs_4": { "locked": { "lastModified": 1727129439, "narHash": "sha256-nPyrcFm6FSk7CxzVW4x2hu62aLDghNcv9dX6DF3dXw8=", @@ -565,7 +555,7 @@ "fenix": "fenix_2", "flake-utils": "flake-utils_5", "flakebox": "flakebox_2", - "nixpkgs": "nixpkgs_5" + "nixpkgs": "nixpkgs_4" } }, "rust-analyzer-src": { @@ -721,6 +711,39 @@ "repo": "default", "type": "github" } + }, + "systems_9": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1726560853, + "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 212b257ee67..4980b1ab72f 100644 --- a/flake.nix +++ b/flake.nix @@ -18,8 +18,8 @@ inputs.nixpkgs.follows = "nixpkgs"; }; bundlers = { - # TODO: switch back to upstream after https://github.com/matthewbauer/nix-bundle/pull/103 is available - url = "github:dpc/bundlers?branch=24-02-21-tar-deterministic&rev=e8aafe89a11ae0a5f3ce97d1d7d0fcfb354c79eb"; + url = "github:NixOS/bundlers?rev=ea1e72ad1dbb0864fd55b3ba52ed166cd190afa2"; + inputs.nixpkgs.follows = "nixpkgs"; }; advisory-db = { url = "github:rustsec/advisory-db"; @@ -53,9 +53,6 @@ in { overlays = { - # technically overlay outputs are supposed to be just a function, - # instead of a list, but keeping this one just to phase it out smoothly - fedimint = [ overlayAll ]; all = overlayAll; wasm-bindgen = import ./nix/overlays/wasm-bindgen.nix; darwin-compile-fixes = import ./nix/overlays/darwin-compile-fixes.nix; @@ -63,7 +60,6 @@ }; bundlers = bundlers.bundlers; - defaultBundler = bundlers.defaultBundler; nixosModules = { fedimintd = import ./nix/modules/fedimintd.nix;