From 636458ebf93454a61fad3767436144606811d423 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Ci=C4=99=C5=BCarkiewicz?= Date: Wed, 9 Oct 2024 11:17:38 -0700 Subject: [PATCH 1/5] chore: update our fork of esplora-electrs --- nix/pkgs/esplora-electrs.Cargo.lock | 267 ++++++++++++++++++++++++---- nix/pkgs/esplora-electrs.nix | 8 +- 2 files changed, 241 insertions(+), 34 deletions(-) diff --git a/nix/pkgs/esplora-electrs.Cargo.lock b/nix/pkgs/esplora-electrs.Cargo.lock index ba517f5e413..c399a95853a 100644 --- a/nix/pkgs/esplora-electrs.Cargo.lock +++ b/nix/pkgs/esplora-electrs.Cargo.lock @@ -52,6 +52,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "ansi_term" version = "0.12.1" @@ -249,6 +255,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73290177011694f38ec25e165d0387ab7ea749a4b81cd4c80dae5988229f7a57" +[[package]] +name = "bitcoin-test-data" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c188654f9dce3bc6ce1bfa9c49777ad514bcad37e421b5f53e9d0ee10603f34" + [[package]] name = "bitcoin_hashes" version = "0.10.0" @@ -304,22 +316,9 @@ dependencies = [ [[package]] name = "bitcoind" -version = "0.34.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50548a349632abb9a2007d431a302bf0401a786c91f809ff31765fba87e2397" -dependencies = [ - "anyhow", - "bitcoincore-rpc", - "log", - "tempfile", - "which", -] - -[[package]] -name = "bitcoind" -version = "0.35.1" +version = "0.34.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09db6b30b527263d2e3a8768a2c9299763858cb84d01a9d63d174b9f515e1a32" +checksum = "2542fac51d8cd8fce6109f4a3ffd1acfdaa3394c36d4a8207af15b8b0540e2fc" dependencies = [ "anyhow", "bitcoin_hashes 0.13.0", @@ -393,6 +392,12 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.0.85" @@ -437,6 +442,33 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cipher" version = "0.4.4" @@ -468,11 +500,32 @@ dependencies = [ "atty", "bitflags 1.3.2", "strsim", - "textwrap", + "textwrap 0.11.0", "unicode-width", "vec_map", ] +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "bitflags 1.3.2", + "clap_lex", + "indexmap", + "textwrap 0.16.1", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "const_fn" version = "0.4.9" @@ -536,6 +589,42 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" +dependencies = [ + "anes", + "atty", + "cast", + "ciborium", + "clap 3.2.25", + "criterion-plot", + "itertools 0.10.5", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + [[package]] name = "crossbeam-channel" version = "0.5.12" @@ -570,6 +659,12 @@ version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + [[package]] name = "crypto-common" version = "0.1.6" @@ -642,8 +737,10 @@ dependencies = [ "base64 0.22.0", "bincode", "bitcoin 0.31.2", - "bitcoind 0.35.1", - "clap", + "bitcoin-test-data", + "bitcoind", + "clap 2.34.0", + "criterion", "crossbeam-channel", "dirs", "electrum-client", @@ -655,7 +752,7 @@ dependencies = [ "hex-conservative", "hyper", "hyperlocal", - "itertools", + "itertools 0.12.1", "lazy_static", "libc", "log", @@ -673,7 +770,7 @@ dependencies = [ "stderrlog", "sysconf", "tempfile", - "time 0.3.34", + "time 0.3.36", "tiny_http", "tokio", "ureq 2.9.6", @@ -698,7 +795,7 @@ dependencies = [ [[package]] name = "electrumd" version = "0.1.0" -source = "git+https://github.com/shesek/electrumd?rev=996fe2a8e563bc1bde6bbc2e0c2a2f4421abcdbc#996fe2a8e563bc1bde6bbc2e0c2a2f4421abcdbc" +source = "git+https://github.com/shesek/electrumd?rev=b35d9db285d932cb3c2296beab65e571a2506349#b35d9db285d932cb3c2296beab65e571a2506349" dependencies = [ "bitcoin_hashes 0.10.0", "home", @@ -723,12 +820,12 @@ dependencies = [ [[package]] name = "elementsd" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17022ca1a790518f66c46c40ab729cc275f8a009babad583972a946a851cadf" +checksum = "19b08013ae0b60e4a3ac9d126e230338dec55575ddf7811aeaebb096f7c3abbf" dependencies = [ "bitcoin_hashes 0.12.0", - "bitcoind 0.34.1", + "bitcoind", "flate2", "minreq", "tar", @@ -909,6 +1006,22 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -1074,6 +1187,16 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown", +] + [[package]] name = "inout" version = "0.1.3" @@ -1094,6 +1217,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.12.1" @@ -1349,12 +1481,24 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "oorandom" +version = "11.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" + [[package]] name = "option-ext" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + [[package]] name = "page_size" version = "0.6.0" @@ -1461,6 +1605,34 @@ version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" +[[package]] +name = "plotters" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a15b6eccb8484002195a3e44fe65a4ce8e93a625797a063735536fd59cb01cf3" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "414cec62c6634ae900ea1c56128dfe87cf63e7caece0852ec76aba307cebadb7" + +[[package]] +name = "plotters-svg" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81b30686a7d9c3e010b84284bdd26a29f2138574f52f5eb6f794fc0ad924e705" +dependencies = [ + "plotters-backend", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -1839,6 +2011,15 @@ version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2242,6 +2423,12 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "textwrap" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" + [[package]] name = "thiserror" version = "1.0.57" @@ -2300,9 +2487,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.34" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa", @@ -2310,7 +2497,7 @@ dependencies = [ "powerfmt", "serde", "time-core", - "time-macros 0.2.17", + "time-macros 0.2.18", ] [[package]] @@ -2331,9 +2518,9 @@ dependencies = [ [[package]] name = "time-macros" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ "num-conv", "time-core", @@ -2364,6 +2551,16 @@ dependencies = [ "log", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -2544,6 +2741,16 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -2886,7 +3093,7 @@ dependencies = [ "hmac", "pbkdf2", "sha1 0.10.6", - "time 0.3.34", + "time 0.3.36", "zstd", ] diff --git a/nix/pkgs/esplora-electrs.nix b/nix/pkgs/esplora-electrs.nix index 1902a85a9e0..63269a55924 100644 --- a/nix/pkgs/esplora-electrs.nix +++ b/nix/pkgs/esplora-electrs.nix @@ -13,7 +13,7 @@ in rustPlatform.buildRustPackage { pname = "esplora"; # last tagged version is far behind master - version = "20240603"; + version = "20241009"; src = fetchFromGitHub { # original: @@ -25,8 +25,8 @@ rustPlatform.buildRustPackage { # pre-allocation size patch: owner = "dpc"; repo = "esplora-electrs"; - rev = "6cf03773a594de7dbd68a62be79c5c78710de19a"; - hash = "sha256-HGFvOy5sBtfaZ/rS7nkIHHFTDJuRKfggxC4y5gW9dAQ="; + rev = "9339bfaf40b896f9d0b9ee945edc44ef49f99d2b"; + hash = "sha256-4PL+dvdUOTlX0IcpkcSxZ7TchsCI4fI8OmsFmwrtPHo="; }; doCheck = false; @@ -36,7 +36,7 @@ rustPlatform.buildRustPackage { outputHashes = { "electrum-client-0.8.0" = "sha256-HDRdGS7CwWsPXkA1HdurwrVu4lhEx0Ay8vHi08urjZ0="; - "electrumd-0.1.0" = "sha256-M9yd53LsKheS9dQwDQKjcwbBM+66QGoNXIoSgV8G/Ao="; + "electrumd-0.1.0" = "sha256-QsoMD2uVDEITuYmYItfP6BJCq7ApoRztOCs7kdeRL9Y="; "jsonrpc-0.12.0" = "sha256-lSNkkQttb8LnJej4Vfe7MrjiNPOuJ5A6w5iLstl9O1k="; }; }; From 1a10e94fe19368af3a5962c89db099e357ba13e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Ci=C4=99=C5=BCarkiewicz?= Date: Fri, 15 Nov 2024 12:28:23 -0800 Subject: [PATCH 2/5] chore(ci): lower parallelism in CI --- scripts/tests/test-ci-all.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/tests/test-ci-all.sh b/scripts/tests/test-ci-all.sh index cd61af525c1..d23f40e59a7 100755 --- a/scripts/tests/test-ci-all.sh +++ b/scripts/tests/test-ci-all.sh @@ -347,7 +347,9 @@ if [ -n "${FM_TEST_CI_ALL_JOBS:-}" ]; then # when specifically set, use the env var parallel_args+=(--jobs "${FM_TEST_CI_ALL_JOBS}") elif [ -n "${CI:-}" ] || [ "${CARGO_PROFILE:-}" == "ci" ]; then - parallel_args+=(--jobs $(($(nproc) / 2 + 1))) + # our CI runners have 96 CPUs, which is a lot of CPUs, and a lot of jobs + # to run in parallel + parallel_args+=(--jobs $(($(nproc) / 3 + 1))) else # on dev computers default to `num_cpus / 2 + 1` max parallel jobs parallel_args+=(--jobs "${FM_TEST_CI_ALL_JOBS:-$(($(nproc) / 2 + 1))}") From 2b0378e31c0daabda14488f4147cbabbb6291705 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Ci=C4=99=C5=BCarkiewicz?= Date: Fri, 15 Nov 2024 16:03:32 -0800 Subject: [PATCH 3/5] chore(ci): increase the max time of a test Seems like with newer electrs backend takes longer. --- scripts/dev/run-test/fm-run-test | 2 +- scripts/tests/test-ci-all.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/dev/run-test/fm-run-test b/scripts/dev/run-test/fm-run-test index 622d1935001..d96d979c43f 100755 --- a/scripts/dev/run-test/fm-run-test +++ b/scripts/dev/run-test/fm-run-test @@ -37,7 +37,7 @@ on_exit() { trap on_error ERR trap on_exit EXIT -command time -q --format="$time_fmt" -o "$time_out_path" timeout -k 230 245 "$@" 2>&1 | ts -s > "$test_out_path" +command time -q --format="$time_fmt" -o "$time_out_path" timeout -k 310 320 "$@" 2>&1 | ts -s > "$test_out_path" awk 'BEGIN {FS="\t"} {printf "## STAT: %8.2fs %8dB %8dW %8dc\n", $1, $2, $3, $4}' < "$time_out_path" echo "## DONE: $test_name$version_str" diff --git a/scripts/tests/test-ci-all.sh b/scripts/tests/test-ci-all.sh index d23f40e59a7..cfb18ad73c5 100755 --- a/scripts/tests/test-ci-all.sh +++ b/scripts/tests/test-ci-all.sh @@ -355,7 +355,7 @@ else parallel_args+=(--jobs "${FM_TEST_CI_ALL_JOBS:-$(($(nproc) / 2 + 1))}") fi -parallel_args+=(--timeout "${FM_TEST_CI_ALL_TIMEOUT:-300}") +parallel_args+=(--timeout "${FM_TEST_CI_ALL_TIMEOUT:-360}") parallel_args+=(--load "${FM_TEST_CI_ALL_MAX_LOAD:-$(($(nproc) / 3 + 1))}") # --delay to let nix start extracting and bump the load From 1b90f79b6a6e086264a3d38cd8fe87a38721b286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Ci=C4=99=C5=BCarkiewicz?= Date: Fri, 15 Nov 2024 17:19:32 -0800 Subject: [PATCH 4/5] chore(ci): increase restore test deadline --- devimint/src/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devimint/src/tests.rs b/devimint/src/tests.rs index d8fef808c49..2a09cedb57c 100644 --- a/devimint/src/tests.rs +++ b/devimint/src/tests.rs @@ -349,7 +349,7 @@ pub async fn latency_tests( if crate::util::is_backwards_compatibility_test() { assert!(restore_time < Duration::from_secs(160)); } else { - assert!(restore_time < Duration::from_secs(15)); + assert!(restore_time < Duration::from_secs(30)); } } } From 691a20c3747d7898f29ec3a872445956b92f3878 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Ci=C4=99=C5=BCarkiewicz?= Date: Mon, 18 Nov 2024 16:08:42 -0800 Subject: [PATCH 5/5] chore(ci): increase cargo nextest timeout Serialized tests with electrs seems to be taking longer in the CI now. --- .config/nextest.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index c2bcdef5988..b94008fc424 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -9,7 +9,7 @@ slow-timeout = { period = "30s", terminate-after = 4 } [profile.dev] [profile.ci] -slow-timeout = { period = "30s", terminate-after = 2 } +slow-timeout = { period = "30s", terminate-after = 3 } # ccov seems MUCH slower, especially in Nix sandbox # possibly due to writing to a shared tracing file(?)