From c258270d6a874d407e2a465b916f97dbcb2c759a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Mon, 28 Aug 2023 00:55:28 +0000 Subject: [PATCH 01/17] update dependencies (#4639) ## Issue Addressed updates underlying dependencies and removes the ignored `RUSTSEC`'s for `cargo audit`. Also switches `procinfo` to `procfs` on `eth2` to remove the `nom` warning, `procinfo` is unmaintained see [here](https://github.com/danburkert/procinfo-rs/issues/46). --- Cargo.lock | 1180 ++++++++++++++---------- Cargo.toml | 3 +- Makefile | 5 +- beacon_node/execution_layer/Cargo.toml | 2 +- common/eth2/Cargo.toml | 10 +- common/eth2/src/lighthouse.rs | 13 +- common/eth2_network_config/Cargo.toml | 4 +- common/logging/Cargo.toml | 4 +- common/warp_utils/src/cors.rs | 12 +- common/warp_utils/src/metrics.rs | 2 +- lighthouse/tests/beacon_node.rs | 32 +- lighthouse/tests/tls/cert.pem | 24 + lighthouse/tests/tls/key.rsa | 27 + lighthouse/tests/validator_client.rs | 15 + testing/web3signer_tests/Cargo.toml | 4 +- watch/Cargo.toml | 3 +- watch/tests/tests.rs | 38 +- 17 files changed, 871 insertions(+), 507 deletions(-) create mode 100644 lighthouse/tests/tls/cert.pem create mode 100644 lighthouse/tests/tls/key.rsa diff --git a/Cargo.lock b/Cargo.lock index b1779045e30..56d421e0390 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,12 +97,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" dependencies = [ "cfg-if", - "cipher", + "cipher 0.3.0", "cpufeatures", "ctr", "opaque-debug", ] +[[package]] +name = "aes" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" +dependencies = [ + "cfg-if", + "cipher 0.4.4", + "cpufeatures", +] + [[package]] name = "aes-gcm" version = "0.9.4" @@ -110,8 +121,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df5f85a83a7d8b0442b6aa7b504b8212c1733da07b98aae43d4bc21b2cb3cdf6" dependencies = [ "aead", - "aes", - "cipher", + "aes 0.7.5", + "cipher 0.3.0", "ctr", "ghash", "subtle", @@ -141,9 +152,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" +checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" dependencies = [ "memchr", ] @@ -194,9 +205,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.72" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" [[package]] name = "arbitrary" @@ -234,6 +245,45 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +[[package]] +name = "asn1-rs" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6fd5ddaf0351dff5b8da21b2fb4ff8e08ddd02857f0bf69c47639106c0fff0" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "asn1_der" version = "0.7.6" @@ -262,9 +312,9 @@ dependencies = [ [[package]] name = "async-lock" -version = "2.7.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa24f727524730b077666307f2734b4a1a1c57acb79193127dcc8914d5242dd7" +checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" dependencies = [ "event-listener", ] @@ -277,7 +327,7 @@ checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" dependencies = [ "async-stream-impl", "futures-core", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.12", ] [[package]] @@ -288,18 +338,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] name = "async-trait" -version = "0.1.72" +version = "0.1.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09" +checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] @@ -310,7 +360,7 @@ checksum = "b6d7b9decdf35d8908a7e3ef02f64c5e9b1695e230154c0e8de3969142d9b94c" dependencies = [ "futures", "pharos", - "rustc_version 0.4.0", + "rustc_version", ] [[package]] @@ -323,7 +373,7 @@ dependencies = [ "futures-sink", "futures-util", "memchr", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.12", ] [[package]] @@ -378,9 +428,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.6.19" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a1de45611fdb535bfde7b7de4fd54f4fd2b17b1737c0a59b69bf9b92074b8c" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" dependencies = [ "async-trait", "axum-core", @@ -395,7 +445,7 @@ dependencies = [ "memchr", "mime", "percent-encoding", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.12", "rustversion", "serde", "serde_json", @@ -649,9 +699,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" [[package]] name = "bitvec" @@ -744,11 +794,10 @@ dependencies = [ [[package]] name = "bollard-stubs" -version = "1.41.0" +version = "1.42.0-rc.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2f2e73fffe9455141e170fb9c1feb0ac521ec7e7dcd47a7cab72a658490fb8" +checksum = "ed59b5c00048f48d7af971b71f800fdf23e858844a6f9e4d32ca72e9399e7864" dependencies = [ - "chrono", "serde", "serde_with", ] @@ -794,16 +843,6 @@ dependencies = [ "tinyvec", ] -[[package]] -name = "buf_redux" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b953a6887648bb07a535631f2bc00fbdb2a2216f135552cb3f534ed136b9c07f" -dependencies = [ - "memchr", - "safemem", -] - [[package]] name = "builder_client" version = "0.1.0" @@ -905,7 +944,7 @@ checksum = "eee4243f1f26fc7a42710e7439c149e2b10b05472f88090acce52632f231a73a" dependencies = [ "camino", "cargo-platform", - "semver 1.0.18", + "semver", "serde", "serde_json", "thiserror", @@ -919,9 +958,13 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.79" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01" +dependencies = [ + "jobserver", + "libc", +] [[package]] name = "cexpr" @@ -929,7 +972,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom 7.1.3", + "nom", ] [[package]] @@ -945,7 +988,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c80e5460aa66fe3b91d40bcbdab953a597b60053e34d684ac6903f863b680a6" dependencies = [ "cfg-if", - "cipher", + "cipher 0.3.0", "cpufeatures", "zeroize", ] @@ -958,7 +1001,7 @@ checksum = "a18446b09be63d457bbec447509e85f662f32952b035ce892290396bc0b0cff5" dependencies = [ "aead", "chacha20", - "cipher", + "cipher 0.3.0", "poly1305", "zeroize", ] @@ -971,11 +1014,7 @@ checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" dependencies = [ "android-tzdata", "iana-time-zone", - "js-sys", "num-traits", - "serde", - "time 0.1.45", - "wasm-bindgen", "winapi", ] @@ -988,6 +1027,16 @@ dependencies = [ "generic-array", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clang-sys" version = "1.6.1" @@ -1066,7 +1115,7 @@ dependencies = [ "state_processing", "store", "task_executor", - "time 0.3.24", + "time", "timer", "tokio", "types", @@ -1107,9 +1156,15 @@ dependencies = [ [[package]] name = "const-oid" -version = "0.9.4" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" + +[[package]] +name = "constant_time_eq" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "795bc6e66a8e340f075fcf6227e417a2dc976b92b91f3cdc778bb858778b6747" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" [[package]] name = "convert_case" @@ -1326,7 +1381,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "049bb91fb4aaf0e3c7efa6cd5ef877dbbbd15b39dad06d9948de4ec8a75761ea" dependencies = [ - "cipher", + "cipher 0.3.0", ] [[package]] @@ -1336,7 +1391,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a011bbe2c35ce9c1f143b7af6f94f29a167beb4cd1d29e6740ce836f723120e" dependencies = [ "nix 0.26.2", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -1354,19 +1409,32 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.0.0-rc.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d4ba9852b42210c7538b75484f9daa0655e9a3ac04f693747bb0f02cf3cfe16" +checksum = "f711ade317dd348950a9910f81c5947e3d8907ebd2b83f76203ff1807e6a2bc2" dependencies = [ "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", "digest 0.10.7", "fiat-crypto", - "packed_simd_2", "platforms 3.0.2", + "rustc_version", "subtle", "zeroize", ] +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.29", +] + [[package]] name = "darling" version = "0.13.4" @@ -1508,20 +1576,34 @@ dependencies = [ [[package]] name = "der" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7ed52955ce76b1554f509074bb357d3fb8ac9b51288a65a3fd480d1dfba946" +checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" dependencies = [ "const-oid", "pem-rfc7468", "zeroize", ] +[[package]] +name = "der-parser" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbd676fbbab537128ef0278adb5576cf363cff6aa22a7b24effe97347cfab61e" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "deranged" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8810e7e2cf385b1e9b50d68264908ec367ba642c96d02edfe61c39e88e2a3c01" +checksum = "7684a49fb1af197853ef7b2ee694bc1f5b4179556f1e5710e1760c5db6f5e929" [[package]] name = "derivative" @@ -1542,7 +1624,7 @@ checksum = "53e0efad4403bfc52dc201159c4b842a246a14b98c64b55dfd0f2d89729dfeb8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] @@ -1554,7 +1636,7 @@ dependencies = [ "convert_case", "proc-macro2", "quote", - "rustc_version 0.4.0", + "rustc_version", "syn 1.0.109", ] @@ -1564,7 +1646,7 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7a532c1f99a0f596f6960a60d1e119e91582b24b39e2d83a190e61262c3ef0c" dependencies = [ - "bitflags 2.3.3", + "bitflags 2.4.0", "byteorder", "diesel_derives", "itoa", @@ -1581,7 +1663,7 @@ dependencies = [ "diesel_table_macro_syntax", "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] @@ -1601,7 +1683,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5" dependencies = [ - "syn 2.0.28", + "syn 2.0.29", ] [[package]] @@ -1681,7 +1763,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98c05fa26996c6141f78ac4fafbe297a7fa69690565ba4e0d1f2e60bde5ce501" dependencies = [ - "aes", + "aes 0.7.5", "aes-gcm", "arrayvec", "delay_map", @@ -1708,6 +1790,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "displaydoc" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.29", +] + [[package]] name = "dtoa" version = "1.0.9" @@ -1738,7 +1831,7 @@ version = "0.16.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4b1e0c257a9e9f25f90ff76d7a68360ed497ee519c8e428d1825ef0000799d4" dependencies = [ - "der 0.7.7", + "der 0.7.8", "digest 0.10.7", "elliptic-curve 0.13.5", "rfc6979 0.4.0", @@ -1748,18 +1841,9 @@ dependencies = [ [[package]] name = "ed25519" -version = "1.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91cff35c70bba8a626e3185d8cd48cc11b5437e1a5bcd15b9b5fa3c64b6dfee7" -dependencies = [ - "signature 1.6.4", -] - -[[package]] -name = "ed25519" -version = "2.2.1" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fb04eee5d9d907f29e80ee6b0e78f7e2c82342c63e3580d8c4f69d9d5aad963" +checksum = "60f6d271ca33075c88028be6f04d502853d63a5ece419d269c15315d4fc1cf1d" dependencies = [ "pkcs8 0.10.2", "signature 2.1.0", @@ -1767,26 +1851,12 @@ dependencies = [ [[package]] name = "ed25519-dalek" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c762bae6dcaf24c4c84667b8579785430908723d5c889f469d76a41d59cc7a9d" -dependencies = [ - "curve25519-dalek 3.2.0", - "ed25519 1.5.3", - "rand 0.7.3", - "serde", - "sha2 0.9.9", - "zeroize", -] - -[[package]] -name = "ed25519-dalek" -version = "2.0.0-pre.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd577ba9d4bcab443cac60003d8fd32c638e7024a3ec92c200d7af5d2c397ed" +checksum = "7277392b266383ef8396db7fdeb1e77b6c52fed775f5df15bb24f35b72156980" dependencies = [ - "curve25519-dalek 4.0.0-rc.1", - "ed25519 2.2.1", + "curve25519-dalek 4.0.0", + "ed25519", "rand_core 0.6.4", "serde", "sha2 0.10.7", @@ -1906,7 +1976,7 @@ checksum = "0be7b2ac146c1f99fe245c02d16af0696450d8e06c135db75e10eeb9e642c20d" dependencies = [ "base64 0.21.2", "bytes", - "ed25519-dalek 2.0.0-pre.0", + "ed25519-dalek", "hex", "k256 0.13.1", "log", @@ -1989,7 +2059,7 @@ checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" dependencies = [ "errno-dragonfly", "libc", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -2075,7 +2145,7 @@ dependencies = [ "mediatype", "mime", "pretty_reqwest_error", - "procinfo", + "procfs", "proto_array", "psutil", "reqwest", @@ -2128,7 +2198,7 @@ dependencies = [ name = "eth2_keystore" version = "0.1.0" dependencies = [ - "aes", + "aes 0.7.5", "bls", "eth2_key_derivation", "hex", @@ -2637,7 +2707,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38e2275cc4e4fc009b0669731a1e5ab7ebf11f469eaede2bab9309a5b4d6057f" dependencies = [ "memoffset 0.9.0", - "rustc_version 0.4.0", + "rustc_version", ] [[package]] @@ -2675,9 +2745,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" +checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" dependencies = [ "crc32fast", "libz-sys", @@ -2811,7 +2881,7 @@ dependencies = [ "futures-io", "memchr", "parking", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.12", "waker-fn", ] @@ -2823,18 +2893,17 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] name = "futures-rustls" -version = "0.22.2" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2411eed028cdf8c8034eaf21f9915f956b6c3abec4d4c7949ee67f0721127bd" +checksum = "35bd3cf68c183738046838e300353e4716c674dc5e56890de4826801a6622a28" dependencies = [ "futures-io", - "rustls 0.20.8", - "webpki 0.22.0", + "rustls 0.21.6", ] [[package]] @@ -2879,7 +2948,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.12", "pin-utils", "slab", ] @@ -3258,7 +3327,7 @@ checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", "http", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.12", ] [[package]] @@ -3338,9 +3407,9 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" @@ -3364,7 +3433,7 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.12", "socket2 0.4.9", "tokio", "tower-service", @@ -3381,7 +3450,7 @@ dependencies = [ "futures-util", "http", "hyper", - "rustls 0.21.5", + "rustls 0.21.6", "tokio", "tokio-rustls 0.24.1", ] @@ -3594,6 +3663,15 @@ dependencies = [ "hashbrown 0.14.0", ] +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "generic-array", +] + [[package]] name = "instant" version = "0.1.12" @@ -3632,7 +3710,7 @@ checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ "hermit-abi 0.3.2", "libc", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -3643,7 +3721,7 @@ checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ "socket2 0.5.3", "widestring 1.0.2", - "windows-sys", + "windows-sys 0.48.0", "winreg 0.50.0", ] @@ -3699,6 +3777,15 @@ dependencies = [ "libc", ] +[[package]] +name = "jobserver" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.64" @@ -3774,7 +3861,7 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" dependencies = [ - "spin", + "spin 0.5.2", ] [[package]] @@ -3880,12 +3967,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "libm" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fc7aa29613bd6a620df431842069224d8bc9011086b1db4c0e0cd47fa03ec9a" - [[package]] name = "libm" version = "0.2.7" @@ -3909,9 +3990,9 @@ dependencies = [ [[package]] name = "libp2p" -version = "0.52.1" +version = "0.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38039ba2df4f3255842050845daef4a004cc1f26da03dbc645535088b51910ef" +checksum = "32d07d1502a027366d55afe187621c2d7895dc111a3df13b35fed698049681d7" dependencies = [ "bytes", "futures", @@ -3929,6 +4010,7 @@ dependencies = [ "libp2p-metrics", "libp2p-noise", "libp2p-plaintext", + "libp2p-quic", "libp2p-swarm", "libp2p-tcp", "libp2p-websocket", @@ -4006,9 +4088,9 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" -version = "0.45.0" +version = "0.45.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e378da62e8c9251f6e885ed173a561663f29b251e745586cf6ae6150b295c37" +checksum = "2d157562dba6017193e5285acf6b1054759e83540bfd79f75b69d6ce774c88da" dependencies = [ "asynchronous-codec", "base64 0.21.2", @@ -4060,13 +4142,13 @@ dependencies = [ [[package]] name = "libp2p-identity" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a38d6012784fe4cc14e6d443eb415b11fc7c456dc15d9f0d90d9b70bc7ac3ec1" +checksum = "686e73aff5e23efbb99bc85340ea6fd8686986aa7b283a881ba182cfca535ca9" dependencies = [ "asn1_der", "bs58 0.5.0", - "ed25519-dalek 1.0.1", + "ed25519-dalek", "libsecp256k1", "log", "multihash 0.19.0", @@ -4103,9 +4185,9 @@ dependencies = [ [[package]] name = "libp2p-metrics" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3787ea81798dcc5bf1d8b40a8e8245cf894b168d04dd70aa48cb3ff2fff141d2" +checksum = "239ba7d28f8d0b5d77760dc6619c05c7e88e74ec8fbbe97f856f20a56745e620" dependencies = [ "instant", "libp2p-core", @@ -4138,12 +4220,12 @@ dependencies = [ [[package]] name = "libp2p-noise" -version = "0.43.0" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87945db2b3f977af09b62b9aa0a5f3e4870995a577ecd845cdeba94cdf6bbca7" +checksum = "71ce70757f2c0d82e9a3ef738fb10ea0723d16cec37f078f719e2c247704c1bb" dependencies = [ "bytes", - "curve25519-dalek 3.2.0", + "curve25519-dalek 4.0.0", "futures", "libp2p-core", "libp2p-identity", @@ -4177,11 +4259,34 @@ dependencies = [ "unsigned-varint 0.7.1", ] +[[package]] +name = "libp2p-quic" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cb763e88f9a043546bfebd3575f340e7dd3d6c1b2cf2629600ec8965360c63a" +dependencies = [ + "bytes", + "futures", + "futures-timer", + "if-watch", + "libp2p-core", + "libp2p-identity", + "libp2p-tls", + "log", + "parking_lot 0.12.1", + "quinn", + "rand 0.8.5", + "rustls 0.21.6", + "socket2 0.5.3", + "thiserror", + "tokio", +] + [[package]] name = "libp2p-swarm" -version = "0.43.2" +version = "0.43.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43106820057e0f65c77b01a3873593f66e676da4e40c70c3a809b239109f1d30" +checksum = "28016944851bd73526d3c146aabf0fa9bbe27c558f080f9e5447da3a1772c01a" dependencies = [ "either", "fnv", @@ -4210,7 +4315,7 @@ dependencies = [ "proc-macro-warning", "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] @@ -4230,11 +4335,30 @@ dependencies = [ "tokio", ] +[[package]] +name = "libp2p-tls" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8218d1d5482b122ccae396bbf38abdcb283ecc96fa54760e1dfd251f0546ac61" +dependencies = [ + "futures", + "futures-rustls", + "libp2p-core", + "libp2p-identity", + "rcgen", + "ring", + "rustls 0.21.6", + "rustls-webpki", + "thiserror", + "x509-parser", + "yasna", +] + [[package]] name = "libp2p-websocket" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "956d981ebc84abc3377e5875483c06d94ff57bc6b25f725047f9fd52592f72d4" +checksum = "3facf0691bab65f571bc97c6c65ffa836248ca631d631b7691ac91deb7fceb5f" dependencies = [ "either", "futures", @@ -4247,14 +4371,14 @@ dependencies = [ "rw-stream-sink", "soketto", "url", - "webpki-roots 0.23.1", + "webpki-roots 0.25.2", ] [[package]] name = "libp2p-yamux" -version = "0.44.0" +version = "0.44.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0a9b42ab6de15c6f076d8fb11dc5f48d899a10b55a2e16b12be9012a05287b0" +checksum = "8eedcb62824c4300efb9cfd4e2a6edaf3ca097b9e68b36dabe45a44469fd6a85" dependencies = [ "futures", "libp2p-core", @@ -4452,6 +4576,12 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +[[package]] +name = "linux-raw-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" + [[package]] name = "linux-raw-sys" version = "0.3.8" @@ -4505,9 +4635,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "logging" @@ -4611,9 +4741,9 @@ checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" [[package]] name = "matchit" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67827e6ea8ee8a7c4a72227ef4fc08957040acffdb5f122733b24fa12daff41b" +checksum = "ed1202b2a6f884ae56f04cff409ab315c5ce26b5e58d7412e484f01fd52f52ef" [[package]] name = "maybe-uninit" @@ -4799,7 +4929,7 @@ checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -4828,6 +4958,24 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7843ec2de400bcbc6a6328c958dc38e5359da6e93e72e37bc5246bf1ae776389" +[[package]] +name = "multer" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01acbdc23469fd8fe07ab135923371d5f5a422fbf9c522158677c8eb15bc51c2" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http", + "httparse", + "log", + "memchr", + "mime", + "spin 0.9.8", + "version_check", +] + [[package]] name = "multiaddr" version = "0.14.0" @@ -4913,24 +5061,6 @@ dependencies = [ "synstructure", ] -[[package]] -name = "multipart" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00dec633863867f29cb39df64a397cdf4a6354708ddd7759f70c7fb51c5f9182" -dependencies = [ - "buf_redux", - "httparse", - "log", - "mime", - "mime_guess", - "quick-error", - "rand 0.8.5", - "safemem", - "tempfile", - "twoway", -] - [[package]] name = "multistream-select" version = "0.13.0" @@ -5140,12 +5270,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" -[[package]] -name = "nom" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf51a729ecf40266a2368ad335a5fdde43471f545a967109cd62146ecf8b66ff" - [[package]] name = "nom" version = "7.1.3" @@ -5195,7 +5319,7 @@ dependencies = [ "autocfg 0.1.8", "byteorder", "lazy_static", - "libm 0.2.7", + "libm", "num-integer", "num-iter", "num-traits", @@ -5263,6 +5387,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "oid-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bedf36ffb6ba96c2eb7144ef6270557b52e54b20c0a8e1eb2ff99a6c6959bff" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.18.0" @@ -5315,9 +5448,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.55" +version = "0.10.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d" +checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e" dependencies = [ "bitflags 1.3.2", "cfg-if", @@ -5336,7 +5469,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] @@ -5347,18 +5480,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "111.26.0+1.1.1u" +version = "111.27.0+1.1.1v" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efc62c9f12b22b8f5208c23a7200a442b2e5999f8bdf80233852122b5a4f6f37" +checksum = "06e8f197c82d7511c5b014030c9b1efeda40d7d5f99d23b4ceed3524a5e63f02" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.90" +version = "0.9.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6" +checksum = "866b5f16f90776b9bb8dc1e1802ac6f0513de3a7a7465867bfbc563dc737faac" dependencies = [ "cc", "libc", @@ -5409,16 +5542,6 @@ dependencies = [ "sha2 0.10.7", ] -[[package]] -name = "packed_simd_2" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1914cd452d8fccd6f9db48147b29fd4ae05bea9dc5d9ad578509f72415de282" -dependencies = [ - "cfg-if", - "libm 0.1.4", -] - [[package]] name = "parity-scale-codec" version = "2.3.1" @@ -5522,7 +5645,18 @@ dependencies = [ "libc", "redox_syscall 0.3.5", "smallvec 1.11.0", - "windows-targets", + "windows-targets 0.48.3", +] + +[[package]] +name = "password-hash" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" +dependencies = [ + "base64ct", + "rand_core 0.6.4", + "subtle", ] [[package]] @@ -5549,6 +5683,18 @@ dependencies = [ "crypto-mac 0.11.1", ] +[[package]] +name = "pbkdf2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest 0.10.7", + "hmac 0.12.1", + "password-hash", + "sha2 0.10.7", +] + [[package]] name = "peeking_take_while" version = "0.1.2" @@ -5586,7 +5732,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9567389417feee6ce15dd6527a8a1ecac205ef62c2932bcf3d9f6fc5b78b414" dependencies = [ "futures", - "rustc_version 0.4.0", + "rustc_version", ] [[package]] @@ -5609,22 +5755,22 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "030ad2bc4db10a8944cb0d837f158bdfec4d4a4873ab701a95046770d11f8842" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] @@ -5635,9 +5781,9 @@ checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777" [[package]] name = "pin-project-lite" -version = "0.2.10" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57" +checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" [[package]] name = "pin-utils" @@ -5661,7 +5807,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der 0.7.7", + "der 0.7.8", "spki 0.7.2", ] @@ -5723,8 +5869,8 @@ dependencies = [ "concurrent-queue", "libc", "log", - "pin-project-lite 0.2.10", - "windows-sys", + "pin-project-lite 0.2.12", + "windows-sys 0.48.0", ] [[package]] @@ -5886,7 +6032,7 @@ checksum = "70550716265d1ec349c41f70dd4f964b4fd88394efe4405f0c1da679c4799a07" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] @@ -5899,17 +6045,20 @@ dependencies = [ ] [[package]] -name = "procinfo" -version = "0.4.2" +name = "procfs" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ab1427f3d2635891f842892dda177883dca0639e05fe66796a62c9d2f23b49c" +checksum = "943ca7f9f29bab5844ecd8fdb3992c5969b6622bb9609b9502fef9b4310e3f1f" dependencies = [ + "bitflags 1.3.2", "byteorder", - "libc", - "nom 2.2.1", - "rustc_version 0.2.3", -] - + "chrono", + "flate2", + "hex", + "lazy_static", + "rustix 0.36.15", +] + [[package]] name = "prometheus" version = "0.13.3" @@ -5939,13 +6088,13 @@ dependencies = [ [[package]] name = "prometheus-client-derive-encode" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b6a5217beb0ad503ee7fa752d451c905113d70721b937126158f3106a48cc1" +checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.29", ] [[package]] @@ -6049,11 +6198,59 @@ dependencies = [ "pin-project-lite 0.1.12", ] +[[package]] +name = "quinn" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cc2c5017e4b43d5995dcea317bc46c1e09404c0a9664d2908f7f02dfe943d75" +dependencies = [ + "bytes", + "futures-io", + "pin-project-lite 0.2.12", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls 0.21.6", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "quinn-proto" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13f81c9a9d574310b8351f8666f5a93ac3b0069c45c28ad52c10291389a7cf9" +dependencies = [ + "bytes", + "rand 0.8.5", + "ring", + "rustc-hash", + "rustls 0.21.6", + "slab", + "thiserror", + "tinyvec", + "tracing", +] + +[[package]] +name = "quinn-udp" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "055b4e778e8feb9f93c4e439f71dc2156ef13360b432b799e179a8c4cdf0b1d7" +dependencies = [ + "bytes", + "libc", + "socket2 0.5.3", + "tracing", + "windows-sys 0.48.0", +] + [[package]] name = "quote" -version = "1.0.32" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2", ] @@ -6193,6 +6390,18 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "rcgen" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" +dependencies = [ + "pem", + "ring", + "time", + "yasna", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -6224,13 +6433,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.1" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" +checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.4", + "regex-automata 0.3.6", "regex-syntax 0.7.4", ] @@ -6245,9 +6454,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.4" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7b6d6190b7594385f61bd3911cd1be99dfddcfc365a4160cc2ab5bff4aed294" +checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" dependencies = [ "aho-corasick", "memchr", @@ -6290,8 +6499,8 @@ dependencies = [ "native-tls", "once_cell", "percent-encoding", - "pin-project-lite 0.2.10", - "rustls 0.21.5", + "pin-project-lite 0.2.12", + "rustls 0.21.6", "rustls-pemfile", "serde", "serde_json", @@ -6350,7 +6559,7 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", + "spin 0.5.2", "untrusted", "web-sys", "winapi", @@ -6442,60 +6651,61 @@ checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" [[package]] name = "rustc_version" -version = "0.2.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 0.9.0", + "semver", ] [[package]] -name = "rustc_version" -version = "0.4.0" +name = "rusticata-macros" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" dependencies = [ - "semver 1.0.18", + "nom", ] [[package]] name = "rustix" -version = "0.37.23" +version = "0.36.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" +checksum = "c37f1bd5ef1b5422177b7646cba67430579cfe2ace80f284fee876bca52ad941" dependencies = [ "bitflags 1.3.2", "errno", "io-lifetimes", "libc", - "linux-raw-sys 0.3.8", - "windows-sys", + "linux-raw-sys 0.1.4", + "windows-sys 0.45.0", ] [[package]] name = "rustix" -version = "0.38.4" +version = "0.37.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" +checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" dependencies = [ - "bitflags 2.3.3", + "bitflags 1.3.2", "errno", + "io-lifetimes", "libc", - "linux-raw-sys 0.4.5", - "windows-sys", + "linux-raw-sys 0.3.8", + "windows-sys 0.48.0", ] [[package]] -name = "rustls" -version = "0.19.1" +name = "rustix" +version = "0.38.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" dependencies = [ - "base64 0.13.1", - "log", - "ring", - "sct 0.6.1", - "webpki 0.21.4", + "bitflags 2.4.0", + "errno", + "libc", + "linux-raw-sys 0.4.5", + "windows-sys 0.48.0", ] [[package]] @@ -6506,20 +6716,20 @@ checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" dependencies = [ "log", "ring", - "sct 0.7.0", - "webpki 0.22.0", + "sct", + "webpki", ] [[package]] name = "rustls" -version = "0.21.5" +version = "0.21.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79ea77c539259495ce8ca47f53e66ae0330a8819f67e23ac96ca02f50e7b7d36" +checksum = "1d1feddffcfcc0b33f5c6ce9a29e341e4cd59c3f78e7ee45f4a40c038b1d6cbb" dependencies = [ "log", "ring", - "rustls-webpki 0.101.2", - "sct 0.7.0", + "rustls-webpki", + "sct", ] [[package]] @@ -6533,19 +6743,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.100.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "rustls-webpki" -version = "0.101.2" +version = "0.101.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "513722fd73ad80a71f72b61009ea1b584bcfa1483ca93949c8f290298837fa59" +checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d" dependencies = [ "ring", "untrusted", @@ -6578,19 +6778,13 @@ checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" name = "safe_arith" version = "0.1.0" -[[package]] -name = "safemem" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" - [[package]] name = "salsa20" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecbd2eb639fd7cab5804a0837fe373cc2172d15437e804c054a9fb885cb923b0" dependencies = [ - "cipher", + "cipher 0.3.0", ] [[package]] @@ -6632,7 +6826,7 @@ version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" dependencies = [ - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -6668,16 +6862,6 @@ dependencies = [ "sha2 0.9.9", ] -[[package]] -name = "sct" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "sct" version = "0.7.0" @@ -6709,7 +6893,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct 0.2.0", - "der 0.7.7", + "der 0.7.8", "generic-array", "pkcs8 0.10.2", "subtle", @@ -6739,15 +6923,6 @@ dependencies = [ "libc", ] -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser", -] - [[package]] name = "semver" version = "1.0.18" @@ -6757,12 +6932,6 @@ dependencies = [ "serde", ] -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" - [[package]] name = "send_wrapper" version = "0.6.0" @@ -6779,9 +6948,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.180" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea67f183f058fe88a4e3ec6e2788e003840893b91bac4559cabedd00863b3ed" +checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" dependencies = [ "serde_derive", ] @@ -6819,20 +6988,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.180" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24e744d7782b686ab3b73267ef05697159cc0e5abbed3f47f9933165e5219036" +checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] name = "serde_json" -version = "1.0.104" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" +checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" dependencies = [ "itoa", "ryu", @@ -6857,7 +7026,7 @@ checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] @@ -7049,7 +7218,7 @@ dependencies = [ "num-bigint", "num-traits", "thiserror", - "time 0.3.24", + "time", ] [[package]] @@ -7181,7 +7350,7 @@ dependencies = [ "serde", "serde_json", "slog", - "time 0.3.24", + "time", ] [[package]] @@ -7226,14 +7395,14 @@ dependencies = [ "slog", "term", "thread_local", - "time 0.3.24", + "time", ] [[package]] name = "sloggers" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e20d36cb80da75a9c5511872f15247ddad14ead8c1dd97a86b56d1be9f5d4a0e" +checksum = "7a0a4d8569a69ee56f277bffc2f6eee637b98ed468448e8a5a84fa63efe4de9d" dependencies = [ "chrono", "libc", @@ -7286,17 +7455,17 @@ checksum = "5e9f0ab6ef7eb7353d9119c170a436d1bf248eea575ac42d19d12f4e34130831" [[package]] name = "snow" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ccba027ba85743e09d15c03296797cad56395089b832b48b5a5217880f57733" +checksum = "0c9d1425eb528a21de2755c75af4c9b5d57f50a0d4c3b7f1828a4cd03f8ba155" dependencies = [ "aes-gcm", "blake2", "chacha20poly1305", - "curve25519-dalek 4.0.0-rc.1", + "curve25519-dalek 4.0.0", "rand_core 0.6.4", "ring", - "rustc_version 0.4.0", + "rustc_version", "sha2 0.10.7", "subtle", ] @@ -7318,7 +7487,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -7342,6 +7511,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "spki" version = "0.6.0" @@ -7359,7 +7534,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" dependencies = [ "base64ct", - "der 0.7.7", + "der 0.7.8", ] [[package]] @@ -7575,9 +7750,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.28" +version = "2.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" +checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a" dependencies = [ "proc-macro2", "quote", @@ -7691,15 +7866,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.7.0" +version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998" +checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651" dependencies = [ "cfg-if", "fastrand 2.0.0", "redox_syscall 0.3.5", - "rustix 0.38.4", - "windows-sys", + "rustix 0.38.8", + "windows-sys 0.48.0", ] [[package]] @@ -7741,8 +7916,7 @@ dependencies = [ [[package]] name = "testcontainers" version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e2b1567ca8a2b819ea7b28c92be35d9f76fb9edb214321dcc86eb96023d1f87" +source = "git+https://github.com/testcontainers/testcontainers-rs/?rev=0f2c9851#0f2c985160e51a200cfc847097c15b8d85ed7df1" dependencies = [ "bollard-stubs", "futures", @@ -7766,22 +7940,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.44" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" +checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.44" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" +checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] @@ -7805,20 +7979,9 @@ dependencies = [ [[package]] name = "time" -version = "0.1.45" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" -dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", -] - -[[package]] -name = "time" -version = "0.3.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b79eabcd964882a646b3584543ccabeae7869e9ac32a46f6f22b7a5bd405308b" +checksum = "b0fdd63d58b18d663fbdf70e049f00a22c8e42be082203be7f26589213cd75ea" dependencies = [ "deranged", "itoa", @@ -7910,22 +8073,21 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.29.1" +version = "1.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ - "autocfg 1.1.0", "backtrace", "bytes", "libc", "mio", "num_cpus", "parking_lot 0.12.1", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.12", "signal-hook-registry", - "socket2 0.4.9", + "socket2 0.5.3", "tokio-macros", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -7934,7 +8096,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" dependencies = [ - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.12", "tokio", ] @@ -7946,7 +8108,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] @@ -7975,7 +8137,7 @@ dependencies = [ "parking_lot 0.12.1", "percent-encoding", "phf", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.12", "postgres-protocol", "postgres-types", "socket2 0.5.3", @@ -7983,17 +8145,6 @@ dependencies = [ "tokio-util 0.7.8", ] -[[package]] -name = "tokio-rustls" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" -dependencies = [ - "rustls 0.19.1", - "tokio", - "webpki 0.21.4", -] - [[package]] name = "tokio-rustls" version = "0.23.4" @@ -8002,7 +8153,7 @@ checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ "rustls 0.20.8", "tokio", - "webpki 0.22.0", + "webpki", ] [[package]] @@ -8011,7 +8162,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.5", + "rustls 0.21.6", "tokio", ] @@ -8022,38 +8173,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.12", "tokio", "tokio-util 0.7.8", ] [[package]] name = "tokio-tungstenite" -version = "0.15.0" +version = "0.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "511de3f85caf1c98983545490c3d09685fa8eb634e57eec22bb4db271f46cbd8" +checksum = "f714dd15bead90401d77e04243611caec13726c2408afd5b31901dfcdcb3b181" dependencies = [ "futures-util", "log", - "pin-project", + "rustls 0.20.8", "tokio", - "tungstenite 0.14.0", + "tokio-rustls 0.23.4", + "tungstenite 0.17.3", + "webpki", + "webpki-roots 0.22.6", ] [[package]] name = "tokio-tungstenite" -version = "0.17.2" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f714dd15bead90401d77e04243611caec13726c2408afd5b31901dfcdcb3b181" +checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd" dependencies = [ "futures-util", "log", - "rustls 0.20.8", "tokio", - "tokio-rustls 0.23.4", - "tungstenite 0.17.3", - "webpki 0.22.0", - "webpki-roots 0.22.6", + "tungstenite 0.18.0", ] [[package]] @@ -8067,7 +8217,7 @@ dependencies = [ "futures-io", "futures-sink", "log", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.12", "slab", "tokio", ] @@ -8081,7 +8231,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.12", "slab", "tokio", "tracing", @@ -8139,7 +8289,7 @@ dependencies = [ "futures-core", "futures-util", "pin-project", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.12", "tokio", "tower-layer", "tower-service", @@ -8166,7 +8316,7 @@ checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", "log", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.12", "tracing-attributes", "tracing-core", ] @@ -8179,7 +8329,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] @@ -8336,9 +8486,9 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "tungstenite" -version = "0.14.0" +version = "0.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0b2d8558abd2e276b0a8df5c05a2ec762609344191e5fd23e292c910e9165b5" +checksum = "e27992fd6a8c29ee7eef28fc78349aa244134e10ad447ce3b9f0ac0ed0fa4ce0" dependencies = [ "base64 0.13.1", "byteorder", @@ -8347,17 +8497,19 @@ dependencies = [ "httparse", "log", "rand 0.8.5", - "sha-1 0.9.8", + "rustls 0.20.8", + "sha-1 0.10.1", "thiserror", "url", "utf-8", + "webpki", ] [[package]] name = "tungstenite" -version = "0.17.3" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e27992fd6a8c29ee7eef28fc78349aa244134e10ad447ce3b9f0ac0ed0fa4ce0" +checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788" dependencies = [ "base64 0.13.1", "byteorder", @@ -8366,21 +8518,10 @@ dependencies = [ "httparse", "log", "rand 0.8.5", - "rustls 0.20.8", - "sha-1 0.10.1", + "sha1", "thiserror", "url", "utf-8", - "webpki 0.22.0", -] - -[[package]] -name = "twoway" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b11b2b5241ba34be09c3cc85a36e56e48f9888862e19cedf23336d35316ed1" -dependencies = [ - "memchr", ] [[package]] @@ -8731,8 +8872,8 @@ dependencies = [ [[package]] name = "warp" -version = "0.3.2" -source = "git+https://github.com/macladson/warp?rev=7e75acc368229a46a236a8c991bf251fe7fe50ef#7e75acc368229a46a236a8c991bf251fe7fe50ef" +version = "0.3.5" +source = "git+https://github.com/seanmonstar/warp.git?rev=149913fe#149913fed948bbe2149b52b9016170bcaef950ab" dependencies = [ "bytes", "futures-channel", @@ -8743,18 +8884,19 @@ dependencies = [ "log", "mime", "mime_guess", - "multipart", + "multer", "percent-encoding", "pin-project", + "rustls-pemfile", "scoped-tls", "serde", "serde_json", "serde_urlencoded", "tokio", - "tokio-rustls 0.22.0", + "tokio-rustls 0.23.4", "tokio-stream", - "tokio-tungstenite 0.15.0", - "tokio-util 0.6.10", + "tokio-tungstenite 0.18.0", + "tokio-util 0.7.8", "tower-service", "tracing", ] @@ -8783,12 +8925,6 @@ version = "0.9.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -8816,7 +8952,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", "wasm-bindgen-shared", ] @@ -8850,7 +8986,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -8960,16 +9096,6 @@ dependencies = [ "zip", ] -[[package]] -name = "webpki" -version = "0.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "webpki" version = "0.22.0" @@ -8986,17 +9112,14 @@ version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" dependencies = [ - "webpki 0.22.0", + "webpki", ] [[package]] name = "webpki-roots" -version = "0.23.1" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338" -dependencies = [ - "rustls-webpki 0.100.1", -] +checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" [[package]] name = "widestring" @@ -9066,7 +9189,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows-targets", + "windows-targets 0.48.3", ] [[package]] @@ -9081,35 +9204,65 @@ dependencies = [ "winapi", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.3", ] [[package]] name = "windows-targets" -version = "0.48.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.48.0", + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f51fb4c64f8b770a823c043c7fad036323e1c48f55287b7bbb7987b2fcdf3b" +dependencies = [ + "windows_aarch64_gnullvm 0.48.3", + "windows_aarch64_msvc 0.48.3", + "windows_i686_gnu 0.48.3", + "windows_i686_msvc 0.48.3", + "windows_x86_64_gnu 0.48.3", + "windows_x86_64_gnullvm 0.48.3", + "windows_x86_64_msvc 0.48.3", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.0" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "fde1bb55ae4ce76a597a8566d82c57432bc69c039449d61572a7a353da28f68c" [[package]] name = "windows_aarch64_msvc" @@ -9119,9 +9272,15 @@ checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d" [[package]] name = "windows_aarch64_msvc" -version = "0.48.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1513e8d48365a78adad7322fd6b5e4c4e99d92a69db8df2d435b25b1f1f286d4" [[package]] name = "windows_i686_gnu" @@ -9131,9 +9290,15 @@ checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed" [[package]] name = "windows_i686_gnu" -version = "0.48.0" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "60587c0265d2b842298f5858e1a5d79d146f9ee0c37be5782e92a6eb5e1d7a83" [[package]] name = "windows_i686_msvc" @@ -9143,9 +9308,15 @@ checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956" [[package]] name = "windows_i686_msvc" -version = "0.48.0" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "224fe0e0ffff5d2ea6a29f82026c8f43870038a0ffc247aa95a52b47df381ac4" [[package]] name = "windows_x86_64_gnu" @@ -9155,15 +9326,27 @@ checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62fc52a0f50a088de499712cbc012df7ebd94e2d6eb948435449d76a6287e7ad" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2093925509d91ea3d69bcd20238f4c2ecdb1a29d3c281d026a09705d0dd35f3d" [[package]] name = "windows_x86_64_msvc" @@ -9173,15 +9356,21 @@ checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" [[package]] name = "windows_x86_64_msvc" -version = "0.48.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6ade45bc8bf02ae2aa34a9d54ba660a1a58204da34ba793c00d83ca3730b5f1" [[package]] name = "winnow" -version = "0.5.3" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f46aab759304e4d7b2075a9aecba26228bb073ee8c50db796b2c72c676b5d807" +checksum = "d09770118a7eb1ccaf4a594a221334119a44a814fcb0d31c5b85e83e97227a97" dependencies = [ "memchr", ] @@ -9202,7 +9391,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ "cfg-if", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -9216,7 +9405,7 @@ dependencies = [ "js-sys", "log", "pharos", - "rustc_version 0.4.0", + "rustc_version", "send_wrapper", "thiserror", "wasm-bindgen", @@ -9250,6 +9439,23 @@ dependencies = [ "zeroize", ] +[[package]] +name = "x509-parser" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7069fba5b66b9193bd2c5d3d4ff12b839118f6bcbef5328efafafb5395cf63da" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "rusticata-macros", + "thiserror", + "time", +] + [[package]] name = "xml-rs" version = "0.8.16" @@ -9276,18 +9482,28 @@ dependencies = [ [[package]] name = "yamux" -version = "0.10.2" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d9ba232399af1783a58d8eb26f6b5006fbefe2dc9ef36bd283324792d03ea5" +checksum = "0329ef377816896f014435162bb3711ea7a07729c23d0960e6f8048b21b8fe91" dependencies = [ "futures", "log", "nohash-hasher", "parking_lot 0.12.1", + "pin-project", "rand 0.8.5", "static_assertions", ] +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + [[package]] name = "zeroize" version = "1.6.0" @@ -9305,19 +9521,55 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] name = "zip" -version = "0.5.13" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93ab48844d61251bb3835145c521d88aa4031d7139e8485990f60ca911fa0815" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" dependencies = [ + "aes 0.8.3", "byteorder", "bzip2", + "constant_time_eq", "crc32fast", + "crossbeam-utils", "flate2", - "thiserror", - "time 0.1.45", + "hmac 0.12.1", + "pbkdf2 0.11.0", + "sha1", + "time", + "zstd", +] + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.8+zstd.1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +dependencies = [ + "cc", + "libc", + "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index 15906a03065..9930658e652 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,7 +91,8 @@ resolver = "2" [patch] [patch.crates-io] -warp = { git = "https://github.com/macladson/warp", rev="7e75acc368229a46a236a8c991bf251fe7fe50ef" } +# TODO: remove when 0.3.6 get's released. +warp = { git = "https://github.com/seanmonstar/warp.git", rev="149913fe" } [profile.maxperf] inherits = "release" diff --git a/Makefile b/Makefile index bc84cdede09..1e99b3dbb33 100644 --- a/Makefile +++ b/Makefile @@ -206,9 +206,8 @@ arbitrary-fuzz: # Runs cargo audit (Audit Cargo.lock files for crates with security vulnerabilities reported to the RustSec Advisory Database) audit: - # cargo install --force cargo-audit - cargo audit --ignore RUSTSEC-2020-0071 --ignore RUSTSEC-2022-0093 \ - --ignore RUSTSEC-2023-0052 --ignore RUSTSEC-2023-0053 + cargo install --force cargo-audit + cargo audit --ignore RUSTSEC-2023-0052 # Runs `cargo vendor` to make sure dependencies can be vendored for packaging, reproducibility and archival purpose. vendor: diff --git a/beacon_node/execution_layer/Cargo.toml b/beacon_node/execution_layer/Cargo.toml index 8492a5159e2..f2259a4813b 100644 --- a/beacon_node/execution_layer/Cargo.toml +++ b/beacon_node/execution_layer/Cargo.toml @@ -50,4 +50,4 @@ keccak-hash = "0.10.0" hash256-std-hasher = "0.15.2" triehash = "0.8.4" hash-db = "0.15.2" -pretty_reqwest_error = { path = "../../common/pretty_reqwest_error" } \ No newline at end of file +pretty_reqwest_error = { path = "../../common/pretty_reqwest_error" } diff --git a/common/eth2/Cargo.toml b/common/eth2/Cargo.toml index d8e1a375fd5..8cb3de3a002 100644 --- a/common/eth2/Cargo.toml +++ b/common/eth2/Cargo.toml @@ -35,14 +35,8 @@ tokio = { version = "1.14.0", features = ["full"] } [target.'cfg(target_os = "linux")'.dependencies] psutil = { version = "3.2.2", optional = true } -procinfo = { version = "0.4.2", optional = true } +procfs = { version = "0.15.1", optional = true } [features] default = ["lighthouse"] -lighthouse = [ - "proto_array", - "psutil", - "procinfo", - "store", - "slashing_protection", -] +lighthouse = ["proto_array", "psutil", "procfs", "store", "slashing_protection"] diff --git a/common/eth2/src/lighthouse.rs b/common/eth2/src/lighthouse.rs index 1b4bcc0e395..dfc19db4928 100644 --- a/common/eth2/src/lighthouse.rs +++ b/common/eth2/src/lighthouse.rs @@ -95,8 +95,8 @@ pub struct ValidatorInclusionData { #[cfg(target_os = "linux")] use { - procinfo::pid, psutil::cpu::os::linux::CpuTimesExt, - psutil::memory::os::linux::VirtualMemoryExt, psutil::process::Process, + psutil::cpu::os::linux::CpuTimesExt, psutil::memory::os::linux::VirtualMemoryExt, + psutil::process::Process, }; /// Reports on the health of the Lighthouse instance. @@ -238,7 +238,7 @@ pub struct ProcessHealth { /// The pid of this process. pub pid: u32, /// The number of threads used by this pid. - pub pid_num_threads: i32, + pub pid_num_threads: i64, /// The total resident memory used by this pid. pub pid_mem_resident_set_size: u64, /// The total virtual memory used by this pid. @@ -262,7 +262,12 @@ impl ProcessHealth { .memory_info() .map_err(|e| format!("Unable to get process memory info: {:?}", e))?; - let stat = pid::stat_self().map_err(|e| format!("Unable to get stat: {:?}", e))?; + let me = procfs::process::Process::myself() + .map_err(|e| format!("Unable to get process: {:?}", e))?; + let stat = me + .stat() + .map_err(|e| format!("Unable to get stat: {:?}", e))?; + let process_times = process .cpu_times() .map_err(|e| format!("Unable to get process cpu times : {:?}", e))?; diff --git a/common/eth2_network_config/Cargo.toml b/common/eth2_network_config/Cargo.toml index 338a2d243bc..d1fe25f1cc1 100644 --- a/common/eth2_network_config/Cargo.toml +++ b/common/eth2_network_config/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" build = "build.rs" [build-dependencies] -zip = "0.5.8" +zip = "0.6" eth2_config = { path = "../eth2_config"} [dev-dependencies] @@ -18,4 +18,4 @@ serde_yaml = "0.8.13" types = { path = "../../consensus/types"} ethereum_ssz = "0.5.0" eth2_config = { path = "../eth2_config"} -discv5 = "0.3.1" \ No newline at end of file +discv5 = "0.3.1" diff --git a/common/logging/Cargo.toml b/common/logging/Cargo.toml index b6179d9e782..0e158f58ff5 100644 --- a/common/logging/Cargo.toml +++ b/common/logging/Cargo.toml @@ -17,6 +17,6 @@ sloggers = { version = "2.1.1", features = ["json"] } slog-async = "2.7.0" take_mut = "0.2.2" parking_lot = "0.12.1" -serde = "1.0.153" +serde = "1.0.153" serde_json = "1.0.94" -chrono = "0.4.23" +chrono = { version = "0.4", default-features = false, features = ["clock", "std"] } diff --git a/common/warp_utils/src/cors.rs b/common/warp_utils/src/cors.rs index 314ea9c8f74..55043dfd7dd 100644 --- a/common/warp_utils/src/cors.rs +++ b/common/warp_utils/src/cors.rs @@ -10,10 +10,14 @@ pub fn set_builder_origins( default_origin: (IpAddr, u16), ) -> Result { if let Some(allow_origin) = allow_origin { - let origins = allow_origin - .split(',') - .map(|s| verify_cors_origin_str(s).map(|_| s)) - .collect::, _>>()?; + let mut origins = vec![]; + for origin in allow_origin.split(',') { + verify_cors_origin_str(origin)?; + if origin == "*" { + return Ok(builder.allow_any_origin()); + } + origins.push(origin) + } Ok(builder.allow_origins(origins)) } else { let origin = match default_origin.0 { diff --git a/common/warp_utils/src/metrics.rs b/common/warp_utils/src/metrics.rs index 1b9d89db91a..d93b74ca956 100644 --- a/common/warp_utils/src/metrics.rs +++ b/common/warp_utils/src/metrics.rs @@ -87,7 +87,7 @@ pub fn scrape_process_health_metrics() { // This will silently fail if we are unable to observe the health. This is desired behaviour // since we don't support `Health` for all platforms. if let Ok(health) = ProcessHealth::observe() { - set_gauge(&PROCESS_NUM_THREADS, health.pid_num_threads as i64); + set_gauge(&PROCESS_NUM_THREADS, health.pid_num_threads); set_gauge(&PROCESS_RES_MEM, health.pid_mem_resident_set_size as i64); set_gauge(&PROCESS_VIRT_MEM, health.pid_mem_virtual_memory_size as i64); set_gauge(&PROCESS_SECONDS, health.pid_process_seconds_total as i64); diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 37d224dbc31..626be85f197 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -11,6 +11,7 @@ use lighthouse_network::PeerId; use std::fs::File; use std::io::{Read, Write}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::path::Path; use std::path::PathBuf; use std::process::Command; use std::str::FromStr; @@ -1460,15 +1461,20 @@ fn disable_inbound_rate_limiter_flag() { #[test] fn http_allow_origin_flag() { CommandLineTest::new() - .flag("http-allow-origin", Some("127.0.0.99")) + .flag("http", None) + .flag("http-allow-origin", Some("http://127.0.0.99")) .run_with_zero_port() .with_config(|config| { - assert_eq!(config.http_api.allow_origin, Some("127.0.0.99".to_string())); + assert_eq!( + config.http_api.allow_origin, + Some("http://127.0.0.99".to_string()) + ); }); } #[test] fn http_allow_origin_all_flag() { CommandLineTest::new() + .flag("http", None) .flag("http-allow-origin", Some("*")) .run_with_zero_port() .with_config(|config| assert_eq!(config.http_api.allow_origin, Some("*".to_string()))); @@ -1476,6 +1482,7 @@ fn http_allow_origin_all_flag() { #[test] fn http_allow_sync_stalled_flag() { CommandLineTest::new() + .flag("http", None) .flag("http-allow-sync-stalled", None) .run_with_zero_port() .with_config(|config| assert_eq!(config.http_api.allow_sync_stalled, true)); @@ -1483,32 +1490,29 @@ fn http_allow_sync_stalled_flag() { #[test] fn http_enable_beacon_processor() { CommandLineTest::new() + .flag("http", None) .run_with_zero_port() .with_config(|config| assert_eq!(config.http_api.enable_beacon_processor, true)); CommandLineTest::new() + .flag("http", None) .flag("http-enable-beacon-processor", Some("true")) .run_with_zero_port() .with_config(|config| assert_eq!(config.http_api.enable_beacon_processor, true)); CommandLineTest::new() + .flag("http", None) .flag("http-enable-beacon-processor", Some("false")) .run_with_zero_port() .with_config(|config| assert_eq!(config.http_api.enable_beacon_processor, false)); } #[test] fn http_tls_flags() { - let dir = TempDir::new().expect("Unable to create temporary directory"); CommandLineTest::new() + .flag("http", None) .flag("http-enable-tls", None) - .flag( - "http-tls-cert", - dir.path().join("certificate.crt").as_os_str().to_str(), - ) - .flag( - "http-tls-key", - dir.path().join("private.key").as_os_str().to_str(), - ) + .flag("http-tls-cert", Some("tests/tls/cert.pem")) + .flag("http-tls-key", Some("tests/tls/key.rsa")) .run_with_zero_port() .with_config(|config| { let tls_config = config @@ -1516,14 +1520,15 @@ fn http_tls_flags() { .tls_config .as_ref() .expect("tls_config was empty."); - assert_eq!(tls_config.cert, dir.path().join("certificate.crt")); - assert_eq!(tls_config.key, dir.path().join("private.key")); + assert_eq!(tls_config.cert, Path::new("tests/tls/cert.pem")); + assert_eq!(tls_config.key, Path::new("tests/tls/key.rsa")); }); } #[test] fn http_spec_fork_default() { CommandLineTest::new() + .flag("http", None) .run_with_zero_port() .with_config(|config| assert_eq!(config.http_api.spec_fork_name, None)); } @@ -1531,6 +1536,7 @@ fn http_spec_fork_default() { #[test] fn http_spec_fork_override() { CommandLineTest::new() + .flag("http", None) .flag("http-spec-fork", Some("altair")) .run_with_zero_port() .with_config(|config| assert_eq!(config.http_api.spec_fork_name, Some(ForkName::Altair))); diff --git a/lighthouse/tests/tls/cert.pem b/lighthouse/tests/tls/cert.pem new file mode 100644 index 00000000000..03af12ff819 --- /dev/null +++ b/lighthouse/tests/tls/cert.pem @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIIEADCCAmigAwIBAgICAcgwDQYJKoZIhvcNAQELBQAwLDEqMCgGA1UEAwwhcG9u +eXRvd24gUlNBIGxldmVsIDIgaW50ZXJtZWRpYXRlMB4XDTE2MDgxMzE2MDcwNFoX +DTIyMDIwMzE2MDcwNFowGTEXMBUGA1UEAwwOdGVzdHNlcnZlci5jb20wggEiMA0G +CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCpVhh1/FNP2qvWenbZSghari/UThwe +dynfnHG7gc3JmygkEdErWBO/CHzHgsx7biVE5b8sZYNEDKFojyoPHGWK2bQM/FTy +niJCgNCLdn6hUqqxLAml3cxGW77hAWu94THDGB1qFe+eFiAUnDmob8gNZtAzT6Ky +b/JGJdrEU0wj+Rd7wUb4kpLInNH/Jc+oz2ii2AjNbGOZXnRz7h7Kv3sO9vABByYe +LcCj3qnhejHMqVhbAT1MD6zQ2+YKBjE52MsQKU/xhUpu9KkUyLh0cxkh3zrFiKh4 +Vuvtc+n7aeOv2jJmOl1dr0XLlSHBlmoKqH6dCTSbddQLmlK7dms8vE01AgMBAAGj +gb4wgbswDAYDVR0TAQH/BAIwADALBgNVHQ8EBAMCBsAwHQYDVR0OBBYEFMeUzGYV +bXwJNQVbY1+A8YXYZY8pMEIGA1UdIwQ7MDmAFJvEsUi7+D8vp8xcWvnEdVBGkpoW +oR6kHDAaMRgwFgYDVQQDDA9wb255dG93biBSU0EgQ0GCAXswOwYDVR0RBDQwMoIO +dGVzdHNlcnZlci5jb22CFXNlY29uZC50ZXN0c2VydmVyLmNvbYIJbG9jYWxob3N0 +MA0GCSqGSIb3DQEBCwUAA4IBgQBsk5ivAaRAcNgjc7LEiWXFkMg703AqDDNx7kB1 +RDgLalLvrjOfOp2jsDfST7N1tKLBSQ9bMw9X4Jve+j7XXRUthcwuoYTeeo+Cy0/T +1Q78ctoX74E2nB958zwmtRykGrgE/6JAJDwGcgpY9kBPycGxTlCN926uGxHsDwVs +98cL6ZXptMLTR6T2XP36dAJZuOICSqmCSbFR8knc/gjUO36rXTxhwci8iDbmEVaf +BHpgBXGU5+SQ+QM++v6bHGf4LNQC5NZ4e4xvGax8ioYu/BRsB/T3Lx+RlItz4zdU +XuxCNcm3nhQV2ZHquRdbSdoyIxV5kJXel4wCmOhWIq7A2OBKdu5fQzIAzzLi65EN +RPAKsKB4h7hGgvciZQ7dsMrlGw0DLdJ6UrFyiR5Io7dXYT/+JP91lP5xsl6Lhg9O +FgALt7GSYRm2cZdgi9pO9rRr83Br1VjQT1vHz6yoZMXSqc4A2zcN2a2ZVq//rHvc +FZygs8miAhWPzqnpmgTj1cPiU1M= +-----END CERTIFICATE----- diff --git a/lighthouse/tests/tls/key.rsa b/lighthouse/tests/tls/key.rsa new file mode 100644 index 00000000000..b13bf5d07f9 --- /dev/null +++ b/lighthouse/tests/tls/key.rsa @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAqVYYdfxTT9qr1np22UoIWq4v1E4cHncp35xxu4HNyZsoJBHR +K1gTvwh8x4LMe24lROW/LGWDRAyhaI8qDxxlitm0DPxU8p4iQoDQi3Z+oVKqsSwJ +pd3MRlu+4QFrveExwxgdahXvnhYgFJw5qG/IDWbQM0+ism/yRiXaxFNMI/kXe8FG ++JKSyJzR/yXPqM9ootgIzWxjmV50c+4eyr97DvbwAQcmHi3Ao96p4XoxzKlYWwE9 +TA+s0NvmCgYxOdjLEClP8YVKbvSpFMi4dHMZId86xYioeFbr7XPp+2njr9oyZjpd +Xa9Fy5UhwZZqCqh+nQk0m3XUC5pSu3ZrPLxNNQIDAQABAoIBAFKtZJgGsK6md4vq +kyiYSufrcBLaaEQ/rkQtYCJKyC0NAlZKFLRy9oEpJbNLm4cQSkYPXn3Qunx5Jj2k +2MYz+SgIDy7f7KHgr52Ew020dzNQ52JFvBgt6NTZaqL1TKOS1fcJSSNIvouTBerK +NCSXHzfb4P+MfEVe/w1c4ilE+kH9SzdEo2jK/sRbzHIY8TX0JbmQ4SCLLayr22YG +usIxtIYcWt3MMP/G2luRnYzzBCje5MXdpAhlHLi4TB6x4h5PmBKYc57uOVNngKLd +YyrQKcszW4Nx5v0a4HG3A5EtUXNCco1+5asXOg2lYphQYVh2R+1wgu5WiDjDVu+6 +EYgjFSkCgYEA0NBk6FDoxE/4L/4iJ4zIhu9BptN8Je/uS5c6wRejNC/VqQyw7SHb +hRFNrXPvq5Y+2bI/DxtdzZLKAMXOMjDjj0XEgfOIn2aveOo3uE7zf1i+njxwQhPu +uSYA9AlBZiKGr2PCYSDPnViHOspVJjxRuAgyWM1Qf+CTC0D95aj0oz8CgYEAz5n4 +Cb3/WfUHxMJLljJ7PlVmlQpF5Hk3AOR9+vtqTtdxRjuxW6DH2uAHBDdC3OgppUN4 +CFj55kzc2HUuiHtmPtx8mK6G+otT7Lww+nLSFL4PvZ6CYxqcio5MPnoYd+pCxrXY +JFo2W7e4FkBOxb5PF5So5plg+d0z/QiA7aFP1osCgYEAtgi1rwC5qkm8prn4tFm6 +hkcVCIXc+IWNS0Bu693bXKdGr7RsmIynff1zpf4ntYGpEMaeymClCY0ppDrMYlzU +RBYiFNdlBvDRj6s/H+FTzHRk2DT/99rAhY9nzVY0OQFoQIXK8jlURGrkmI/CYy66 +XqBmo5t4zcHM7kaeEBOWEKkCgYAYnO6VaRtPNQfYwhhoFFAcUc+5t+AVeHGW/4AY +M5qlAlIBu64JaQSI5KqwS0T4H+ZgG6Gti68FKPO+DhaYQ9kZdtam23pRVhd7J8y+ +xMI3h1kiaBqZWVxZ6QkNFzizbui/2mtn0/JB6YQ/zxwHwcpqx0tHG8Qtm5ZAV7PB +eLCYhQKBgQDALJxU/6hMTdytEU5CLOBSMby45YD/RrfQrl2gl/vA0etPrto4RkVq +UrkDO/9W4mZORClN3knxEFSTlYi8YOboxdlynpFfhcs82wFChs+Ydp1eEsVHAqtu +T+uzn0sroycBiBfVB949LExnzGDFUkhG0i2c2InarQYLTsIyHCIDEA== +-----END RSA PRIVATE KEY----- diff --git a/lighthouse/tests/validator_client.rs b/lighthouse/tests/validator_client.rs index 9bcfe2a1d50..062b7e7786a 100644 --- a/lighthouse/tests/validator_client.rs +++ b/lighthouse/tests/validator_client.rs @@ -260,6 +260,7 @@ fn http_flag() { fn http_address_flag() { let addr = "127.0.0.99".parse::().unwrap(); CommandLineTest::new() + .flag("http", None) .flag("http-address", Some("127.0.0.99")) .flag("unencrypted-http-transport", None) .run() @@ -269,6 +270,7 @@ fn http_address_flag() { fn http_address_ipv6_flag() { let addr = "::1".parse::().unwrap(); CommandLineTest::new() + .flag("http", None) .flag("http-address", Some("::1")) .flag("unencrypted-http-transport", None) .run() @@ -279,6 +281,7 @@ fn http_address_ipv6_flag() { fn missing_unencrypted_http_transport_flag() { let addr = "127.0.0.99".parse::().unwrap(); CommandLineTest::new() + .flag("http", None) .flag("http-address", Some("127.0.0.99")) .run() .with_config(|config| assert_eq!(config.http_api.listen_addr, addr)); @@ -286,6 +289,7 @@ fn missing_unencrypted_http_transport_flag() { #[test] fn http_port_flag() { CommandLineTest::new() + .flag("http", None) .flag("http-port", Some("9090")) .run() .with_config(|config| assert_eq!(config.http_api.listen_port, 9090)); @@ -293,6 +297,7 @@ fn http_port_flag() { #[test] fn http_allow_origin_flag() { CommandLineTest::new() + .flag("http", None) .flag("http-allow-origin", Some("http://localhost:9009")) .run() .with_config(|config| { @@ -305,6 +310,7 @@ fn http_allow_origin_flag() { #[test] fn http_allow_origin_all_flag() { CommandLineTest::new() + .flag("http", None) .flag("http-allow-origin", Some("*")) .run() .with_config(|config| assert_eq!(config.http_api.allow_origin, Some("*".to_string()))); @@ -312,12 +318,14 @@ fn http_allow_origin_all_flag() { #[test] fn http_allow_keystore_export_default() { CommandLineTest::new() + .flag("http", None) .run() .with_config(|config| assert!(!config.http_api.allow_keystore_export)); } #[test] fn http_allow_keystore_export_present() { CommandLineTest::new() + .flag("http", None) .flag("http-allow-keystore-export", None) .run() .with_config(|config| assert!(config.http_api.allow_keystore_export)); @@ -325,12 +333,14 @@ fn http_allow_keystore_export_present() { #[test] fn http_store_keystore_passwords_in_secrets_dir_default() { CommandLineTest::new() + .flag("http", None) .run() .with_config(|config| assert!(!config.http_api.store_passwords_in_secrets_dir)); } #[test] fn http_store_keystore_passwords_in_secrets_dir_present() { CommandLineTest::new() + .flag("http", None) .flag("http-store-passwords-in-secrets-dir", None) .run() .with_config(|config| assert!(config.http_api.store_passwords_in_secrets_dir)); @@ -348,6 +358,7 @@ fn metrics_flag() { fn metrics_address_flag() { let addr = "127.0.0.99".parse::().unwrap(); CommandLineTest::new() + .flag("metrics", None) .flag("metrics-address", Some("127.0.0.99")) .run() .with_config(|config| assert_eq!(config.http_metrics.listen_addr, addr)); @@ -356,6 +367,7 @@ fn metrics_address_flag() { fn metrics_address_ipv6_flag() { let addr = "::1".parse::().unwrap(); CommandLineTest::new() + .flag("metrics", None) .flag("metrics-address", Some("::1")) .run() .with_config(|config| assert_eq!(config.http_metrics.listen_addr, addr)); @@ -363,6 +375,7 @@ fn metrics_address_ipv6_flag() { #[test] fn metrics_port_flag() { CommandLineTest::new() + .flag("metrics", None) .flag("metrics-port", Some("9090")) .run() .with_config(|config| assert_eq!(config.http_metrics.listen_port, 9090)); @@ -370,6 +383,7 @@ fn metrics_port_flag() { #[test] fn metrics_allow_origin_flag() { CommandLineTest::new() + .flag("metrics", None) .flag("metrics-allow-origin", Some("http://localhost:9009")) .run() .with_config(|config| { @@ -382,6 +396,7 @@ fn metrics_allow_origin_flag() { #[test] fn metrics_allow_origin_all_flag() { CommandLineTest::new() + .flag("metrics", None) .flag("metrics-allow-origin", Some("*")) .run() .with_config(|config| assert_eq!(config.http_metrics.allow_origin, Some("*".to_string()))); diff --git a/testing/web3signer_tests/Cargo.toml b/testing/web3signer_tests/Cargo.toml index c0fbf667236..faad76a19c3 100644 --- a/testing/web3signer_tests/Cargo.toml +++ b/testing/web3signer_tests/Cargo.toml @@ -26,6 +26,6 @@ serde_derive = "1.0.116" serde_yaml = "0.8.13" eth2_network_config = { path = "../../common/eth2_network_config" } serde_json = "1.0.58" -zip = "0.5.13" +zip = "0.6" lazy_static = "1.4.0" -parking_lot = "0.12.0" \ No newline at end of file +parking_lot = "0.12.0" diff --git a/watch/Cargo.toml b/watch/Cargo.toml index 23e2c566dc1..b29df14439e 100644 --- a/watch/Cargo.toml +++ b/watch/Cargo.toml @@ -41,6 +41,7 @@ tokio-postgres = "0.7.5" http_api = { path = "../beacon_node/http_api" } beacon_chain = { path = "../beacon_node/beacon_chain" } network = { path = "../beacon_node/network" } -testcontainers = "0.14.0" +# TODO: update to 0.15 when released: https://github.com/testcontainers/testcontainers-rs/issues/497 +testcontainers = { git = "https://github.com/testcontainers/testcontainers-rs/", rev = "0f2c9851" } unused_port = { path = "../common/unused_port" } task_executor = { path = "../common/task_executor" } diff --git a/watch/tests/tests.rs b/watch/tests/tests.rs index 9032b124ab8..a54386eb753 100644 --- a/watch/tests/tests.rs +++ b/watch/tests/tests.rs @@ -23,6 +23,7 @@ use watch::{ }; use log::error; +use std::collections::HashMap; use std::env; use std::net::SocketAddr; use std::time::Duration; @@ -30,7 +31,42 @@ use tokio::{runtime, task::JoinHandle}; use tokio_postgres::{config::Config as PostgresConfig, Client, NoTls}; use unused_port::unused_tcp4_port; -use testcontainers::{clients::Cli, images::postgres::Postgres, RunnableImage}; +use testcontainers::{clients::Cli, core::WaitFor, Image, RunnableImage}; + +#[derive(Debug)] +pub struct Postgres(HashMap); + +impl Default for Postgres { + fn default() -> Self { + let mut env_vars = HashMap::new(); + env_vars.insert("POSTGRES_DB".to_owned(), "postgres".to_owned()); + env_vars.insert("POSTGRES_HOST_AUTH_METHOD".into(), "trust".into()); + + Self(env_vars) + } +} + +impl Image for Postgres { + type Args = (); + + fn name(&self) -> String { + "postgres".to_owned() + } + + fn tag(&self) -> String { + "11-alpine".to_owned() + } + + fn ready_conditions(&self) -> Vec { + vec![WaitFor::message_on_stderr( + "database system is ready to accept connections", + )] + } + + fn env_vars(&self) -> Box + '_> { + Box::new(self.0.iter()) + } +} type E = MainnetEthSpec; From 8e95b69a1a9a449f352ec7a3904c549accb75111 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 28 Aug 2023 00:55:31 +0000 Subject: [PATCH 02/17] Send success code for duplicate blocks on HTTP (#4655) ## Issue Addressed Closes #4473 (take 3) ## Proposed Changes - Send a 202 status code by default for duplicate blocks, instead of 400. This conveys to the caller that the block was published, but makes no guarantees about its validity. Block relays can count this as a success or a failure as they wish. - For users wanting finer-grained control over which status is returned for duplicates, a flag `--http-duplicate-block-status` can be used to adjust the behaviour. A 400 status can be supplied to restore the old (spec-compliant) behaviour, or a 200 status can be used to silence VCs that warn loudly for non-200 codes (e.g. Lighthouse prior to v4.4.0). - Update the Lighthouse VC to gracefully handle success codes other than 200. The info message isn't the nicest thing to read, but it covers all bases and isn't a nasty `ERRO`/`CRIT` that will wake anyone up. ## Additional Info I'm planning to raise a PR to `beacon-APIs` to specify that clients may return 202 for duplicate blocks. Really it would be nice to use some 2xx code that _isn't_ the same as the code for "published but invalid". I think unfortunately there aren't any suitable codes, and maybe the best fit is `409 CONFLICT`. Given that we need to fix this promptly for our release, I think using the 202 code temporarily with configuration strikes a nice compromise. --- beacon_node/http_api/src/lib.rs | 185 ++++++++---------- beacon_node/http_api/src/publish_blocks.rs | 47 +++-- beacon_node/http_api/src/task_spawner.rs | 40 ---- beacon_node/http_api/src/test_utils.rs | 10 +- .../tests/broadcast_validation_tests.rs | 9 +- beacon_node/http_api/tests/tests.rs | 67 ++++++- beacon_node/src/cli.rs | 9 + beacon_node/src/config.rs | 3 + common/eth2/src/types.rs | 20 ++ lighthouse/tests/beacon_node.rs | 19 ++ validator_client/src/block_service.rs | 43 ++-- 11 files changed, 271 insertions(+), 181 deletions(-) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 3aa10139b0b..19d62c060d9 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -139,6 +139,8 @@ pub struct Config { pub data_dir: PathBuf, pub sse_capacity_multiplier: usize, pub enable_beacon_processor: bool, + #[serde(with = "eth2::types::serde_status_code")] + pub duplicate_block_status_code: StatusCode, } impl Default for Config { @@ -154,6 +156,7 @@ impl Default for Config { data_dir: PathBuf::from(DEFAULT_ROOT_DIR), sse_capacity_multiplier: 1, enable_beacon_processor: true, + duplicate_block_status_code: StatusCode::ACCEPTED, } } } @@ -510,6 +513,8 @@ pub fn serve( let task_spawner_filter = warp::any().map(move || TaskSpawner::new(beacon_processor_send.clone())); + let duplicate_block_status_code = ctx.config.duplicate_block_status_code; + /* * * Start of HTTP method definitions. @@ -1284,11 +1289,11 @@ pub fn serve( .and(network_tx_filter.clone()) .and(log_filter.clone()) .then( - |block: Arc>, - task_spawner: TaskSpawner, - chain: Arc>, - network_tx: UnboundedSender>, - log: Logger| { + move |block: Arc>, + task_spawner: TaskSpawner, + chain: Arc>, + network_tx: UnboundedSender>, + log: Logger| { task_spawner.spawn_async_with_rejection(Priority::P0, async move { publish_blocks::publish_block( None, @@ -1297,9 +1302,9 @@ pub fn serve( &network_tx, log, BroadcastValidation::default(), + duplicate_block_status_code, ) .await - .map(|()| warp::reply().into_response()) }) }, ); @@ -1314,11 +1319,11 @@ pub fn serve( .and(network_tx_filter.clone()) .and(log_filter.clone()) .then( - |block_bytes: Bytes, - task_spawner: TaskSpawner, - chain: Arc>, - network_tx: UnboundedSender>, - log: Logger| { + move |block_bytes: Bytes, + task_spawner: TaskSpawner, + chain: Arc>, + network_tx: UnboundedSender>, + log: Logger| { task_spawner.spawn_async_with_rejection(Priority::P0, async move { let block = SignedBeaconBlock::::from_ssz_bytes(&block_bytes, &chain.spec) @@ -1334,9 +1339,9 @@ pub fn serve( &network_tx, log, BroadcastValidation::default(), + duplicate_block_status_code, ) .await - .map(|()| warp::reply().into_response()) }) }, ); @@ -1352,12 +1357,12 @@ pub fn serve( .and(network_tx_filter.clone()) .and(log_filter.clone()) .then( - |validation_level: api_types::BroadcastValidationQuery, - block: Arc>, - task_spawner: TaskSpawner, - chain: Arc>, - network_tx: UnboundedSender>, - log: Logger| { + move |validation_level: api_types::BroadcastValidationQuery, + block: Arc>, + task_spawner: TaskSpawner, + chain: Arc>, + network_tx: UnboundedSender>, + log: Logger| { task_spawner.spawn_async_with_rejection(Priority::P0, async move { publish_blocks::publish_block( None, @@ -1366,9 +1371,9 @@ pub fn serve( &network_tx, log, validation_level.broadcast_validation, + duplicate_block_status_code, ) .await - .map(|()| warp::reply().into_response()) }) }, ); @@ -1384,12 +1389,12 @@ pub fn serve( .and(network_tx_filter.clone()) .and(log_filter.clone()) .then( - |validation_level: api_types::BroadcastValidationQuery, - block_bytes: Bytes, - task_spawner: TaskSpawner, - chain: Arc>, - network_tx: UnboundedSender>, - log: Logger| { + move |validation_level: api_types::BroadcastValidationQuery, + block_bytes: Bytes, + task_spawner: TaskSpawner, + chain: Arc>, + network_tx: UnboundedSender>, + log: Logger| { task_spawner.spawn_async_with_rejection(Priority::P0, async move { let block = SignedBeaconBlock::::from_ssz_bytes(&block_bytes, &chain.spec) @@ -1405,9 +1410,9 @@ pub fn serve( &network_tx, log, validation_level.broadcast_validation, + duplicate_block_status_code, ) .await - .map(|()| warp::reply().into_response()) }) }, ); @@ -1427,11 +1432,11 @@ pub fn serve( .and(network_tx_filter.clone()) .and(log_filter.clone()) .then( - |block: SignedBeaconBlock>, - task_spawner: TaskSpawner, - chain: Arc>, - network_tx: UnboundedSender>, - log: Logger| { + move |block: SignedBlindedBeaconBlock, + task_spawner: TaskSpawner, + chain: Arc>, + network_tx: UnboundedSender>, + log: Logger| { task_spawner.spawn_async_with_rejection(Priority::P0, async move { publish_blocks::publish_blinded_block( block, @@ -1439,9 +1444,9 @@ pub fn serve( &network_tx, log, BroadcastValidation::default(), + duplicate_block_status_code, ) .await - .map(|()| warp::reply().into_response()) }) }, ); @@ -1457,13 +1462,13 @@ pub fn serve( .and(network_tx_filter.clone()) .and(log_filter.clone()) .then( - |block_bytes: Bytes, - task_spawner: TaskSpawner, - chain: Arc>, - network_tx: UnboundedSender>, - log: Logger| { + move |block_bytes: Bytes, + task_spawner: TaskSpawner, + chain: Arc>, + network_tx: UnboundedSender>, + log: Logger| { task_spawner.spawn_async_with_rejection(Priority::P0, async move { - let block = SignedBeaconBlock::>::from_ssz_bytes( + let block = SignedBlindedBeaconBlock::::from_ssz_bytes( &block_bytes, &chain.spec, ) @@ -1476,9 +1481,9 @@ pub fn serve( &network_tx, log, BroadcastValidation::default(), + duplicate_block_status_code, ) .await - .map(|()| warp::reply().into_response()) }) }, ); @@ -1494,87 +1499,63 @@ pub fn serve( .and(network_tx_filter.clone()) .and(log_filter.clone()) .then( - |validation_level: api_types::BroadcastValidationQuery, - block: SignedBeaconBlock>, - task_spawner: TaskSpawner, - chain: Arc>, - network_tx: UnboundedSender>, - log: Logger| { - task_spawner.spawn_async(Priority::P0, async move { - match publish_blocks::publish_blinded_block( + move |validation_level: api_types::BroadcastValidationQuery, + block: SignedBlindedBeaconBlock, + task_spawner: TaskSpawner, + chain: Arc>, + network_tx: UnboundedSender>, + log: Logger| { + task_spawner.spawn_async_with_rejection(Priority::P0, async move { + publish_blocks::publish_blinded_block( block, chain, &network_tx, log, validation_level.broadcast_validation, + duplicate_block_status_code, ) .await - { - Ok(()) => warp::reply().into_response(), - Err(e) => match warp_utils::reject::handle_rejection(e).await { - Ok(reply) => reply.into_response(), - Err(_) => warp::reply::with_status( - StatusCode::INTERNAL_SERVER_ERROR, - eth2::StatusCode::INTERNAL_SERVER_ERROR, - ) - .into_response(), - }, - } }) }, ); - let post_beacon_blinded_blocks_v2_ssz = - eth_v2 - .and(warp::path("beacon")) - .and(warp::path("blinded_blocks")) - .and(warp::query::()) - .and(warp::path::end()) - .and(warp::body::bytes()) - .and(chain_filter.clone()) - .and(network_tx_filter.clone()) - .and(log_filter.clone()) - .then( - |validation_level: api_types::BroadcastValidationQuery, - block_bytes: Bytes, - chain: Arc>, - network_tx: UnboundedSender>, - log: Logger| async move { - let block = - match SignedBeaconBlock::>::from_ssz_bytes( - &block_bytes, - &chain.spec, - ) { - Ok(data) => data, - Err(_) => { - return warp::reply::with_status( - StatusCode::BAD_REQUEST, - eth2::StatusCode::BAD_REQUEST, - ) - .into_response(); - } - }; - match publish_blocks::publish_blinded_block( + let post_beacon_blinded_blocks_v2_ssz = eth_v2 + .and(warp::path("beacon")) + .and(warp::path("blinded_blocks")) + .and(warp::query::()) + .and(warp::path::end()) + .and(warp::body::bytes()) + .and(task_spawner_filter.clone()) + .and(chain_filter.clone()) + .and(network_tx_filter.clone()) + .and(log_filter.clone()) + .then( + move |validation_level: api_types::BroadcastValidationQuery, + block_bytes: Bytes, + task_spawner: TaskSpawner, + chain: Arc>, + network_tx: UnboundedSender>, + log: Logger| { + task_spawner.spawn_async_with_rejection(Priority::P0, async move { + let block = SignedBlindedBeaconBlock::::from_ssz_bytes( + &block_bytes, + &chain.spec, + ) + .map_err(|e| { + warp_utils::reject::custom_bad_request(format!("invalid SSZ: {e:?}")) + })?; + publish_blocks::publish_blinded_block( block, chain, &network_tx, log, validation_level.broadcast_validation, + duplicate_block_status_code, ) .await - { - Ok(()) => warp::reply().into_response(), - Err(e) => match warp_utils::reject::handle_rejection(e).await { - Ok(reply) => reply.into_response(), - Err(_) => warp::reply::with_status( - StatusCode::INTERNAL_SERVER_ERROR, - eth2::StatusCode::INTERNAL_SERVER_ERROR, - ) - .into_response(), - }, - } - }, - ); + }) + }, + ); let block_id_or_err = warp::path::param::().or_else(|_| async { Err(warp_utils::reject::custom_bad_request( diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 0f2f7b361c7..58524f06981 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -4,7 +4,7 @@ use beacon_chain::{ BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, IntoGossipVerifiedBlock, NotifyExecutionLayer, }; -use eth2::types::BroadcastValidation; +use eth2::types::{BroadcastValidation, ErrorMessage}; use execution_layer::ProvenancedPayload; use lighthouse_network::PubsubMessage; use network::NetworkMessage; @@ -19,7 +19,8 @@ use types::{ AbstractExecPayload, BeaconBlockRef, BlindedPayload, EthSpec, ExecPayload, ExecutionBlockHash, FullPayload, Hash256, SignedBeaconBlock, }; -use warp::Rejection; +use warp::http::StatusCode; +use warp::{reply::Response, Rejection, Reply}; pub enum ProvenancedBlock> { /// The payload was built using a local EE. @@ -47,7 +48,8 @@ pub async fn publish_block>( network_tx: &UnboundedSender>, log: Logger, validation_level: BroadcastValidation, -) -> Result<(), Rejection> { + duplicate_status_code: StatusCode, +) -> Result { let seen_timestamp = timestamp_now(); let (block, is_locally_built_block) = match provenanced_block { ProvenancedBlock::Local(block, _) => (block, true), @@ -75,10 +77,30 @@ pub async fn publish_block>( }; /* if we can form a `GossipVerifiedBlock`, we've passed our basic gossip checks */ - let gossip_verified_block = block.into_gossip_verified_block(&chain).map_err(|e| { - warn!(log, "Not publishing block, not gossip verified"; "slot" => beacon_block.slot(), "error" => ?e); - warp_utils::reject::custom_bad_request(e.to_string()) - })?; + let gossip_verified_block = match block.into_gossip_verified_block(&chain) { + Ok(b) => b, + Err(BlockError::BlockIsAlreadyKnown) => { + // Allow the status code for duplicate blocks to be overridden based on config. + return Ok(warp::reply::with_status( + warp::reply::json(&ErrorMessage { + code: duplicate_status_code.as_u16(), + message: "duplicate block".to_string(), + stacktraces: vec![], + }), + duplicate_status_code, + ) + .into_response()); + } + Err(e) => { + warn!( + log, + "Not publishing block - not gossip verified"; + "slot" => beacon_block.slot(), + "error" => ?e + ); + return Err(warp_utils::reject::custom_bad_request(e.to_string())); + } + }; let block_root = block_root.unwrap_or(gossip_verified_block.block_root); @@ -167,8 +189,7 @@ pub async fn publish_block>( &log, ) } - - Ok(()) + Ok(warp::reply().into_response()) } Err(BlockError::BeaconChainError(BeaconChainError::UnableToPublish)) => { Err(warp_utils::reject::custom_server_error( @@ -178,10 +199,6 @@ pub async fn publish_block>( Err(BlockError::Slashable) => Err(warp_utils::reject::custom_bad_request( "proposal for this slot and proposer has already been seen".to_string(), )), - Err(BlockError::BlockIsAlreadyKnown) => { - info!(log, "Block from HTTP API already known"; "block" => ?block_root); - Ok(()) - } Err(e) => { if let BroadcastValidation::Gossip = validation_level { Err(warp_utils::reject::broadcast_without_import(format!("{e}"))) @@ -208,7 +225,8 @@ pub async fn publish_blinded_block( network_tx: &UnboundedSender>, log: Logger, validation_level: BroadcastValidation, -) -> Result<(), Rejection> { + duplicate_status_code: StatusCode, +) -> Result { let block_root = block.canonical_root(); let full_block: ProvenancedBlock>> = reconstruct_block(chain.clone(), block_root, block, log.clone()).await?; @@ -219,6 +237,7 @@ pub async fn publish_blinded_block( network_tx, log, validation_level, + duplicate_status_code, ) .await } diff --git a/beacon_node/http_api/src/task_spawner.rs b/beacon_node/http_api/src/task_spawner.rs index 503faff717b..8768e057dac 100644 --- a/beacon_node/http_api/src/task_spawner.rs +++ b/beacon_node/http_api/src/task_spawner.rs @@ -159,46 +159,6 @@ impl TaskSpawner { .and_then(|x| x) } } - - /// Executes an async task which always returns a `Response`. - pub async fn spawn_async( - self, - priority: Priority, - func: impl Future + Send + Sync + 'static, - ) -> Response { - if let Some(beacon_processor_send) = &self.beacon_processor_send { - // Create a wrapper future that will execute `func` and send the - // result to a channel held by this thread. - let (tx, rx) = oneshot::channel(); - let process_fn = async move { - // Await the future, collect the return value. - let func_result = func.await; - // Send the result down the channel. Ignore any failures; the - // send can only fail if the receiver is dropped. - let _ = tx.send(func_result); - }; - - // Send the function to the beacon processor for execution at some arbitrary time. - let result = send_to_beacon_processor( - beacon_processor_send, - priority, - BlockingOrAsync::Async(Box::pin(process_fn)), - rx, - ) - .await; - convert_rejection(result).await - } else { - // There is no beacon processor so spawn a task directly on the - // tokio executor. - tokio::task::spawn(func).await.unwrap_or_else(|e| { - warp::reply::with_status( - warp::reply::json(&format!("Tokio did not execute task: {e:?}")), - eth2::StatusCode::INTERNAL_SERVER_ERROR, - ) - .into_response() - }) - } - } } /// Send a task to the beacon processor and await execution. diff --git a/beacon_node/http_api/src/test_utils.rs b/beacon_node/http_api/src/test_utils.rs index b1478f3f290..33834d58ca0 100644 --- a/beacon_node/http_api/src/test_utils.rs +++ b/beacon_node/http_api/src/test_utils.rs @@ -23,7 +23,7 @@ use network::{NetworkReceivers, NetworkSenders}; use sensitive_url::SensitiveUrl; use slog::Logger; use std::future::Future; -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::net::SocketAddr; use std::sync::Arc; use std::time::Duration; use store::MemoryStore; @@ -220,15 +220,9 @@ pub async fn create_api_server_on_port( let ctx = Arc::new(Context { config: Config { enabled: true, - listen_addr: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), listen_port: port, - allow_origin: None, - tls_config: None, - allow_sync_stalled: false, data_dir: std::path::PathBuf::from(DEFAULT_ROOT_DIR), - spec_fork_name: None, - sse_capacity_multiplier: 1, - enable_beacon_processor: true, + ..Config::default() }, chain: Some(chain), network_senders: Some(network_senders), diff --git a/beacon_node/http_api/tests/broadcast_validation_tests.rs b/beacon_node/http_api/tests/broadcast_validation_tests.rs index 00825890009..96ff37d81af 100644 --- a/beacon_node/http_api/tests/broadcast_validation_tests.rs +++ b/beacon_node/http_api/tests/broadcast_validation_tests.rs @@ -364,13 +364,14 @@ pub async fn consensus_partial_pass_only_consensus() { /* submit `block_b` which should induce equivocation */ let channel = tokio::sync::mpsc::unbounded_channel(); - let publication_result: Result<(), Rejection> = publish_block( + let publication_result = publish_block( None, ProvenancedBlock::local(gossip_block_b.unwrap()), tester.harness.chain.clone(), &channel.0, test_logger, validation_level.unwrap(), + StatusCode::ACCEPTED, ) .await; @@ -641,13 +642,14 @@ pub async fn equivocation_consensus_late_equivocation() { let channel = tokio::sync::mpsc::unbounded_channel(); - let publication_result: Result<(), Rejection> = publish_block( + let publication_result = publish_block( None, ProvenancedBlock::local(gossip_block_b.unwrap()), tester.harness.chain, &channel.0, test_logger, validation_level.unwrap(), + StatusCode::ACCEPTED, ) .await; @@ -1294,12 +1296,13 @@ pub async fn blinded_equivocation_consensus_late_equivocation() { let channel = tokio::sync::mpsc::unbounded_channel(); - let publication_result: Result<(), Rejection> = publish_blinded_block( + let publication_result = publish_blinded_block( block_b, tester.harness.chain, &channel.0, test_logger, validation_level.unwrap(), + StatusCode::ACCEPTED, ) .await; diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 46cc55591c7..adaf1a0f2d4 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -8,7 +8,7 @@ use eth2::{ mixin::{RequestAccept, ResponseForkName, ResponseOptional}, reqwest::RequestBuilder, types::{BlockId as CoreBlockId, ForkChoiceNode, StateId as CoreStateId, *}, - BeaconNodeHttpClient, Error, Timeouts, + BeaconNodeHttpClient, Error, StatusCode, Timeouts, }; use execution_layer::test_utils::TestingBuilder; use execution_layer::test_utils::DEFAULT_BUILDER_THRESHOLD_WEI; @@ -1318,6 +1318,63 @@ impl ApiTester { self } + pub async fn test_post_beacon_blocks_duplicate(self) -> Self { + let block = self + .harness + .make_block( + self.harness.get_current_state(), + self.harness.get_current_slot(), + ) + .await + .0; + + assert!(self.client.post_beacon_blocks(&block).await.is_ok()); + + let blinded_block = block.clone_as_blinded(); + + // Test all the POST methods in sequence, they should all behave the same. + let responses = vec![ + self.client.post_beacon_blocks(&block).await.unwrap_err(), + self.client + .post_beacon_blocks_v2(&block, None) + .await + .unwrap_err(), + self.client + .post_beacon_blocks_ssz(&block) + .await + .unwrap_err(), + self.client + .post_beacon_blocks_v2_ssz(&block, None) + .await + .unwrap_err(), + self.client + .post_beacon_blinded_blocks(&blinded_block) + .await + .unwrap_err(), + self.client + .post_beacon_blinded_blocks_v2(&blinded_block, None) + .await + .unwrap_err(), + self.client + .post_beacon_blinded_blocks_ssz(&blinded_block) + .await + .unwrap_err(), + self.client + .post_beacon_blinded_blocks_v2_ssz(&blinded_block, None) + .await + .unwrap_err(), + ]; + for (i, response) in responses.into_iter().enumerate() { + assert_eq!( + response.status().unwrap(), + StatusCode::ACCEPTED, + "response {i}" + ); + } + + self + } + pub async fn test_beacon_blocks(self) -> Self { for block_id in self.interesting_block_ids() { let expected = block_id @@ -4651,6 +4708,14 @@ async fn post_beacon_blocks_invalid() { .await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn post_beacon_blocks_duplicate() { + ApiTester::new() + .await + .test_post_beacon_blocks_duplicate() + .await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn beacon_pools_post_attestations_valid() { ApiTester::new() diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index 974dabbf0cf..ed25748133f 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -391,6 +391,15 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .help("Multiplier to apply to the length of HTTP server-sent-event (SSE) channels. \ Increasing this value can prevent messages from being dropped.") ) + .arg( + Arg::with_name("http-duplicate-block-status") + .long("http-duplicate-block-status") + .takes_value(true) + .default_value("202") + .value_name("STATUS_CODE") + .help("Status code to send when a block that is already known is POSTed to the \ + HTTP API.") + ) .arg( Arg::with_name("http-enable-beacon-processor") .long("http-enable-beacon-processor") diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index d2e92b48f33..b03a8cb3e5f 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -155,6 +155,9 @@ pub fn get_config( client_config.http_api.enable_beacon_processor = parse_required(cli_args, "http-enable-beacon-processor")?; + client_config.http_api.duplicate_block_status_code = + parse_required(cli_args, "http-duplicate-block-status")?; + if let Some(cache_size) = clap_utils::parse_optional(cli_args, "shuffling-cache-size")? { client_config.chain.shuffling_cache_size = cache_size; } diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 28fd09c09bc..822f8817991 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -1316,6 +1316,26 @@ pub struct BroadcastValidationQuery { pub broadcast_validation: BroadcastValidation, } +pub mod serde_status_code { + use crate::StatusCode; + use serde::{de::Error, Deserialize, Serialize}; + + pub fn serialize(status_code: &StatusCode, ser: S) -> Result + where + S: serde::Serializer, + { + status_code.as_u16().serialize(ser) + } + + pub fn deserialize<'de, D>(de: D) -> Result + where + D: serde::de::Deserializer<'de>, + { + let status_code = u16::deserialize(de)?; + StatusCode::try_from(status_code).map_err(D::Error::custom) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 626be85f197..5069b0261f6 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -2387,3 +2387,22 @@ fn http_sse_capacity_multiplier_override() { .run_with_zero_port() .with_config(|config| assert_eq!(config.http_api.sse_capacity_multiplier, 10)); } + +#[test] +fn http_duplicate_block_status_default() { + CommandLineTest::new() + .run_with_zero_port() + .with_config(|config| { + assert_eq!(config.http_api.duplicate_block_status_code.as_u16(), 202) + }); +} + +#[test] +fn http_duplicate_block_status_override() { + CommandLineTest::new() + .flag("http-duplicate-block-status", Some("301")) + .run_with_zero_port() + .with_config(|config| { + assert_eq!(config.http_api.duplicate_block_status_code.as_u16(), 301) + }); +} diff --git a/validator_client/src/block_service.rs b/validator_client/src/block_service.rs index 2a09455b6ff..094b85bf810 100644 --- a/validator_client/src/block_service.rs +++ b/validator_client/src/block_service.rs @@ -10,7 +10,8 @@ use crate::{ validator_store::{Error as ValidatorStoreError, ValidatorStore}, }; use environment::RuntimeContext; -use eth2::BeaconNodeHttpClient; +use eth2::{BeaconNodeHttpClient, StatusCode}; +use slog::Logger; use slog::{crit, debug, error, info, trace, warn}; use slot_clock::SlotClock; use std::fmt::Debug; @@ -593,12 +594,7 @@ impl BlockService { beacon_node .post_beacon_blocks(&signed_block) .await - .map_err(|e| { - BlockError::Irrecoverable(format!( - "Error from beacon node when publishing block: {:?}", - e - )) - })? + .or_else(|e| handle_block_post_error(e, slot, log))? } BlockType::Blinded => { let _post_timer = metrics::start_timer_vec( @@ -608,12 +604,7 @@ impl BlockService { beacon_node .post_beacon_blinded_blocks(&signed_block) .await - .map_err(|e| { - BlockError::Irrecoverable(format!( - "Error from beacon node when publishing block: {:?}", - e - )) - })? + .or_else(|e| handle_block_post_error(e, slot, log))? } } Ok::<_, BlockError>(()) @@ -634,3 +625,29 @@ impl BlockService { Ok(()) } } + +fn handle_block_post_error(err: eth2::Error, slot: Slot, log: &Logger) -> Result<(), BlockError> { + // Handle non-200 success codes. + if let Some(status) = err.status() { + if status == StatusCode::ACCEPTED { + info!( + log, + "Block is already known to BN or might be invalid"; + "slot" => slot, + "status_code" => status.as_u16(), + ); + return Ok(()); + } else if status.is_success() { + debug!( + log, + "Block published with non-standard success code"; + "slot" => slot, + "status_code" => status.as_u16(), + ); + return Ok(()); + } + } + Err(BlockError::Irrecoverable(format!( + "Error from beacon node when publishing block: {err:?}", + ))) +} From 9c24cd4ad4ab6eaa7d5725d4a300c271a1a2c25e Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Mon, 28 Aug 2023 00:55:32 +0000 Subject: [PATCH 03/17] Do not log slot clock error prior to genesis (#4657) ## Issue Addressed #4654 ## Proposed Changes Only log error if we're unable to read slot clock after genesis. I thought about simply down grading the `error` to a `warn`, but feel like it's still unnecessary noise before genesis, and it would be good to retain error log if we're pass genesis. But I'd be ok with just downgrading the log level, too. --- .../src/subnet_service/attestation_subnets.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/beacon_node/network/src/subnet_service/attestation_subnets.rs b/beacon_node/network/src/subnet_service/attestation_subnets.rs index b4f52df39d3..1cae6299e1c 100644 --- a/beacon_node/network/src/subnet_service/attestation_subnets.rs +++ b/beacon_node/network/src/subnet_service/attestation_subnets.rs @@ -302,9 +302,16 @@ impl AttestationService { /// Gets the long lived subnets the node should be subscribed to during the current epoch and /// the remaining duration for which they remain valid. fn recompute_long_lived_subnets_inner(&mut self) -> Result { - let current_epoch = self.beacon_chain.epoch().map_err( - |e| error!(self.log, "Failed to get the current epoch from clock"; "err" => ?e), - )?; + let current_epoch = self.beacon_chain.epoch().map_err(|e| { + if !self + .beacon_chain + .slot_clock + .is_prior_to_genesis() + .unwrap_or(false) + { + error!(self.log, "Failed to get the current epoch from clock"; "err" => ?e) + } + })?; let (subnets, next_subscription_epoch) = SubnetId::compute_subnets_for_epoch::( self.node_id.raw().into(), From 55e02e7c3f92e46900b6c7df6e0eaab971718027 Mon Sep 17 00:00:00 2001 From: Mac L Date: Mon, 28 Aug 2023 00:55:33 +0000 Subject: [PATCH 04/17] Show `--gui` flag in help text (#4660) ## Issue Addressed N/A ## Proposed Changes Remove the `hidden(true)` modifier on the `--gui` flag so it shows up when running `lighthouse bn --help` ## Additional Info We need to include this now that Siren has had its first stable release. --- beacon_node/src/cli.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index ed25748133f..837625e12a7 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -1134,7 +1134,6 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .arg( Arg::with_name("gui") .long("gui") - .hidden(true) .help("Enable the graphical user interface and all its requirements. \ This enables --http and --validator-monitor-auto and enables SSE logging.") .takes_value(false) From e056c279aa7f476baeaad6690ac86001d42af597 Mon Sep 17 00:00:00 2001 From: Mac L Date: Mon, 28 Aug 2023 00:55:34 +0000 Subject: [PATCH 05/17] Increase `web3signer_tests` timeouts (#4662) ## Issue Addressed `web3signer_tests` can sometimes timeout. ## Proposed Changes Increase the `web3signer_tests` timeout from 20s to 30s ## Additional Info Previously I believed the consistent CI failures were due to this, but it ended up being something different. See below: --- The timing of this makes it very likely it is related to the [latest release of `web3-signer`](https://github.com/Consensys/web3signer/releases/tag/23.8.1). I now believe this is due to an out of date Java runtime on our runners. A newer version of Java became a requirement with the new `web3-signer` release. However, I was getting timeouts locally, which implies that the margin before timeout is quite small at 20s so bumping it up to 30s could be a good idea regardless. --- testing/web3signer_tests/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/web3signer_tests/src/lib.rs b/testing/web3signer_tests/src/lib.rs index dd17ae23b15..463de0c8b3a 100644 --- a/testing/web3signer_tests/src/lib.rs +++ b/testing/web3signer_tests/src/lib.rs @@ -51,7 +51,7 @@ mod tests { /// If the we are unable to reach the Web3Signer HTTP API within this time out then we will /// assume it failed to start. - const UPCHECK_TIMEOUT: Duration = Duration::from_secs(20); + const UPCHECK_TIMEOUT: Duration = Duration::from_secs(30); /// Set to `false` to send the Web3Signer logs to the console during tests. Logs are useful when /// debugging. From d61f5071849723f56c4cdd02e8a4f0c6100d1543 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 28 Aug 2023 05:34:27 +0000 Subject: [PATCH 06/17] Add Holesky (#4653) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Issue Addressed NA ## Proposed Changes Add the Holesky network config as per https://github.com/eth-clients/holesky/tree/36e4ff2d5138dcb2eb614f0f60fdb060b2adc1e2/custom_config_data. Since the genesis state is ~190MB, I've opted to *not* include it in the binary and instead download it at runtime (see #4564 for context). To download this file we have: - A hard-coded URL for a SigP-hosted S3 bucket with the Holesky genesis state. Assuming this download works correctly, users will be none the wiser that the state wasn't included in the binary (apart from some additional logs) - If the user provides a `--checkpoint-sync-url` flag, then LH will download the genesis state from that server rather than our S3 bucket. - If the user provides a `--genesis-state-url` flag, then LH will download the genesis state from that server regardless of the S3 bucket or `--checkpoint-sync-url` flag. - Whenever a genesis state is downloaded it is checked against a checksum baked into the binary. - A genesis state will never be downloaded if it's already included in the binary. - There is a `--genesis-state-url-timeout` flag to tweak the timeout for downloading the genesis state file. ## Log Output Example of log output when a state is downloaded: ```bash Aug 23 05:40:13.424 INFO Logging to file path: "/Users/paul/.lighthouse/holesky/beacon/logs/beacon.log" Aug 23 05:40:13.425 INFO Lighthouse started version: Lighthouse/v4.3.0-bd9931f+ Aug 23 05:40:13.425 INFO Configured for network name: holesky Aug 23 05:40:13.426 INFO Data directory initialised datadir: /Users/paul/.lighthouse/holesky Aug 23 05:40:13.427 INFO Deposit contract address: 0x4242424242424242424242424242424242424242, deploy_block: 0 Aug 23 05:40:13.427 INFO Downloading genesis state info: this may take some time on testnets with large validator counts, timeout: 60s, server: https://sigp-public-genesis-states.s3.ap-southeast-2.amazonaws.com/ Aug 23 05:40:29.895 INFO Starting from known genesis state service: beacon ``` Example of log output when there are no URLs specified: ``` Aug 23 06:29:51.645 INFO Logging to file path: "/Users/paul/.lighthouse/goerli/beacon/logs/beacon.log" Aug 23 06:29:51.646 INFO Lighthouse started version: Lighthouse/v4.3.0-666a39c+ Aug 23 06:29:51.646 INFO Configured for network name: goerli Aug 23 06:29:51.647 INFO Data directory initialised datadir: /Users/paul/.lighthouse/goerli Aug 23 06:29:51.647 INFO Deposit contract address: 0xff50ed3d0ec03ac01d4c79aad74928bff48a7b2b, deploy_block: 4367322 The genesis state is not present in the binary and there are no known download URLs. Please use --checkpoint-sync-url or --genesis-state-url. ``` ## Additional Info I tested the `--genesis-state-url` flag with all 9 Goerli checkpoint sync servers on https://eth-clients.github.io/checkpoint-sync-endpoints/ and they all worked 🎉 My IDE eagerly formatted some `Cargo.toml`. I've disabled it but I don't see the value in spending time reverting the changes that are already there. I also added the `GenesisStateBytes` enum to avoid an unnecessary clone on the genesis state bytes baked into the binary. This is not a huge deal on Mainnet, but will become more relevant when testing with big genesis states. When we do a fresh checkpoint sync we're downloading the genesis state to check the `genesis_validators_root` against the finalised state we receive. This is not *entirely* pointless, since we verify the checksum when we download the genesis state so we are actually guaranteeing that the finalised state is on the same network. There might be a smarter/less-download-y way to go about this, but I've run out of cycles to figure that out. Perhaps we can grab it in the next release? --- Cargo.lock | 8 + account_manager/Cargo.toml | 10 +- account_manager/src/validator/exit.rs | 19 +- .../src/validator/slashing_protection.rs | 24 +- beacon_node/Cargo.toml | 18 +- beacon_node/client/src/builder.rs | 41 ++- beacon_node/client/src/config.rs | 15 +- beacon_node/src/config.rs | 40 ++- boot_node/src/config.rs | 16 +- common/eth2_config/src/lib.rs | 90 ++++-- common/eth2_network_config/Cargo.toml | 13 +- common/eth2_network_config/build.rs | 9 +- .../holesky/boot_enr.yaml | 8 + .../holesky/config.yaml | 117 ++++++++ .../holesky/deploy_block.txt | 1 + common/eth2_network_config/src/lib.rs | 275 ++++++++++++++++-- common/pretty_reqwest_error/src/lib.rs | 6 + lcli/src/eth1_genesis.rs | 2 +- lcli/src/interop_genesis.rs | 2 +- lcli/src/new_testnet.rs | 5 +- lighthouse/src/main.rs | 24 ++ lighthouse/tests/beacon_node.rs | 25 ++ 22 files changed, 651 insertions(+), 117 deletions(-) create mode 100644 common/eth2_network_config/built_in_network_configs/holesky/boot_enr.yaml create mode 100644 common/eth2_network_config/built_in_network_configs/holesky/config.yaml create mode 100644 common/eth2_network_config/built_in_network_configs/holesky/deploy_block.txt diff --git a/Cargo.lock b/Cargo.lock index 56d421e0390..75c15ba7813 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -33,6 +33,7 @@ dependencies = [ "serde", "serde_json", "slashing_protection", + "slog", "slot_clock", "tempfile", "tokio", @@ -2223,9 +2224,16 @@ dependencies = [ "discv5", "eth2_config", "ethereum_ssz", + "logging", + "pretty_reqwest_error", + "reqwest", + "sensitive_url", "serde_yaml", + "sha2 0.10.7", + "slog", "tempfile", "types", + "url", "zip", ] diff --git a/account_manager/Cargo.toml b/account_manager/Cargo.toml index 7d90cbb427d..238e4a77e08 100644 --- a/account_manager/Cargo.toml +++ b/account_manager/Cargo.toml @@ -1,7 +1,10 @@ [package] name = "account_manager" version = "0.3.5" -authors = ["Paul Hauner ", "Luke Anderson "] +authors = [ + "Paul Hauner ", + "Luke Anderson ", +] edition = "2021" [dependencies] @@ -19,13 +22,14 @@ tokio = { version = "1.14.0", features = ["full"] } eth2_keystore = { path = "../crypto/eth2_keystore" } account_utils = { path = "../common/account_utils" } slashing_protection = { path = "../validator_client/slashing_protection" } -eth2 = {path = "../common/eth2"} -safe_arith = {path = "../consensus/safe_arith"} +eth2 = { path = "../common/eth2" } +safe_arith = { path = "../consensus/safe_arith" } slot_clock = { path = "../common/slot_clock" } filesystem = { path = "../common/filesystem" } sensitive_url = { path = "../common/sensitive_url" } serde = { version = "1.0.116", features = ["derive"] } serde_json = "1.0.58" +slog = { version = "2.5.2" } [dev-dependencies] tempfile = "3.1.0" diff --git a/account_manager/src/validator/exit.rs b/account_manager/src/validator/exit.rs index 5755a355f31..1ff61a7c01d 100644 --- a/account_manager/src/validator/exit.rs +++ b/account_manager/src/validator/exit.rs @@ -10,6 +10,7 @@ use eth2_keystore::Keystore; use eth2_network_config::Eth2NetworkConfig; use safe_arith::SafeArith; use sensitive_url::SensitiveUrl; +use slog::Logger; use slot_clock::{SlotClock, SystemTimeSlotClock}; use std::path::{Path, PathBuf}; use std::time::Duration; @@ -78,6 +79,12 @@ pub fn cli_run(matches: &ArgMatches, env: Environment) -> Result< let password_file_path: Option = clap_utils::parse_optional(matches, PASSWORD_FILE_FLAG)?; + let genesis_state_url: Option = + clap_utils::parse_optional(matches, "genesis-state-url")?; + let genesis_state_url_timeout = + clap_utils::parse_required(matches, "genesis-state-url-timeout") + .map(Duration::from_secs)?; + let stdin_inputs = cfg!(windows) || matches.is_present(STDIN_INPUTS_FLAG); let no_wait = matches.is_present(NO_WAIT); let no_confirmation = matches.is_present(NO_CONFIRMATION); @@ -104,6 +111,9 @@ pub fn cli_run(matches: &ArgMatches, env: Environment) -> Result< ð2_network_config, no_wait, no_confirmation, + genesis_state_url, + genesis_state_url_timeout, + env.core_context().log(), ))?; Ok(()) @@ -120,13 +130,14 @@ async fn publish_voluntary_exit( eth2_network_config: &Eth2NetworkConfig, no_wait: bool, no_confirmation: bool, + genesis_state_url: Option, + genesis_state_url_timeout: Duration, + log: &Logger, ) -> Result<(), String> { let genesis_data = get_geneisis_data(client).await?; let testnet_genesis_root = eth2_network_config - .beacon_state::() - .as_ref() - .expect("network should have valid genesis state") - .genesis_validators_root(); + .genesis_validators_root::(genesis_state_url.as_deref(), genesis_state_url_timeout, log)? + .ok_or("Genesis state is unknown")?; // Verify that the beacon node and validator being exited are on the same network. if genesis_data.genesis_validators_root != testnet_genesis_root { diff --git a/account_manager/src/validator/slashing_protection.rs b/account_manager/src/validator/slashing_protection.rs index f25bbd8159f..570f29b4ad6 100644 --- a/account_manager/src/validator/slashing_protection.rs +++ b/account_manager/src/validator/slashing_protection.rs @@ -7,7 +7,8 @@ use slashing_protection::{ use std::fs::File; use std::path::PathBuf; use std::str::FromStr; -use types::{BeaconState, Epoch, EthSpec, PublicKeyBytes, Slot}; +use std::time::Duration; +use types::{Epoch, EthSpec, PublicKeyBytes, Slot}; pub const CMD: &str = "slashing-protection"; pub const IMPORT_CMD: &str = "import"; @@ -82,19 +83,24 @@ pub fn cli_run( ) -> Result<(), String> { let slashing_protection_db_path = validator_base_dir.join(SLASHING_PROTECTION_FILENAME); + let genesis_state_url: Option = + clap_utils::parse_optional(matches, "genesis-state-url")?; + let genesis_state_url_timeout = + clap_utils::parse_required(matches, "genesis-state-url-timeout") + .map(Duration::from_secs)?; + + let context = env.core_context(); let eth2_network_config = env .eth2_network_config .ok_or("Unable to get testnet configuration from the environment")?; let genesis_validators_root = eth2_network_config - .beacon_state::() - .map(|state: BeaconState| state.genesis_validators_root()) - .map_err(|e| { - format!( - "Unable to get genesis state, has genesis occurred? Detail: {:?}", - e - ) - })?; + .genesis_validators_root::( + genesis_state_url.as_deref(), + genesis_state_url_timeout, + context.log(), + )? + .ok_or_else(|| "Unable to get genesis state, has genesis occurred?".to_string())?; match matches.subcommand() { (IMPORT_CMD, Some(matches)) => { diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index f55c724dc39..a0e46705b74 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -1,7 +1,10 @@ [package] name = "beacon_node" version = "4.3.0" -authors = ["Paul Hauner ", "Age Manning ", + "Age Manning { + ClientGenesis::GenesisState => { info!( context.log(), "Starting from known genesis state"; ); - let genesis_state = BeaconState::from_ssz_bytes(&genesis_state_bytes, &spec) - .map_err(|e| format!("Unable to parse genesis state SSZ: {:?}", e))?; + let genesis_state = genesis_state(&runtime_context, &config, log)?; builder.genesis_state(genesis_state).map(|v| (v, None))? } ClientGenesis::WeakSubjSszBytes { anchor_state_bytes, anchor_block_bytes, - genesis_state_bytes, } => { info!(context.log(), "Starting checkpoint sync"); if config.chain.genesis_backfill { @@ -279,17 +276,13 @@ where .map_err(|e| format!("Unable to parse weak subj state SSZ: {:?}", e))?; let anchor_block = SignedBeaconBlock::from_ssz_bytes(&anchor_block_bytes, &spec) .map_err(|e| format!("Unable to parse weak subj block SSZ: {:?}", e))?; - let genesis_state = BeaconState::from_ssz_bytes(&genesis_state_bytes, &spec) - .map_err(|e| format!("Unable to parse genesis state SSZ: {:?}", e))?; + let genesis_state = genesis_state(&runtime_context, &config, log)?; builder .weak_subjectivity_state(anchor_state, anchor_block, genesis_state) .map(|v| (v, None))? } - ClientGenesis::CheckpointSyncUrl { - genesis_state_bytes, - url, - } => { + ClientGenesis::CheckpointSyncUrl { url } => { info!( context.log(), "Starting checkpoint sync"; @@ -384,8 +377,7 @@ where debug!(context.log(), "Downloaded finalized block"); - let genesis_state = BeaconState::from_ssz_bytes(&genesis_state_bytes, &spec) - .map_err(|e| format!("Unable to parse genesis state SSZ: {:?}", e))?; + let genesis_state = genesis_state(&runtime_context, &config, log)?; info!( context.log(), @@ -1089,3 +1081,22 @@ where Ok(self) } } + +/// Obtain the genesis state from the `eth2_network_config` in `context`. +fn genesis_state( + context: &RuntimeContext, + config: &ClientConfig, + log: &Logger, +) -> Result, String> { + let eth2_network_config = context + .eth2_network_config + .as_ref() + .ok_or("An eth2_network_config is required to obtain the genesis state")?; + eth2_network_config + .genesis_state::( + config.genesis_state_url.as_deref(), + config.genesis_state_url_timeout, + log, + )? + .ok_or_else(|| "Genesis state is unknown".to_string()) +} diff --git a/beacon_node/client/src/config.rs b/beacon_node/client/src/config.rs index 283efe9c3fc..adaf0279847 100644 --- a/beacon_node/client/src/config.rs +++ b/beacon_node/client/src/config.rs @@ -7,6 +7,7 @@ use sensitive_url::SensitiveUrl; use serde_derive::{Deserialize, Serialize}; use std::fs; use std::path::PathBuf; +use std::time::Duration; use types::{Graffiti, PublicKeyBytes}; /// Default directory name for the freezer database under the top-level data dir. const DEFAULT_FREEZER_DB_DIR: &str = "freezer_db"; @@ -25,18 +26,13 @@ pub enum ClientGenesis { /// contract. #[default] DepositContract, - /// Loads the genesis state from SSZ-encoded `BeaconState` bytes. - /// - /// We include the bytes instead of the `BeaconState` because the `EthSpec` type - /// parameter would be very annoying. - SszBytes { genesis_state_bytes: Vec }, + /// Loads the genesis state from the genesis state in the `Eth2NetworkConfig`. + GenesisState, WeakSubjSszBytes { - genesis_state_bytes: Vec, anchor_state_bytes: Vec, anchor_block_bytes: Vec, }, CheckpointSyncUrl { - genesis_state_bytes: Vec, url: SensitiveUrl, }, } @@ -81,6 +77,8 @@ pub struct Config { pub slasher: Option, pub logger_config: LoggerConfig, pub beacon_processor: BeaconProcessorConfig, + pub genesis_state_url: Option, + pub genesis_state_url_timeout: Duration, } impl Default for Config { @@ -108,6 +106,9 @@ impl Default for Config { validator_monitor_individual_tracking_threshold: DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD, logger_config: LoggerConfig::default(), beacon_processor: <_>::default(), + genesis_state_url: <_>::default(), + // This default value should always be overwritten by the CLI default value. + genesis_state_url_timeout: Duration::from_secs(60), } } } diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index b03a8cb3e5f..70495777e28 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -471,9 +471,30 @@ pub fn get_config( client_config.chain.checkpoint_sync_url_timeout = clap_utils::parse_required::(cli_args, "checkpoint-sync-url-timeout")?; - client_config.genesis = if let Some(genesis_state_bytes) = - eth2_network_config.genesis_state_bytes.clone() - { + client_config.genesis_state_url_timeout = + clap_utils::parse_required(cli_args, "genesis-state-url-timeout") + .map(Duration::from_secs)?; + + let genesis_state_url_opt = + clap_utils::parse_optional::(cli_args, "genesis-state-url")?; + let checkpoint_sync_url_opt = + clap_utils::parse_optional::(cli_args, "checkpoint-sync-url")?; + + // If the `--genesis-state-url` is defined, use that to download the + // genesis state bytes. If it's not defined, try `--checkpoint-sync-url`. + client_config.genesis_state_url = if let Some(genesis_state_url) = genesis_state_url_opt { + Some(genesis_state_url) + } else if let Some(checkpoint_sync_url) = checkpoint_sync_url_opt { + // If the checkpoint sync URL is going to be used to download the + // genesis state, adopt the timeout from the checkpoint sync URL too. + client_config.genesis_state_url_timeout = + Duration::from_secs(client_config.chain.checkpoint_sync_url_timeout); + Some(checkpoint_sync_url) + } else { + None + }; + + client_config.genesis = if eth2_network_config.genesis_state_is_known() { // Set up weak subjectivity sync, or start from the hardcoded genesis state. if let (Some(initial_state_path), Some(initial_block_path)) = ( cli_args.value_of("checkpoint-state"), @@ -495,7 +516,6 @@ pub fn get_config( let anchor_block_bytes = read(initial_block_path)?; ClientGenesis::WeakSubjSszBytes { - genesis_state_bytes, anchor_state_bytes, anchor_block_bytes, } @@ -503,17 +523,9 @@ pub fn get_config( let url = SensitiveUrl::parse(remote_bn_url) .map_err(|e| format!("Invalid checkpoint sync URL: {:?}", e))?; - ClientGenesis::CheckpointSyncUrl { - genesis_state_bytes, - url, - } + ClientGenesis::CheckpointSyncUrl { url } } else { - // Note: re-serializing the genesis state is not so efficient, however it avoids adding - // trait bounds to the `ClientGenesis` enum. This would have significant flow-on - // effects. - ClientGenesis::SszBytes { - genesis_state_bytes, - } + ClientGenesis::GenesisState } } else { if cli_args.is_present("checkpoint-state") || cli_args.is_present("checkpoint-sync-url") { diff --git a/boot_node/src/config.rs b/boot_node/src/config.rs index d006156bf9d..779269921a5 100644 --- a/boot_node/src/config.rs +++ b/boot_node/src/config.rs @@ -10,6 +10,7 @@ use lighthouse_network::{ use serde_derive::{Deserialize, Serialize}; use ssz::Encode; use std::net::{SocketAddrV4, SocketAddrV6}; +use std::time::Duration; use std::{marker::PhantomData, path::PathBuf}; use types::EthSpec; @@ -90,8 +91,19 @@ impl BootNodeConfig { let enr_fork = { let spec = eth2_network_config.chain_spec::()?; - if eth2_network_config.beacon_state_is_known() { - let genesis_state = eth2_network_config.beacon_state::()?; + let genesis_state_url: Option = + clap_utils::parse_optional(matches, "genesis-state-url")?; + let genesis_state_url_timeout = + clap_utils::parse_required(matches, "genesis-state-url-timeout") + .map(Duration::from_secs)?; + + if eth2_network_config.genesis_state_is_known() { + let genesis_state = eth2_network_config + .genesis_state::(genesis_state_url.as_deref(), genesis_state_url_timeout, &logger)? + .ok_or_else(|| { + "The genesis state for this network is not known, this is an unsupported mode" + .to_string() + })?; slog::info!(logger, "Genesis state found"; "root" => genesis_state.canonical_root().to_string()); let enr_fork = spec.enr_fork_id::( diff --git a/common/eth2_config/src/lib.rs b/common/eth2_config/src/lib.rs index 7e5506667ff..bb14d067569 100644 --- a/common/eth2_config/src/lib.rs +++ b/common/eth2_config/src/lib.rs @@ -23,6 +23,16 @@ pub const PREDEFINED_NETWORKS_DIR: &str = predefined_networks_dir!(); pub const GENESIS_FILE_NAME: &str = "genesis.ssz"; pub const GENESIS_ZIP_FILE_NAME: &str = "genesis.ssz.zip"; +const HOLESKY_GENESIS_STATE_SOURCE: GenesisStateSource = GenesisStateSource::Url { + urls: &[ + // This is an AWS S3 bucket hosted by Sigma Prime. See Paul Hauner for + // more details. + "https://sigp-public-genesis-states.s3.ap-southeast-2.amazonaws.com/holesky/", + ], + checksum: "0x76631cd0b9ddc5b2c766b496e23f16759ce1181446a4efb40e5540cd15b78a07", + genesis_validators_root: "0x9143aa7c615a7f7115e2b6aac319c03529df8242ae705fba9df39b79c59fa8b1", +}; + /// The core configuration of a Lighthouse beacon node. #[derive(Debug, Clone)] pub struct Eth2Config { @@ -62,6 +72,32 @@ impl Eth2Config { } } +/// Describes how a genesis state may be obtained. +#[derive(Copy, Clone, Debug, PartialEq)] +pub enum GenesisStateSource { + /// The genesis state for this network is not yet known. + Unknown, + /// The genesis state for this network is included in the binary via + /// `include_bytes!` or by loading from a testnet dir. + IncludedBytes, + /// The genesis state for this network should be downloaded from a URL. + Url { + /// URLs to try to download the file from, in order. + urls: &'static [&'static str], + /// The SHA256 of the genesis state bytes. This is *not* a hash tree + /// root to simplify the types (i.e., to avoid getting EthSpec + /// involved). + /// + /// The format should be 0x-prefixed ASCII bytes. + checksum: &'static str, + /// The `genesis_validators_root` of the genesis state. Used to avoid + /// downloading the state for simple signing operations. + /// + /// The format should be 0x-prefixed ASCII bytes. + genesis_validators_root: &'static str, + }, +} + /// A directory that can be built by downloading files via HTTP. /// /// Used by the `eth2_network_config` crate to initialize the network directories during build and @@ -70,7 +106,7 @@ impl Eth2Config { pub struct Eth2NetArchiveAndDirectory<'a> { pub name: &'a str, pub config_dir: &'a str, - pub genesis_is_known: bool, + pub genesis_state_source: GenesisStateSource, } impl<'a> Eth2NetArchiveAndDirectory<'a> { @@ -89,15 +125,11 @@ impl<'a> Eth2NetArchiveAndDirectory<'a> { } } -/// Indicates that the `genesis.ssz.zip` file is present on the filesystem. This means that the -/// deposit ceremony has concluded and the final genesis `BeaconState` is known. -const GENESIS_STATE_IS_KNOWN: bool = true; - -#[derive(Copy, Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub struct HardcodedNet { pub name: &'static str, pub config_dir: &'static str, - pub genesis_is_known: bool, + pub genesis_state_source: GenesisStateSource, pub config: &'static [u8], pub deploy_block: &'static [u8], pub boot_enr: &'static [u8], @@ -109,7 +141,7 @@ pub struct HardcodedNet { /// It also defines a `include__file!` macro which provides a wrapper around /// `std::include_bytes`, allowing the inclusion of bytes from the specific testnet directory. macro_rules! define_archive { - ($name_ident: ident, $config_dir: tt, $genesis_is_known: ident) => { + ($name_ident: ident, $config_dir: tt, $genesis_state_source: path) => { paste! { #[macro_use] pub mod $name_ident { @@ -118,7 +150,7 @@ macro_rules! define_archive { pub const ETH2_NET_DIR: Eth2NetArchiveAndDirectory = Eth2NetArchiveAndDirectory { name: stringify!($name_ident), config_dir: $config_dir, - genesis_is_known: $genesis_is_known, + genesis_state_source: $genesis_state_source, }; /// A wrapper around `std::include_bytes` which includes a file from a specific network @@ -151,7 +183,7 @@ macro_rules! define_net { $this_crate::HardcodedNet { name: ETH2_NET_DIR.name, config_dir: ETH2_NET_DIR.config_dir, - genesis_is_known: ETH2_NET_DIR.genesis_is_known, + genesis_state_source: ETH2_NET_DIR.genesis_state_source, config: $this_crate::$include_file!($this_crate, "../", "config.yaml"), deploy_block: $this_crate::$include_file!($this_crate, "../", "deploy_block.txt"), boot_enr: $this_crate::$include_file!($this_crate, "../", "boot_enr.yaml"), @@ -199,9 +231,9 @@ macro_rules! define_nets { /// `build.rs` which will unzip the genesis states. Then, that `eth2_network_configs` crate can /// perform the final step of using `std::include_bytes` to bake the files (bytes) into the binary. macro_rules! define_hardcoded_nets { - ($(($name_ident: ident, $config_dir: tt, $genesis_is_known: ident)),+) => { + ($(($name_ident: ident, $config_dir: tt, $genesis_state_source: path)),+) => { $( - define_archive!($name_ident, $config_dir, $genesis_is_known); + define_archive!($name_ident, $config_dir, $genesis_state_source); )+ pub const ETH2_NET_DIRS: &[Eth2NetArchiveAndDirectory<'static>] = &[$($name_ident::ETH2_NET_DIR,)+]; @@ -242,9 +274,8 @@ define_hardcoded_nets!( // The name of the directory in the `eth2_network_config/built_in_network_configs` // directory where the configuration files are located for this network. "mainnet", - // Set to `true` if the genesis state can be found in the `built_in_network_configs` - // directory. - GENESIS_STATE_IS_KNOWN + // Describes how the genesis state can be obtained. + GenesisStateSource::IncludedBytes ), ( // Network name (must be unique among all networks). @@ -252,9 +283,8 @@ define_hardcoded_nets!( // The name of the directory in the `eth2_network_config/built_in_network_configs` // directory where the configuration files are located for this network. "prater", - // Set to `true` if the genesis state can be found in the `built_in_network_configs` - // directory. - GENESIS_STATE_IS_KNOWN + // Describes how the genesis state can be obtained. + GenesisStateSource::IncludedBytes ), ( // Network name (must be unique among all networks). @@ -264,9 +294,8 @@ define_hardcoded_nets!( // // The Goerli network is effectively an alias to Prater. "prater", - // Set to `true` if the genesis state can be found in the `built_in_network_configs` - // directory. - GENESIS_STATE_IS_KNOWN + // Describes how the genesis state can be obtained. + GenesisStateSource::IncludedBytes ), ( // Network name (must be unique among all networks). @@ -274,9 +303,8 @@ define_hardcoded_nets!( // The name of the directory in the `eth2_network_config/built_in_network_configs` // directory where the configuration files are located for this network. "gnosis", - // Set to `true` if the genesis state can be found in the `built_in_network_configs` - // directory. - GENESIS_STATE_IS_KNOWN + // Describes how the genesis state can be obtained. + GenesisStateSource::IncludedBytes ), ( // Network name (must be unique among all networks). @@ -284,8 +312,16 @@ define_hardcoded_nets!( // The name of the directory in the `eth2_network_config/built_in_network_configs` // directory where the configuration files are located for this network. "sepolia", - // Set to `true` if the genesis state can be found in the `built_in_network_configs` - // directory. - GENESIS_STATE_IS_KNOWN + // Describes how the genesis state can be obtained. + GenesisStateSource::IncludedBytes + ), + ( + // Network name (must be unique among all networks). + holesky, + // The name of the directory in the `eth2_network_config/built_in_network_configs` + // directory where the configuration files are located for this network. + "holesky", + // Describes how the genesis state can be obtained. + HOLESKY_GENESIS_STATE_SOURCE ) ); diff --git a/common/eth2_network_config/Cargo.toml b/common/eth2_network_config/Cargo.toml index d1fe25f1cc1..e73f64d5a83 100644 --- a/common/eth2_network_config/Cargo.toml +++ b/common/eth2_network_config/Cargo.toml @@ -8,14 +8,21 @@ build = "build.rs" [build-dependencies] zip = "0.6" -eth2_config = { path = "../eth2_config"} +eth2_config = { path = "../eth2_config" } [dev-dependencies] tempfile = "3.1.0" [dependencies] serde_yaml = "0.8.13" -types = { path = "../../consensus/types"} +types = { path = "../../consensus/types" } ethereum_ssz = "0.5.0" -eth2_config = { path = "../eth2_config"} +eth2_config = { path = "../eth2_config" } discv5 = "0.3.1" +reqwest = { version = "0.11.0", features = ["blocking"] } +pretty_reqwest_error = { path = "../pretty_reqwest_error" } +sha2 = "0.10" +url = "2.2.2" +sensitive_url = { path = "../sensitive_url" } +slog = "2.5.2" +logging = { path = "../logging" } diff --git a/common/eth2_network_config/build.rs b/common/eth2_network_config/build.rs index fa45fafa4e7..3165930f4a8 100644 --- a/common/eth2_network_config/build.rs +++ b/common/eth2_network_config/build.rs @@ -1,5 +1,7 @@ //! Extracts zipped genesis states on first run. -use eth2_config::{Eth2NetArchiveAndDirectory, ETH2_NET_DIRS, GENESIS_FILE_NAME}; +use eth2_config::{ + Eth2NetArchiveAndDirectory, GenesisStateSource, ETH2_NET_DIRS, GENESIS_FILE_NAME, +}; use std::fs::File; use std::io; use zip::ZipArchive; @@ -26,7 +28,7 @@ fn uncompress_state(network: &Eth2NetArchiveAndDirectory<'static>) -> Result<(), return Ok(()); } - if network.genesis_is_known { + if network.genesis_state_source == GenesisStateSource::IncludedBytes { // Extract genesis state from genesis.ssz.zip let archive_path = network.genesis_state_archive(); let archive_file = File::open(&archive_path) @@ -46,7 +48,8 @@ fn uncompress_state(network: &Eth2NetArchiveAndDirectory<'static>) -> Result<(), io::copy(&mut file, &mut outfile) .map_err(|e| format!("Error writing file {:?}: {}", genesis_ssz_path, e))?; } else { - // Create empty genesis.ssz if genesis is unknown + // Create empty genesis.ssz if genesis is unknown or to be downloaded via URL. + // This is a bit of a hack to make `include_bytes!` easier to deal with. File::create(genesis_ssz_path) .map_err(|e| format!("Failed to create {}: {}", GENESIS_FILE_NAME, e))?; } diff --git a/common/eth2_network_config/built_in_network_configs/holesky/boot_enr.yaml b/common/eth2_network_config/built_in_network_configs/holesky/boot_enr.yaml new file mode 100644 index 00000000000..616d41d672a --- /dev/null +++ b/common/eth2_network_config/built_in_network_configs/holesky/boot_enr.yaml @@ -0,0 +1,8 @@ +# EF +- enr:-Iq4QJk4WqRkjsX5c2CXtOra6HnxN-BMXnWhmhEQO9Bn9iABTJGdjUOurM7Btj1ouKaFkvTRoju5vz2GPmVON2dffQKGAX53x8JigmlkgnY0gmlwhLKAlv6Jc2VjcDI1NmsxoQK6S-Cii_KmfFdUJL2TANL3ksaKUnNXvTCv1tLwXs0QgIN1ZHCCIyk +- enr:-KG4QF6d6vMSboSujAXTI4vYqArccm0eIlXfcxf2Lx_VE1q6IkQo_2D5LAO3ZSBVUs0w5rrVDmABJZuMzISe_pZundADhGV0aDKQqX6DZjABcAAAAQAAAAAAAIJpZIJ2NIJpcISygIjpiXNlY3AyNTZrMaEDF3aSa7QSCvdqLpANNd8GML4PLEZVg45fKQwMWhDZjd2DdGNwgiMog3VkcIIjKA +- enr:-Ly4QJLXSSAj3ggPBIcodvBU6IyfpU_yW7E9J-5syoJorBuvcYj_Fokcjr303bQoTdWXADf8po0ssh75Mr5wVGzZZsMBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpCpfoNmMAFwAAABAAAAAAAAgmlkgnY0gmlwhJK-DYCJc2VjcDI1NmsxoQJrIlXIQDvQ6t9yDySqJYDXgZgLXzTvq8W7OI51jfmxJohzeW5jbmV0cwCDdGNwgiMog3VkcIIjKA +# Teku +- enr:-LK4QMlzEff6d-M0A1pSFG5lJ2c56i_I-ZftdojZbW3ehkGNM4pkQuHQqzVvF1BG9aDjIakjnmO23mCBFFZ2w5zOsugEh2F0dG5ldHOIAAAAAAYAAACEZXRoMpCpfoNmMAFwAAABAAAAAAAAgmlkgnY0gmlwhKyuI_mJc2VjcDI1NmsxoQIH1kQRCZW-4AIVyAeXj5o49m_IqNFKRHp6tSpfXMUrSYN0Y3CCIyiDdWRwgiMo +# Sigma Prime +- enr:-Le4QI88slOwzz66Ksq8Vnz324DPb1BzSiY-WYPvnoJIl-lceW9bmSJnwDzgNbCjp5wsBigg76x4tValvGgQPxxSjrMBhGV0aDKQqX6DZjABcAAAAQAAAAAAAIJpZIJ2NIJpcIQ5gR6Wg2lwNpAgAUHQBwEQAAAAAAAAADR-iXNlY3AyNTZrMaEDPMSNdcL92uNIyCsS177Z6KTXlbZakQqxv3aQcWawNXeDdWRwgiMohHVkcDaCI4I diff --git a/common/eth2_network_config/built_in_network_configs/holesky/config.yaml b/common/eth2_network_config/built_in_network_configs/holesky/config.yaml new file mode 100644 index 00000000000..a6bfd87adec --- /dev/null +++ b/common/eth2_network_config/built_in_network_configs/holesky/config.yaml @@ -0,0 +1,117 @@ +# Extends the mainnet preset +PRESET_BASE: 'mainnet' +CONFIG_NAME: holesky + +# Genesis +# --------------------------------------------------------------- +# `2**14` (= 16,384) +MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 16384 +# Sep-15-2023 13:55:00 +UTC +MIN_GENESIS_TIME: 1694786100 +GENESIS_FORK_VERSION: 0x00017000 +# Genesis delay 5 mins +GENESIS_DELAY: 300 + + +# Forking +# --------------------------------------------------------------- +# Some forks are disabled for now: +# - These may be re-assigned to another fork-version later +# - Temporarily set to max uint64 value: 2**64 - 1 + +# Altair +ALTAIR_FORK_VERSION: 0x10017000 +ALTAIR_FORK_EPOCH: 0 +# Merge +BELLATRIX_FORK_VERSION: 0x20017000 +BELLATRIX_FORK_EPOCH: 0 +TERMINAL_TOTAL_DIFFICULTY: 0 +TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 +TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 + +# Capella +CAPELLA_FORK_VERSION: 0x30017000 +CAPELLA_FORK_EPOCH: 256 + +# DENEB +DENEB_FORK_VERSION: 0x40017000 +DENEB_FORK_EPOCH: 18446744073709551615 + +# Time parameters +# --------------------------------------------------------------- +# 12 seconds +SECONDS_PER_SLOT: 12 +# 14 (estimate from Eth1 mainnet) +SECONDS_PER_ETH1_BLOCK: 14 +# 2**8 (= 256) epochs ~27 hours +MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 +# 2**8 (= 256) epochs ~27 hours +SHARD_COMMITTEE_PERIOD: 256 +# 2**11 (= 2,048) Eth1 blocks ~8 hours +ETH1_FOLLOW_DISTANCE: 2048 + + +# Validator cycle +# --------------------------------------------------------------- +# 2**2 (= 4) +INACTIVITY_SCORE_BIAS: 4 +# 2**4 (= 16) +INACTIVITY_SCORE_RECOVERY_RATE: 16 +# 28,000,000,000 Gwei to ensure quicker ejection +EJECTION_BALANCE: 28000000000 +# 2**2 (= 4) +MIN_PER_EPOCH_CHURN_LIMIT: 4 +# 2**16 (= 65,536) +CHURN_LIMIT_QUOTIENT: 65536 + +# Fork choice +# --------------------------------------------------------------- +# 40% +PROPOSER_SCORE_BOOST: 40 + +# Deposit contract +# --------------------------------------------------------------- +DEPOSIT_CHAIN_ID: 17000 +DEPOSIT_NETWORK_ID: 17000 +DEPOSIT_CONTRACT_ADDRESS: 0x4242424242424242424242424242424242424242 + +# Networking +# --------------------------------------------------------------- +# `10 * 2**20` (= 10485760, 10 MiB) +GOSSIP_MAX_SIZE: 10485760 +# `2**10` (= 1024) +MAX_REQUEST_BLOCKS: 1024 +# `2**8` (= 256) +EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 +# `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) +MIN_EPOCHS_FOR_BLOCK_REQUESTS: 33024 +# `10 * 2**20` (=10485760, 10 MiB) +MAX_CHUNK_SIZE: 10485760 +# 5s +TTFB_TIMEOUT: 5 +# 10s +RESP_TIMEOUT: 10 +ATTESTATION_PROPAGATION_SLOT_RANGE: 32 +# 500ms +MAXIMUM_GOSSIP_CLOCK_DISPARITY: 500 +MESSAGE_DOMAIN_INVALID_SNAPPY: 0x00000000 +MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000 +# 2 subnets per node +SUBNETS_PER_NODE: 2 +# 2**8 (= 64) +ATTESTATION_SUBNET_COUNT: 64 +ATTESTATION_SUBNET_EXTRA_BITS: 0 +# ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS +ATTESTATION_SUBNET_PREFIX_BITS: 6 + +# Deneb +# `2**7` (=128) +MAX_REQUEST_BLOCKS_DENEB: 128 +# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK +MAX_REQUEST_BLOB_SIDECARS: 768 +# `2**12` (= 4096 epochs, ~18 days) +MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 +# `6` +BLOB_SIDECAR_SUBNET_COUNT: 6 +# `uint64(6)` +MAX_BLOBS_PER_BLOCK: 6 diff --git a/common/eth2_network_config/built_in_network_configs/holesky/deploy_block.txt b/common/eth2_network_config/built_in_network_configs/holesky/deploy_block.txt new file mode 100644 index 00000000000..573541ac970 --- /dev/null +++ b/common/eth2_network_config/built_in_network_configs/holesky/deploy_block.txt @@ -0,0 +1 @@ +0 diff --git a/common/eth2_network_config/src/lib.rs b/common/eth2_network_config/src/lib.rs index 7274bbf029b..1206b8b8caa 100644 --- a/common/eth2_network_config/src/lib.rs +++ b/common/eth2_network_config/src/lib.rs @@ -13,10 +13,20 @@ use discv5::enr::{CombinedKey, Enr}; use eth2_config::{instantiate_hardcoded_nets, HardcodedNet}; +use pretty_reqwest_error::PrettyReqwestError; +use reqwest::blocking::Client; +use sensitive_url::SensitiveUrl; +use sha2::{Digest, Sha256}; +use slog::{info, warn, Logger}; use std::fs::{create_dir_all, File}; use std::io::{Read, Write}; use std::path::PathBuf; -use types::{BeaconState, ChainSpec, Config, EthSpec, EthSpecId}; +use std::str::FromStr; +use std::time::Duration; +use types::{BeaconState, ChainSpec, Config, EthSpec, EthSpecId, Hash256}; +use url::Url; + +pub use eth2_config::GenesisStateSource; pub const DEPLOY_BLOCK_FILE: &str = "deploy_block.txt"; pub const BOOT_ENR_FILE: &str = "boot_enr.yaml"; @@ -32,6 +42,35 @@ instantiate_hardcoded_nets!(eth2_config); pub const DEFAULT_HARDCODED_NETWORK: &str = "mainnet"; +/// A simple slice-or-vec enum to avoid cloning the beacon state bytes in the +/// binary whilst also supporting loading them from a file at runtime. +#[derive(Clone, PartialEq, Debug)] +pub enum GenesisStateBytes { + Slice(&'static [u8]), + Vec(Vec<u8>), +} + +impl AsRef<[u8]> for GenesisStateBytes { + fn as_ref(&self) -> &[u8] { + match self { + GenesisStateBytes::Slice(slice) => slice, + GenesisStateBytes::Vec(vec) => vec.as_ref(), + } + } +} + +impl From<&'static [u8]> for GenesisStateBytes { + fn from(slice: &'static [u8]) -> Self { + GenesisStateBytes::Slice(slice) + } +} + +impl From<Vec<u8>> for GenesisStateBytes { + fn from(vec: Vec<u8>) -> Self { + GenesisStateBytes::Vec(vec) + } +} + /// Specifies an Eth2 network. /// /// See the crate-level documentation for more details. @@ -41,7 +80,8 @@ pub struct Eth2NetworkConfig { /// value to be the block number where the first deposit occurs. pub deposit_contract_deploy_block: u64, pub boot_enr: Option<Vec<Enr<CombinedKey>>>, - pub genesis_state_bytes: Option<Vec<u8>>, + pub genesis_state_source: GenesisStateSource, + pub genesis_state_bytes: Option<GenesisStateBytes>, pub config: Config, } @@ -65,8 +105,10 @@ impl Eth2NetworkConfig { serde_yaml::from_reader(net.boot_enr) .map_err(|e| format!("Unable to parse boot enr: {:?}", e))?, ), - genesis_state_bytes: Some(net.genesis_state_bytes.to_vec()) - .filter(|bytes| !bytes.is_empty()), + genesis_state_source: net.genesis_state_source, + genesis_state_bytes: Some(net.genesis_state_bytes) + .filter(|bytes| !bytes.is_empty()) + .map(Into::into), config: serde_yaml::from_reader(net.config) .map_err(|e| format!("Unable to parse yaml config: {:?}", e))?, }) @@ -81,8 +123,37 @@ impl Eth2NetworkConfig { } /// Returns `true` if this configuration contains a `BeaconState`. - pub fn beacon_state_is_known(&self) -> bool { - self.genesis_state_bytes.is_some() + pub fn genesis_state_is_known(&self) -> bool { + self.genesis_state_source != GenesisStateSource::Unknown + } + + /// The `genesis_validators_root` of the genesis state. May download the + /// genesis state if the value is not already available. + pub fn genesis_validators_root<E: EthSpec>( + &self, + genesis_state_url: Option<&str>, + timeout: Duration, + log: &Logger, + ) -> Result<Option<Hash256>, String> { + if let GenesisStateSource::Url { + genesis_validators_root, + .. + } = self.genesis_state_source + { + Hash256::from_str(genesis_validators_root) + .map(Option::Some) + .map_err(|e| { + format!( + "Unable to parse genesis state genesis_validators_root: {:?}", + e + ) + }) + } else { + self.genesis_state::<E>(genesis_state_url, timeout, log)? + .map(|state| state.genesis_validators_root()) + .map(Result::Ok) + .transpose() + } } /// Construct a consolidated `ChainSpec` from the YAML config. @@ -96,15 +167,65 @@ impl Eth2NetworkConfig { } /// Attempts to deserialize `self.beacon_state`, returning an error if it's missing or invalid. - pub fn beacon_state<E: EthSpec>(&self) -> Result<BeaconState<E>, String> { + /// + /// If the genesis state is configured to be downloaded from a URL, then the + /// `genesis_state_url` will override the built-in list of download URLs. + pub fn genesis_state<E: EthSpec>( + &self, + genesis_state_url: Option<&str>, + timeout: Duration, + log: &Logger, + ) -> Result<Option<BeaconState<E>>, String> { let spec = self.chain_spec::<E>()?; - let genesis_state_bytes = self - .genesis_state_bytes - .as_ref() - .ok_or("Genesis state is unknown")?; + match &self.genesis_state_source { + GenesisStateSource::Unknown => Ok(None), + GenesisStateSource::IncludedBytes => { + let state = self + .genesis_state_bytes + .as_ref() + .map(|bytes| { + BeaconState::from_ssz_bytes(bytes.as_ref(), &spec).map_err(|e| { + format!("Built-in genesis state SSZ bytes are invalid: {:?}", e) + }) + }) + .ok_or("Genesis state bytes missing from Eth2NetworkConfig")??; + Ok(Some(state)) + } + GenesisStateSource::Url { + urls: built_in_urls, + checksum, + genesis_validators_root, + } => { + let checksum = Hash256::from_str(checksum).map_err(|e| { + format!("Unable to parse genesis state bytes checksum: {:?}", e) + })?; + let bytes = if let Some(specified_url) = genesis_state_url { + download_genesis_state(&[specified_url], timeout, checksum, log) + } else { + download_genesis_state(built_in_urls, timeout, checksum, log) + }?; + let state = BeaconState::from_ssz_bytes(bytes.as_ref(), &spec).map_err(|e| { + format!("Downloaded genesis state SSZ bytes are invalid: {:?}", e) + })?; + + let genesis_validators_root = + Hash256::from_str(genesis_validators_root).map_err(|e| { + format!( + "Unable to parse genesis state genesis_validators_root: {:?}", + e + ) + })?; + if state.genesis_validators_root() != genesis_validators_root { + return Err(format!( + "Downloaded genesis validators root {:?} does not match expected {:?}", + state.genesis_validators_root(), + genesis_validators_root + )); + } - BeaconState::from_ssz_bytes(genesis_state_bytes, &spec) - .map_err(|e| format!("Genesis state SSZ bytes are invalid: {:?}", e)) + Ok(Some(state)) + } + } } /// Write the files to the directory. @@ -162,7 +283,7 @@ impl Eth2NetworkConfig { File::create(&file) .map_err(|e| format!("Unable to create {:?}: {:?}", file, e)) .and_then(|mut file| { - file.write_all(genesis_state_bytes) + file.write_all(genesis_state_bytes.as_ref()) .map_err(|e| format!("Unable to write {:?}: {:?}", file, e)) })?; } @@ -198,7 +319,7 @@ impl Eth2NetworkConfig { // The genesis state is a special case because it uses SSZ, not YAML. let genesis_file_path = base_dir.join(GENESIS_STATE_FILE); - let genesis_state_bytes = if genesis_file_path.exists() { + let (genesis_state_bytes, genesis_state_source) = if genesis_file_path.exists() { let mut bytes = vec![]; File::open(&genesis_file_path) .map_err(|e| format!("Unable to open {:?}: {:?}", genesis_file_path, e)) @@ -207,20 +328,105 @@ impl Eth2NetworkConfig { .map_err(|e| format!("Unable to read {:?}: {:?}", file, e)) })?; - Some(bytes).filter(|bytes| !bytes.is_empty()) + let state = Some(bytes).filter(|bytes| !bytes.is_empty()); + let genesis_state_source = if state.is_some() { + GenesisStateSource::IncludedBytes + } else { + GenesisStateSource::Unknown + }; + (state, genesis_state_source) } else { - None + (None, GenesisStateSource::Unknown) }; Ok(Self { deposit_contract_deploy_block, boot_enr, - genesis_state_bytes, + genesis_state_source, + genesis_state_bytes: genesis_state_bytes.map(Into::into), config, }) } } +/// Try to download a genesis state from each of the `urls` in the order they +/// are defined. Return `Ok` if any url returns a response that matches the +/// given `checksum`. +fn download_genesis_state( + urls: &[&str], + timeout: Duration, + checksum: Hash256, + log: &Logger, +) -> Result<Vec<u8>, String> { + if urls.is_empty() { + return Err( + "The genesis state is not present in the binary and there are no known download URLs. \ + Please use --checkpoint-sync-url or --genesis-state-url." + .to_string(), + ); + } + + let mut errors = vec![]; + for url in urls { + // URLs are always expected to be the base URL of a server that supports + // the beacon-API. + let url = parse_state_download_url(url)?; + let redacted_url = SensitiveUrl::new(url.clone()) + .map(|url| url.to_string()) + .unwrap_or_else(|_| "<REDACTED>".to_string()); + + info!( + log, + "Downloading genesis state"; + "server" => &redacted_url, + "timeout" => ?timeout, + "info" => "this may take some time on testnets with large validator counts" + ); + + let client = Client::new(); + let response = client + .get(url) + .header("Accept", "application/octet-stream") + .timeout(timeout) + .send() + .and_then(|r| r.error_for_status().and_then(|r| r.bytes())); + + match response { + Ok(bytes) => { + // Check the server response against our local checksum. + if Sha256::digest(bytes.as_ref())[..] == checksum[..] { + return Ok(bytes.into()); + } else { + warn!( + log, + "Genesis state download failed"; + "server" => &redacted_url, + "timeout" => ?timeout, + ); + errors.push(format!( + "Response from {} did not match local checksum", + redacted_url + )) + } + } + Err(e) => errors.push(PrettyReqwestError::from(e).to_string()), + } + } + Err(format!( + "Unable to download a genesis state from {} source(s): {}", + errors.len(), + errors.join(",") + )) +} + +/// Parses the `url` and joins the necessary state download path. +fn parse_state_download_url(url: &str) -> Result<Url, String> { + Url::parse(url) + .map_err(|e| format!("Invalid genesis state URL: {:?}", e))? + .join("eth/v2/debug/beacon/states/genesis") + .map_err(|e| format!("Failed to append genesis state path to URL: {:?}", e)) +} + #[cfg(test)] mod tests { use super::*; @@ -260,7 +466,9 @@ mod tests { #[test] fn mainnet_genesis_state() { let config = Eth2NetworkConfig::from_hardcoded_net(&MAINNET).unwrap(); - config.beacon_state::<E>().expect("beacon state can decode"); + config + .genesis_state::<E>(None, Duration::from_secs(1), &logging::test_logger()) + .expect("beacon state can decode"); } #[test] @@ -285,10 +493,25 @@ mod tests { assert_eq!( config.genesis_state_bytes.is_some(), - net.genesis_is_known, + net.genesis_state_source == GenesisStateSource::IncludedBytes, "{:?}", net.name ); + + if let GenesisStateSource::Url { + urls, + checksum, + genesis_validators_root, + } = net.genesis_state_source + { + Hash256::from_str(checksum).expect("the checksum must be a valid 32-byte value"); + Hash256::from_str(genesis_validators_root) + .expect("the GVR must be a valid 32-byte value"); + for url in urls { + parse_state_download_url(url).expect("url must be valid"); + } + } + assert_eq!(config.config.config_name, Some(net.config_dir.to_string())); } } @@ -324,10 +547,20 @@ mod tests { let base_dir = temp_dir.path().join("my_testnet"); let deposit_contract_deploy_block = 42; + let genesis_state_source = if genesis_state.is_some() { + GenesisStateSource::IncludedBytes + } else { + GenesisStateSource::Unknown + }; + let testnet: Eth2NetworkConfig = Eth2NetworkConfig { deposit_contract_deploy_block, boot_enr, - genesis_state_bytes: genesis_state.as_ref().map(Encode::as_ssz_bytes), + genesis_state_source, + genesis_state_bytes: genesis_state + .as_ref() + .map(Encode::as_ssz_bytes) + .map(Into::into), config, }; diff --git a/common/pretty_reqwest_error/src/lib.rs b/common/pretty_reqwest_error/src/lib.rs index 4c605f38aeb..0aaee5965ee 100644 --- a/common/pretty_reqwest_error/src/lib.rs +++ b/common/pretty_reqwest_error/src/lib.rs @@ -55,6 +55,12 @@ impl fmt::Debug for PrettyReqwestError { } } +impl fmt::Display for PrettyReqwestError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{:?}", self) + } +} + impl From<reqwest::Error> for PrettyReqwestError { fn from(inner: reqwest::Error) -> Self { Self(inner) diff --git a/lcli/src/eth1_genesis.rs b/lcli/src/eth1_genesis.rs index 34144cd86db..bddd4baad8b 100644 --- a/lcli/src/eth1_genesis.rs +++ b/lcli/src/eth1_genesis.rs @@ -49,7 +49,7 @@ pub fn run<T: EthSpec>( .wait_for_genesis_state::<T>(ETH1_GENESIS_UPDATE_INTERVAL, spec) .await .map(move |genesis_state| { - eth2_network_config.genesis_state_bytes = Some(genesis_state.as_ssz_bytes()); + eth2_network_config.genesis_state_bytes = Some(genesis_state.as_ssz_bytes().into()); eth2_network_config.force_write_to_file(testnet_dir) }) .map_err(|e| format!("Failed to find genesis: {}", e))?; diff --git a/lcli/src/interop_genesis.rs b/lcli/src/interop_genesis.rs index 57a5ba00988..1a0b81fcb7f 100644 --- a/lcli/src/interop_genesis.rs +++ b/lcli/src/interop_genesis.rs @@ -42,7 +42,7 @@ pub fn run<T: EthSpec>(testnet_dir: PathBuf, matches: &ArgMatches) -> Result<(), &spec, )?; - eth2_network_config.genesis_state_bytes = Some(genesis_state.as_ssz_bytes()); + eth2_network_config.genesis_state_bytes = Some(genesis_state.as_ssz_bytes().into()); eth2_network_config.force_write_to_file(testnet_dir)?; Ok(()) diff --git a/lcli/src/new_testnet.rs b/lcli/src/new_testnet.rs index 01a44cabef7..973993f9790 100644 --- a/lcli/src/new_testnet.rs +++ b/lcli/src/new_testnet.rs @@ -1,7 +1,7 @@ use account_utils::eth2_keystore::keypair_from_secret; use clap::ArgMatches; use clap_utils::{parse_optional, parse_required, parse_ssz_optional}; -use eth2_network_config::Eth2NetworkConfig; +use eth2_network_config::{Eth2NetworkConfig, GenesisStateSource}; use eth2_wallet::bip39::Seed; use eth2_wallet::bip39::{Language, Mnemonic}; use eth2_wallet::{recover_validator_secret_from_mnemonic, KeyType}; @@ -190,7 +190,8 @@ pub fn run<T: EthSpec>(testnet_dir_path: PathBuf, matches: &ArgMatches) -> Resul let testnet = Eth2NetworkConfig { deposit_contract_deploy_block, boot_enr: Some(vec![]), - genesis_state_bytes, + genesis_state_bytes: genesis_state_bytes.map(Into::into), + genesis_state_source: GenesisStateSource::IncludedBytes, config: Config::from_chain_spec::<T>(&spec), }; diff --git a/lighthouse/src/main.rs b/lighthouse/src/main.rs index d8b522307c4..6384fc53cd6 100644 --- a/lighthouse/src/main.rs +++ b/lighthouse/src/main.rs @@ -324,6 +324,30 @@ fn main() { .takes_value(true) .global(true) ) + .arg( + Arg::with_name("genesis-state-url") + .long("genesis-state-url") + .value_name("URL") + .help( + "A URL of a beacon-API compatible server from which to download the genesis state. \ + Checkpoint sync server URLs can generally be used with this flag. \ + If not supplied, a default URL or the --checkpoint-sync-url may be used. \ + If the genesis state is already included in this binary then this value will be ignored.", + ) + .takes_value(true) + .global(true), + ) + .arg( + Arg::with_name("genesis-state-url-timeout") + .long("genesis-state-url-timeout") + .value_name("SECONDS") + .help( + "The timeout in seconds for the request to --genesis-state-url.", + ) + .takes_value(true) + .default_value("180") + .global(true), + ) .subcommand(beacon_node::cli_app()) .subcommand(boot_node::cli_app()) .subcommand(validator_client::cli_app()) diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 5069b0261f6..05b4358509b 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -2406,3 +2406,28 @@ fn http_duplicate_block_status_override() { assert_eq!(config.http_api.duplicate_block_status_code.as_u16(), 301) }); } + +#[test] +fn genesis_state_url_default() { + CommandLineTest::new() + .run_with_zero_port() + .with_config(|config| { + assert_eq!(config.genesis_state_url, None); + assert_eq!(config.genesis_state_url_timeout, Duration::from_secs(180)); + }); +} + +#[test] +fn genesis_state_url_value() { + CommandLineTest::new() + .flag("genesis-state-url", Some("http://genesis.com")) + .flag("genesis-state-url-timeout", Some("42")) + .run_with_zero_port() + .with_config(|config| { + assert_eq!( + config.genesis_state_url.as_deref(), + Some("http://genesis.com") + ); + assert_eq!(config.genesis_state_url_timeout, Duration::from_secs(42)); + }); +} From f284e0e26445f776e470ad694a947cb10adc5686 Mon Sep 17 00:00:00 2001 From: Michael Sproul <michael@sigmaprime.io> Date: Mon, 28 Aug 2023 05:34:28 +0000 Subject: [PATCH 07/17] Fix bug in block root storage (#4663) ## Issue Addressed Fix a bug in the storage of the linear block roots array in the freezer DB. Previously this array was always written as part of state storage (or block backfill). With state pruning enabled by #4610, these states were no longer being written and as a result neither were the block roots. The impact is quite low, we would just log an error when trying to forwards-iterate the block roots, which for validating nodes only happens when they try to look up blocks for peers: > Aug 25 03:42:36.980 ERRO Missing chunk in forwards iterator chunk index: 49726, service: freezer_db Any node checkpoint synced off `unstable` is affected and has a corrupt database. If you see the log above, you need to re-sync with the fix. Nodes that haven't checkpoint synced recently should _not_ be corrupted, even if they ran the buggy version. ## Proposed Changes - Use a `ChunkWriter` to write the block roots when states are not being stored. - Tweak the usage of `get_latest_restore_point` so that it doesn't return a nonsense value when state pruning is enabled. - Tweak the guarantee on the block roots array so that block roots are assumed available up to the split slot (exclusive). This is a bit nicer than relying on anything to do with the latest restore point, which is a nonsensical concept when there aren't any restore points. ## Additional Info I'm looking forward to deleting the chunked vector code for good when we merge tree-states :grin: --- Cargo.lock | 1 + beacon_node/beacon_chain/src/builder.rs | 15 ++++ beacon_node/beacon_chain/tests/store_tests.rs | 2 +- beacon_node/store/src/chunked_iter.rs | 8 +- beacon_node/store/src/forwards_iter.rs | 54 ++++++++++-- beacon_node/store/src/hot_cold_store.rs | 82 +++++++++++++++---- watch/Cargo.toml | 1 + watch/tests/tests.rs | 23 +++--- 8 files changed, 144 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 75c15ba7813..27d7b655e30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9051,6 +9051,7 @@ dependencies = [ "http_api", "hyper", "log", + "logging", "network", "r2d2", "rand 0.7.3", diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index bb629d66249..54739f2b8ac 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -481,6 +481,21 @@ where let (_, updated_builder) = self.set_genesis_state(genesis_state)?; self = updated_builder; + // Fill in the linear block roots between the checkpoint block's slot and the aligned + // state's slot. All slots less than the block's slot will be handled by block backfill, + // while states greater or equal to the checkpoint state will be handled by `migrate_db`. + let block_root_batch = store + .store_frozen_block_root_at_skip_slots( + weak_subj_block.slot(), + weak_subj_state.slot(), + weak_subj_block_root, + ) + .map_err(|e| format!("Error writing frozen block roots: {e:?}"))?; + store + .cold_db + .do_atomically(block_root_batch) + .map_err(|e| format!("Error writing frozen block roots: {e:?}"))?; + // Write the state and block non-atomically, it doesn't matter if they're forgotten // about on a crash restart. store diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index bf68657b4f5..ab54af42c78 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -421,7 +421,7 @@ async fn forwards_iter_block_and_state_roots_until() { // The last restore point slot is the point at which the hybrid forwards iterator behaviour // changes. - let last_restore_point_slot = store.get_latest_restore_point_slot(); + let last_restore_point_slot = store.get_latest_restore_point_slot().unwrap(); assert!(last_restore_point_slot > 0); let chain = &harness.chain; diff --git a/beacon_node/store/src/chunked_iter.rs b/beacon_node/store/src/chunked_iter.rs index 8ef0b6d201d..b3322b5225d 100644 --- a/beacon_node/store/src/chunked_iter.rs +++ b/beacon_node/store/src/chunked_iter.rs @@ -30,16 +30,16 @@ where /// Create a new iterator which can yield elements from `start_vindex` up to the last /// index stored by the restore point at `last_restore_point_slot`. /// - /// The `last_restore_point` slot should be the slot of a recent restore point as obtained from - /// `HotColdDB::get_latest_restore_point_slot`. We pass it as a parameter so that the caller can + /// The `freezer_upper_limit` slot should be the slot of a recent restore point as obtained from + /// `Root::freezer_upper_limit`. We pass it as a parameter so that the caller can /// maintain a stable view of the database (see `HybridForwardsBlockRootsIterator`). pub fn new( store: &'a HotColdDB<E, Hot, Cold>, start_vindex: usize, - last_restore_point_slot: Slot, + freezer_upper_limit: Slot, spec: &ChainSpec, ) -> Self { - let (_, end_vindex) = F::start_and_end_vindex(last_restore_point_slot, spec); + let (_, end_vindex) = F::start_and_end_vindex(freezer_upper_limit, spec); // Set the next chunk to the one containing `start_vindex`. let next_cindex = start_vindex / F::chunk_size(); diff --git a/beacon_node/store/src/forwards_iter.rs b/beacon_node/store/src/forwards_iter.rs index 353be6bf058..125b73a458f 100644 --- a/beacon_node/store/src/forwards_iter.rs +++ b/beacon_node/store/src/forwards_iter.rs @@ -19,6 +19,14 @@ pub trait Root<E: EthSpec>: Field<E, Value = Hash256> { end_state: BeaconState<E>, end_root: Hash256, ) -> Result<SimpleForwardsIterator>; + + /// The first slot for which this field is *no longer* stored in the freezer database. + /// + /// If `None`, then this field is not stored in the freezer database at all due to pruning + /// configuration. + fn freezer_upper_limit<Hot: ItemStore<E>, Cold: ItemStore<E>>( + store: &HotColdDB<E, Hot, Cold>, + ) -> Option<Slot>; } impl<E: EthSpec> Root<E> for BlockRoots { @@ -39,6 +47,13 @@ impl<E: EthSpec> Root<E> for BlockRoots { )?; Ok(SimpleForwardsIterator { values }) } + + fn freezer_upper_limit<Hot: ItemStore<E>, Cold: ItemStore<E>>( + store: &HotColdDB<E, Hot, Cold>, + ) -> Option<Slot> { + // Block roots are stored for all slots up to the split slot (exclusive). + Some(store.get_split_slot()) + } } impl<E: EthSpec> Root<E> for StateRoots { @@ -59,6 +74,15 @@ impl<E: EthSpec> Root<E> for StateRoots { )?; Ok(SimpleForwardsIterator { values }) } + + fn freezer_upper_limit<Hot: ItemStore<E>, Cold: ItemStore<E>>( + store: &HotColdDB<E, Hot, Cold>, + ) -> Option<Slot> { + // State roots are stored for all slots up to the latest restore point (exclusive). + // There may not be a latest restore point if state pruning is enabled, in which + // case this function will return `None`. + store.get_latest_restore_point_slot() + } } /// Forwards root iterator that makes use of a flat field table in the freezer DB. @@ -118,6 +142,7 @@ impl Iterator for SimpleForwardsIterator { pub enum HybridForwardsIterator<'a, E: EthSpec, F: Root<E>, Hot: ItemStore<E>, Cold: ItemStore<E>> { PreFinalization { iter: Box<FrozenForwardsIterator<'a, E, F, Hot, Cold>>, + end_slot: Option<Slot>, /// Data required by the `PostFinalization` iterator when we get to it. continuation_data: Option<Box<(BeaconState<E>, Hash256)>>, }, @@ -129,6 +154,7 @@ pub enum HybridForwardsIterator<'a, E: EthSpec, F: Root<E>, Hot: ItemStore<E>, C PostFinalization { iter: SimpleForwardsIterator, }, + Finished, } impl<'a, E: EthSpec, F: Root<E>, Hot: ItemStore<E>, Cold: ItemStore<E>> @@ -138,8 +164,8 @@ impl<'a, E: EthSpec, F: Root<E>, Hot: ItemStore<E>, Cold: ItemStore<E>> /// /// The `get_state` closure should return a beacon state and final block/state root to backtrack /// from in the case where the iterated range does not lie entirely within the frozen portion of - /// the database. If an `end_slot` is provided and it is before the database's latest restore - /// point slot then the `get_state` closure will not be called at all. + /// the database. If an `end_slot` is provided and it is before the database's freezer upper + /// limit for the field then the `get_state` closure will not be called at all. /// /// It is OK for `get_state` to hold a lock while this function is evaluated, as the returned /// iterator is as lazy as possible and won't do any work apart from calling `get_state`. @@ -155,13 +181,15 @@ impl<'a, E: EthSpec, F: Root<E>, Hot: ItemStore<E>, Cold: ItemStore<E>> ) -> Result<Self> { use HybridForwardsIterator::*; - let latest_restore_point_slot = store.get_latest_restore_point_slot(); + // First slot at which this field is *not* available in the freezer. i.e. all slots less + // than this slot have their data available in the freezer. + let freezer_upper_limit = F::freezer_upper_limit(store).unwrap_or(Slot::new(0)); - let result = if start_slot < latest_restore_point_slot { + let result = if start_slot < freezer_upper_limit { let iter = Box::new(FrozenForwardsIterator::new( store, start_slot, - latest_restore_point_slot, + freezer_upper_limit, spec, )); @@ -169,13 +197,14 @@ impl<'a, E: EthSpec, F: Root<E>, Hot: ItemStore<E>, Cold: ItemStore<E>> // `end_slot`. If it tries to continue further a `NoContinuationData` error will be // returned. let continuation_data = - if end_slot.map_or(false, |end_slot| end_slot < latest_restore_point_slot) { + if end_slot.map_or(false, |end_slot| end_slot < freezer_upper_limit) { None } else { Some(Box::new(get_state())) }; PreFinalization { iter, + end_slot, continuation_data, } } else { @@ -195,6 +224,7 @@ impl<'a, E: EthSpec, F: Root<E>, Hot: ItemStore<E>, Cold: ItemStore<E>> match self { PreFinalization { iter, + end_slot, continuation_data, } => { match iter.next() { @@ -203,10 +233,17 @@ impl<'a, E: EthSpec, F: Root<E>, Hot: ItemStore<E>, Cold: ItemStore<E>> // to a post-finalization iterator beginning from the last slot // of the pre iterator. None => { + // If the iterator has an end slot (inclusive) which has already been + // covered by the (exclusive) frozen forwards iterator, then we're done! + let iter_end_slot = Slot::from(iter.inner.end_vindex); + if end_slot.map_or(false, |end_slot| iter_end_slot == end_slot + 1) { + *self = Finished; + return Ok(None); + } + let continuation_data = continuation_data.take(); let store = iter.inner.store; - let start_slot = Slot::from(iter.inner.end_vindex); - + let start_slot = iter_end_slot; *self = PostFinalizationLazy { continuation_data, store, @@ -230,6 +267,7 @@ impl<'a, E: EthSpec, F: Root<E>, Hot: ItemStore<E>, Cold: ItemStore<E>> self.do_next() } PostFinalization { iter } => iter.next().transpose(), + Finished => Ok(None), } } } diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index 47839ed3e98..87f8e0ffc36 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -18,7 +18,7 @@ use crate::metadata::{ }; use crate::metrics; use crate::{ - get_key_for_col, DBColumn, DatabaseBlock, Error, ItemStore, KeyValueStoreOp, + get_key_for_col, ChunkWriter, DBColumn, DatabaseBlock, Error, ItemStore, KeyValueStoreOp, PartialBeaconState, StoreItem, StoreOp, }; use itertools::process_results; @@ -963,6 +963,9 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold> ops.push(op); // 2. Store updated vector entries. + // Block roots need to be written here as well as by the `ChunkWriter` in `migrate_db` + // because states may require older block roots, and the writer only stores block roots + // between the previous split point and the new split point. let db = &self.cold_db; store_updated_vector(BlockRoots, db, state, &self.spec, ops)?; store_updated_vector(StateRoots, db, state, &self.spec, ops)?; @@ -1243,10 +1246,21 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold> }; } - /// Fetch the slot of the most recently stored restore point. - pub fn get_latest_restore_point_slot(&self) -> Slot { - (self.get_split_slot() - 1) / self.config.slots_per_restore_point - * self.config.slots_per_restore_point + /// Fetch the slot of the most recently stored restore point (if any). + pub fn get_latest_restore_point_slot(&self) -> Option<Slot> { + let split_slot = self.get_split_slot(); + let anchor = self.get_anchor_info(); + + // There are no restore points stored if the state upper limit lies in the hot database. + // It hasn't been reached yet, and may never be. + if anchor.map_or(false, |a| a.state_upper_limit >= split_slot) { + None + } else { + Some( + (split_slot - 1) / self.config.slots_per_restore_point + * self.config.slots_per_restore_point, + ) + } } /// Load the database schema version from disk. @@ -1585,6 +1599,25 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold> ) } + /// Update the linear array of frozen block roots with the block root for several skipped slots. + /// + /// Write the block root at all slots from `start_slot` (inclusive) to `end_slot` (exclusive). + pub fn store_frozen_block_root_at_skip_slots( + &self, + start_slot: Slot, + end_slot: Slot, + block_root: Hash256, + ) -> Result<Vec<KeyValueStoreOp>, Error> { + let mut ops = vec![]; + let mut block_root_writer = + ChunkWriter::<BlockRoots, _, _>::new(&self.cold_db, start_slot.as_usize())?; + for slot in start_slot.as_usize()..end_slot.as_usize() { + block_root_writer.set(slot, block_root, &mut ops)?; + } + block_root_writer.write(&mut ops)?; + Ok(ops) + } + /// Try to prune all execution payloads, returning early if there is no need to prune. pub fn try_prune_execution_payloads(&self, force: bool) -> Result<(), Error> { let split = self.get_split_info(); @@ -1725,7 +1758,14 @@ pub fn migrate_database<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>( return Err(HotColdDBError::FreezeSlotUnaligned(finalized_state.slot()).into()); } - let mut hot_db_ops: Vec<StoreOp<E>> = Vec::new(); + let mut hot_db_ops = vec![]; + let mut cold_db_ops = vec![]; + + // Chunk writer for the linear block roots in the freezer DB. + // Start at the new upper limit because we iterate backwards. + let new_frozen_block_root_upper_limit = finalized_state.slot().as_usize().saturating_sub(1); + let mut block_root_writer = + ChunkWriter::<BlockRoots, _, _>::new(&store.cold_db, new_frozen_block_root_upper_limit)?; // 1. Copy all of the states between the new finalized state and the split slot, from the hot DB // to the cold DB. Delete the execution payloads of these now-finalized blocks. @@ -1750,6 +1790,9 @@ pub fn migrate_database<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>( // Delete the old summary, and the full state if we lie on an epoch boundary. hot_db_ops.push(StoreOp::DeleteState(state_root, Some(slot))); + // Store the block root for this slot in the linear array of frozen block roots. + block_root_writer.set(slot.as_usize(), block_root, &mut cold_db_ops)?; + // Do not try to store states if a restore point is yet to be stored, or will never be // stored (see `STATE_UPPER_LIMIT_NO_RETAIN`). Make an exception for the genesis state // which always needs to be copied from the hot DB to the freezer and should not be deleted. @@ -1759,29 +1802,34 @@ pub fn migrate_database<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>( .map_or(false, |anchor| slot < anchor.state_upper_limit) { debug!(store.log, "Pruning finalized state"; "slot" => slot); + continue; } - let mut cold_db_ops: Vec<KeyValueStoreOp> = Vec::new(); + // Store a pointer from this state root to its slot, so we can later reconstruct states + // from their state root alone. + let cold_state_summary = ColdStateSummary { slot }; + let op = cold_state_summary.as_kv_store_op(state_root); + cold_db_ops.push(op); if slot % store.config.slots_per_restore_point == 0 { let state: BeaconState<E> = get_full_state(&store.hot_db, &state_root, &store.spec)? .ok_or(HotColdDBError::MissingStateToFreeze(state_root))?; store.store_cold_state(&state_root, &state, &mut cold_db_ops)?; - } - // Store a pointer from this state root to its slot, so we can later reconstruct states - // from their state root alone. - let cold_state_summary = ColdStateSummary { slot }; - let op = cold_state_summary.as_kv_store_op(state_root); - cold_db_ops.push(op); - - // There are data dependencies between calls to `store_cold_state()` that prevent us from - // doing one big call to `store.cold_db.do_atomically()` at end of the loop. - store.cold_db.do_atomically(cold_db_ops)?; + // Commit the batch of cold DB ops whenever a full state is written. Each state stored + // may read the linear fields of previous states stored. + store + .cold_db + .do_atomically(std::mem::take(&mut cold_db_ops))?; + } } + // Finish writing the block roots and commit the remaining cold DB ops. + block_root_writer.write(&mut cold_db_ops)?; + store.cold_db.do_atomically(cold_db_ops)?; + // Warning: Critical section. We have to take care not to put any of the two databases in an // inconsistent state if the OS process dies at any point during the freezeing // procedure. diff --git a/watch/Cargo.toml b/watch/Cargo.toml index b29df14439e..3dc3b7c1905 100644 --- a/watch/Cargo.toml +++ b/watch/Cargo.toml @@ -45,3 +45,4 @@ network = { path = "../beacon_node/network" } testcontainers = { git = "https://github.com/testcontainers/testcontainers-rs/", rev = "0f2c9851" } unused_port = { path = "../common/unused_port" } task_executor = { path = "../common/task_executor" } +logging = { path = "../common/logging" } diff --git a/watch/tests/tests.rs b/watch/tests/tests.rs index a54386eb753..dc0b8af6e34 100644 --- a/watch/tests/tests.rs +++ b/watch/tests/tests.rs @@ -7,12 +7,21 @@ use beacon_chain::{ }; use eth2::{types::BlockId, BeaconNodeHttpClient, SensitiveUrl, Timeouts}; use http_api::test_utils::{create_api_server, ApiServer}; +use log::error; +use logging::test_logger; use network::NetworkReceivers; - use rand::distributions::Alphanumeric; use rand::{thread_rng, Rng}; +use std::collections::HashMap; +use std::env; +use std::net::SocketAddr; +use std::time::Duration; +use testcontainers::{clients::Cli, core::WaitFor, Image, RunnableImage}; use tokio::sync::oneshot; +use tokio::{runtime, task::JoinHandle}; +use tokio_postgres::{config::Config as PostgresConfig, Client, NoTls}; use types::{Hash256, MainnetEthSpec, Slot}; +use unused_port::unused_tcp4_port; use url::Url; use watch::{ client::WatchHttpClient, @@ -22,17 +31,6 @@ use watch::{ updater::{handler::*, run_updater, Config as UpdaterConfig, WatchSpec}, }; -use log::error; -use std::collections::HashMap; -use std::env; -use std::net::SocketAddr; -use std::time::Duration; -use tokio::{runtime, task::JoinHandle}; -use tokio_postgres::{config::Config as PostgresConfig, Client, NoTls}; -use unused_port::unused_tcp4_port; - -use testcontainers::{clients::Cli, core::WaitFor, Image, RunnableImage}; - #[derive(Debug)] pub struct Postgres(HashMap<String, String>); @@ -132,6 +130,7 @@ impl TesterBuilder { reconstruct_historic_states: true, ..ChainConfig::default() }) + .logger(test_logger()) .deterministic_keypairs(VALIDATOR_COUNT) .fresh_ephemeral_store() .build(); From 0c23c86849401ea38456e8b1b9a9b0ea4b826ea7 Mon Sep 17 00:00:00 2001 From: Philippe Schommers <philippe@schommers.be> Date: Tue, 29 Aug 2023 05:56:30 +0000 Subject: [PATCH 08/17] feat: add chiado (#4530) ## Issue Addressed N/A ## Proposed Changes Adds the Chiado (Gnosis testnet) network to the builtin one. ## Additional Info It's a fairly trivial change all things considered as the preset already exists, so shouldn't be hard to maintain. It compiles and seems to work, but I'm sure I missed something? Co-authored-by: Paul Hauner <paul@paulhauner.com> --- book/src/merge-migration.md | 3 +- book/src/run_a_node.md | 3 +- common/eth2_config/src/lib.rs | 17 ++ .../chiado/boot_enr.yaml | 8 + .../chiado/config.yaml | 154 ++++++++++++++++++ .../chiado/deploy_block.txt | 1 + common/eth2_network_config/src/lib.rs | 4 +- 7 files changed, 186 insertions(+), 4 deletions(-) create mode 100644 common/eth2_network_config/built_in_network_configs/chiado/boot_enr.yaml create mode 100644 common/eth2_network_config/built_in_network_configs/chiado/config.yaml create mode 100644 common/eth2_network_config/built_in_network_configs/chiado/deploy_block.txt diff --git a/book/src/merge-migration.md b/book/src/merge-migration.md index acca0bbeb3e..bab520b569b 100644 --- a/book/src/merge-migration.md +++ b/book/src/merge-migration.md @@ -21,7 +21,7 @@ engine to a merge-ready version. ## When? -All networks (**Mainnet**, **Goerli (Prater)**, **Ropsten**, **Sepolia**, **Kiln**, **Gnosis**) have successfully undergone the Bellatrix fork and transitioned to a post-merge Network. Your node must have a merge-ready configuration to continue operating. Table below lists the date at which Bellatrix and The Merge occurred: +All networks (**Mainnet**, **Goerli (Prater)**, **Ropsten**, **Sepolia**, **Kiln**, **Chiado**, **Gnosis**) have successfully undergone the Bellatrix fork and transitioned to a post-merge Network. Your node must have a merge-ready configuration to continue operating. Table below lists the date at which Bellatrix and The Merge occurred: <div align="center"> @@ -31,6 +31,7 @@ All networks (**Mainnet**, **Goerli (Prater)**, **Ropsten**, **Sepolia**, **Kiln | Sepolia | 20<sup>th</sup> June 2022 | 6<sup>th</sup> July 2022 | | | Goerli | 4<sup>th</sup> August 2022 | 10<sup>th</sup> August 2022 | Previously named `Prater`| | Mainnet | 6<sup>th</sup> September 2022 | 15<sup>th</sup> September 2022 | +| Chiado | 10<sup>th</sup> October 2022 | 4<sup>th</sup> November 2022 | | Gnosis| 30<sup>th</sup> November 2022 | 8<sup>th</sup> December 2022 </div> diff --git a/book/src/run_a_node.md b/book/src/run_a_node.md index a31aedf785a..1ea14273357 100644 --- a/book/src/run_a_node.md +++ b/book/src/run_a_node.md @@ -58,7 +58,8 @@ Notable flags: - `lighthouse --network mainnet`: Mainnet. - `lighthouse --network goerli`: Goerli (testnet). - `lighthouse --network sepolia`: Sepolia (testnet). - - `lighthouse --network gnosis`: Gnosis chain + - `lighthouse --network chiado`: Chiado (testnet). + - `lighthouse --network gnosis`: Gnosis chain. > Note: Using the correct `--network` flag is very important; using the wrong flag can result in penalties, slashings or lost deposits. As a rule of thumb, *always* diff --git a/common/eth2_config/src/lib.rs b/common/eth2_config/src/lib.rs index bb14d067569..6487151a92c 100644 --- a/common/eth2_config/src/lib.rs +++ b/common/eth2_config/src/lib.rs @@ -33,6 +33,13 @@ const HOLESKY_GENESIS_STATE_SOURCE: GenesisStateSource = GenesisStateSource::Url genesis_validators_root: "0x9143aa7c615a7f7115e2b6aac319c03529df8242ae705fba9df39b79c59fa8b1", }; +const CHIADO_GENESIS_STATE_SOURCE: GenesisStateSource = GenesisStateSource::Url { + // No default checkpoint sources are provided. + urls: &[], + checksum: "0xd4a039454c7429f1dfaa7e11e397ef3d0f50d2d5e4c0e4dc04919d153aa13af1", + genesis_validators_root: "0x9d642dac73058fbf39c0ae41ab1e34e4d889043cb199851ded7095bc99eb4c1e", +}; + /// The core configuration of a Lighthouse beacon node. #[derive(Debug, Clone)] pub struct Eth2Config { @@ -306,6 +313,16 @@ define_hardcoded_nets!( // Describes how the genesis state can be obtained. GenesisStateSource::IncludedBytes ), + ( + // Network name (must be unique among all networks). + chiado, + // The name of the directory in the `eth2_network_config/built_in_network_configs` + // directory where the configuration files are located for this network. + "chiado", + // Set to `true` if the genesis state can be found in the `built_in_network_configs` + // directory. + CHIADO_GENESIS_STATE_SOURCE + ), ( // Network name (must be unique among all networks). sepolia, diff --git a/common/eth2_network_config/built_in_network_configs/chiado/boot_enr.yaml b/common/eth2_network_config/built_in_network_configs/chiado/boot_enr.yaml new file mode 100644 index 00000000000..96baffde6fb --- /dev/null +++ b/common/eth2_network_config/built_in_network_configs/chiado/boot_enr.yaml @@ -0,0 +1,8 @@ +# chiado-teku-0 +- "enr:-Ly4QLYLNqrjvSxD3lpAPBUNlxa6cIbe79JqLZLFcZZjWoCjZcw-85agLUErHiygG2weRSCLnd5V460qTbLbwJQsfZkoh2F0dG5ldHOI__________-EZXRoMpAxNnBDAgAAb___________gmlkgnY0gmlwhKq7mu-Jc2VjcDI1NmsxoQP900YAYa9kdvzlSKGjVo-F3XVzATjOYp3BsjLjSophO4hzeW5jbmV0cw-DdGNwgiMog3VkcIIjKA" +# chiado-teku-1 +- "enr:-Ly4QCGeYvTCNOGKi0mKRUd45rLj96b4pH98qG7B9TCUGXGpHZALtaL2-XfjASQyhbCqENccI4PGXVqYTIehNT9KJMQgh2F0dG5ldHOI__________-EZXRoMpAxNnBDAgAAb___________gmlkgnY0gmlwhIuQrVSJc2VjcDI1NmsxoQP9iDchx2PGl3JyJ29B9fhLCvVMN6n23pPAIIeFV-sHOIhzeW5jbmV0cw-DdGNwgiMog3VkcIIjKA" +#GnosisDAO Bootnode: 3.71.132.231 +- "enr:-Ly4QAtr21x5Ps7HYhdZkIBRBgcBkvlIfEel1YNjtFWf4cV3au2LgBGICz9PtEs9-p2HUl_eME8m1WImxTxSB3AkCMwBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpAxNnBDAgAAb___________gmlkgnY0gmlwhANHhOeJc2VjcDI1NmsxoQNLp1QPV8-pyMCohOtj6xGtSBM_GtVTqzlbvNsCF4ezkYhzeW5jbmV0cwCDdGNwgiMog3VkcIIjKA" +#GnosisDAO Bootnode: 3.69.35.13 +- "enr:-Ly4QLgn8Bx6faigkKUGZQvd1HDToV2FAxZIiENK-lczruzQb90qJK-4E65ADly0s4__dQOW7IkLMW7ZAyJy2vtiLy8Bh2F0dG5ldHOIAAAAAAAAAACEZXRoMpAxNnBDAgAAb___________gmlkgnY0gmlwhANFIw2Jc2VjcDI1NmsxoQMa-fWEy9UJHfOl_lix3wdY5qust78sHAqZnWwEiyqKgYhzeW5jbmV0cwCDdGNwgiMog3VkcIIjKA" diff --git a/common/eth2_network_config/built_in_network_configs/chiado/config.yaml b/common/eth2_network_config/built_in_network_configs/chiado/config.yaml new file mode 100644 index 00000000000..47b285a654f --- /dev/null +++ b/common/eth2_network_config/built_in_network_configs/chiado/config.yaml @@ -0,0 +1,154 @@ +# Extends the mainnet preset +PRESET_BASE: gnosis +# needs to exist because of Prysm. Otherwise it conflicts with mainnet genesis +CONFIG_NAME: chiado + +# Genesis +MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 6000 +# 10 October 2022 10:00:00 GMT+0000 +MIN_GENESIS_TIME: 1665396000 +GENESIS_DELAY: 300 + +# Projected time: 2022-11-04T15:00:00.000Z, block: 680928 +TERMINAL_TOTAL_DIFFICULTY: 231707791542740786049188744689299064356246512 + +# Deposit contract +# --------------------------------------------------------------- +# NOTE: Don't use a value too high, or Teku rejects it (4294906129 NOK) +DEPOSIT_CHAIN_ID: 10200 +DEPOSIT_NETWORK_ID: 10200 +DEPOSIT_CONTRACT_ADDRESS: 0xb97036A26259B7147018913bD58a774cf91acf25 + +# Misc +# --------------------------------------------------------------- +# 2**6 (= 64) +MAX_COMMITTEES_PER_SLOT: 64 +# 2**7 (= 128) +TARGET_COMMITTEE_SIZE: 128 +# 2**11 (= 2,048) +MAX_VALIDATORS_PER_COMMITTEE: 2048 +# 2**2 (= 4) +MIN_PER_EPOCH_CHURN_LIMIT: 4 +# 2**12 (= 4096) +CHURN_LIMIT_QUOTIENT: 4096 +# See issue 563 +SHUFFLE_ROUND_COUNT: 90 +# 4 +HYSTERESIS_QUOTIENT: 4 +# 1 (minus 0.25) +HYSTERESIS_DOWNWARD_MULTIPLIER: 1 +# 5 (plus 1.25) +HYSTERESIS_UPWARD_MULTIPLIER: 5 +# Validator +# --------------------------------------------------------------- +# 2**10 (= 1024) ~1.4 hour +ETH1_FOLLOW_DISTANCE: 1024 +# 2**4 (= 16) +TARGET_AGGREGATORS_PER_COMMITTEE: 16 +# 2**0 (= 1) +RANDOM_SUBNETS_PER_VALIDATOR: 1 +# 2**8 (= 256) +EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION: 256 +# 6 (estimate from xDai mainnet) +SECONDS_PER_ETH1_BLOCK: 6 + +# Gwei values +# --------------------------------------------------------------- +# 2**0 * 10**9 (= 1,000,000,000) Gwei +MIN_DEPOSIT_AMOUNT: 1000000000 +# 2**5 * 10**9 (= 32,000,000,000) Gwei +MAX_EFFECTIVE_BALANCE: 32000000000 +# 2**4 * 10**9 (= 16,000,000,000) Gwei +EJECTION_BALANCE: 16000000000 +# 2**0 * 10**9 (= 1,000,000,000) Gwei +EFFECTIVE_BALANCE_INCREMENT: 1000000000 +# Initial values +# --------------------------------------------------------------- +# GBC area code +GENESIS_FORK_VERSION: 0x0000006f +BLS_WITHDRAWAL_PREFIX: 0x00 +# Time parameters +# --------------------------------------------------------------- +# 5 seconds +SECONDS_PER_SLOT: 5 +# 2**0 (= 1) slots 12 seconds +MIN_ATTESTATION_INCLUSION_DELAY: 1 +# 2**4 (= 16) slots 1.87 minutes +SLOTS_PER_EPOCH: 16 +# 2**0 (= 1) epochs 1.87 minutes +MIN_SEED_LOOKAHEAD: 1 +# 2**2 (= 4) epochs 7.47 minutes +MAX_SEED_LOOKAHEAD: 4 +# 2**6 (= 64) epochs ~2 hours +EPOCHS_PER_ETH1_VOTING_PERIOD: 64 +# 2**13 (= 8,192) slots ~15.9 hours +SLOTS_PER_HISTORICAL_ROOT: 8192 +# 2**8 (= 256) epochs ~8 hours +MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 +# 2**8 (= 256) epochs ~8 hours +SHARD_COMMITTEE_PERIOD: 256 +# 2**2 (= 4) epochs 7.47 minutes +MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 + +# State vector lengths +# --------------------------------------------------------------- +# 2**16 (= 65,536) epochs ~85 days +EPOCHS_PER_HISTORICAL_VECTOR: 65536 +# 2**13 (= 8,192) epochs ~10.6 days +EPOCHS_PER_SLASHINGS_VECTOR: 8192 +# 2**24 (= 16,777,216) historical roots, ~15,243 years +HISTORICAL_ROOTS_LIMIT: 16777216 +# 2**40 (= 1,099,511,627,776) validator spots +VALIDATOR_REGISTRY_LIMIT: 1099511627776 +# Reward and penalty quotients +# --------------------------------------------------------------- +# 25 +BASE_REWARD_FACTOR: 25 +# 2**9 (= 512) +WHISTLEBLOWER_REWARD_QUOTIENT: 512 +# 2**3 (= 8) +PROPOSER_REWARD_QUOTIENT: 8 +# 2**26 (= 67,108,864) +INACTIVITY_PENALTY_QUOTIENT: 67108864 +# 2**7 (= 128) (lower safety margin at Phase 0 genesis) +MIN_SLASHING_PENALTY_QUOTIENT: 128 +# 1 (lower safety margin at Phase 0 genesis) +PROPORTIONAL_SLASHING_MULTIPLIER: 1 +# Max operations per block +# --------------------------------------------------------------- +# 2**4 (= 16) +MAX_PROPOSER_SLASHINGS: 16 +# 2**1 (= 2) +MAX_ATTESTER_SLASHINGS: 2 +# 2**7 (= 128) +MAX_ATTESTATIONS: 128 +# 2**4 (= 16) +MAX_DEPOSITS: 16 +# 2**4 (= 16) +MAX_VOLUNTARY_EXITS: 16 +# Signature domains +# --------------------------------------------------------------- +DOMAIN_BEACON_PROPOSER: 0x00000000 +DOMAIN_BEACON_ATTESTER: 0x01000000 +DOMAIN_RANDAO: 0x02000000 +DOMAIN_DEPOSIT: 0x03000000 +DOMAIN_VOLUNTARY_EXIT: 0x04000000 +DOMAIN_SELECTION_PROOF: 0x05000000 +DOMAIN_AGGREGATE_AND_PROOF: 0x06000000 +DOMAIN_SYNC_COMMITTEE: 0x07000000 +DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF: 0x08000000 +DOMAIN_CONTRIBUTION_AND_PROOF: 0x09000000 + +# Altair +ALTAIR_FORK_VERSION: 0x0100006f +ALTAIR_FORK_EPOCH: 90 # Mon Oct 10 2022 12:00:00 GMT+0000 +# Bellatrix +BELLATRIX_FORK_VERSION: 0x0200006f +BELLATRIX_FORK_EPOCH: 180 # Mon Oct 10 2022 14:00:00 GMT+0000 +# Capella +CAPELLA_FORK_VERSION: 0x0300006f +CAPELLA_FORK_EPOCH: 244224 # Wed May 24 2023 13:12:00 GMT+0000 + +INACTIVITY_SCORE_BIAS: 4 +# 2**4 (= 16) +INACTIVITY_SCORE_RECOVERY_RATE: 16 diff --git a/common/eth2_network_config/built_in_network_configs/chiado/deploy_block.txt b/common/eth2_network_config/built_in_network_configs/chiado/deploy_block.txt new file mode 100644 index 00000000000..573541ac970 --- /dev/null +++ b/common/eth2_network_config/built_in_network_configs/chiado/deploy_block.txt @@ -0,0 +1 @@ +0 diff --git a/common/eth2_network_config/src/lib.rs b/common/eth2_network_config/src/lib.rs index 1206b8b8caa..769f656e85d 100644 --- a/common/eth2_network_config/src/lib.rs +++ b/common/eth2_network_config/src/lib.rs @@ -482,10 +482,10 @@ mod tests { fn hard_coded_nets_work() { for net in HARDCODED_NETS { let config = Eth2NetworkConfig::from_hardcoded_net(net) - .unwrap_or_else(|_| panic!("{:?}", net.name)); + .unwrap_or_else(|e| panic!("{:?}: {:?}", net.name, e)); // Ensure we can parse the YAML config to a chain spec. - if net.name == types::GNOSIS { + if config.config.preset_base == types::GNOSIS { config.chain_spec::<GnosisEthSpec>().unwrap(); } else { config.chain_spec::<MainnetEthSpec>().unwrap(); From 41ac9b6a08725e66b617427caa68bc413bb7f4cd Mon Sep 17 00:00:00 2001 From: Jimmy Chen <jchen.tc@gmail.com> Date: Wed, 30 Aug 2023 21:41:35 +1000 Subject: [PATCH 09/17] Pin foundry toolchain version to fix stuck CI jobs --- .github/workflows/test-suite.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 91a0b734537..5819f804847 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -67,6 +67,8 @@ jobs: run: rustup update stable - name: Install Foundry (anvil) uses: foundry-rs/foundry-toolchain@v1 + with: + version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d - name: Run tests in release run: make test-release release-tests-windows: @@ -88,6 +90,8 @@ jobs: npm config set msvs_version 2019 - name: Install Foundry (anvil) uses: foundry-rs/foundry-toolchain@v1 + with: + version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d - name: Install make run: choco install -y make - uses: KyleMayes/install-llvm-action@v1 @@ -143,6 +147,8 @@ jobs: run: rustup update stable - name: Install Foundry (anvil) uses: foundry-rs/foundry-toolchain@v1 + with: + version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d - name: Run tests in debug run: make test-debug state-transition-vectors-ubuntu: @@ -189,6 +195,8 @@ jobs: run: rustup update stable - name: Install Foundry (anvil) uses: foundry-rs/foundry-toolchain@v1 + with: + version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d - name: Run the beacon chain sim that starts from an eth1 contract run: cargo run --release --bin simulator eth1-sim merge-transition-ubuntu: @@ -201,6 +209,8 @@ jobs: run: rustup update stable - name: Install Foundry (anvil) uses: foundry-rs/foundry-toolchain@v1 + with: + version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d - name: Run the beacon chain sim and go through the merge transition run: cargo run --release --bin simulator eth1-sim --post-merge no-eth1-simulator-ubuntu: @@ -223,6 +233,8 @@ jobs: run: rustup update stable - name: Install Foundry (anvil) uses: foundry-rs/foundry-toolchain@v1 + with: + version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d - name: Run the syncing simulator run: cargo run --release --bin simulator syncing-sim doppelganger-protection-test: From e99ba3a14e5f85011ffed081c9c4cb1dabb772fe Mon Sep 17 00:00:00 2001 From: Paul Hauner <paul@paulhauner.com> Date: Thu, 31 Aug 2023 02:12:35 +0000 Subject: [PATCH 10/17] Release v4.4.0 (#4673) ## Issue Addressed NA ## Proposed Changes Bump versions from `v4.3.0` to `v4.4.0`. ## Additional Info NA --- Cargo.lock | 8 ++++---- beacon_node/Cargo.toml | 2 +- boot_node/Cargo.toml | 2 +- common/lighthouse_version/src/lib.rs | 4 ++-- lcli/Cargo.toml | 2 +- lighthouse/Cargo.toml | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 27d7b655e30..9afe6fe183f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -606,7 +606,7 @@ dependencies = [ [[package]] name = "beacon_node" -version = "4.3.0" +version = "4.4.0" dependencies = [ "beacon_chain", "clap", @@ -805,7 +805,7 @@ dependencies = [ [[package]] name = "boot_node" -version = "4.3.0" +version = "4.4.0" dependencies = [ "beacon_node", "clap", @@ -3880,7 +3880,7 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "lcli" -version = "4.3.0" +version = "4.4.0" dependencies = [ "account_utils", "beacon_chain", @@ -4467,7 +4467,7 @@ dependencies = [ [[package]] name = "lighthouse" -version = "4.3.0" +version = "4.4.0" dependencies = [ "account_manager", "account_utils", diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index a0e46705b74..e7d24e8ab46 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "beacon_node" -version = "4.3.0" +version = "4.4.0" authors = [ "Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com", diff --git a/boot_node/Cargo.toml b/boot_node/Cargo.toml index c3dd3bd1936..3db521c0e27 100644 --- a/boot_node/Cargo.toml +++ b/boot_node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "boot_node" -version = "4.3.0" +version = "4.4.0" authors = ["Sigma Prime <contact@sigmaprime.io>"] edition = "2021" diff --git a/common/lighthouse_version/src/lib.rs b/common/lighthouse_version/src/lib.rs index e874432fbca..65f57531d33 100644 --- a/common/lighthouse_version/src/lib.rs +++ b/common/lighthouse_version/src/lib.rs @@ -17,8 +17,8 @@ pub const VERSION: &str = git_version!( // NOTE: using --match instead of --exclude for compatibility with old Git "--match=thiswillnevermatchlol" ], - prefix = "Lighthouse/v4.3.0-", - fallback = "Lighthouse/v4.3.0" + prefix = "Lighthouse/v4.4.0-", + fallback = "Lighthouse/v4.4.0" ); /// Returns `VERSION`, but with platform information appended to the end. diff --git a/lcli/Cargo.toml b/lcli/Cargo.toml index f9d0a6a31c7..8b838ac6f82 100644 --- a/lcli/Cargo.toml +++ b/lcli/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lcli" description = "Lighthouse CLI (modeled after zcli)" -version = "4.3.0" +version = "4.4.0" authors = ["Paul Hauner <paul@paulhauner.com>"] edition = "2021" diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index 8003236f2df..d836c4d96de 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lighthouse" -version = "4.3.0" +version = "4.4.0" authors = ["Sigma Prime <contact@sigmaprime.io>"] edition = "2021" autotests = false From 74eb26764371ba86d32ed7fe2b2fe6a83562b053 Mon Sep 17 00:00:00 2001 From: Michael Sproul <michael@sigmaprime.io> Date: Thu, 31 Aug 2023 11:18:00 +0000 Subject: [PATCH 11/17] Remove double-locking deadlock from HTTP API (#4687) ## Issue Addressed Fix a deadlock introduced in #4236 which was caught during the v4.4.0 release testing cycle (with thanks to @paulhauner and `gdb`). ## Proposed Changes Avoid re-locking the fork choice read lock when querying a state by root in the HTTP API. This avoids a deadlock due to the lock already being held. ## Additional Info The [RwLock docs](https://docs.rs/lock_api/latest/lock_api/struct.RwLock.html#method.read) explicitly advise against re-locking: > Note that attempts to recursively acquire a read lock on a RwLock when the current thread already holds one may result in a deadlock. --- beacon_node/http_api/src/state_id.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/beacon_node/http_api/src/state_id.rs b/beacon_node/http_api/src/state_id.rs index 5e86053771e..1a76333e2d4 100644 --- a/beacon_node/http_api/src/state_id.rs +++ b/beacon_node/http_api/src/state_id.rs @@ -89,9 +89,7 @@ impl StateId { } else { // This block is either old and finalized, or recent and unfinalized, so // it's safe to fallback to the optimistic status of the finalized block. - chain - .canonical_head - .fork_choice_read_lock() + fork_choice .is_optimistic_or_invalid_block(&hot_summary.latest_block_root) .map_err(BeaconChainError::ForkChoiceError) .map_err(warp_utils::reject::beacon_chain_error)? From 2841f60686d642fcc0785c884d43e34e47a800dc Mon Sep 17 00:00:00 2001 From: Michael Sproul <michael@sigmaprime.io> Date: Mon, 4 Sep 2023 02:56:52 +0000 Subject: [PATCH 12/17] Release v4.4.1 (#4690) ## Proposed Changes New release to replace the cancelled v4.4.0 release. This release includes the bugfix #4687 which avoids a deadlock that was present in v4.4.0. ## Additional Info Awaiting testing over the weekend this will be merged Monday September 4th. --- Cargo.lock | 8 ++++---- beacon_node/Cargo.toml | 2 +- boot_node/Cargo.toml | 2 +- common/lighthouse_version/src/lib.rs | 4 ++-- lcli/Cargo.toml | 2 +- lighthouse/Cargo.toml | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9afe6fe183f..ad7b6fe14ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -606,7 +606,7 @@ dependencies = [ [[package]] name = "beacon_node" -version = "4.4.0" +version = "4.4.1" dependencies = [ "beacon_chain", "clap", @@ -805,7 +805,7 @@ dependencies = [ [[package]] name = "boot_node" -version = "4.4.0" +version = "4.4.1" dependencies = [ "beacon_node", "clap", @@ -3880,7 +3880,7 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "lcli" -version = "4.4.0" +version = "4.4.1" dependencies = [ "account_utils", "beacon_chain", @@ -4467,7 +4467,7 @@ dependencies = [ [[package]] name = "lighthouse" -version = "4.4.0" +version = "4.4.1" dependencies = [ "account_manager", "account_utils", diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index e7d24e8ab46..3f4c93fa4df 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "beacon_node" -version = "4.4.0" +version = "4.4.1" authors = [ "Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com", diff --git a/boot_node/Cargo.toml b/boot_node/Cargo.toml index 3db521c0e27..fcbe8f84d27 100644 --- a/boot_node/Cargo.toml +++ b/boot_node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "boot_node" -version = "4.4.0" +version = "4.4.1" authors = ["Sigma Prime <contact@sigmaprime.io>"] edition = "2021" diff --git a/common/lighthouse_version/src/lib.rs b/common/lighthouse_version/src/lib.rs index 65f57531d33..567714e5ca9 100644 --- a/common/lighthouse_version/src/lib.rs +++ b/common/lighthouse_version/src/lib.rs @@ -17,8 +17,8 @@ pub const VERSION: &str = git_version!( // NOTE: using --match instead of --exclude for compatibility with old Git "--match=thiswillnevermatchlol" ], - prefix = "Lighthouse/v4.4.0-", - fallback = "Lighthouse/v4.4.0" + prefix = "Lighthouse/v4.4.1-", + fallback = "Lighthouse/v4.4.1" ); /// Returns `VERSION`, but with platform information appended to the end. diff --git a/lcli/Cargo.toml b/lcli/Cargo.toml index 8b838ac6f82..704e0a657e8 100644 --- a/lcli/Cargo.toml +++ b/lcli/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lcli" description = "Lighthouse CLI (modeled after zcli)" -version = "4.4.0" +version = "4.4.1" authors = ["Paul Hauner <paul@paulhauner.com>"] edition = "2021" diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index d836c4d96de..012276f4cec 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lighthouse" -version = "4.4.0" +version = "4.4.1" authors = ["Sigma Prime <contact@sigmaprime.io>"] edition = "2021" autotests = false From 291ff640f1ddac3e2b93469d41a298671bc4f109 Mon Sep 17 00:00:00 2001 From: chonghe <tanck2005@gmail.com> Date: Wed, 6 Sep 2023 04:37:02 +0000 Subject: [PATCH 13/17] Minor revision to Lighthouse Book on validator-manager (#4638) Correct the formatting and remove `http-port 5062` to make the command simpler Co-authored-by: chonghe <44791194+chong-he@users.noreply.github.com> --- book/src/faq.md | 40 +++++++++++++++++++--------- book/src/validator-manager-create.md | 2 -- book/src/validator-manager-move.md | 7 +++-- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/book/src/faq.md b/book/src/faq.md index 15c5757064f..b3f3d55edf1 100644 --- a/book/src/faq.md +++ b/book/src/faq.md @@ -40,6 +40,7 @@ - [How do I check the version of Lighthouse that is running?](#misc-version) - [Does Lighthouse have pruning function like the execution client to save disk space?](#misc-prune) - [Can I use a HDD for the freezer database and only have the hot db on SSD?](#misc-freezer) +- [Can Lighthouse log in local timestamp instead of UTC?](#misc-timestamp) ## Beacon Node @@ -436,16 +437,14 @@ Monitoring](./validator-monitoring.md) for more information. Lighthouse has also ### <a name="net-bn-vc"></a> My beacon node and validator client are on different servers. How can I point the validator client to the beacon node? -The settings are as follows: - -1. On the beacon node: - - Specify `lighthouse bn --http-address local_IP` so that the beacon node is listening on the local network rather than on the `localhost`. - -1. On the validator client: +The setting on the beacon node is the same for both cases below. In the beacon node, specify `lighthouse bn --http-address local_IP` so that the beacon node is listening on the local network rather than `localhost`. You can find the `local_IP` by running the command `hostname -I | awk '{print $1}'` on the server running the beacon node. +1. If the beacon node and validator clients are on different servers *in the same network*, the setting in the validator client is as follows: + Use the flag `--beacon-nodes` to point to the beacon node. For example, `lighthouse vc --beacon-nodes http://local_IP:5052` where `local_IP` is the local IP address of the beacon node and `5052` is the default `http-port` of the beacon node. + If you have firewall setup, e.g., `ufw`, you will need to allow port 5052 (assuming that the default port is used) with `sudo ufw allow 5052`. Note: this will allow all IP addresses to access the HTTP API of the beacon node. If you are on an untrusted network (e.g., a university or public WiFi) or the host is exposed to the internet, use apply IP-address filtering as described later in this section. + You can test that the setup is working with by running the following command on the validator client host: ```bash @@ -453,8 +452,25 @@ The settings are as follows: ``` You can refer to [Redundancy](./redundancy.md) for more information. - - It is also worth noting that the `--beacon-nodes` flag can also be used for redundancy of beacon nodes. For example, let's say you have a beacon node and a validator client running on the same host, and a second beacon node on another server as a backup. In this case, you can use `lighthouse vc --beacon-nodes http://localhost:5052, http://local_IP:5052` on the validator client. + +2. If the beacon node and validator clients are on different servers *and different networks*, it is necessary to perform port forwarding of the SSH port (e.g., the default port 22) on the router, and also allow firewall on the SSH port. The connection can be established via port forwarding on the router. + + + + In the validator client, use the flag `--beacon-nodes` to point to the beacon node. However, since the beacon node and the validator client are on different networks, the IP address to use is the public IP address of the beacon node, i.e., `lighthouse vc --beacon-nodes http://public_IP:5052`. You can get the public IP address of the beacon node by running the command ` dig +short myip.opendns.com @resolver1.opendns.com` on the server running the beacon node. + + Additionally, port forwarding of port 5052 on the router connected to the beacon node is required for the vc to connect to the bn. To do port forwarding, refer to [how to open ports](./advanced_networking.md#how-to-open-ports). + + + If you have firewall setup, e.g., `ufw`, you will need to allow connections to port 5052 (assuming that the default port is used). Since the beacon node HTTP/HTTPS API is public-facing (i.e., the 5052 port is now exposed to the internet due to port forwarding), we strongly recommend users to apply IP-address filtering to the BN/VC connection from malicious actors. This can be done using the command: + + ``` + sudo ufw allow from vc_IP_address proto tcp to any port 5052 + ``` + where `vc_IP_address` is the public IP address of the validator client. The command will only allow connections to the beacon node from the validator client IP address to prevent malicious attacks on the beacon node over the internet. + + +It is also worth noting that the `--beacon-nodes` flag can also be used for redundancy of beacon nodes. For example, let's say you have a beacon node and a validator client running on the same host, and a second beacon node on another server as a backup. In this case, you can use `lighthouse vc --beacon-nodes http://localhost:5052, http://IP-address:5052` on the validator client. ### <a name="net-ip"></a> Should I do anything to the beacon node or validator client settings if I have a relocation of the node / change of IP address? No. Lighthouse will auto-detect the change and update your Ethereum Node Record (ENR). You just need to make sure you are not manually setting the ENR with `--enr-address` (which, for common use cases, this flag is not used). @@ -513,11 +529,9 @@ There is no pruning of Lighthouse database for now. However, since v4.2.0, a fea Yes, you can do so by using the flag `--freezer-dir /path/to/freezer_db` in the beacon node. +### <a name="misc-timestamp"></a> Can Lighthouse log in local timestamp instead of UTC? - - - - +The reason why Lighthouse logs in UTC is due to the dependency on an upstream library that is [yet to be resolved](https://github.com/sigp/lighthouse/issues/3130). Alternatively, using the flag `disable-log-timestamp` in combination with systemd will suppress the UTC timestamps and print the logs in local timestamps. diff --git a/book/src/validator-manager-create.md b/book/src/validator-manager-create.md index 779c159276e..0cec150dab7 100644 --- a/book/src/validator-manager-create.md +++ b/book/src/validator-manager-create.md @@ -124,7 +124,6 @@ The command will create two files: The VC which will receive the validators needs to have the following flags at a minimum: - `--http` -- `--http-port 5062` - `--enable-doppelganger-protection` Therefore, the VC command might look like: @@ -133,7 +132,6 @@ Therefore, the VC command might look like: lighthouse \ vc \ --http \ - --http-port 5062 \ --enable-doppelganger-protection ``` diff --git a/book/src/validator-manager-move.md b/book/src/validator-manager-move.md index 98932604d5d..15089d65c5d 100644 --- a/book/src/validator-manager-move.md +++ b/book/src/validator-manager-move.md @@ -69,7 +69,6 @@ In reality, many host configurations are possible. For example: The source VC needs to have the following flags at a minimum: - `--http` -- `--http-port 5062` - `--http-allow-keystore-export` Therefore, the source VC command might look like: @@ -78,7 +77,6 @@ Therefore, the source VC command might look like: lighthouse \ vc \ --http \ - --http-port 5062 \ --http-allow-keystore-export ``` @@ -87,7 +85,6 @@ lighthouse \ The destination VC needs to have the following flags at a minimum: - `--http` -- `--http-port 5062` - `--enable-doppelganger-protection` Therefore, the destination VC command might look like: @@ -96,7 +93,6 @@ Therefore, the destination VC command might look like: lighthouse \ vc \ --http \ - --http-port 5062 \ --enable-doppelganger-protection ``` @@ -167,6 +163,8 @@ At the same time, `lighthouse vc` will log: INFO Importing keystores via standard HTTP API, count: 1 INFO Enabled validator voting_pubkey: 0xab6e29f1b98fedfca878edce2b471f1b5ee58ee4c3bd216201f98254ef6f6eac40a53d74c8b7da54f51d3e85cacae92f, signing_method: local_keystore INFO Modified key_cache saved successfully +``` + Once the operation completes successfully, there is nothing else to be done. The validators have been removed from the `src-host` and enabled at the `dest-host`. If the `--enable-doppelganger-protection` flag was used it may take 2-3 epochs @@ -183,6 +181,7 @@ lighthouse \ --dest-vc-token ~/.lighthouse/mainnet/validators/api-token.txt \ --validators 0x9096aab771e44da149bd7c9926d6f7bb96ef465c0eeb4918be5178cd23a1deb4aec232c61d85ff329b54ed4a3bdfff3a,0x90fc4f72d898a8f01ab71242e36f4545aaf87e3887be81632bb8ba4b2ae8fb70753a62f866344d7905e9a07f5a9cdda1 ``` + Any errors encountered during the operation should include information on how to proceed. Assistance is also available on our [Discord](https://discord.gg/cyAszAh). \ No newline at end of file From 48e2b205e88dfd16a74f2ceeba74ab3e20c7346d Mon Sep 17 00:00:00 2001 From: Daichuan Wu <109904112+daichuanwu21@users.noreply.github.com> Date: Wed, 6 Sep 2023 04:37:03 +0000 Subject: [PATCH 14/17] Fix some typos in "Advanced Networking" documentation (#4672) ## Issue Addressed N/A ## Proposed Changes The current Advanced Networking page references the ["--listen-addresses"](https://github.com/sigp/lighthouse/blob/14924dbc955732a742d52eafcad32cb0f790f027/book/src/advanced_networking.md?plain=1#L124C8-L124C8) argument, which does not exist in the beacon node. This PR changes such instances of "--listen-addresses" to "--listen-address". Additionally, the page mentions using sockets that [both listen to IPv6](https://github.com/sigp/lighthouse/blob/14924dbc955732a742d52eafcad32cb0f790f027/book/src/advanced_networking.md?plain=1#L151) in a dual-stack setup? Hence, this PR also changes said line to "using one socket for IPv4 and another socket for IPv6". ## Additional Info None. --- book/src/advanced_networking.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/book/src/advanced_networking.md b/book/src/advanced_networking.md index ba07a6f87f9..b1f05450c48 100644 --- a/book/src/advanced_networking.md +++ b/book/src/advanced_networking.md @@ -121,13 +121,13 @@ nodes that do not run directly on a public network. To listen over only IPv6 use the same parameters as done when listening over IPv4 only: -- `--listen-addresses :: --port 9909` will listen over IPv6 using port `9909` for +- `--listen-address :: --port 9909` will listen over IPv6 using port `9909` for TCP and UDP. -- `--listen-addresses :: --port 9909 --discovery-port 9999` will listen over +- `--listen-address :: --port 9909 --discovery-port 9999` will listen over IPv6 using port `9909` for TCP and port `9999` for UDP. To listen over both IPv4 and IPv6: -- Set two listening addresses using the `--listen-addresses` flag twice ensuring +- Set two listening addresses using the `--listen-address` flag twice ensuring the two addresses are one IPv4, and the other IPv6. When doing so, the `--port` and `--discovery-port` flags will apply exclusively to IPv4. Note that this behaviour differs from the Ipv6 only case described above. @@ -139,16 +139,16 @@ To listen over both IPv4 and IPv6: ##### Configuration Examples -- `--listen-addresses :: --listen-addresses 0.0.0.0 --port 9909` will listen +- `--listen-address :: --listen-address 0.0.0.0 --port 9909` will listen over IPv4 using port `9909` for TCP and UDP. It will also listen over IPv6 but using the default value for `--port6` for UDP and TCP (`9090`). -- `--listen-addresses :: --listen-addresses --port 9909 --discovery-port6 9999` +- `--listen-address :: --listen-address --port 9909 --discovery-port6 9999` will have the same configuration as before except for the IPv6 UDP socket, which will use port `9999`. #### Configuring Lighthouse to advertise IPv6 reachable addresses Lighthouse supports IPv6 to connect to other nodes both over IPv6 exclusively, -and dual stack using one socket for IPv6 and another socket for IPv6. In both +and dual stack using one socket for IPv4 and another socket for IPv6. In both scenarios, the previous sections still apply. In summary: > Beacon nodes must advertise their publicly reachable socket address From 0caf2af7712cf02e32c9af20aa1a08b545f441a3 Mon Sep 17 00:00:00 2001 From: Ricki Moore <mrricki.m.usmc@gmail.com> Date: Wed, 6 Sep 2023 04:37:04 +0000 Subject: [PATCH 15/17] Feat: siren faq update (#4685) ## Issue Addressed Siren FAQ requires more information regarding network connections to lighthouse BN/VC ## Proposed Changes Added more info regarding port, BN/VC flags, ssh tunneling and VPNs access --- book/src/ui-configuration.md | 23 ++++++++---------- book/src/ui-faqs.md | 47 +++++++++++++++++++++++++++++++----- 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/book/src/ui-configuration.md b/book/src/ui-configuration.md index 98f30413913..1a5daa54418 100644 --- a/book/src/ui-configuration.md +++ b/book/src/ui-configuration.md @@ -9,23 +9,20 @@ following configuration screen. ## Connecting to the Clients -This allows you to enter the address and ports of the associated Lighthouse +Both the Beacon node and the Validator client need to have their HTTP APIs enabled. These ports should be accessible from the computer running Siren. This allows you to enter the address and ports of the associated Lighthouse Beacon node and Lighthouse Validator client. -> The Beacon Node must be run with the `--gui` flag set. +To enable the HTTP API for the beacon node, utilize the `--gui` CLI flag. This action ensures that the HTTP API can be accessed by other software on the same machine. -If you run Siren in the browser (by entering `localhost` in the browser), you will need to allow CORS in the HTTP API. This can be done by adding the flag `--http-allow-origin "*"` for both beacon node and validator client. If you would like to access Siren beyond the local computer, we recommend using an SSH tunnel. This requires a tunnel for 3 ports: `80` (assuming the port is unchanged as per the [installation guide](./ui-installation.md#docker-recommended), `5052` (for beacon node) and `5062` (for validator client). You can use the command below to perform SSH tunneling: -```bash -ssh -N -L 80:127.0.0.1:80 -L 5052:127.0.0.1:5052 -L 5062:127.0.0.1:5062 username@local_ip -``` +> The Beacon Node must be run with the `--gui` flag set. -where `username` is the username of the server and `local_ip` is the local IP address of the server. Note that with the `-N` option in an SSH session, you will not be able to execute commands in the CLI to avoid confusion with ordinary shell sessions. The connection will appear to be "hung" upon a successful connection, but that is normal. Once you have successfully connected to the server via SSH tunneling, you should be able to access Siren by entering `localhost` in a web browser. +If you require accessibility from another machine within the network, configure the `--http-address` to match the local LAN IP of the system running the Beacon Node and Validator Client. -You can also access Siren using the app downloaded in the [Siren release page](https://github.com/sigp/siren/releases). To access Siren beyond the local computer, you can use SSH tunneling for ports `5052` and `5062` using the command: +> To access from another machine on the same network (192.168.0.200) set the Beacon Node and Validator Client `--http-address` as `192.168.0.200`. -```bash -ssh -N -L 5052:127.0.0.1:5052 -L 5062:127.0.0.1:5062 username@local_ip -``` +In a similar manner, the validator client requires activation of the `--http` flag, along with the optional consideration of configuring the `--http-address` flag. If `--http-address` flag is set on the Validator Client, then the `--unencrypted-http-transport` flag is required as well. These settings will ensure compatibility with Siren's connectivity requirements. + +If you run Siren in the browser (by entering `localhost` in the browser), you will need to allow CORS in the HTTP API. This can be done by adding the flag `--http-allow-origin "*"` for both beacon node and validator client. A green tick will appear once Siren is able to connect to both clients. You can specify different ports for each client by clicking on the advanced tab. @@ -37,7 +34,7 @@ The API Token is a secret key that allows you to connect to the validator client. The validator client's HTTP API is guarded by this key because it contains sensitive validator information and the ability to modify validators. Please see [`Validator Authorization`](./api-vc-auth-header.md) -for further details. +for further details. Siren requires this token in order to connect to the Validator client. The token is located in the default data directory of the validator @@ -49,7 +46,7 @@ entered. ## Name -This is your name, it can be modified and is solely used for aesthetics. +This is your name, it can be modified and is solely used for aesthetics. ## Device diff --git a/book/src/ui-faqs.md b/book/src/ui-faqs.md index f4cbf1c40ac..92e06270c2e 100644 --- a/book/src/ui-faqs.md +++ b/book/src/ui-faqs.md @@ -1,7 +1,7 @@ # Frequently Asked Questions ## 1. Are there any requirements to run Siren? -Yes, Siren requires Lighthouse v3.5.1 or higher to function properly. These releases can be found on the [releases](https://github.com/sigp/lighthouse/releases) page of the Lighthouse repository. +Yes, the most current Siren version requires Lighthouse v4.3.0 or higher to function properly. These releases can be found on the [releases](https://github.com/sigp/lighthouse/releases) page of the Lighthouse repository. ## 2. Where can I find my API token? The required Api token may be found in the default data directory of the validator client. For more information please refer to the lighthouse ui configuration [`api token section`](./api-vc-auth-header.md). @@ -9,13 +9,41 @@ The required Api token may be found in the default data directory of the validat ## 3. How do I fix the Node Network Errors? If you receive a red notification with a BEACON or VALIDATOR NODE NETWORK ERROR you can refer to the lighthouse ui configuration and [`connecting to clients section`](./ui-configuration.md#connecting-to-the-clients). -## 4. How do I change my Beacon or Validator address after logging in? -Once you have successfully arrived to the main dashboard, use the sidebar to access the settings view. In the top right hand corner there is a `Configuration` action button that will redirect you back to the configuration screen where you can make appropriate changes. +## 4. How do I connect Siren to Lighthouse from a different computer on the same network? +The most effective approach to enable access for a local network computer to Lighthouse's HTTP API ports is by configuring the `--http-address` to match the local LAN IP of the system running the beacon node and validator client. For instance, if the said node operates at `192.168.0.200`, this IP can be specified using the `--http-address` parameter as `--http-address 192.168.0.200`. +Subsequently, by designating the host as `192.168.0.200`, you can seamlessly connect Siren to this specific beacon node and validator client pair from any computer situated within the same network. -## 5. Why doesn't my validator balance graph show any data? -If your graph is not showing data, it usually means your validator node is still caching data. The application must wait at least 3 epochs before it can render any graphical visualizations. This could take up to 20min. +## 5. How can I use Siren to monitor my validators remotely when I am not at home? + +There are two primary methods to access your Beacon Node and Validator Client remotely: setting up a VPN or utilizing SSH-reverse tunneling. + +Most contemporary home routers provide options for VPN access in various ways. A VPN permits a remote computer to establish a connection with internal computers within a home network. With a VPN configuration in place, connecting to the VPN enables you to treat your computer as if it is part of your local home network. The connection process involves following the setup steps for connecting via another machine on the same network on the Siren configuration page and [`connecting to clients section`](./ui-configuration.md#connecting-to-the-clients). + +In the absence of a VPN, an alternative approach involves utilizing an SSH tunnel. To achieve this, you need remote SSH access to the computer hosting the Beacon Node and Validator Client pair (which necessitates a port forward in your router). In this context, while it is not obligatory to set a `--http-address` flag on the Beacon Node and Validator Client, you can configure an SSH tunnel to the local ports on the node and establish a connection through the tunnel. For instructions on setting up an SSH tunnel, refer to [`Connecting Siren via SSH tunnel`](./ui-faqs.md#6-how-do-i-connect-siren-to-lighthouse-via-a-ssh-tunnel) for detailed guidance. + +## 6. How do I connect Siren to Lighthouse via a ssh tunnel? +If you would like to access Siren beyond the local network (i.e across the internet), we recommend using an SSH tunnel. This requires a tunnel for 3 ports: `80` (assuming the port is unchanged as per the [installation guide](./ui-installation.md#docker-recommended), `5052` (for beacon node) and `5062` (for validator client). You can use the command below to perform SSH tunneling: + +```bash + +ssh -N -L 80:127.0.0.1:80 -L 5052:127.0.0.1:5052 -L 5062:127.0.0.1:5062 username@local_ip + +``` + + +Where `username` is the username of the server and `local_ip` is the local IP address of the server. Note that with the `-N` option in an SSH session, you will not be able to execute commands in the CLI to avoid confusion with ordinary shell sessions. The connection will appear to be "hung" upon a successful connection, but that is normal. Once you have successfully connected to the server via SSH tunneling, you should be able to access Siren by entering `localhost` in a web browser. -## 4. Does Siren support reverse proxy or DNS named addresses? + +You can also access Siren using the app downloaded in the [Siren release page](https://github.com/sigp/siren/releases). To access Siren beyond the local computer, you can use SSH tunneling for ports `5052` and `5062` using the command: + + +```bash + +ssh -N -L 5052:127.0.0.1:5052 -L 5062:127.0.0.1:5062 username@local_ip + +``` + +## 7. Does Siren support reverse proxy or DNS named addresses? Yes, if you need to access your beacon or validator from an address such as `https://merp-server:9909/eth2-vc` you should follow the following steps for configuration: 1. Toggle `https` as your protocol 2. Add your address as `merp-server/eth2-vc` @@ -24,3 +52,10 @@ Yes, if you need to access your beacon or validator from an address such as `htt If you have configured it correctly you should see a green checkmark indicating Siren is now connected to your Validator Client and Beacon Node. If you have separate address setups for your Validator Client and Beacon Node respectively you should access the `Advance Settings` on the configuration and repeat the steps above for each address. + + +## 8. How do I change my Beacon or Validator address after logging in? +Once you have successfully arrived to the main dashboard, use the sidebar to access the settings view. In the top right hand corner there is a `Configuration` action button that will redirect you back to the configuration screen where you can make appropriate changes. + +## 9. Why doesn't my validator balance graph show any data? +If your graph is not showing data, it usually means your validator node is still caching data. The application must wait at least 3 epochs before it can render any graphical visualizations. This could take up to 20min. From 1ff4033830a21d076ea0c6b2720bef28f342e4e4 Mon Sep 17 00:00:00 2001 From: Jimmy Chen <jchen.tc@gmail.com> Date: Wed, 6 Sep 2023 04:37:05 +0000 Subject: [PATCH 16/17] Remove Node.js from release-tests CI job since we no longer use ganache (#4691) ## Issue Addressed I noticed a node.js version warning on our CI, and thought about updating the version to get rid of this warning, but then realized we may not need node.js anymore now we're using `anvil` instead of `ganache`. > The following actions uses node12 which is deprecated and will be forced to run on node16: actions/setup-node@v2. For more info: https://github.blog/changelog/2023-06-13-github-actions-all-actions-will-run-on-node16-instead-of-node12-by-default/ --- .github/workflows/test-suite.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 5819f804847..1d9856b93cf 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -80,14 +80,6 @@ jobs: - name: Get latest version of stable Rust if: env.SELF_HOSTED_RUNNERS == false run: rustup update stable - - name: Use Node.js - uses: actions/setup-node@v2 - with: - node-version: '14' - - name: Install windows build tools - run: | - choco install python visualstudio2019-workload-vctools -y - npm config set msvs_version 2019 - name: Install Foundry (anvil) uses: foundry-rs/foundry-toolchain@v1 with: From d4aedab21fd9ff78caef4889e72f8d764ed5915e Mon Sep 17 00:00:00 2001 From: Jimmy Chen <jchen.tc@gmail.com> Date: Mon, 11 Sep 2023 12:49:31 +1000 Subject: [PATCH 17/17] Fix some compilation errors in tests --- .../tests/broadcast_validation_tests.rs | 8 ++--- beacon_node/http_api/tests/tests.rs | 20 ++++++------ common/eth2/src/lib.rs | 2 +- common/eth2/src/types.rs | 20 ++++++++++++ consensus/types/src/blob_sidecar.rs | 29 +---------------- consensus/types/src/sidecar.rs | 32 ++++++++++++++++++- 6 files changed, 66 insertions(+), 45 deletions(-) diff --git a/beacon_node/http_api/tests/broadcast_validation_tests.rs b/beacon_node/http_api/tests/broadcast_validation_tests.rs index be0e30a84d4..fe300ae5e1d 100644 --- a/beacon_node/http_api/tests/broadcast_validation_tests.rs +++ b/beacon_node/http_api/tests/broadcast_validation_tests.rs @@ -912,19 +912,19 @@ pub async fn blinded_gossip_full_pass_ssz() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let ((block, _), _): ((SignedBlindedBeaconBlock<E>, _), _) = - tester.harness.make_blinded_block(state_a, slot_b).await; + let (block_contents_tuple, _) = tester.harness.make_blinded_block(state_a, slot_b).await; + let block_contents = block_contents_tuple.into(); let response: Result<(), eth2::Error> = tester .client - .post_beacon_blinded_blocks_v2_ssz(&block, validation_level) + .post_beacon_blinded_blocks_v2_ssz(&block_contents, validation_level) .await; assert!(response.is_ok()); assert!(tester .harness .chain - .block_is_known_to_fork_choice(&block.canonical_root())); + .block_is_known_to_fork_choice(&block_contents.signed_block().canonical_root())); } /// This test checks that a block that is **invalid** from a gossip perspective gets rejected when using `broadcast_validation=consensus`. diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 7c2cd917b64..40e15752f1c 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -2656,13 +2656,10 @@ impl ApiTester { .get_validator_blinded_blocks::<E, Payload>(slot, &randao_reveal, None) .await .unwrap() - .data - .deconstruct() - .0; + .data; - let signed_block = block.sign(&sk, &fork, genesis_validators_root, &self.chain.spec); let signed_block_contents = - SignedBlockContents::<E, Payload>::Block(signed_block.clone()); + block.sign(&sk, &fork, genesis_validators_root, &self.chain.spec); self.client .post_beacon_blinded_blocks(&signed_block_contents) @@ -2670,6 +2667,7 @@ impl ApiTester { .unwrap(); // This converts the generic `Payload` to a concrete type for comparison. + let signed_block = signed_block_contents.deconstruct().0; let head_block = SignedBeaconBlock::from(signed_block.clone()); assert_eq!(head_block, signed_block); @@ -2715,23 +2713,23 @@ impl ApiTester { sk.sign(message).into() }; - let block = self + let block_contents = self .client .get_validator_blinded_blocks::<E, Payload>(slot, &randao_reveal, None) .await .unwrap() - .data - .deconstruct() - .0; + .data; - let signed_block = block.sign(&sk, &fork, genesis_validators_root, &self.chain.spec); + let signed_block_contents = + block_contents.sign(&sk, &fork, genesis_validators_root, &self.chain.spec); self.client - .post_beacon_blinded_blocks_ssz(&signed_block) + .post_beacon_blinded_blocks_ssz(&signed_block_contents) .await .unwrap(); // This converts the generic `Payload` to a concrete type for comparison. + let signed_block = signed_block_contents.deconstruct().0; let head_block = SignedBeaconBlock::from(signed_block.clone()); assert_eq!(head_block, signed_block); diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index be03b96a62f..d19a2981290 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -751,7 +751,7 @@ impl BeaconNodeHttpClient { /// Returns `Ok(None)` on a 404 error. pub async fn post_beacon_blinded_blocks_ssz<T: EthSpec, Payload: AbstractExecPayload<T>>( &self, - block: &SignedBlindedBlockContents<T>, + block: &SignedBlockContents<T, Payload>, ) -> Result<(), Error> { let mut path = self.eth_path(V1)?; diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 7aea184e942..53b57f456f0 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -1431,6 +1431,26 @@ impl<T: EthSpec, Payload: AbstractExecPayload<T>> BlockContents<T, Payload> { BlockContents::Block(block) => (block, None), } } + + /// Signs `self`, producing a `SignedBlockContents`. + pub fn sign( + self, + secret_key: &SecretKey, + fork: &Fork, + genesis_validators_root: Hash256, + spec: &ChainSpec, + ) -> SignedBlockContents<T, Payload> { + let (block, maybe_blobs) = self.deconstruct(); + let signed_block = block.sign(secret_key, fork, genesis_validators_root, spec); + let signed_blobs = maybe_blobs.map(|blobs| { + blobs + .into_iter() + .map(|blob| blob.sign(secret_key, fork, genesis_validators_root, spec)) + .collect::<Vec<_>>() + .into() + }); + SignedBlockContents::new(signed_block, signed_blobs) + } } impl<T: EthSpec, Payload: AbstractExecPayload<T>> ForkVersionDeserialize diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index a48c38421fa..b6eef688b51 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -1,6 +1,5 @@ use crate::test_utils::TestRandom; -use crate::{Blob, ChainSpec, Domain, EthSpec, Fork, Hash256, SignedBlobSidecar, SignedRoot, Slot}; -use bls::SecretKey; +use crate::{Blob, EthSpec, Hash256, SignedRoot, Slot}; use derivative::Derivative; use kzg::{Kzg, KzgCommitment, KzgPreset, KzgProof}; use rand::Rng; @@ -10,7 +9,6 @@ use ssz_derive::{Decode, Encode}; use ssz_types::{FixedVector, VariableList}; use std::fmt::Debug; use std::hash::Hash; -use std::marker::PhantomData; use std::sync::Arc; use test_random_derive::TestRandom; use tree_hash::TreeHash; @@ -171,31 +169,6 @@ impl<T: EthSpec> BlobSidecar<T> { // Fixed part Self::empty().as_ssz_bytes().len() } - - // this is mostly not used except for in testing - pub fn sign( - self: Arc<Self>, - secret_key: &SecretKey, - fork: &Fork, - genesis_validators_root: Hash256, - spec: &ChainSpec, - ) -> SignedBlobSidecar<T> { - let signing_epoch = self.slot.epoch(T::slots_per_epoch()); - let domain = spec.get_domain( - signing_epoch, - Domain::BlobSidecar, - fork, - genesis_validators_root, - ); - let message = self.signing_root(domain); - let signature = secret_key.sign(message); - - SignedBlobSidecar { - message: self, - signature, - _phantom: PhantomData, - } - } } #[derive( diff --git a/consensus/types/src/sidecar.rs b/consensus/types/src/sidecar.rs index 89e5e612873..e784cc57f1f 100644 --- a/consensus/types/src/sidecar.rs +++ b/consensus/types/src/sidecar.rs @@ -2,14 +2,17 @@ use crate::beacon_block_body::KzgCommitments; use crate::test_utils::TestRandom; use crate::{ AbstractExecPayload, BeaconBlock, BlindedBlobSidecar, BlindedBlobSidecarList, BlobRootsList, - BlobSidecar, BlobSidecarList, BlobsList, EthSpec, SidecarList, SignedRoot, Slot, + BlobSidecar, BlobSidecarList, BlobsList, ChainSpec, Domain, EthSpec, Fork, Hash256, + SidecarList, SignedRoot, SignedSidecar, Slot, }; +use bls::SecretKey; use kzg::KzgProof; use serde::de::DeserializeOwned; use ssz::{Decode, Encode}; use ssz_types::VariableList; use std::fmt::Debug; use std::hash::Hash; +use std::marker::PhantomData; use std::sync::Arc; use tree_hash::TreeHash; @@ -29,13 +32,40 @@ pub trait Sidecar<E: EthSpec>: + for<'a> arbitrary::Arbitrary<'a> { type BlobItems: BlobItems<E>; + fn slot(&self) -> Slot; + fn build_sidecar<Payload: AbstractExecPayload<E>>( blob_items: Self::BlobItems, block: &BeaconBlock<E, Payload>, expected_kzg_commitments: &KzgCommitments<E>, kzg_proofs: Vec<KzgProof>, ) -> Result<SidecarList<E, Self>, String>; + + // this is mostly not used except for in testing + fn sign( + self: Arc<Self>, + secret_key: &SecretKey, + fork: &Fork, + genesis_validators_root: Hash256, + spec: &ChainSpec, + ) -> SignedSidecar<E, Self> { + let signing_epoch = self.slot().epoch(E::slots_per_epoch()); + let domain = spec.get_domain( + signing_epoch, + Domain::BlobSidecar, + fork, + genesis_validators_root, + ); + let message = self.signing_root(domain); + let signature = secret_key.sign(message); + + SignedSidecar { + message: self, + signature, + _phantom: PhantomData, + } + } } pub trait BlobItems<T: EthSpec>: Sync + Send + Sized {