From 7ea6b2b13d71653d39cb019caca256658da187d5 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 22 Dec 2023 17:13:17 +0100 Subject: [PATCH 01/56] update connector to log the connecdtion problem --- cluster-endpoints/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-endpoints/Cargo.toml b/cluster-endpoints/Cargo.toml index b105dfeb..d937277c 100644 --- a/cluster-endpoints/Cargo.toml +++ b/cluster-endpoints/Cargo.toml @@ -10,7 +10,7 @@ license = "AGPL" [dependencies] #geyser-grpc-connector = { path = "../../geyser-grpc-connector" } #geyser-grpc-connector = { tag = "v0.5.0+yellowstone.1.11+solana.1.16.17", git = "https://github.com/blockworks-foundation/geyser-grpc-connector.git" } -geyser-grpc-connector = "0.7.0+yellowstone.1.11" +geyser-grpc-connector = "0.7.1+yellowstone.1.11" solana-sdk = { workspace = true } solana-rpc-client-api = { workspace = true } From 68ee942b4b2f3f1ba286963ccbde4dbc8728ea82 Mon Sep 17 00:00:00 2001 From: Aniket Prajapati <42654595+aniketfuryrocks@users.noreply.github.com> Date: Tue, 2 Jan 2024 20:51:39 +0530 Subject: [PATCH 02/56] removed DEFAULT_LITE_RPC_HTTP_ADDR; config to Config::default_lite_rpc_http_addr (#254) * -D clippy * scripts * fmt * commented out * diff clippy and build * fix * push -> main --- config.example.json | 2 +- lite-rpc/src/lib.rs | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/config.example.json b/config.example.json index 2074c7ea..665d2a91 100644 --- a/config.example.json +++ b/config.example.json @@ -1,7 +1,7 @@ { "rpc_addr": "http://0.0.0.0:8899", "ws_addr": "ws://0.0.0.0:8900", - "lite_rpc_http_addr": "http://0.0.0.0:8890", + "lite_rpc_http_addr": "[::]:8890", "lite_rpc_ws_addr": "[::]:8891", "fanout_size": 18, "identity_keypair": null, diff --git a/lite-rpc/src/lib.rs b/lite-rpc/src/lib.rs index 4b15fef0..f5c8da9a 100644 --- a/lite-rpc/src/lib.rs +++ b/lite-rpc/src/lib.rs @@ -13,8 +13,6 @@ pub mod service_spawner; #[from_env] pub const DEFAULT_RPC_ADDR: &str = "http://0.0.0.0:8899"; #[from_env] -pub const DEFAULT_LITE_RPC_ADDR: &str = "http://0.0.0.0:8890"; -#[from_env] pub const DEFAULT_WS_ADDR: &str = "ws://0.0.0.0:8900"; #[from_env] From 95a3f7d66c4b3e549124b0ad4282d89aeadfff4d Mon Sep 17 00:00:00 2001 From: vuittont60 <81072379+vuittont60@users.noreply.github.com> Date: Tue, 2 Jan 2024 23:22:46 +0800 Subject: [PATCH 03/56] Fix some typos (#260) * fix typos in benches/ * fix typos in cluster-endpoints/ * fix typos in quic-forward-proxy/ * fix typo in README.md --- README.md | 1 - benches/bench_tx_send_and_confirm.ts | 2 +- benches/bench_txs_send.ts | 2 +- cluster-endpoints/src/rpc_polling/poll_blocks.rs | 2 +- cluster-endpoints/src/rpc_polling/poll_slots.rs | 2 +- quic-forward-proxy/README.md | 2 +- quic-forward-proxy/src/quinn_auto_reconnect.rs | 2 +- 7 files changed, 6 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 58d6eed9..72b43702 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,6 @@ solana tpu ([details](quic-forward-proxy/README.md)). ### Confirmation strategies 1) Subscribe to new blocks using websockets (deprecated) -(depricated) 2) Polling blocks over RPC.(Current) 3) Subscribe blocks over gRPC. (Current) diff --git a/benches/bench_tx_send_and_confirm.ts b/benches/bench_tx_send_and_confirm.ts index 13cac472..1b0a4d09 100644 --- a/benches/bench_tx_send_and_confirm.ts +++ b/benches/bench_tx_send_and_confirm.ts @@ -76,7 +76,7 @@ export async function main() { } main().then(x => { - console.log('finished sucessfully') + console.log('finished successfully') }).catch(e => { console.log('caught an error : ' + e) }) \ No newline at end of file diff --git a/benches/bench_txs_send.ts b/benches/bench_txs_send.ts index 2d1d8a02..a89da7ad 100644 --- a/benches/bench_txs_send.ts +++ b/benches/bench_txs_send.ts @@ -89,7 +89,7 @@ const skip_confirmations = get_postional_arg(5, false) === "true"; } } - console.log("sucesses : " + successes) + console.log("successes : " + successes) console.log("failures : " + failures) //console.log("time taken to send : " + time_taken_to_send) } diff --git a/cluster-endpoints/src/rpc_polling/poll_blocks.rs b/cluster-endpoints/src/rpc_polling/poll_blocks.rs index ac8a40fb..0c4d60b8 100644 --- a/cluster-endpoints/src/rpc_polling/poll_blocks.rs +++ b/cluster-endpoints/src/rpc_polling/poll_blocks.rs @@ -126,7 +126,7 @@ pub fn poll_block( //slot poller let slot_poller = tokio::spawn(async move { - log::info!("block listner started"); + log::info!("block listener started"); let current_slot = rpc_client .get_slot() .await diff --git a/cluster-endpoints/src/rpc_polling/poll_slots.rs b/cluster-endpoints/src/rpc_polling/poll_slots.rs index fd363f4f..d8ffc84e 100644 --- a/cluster-endpoints/src/rpc_polling/poll_slots.rs +++ b/cluster-endpoints/src/rpc_polling/poll_slots.rs @@ -90,7 +90,7 @@ pub fn poll_slots( processed_slot: current_slot, estimated_processed_slot: estimated_slot, }) - .context("Connot send slot notification")?; + .context("Cannot send slot notification")?; } } } diff --git a/quic-forward-proxy/README.md b/quic-forward-proxy/README.md index 981dcaef..9ae45b30 100644 --- a/quic-forward-proxy/README.md +++ b/quic-forward-proxy/README.md @@ -95,7 +95,7 @@ Implementation Details * _proxy_ will maintain multiple quic connection per TPU according to the Solana validator quic policy * _proxy_ will use lightweight quic streams to send the transactions * inbound traffic (from Lite RPC) - * client-proxy-communcation is done via QUIC using a custom wire format + * client-proxy-communication is done via QUIC using a custom wire format * _proxy_ supports only quic ATM but that could be extended to support other protocols * _proxy_ should perform client authentication by TLS (see [issue](https://github.com/blockworks-foundation/lite-rpc/issues/167)) * _proxy_ uses a single queue (channel) for buffering the transactions from any inbound connection diff --git a/quic-forward-proxy/src/quinn_auto_reconnect.rs b/quic-forward-proxy/src/quinn_auto_reconnect.rs index 14c0120c..fa528c18 100644 --- a/quic-forward-proxy/src/quinn_auto_reconnect.rs +++ b/quic-forward-proxy/src/quinn_auto_reconnect.rs @@ -29,7 +29,7 @@ enum ConnectionState { } pub struct AutoReconnect { - // endoint should be configures with keep-alive and idle timeout + // endpoint should be configures with keep-alive and idle timeout endpoint: Endpoint, current: RwLock, pub target_address: SocketAddr, From 3e4c81c56742d30dd3265bf31377b06364f0afe0 Mon Sep 17 00:00:00 2001 From: musitdev Date: Tue, 10 Oct 2023 15:07:44 +0200 Subject: [PATCH 04/56] Stake and vote management. Integration of the main loop in current architecture --- Cargo.lock | 2319 +++++++++++++++++++------ Cargo.toml | 4 + core/src/stores/data_cache.rs | 6 +- core/src/structures/epoch.rs | 4 + core/src/structures/leaderschedule.rs | 48 + core/src/structures/mod.rs | 1 + lite-rpc/Cargo.toml | 1 + lite-rpc/src/bridge.rs | 21 +- lite-rpc/src/main.rs | 15 + lite-rpc/src/rpc.rs | 23 +- stake_vote/Cargo.toml | 43 + stake_vote/src/account.rs | 93 + stake_vote/src/bootstrap.rs | 46 + stake_vote/src/epoch.rs | 56 + stake_vote/src/leader_schedule.rs | 31 + stake_vote/src/lib.rs | 190 ++ stake_vote/src/stake.rs | 131 ++ stake_vote/src/utils.rs | 18 + stake_vote/src/vote.rs | 84 + 19 files changed, 2584 insertions(+), 550 deletions(-) create mode 100644 core/src/structures/leaderschedule.rs create mode 100644 stake_vote/Cargo.toml create mode 100644 stake_vote/src/account.rs create mode 100644 stake_vote/src/bootstrap.rs create mode 100644 stake_vote/src/epoch.rs create mode 100644 stake_vote/src/leader_schedule.rs create mode 100644 stake_vote/src/lib.rs create mode 100644 stake_vote/src/stake.rs create mode 100644 stake_vote/src/utils.rs create mode 100644 stake_vote/src/vote.rs diff --git a/Cargo.lock b/Cargo.lock index 2b4d502c..8c8d2f88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,36 +65,43 @@ dependencies = [ [[package]] name = "ahash" -version = "0.7.6" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +checksum = "5a824f2aa7e75a0c98c5a504fceb80649e9c35265d44525b5f94de4771a395cd" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.11", "once_cell", "version_check", ] [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" dependencies = [ "cfg-if", - "getrandom 0.2.10", + "getrandom 0.2.11", "once_cell", "version_check", + "zerocopy", ] [[package]] name = "aho-corasick" -version = "1.0.5" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + [[package]] name = "alloc-no-stdlib" version = "2.0.4" @@ -136,9 +143,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.5.0" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" +checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" dependencies = [ "anstyle", "anstyle-parse", @@ -150,15 +157,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] name = "anstyle-parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" dependencies = [ "utf8parse", ] @@ -174,9 +181,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "2.1.0" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" +checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" dependencies = [ "anstyle", "windows-sys 0.48.0", @@ -188,6 +195,12 @@ version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +[[package]] +name = "arc-swap" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" + [[package]] name = "ark-bn254" version = "0.4.0" @@ -211,8 +224,8 @@ dependencies = [ "ark-std", "derivative", "hashbrown 0.13.2", - "itertools", - "num-traits 0.2.16", + "itertools 0.10.5", + "num-traits 0.2.17", "zeroize", ] @@ -228,9 +241,9 @@ dependencies = [ "ark-std", "derivative", "digest 0.10.7", - "itertools", + "itertools 0.10.5", "num-bigint 0.4.4", - "num-traits 0.2.16", + "num-traits 0.2.17", "paste", "rustc_version", "zeroize", @@ -253,8 +266,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7abe79b0e4288889c4574159ab790824d0033b9fdcb2a112a3182fac2e514565" dependencies = [ "num-bigint 0.4.4", - "num-traits 0.2.16", - "proc-macro2 1.0.66", + "num-traits 0.2.17", + "proc-macro2 1.0.70", "quote 1.0.33", "syn 1.0.109", ] @@ -290,7 +303,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae3281bc6d0fd7e549af32b52511e1302185bd688fd3359fa36423346ff682ea" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", "syn 1.0.109", ] @@ -301,7 +314,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94893f1e0c6eeab764ade8dc4c0db24caf4fe7cbbaafc0eba0a9030f447b5185" dependencies = [ - "num-traits 0.2.16", + "num-traits 0.2.17", "rand 0.8.5", ] @@ -339,10 +352,10 @@ dependencies = [ "asn1-rs-impl", "displaydoc", "nom", - "num-traits 0.2.16", + "num-traits 0.2.17", "rusticata-macros", "thiserror", - "time 0.3.28", + "time", ] [[package]] @@ -351,7 +364,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", "syn 1.0.109", "synstructure", @@ -363,7 +376,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", "syn 1.0.109", ] @@ -387,9 +400,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.2" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d495b6dc0184693324491a5ac05f559acc97bf937ab31d7a1c33dd0016be6d2b" +checksum = "bc2d0cfb2a7388d34f590e76686704c494ed7aaceed62ee1ba35cbf363abc2a5" dependencies = [ "brotli", "flate2", @@ -434,20 +447,20 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", ] [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", ] [[package]] @@ -521,6 +534,20 @@ dependencies = [ "tower-service", ] +[[package]] +name = "backoff" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +dependencies = [ + "futures-core", + "getrandom 0.2.11", + "instant", + "pin-project-lite", + "rand 0.8.5", + "tokio", +] + [[package]] name = "backtrace" version = "0.3.69" @@ -575,12 +602,12 @@ version = "0.2.3" dependencies = [ "anyhow", "bincode", - "clap 4.4.2", + "clap 4.4.10", "csv", - "dashmap", + "dashmap 5.5.3", "dirs", "futures", - "itertools", + "itertools 0.10.5", "lazy_static", "log", "rand 0.8.5", @@ -603,6 +630,27 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.65.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5" +dependencies = [ + "bitflags 1.3.2", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "peeking_take_while", + "prettyplease 0.2.15", + "proc-macro2 1.0.70", + "quote 1.0.33", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.39", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -611,9 +659,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" [[package]] name = "bitmaps" @@ -626,9 +674,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "199c42ab6972d92c9f8995f086273d25c42fc0f7b2a1fcefba465c1352d25ba5" +checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87" dependencies = [ "arrayref", "arrayvec", @@ -692,7 +740,7 @@ dependencies = [ "borsh-derive-internal 0.9.3", "borsh-schema-derive-internal 0.9.3", "proc-macro-crate 0.1.5", - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "syn 1.0.109", ] @@ -705,7 +753,7 @@ dependencies = [ "borsh-derive-internal 0.10.3", "borsh-schema-derive-internal 0.10.3", "proc-macro-crate 0.1.5", - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "syn 1.0.109", ] @@ -715,7 +763,7 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5449c28a7b352f2d1e592a8a28bf139bc71afb0764a14f3c02500935d8c44065" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", "syn 1.0.109", ] @@ -726,7 +774,7 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afb438156919598d2c7bad7e1c0adf3d26ed3840dbc010db1a882a65583ca2fb" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", "syn 1.0.109", ] @@ -737,7 +785,7 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdbd5696d8bfa21d53d9fe39a714a18538bad11492a42d066dbbc395fb1951c0" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", "syn 1.0.109", ] @@ -748,16 +796,16 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "634205cc43f74a1b9046ef87c4540ebda95696ec0f315024860cad7c5b0f5ccd" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", "syn 1.0.109", ] [[package]] name = "brotli" -version = "3.3.4" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -766,9 +814,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.3.4" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744" +checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -782,9 +830,9 @@ checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" [[package]] name = "bstr" -version = "1.6.2" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c2f7349907b712260e64b0afe2f84692af14a454be26187d9df565c7f69266a" +checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c" dependencies = [ "memchr", "serde", @@ -792,9 +840,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "bv" @@ -817,26 +865,47 @@ dependencies = [ [[package]] name = "bytemuck_derive" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdde5c9cd29ebd706ce1b35600920a33550e402fc998a2e53ad3b42c3c47a192" +checksum = "965ab7eb5f8f97d2a083c799f3a1b994fc397b2fe2da5d1da1626ce15a39f2b1" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", ] [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] [[package]] name = "caps" @@ -858,6 +927,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -866,20 +944,28 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.28" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ed24df0632f708f5f6d8082675bef2596f7084dee3dd55f632290bf35bfe0f" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" dependencies = [ "android-tzdata", "iana-time-zone", "js-sys", - "num-traits 0.2.16", + "num-traits 0.2.17", "serde", - "time 0.1.45", "wasm-bindgen", "windows-targets 0.48.5", ] +[[package]] +name = "chrono-humanize" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799627e6b4d27827a814e837b9d8a504832086081806d45b1afa34dc982b023b" +dependencies = [ + "chrono", +] + [[package]] name = "cipher" version = "0.3.0" @@ -889,6 +975,17 @@ dependencies = [ "generic-array", ] +[[package]] +name = "clang-sys" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "2.34.0" @@ -922,9 +1019,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.2" +version = "4.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a13b88d2c62ff462f88e4a121f17a82c1af05693a2f192b5c38d14de73c19f6" +checksum = "41fffed7514f420abec6d183b1d3acfd9099c79c3a10a06ade4f8203f1411272" dependencies = [ "clap_builder", "clap_derive", @@ -932,26 +1029,26 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.2" +version = "4.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" +checksum = "63361bae7eef3771745f02d8d892bec2fee5f6e34af316ba556e7f97a7069ff1" dependencies = [ "anstream", "anstyle", - "clap_lex 0.5.1", + "clap_lex 0.6.0", "strsim 0.10.0", ] [[package]] name = "clap_derive" -version = "4.4.2" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck", - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", ] [[package]] @@ -965,9 +1062,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" [[package]] name = "colorchoice" @@ -990,9 +1087,9 @@ dependencies = [ [[package]] name = "concurrent-queue" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c" +checksum = "f057a694a54f12365049b0958a1685bb52d567f5593b355fbf685838e873d400" dependencies = [ "crossbeam-utils", ] @@ -1051,7 +1148,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a4f51209740b5e1589e702b3044cdd4562cef41b6da404904192ffffb852d62" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", "syn 1.0.109", ] @@ -1095,9 +1192,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" dependencies = [ "libc", ] @@ -1182,9 +1279,9 @@ dependencies = [ [[package]] name = "csv" -version = "1.2.2" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626ae34994d3d8d668f4269922248239db4ae42d538b14c398b74a52208e8086" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" dependencies = [ "csv-core", "itoa", @@ -1194,9 +1291,9 @@ dependencies = [ [[package]] name = "csv-core" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" dependencies = [ "memchr", ] @@ -1242,10 +1339,10 @@ checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", "strsim 0.10.0", - "syn 2.0.31", + "syn 2.0.39", ] [[package]] @@ -1256,7 +1353,18 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", +] + +[[package]] +name = "dashmap" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" +dependencies = [ + "cfg-if", + "num_cpus", + "rayon", ] [[package]] @@ -1266,17 +1374,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.0", + "hashbrown 0.14.3", "lock_api", "once_cell", - "parking_lot_core", + "parking_lot_core 0.9.9", ] [[package]] name = "data-encoding" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" +checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" [[package]] name = "der" @@ -1297,15 +1405,18 @@ dependencies = [ "displaydoc", "nom", "num-bigint 0.4.4", - "num-traits 0.2.16", + "num-traits 0.2.17", "rusticata-macros", ] [[package]] name = "deranged" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" +dependencies = [ + "powerfmt", +] [[package]] name = "derivation-path" @@ -1319,7 +1430,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", "syn 1.0.109", ] @@ -1331,7 +1442,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case", - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", "rustc_version", "syn 1.0.109", @@ -1369,6 +1480,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "dir-diff" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2860407d7d7e2e004bb2128510ad9e8d669e76fa005ccf567977b5d71b8b4a0b" +dependencies = [ + "walkdir", +] + [[package]] name = "dirs" version = "5.0.1" @@ -1396,9 +1516,9 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", ] [[package]] @@ -1468,7 +1588,7 @@ dependencies = [ "derivation-path", "ed25519-dalek", "hmac 0.12.1", - "sha2 0.10.7", + "sha2 0.10.8", ] [[package]] @@ -1507,9 +1627,9 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", ] [[package]] @@ -1533,23 +1653,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" -dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.2" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ - "cc", "libc", + "windows-sys 0.52.0", ] [[package]] @@ -1564,11 +1673,20 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +[[package]] +name = "fast-math" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2465292146cdfc2011350fe3b1c616ac83cf0faeedb33463ba1c332ed8948d66" +dependencies = [ + "ieee754", +] + [[package]] name = "fastrand" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "feature-probe" @@ -1576,6 +1694,24 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "835a3dc7d1ec9e75e2b5fb4ba75396837112d2060b03f7d43bc1897c7f7211da" +[[package]] +name = "filetime" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.3.5", + "windows-sys 0.48.0", +] + +[[package]] +name = "finl_unicode" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" + [[package]] name = "fixedbitset" version = "0.4.2" @@ -1584,9 +1720,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", "miniz_oxide", @@ -1622,11 +1758,17 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" dependencies = [ "futures-channel", "futures-core", @@ -1639,9 +1781,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" dependencies = [ "futures-core", "futures-sink", @@ -1649,15 +1791,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" dependencies = [ "futures-core", "futures-task", @@ -1666,32 +1808,32 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", ] [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" [[package]] name = "futures-timer" @@ -1705,9 +1847,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" dependencies = [ "futures-channel", "futures-core", @@ -1757,9 +1899,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" dependencies = [ "cfg-if", "js-sys", @@ -1793,21 +1935,27 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.0" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "glob" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "globset" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759c97c1e17c55525b57192c06a267cda0ac5210b222d6b82189a2338fa1c13d" +checksum = "57da3b9b5b85bd66f31093f8c408b90a74431672542466497dcbdfdc02034be1" dependencies = [ "aho-corasick", "bstr", - "fnv", "log", - "regex", + "regex-automata 0.4.3", + "regex-syntax 0.8.2", ] [[package]] @@ -1855,6 +2003,25 @@ dependencies = [ "web-sys", ] +[[package]] +name = "goauth" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8af59a261bcf42f45d1b261232847b9b850ba0a1419d6100698246fb66e9240" +dependencies = [ + "arc-swap", + "futures", + "log", + "reqwest", + "serde", + "serde_derive", + "serde_json", + "simpl", + "smpl_jwt", + "time", + "tokio", +] + [[package]] name = "goblin" version = "0.5.4" @@ -1868,9 +2035,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.21" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" dependencies = [ "bytes", "fnv", @@ -1878,7 +2045,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 1.9.3", + "indexmap 2.1.0", "slab", "tokio", "tokio-util", @@ -1900,7 +2067,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" dependencies = [ - "ahash 0.7.6", + "ahash 0.7.7", ] [[package]] @@ -1908,6 +2075,9 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.7", +] [[package]] name = "hashbrown" @@ -1915,14 +2085,38 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.6", ] [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + +[[package]] +name = "headers" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270" +dependencies = [ + "base64 0.21.5", + "bytes", + "headers-core", + "http", + "httpdate", + "mime", + "sha1", +] + +[[package]] +name = "headers-core" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" +dependencies = [ + "http", +] [[package]] name = "heck" @@ -1941,9 +2135,15 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.2" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" + +[[package]] +name = "hex" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "histogram" @@ -1981,11 +2181,20 @@ dependencies = [ "hmac 0.8.1", ] +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys 0.48.0", +] + [[package]] name = "http" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" dependencies = [ "bytes", "fnv", @@ -2038,24 +2247,42 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2 0.4.10", "tokio", "tower-service", "tracing", "want", ] +[[package]] +name = "hyper-proxy" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca815a891b24fdfb243fa3239c86154392b0953ee584aa1a2a1f66d20cbe75cc" +dependencies = [ + "bytes", + "futures", + "headers", + "http", + "hyper", + "hyper-tls", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-rustls" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http", "hyper", "log", - "rustls 0.21.7", + "rustls 0.21.9", "rustls-native-certs", "tokio", "tokio-rustls 0.24.1", @@ -2073,18 +2300,31 @@ dependencies = [ "tokio-io-timeout", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" -version = "0.1.57" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows", + "windows-core", ] [[package]] @@ -2112,6 +2352,12 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "ieee754" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9007da9cacbd3e6343da136e98b0d2df013f553d35bdec8b518f07bea768e19c" + [[package]] name = "im" version = "15.1.0" @@ -2128,6 +2374,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "index_list" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70891286cb8e844fdfcf1178b47569699f9e20b5ecc4b45a6240a64771444638" + [[package]] name = "indexmap" version = "1.9.3" @@ -2140,19 +2392,19 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown 0.14.3", ] [[package]] name = "indicatif" -version = "0.17.6" +version = "0.17.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b297dc40733f23a0e52728a58fa9489a5b7638a324932de16b41adc3ef80730" +checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" dependencies = [ "console", "instant", @@ -2172,9 +2424,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "itertools" @@ -2185,6 +2437,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.9" @@ -2193,18 +2454,18 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jobserver" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" dependencies = [ "wasm-bindgen", ] @@ -2278,7 +2539,7 @@ dependencies = [ "globset", "hyper", "jsonrpsee-types", - "parking_lot", + "parking_lot 0.12.1", "rand 0.8.5", "rustc-hash", "serde", @@ -2318,7 +2579,7 @@ checksum = "d814a21d9a819f8de1a41b819a263ffd68e4bb5f043d936db1c49b54684bde0a" dependencies = [ "heck", "proc-macro-crate 1.3.1", - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", "syn 1.0.109", ] @@ -2396,10 +2657,58 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] -name = "libc" -version = "0.2.147" +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" + +[[package]] +name = "libloading" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +dependencies = [ + "cfg-if", + "winapi", +] + +[[package]] +name = "libm" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + +[[package]] +name = "libredox" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +dependencies = [ + "bitflags 2.4.1", + "libc", + "redox_syscall 0.4.1", +] + +[[package]] +name = "librocksdb-sys" +version = "0.11.0+8.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3386f101bcb4bd252d8e9d2fb41ec3b0862a15a62b478c355b2982efa469e3e" +dependencies = [ + "bindgen", + "bzip2-sys", + "cc", + "glob", + "libc", + "libz-sys", + "lz4-sys", +] [[package]] name = "libsecp256k1" @@ -2449,11 +2758,22 @@ dependencies = [ "libsecp256k1-core", ] +[[package]] +name = "libz-sys" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" -version = "0.4.5" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" +checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" [[package]] name = "lite-rpc" @@ -2468,9 +2788,9 @@ dependencies = [ "bs58", "bytes", "chrono", - "clap 4.4.2", + "clap 4.4.10", "const_env", - "dashmap", + "dashmap 5.5.3", "dotenv", "futures", "jsonrpsee", @@ -2486,6 +2806,7 @@ dependencies = [ "solana-lite-rpc-core", "solana-lite-rpc-history", "solana-lite-rpc-services", + "solana-lite-rpc-stakevote", "solana-rpc-client", "solana-rpc-client-api", "solana-sdk", @@ -2499,9 +2820,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -2513,6 +2834,35 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "lru" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" +dependencies = [ + "hashbrown 0.12.3", +] + +[[package]] +name = "lz4" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e9e2dd86df36ce760a60f6ff6ad526f7ba1f14ba0356f8254fb6905e6494df1" +dependencies = [ + "libc", + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "matchers" version = "0.1.0" @@ -2522,26 +2872,33 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "matches" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" + [[package]] name = "matchit" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed1202b2a6f884ae56f04cff409ab315c5ce26b5e58d7412e484f01fd52f52ef" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "md-5" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ + "cfg-if", "digest 0.10.7", ] [[package]] name = "memchr" -version = "2.6.3" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memmap2" @@ -2615,15 +2972,36 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" dependencies = [ "libc", "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.48.0", ] +[[package]] +name = "modular-bitfield" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a53d79ba8304ac1c4f9eb3b9d281f21f7be9d4626f72ce7df4ad8fbde4f38a74" +dependencies = [ + "modular-bitfield-impl", + "static_assertions", +] + +[[package]] +name = "modular-bitfield-impl" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a7d5f7076603ebc68de2dc6a650ec331a062a13abaa346975be747bbfa4b789" +dependencies = [ + "proc-macro2 1.0.70", + "quote 1.0.33", + "syn 1.0.109", +] + [[package]] name = "multimap" version = "0.8.3" @@ -2692,7 +3070,7 @@ dependencies = [ "num-integer", "num-iter", "num-rational", - "num-traits 0.2.16", + "num-traits 0.2.17", ] [[package]] @@ -2703,7 +3081,7 @@ checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" dependencies = [ "autocfg", "num-integer", - "num-traits 0.2.16", + "num-traits 0.2.17", ] [[package]] @@ -2714,7 +3092,7 @@ checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" dependencies = [ "autocfg", "num-integer", - "num-traits 0.2.16", + "num-traits 0.2.17", ] [[package]] @@ -2724,7 +3102,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" dependencies = [ "autocfg", - "num-traits 0.2.16", + "num-traits 0.2.17", ] [[package]] @@ -2733,7 +3111,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", "syn 1.0.109", ] @@ -2744,9 +3122,9 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfb77679af88f8b125209d354a202862602672222e7f2313fdd6dc349bad4712" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", ] [[package]] @@ -2756,7 +3134,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ "autocfg", - "num-traits 0.2.16", + "num-traits 0.2.17", ] [[package]] @@ -2767,7 +3145,7 @@ checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" dependencies = [ "autocfg", "num-integer", - "num-traits 0.2.16", + "num-traits 0.2.17", ] [[package]] @@ -2779,7 +3157,7 @@ dependencies = [ "autocfg", "num-bigint 0.2.6", "num-integer", - "num-traits 0.2.16", + "num-traits 0.2.17", ] [[package]] @@ -2788,14 +3166,14 @@ version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" dependencies = [ - "num-traits 0.2.16", + "num-traits 0.2.17", ] [[package]] name = "num-traits" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ "autocfg", ] @@ -2806,7 +3184,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.2", + "hermit-abi 0.3.3", "libc", ] @@ -2835,9 +3213,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96667db765a921f7b295ffee8b60472b686a51d4f21c2ee4ffdb94c7013b65a6" dependencies = [ "proc-macro-crate 1.3.1", - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", ] [[package]] @@ -2846,10 +3224,10 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c11e44798ad209ccdd91fc192f0526a369a01234f7373e1b141c96d7cee4f0e" dependencies = [ - "proc-macro-crate 1.3.1", - "proc-macro2 1.0.66", + "proc-macro-crate 2.0.0", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", ] [[package]] @@ -2890,11 +3268,11 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.57" +version = "0.10.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "cfg-if", "foreign-types", "libc", @@ -2909,9 +3287,9 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", ] [[package]] @@ -2920,14 +3298,24 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-src" +version = "300.1.6+3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439fac53e092cd7442a3660c85dde4643ab3b5bd39040912388dcdabf6b88085" +dependencies = [ + "cc", +] + [[package]] name = "openssl-sys" -version = "0.9.92" +version = "0.9.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db7e971c2c2bba161b2d2fdf37080177eff520b3bc044787c7f1f5f9e78d869b" +checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f" dependencies = [ "cc", "libc", + "openssl-src", "pkg-config", "vcpkg", ] @@ -2940,9 +3328,32 @@ checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] name = "os_str_bytes" -version = "6.5.1" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + +[[package]] +name = "ouroboros" +version = "0.15.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1358bd1558bd2a083fed428ffeda486fbfb323e698cdda7794259d592ca72db" +dependencies = [ + "aliasable", + "ouroboros_macro", +] + +[[package]] +name = "ouroboros_macro" +version = "0.15.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d5d9eb14b174ee9aa2ef96dc2b94637a2d4b6e7cb873c7e171f0c20c6cf3eac" +checksum = "5f7d21ccd03305a674437ee1248f3ab5d4b1db095cf1caf49f1713ddf61956b7" +dependencies = [ + "Inflector", + "proc-macro-error", + "proc-macro2 1.0.70", + "quote 1.0.33", + "syn 1.0.109", +] [[package]] name = "overload" @@ -2950,6 +3361,17 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -2957,18 +3379,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.9", ] [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" dependencies = [ "cfg-if", + "instant", "libc", - "redox_syscall 0.3.5", + "redox_syscall 0.2.16", + "smallvec", + "winapi", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.4.1", "smallvec", "windows-targets 0.48.5", ] @@ -2997,6 +3433,12 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "pem" version = "1.1.1" @@ -3028,7 +3470,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.0.0", + "indexmap 2.1.0", ] [[package]] @@ -3064,9 +3506,9 @@ version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", ] [[package]] @@ -3118,9 +3560,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.4.3" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b" +checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b" [[package]] name = "postgres-native-tls" @@ -3149,7 +3591,7 @@ dependencies = [ "md-5", "memchr", "rand 0.8.5", - "sha2 0.10.7", + "sha2 0.10.8", "stringprep", ] @@ -3165,20 +3607,36 @@ dependencies = [ "postgres-protocol", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "prettyplease" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +dependencies = [ + "proc-macro2 1.0.70", + "syn 1.0.109", +] + [[package]] name = "prettyplease" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ - "proc-macro2 1.0.66", - "syn 2.0.31", + "proc-macro2 1.0.70", + "syn 2.0.39", ] [[package]] @@ -3197,7 +3655,40 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" dependencies = [ "once_cell", - "toml_edit", + "toml_edit 0.19.15", +] + +[[package]] +name = "proc-macro-crate" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8366a6159044a37876a2b9817124296703c586a5c92e2c53751fa06d8d43e8" +dependencies = [ + "toml_edit 0.20.7", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2 1.0.70", + "quote 1.0.33", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2 1.0.70", + "quote 1.0.33", + "version_check", ] [[package]] @@ -3211,9 +3702,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" dependencies = [ "unicode-ident", ] @@ -3228,63 +3719,117 @@ dependencies = [ "fnv", "lazy_static", "memchr", - "parking_lot", + "parking_lot 0.12.1", "protobuf", "thiserror", ] [[package]] name = "prost" -version = "0.12.1" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive 0.11.9", +] + +[[package]] +name = "prost" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d" +checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.12.3", ] [[package]] name = "prost-build" -version = "0.12.1" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bdf592881d821b83d471f8af290226c8d51402259e9bb5be7f9f8bdebbb11ac" +checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" dependencies = [ "bytes", "heck", - "itertools", + "itertools 0.10.5", + "lazy_static", + "log", + "multimap", + "petgraph", + "prettyplease 0.1.25", + "prost 0.11.9", + "prost-types 0.11.9", + "regex", + "syn 1.0.109", + "tempfile", + "which", +] + +[[package]] +name = "prost-build" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c55e02e35260070b6f716a2423c2ff1c3bb1642ddca6f99e1f26d06268a0e2d2" +dependencies = [ + "bytes", + "heck", + "itertools 0.11.0", "log", "multimap", "once_cell", "petgraph", - "prettyplease", - "prost", - "prost-types", + "prettyplease 0.2.15", + "prost 0.12.3", + "prost-types 0.12.3", "regex", - "syn 2.0.31", + "syn 2.0.39", "tempfile", "which", ] [[package]] name = "prost-derive" -version = "0.12.1" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" dependencies = [ "anyhow", - "itertools", - "proc-macro2 1.0.66", + "itertools 0.10.5", + "proc-macro2 1.0.70", + "quote 1.0.33", + "syn 1.0.109", +] + +[[package]] +name = "prost-derive" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" +dependencies = [ + "anyhow", + "itertools 0.11.0", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", ] [[package]] name = "prost-types" -version = "0.12.1" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +dependencies = [ + "prost 0.11.9", +] + +[[package]] +name = "prost-types" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e081b29f63d83a4bc75cfc9f3fe424f9156cf92d8a4f0c9407cce9a1b67327cf" +checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e" dependencies = [ - "prost", + "prost 0.12.3", ] [[package]] @@ -3337,7 +3882,7 @@ checksum = "94b0b33c13a79f669c85defaf4c275dc86a0c0372807d0ca3d78e0bb87274863" dependencies = [ "bytes", "rand 0.8.5", - "ring", + "ring 0.16.20", "rustc-hash", "rustls 0.20.9", "rustls-native-certs", @@ -3356,7 +3901,7 @@ checksum = "641538578b21f5e5c8ea733b736895576d0fe329bb883b937db6f4d163dbaaf4" dependencies = [ "libc", "quinn-proto", - "socket2 0.4.9", + "socket2 0.4.10", "tracing", "windows-sys 0.42.0", ] @@ -3376,7 +3921,7 @@ version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", ] [[package]] @@ -3438,7 +3983,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.11", ] [[package]] @@ -3461,9 +4006,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" dependencies = [ "either", "rayon-core", @@ -3471,14 +4016,12 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", ] [[package]] @@ -3488,8 +4031,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6413f3de1edee53342e6138e75b56d32e7bc6e332b3bd62d497b1929d4cfbcdd" dependencies = [ "pem", - "ring", - "time 0.3.28", + "ring 0.16.20", + "time", "yasna", ] @@ -3500,8 +4043,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" dependencies = [ "pem", - "ring", - "time 0.3.28", + "ring 0.16.20", + "time", "yasna", ] @@ -3523,27 +4066,51 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_users" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" dependencies = [ - "getrandom 0.2.10", - "redox_syscall 0.2.16", + "getrandom 0.2.11", + "libredox", "thiserror", ] +[[package]] +name = "reed-solomon-erasure" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7263373d500d4d4f505d43a2a662d475a894aa94503a1ee28e9188b5f3960d4f" +dependencies = [ + "cc", + "libc", + "libm", + "lru", + "parking_lot 0.11.2", + "smallvec", + "spin 0.9.8", +] + [[package]] name = "regex" -version = "1.9.5" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.8", - "regex-syntax 0.7.5", + "regex-automata 0.4.3", + "regex-syntax 0.8.2", ] [[package]] @@ -3557,13 +4124,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.8" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.5", + "regex-syntax 0.8.2", ] [[package]] @@ -3574,15 +4141,15 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" -version = "0.11.20" +version = "0.11.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" +checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" dependencies = [ "async-compression", "base64 0.21.5", @@ -3595,19 +4162,23 @@ dependencies = [ "http-body", "hyper", "hyper-rustls", + "hyper-tls", "ipnet", "js-sys", "log", "mime", + "native-tls", "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.7", + "rustls 0.21.9", "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", + "system-configuration", "tokio", + "tokio-native-tls", "tokio-rustls 0.24.1", "tokio-util", "tower-service", @@ -3615,7 +4186,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 0.25.2", + "webpki-roots 0.25.3", "winreg", ] @@ -3628,39 +4199,63 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", - "untrusted", + "spin 0.5.2", + "untrusted 0.7.1", "web-sys", "winapi", ] [[package]] -name = "rpassword" -version = "7.2.0" +name = "ring" +version = "0.17.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6678cf63ab3491898c0d021b493c94c9b221d91295294a2a5746eacbe5928322" +checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" dependencies = [ + "cc", + "getrandom 0.2.11", "libc", - "rtoolbox", - "winapi", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys 0.48.0", ] [[package]] -name = "rtoolbox" -version = "0.0.1" +name = "rocksdb" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "034e22c514f5c0cb8a10ff341b9b048b5ceb21591f31c8f44c43b960f9b3524a" +checksum = "bb6f170a4041d50a0ce04b0d2e14916d6ca863ea2e422689a5b694395d299ffe" dependencies = [ "libc", - "winapi", + "librocksdb-sys", ] [[package]] -name = "rustc-demangle" -version = "0.1.23" +name = "rpassword" +version = "7.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" - +checksum = "80472be3c897911d0137b2d2b9055faf6eeac5b14e324073d83bc17b191d7e3f" +dependencies = [ + "libc", + "rtoolbox", + "windows-sys 0.48.0", +] + +[[package]] +name = "rtoolbox" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c247d24e63230cdb56463ae328478bd5eac8b8faa8c69461a77e8e323afac90e" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustc-hash" version = "1.1.0" @@ -3687,11 +4282,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.11" +version = "0.38.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0c3dde1fc030af041adc40e79c0e7fbcf431dd24870053d187d7c66e4b87453" +checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "errno", "libc", "linux-raw-sys", @@ -3705,20 +4300,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" dependencies = [ "log", - "ring", + "ring 0.16.20", "sct", "webpki", ] [[package]] name = "rustls" -version = "0.21.7" +version = "0.21.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" dependencies = [ "log", - "ring", - "rustls-webpki 0.101.4", + "ring 0.17.5", + "rustls-webpki 0.101.7", "sct", ] @@ -3736,31 +4331,31 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ "base64 0.21.5", ] [[package]] name = "rustls-webpki" -version = "0.100.2" +version = "0.100.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e98ff011474fa39949b7e5c0428f9b4937eda7da7848bbb947786b7be0b27dab" +checksum = "5f6a5fc258f1c1276dfe3016516945546e2d5383911efc0fc4f1cdc5df3a4ae3" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] name = "rustls-webpki" -version = "0.101.4" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring", - "untrusted", + "ring 0.17.5", + "untrusted 0.9.0", ] [[package]] @@ -3775,6 +4370,15 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.22" @@ -3805,19 +4409,19 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", ] [[package]] name = "sct" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring", - "untrusted", + "ring 0.17.5", + "untrusted 0.9.0", ] [[package]] @@ -3845,9 +4449,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.18" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" [[package]] name = "send_wrapper" @@ -3857,9 +4461,9 @@ checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" [[package]] name = "serde" -version = "1.0.188" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" dependencies = [ "serde_derive", ] @@ -3875,20 +4479,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", ] [[package]] name = "serde_json" -version = "1.0.105" +version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ "itoa", "ryu", @@ -3924,9 +4528,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "881b6f881b17d13214e5d494c939ebab463d01264ce1811e9d4ac3a882e7695f" dependencies = [ "darling", - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", ] [[package]] @@ -3953,6 +4557,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.10.7", +] + [[package]] name = "sha2" version = "0.9.9" @@ -3968,9 +4583,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.7" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", @@ -4001,9 +4616,9 @@ dependencies = [ [[package]] name = "sharded-slab" -version = "0.1.4" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ "lazy_static", ] @@ -4014,6 +4629,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" +[[package]] +name = "shlex" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -4029,6 +4650,12 @@ version = "1.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +[[package]] +name = "simpl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a30f10c911c0355f80f1c2faa8096efc4a58cdf8590b954d5b395efa071c711" + [[package]] name = "siphasher" version = "0.3.11" @@ -4056,15 +4683,31 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.0" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" + +[[package]] +name = "smpl_jwt" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +checksum = "95b6ff8c21c74ce7744643a7cddbb02579a44f1f77e4316bff1ddb741aca8ac9" +dependencies = [ + "base64 0.13.1", + "log", + "openssl", + "serde", + "serde_derive", + "serde_json", + "simpl", + "time", +] [[package]] name = "socket2" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" dependencies = [ "libc", "winapi", @@ -4072,9 +4715,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.3" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" dependencies = [ "libc", "windows-sys 0.48.0", @@ -4131,7 +4774,7 @@ dependencies = [ "bytemuck", "log", "num-derive 0.3.3", - "num-traits 0.2.16", + "num-traits 0.2.17", "rustc_version", "serde", "solana-frozen-abi", @@ -4142,6 +4785,42 @@ dependencies = [ "thiserror", ] +[[package]] +name = "solana-bpf-loader-program" +version = "1.16.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6302bcae9ca7cddffc5af54c9407abbbb14b94539300aca01d879a3f23379f42" +dependencies = [ + "bincode", + "byteorder", + "libsecp256k1", + "log", + "rand 0.7.3", + "solana-measure", + "solana-program-runtime", + "solana-sdk", + "solana-zk-token-sdk", + "solana_rbpf", + "thiserror", +] + +[[package]] +name = "solana-bucket-map" +version = "1.16.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a403fe9ea9d8f7a829918cc53023ab62802e5628483b78a92a33edf7a5d4e54c" +dependencies = [ + "bv", + "log", + "memmap2", + "modular-bitfield", + "num_enum 0.6.1", + "rand 0.7.3", + "solana-measure", + "solana-sdk", + "tempfile", +] + [[package]] name = "solana-clap-utils" version = "1.16.17" @@ -4193,6 +4872,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "solana-compute-budget-program" +version = "1.16.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa1c6c014d6b34f2184598e115fc1de59c5ae52080677ae1fa8000aa0400e23f" +dependencies = [ + "solana-program-runtime", + "solana-sdk", +] + [[package]] name = "solana-config-program" version = "1.16.17" @@ -4228,13 +4917,36 @@ dependencies = [ "tokio", ] +[[package]] +name = "solana-entry" +version = "1.16.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843f27e05805a97c09b94f177897986ec90c3a53fa2f7483b6f78ea807dcdb0f" +dependencies = [ + "bincode", + "crossbeam-channel", + "dlopen", + "dlopen_derive", + "lazy_static", + "log", + "rand 0.7.3", + "rayon", + "serde", + "solana-measure", + "solana-merkle-tree", + "solana-metrics", + "solana-perf", + "solana-rayon-threadlimit", + "solana-sdk", +] + [[package]] name = "solana-frozen-abi" version = "1.16.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d266bf0311bb403d31206aa2904b8741f57c7f5e27580b6810ad5e22fc7c3282" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.6", "blake3", "block-buffer 0.10.4", "bs58", @@ -4255,7 +4967,7 @@ dependencies = [ "serde_bytes", "serde_derive", "serde_json", - "sha2 0.10.7", + "sha2 0.10.8", "solana-frozen-abi-macro", "subtle", "thiserror", @@ -4267,10 +4979,71 @@ version = "1.16.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dfe18c5155015dcb494c6de84a03b725fcf90ec2006a047769018b94c2cf0de" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", "rustc_version", - "syn 2.0.31", + "syn 2.0.39", +] + +[[package]] +name = "solana-ledger" +version = "1.16.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "414fb121473618de741b4e3b0e154d3546f019df69d4b1675831e698ba71b41b" +dependencies = [ + "assert_matches", + "bincode", + "bitflags 1.3.2", + "byteorder", + "chrono", + "chrono-humanize", + "crossbeam-channel", + "dashmap 4.0.2", + "fs_extra", + "futures", + "itertools 0.10.5", + "lazy_static", + "libc", + "log", + "lru", + "num_cpus", + "num_enum 0.6.1", + "prost 0.11.9", + "rand 0.7.3", + "rand_chacha 0.2.2", + "rayon", + "reed-solomon-erasure", + "rocksdb", + "rustc_version", + "scopeguard", + "serde", + "serde_bytes", + "sha2 0.10.8", + "solana-account-decoder", + "solana-bpf-loader-program", + "solana-entry", + "solana-frozen-abi", + "solana-frozen-abi-macro", + "solana-measure", + "solana-metrics", + "solana-perf", + "solana-program-runtime", + "solana-rayon-threadlimit", + "solana-runtime", + "solana-sdk", + "solana-stake-program", + "solana-storage-bigtable", + "solana-storage-proto", + "solana-transaction-status", + "solana-vote-program", + "spl-token", + "spl-token-2022", + "static_assertions", + "tempfile", + "thiserror", + "tokio", + "tokio-stream", + "trees", ] [[package]] @@ -4285,11 +5058,15 @@ dependencies = [ "bs58", "bytes", "chrono", - "dashmap", + "dashmap 5.5.3", "derive_more", "futures", +<<<<<<< HEAD "geyser-grpc-connector", "itertools", +======= + "itertools 0.10.5", +>>>>>>> a084068 (Stake and vote management. Integration of the main loop in current architecture) "lazy_static", "log", "merge-streams", @@ -4324,9 +5101,9 @@ dependencies = [ "bs58", "bytes", "chrono", - "dashmap", + "dashmap 5.5.3", "futures", - "itertools", + "itertools 0.10.5", "log", "quinn", "rustls 0.20.9", @@ -4355,8 +5132,8 @@ dependencies = [ "base64 0.21.5", "bincode", "chrono", - "dashmap", - "itertools", + "dashmap 5.5.3", + "itertools 0.10.5", "log", "native-tls", "postgres-native-tls", @@ -4382,11 +5159,11 @@ dependencies = [ "bs58", "bytes", "chrono", - "clap 4.4.2", - "dashmap", + "clap 4.4.10", + "dashmap 5.5.3", "dotenv", "futures", - "itertools", + "itertools 0.10.5", "lazy_static", "log", "native-tls", @@ -4420,13 +5197,13 @@ dependencies = [ "bs58", "bytes", "chrono", - "clap 4.4.2", + "clap 4.4.10", "countmap", "crossbeam-channel", - "dashmap", + "dashmap 5.5.3", "dotenv", "futures", - "itertools", + "itertools 0.10.5", "lazy_static", "log", "native-tls", @@ -4463,9 +5240,9 @@ dependencies = [ "chrono", "countmap", "crossbeam-channel", - "dashmap", + "dashmap 5.5.3", "futures", - "itertools", + "itertools 0.10.5", "lazy_static", "log", "prometheus", @@ -4488,6 +5265,53 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "solana-lite-rpc-stakevote" +version = "0.2.3" +dependencies = [ + "anyhow", + "bincode", + "borsh 0.10.3", + "bs58", + "futures", + "futures-util", + "hex", + "itertools 0.11.0", + "log", + "nom", + "reqwest", + "serde", + "serde_json", + "solana-account-decoder", + "solana-client", + "solana-ledger", + "solana-lite-rpc-core", + "solana-program", + "solana-rpc-client", + "solana-rpc-client-api", + "solana-sdk", + "solana-version", + "thiserror", + "tokio", + "tracing-subscriber", + "yellowstone-grpc-client", + "yellowstone-grpc-proto", +] + +[[package]] +name = "solana-loader-v4-program" +version = "1.16.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "934ab242e2efd5cc99e371fdab05d903e80a74c3cc5d7d14bd25680ca2936435" +dependencies = [ + "log", + "rand 0.7.3", + "solana-measure", + "solana-program-runtime", + "solana-sdk", + "solana_rbpf", +] + [[package]] name = "solana-logger" version = "1.16.17" @@ -4509,6 +5333,17 @@ dependencies = [ "solana-sdk", ] +[[package]] +name = "solana-merkle-tree" +version = "1.16.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b3986083ce6fe3d70972ecb29b318c6ac211af2232b0ab774808fbe77dd865" +dependencies = [ + "fast-math", + "matches", + "solana-program", +] + [[package]] name = "solana-metrics" version = "1.16.17" @@ -4537,7 +5372,7 @@ dependencies = [ "rand 0.7.3", "serde", "serde_derive", - "socket2 0.4.9", + "socket2 0.4.10", "solana-logger", "solana-sdk", "solana-version", @@ -4551,7 +5386,7 @@ version = "1.16.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f900c1015844087cd4f10ba9d2d26a9859f2f5ca07427865cc74942595abc0a7" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.6", "bincode", "bv", "caps", @@ -4596,8 +5431,8 @@ dependencies = [ "console_error_panic_hook", "console_log", "curve25519-dalek", - "getrandom 0.2.10", - "itertools", + "getrandom 0.2.11", + "itertools 0.10.5", "js-sys", "lazy_static", "libc", @@ -4606,8 +5441,8 @@ dependencies = [ "memoffset 0.9.0", "num-bigint 0.4.4", "num-derive 0.3.3", - "num-traits 0.2.16", - "parking_lot", + "num-traits 0.2.17", + "parking_lot 0.12.1", "rand 0.7.3", "rand_chacha 0.2.2", "rustc_version", @@ -4616,7 +5451,7 @@ dependencies = [ "serde_bytes", "serde_derive", "serde_json", - "sha2 0.10.7", + "sha2 0.10.8", "sha3 0.10.8", "solana-frozen-abi", "solana-frozen-abi-macro", @@ -4637,11 +5472,11 @@ dependencies = [ "bincode", "eager", "enum-iterator", - "itertools", + "itertools 0.10.5", "libc", "log", "num-derive 0.3.3", - "num-traits 0.2.16", + "num-traits 0.2.17", "percentage", "rand 0.7.3", "rustc_version", @@ -4689,7 +5524,7 @@ dependencies = [ "async-mutex", "async-trait", "futures", - "itertools", + "itertools 0.10.5", "lazy_static", "log", "quinn", @@ -4728,8 +5563,8 @@ dependencies = [ "dialoguer", "log", "num-derive 0.3.3", - "num-traits 0.2.16", - "parking_lot", + "num-traits 0.2.17", + "parking_lot 0.12.1", "qstring", "semver", "solana-sdk", @@ -4798,6 +5633,75 @@ dependencies = [ "thiserror", ] +[[package]] +name = "solana-runtime" +version = "1.16.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f67421e70b1fdf33e1734f17ed54417091febe3ad23040aa45d5936a5ff6fbb" +dependencies = [ + "arrayref", + "bincode", + "blake3", + "bv", + "bytemuck", + "byteorder", + "bzip2", + "crossbeam-channel", + "dashmap 4.0.2", + "dir-diff", + "flate2", + "fnv", + "im", + "index_list", + "itertools 0.10.5", + "lazy_static", + "log", + "lru", + "lz4", + "memmap2", + "modular-bitfield", + "num-derive 0.3.3", + "num-traits 0.2.17", + "num_cpus", + "num_enum 0.6.1", + "once_cell", + "ouroboros", + "percentage", + "rand 0.7.3", + "rayon", + "regex", + "rustc_version", + "serde", + "serde_derive", + "solana-address-lookup-table-program", + "solana-bpf-loader-program", + "solana-bucket-map", + "solana-compute-budget-program", + "solana-config-program", + "solana-frozen-abi", + "solana-frozen-abi-macro", + "solana-loader-v4-program", + "solana-measure", + "solana-metrics", + "solana-perf", + "solana-program-runtime", + "solana-rayon-threadlimit", + "solana-sdk", + "solana-stake-program", + "solana-system-program", + "solana-vote-program", + "solana-zk-token-proof-program", + "solana-zk-token-sdk", + "static_assertions", + "strum", + "strum_macros", + "symlink", + "tar", + "tempfile", + "thiserror", + "zstd", +] + [[package]] name = "solana-sdk" version = "1.16.17" @@ -4819,14 +5723,14 @@ dependencies = [ "ed25519-dalek-bip32", "generic-array", "hmac 0.12.1", - "itertools", + "itertools 0.10.5", "js-sys", "lazy_static", "libsecp256k1", "log", "memmap2", "num-derive 0.3.3", - "num-traits 0.2.16", + "num-traits 0.2.17", "num_enum 0.6.1", "pbkdf2 0.11.0", "qstring", @@ -4839,7 +5743,7 @@ dependencies = [ "serde_derive", "serde_json", "serde_with", - "sha2 0.10.7", + "sha2 0.10.8", "sha3 0.10.8", "solana-frozen-abi", "solana-frozen-abi-macro", @@ -4855,13 +5759,79 @@ dependencies = [ name = "solana-sdk-macro" version = "1.16.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e560806a3859717eb2220b26e2cd68bb757b63affa3e79c3f1d8d853b5ee78f" +checksum = "1e560806a3859717eb2220b26e2cd68bb757b63affa3e79c3f1d8d853b5ee78f" +dependencies = [ + "bs58", + "proc-macro2 1.0.70", + "quote 1.0.33", + "rustversion", + "syn 2.0.39", +] + +[[package]] +name = "solana-stake-program" +version = "1.16.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0da6a315cd725d03d78f07144904207811767c784a0dd0db6f042ce14b2cb57" +dependencies = [ + "bincode", + "log", + "rustc_version", + "solana-config-program", + "solana-program-runtime", + "solana-sdk", + "solana-vote-program", +] + +[[package]] +name = "solana-storage-bigtable" +version = "1.16.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "934fec68868a72b28c9d51eac6558f2fc9490def4013b99d3e4171ba8f67b433" +dependencies = [ + "backoff", + "bincode", + "bytes", + "bzip2", + "enum-iterator", + "flate2", + "futures", + "goauth", + "http", + "hyper", + "hyper-proxy", + "log", + "openssl", + "prost 0.11.9", + "prost-types 0.11.9", + "serde", + "serde_derive", + "smpl_jwt", + "solana-metrics", + "solana-sdk", + "solana-storage-proto", + "solana-transaction-status", + "thiserror", + "tokio", + "tonic 0.8.3", + "zstd", +] + +[[package]] +name = "solana-storage-proto" +version = "1.16.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e38056b84ed3dc1fbfc6e35b34c6becd7739c0874f2d49d0d1e144d4cf4c4ac" dependencies = [ + "bincode", "bs58", - "proc-macro2 1.0.66", - "quote 1.0.33", - "rustversion", - "syn 2.0.31", + "prost 0.11.9", + "protobuf-src", + "serde", + "solana-account-decoder", + "solana-sdk", + "solana-transaction-status", + "tonic-build 0.8.4", ] [[package]] @@ -4876,7 +5846,7 @@ dependencies = [ "futures-util", "histogram", "indexmap 1.9.3", - "itertools", + "itertools 0.10.5", "libc", "log", "nix", @@ -4897,6 +5867,20 @@ dependencies = [ "x509-parser", ] +[[package]] +name = "solana-system-program" +version = "1.16.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa938573c40c2611bc9d875a4fe9a076c247b0ec9e557e9801de64c52ee4c12" +dependencies = [ + "bincode", + "log", + "serde", + "serde_derive", + "solana-program-runtime", + "solana-sdk", +] + [[package]] name = "solana-thin-client" version = "1.16.17" @@ -5003,7 +5987,7 @@ dependencies = [ "bincode", "log", "num-derive 0.3.3", - "num-traits 0.2.16", + "num-traits 0.2.17", "rustc_version", "serde", "serde_derive", @@ -5016,6 +6000,21 @@ dependencies = [ "thiserror", ] +[[package]] +name = "solana-zk-token-proof-program" +version = "1.16.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a92de0bab3e238e17210b9ecb6bb35e0d12488c17363eaaed03400557c16c45" +dependencies = [ + "bytemuck", + "getrandom 0.1.16", + "num-derive 0.3.3", + "num-traits 0.2.17", + "solana-program-runtime", + "solana-sdk", + "solana-zk-token-sdk", +] + [[package]] name = "solana-zk-token-sdk" version = "1.16.17" @@ -5029,11 +6028,11 @@ dependencies = [ "byteorder", "curve25519-dalek", "getrandom 0.1.16", - "itertools", + "itertools 0.10.5", "lazy_static", "merlin", "num-derive 0.3.3", - "num-traits 0.2.16", + "num-traits 0.2.17", "rand 0.7.3", "serde", "serde_json", @@ -5070,6 +6069,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "spki" version = "0.5.4" @@ -5089,7 +6094,7 @@ dependencies = [ "assert_matches", "borsh 0.10.3", "num-derive 0.4.1", - "num-traits 0.2.16", + "num-traits 0.2.17", "solana-program", "spl-token", "spl-token-2022", @@ -5115,7 +6120,7 @@ checksum = "fadbefec4f3c678215ca72bd71862697bb06b41fd77c0088902dd3203354387b" dependencies = [ "quote 1.0.33", "spl-discriminator-syn", - "syn 2.0.31", + "syn 2.0.39", ] [[package]] @@ -5124,10 +6129,10 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e5f2044ca42c8938d54d1255ce599c79a1ffd86b677dfab695caa20f9ffc3f2" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", - "sha2 0.10.7", - "syn 2.0.31", + "sha2 0.10.8", + "syn 2.0.39", "thiserror", ] @@ -5169,7 +6174,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "249e0318493b6bcf27ae9902600566c689b7dfba9f1bdff5893e92253374e78c" dependencies = [ "num-derive 0.4.1", - "num-traits 0.2.16", + "num-traits 0.2.17", "solana-program", "spl-program-error-derive", "thiserror", @@ -5181,10 +6186,10 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5269c8e868da17b6552ef35a51355a017bd8e0eae269c201fef830d35fa52c" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", - "sha2 0.10.7", - "syn 2.0.31", + "sha2 0.10.8", + "syn 2.0.39", ] [[package]] @@ -5210,7 +6215,7 @@ dependencies = [ "arrayref", "bytemuck", "num-derive 0.3.3", - "num-traits 0.2.16", + "num-traits 0.2.17", "num_enum 0.6.1", "solana-program", "thiserror", @@ -5225,7 +6230,7 @@ dependencies = [ "arrayref", "bytemuck", "num-derive 0.4.1", - "num-traits 0.2.16", + "num-traits 0.2.17", "num_enum 0.7.1", "solana-program", "solana-zk-token-sdk", @@ -5281,12 +6286,19 @@ dependencies = [ "spl-program-error", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "stringprep" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3737bde7edce97102e0e2b15365bf7a20bfdb5f60f4f9e8d7004258a51a8da" +checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" dependencies = [ + "finl_unicode", "unicode-bidi", "unicode-normalization", ] @@ -5303,12 +6315,40 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strum" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +dependencies = [ + "heck", + "proc-macro2 1.0.70", + "quote 1.0.33", + "rustversion", + "syn 1.0.109", +] + [[package]] name = "subtle" version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + [[package]] name = "syn" version = "0.15.44" @@ -5326,18 +6366,18 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.31" +version = "2.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", "unicode-ident", ] @@ -5354,30 +6394,62 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", "syn 1.0.109", "unicode-xid 0.2.4", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tar" +version = "0.4.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempfile" -version = "3.8.0" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" dependencies = [ "cfg-if", "fastrand", - "redox_syscall 0.3.5", + "redox_syscall 0.4.1", "rustix", "windows-sys 0.48.0", ] [[package]] name = "termcolor" -version = "1.2.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" dependencies = [ "winapi-util", ] @@ -5399,22 +6471,22 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.48" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.48" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", ] [[package]] @@ -5429,23 +6501,13 @@ dependencies = [ [[package]] name = "time" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" -dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", -] - -[[package]] -name = "time" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" +checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" dependencies = [ "deranged", "itoa", + "powerfmt", "serde", "time-core", "time-macros", @@ -5453,15 +6515,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" dependencies = [ "time-core", ] @@ -5502,19 +6564,19 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.32.0" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" dependencies = [ "backtrace", "bytes", "libc", "mio", "num_cpus", - "parking_lot", + "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.3", + "socket2 0.5.5", "tokio-macros", "windows-sys 0.48.0", ] @@ -5531,13 +6593,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", ] [[package]] @@ -5563,14 +6625,14 @@ dependencies = [ "futures-channel", "futures-util", "log", - "parking_lot", + "parking_lot 0.12.1", "percent-encoding", "phf", "pin-project-lite", "postgres-protocol", "postgres-types", "rand 0.8.5", - "socket2 0.5.3", + "socket2 0.5.5", "tokio", "tokio-util", "whoami", @@ -5593,7 +6655,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.7", + "rustls 0.21.9", "tokio", ] @@ -5626,9 +6688,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.8" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", @@ -5650,21 +6712,66 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.3" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" + +[[package]] +name = "toml_edit" +version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap 2.1.0", + "toml_datetime", + "winnow", +] [[package]] name = "toml_edit" -version = "0.19.14" +version = "0.20.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" +checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" dependencies = [ - "indexmap 2.0.0", + "indexmap 2.1.0", "toml_datetime", "winnow", ] +[[package]] +name = "tonic" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f219fad3b929bef19b1f86fbc0358d35daed8f2cac972037ac0dc10bbb8d5fb" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.13.1", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost 0.11.9", + "prost-derive 0.11.9", + "rustls-pemfile", + "tokio", + "tokio-rustls 0.23.4", + "tokio-stream", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "tracing", + "tracing-futures", +] + [[package]] name = "tonic" version = "0.10.2" @@ -5684,8 +6791,8 @@ dependencies = [ "hyper-timeout", "percent-encoding", "pin-project", - "prost", - "rustls 0.21.7", + "prost 0.12.3", + "rustls 0.21.9", "rustls-native-certs", "rustls-pemfile", "tokio", @@ -5697,17 +6804,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "tonic-build" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bf5e9b9c0f7e0a7c027dcfaba7b2c60816c7049171f679d99ee2ff65d0de8c4" +dependencies = [ + "prettyplease 0.1.25", + "proc-macro2 1.0.70", + "prost-build 0.11.9", + "quote 1.0.33", + "syn 1.0.109", +] + [[package]] name = "tonic-build" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d021fc044c18582b9a2408cd0dd05b1596e3ecdb5c4df822bb0183545683889" dependencies = [ - "prettyplease", - "proc-macro2 1.0.66", - "prost-build", + "prettyplease 0.2.15", + "proc-macro2 1.0.70", + "prost-build 0.12.3", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", ] [[package]] @@ -5717,10 +6837,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f80db390246dfb46553481f6024f0082ba00178ea495dbb99e70ba9a4fafb5e1" dependencies = [ "async-stream", - "prost", + "prost 0.12.3", "tokio", "tokio-stream", - "tonic", + "tonic 0.10.2", ] [[package]] @@ -5757,11 +6877,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -5770,41 +6889,51 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", ] [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", "valuable", ] +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + [[package]] name = "tracing-log" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" dependencies = [ - "lazy_static", "log", + "once_cell", "tracing-core", ] [[package]] name = "tracing-subscriber" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ "matchers", "nu-ansi-term", @@ -5818,6 +6947,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "trees" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de5f738ceab88e2491a94ddc33c3feeadfa95fedc60363ef110845df12f3878" + [[package]] name = "try-lock" version = "0.2.4" @@ -5848,9 +6983,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-bidi" @@ -5860,9 +6995,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" @@ -5875,9 +7010,9 @@ dependencies = [ [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "unicode-xid" @@ -5916,6 +7051,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "uriparse" version = "0.6.4" @@ -5979,6 +7120,16 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "walkdir" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -5994,12 +7145,6 @@ version = "0.9.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -6008,9 +7153,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -6018,24 +7163,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.37" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" +checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" dependencies = [ "cfg-if", "js-sys", @@ -6045,9 +7190,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" dependencies = [ "quote 1.0.33", "wasm-bindgen-macro-support", @@ -6055,28 +7200,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" dependencies = [ "js-sys", "wasm-bindgen", @@ -6084,12 +7229,12 @@ dependencies = [ [[package]] name = "webpki" -version = "0.22.1" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0e74f82d49d545ad128049b7e88f6576df2da6b02e9ce565c6f533be576957e" +checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" dependencies = [ - "ring", - "untrusted", + "ring 0.17.5", + "untrusted 0.9.0", ] [[package]] @@ -6107,23 +7252,23 @@ version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338" dependencies = [ - "rustls-webpki 0.100.2", + "rustls-webpki 0.100.3", ] [[package]] name = "webpki-roots" -version = "0.25.2" +version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" +checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" [[package]] name = "which" -version = "4.4.1" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ad25fe5717e59ada8ea33511bbbf7420b11031730a24c65e82428766c307006" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" dependencies = [ - "dirs", "either", + "home", "once_cell", "rustix", ] @@ -6156,9 +7301,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" dependencies = [ "winapi", ] @@ -6170,10 +7315,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows" -version = "0.48.0" +name = "windows-core" +version = "0.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" dependencies = [ "windows-targets 0.48.5", ] @@ -6211,6 +7356,15 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -6241,6 +7395,21 @@ dependencies = [ "windows_x86_64_msvc 0.48.5", ] +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -6253,6 +7422,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -6265,6 +7440,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -6277,6 +7458,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -6289,6 +7476,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -6301,6 +7494,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -6313,6 +7512,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -6325,11 +7530,17 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + [[package]] name = "winnow" -version = "0.5.15" +version = "0.5.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" +checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b" dependencies = [ "memchr", ] @@ -6359,7 +7570,16 @@ dependencies = [ "oid-registry", "rusticata-macros", "thiserror", - "time 0.3.28", + "time", +] + +[[package]] +name = "xattr" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4686009f71ff3e5c4dbcf1a282d0a44db3f021ba69350cd42086b3e5f1c6985" +dependencies = [ + "libc", ] [[package]] @@ -6368,7 +7588,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" dependencies = [ - "time 0.3.28", + "time", ] [[package]] @@ -6381,7 +7601,7 @@ dependencies = [ "futures", "http", "thiserror", - "tonic", + "tonic 0.10.2", "tonic-health", "yellowstone-grpc-proto", ] @@ -6394,13 +7614,33 @@ checksum = "00d751c6ef3093ec90ab1e16c6a504b5bea99aca6c688c429fed4cc56782f57e" dependencies = [ "anyhow", "bincode", - "prost", + "prost 0.12.3", "protobuf-src", "solana-account-decoder", "solana-sdk", "solana-transaction-status", - "tonic", - "tonic-build", + "tonic 0.10.2", + "tonic-build 0.10.2", +] + +[[package]] +name = "zerocopy" +version = "0.7.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e97e415490559a91254a2979b4829267a57d2fcd741a98eee8b722fb57289aa0" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f" +dependencies = [ + "proc-macro2 1.0.70", + "quote 1.0.33", + "syn 2.0.39", ] [[package]] @@ -6418,9 +7658,9 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ - "proc-macro2 1.0.66", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.31", + "syn 2.0.39", ] [[package]] @@ -6444,11 +7684,10 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.8+zstd.1.5.5" +version = "2.0.9+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" dependencies = [ "cc", - "libc", "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index be24c5bf..99ee94da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "quic-forward-proxy-integration-test", "cluster-endpoints", "history", + "stake_vote", "bench" ] @@ -28,6 +29,8 @@ solana-net-utils = "~1.16.3" solana-pubsub-client = "~1.16.3" solana-streamer = "~1.16.3" solana-account-decoder = "~1.16.3" +solana-ledger = "~1.16.3" +solana-program = "~1.16.3" itertools = "0.10.5" serde = { version = "1.0.160", features = ["derive"] } serde_json = "1.0.96" @@ -58,6 +61,7 @@ solana-lite-rpc-services = {path = "services", version="0.2.3"} solana-lite-rpc-core = {path = "core", version="0.2.3"} solana-lite-rpc-cluster-endpoints = {path = "cluster-endpoints", version="0.2.3"} solana-lite-rpc-history = {path = "history", version="0.2.3"} +solana-lite-rpc-stakevote = {path = "stake_vote", version="0.2.3"} async-trait = "0.1.68" yellowstone-grpc-client = "1.11.0" diff --git a/core/src/stores/data_cache.rs b/core/src/stores/data_cache.rs index f72948a1..975f5b74 100644 --- a/core/src/stores/data_cache.rs +++ b/core/src/stores/data_cache.rs @@ -1,9 +1,9 @@ -use std::sync::{atomic::AtomicU64, Arc}; - +use crate::structures::leaderschedule::CalculatedSchedule; use dashmap::DashMap; use solana_sdk::hash::Hash; use solana_sdk::slot_history::Slot; use solana_sdk::{commitment_config::CommitmentConfig, pubkey::Pubkey}; +use std::sync::{atomic::AtomicU64, Arc}; use crate::{ stores::{ @@ -37,6 +37,7 @@ pub struct DataCache { pub identity_stakes: IdentityStakes, pub cluster_info: ClusterInfo, pub epoch_data: EpochCache, + pub leader_schedule: Arc, } impl DataCache { @@ -87,6 +88,7 @@ impl DataCache { store: Arc::new(DashMap::new()), }, epoch_data: EpochCache::new_for_tests(), + leader_schedule: Arc::new(CalculatedSchedule::default()), } } } diff --git a/core/src/structures/epoch.rs b/core/src/structures/epoch.rs index 521d22ce..15cb3a1d 100644 --- a/core/src/structures/epoch.rs +++ b/core/src/structures/epoch.rs @@ -44,6 +44,10 @@ impl EpochCache { } } + pub fn get_epoch_schedule(&self) -> &EpochSchedule { + self.epoch_schedule.as_ref() + } + pub fn get_slots_in_epoch(&self, epoch: u64) -> u64 { self.epoch_schedule.get_slots_in_epoch(epoch) } diff --git a/core/src/structures/leaderschedule.rs b/core/src/structures/leaderschedule.rs new file mode 100644 index 00000000..441af12e --- /dev/null +++ b/core/src/structures/leaderschedule.rs @@ -0,0 +1,48 @@ +use crate::stores::block_information_store::BlockInformation; +use crate::stores::data_cache::DataCache; +use solana_sdk::commitment_config::CommitmentConfig; +use solana_sdk::pubkey::Pubkey; +use solana_sdk::vote::state::VoteState; +use std::collections::HashMap; + +#[derive(Clone, Default)] +pub struct CalculatedSchedule { + pub current: Option, + pub next: Option, +} + +impl CalculatedSchedule { + pub async fn get_leader_schedule_for_slot( + &self, + slot: Option, + commitment: Option, + data_cache: &DataCache, + ) -> Option>> { + let commitment = commitment.unwrap_or_else(|| CommitmentConfig::confirmed()); + let slot = match slot { + Some(slot) => slot, + None => { + let BlockInformation { slot, .. } = data_cache + .block_information_store + .get_latest_block(commitment) + .await; + slot + } + }; + let epoch = data_cache.epoch_data.get_epoch_at_slot(slot); + + let get_schedule = |schedule_data: Option<&LeaderScheduleData>| { + schedule_data.and_then(|current| { + (current.epoch == epoch.epoch).then_some(current.schedule.clone()) + }) + }; + get_schedule(self.current.as_ref()).or_else(|| get_schedule(self.next.as_ref())) + } +} + +#[derive(Clone)] +pub struct LeaderScheduleData { + pub schedule: HashMap>, + pub vote_stakes: HashMap, + pub epoch: u64, +} diff --git a/core/src/structures/mod.rs b/core/src/structures/mod.rs index 87b04083..ba7db841 100644 --- a/core/src/structures/mod.rs +++ b/core/src/structures/mod.rs @@ -3,6 +3,7 @@ pub mod epoch; pub mod identity_stakes; pub mod leader_data; +pub mod leaderschedule; pub mod notifications; pub mod produced_block; pub mod proxy_request_format; diff --git a/lite-rpc/Cargo.toml b/lite-rpc/Cargo.toml index 192b6d9a..6d4defa7 100644 --- a/lite-rpc/Cargo.toml +++ b/lite-rpc/Cargo.toml @@ -46,6 +46,7 @@ solana-lite-rpc-core = { workspace = true } solana-lite-rpc-services = { workspace = true } solana-lite-rpc-cluster-endpoints = { workspace = true } solana-lite-rpc-history = { workspace = true } +solana-lite-rpc-stakevote = { workspace = true } [dev-dependencies] bench = { path = "../bench" } diff --git a/lite-rpc/src/bridge.rs b/lite-rpc/src/bridge.rs index 9e785f4b..de36bd6c 100644 --- a/lite-rpc/src/bridge.rs +++ b/lite-rpc/src/bridge.rs @@ -4,6 +4,7 @@ use crate::{ rpc::LiteRpcServer, }; use solana_sdk::epoch_info::EpochInfo; +use std::collections::HashMap; use solana_lite_rpc_services::{ transaction_service::TransactionService, tx_sender::TXS_IN_CHANNEL, @@ -23,9 +24,9 @@ use solana_rpc_client_api::{ config::{ RpcBlockConfig, RpcBlockSubscribeConfig, RpcBlockSubscribeFilter, RpcBlocksConfigWrapper, RpcContextConfig, RpcEncodingConfigWrapper, RpcEpochConfig, RpcGetVoteAccountsConfig, - RpcProgramAccountsConfig, RpcRequestAirdropConfig, RpcSignatureStatusConfig, - RpcSignatureSubscribeConfig, RpcSignaturesForAddressConfig, RpcTransactionLogsConfig, - RpcTransactionLogsFilter, + RpcLeaderScheduleConfig, RpcProgramAccountsConfig, RpcRequestAirdropConfig, + RpcSignatureStatusConfig, RpcSignatureSubscribeConfig, RpcSignaturesForAddressConfig, + RpcTransactionLogsConfig, RpcTransactionLogsFilter, }, response::{ Response as RpcResponse, RpcBlockhash, RpcConfirmedTransactionStatusWithSignature, @@ -470,4 +471,18 @@ impl LiteRpcServer for LiteBridge { async fn vote_subscribe(&self, _pending: PendingSubscriptionSink) -> SubscriptionResult { todo!() } + + async fn get_leader_schedule( + &self, + slot: Option, + config: Option, + ) -> crate::rpc::Result>>> { + //TODO verify leader identity. + let schedule = self + .data_cache + .leader_schedule + .get_leader_schedule_for_slot(slot, config.and_then(|c| c.commitment), &self.data_cache) + .await; + Ok(schedule) + } } diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index a3a89454..af9ca54e 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -1,5 +1,6 @@ pub mod rpc_tester; +use solana_lite_rpc_core::structures::leaderschedule::CalculatedSchedule; use std::time::Duration; use anyhow::bail; @@ -159,6 +160,7 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: store: Arc::new(DashMap::new()), }, epoch_data, + leader_schedule: Arc::new(CalculatedSchedule::default()), }; let lata_cache_service = DataCachingService { @@ -217,6 +219,16 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: maximum_retries_per_tx, slot_notifier.resubscribe(), ); + + //start stake vote and leader schedule. + let stake_vote_jh = solana_lite_rpc_stakevote::start_stakes_and_votes_loop( + data_cache.clone(), + slot_notifier.resubscribe(), + Arc::clone(&rpc_client), + grpc_addr, + ) + .await?; + drop(slot_notifier); let support_service = tokio::spawn(async move { spawner.spawn_support_services().await }); @@ -235,6 +247,9 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: .start(lite_rpc_http_addr, lite_rpc_ws_addr), ); tokio::select! { + res = stake_vote_jh => { + anyhow::bail!("Stakes Votes Services {res:?}") + } res = tx_service_jh => { anyhow::bail!("Tx Services {res:?}") } diff --git a/lite-rpc/src/rpc.rs b/lite-rpc/src/rpc.rs index 3164a1bf..81843f0b 100644 --- a/lite-rpc/src/rpc.rs +++ b/lite-rpc/src/rpc.rs @@ -1,11 +1,12 @@ +use crate::configs::{IsBlockHashValidConfig, SendTransactionConfig}; use jsonrpsee::core::SubscriptionResult; use jsonrpsee::proc_macros::rpc; use solana_rpc_client_api::config::{ RpcBlockConfig, RpcBlockSubscribeConfig, RpcBlockSubscribeFilter, RpcBlocksConfigWrapper, RpcContextConfig, RpcEncodingConfigWrapper, RpcEpochConfig, RpcGetVoteAccountsConfig, - RpcProgramAccountsConfig, RpcRequestAirdropConfig, RpcSignatureStatusConfig, - RpcSignatureSubscribeConfig, RpcSignaturesForAddressConfig, RpcTransactionLogsConfig, - RpcTransactionLogsFilter, + RpcLeaderScheduleConfig, RpcProgramAccountsConfig, RpcRequestAirdropConfig, + RpcSignatureStatusConfig, RpcSignatureSubscribeConfig, RpcSignaturesForAddressConfig, + RpcTransactionLogsConfig, RpcTransactionLogsFilter, }; use solana_rpc_client_api::response::{ Response as RpcResponse, RpcBlockhash, RpcConfirmedTransactionStatusWithSignature, @@ -17,8 +18,7 @@ use solana_sdk::commitment_config::CommitmentConfig; use solana_sdk::epoch_info::EpochInfo; use solana_sdk::slot_history::Slot; use solana_transaction_status::{TransactionStatus, UiConfirmedBlock}; - -use crate::configs::{IsBlockHashValidConfig, SendTransactionConfig}; +use std::collections::HashMap; pub type Result = std::result::Result; @@ -225,4 +225,17 @@ pub trait LiteRpc { #[subscription(name = "voteSubscribe" => "voteNotification", unsubscribe="voteUnsubscribe", item=RpcVote)] async fn vote_subscribe(&self) -> SubscriptionResult; + + #[method(name = "getEpochInfo")] + async fn get_epoch_info( + &self, + config: Option, + ) -> crate::rpc::Result; + + #[method(name = "getLeaderSchedule")] + async fn get_leader_schedule( + &self, + slot: Option, + config: Option, + ) -> crate::rpc::Result>>>; } diff --git a/stake_vote/Cargo.toml b/stake_vote/Cargo.toml new file mode 100644 index 00000000..a6c66004 --- /dev/null +++ b/stake_vote/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "solana-lite-rpc-stakevote" +version = "0.2.3" +edition = "2021" +description = "History implementations used by solana lite rpc" +rust-version = "1.70.0" +repository = "https://github.com/blockworks-foundation/lite-rpc" +license = "AGPL" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "1.0.71" +bincode = "1.3.3" +borsh = "0.10.3" +bs58 = "0.4.0" +futures = { version = "0.3.28", default-features = false } +futures-util = "0.3.28" +hex = "0.4.3" +log = "0.4.17" +tracing-subscriber = "0.3.16" +tokio = { version = "1.*", features = ["full"] } +nom = "7.1.3" +itertools = "0.11.0" + +reqwest = "0.11" +serde = "1.0" +serde_json = "1.0" +thiserror = "1.0.40" + +yellowstone-grpc-client = "1.11.0+solana.1.16.14" +yellowstone-grpc-proto = "1.10.0+solana.1.16.14" + +solana-sdk = { workspace = true } +solana-client = { workspace = true } +solana-ledger = { workspace = true } +solana-rpc-client-api = { workspace = true } +solana-rpc-client = { workspace = true } +solana-version = { workspace = true } +solana-account-decoder = { workspace = true } +solana-program = { workspace = true } + +solana-lite-rpc-core = { workspace = true } diff --git a/stake_vote/src/account.rs b/stake_vote/src/account.rs new file mode 100644 index 00000000..10105a68 --- /dev/null +++ b/stake_vote/src/account.rs @@ -0,0 +1,93 @@ +use anyhow::bail; +use borsh::BorshDeserialize; +use solana_sdk::pubkey::Pubkey; +use solana_sdk::stake::state::Delegation; +use solana_sdk::stake::state::StakeState; +use solana_sdk::vote::state::VoteState; +use yellowstone_grpc_proto::prelude::SubscribeUpdateAccount; + +#[derive(Debug)] +#[allow(dead_code)] +pub struct AccountPretty { + pub is_startup: bool, + pub slot: u64, + pub pubkey: Pubkey, + pub lamports: u64, + pub owner: Pubkey, + pub executable: bool, + pub rent_epoch: u64, + pub data: Vec, + pub write_version: u64, + pub txn_signature: String, +} + +impl AccountPretty { + pub fn new_from_geyzer( + geyser_account: SubscribeUpdateAccount, + current_slot: u64, + ) -> Option { + let Some(inner_account) = geyser_account.account else { + log::warn!("Receive a SubscribeUpdateAccount without account."); + return None; + }; + + if geyser_account.slot != current_slot { + log::trace!( + "Get geyser account on a different slot:{} of the current:{current_slot}", + geyser_account.slot + ); + } + + Some(AccountPretty { + is_startup: geyser_account.is_startup, + slot: geyser_account.slot, + pubkey: Pubkey::try_from(inner_account.pubkey).expect("valid pubkey"), + lamports: inner_account.lamports, + owner: Pubkey::try_from(inner_account.owner).expect("valid pubkey"), + executable: inner_account.executable, + rent_epoch: inner_account.rent_epoch, + data: inner_account.data, + write_version: inner_account.write_version, + txn_signature: bs58::encode(inner_account.txn_signature.unwrap_or_default()) + .into_string(), + }) + } + + pub fn read_stake(&self) -> anyhow::Result> { + if self.data.is_empty() { + log::warn!("Stake account with empty data. Can't read vote."); + bail!("Error: read Stake account with empty data"); + } + + if self.data.is_empty() { + log::warn!("Stake account with empty data. Can't read stake."); + bail!("Error: read Stake account with empty data"); + } + match BorshDeserialize::deserialize(&mut self.data.as_slice())? { + StakeState::Stake(_, stake) => Ok(Some(stake.delegation)), + StakeState::Initialized(_) => Ok(None), + StakeState::Uninitialized => Ok(None), + StakeState::RewardsPool => Ok(None), + } + } + + pub fn read_vote(&self) -> anyhow::Result { + if self.data.is_empty() { + log::warn!("Vote account with empty data. Can't read vote."); + bail!("Error: read Vote account with empty data"); + } + Ok(VoteState::deserialize(&self.data)?) + } +} + +impl std::fmt::Display for AccountPretty { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{} at slot:{} lpt:{}", + self.pubkey.to_string(), + self.slot, + self.lamports + ) + } +} diff --git a/stake_vote/src/bootstrap.rs b/stake_vote/src/bootstrap.rs new file mode 100644 index 00000000..9323c8a8 --- /dev/null +++ b/stake_vote/src/bootstrap.rs @@ -0,0 +1,46 @@ +use crate::epoch::ScheduleEpochData; +use solana_lite_rpc_core::stores::data_cache::DataCache; +use solana_lite_rpc_core::structures::leaderschedule::CalculatedSchedule; +use solana_rpc_client::nonblocking::rpc_client::RpcClient; +use std::sync::Arc; + +pub async fn bootstrap_process_data( + data_cache: &DataCache, + rpc_client: Arc, +) -> (ScheduleEpochData, BootstrapData) { + let new_rate_activation_epoch = solana_sdk::feature_set::FeatureSet::default() + .new_warmup_cooldown_rate_epoch(data_cache.epoch_data.get_epoch_schedule()); + + let bootstrap_epoch = crate::utils::get_current_epoch(data_cache).await; + let current_schedule_epoch = ScheduleEpochData { + current_epoch: bootstrap_epoch.epoch, + slots_in_epoch: bootstrap_epoch.slots_in_epoch, + last_slot_in_epoch: data_cache + .epoch_data + .get_last_slot_in_epoch(bootstrap_epoch.epoch), + current_confirmed_slot: bootstrap_epoch.absolute_slot, + new_rate_activation_epoch, + }; + + //Init bootstrap process + let bootstrap_data = crate::bootstrap::BootstrapData { + done: false, + sleep_time: 1, + rpc_client, + }; + (current_schedule_epoch, bootstrap_data) +} + +pub fn bootstrap_leader_schedule( + current_file_patch: &str, + next_file_patch: &str, + slots_in_epoch: u64, +) -> anyhow::Result { + todo!(); +} + +pub struct BootstrapData { + pub done: bool, + pub sleep_time: u64, + pub rpc_client: Arc, +} diff --git a/stake_vote/src/epoch.rs b/stake_vote/src/epoch.rs new file mode 100644 index 00000000..4d6d8e5c --- /dev/null +++ b/stake_vote/src/epoch.rs @@ -0,0 +1,56 @@ +use crate::leader_schedule::LeaderScheduleEvent; +use serde::{Deserialize, Serialize}; +use solana_lite_rpc_core::stores::data_cache::DataCache; + +#[derive(Debug, Default, Copy, Clone, PartialOrd, PartialEq, Eq, Ord, Serialize, Deserialize)] +pub struct ScheduleEpochData { + pub current_epoch: u64, + pub slots_in_epoch: u64, + pub last_slot_in_epoch: u64, + pub current_confirmed_slot: u64, + pub new_rate_activation_epoch: Option, +} + +impl ScheduleEpochData { + pub async fn process_new_confirmed_slot( + &mut self, + new_slot: u64, + data_cache: &DataCache, + ) -> Option { + if self.current_confirmed_slot < new_slot { + self.current_confirmed_slot = new_slot; + log::trace!("Receive slot slot: {new_slot:?}"); + self.manage_change_epoch(data_cache).await + } else { + None + } + } + + async fn manage_change_epoch(&mut self, data_cache: &DataCache) -> Option { + //execute leaderschedule calculus at the last slot of the current epoch. + //account change of the slot has been send at confirmed slot. + //first epoch slot send all stake change and during this send no slot is send. + //to avoid to delay too much the schedule, start the calculus at the end of the epoch. + //the first epoch slot arrive very late cause of the stake account notification from the validator. + if self.current_confirmed_slot >= self.last_slot_in_epoch { + log::info!("Change epoch at slot:{}", self.current_confirmed_slot); + let next_epoch = data_cache + .epoch_data + .get_epoch_at_slot(self.last_slot_in_epoch + 1); + self.current_epoch = next_epoch.epoch; + self.last_slot_in_epoch = data_cache + .epoch_data + .get_last_slot_in_epoch(next_epoch.epoch); + + //start leader schedule calculus + //at current epoch change the schedule is calculated for the next epoch. + Some(crate::leader_schedule::LeaderScheduleEvent::Init( + self.current_epoch, + self.slots_in_epoch, + self.new_rate_activation_epoch.clone(), + )) + } else { + None + } + } +} diff --git a/stake_vote/src/leader_schedule.rs b/stake_vote/src/leader_schedule.rs new file mode 100644 index 00000000..92b04b8e --- /dev/null +++ b/stake_vote/src/leader_schedule.rs @@ -0,0 +1,31 @@ +use crate::stake::StakeMap; +use crate::stake::StakeStore; +use crate::vote::VoteMap; +use crate::vote::VoteStore; +use futures::stream::FuturesUnordered; +use solana_lite_rpc_core::structures::leaderschedule::LeaderScheduleData; +use solana_program::sysvar::epoch_schedule::EpochSchedule; +use solana_sdk::stake_history::StakeHistory; +use std::sync::Arc; +use tokio::task::JoinHandle; + +pub enum LeaderScheduleEvent { + Init(u64, u64, Option), + MergeStoreAndSaveSchedule( + StakeMap, + VoteMap, + LeaderScheduleData, + (u64, u64, Arc), + Option, + ), +} + +//Execute the leader schedule process. +pub fn run_leader_schedule_events( + event: LeaderScheduleEvent, + bootstrap_tasks: &mut FuturesUnordered>, + stakestore: &mut StakeStore, + votestore: &mut VoteStore, +) -> Option { + todo!(); +} diff --git a/stake_vote/src/lib.rs b/stake_vote/src/lib.rs new file mode 100644 index 00000000..ccf3725c --- /dev/null +++ b/stake_vote/src/lib.rs @@ -0,0 +1,190 @@ +use crate::account::AccountPretty; +use futures::Stream; +use futures_util::stream::FuturesUnordered; +use futures_util::StreamExt; +use solana_lite_rpc_core::stores::data_cache::DataCache; +use solana_lite_rpc_core::types::SlotStream; +use solana_rpc_client::nonblocking::rpc_client::RpcClient; +use std::collections::HashMap; +use std::sync::Arc; +use yellowstone_grpc_client::GeyserGrpcClient; +use yellowstone_grpc_proto::geyser::CommitmentLevel; +use yellowstone_grpc_proto::prelude::SubscribeRequestFilterAccounts; +use yellowstone_grpc_proto::prelude::SubscribeRequestFilterBlocksMeta; +use yellowstone_grpc_proto::prelude::SubscribeUpdate; +use yellowstone_grpc_proto::prelude::{subscribe_update::UpdateOneof, SubscribeRequestFilterSlots}; +use yellowstone_grpc_proto::tonic::Status; + +mod account; +mod bootstrap; +mod epoch; +mod leader_schedule; +mod stake; +mod utils; +mod vote; + +const STAKESTORE_INITIAL_CAPACITY: usize = 600000; +const VOTESTORE_INITIAL_CAPACITY: usize = 600000; + +type Slot = u64; + +pub async fn start_stakes_and_votes_loop( + mut data_cache: DataCache, + mut slot_notification: SlotStream, + rpc_client: Arc, + grpc_url: String, +) -> anyhow::Result> { + let mut account_gyzer_stream = subscribe_geyzer(grpc_url).await?; + let jh = tokio::spawn(async move { + //Stake account management struct + let mut stakestore = stake::StakeStore::new(STAKESTORE_INITIAL_CAPACITY); + + //Vote account management struct + let mut votestore = vote::VoteStore::new(VOTESTORE_INITIAL_CAPACITY); + + //Init bootstrap process + let (mut current_schedule_epoch, bootstrap_data) = + crate::bootstrap::bootstrap_process_data(&data_cache, rpc_client.clone()).await; + + //future execution collection. + let mut spawned_leader_schedule_task = FuturesUnordered::new(); + + loop { + tokio::select! { + //manage confirm new slot notification to detect epoch change. + Ok(_) = slot_notification.recv() => { + let new_slot = crate::utils::get_current_confirmed_slot(&data_cache).await; + let schedule_event = current_schedule_epoch.process_new_confirmed_slot(new_slot, &data_cache).await; + if bootstrap_data.done { + if let Some(init_event) = schedule_event { + crate::leader_schedule::run_leader_schedule_events( + init_event, + &mut spawned_leader_schedule_task, + &mut stakestore, + &mut votestore, + ); + } + } + } + //manage geyser account notification + //Geyser delete account notification patch must be installed on the validator. + //see https://github.com/solana-labs/solana/pull/33292 + ret = account_gyzer_stream.next() => { + match ret { + Some(message) => { + //process the message + match message { + Ok(msg) => { + match msg.update_oneof { + Some(UpdateOneof::Account(account)) => { + //store new account stake. + let current_slot = crate::utils::get_current_confirmed_slot(&data_cache).await; + + if let Some(account) = AccountPretty::new_from_geyzer(account, current_slot) { + //log::trace!("Geyser receive new account"); + match account.owner { + solana_sdk::stake::program::ID => { + log::info!("Geyser notif stake account:{}", account); + if let Err(err) = stakestore.notify_stake_change( + account, + current_schedule_epoch.last_slot_in_epoch, + ) { + log::warn!("Can't add new stake from account data err:{}", err); + continue; + } + } + solana_sdk::vote::program::ID => { + // Generatea lot of logs. log::info!("Geyser notif VOTE account:{}", account); + let account_pubkey = account.pubkey; + //process vote accout notification + if let Err(err) = votestore.add_vote(account, current_schedule_epoch.last_slot_in_epoch) { + log::warn!("Can't add new stake from account data err:{} account:{}", err, account_pubkey); + continue; + } + } + _ => log::warn!("receive an account notification from a unknown owner:{account:?}"), + } + } + } + Some(UpdateOneof::Ping(_)) => log::trace!("UpdateOneof::Ping"), + bad_msg => { + log::info!("Geyser stream unexpected message received:{:?}", bad_msg); + } + } + } + Err(error) => { + log::error!("Geyser stream receive an error has message: {error:?}, try to reconnect and resynchronize."); + //todo reconnect and resynchronize. + //break; + } + } + } + None => { + //TODO Restart geyser connection and the bootstrap. + log::error!("The geyser stream close try to reconnect and resynchronize."); + break; + } + } + } + //Manage leader schedule generation process + Some(Ok(event)) = spawned_leader_schedule_task.next() => { + let new_leader_schedule = crate::leader_schedule::run_leader_schedule_events( + event, + &mut spawned_leader_schedule_task, + &mut stakestore, + &mut votestore, + ); + //clone old schedule values is there's other use. + //only done once epoch. Avoid to use a Mutex. + let data_schedule = Arc::make_mut(&mut data_cache.leader_schedule); + data_schedule.current = data_schedule.next.take(); + data_schedule.next = new_leader_schedule; + } + } + } + }); + Ok(jh) +} + +//subscribe Geyser grpc +async fn subscribe_geyzer( + grpc_url: String, +) -> anyhow::Result>> { + let mut client = GeyserGrpcClient::connect(grpc_url, None::<&'static str>, None)?; + //slot subscription + let mut slots = HashMap::new(); + slots.insert("client".to_string(), SubscribeRequestFilterSlots {}); + + //account subscription + let mut accounts: HashMap = HashMap::new(); + accounts.insert( + "client".to_owned(), + SubscribeRequestFilterAccounts { + account: vec![], + owner: vec![ + solana_sdk::stake::program::ID.to_string(), + solana_sdk::vote::program::ID.to_string(), + ], + filters: vec![], + }, + ); + + //block Meta subscription filter + let mut blocks_meta = HashMap::new(); + blocks_meta.insert("client".to_string(), SubscribeRequestFilterBlocksMeta {}); + + let confirmed_stream = client + .subscribe_once( + slots.clone(), + accounts.clone(), //accounts + Default::default(), //tx + Default::default(), //entry + Default::default(), //full block + Default::default(), //block meta + Some(CommitmentLevel::Confirmed), + vec![], + ) + .await?; + + Ok(confirmed_stream) +} diff --git a/stake_vote/src/stake.rs b/stake_vote/src/stake.rs new file mode 100644 index 00000000..1bfec047 --- /dev/null +++ b/stake_vote/src/stake.rs @@ -0,0 +1,131 @@ +use crate::AccountPretty; +use crate::Slot; +use anyhow::bail; +use serde::{Deserialize, Serialize}; +use solana_sdk::pubkey::Pubkey; +use solana_sdk::stake::state::Delegation; +use solana_sdk::stake_history::StakeHistory; +use std::collections::HashMap; + +pub type StakeMap = HashMap; + +#[derive(Debug, Default)] +pub enum StakeAction { + Notify { + stake: StoredStake, + }, + Remove(Pubkey, Slot), + // Merge { + // source_account: Pubkey, + // destination_account: Pubkey, + // update_slot: Slot, + // }, + #[default] + None, +} + +impl StakeAction { + fn get_update_slot(&self) -> u64 { + match self { + StakeAction::Notify { stake } => stake.last_update_slot, + StakeAction::Remove(_, slot) => *slot, + StakeAction::None => 0, + } + } +} + +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct StoredStake { + pub pubkey: Pubkey, + pub lamports: u64, + pub stake: Delegation, + pub last_update_slot: Slot, + pub write_version: u64, +} + +#[derive(Debug, Default)] +pub struct StakeStore { + stakes: StakeMap, + stake_history: Option, + pub updates: Vec, + pub extracted: bool, +} + +impl StakeStore { + pub fn new(capacity: usize) -> Self { + StakeStore { + stakes: HashMap::with_capacity(capacity), + stake_history: None, + updates: vec![], + extracted: false, + } + } + + pub fn get_stake_history(&self) -> Option { + self.stake_history.clone() + } + + pub fn notify_stake_change( + &mut self, + account: AccountPretty, + current_end_epoch_slot: Slot, + ) -> anyhow::Result<()> { + //if lamport == 0 the account has been removed. + if account.lamports == 0 { + self.notify_stake_action( + StakeAction::Remove(account.pubkey, account.slot), + current_end_epoch_slot, + ); + } else { + let Ok(delegated_stake_opt) = account.read_stake() else { + bail!("Can't read stake from account data"); + }; + + if let Some(delegated_stake) = delegated_stake_opt { + let stake = StoredStake { + pubkey: account.pubkey, + lamports: account.lamports, + stake: delegated_stake, + last_update_slot: account.slot, + write_version: account.write_version, + }; + + self.notify_stake_action(StakeAction::Notify { stake }, current_end_epoch_slot); + } + } + + Ok(()) + } + pub fn notify_stake_action(&mut self, action: StakeAction, current_end_epoch_slot: Slot) { + //during extract push the new update or + //don't insertnow account change that has been done in next epoch. + //put in update pool to be merged next epoch change. + let insert_stake = !self.extracted || action.get_update_slot() > current_end_epoch_slot; + match insert_stake { + false => self.updates.push(action), + true => self.process_stake_action(action), + } + } + + fn process_stake_action(&mut self, action: StakeAction) { + match action { + StakeAction::Notify { stake } => { + todo!(); + } + StakeAction::Remove(account_pk, slot) => self.remove_from_store(&account_pk, slot), + StakeAction::None => (), + } + } + + fn remove_from_store(&mut self, account_pk: &Pubkey, update_slot: Slot) { + if self + .stakes + .get(account_pk) + .map(|stake| stake.last_update_slot <= update_slot) + .unwrap_or(false) + { + log::info!("Stake remove_from_store for {}", account_pk.to_string()); + self.stakes.remove(account_pk); + } + } +} diff --git a/stake_vote/src/utils.rs b/stake_vote/src/utils.rs new file mode 100644 index 00000000..acda274c --- /dev/null +++ b/stake_vote/src/utils.rs @@ -0,0 +1,18 @@ +use solana_lite_rpc_core::stores::block_information_store::BlockInformation; +use solana_lite_rpc_core::stores::data_cache::DataCache; +use solana_lite_rpc_core::structures::epoch::Epoch as LiteRpcEpoch; +use solana_sdk::commitment_config::CommitmentConfig; + +pub async fn get_current_confirmed_slot(data_cache: &DataCache) -> u64 { + let commitment = CommitmentConfig::confirmed(); + let BlockInformation { slot, .. } = data_cache + .block_information_store + .get_latest_block(commitment) + .await; + slot +} + +pub async fn get_current_epoch(data_cache: &DataCache) -> LiteRpcEpoch { + let commitment = CommitmentConfig::confirmed(); + data_cache.get_current_epoch(commitment).await +} diff --git a/stake_vote/src/vote.rs b/stake_vote/src/vote.rs new file mode 100644 index 00000000..f835064e --- /dev/null +++ b/stake_vote/src/vote.rs @@ -0,0 +1,84 @@ +use crate::AccountPretty; +use crate::Slot; +use anyhow::bail; +use serde::{Deserialize, Serialize}; +use solana_sdk::pubkey::Pubkey; +use solana_sdk::vote::state::VoteState; +use std::collections::HashMap; +use std::sync::Arc; + +pub type VoteMap = HashMap>; + +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct StoredVote { + pub pubkey: Pubkey, + pub vote_data: VoteState, + pub last_update_slot: Slot, + pub write_version: u64, +} + +#[derive(Default)] +pub struct VoteStore { + votes: VoteMap, + updates: Vec<(Pubkey, StoredVote)>, + extracted: bool, +} + +impl VoteStore { + pub fn new(capacity: usize) -> Self { + VoteStore { + votes: HashMap::with_capacity(capacity), + updates: vec![], + extracted: false, + } + } + pub fn add_vote( + &mut self, + new_account: AccountPretty, + current_end_epoch_slot: Slot, + ) -> anyhow::Result<()> { + if new_account.lamports == 0 { + self.remove_from_store(&new_account.pubkey, new_account.slot); + } else { + let Ok(vote_data) = new_account.read_vote() else { + bail!("Can't read Vote from account data"); + }; + + //log::info!("add_vote {} :{vote_data:?}", new_account.pubkey); + + let new_voteacc = StoredVote { + pubkey: new_account.pubkey, + vote_data, + last_update_slot: new_account.slot, + write_version: new_account.write_version, + }; + + //during extract push the new update or + //don't insertnow account change that has been done in next epoch. + //put in update pool to be merged next epoch change. + let insert_stake = + !self.extracted || new_voteacc.last_update_slot > current_end_epoch_slot; + match insert_stake { + false => self.updates.push((new_account.pubkey, new_voteacc)), + true => self.insert_vote(new_account.pubkey, new_voteacc), + } + } + + Ok(()) + } + fn insert_vote(&mut self, vote_account: Pubkey, vote_data: StoredVote) { + todo!(); + } + + fn remove_from_store(&mut self, account_pk: &Pubkey, update_slot: Slot) { + if self + .votes + .get(account_pk) + .map(|vote| vote.last_update_slot <= update_slot) + .unwrap_or(true) + { + log::info!("Vote remove_from_store for {}", account_pk.to_string()); + self.votes.remove(account_pk); + } + } +} From dc37b865f6935fb3af8f1b56804fbac0e9ede843 Mon Sep 17 00:00:00 2001 From: musitdev Date: Tue, 10 Oct 2023 15:22:20 +0200 Subject: [PATCH 05/56] update cargo.toml with workplace --- Cargo.lock | 12 +----------- Cargo.toml | 1 + stake_vote/Cargo.toml | 37 ++++++++++++++++--------------------- 3 files changed, 18 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8c8d2f88..0f469b76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2139,12 +2139,6 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - [[package]] name = "histogram" version = "0.6.9" @@ -5275,11 +5269,8 @@ dependencies = [ "bs58", "futures", "futures-util", - "hex", - "itertools 0.11.0", + "itertools 0.10.5", "log", - "nom", - "reqwest", "serde", "serde_json", "solana-account-decoder", @@ -5293,7 +5284,6 @@ dependencies = [ "solana-version", "thiserror", "tokio", - "tracing-subscriber", "yellowstone-grpc-client", "yellowstone-grpc-proto", ] diff --git a/Cargo.toml b/Cargo.toml index 99ee94da..56b79b16 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ serde_json = "1.0.96" bincode = "1.3.3" bs58 = "0.4.0" base64 = "0.21.0" +borsh = "0.10.3" thiserror = "1.0.40" futures = "0.3.28" bytes = "1.4.0" diff --git a/stake_vote/Cargo.toml b/stake_vote/Cargo.toml index a6c66004..07b0813c 100644 --- a/stake_vote/Cargo.toml +++ b/stake_vote/Cargo.toml @@ -10,27 +10,17 @@ license = "AGPL" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -anyhow = "1.0.71" -bincode = "1.3.3" -borsh = "0.10.3" -bs58 = "0.4.0" -futures = { version = "0.3.28", default-features = false } -futures-util = "0.3.28" -hex = "0.4.3" -log = "0.4.17" -tracing-subscriber = "0.3.16" -tokio = { version = "1.*", features = ["full"] } -nom = "7.1.3" -itertools = "0.11.0" - -reqwest = "0.11" -serde = "1.0" -serde_json = "1.0" -thiserror = "1.0.40" - -yellowstone-grpc-client = "1.11.0+solana.1.16.14" -yellowstone-grpc-proto = "1.10.0+solana.1.16.14" - +anyhow = { workspace = true } +bincode = { workspace = true } +borsh = { workspace = true } +bs58 = { workspace = true } +log = { workspace = true } +itertools = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +yellowstone-grpc-client = { workspace = true } +yellowstone-grpc-proto = { workspace = true } solana-sdk = { workspace = true } solana-client = { workspace = true } solana-ledger = { workspace = true } @@ -41,3 +31,8 @@ solana-account-decoder = { workspace = true } solana-program = { workspace = true } solana-lite-rpc-core = { workspace = true } + +futures = { version = "0.3.28", default-features = false } +tokio = { version = "1.28.2", features = ["full"]} +futures-util = "0.3.28" + From 0e5f607ad1ae45541c67faa93dd839a12fe54d4e Mon Sep 17 00:00:00 2001 From: musitdev Date: Thu, 12 Oct 2023 11:13:01 +0200 Subject: [PATCH 06/56] add the leader schedule calculus --- core/src/structures/leaderschedule.rs | 3 - stake_vote/src/account.rs | 36 ++-- stake_vote/src/leader_schedule.rs | 276 +++++++++++++++++++++++++- stake_vote/src/lib.rs | 17 +- stake_vote/src/stake.rs | 114 ++++++++--- stake_vote/src/utils.rs | 135 +++++++++++++ stake_vote/src/vote.rs | 106 ++++++++-- 7 files changed, 617 insertions(+), 70 deletions(-) diff --git a/core/src/structures/leaderschedule.rs b/core/src/structures/leaderschedule.rs index 441af12e..05086b42 100644 --- a/core/src/structures/leaderschedule.rs +++ b/core/src/structures/leaderschedule.rs @@ -1,8 +1,6 @@ use crate::stores::block_information_store::BlockInformation; use crate::stores::data_cache::DataCache; use solana_sdk::commitment_config::CommitmentConfig; -use solana_sdk::pubkey::Pubkey; -use solana_sdk::vote::state::VoteState; use std::collections::HashMap; #[derive(Clone, Default)] @@ -43,6 +41,5 @@ impl CalculatedSchedule { #[derive(Clone)] pub struct LeaderScheduleData { pub schedule: HashMap>, - pub vote_stakes: HashMap, pub epoch: u64, } diff --git a/stake_vote/src/account.rs b/stake_vote/src/account.rs index 10105a68..9e96c708 100644 --- a/stake_vote/src/account.rs +++ b/stake_vote/src/account.rs @@ -1,8 +1,11 @@ use anyhow::bail; use borsh::BorshDeserialize; +use solana_sdk::account::Account; +use solana_sdk::account::AccountSharedData; use solana_sdk::pubkey::Pubkey; use solana_sdk::stake::state::Delegation; use solana_sdk::stake::state::StakeState; +use solana_sdk::stake_history::StakeHistory; use solana_sdk::vote::state::VoteState; use yellowstone_grpc_proto::prelude::SubscribeUpdateAccount; @@ -54,21 +57,7 @@ impl AccountPretty { } pub fn read_stake(&self) -> anyhow::Result> { - if self.data.is_empty() { - log::warn!("Stake account with empty data. Can't read vote."); - bail!("Error: read Stake account with empty data"); - } - - if self.data.is_empty() { - log::warn!("Stake account with empty data. Can't read stake."); - bail!("Error: read Stake account with empty data"); - } - match BorshDeserialize::deserialize(&mut self.data.as_slice())? { - StakeState::Stake(_, stake) => Ok(Some(stake.delegation)), - StakeState::Initialized(_) => Ok(None), - StakeState::Uninitialized => Ok(None), - StakeState::RewardsPool => Ok(None), - } + read_stake_from_account_data(&mut self.data.as_slice()) } pub fn read_vote(&self) -> anyhow::Result { @@ -91,3 +80,20 @@ impl std::fmt::Display for AccountPretty { ) } } + +pub fn read_stake_from_account_data(mut data: &[u8]) -> anyhow::Result> { + if data.is_empty() { + log::warn!("Stake account with empty data. Can't read stake."); + bail!("Error: read Stake account with empty data"); + } + match BorshDeserialize::deserialize(&mut data)? { + StakeState::Stake(_, stake) => Ok(Some(stake.delegation)), + StakeState::Initialized(_) => Ok(None), + StakeState::Uninitialized => Ok(None), + StakeState::RewardsPool => Ok(None), + } +} + +pub fn read_historystake_from_account(account: Account) -> Option { + solana_sdk::account::from_account::(&AccountSharedData::from(account)) +} diff --git a/stake_vote/src/leader_schedule.rs b/stake_vote/src/leader_schedule.rs index 92b04b8e..e3a6cac9 100644 --- a/stake_vote/src/leader_schedule.rs +++ b/stake_vote/src/leader_schedule.rs @@ -1,31 +1,287 @@ -use crate::stake::StakeMap; -use crate::stake::StakeStore; -use crate::vote::VoteMap; -use crate::vote::VoteStore; +use crate::stake::{StakeMap, StakeStore}; +use crate::vote::StoredVote; +use crate::vote::{VoteMap, VoteStore}; use futures::stream::FuturesUnordered; -use solana_lite_rpc_core::structures::leaderschedule::LeaderScheduleData; +use itertools::Itertools; +use serde::{Deserialize, Serialize}; +use solana_client::rpc_client::RpcClient; +use solana_ledger::leader_schedule::LeaderSchedule; use solana_program::sysvar::epoch_schedule::EpochSchedule; +use solana_sdk::clock::NUM_CONSECUTIVE_LEADER_SLOTS; +use solana_sdk::commitment_config::CommitmentConfig; +use solana_sdk::feature_set::FeatureSet; +use solana_sdk::pubkey::Pubkey; +use solana_sdk::stake::state::StakeActivationStatus; use solana_sdk::stake_history::StakeHistory; +use std::collections::BTreeMap; +use std::collections::HashMap; +use std::fs::File; +use std::io::Write; +use std::str::FromStr; use std::sync::Arc; +use std::time::Duration; +use std::time::{SystemTime, UNIX_EPOCH}; use tokio::task::JoinHandle; +#[derive(Debug)] +pub struct LeaderScheduleGeneratedData { + pub schedule: HashMap>, + pub vote_stakes: HashMap)>, + pub epoch: u64, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct EpochStake { + epoch: u64, + stake_vote_map: HashMap)>, +} + +/* +Leader schedule calculus state diagram + +InitLeaderschedule + | + |extract store stake and vote| + | | + Error CalculateScedule(stakes, votes) + | | + | Wait(1s)| |Calculate schedule| + | | +InitLeaderscedule MergeStore(stakes, votes, schedule) + | | + Error SaveSchedule(schedule) + | | + |never occurs restart (wait 1s)| |save schedule and verify (opt)| + | + InitLeaderscedule +*/ + pub enum LeaderScheduleEvent { Init(u64, u64, Option), MergeStoreAndSaveSchedule( StakeMap, VoteMap, - LeaderScheduleData, - (u64, u64, Arc), + LeaderScheduleGeneratedData, + (u64, u64, Option), Option, ), } +enum LeaderScheduleResult { + TaskHandle(JoinHandle), + Event(LeaderScheduleEvent), + End(LeaderScheduleGeneratedData), +} + //Execute the leader schedule process. pub fn run_leader_schedule_events( event: LeaderScheduleEvent, - bootstrap_tasks: &mut FuturesUnordered>, + schedule_tasks: &mut FuturesUnordered>, + stakestore: &mut StakeStore, + votestore: &mut VoteStore, +) -> Option { + let result = process_leadershedule_event(event, stakestore, votestore); + match result { + LeaderScheduleResult::TaskHandle(jh) => { + schedule_tasks.push(jh); + None + } + LeaderScheduleResult::Event(event) => { + run_leader_schedule_events(event, schedule_tasks, stakestore, votestore) + } + LeaderScheduleResult::End(schedule) => Some(schedule), + } +} + +fn process_leadershedule_event( + // rpc_url: String, + event: LeaderScheduleEvent, stakestore: &mut StakeStore, votestore: &mut VoteStore, -) -> Option { - todo!(); +) -> LeaderScheduleResult { + match event { + LeaderScheduleEvent::Init(current_epoch, slots_in_epoch, new_rate_activation_epoch) => { + match ( + StakeStore::take_stakestore(stakestore), + VoteStore::take_votestore(votestore), + ) { + (Ok((stake_map, mut stake_history)), Ok(vote_map)) => { + //For test TODO put in extract and restore process to avoid to clone. + log::info!("LeaderScheduleEvent::CalculateScedule"); + let jh = tokio::task::spawn_blocking({ + move || { + let next_epoch = current_epoch + 1; + let epoch_vote_stakes = calculate_epoch_stakes( + &stake_map, + &vote_map, + current_epoch, + next_epoch, + stake_history.as_mut(), + new_rate_activation_epoch, + ); + + let leader_schedule = calculate_leader_schedule( + &epoch_vote_stakes, + next_epoch, + slots_in_epoch, + ); + log::info!("End calculate leader schedule"); + LeaderScheduleEvent::MergeStoreAndSaveSchedule( + stake_map, + vote_map, + LeaderScheduleGeneratedData { + schedule: leader_schedule, + vote_stakes: epoch_vote_stakes, + epoch: next_epoch, + }, + (current_epoch, slots_in_epoch, new_rate_activation_epoch), + stake_history, + ) + } + }); + LeaderScheduleResult::TaskHandle(jh) + } + _ => { + log::error!("Create leadershedule init event error during extract store"); + LeaderScheduleResult::Event(LeaderScheduleEvent::Init( + current_epoch, + slots_in_epoch, + new_rate_activation_epoch, + )) + } + } + } + LeaderScheduleEvent::MergeStoreAndSaveSchedule( + stake_map, + vote_map, + schedule_data, + (current_epoch, slots_in_epoch, epoch_schedule), + stake_history, + ) => { + log::info!("LeaderScheduleEvent::MergeStoreAndSaveSchedule RECV"); + match ( + StakeStore::merge_stakestore(stakestore, stake_map, stake_history), + VoteStore::merge_votestore(votestore, vote_map), + ) { + (Ok(()), Ok(())) => LeaderScheduleResult::End(schedule_data), + _ => { + //this shoud never arrive because the store has been extracted before. + //TODO remove this error using type state + log::warn!("LeaderScheduleEvent::MergeStoreAndSaveSchedule merge stake or vote fail, -restart Schedule"); + LeaderScheduleResult::Event(LeaderScheduleEvent::Init( + current_epoch, + slots_in_epoch, + epoch_schedule, + )) + } + } + } + } +} + +fn calculate_epoch_stakes( + stake_map: &StakeMap, + vote_map: &VoteMap, + current_epoch: u64, + next_epoch: u64, + mut stake_history: Option<&mut StakeHistory>, + new_rate_activation_epoch: Option, +) -> HashMap)> { + //update stake history with current end epoch stake values. + let stake_history_entry = + stake_map + .values() + .fold(StakeActivationStatus::default(), |acc, stake_account| { + let delegation = stake_account.stake; + acc + delegation.stake_activating_and_deactivating( + current_epoch, + stake_history.as_deref(), + new_rate_activation_epoch, + ) + }); + match stake_history { + Some(ref mut stake_history) => stake_history.add(current_epoch, stake_history_entry), + None => log::warn!("Vote stake calculus without Stake History"), + }; + + //calculate schedule stakes. + let delegated_stakes: HashMap = + stake_map + .values() + .fold(HashMap::default(), |mut delegated_stakes, stake_account| { + let delegation = stake_account.stake; + let entry = delegated_stakes.entry(delegation.voter_pubkey).or_default(); + *entry += delegation.stake( + next_epoch, + stake_history.as_deref(), + new_rate_activation_epoch, + ); + delegated_stakes + }); + + let staked_vote_map: HashMap)> = vote_map + .values() + .map(|vote_account| { + let delegated_stake = delegated_stakes + .get(&vote_account.pubkey) + .copied() + .unwrap_or_else(|| { + log::info!( + "calculate_epoch_stakes stake with no vote account:{}", + vote_account.pubkey + ); + Default::default() + }); + (vote_account.pubkey, (delegated_stake, vote_account.clone())) + }) + .collect(); + staked_vote_map +} + +//Copied from leader_schedule_utils.rs +// Mostly cribbed from leader_schedule_utils +fn calculate_leader_schedule( + stake_vote_map: &HashMap)>, + epoch: u64, + slots_in_epoch: u64, +) -> HashMap> { + let mut stakes: Vec<(Pubkey, u64)> = stake_vote_map + .iter() + .filter_map(|(_, (stake, vote_account))| { + (*stake > 0).then_some((vote_account.vote_data.node_pubkey, *stake)) + }) + .collect(); + + let mut seed = [0u8; 32]; + seed[0..8].copy_from_slice(&epoch.to_le_bytes()); + sort_stakes(&mut stakes); + log::info!("calculate_leader_schedule stakes:{stakes:?} epoch:{epoch}"); + let schedule = LeaderSchedule::new(&stakes, seed, slots_in_epoch, NUM_CONSECUTIVE_LEADER_SLOTS); + + let slot_schedule = schedule + .get_slot_leaders() + .iter() + .enumerate() + .map(|(i, pk)| (pk.to_string(), i)) + .into_group_map() + .into_iter() + .collect(); + slot_schedule +} + +// Cribbed from leader_schedule_utils +fn sort_stakes(stakes: &mut Vec<(Pubkey, u64)>) { + // Sort first by stake. If stakes are the same, sort by pubkey to ensure a + // deterministic result. + // Note: Use unstable sort, because we dedup right after to remove the equal elements. + stakes.sort_unstable_by(|(l_pubkey, l_stake), (r_pubkey, r_stake)| { + if r_stake == l_stake { + r_pubkey.cmp(l_pubkey) + } else { + r_stake.cmp(l_stake) + } + }); + + // Now that it's sorted, we can do an O(n) dedup. + stakes.dedup(); } diff --git a/stake_vote/src/lib.rs b/stake_vote/src/lib.rs index ccf3725c..dbd1821e 100644 --- a/stake_vote/src/lib.rs +++ b/stake_vote/src/lib.rs @@ -1,8 +1,10 @@ use crate::account::AccountPretty; +use crate::leader_schedule::LeaderScheduleGeneratedData; use futures::Stream; use futures_util::stream::FuturesUnordered; use futures_util::StreamExt; use solana_lite_rpc_core::stores::data_cache::DataCache; +use solana_lite_rpc_core::structures::leaderschedule::LeaderScheduleData; use solana_lite_rpc_core::types::SlotStream; use solana_rpc_client::nonblocking::rpc_client::RpcClient; use std::collections::HashMap; @@ -138,7 +140,20 @@ pub async fn start_stakes_and_votes_loop( //only done once epoch. Avoid to use a Mutex. let data_schedule = Arc::make_mut(&mut data_cache.leader_schedule); data_schedule.current = data_schedule.next.take(); - data_schedule.next = new_leader_schedule; + match new_leader_schedule { + //TODO use vote_stakes for vote accounts RPC call. + Some(LeaderScheduleGeneratedData{schedule, vote_stakes, epoch}) => { + let new_schedule_data = LeaderScheduleData{ + schedule, + epoch + }; + data_schedule.next = Some(new_schedule_data); + } + None => { + log::warn!("Error during schedule calculus. No schedule for this epoch."); + data_schedule.next = None; + } + }; } } } diff --git a/stake_vote/src/stake.rs b/stake_vote/src/stake.rs index 1bfec047..428157b2 100644 --- a/stake_vote/src/stake.rs +++ b/stake_vote/src/stake.rs @@ -1,13 +1,17 @@ +use crate::utils::TakableContent; +use crate::utils::TakableMap; use crate::AccountPretty; use crate::Slot; use anyhow::bail; use serde::{Deserialize, Serialize}; +use solana_sdk::account::Account; use solana_sdk::pubkey::Pubkey; use solana_sdk::stake::state::Delegation; use solana_sdk::stake_history::StakeHistory; use std::collections::HashMap; pub type StakeMap = HashMap; +type StakeContent = (StakeMap, Option); #[derive(Debug, Default)] pub enum StakeAction { @@ -43,26 +47,26 @@ pub struct StoredStake { pub write_version: u64, } +impl TakableContent for StakeContent { + fn add_value(&mut self, val: StakeAction) { + StakeStore::process_stake_action(&mut self.0, val); + } +} + #[derive(Debug, Default)] pub struct StakeStore { - stakes: StakeMap, - stake_history: Option, - pub updates: Vec, - pub extracted: bool, + stakes: TakableMap, } impl StakeStore { pub fn new(capacity: usize) -> Self { StakeStore { - stakes: HashMap::with_capacity(capacity), - stake_history: None, - updates: vec![], - extracted: false, + stakes: TakableMap::new((HashMap::with_capacity(capacity), None)), } } pub fn get_stake_history(&self) -> Option { - self.stake_history.clone() + self.stakes.content.1.clone() } pub fn notify_stake_change( @@ -96,36 +100,98 @@ impl StakeStore { Ok(()) } + pub fn notify_stake_action(&mut self, action: StakeAction, current_end_epoch_slot: Slot) { - //during extract push the new update or - //don't insertnow account change that has been done in next epoch. - //put in update pool to be merged next epoch change. - let insert_stake = !self.extracted || action.get_update_slot() > current_end_epoch_slot; - match insert_stake { - false => self.updates.push(action), - true => self.process_stake_action(action), - } + let action_update_slot = action.get_update_slot(); + self.stakes + .add_value(action, action_update_slot <= current_end_epoch_slot); } - fn process_stake_action(&mut self, action: StakeAction) { + fn process_stake_action(stakes: &mut StakeMap, action: StakeAction) { match action { StakeAction::Notify { stake } => { - todo!(); + Self::notify_stake(stakes, stake); } - StakeAction::Remove(account_pk, slot) => self.remove_from_store(&account_pk, slot), + StakeAction::Remove(account_pk, slot) => Self::remove_stake(stakes, &account_pk, slot), StakeAction::None => (), } } + fn notify_stake(map: &mut StakeMap, stake: StoredStake) { + //log::info!("stake_map_notify_stake stake:{stake:?}"); + match map.entry(stake.pubkey) { + // If value already exists, then increment it by one + std::collections::hash_map::Entry::Occupied(occupied) => { + let strstake = occupied.into_mut(); // <-- get mut reference to existing value + //doesn't erase new state with an old one. Can arrive during bootstrapping. + //several instructions can be done in the same slot. + if strstake.last_update_slot <= stake.last_update_slot { + log::info!("stake_map_notify_stake Stake store updated stake: {} old_stake:{strstake:?} stake:{stake:?}", stake.pubkey); + *strstake = stake; + } + } + // If value doesn't exist yet, then insert a new value of 1 + std::collections::hash_map::Entry::Vacant(vacant) => { + log::info!( + "stake_map_notify_stake Stake store insert stake: {} stake:{stake:?}", + stake.pubkey + ); + vacant.insert(stake); + } + }; + } - fn remove_from_store(&mut self, account_pk: &Pubkey, update_slot: Slot) { - if self - .stakes + fn remove_stake(stakes: &mut StakeMap, account_pk: &Pubkey, update_slot: Slot) { + if stakes .get(account_pk) .map(|stake| stake.last_update_slot <= update_slot) .unwrap_or(false) { log::info!("Stake remove_from_store for {}", account_pk.to_string()); - self.stakes.remove(account_pk); + stakes.remove(account_pk); } } + + //helper method to extract and merge stakes. + pub fn take_stakestore( + stakestore: &mut StakeStore, + ) -> anyhow::Result<(StakeMap, Option)> { + crate::utils::take(&mut stakestore.stakes) + } + + pub fn merge_stakestore( + stakestore: &mut StakeStore, + stake_map: StakeMap, + stake_history: Option, + ) -> anyhow::Result<()> { + crate::utils::merge(&mut stakestore.stakes, (stake_map, stake_history)) + } +} + +pub fn merge_program_account_in_strake_map( + stake_map: &mut StakeMap, + stakes_list: Vec<(Pubkey, Account)>, + last_update_slot: Slot, +) { + stakes_list + .into_iter() + .filter_map(|(pk, account)| { + match crate::account::read_stake_from_account_data(&account.data) { + Ok(opt_stake) => opt_stake.map(|stake| (pk, stake, account.lamports)), + Err(err) => { + log::warn!("Error during pa account data deserialisation:{err}"); + None + } + } + }) + .for_each(|(pk, delegated_stake, lamports)| { + let stake = StoredStake { + pubkey: pk, + lamports, + stake: delegated_stake, + last_update_slot, + write_version: 0, + }; + + StakeStore::notify_stake(stake_map, stake); + }); } diff --git a/stake_vote/src/utils.rs b/stake_vote/src/utils.rs index acda274c..faf36858 100644 --- a/stake_vote/src/utils.rs +++ b/stake_vote/src/utils.rs @@ -1,7 +1,11 @@ +use anyhow::bail; use solana_lite_rpc_core::stores::block_information_store::BlockInformation; use solana_lite_rpc_core::stores::data_cache::DataCache; use solana_lite_rpc_core::structures::epoch::Epoch as LiteRpcEpoch; use solana_sdk::commitment_config::CommitmentConfig; +use solana_sdk::pubkey::Pubkey; +use std::collections::HashMap; +use std::default::Default; pub async fn get_current_confirmed_slot(data_cache: &DataCache) -> u64 { let commitment = CommitmentConfig::confirmed(); @@ -16,3 +20,134 @@ pub async fn get_current_epoch(data_cache: &DataCache) -> LiteRpcEpoch { let commitment = CommitmentConfig::confirmed(); data_cache.get_current_epoch(commitment).await } + +//Takable struct code +pub trait TakableContent: Default { + fn add_value(&mut self, val: T); +} + +///A struct that hold a collection call content that can be taken during some time and merged after. +///During the time the content is taken, new added values are cached and added to the content after the merge. +///It allow to process struct content while allowing to still update it without lock. +#[derive(Default, Debug)] +pub struct TakableMap> { + pub content: C, + pub updates: Vec, + taken: bool, +} + +impl + Default> TakableMap { + pub fn new(content: C) -> Self { + TakableMap { + content, + updates: vec![], + taken: false, + } + } + + //add a value to the content if not taken or put it in the update waiting list. + //Use force_in_update to force the insert in update waiting list. + pub fn add_value(&mut self, val: T, force_in_update: bool) { + //during extract push the new update or + //don't insert now account change that has been done in next epoch. + //put in update pool to be merged next epoch change. + match self.taken || force_in_update { + true => self.updates.push(val), + false => self.content.add_value(val), + } + } + + pub fn take(self) -> (Self, C) { + let takenmap = TakableMap { + content: C::default(), + updates: self.updates, + taken: true, + }; + (takenmap, self.content) + } + + pub fn merge(self, content: C) -> Self { + let mut mergedstore = TakableMap { + content, + updates: vec![], + taken: false, + }; + + //apply stake added during extraction. + for val in self.updates { + mergedstore.content.add_value(val); + } + mergedstore + } + + pub fn is_taken(&self) -> bool { + self.taken + } +} + +pub fn take + Default>( + map: &mut TakableMap, +) -> anyhow::Result { + if map.is_taken() { + bail!("TakableMap already taken. Try later"); + } + let new_store = std::mem::take(map); + let (new_store, content) = new_store.take(); + *map = new_store; + Ok(content) +} + +pub fn merge + Default>( + map: &mut TakableMap, + content: C, +) -> anyhow::Result<()> { + if !map.is_taken() { + bail!("TakableMap merge of non taken map. Try later"); + } + let new_store = std::mem::take(map); + let new_store = new_store.merge(content); + *map = new_store; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_takable_struct() { + impl TakableContent for Vec { + fn add_value(&mut self, val: u64) { + self.push(val) + } + } + + let content: Vec = vec![]; + let mut takable = TakableMap::new(content); + takable.add_value(23, false); + assert_eq!(takable.content.len(), 1); + + takable.add_value(24, true); + assert_eq!(takable.content.len(), 1); + assert_eq!(takable.updates.len(), 1); + + let content = take(&mut takable).unwrap(); + assert_eq!(content.len(), 1); + assert_eq!(takable.content.len(), 0); + assert_eq!(takable.updates.len(), 1); + let err_content = take(&mut takable); + assert!(err_content.is_err()); + assert_eq!(content.len(), 1); + assert_eq!(takable.content.len(), 0); + assert_eq!(takable.updates.len(), 1); + takable.add_value(25, false); + assert_eq!(takable.content.len(), 0); + assert_eq!(takable.updates.len(), 2); + merge(&mut takable, content).unwrap(); + assert_eq!(takable.content.len(), 3); + assert_eq!(takable.updates.len(), 0); + + let err = merge(&mut takable, vec![]); + assert!(err.is_err()); + } +} diff --git a/stake_vote/src/vote.rs b/stake_vote/src/vote.rs index f835064e..a3444fd9 100644 --- a/stake_vote/src/vote.rs +++ b/stake_vote/src/vote.rs @@ -1,7 +1,10 @@ +use crate::utils::TakableContent; +use crate::utils::TakableMap; use crate::AccountPretty; use crate::Slot; use anyhow::bail; use serde::{Deserialize, Serialize}; +use solana_sdk::account::Account; use solana_sdk::pubkey::Pubkey; use solana_sdk::vote::state::VoteState; use std::collections::HashMap; @@ -9,6 +12,12 @@ use std::sync::Arc; pub type VoteMap = HashMap>; +impl TakableContent for VoteMap { + fn add_value(&mut self, val: StoredVote) { + VoteStore::vote_map_insert_vote(self, val.pubkey, val); + } +} + #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct StoredVote { pub pubkey: Pubkey, @@ -19,17 +28,13 @@ pub struct StoredVote { #[derive(Default)] pub struct VoteStore { - votes: VoteMap, - updates: Vec<(Pubkey, StoredVote)>, - extracted: bool, + votes: TakableMap, } impl VoteStore { pub fn new(capacity: usize) -> Self { VoteStore { - votes: HashMap::with_capacity(capacity), - updates: vec![], - extracted: false, + votes: TakableMap::new(HashMap::with_capacity(capacity)), } } pub fn add_vote( @@ -53,32 +58,99 @@ impl VoteStore { write_version: new_account.write_version, }; - //during extract push the new update or - //don't insertnow account change that has been done in next epoch. - //put in update pool to be merged next epoch change. - let insert_stake = - !self.extracted || new_voteacc.last_update_slot > current_end_epoch_slot; - match insert_stake { - false => self.updates.push((new_account.pubkey, new_voteacc)), - true => self.insert_vote(new_account.pubkey, new_voteacc), - } + let action_update_slot = new_voteacc.last_update_slot; + self.votes + .add_value(new_voteacc, action_update_slot <= current_end_epoch_slot); } Ok(()) } + //helper method to extract and merge stakes. + pub fn take_votestore(votestore: &mut VoteStore) -> anyhow::Result { + crate::utils::take(&mut votestore.votes) + } + + pub fn merge_votestore(votestore: &mut VoteStore, vote_map: VoteMap) -> anyhow::Result<()> { + crate::utils::merge(&mut votestore.votes, vote_map) + } + fn insert_vote(&mut self, vote_account: Pubkey, vote_data: StoredVote) { - todo!(); + Self::vote_map_insert_vote(&mut self.votes.content, vote_account, vote_data); } fn remove_from_store(&mut self, account_pk: &Pubkey, update_slot: Slot) { if self .votes + .content .get(account_pk) .map(|vote| vote.last_update_slot <= update_slot) .unwrap_or(true) { log::info!("Vote remove_from_store for {}", account_pk.to_string()); - self.votes.remove(account_pk); + self.votes.content.remove(account_pk); } } + + fn vote_map_insert_vote(map: &mut VoteMap, vote_account_pk: Pubkey, vote_data: StoredVote) { + match map.entry(vote_account_pk) { + std::collections::hash_map::Entry::Occupied(occupied) => { + let voteacc = occupied.into_mut(); // <-- get mut reference to existing value + if voteacc.last_update_slot <= vote_data.last_update_slot { + // generate a lot of trace log::trace!( + // "Vote updated for: {vote_account_pk} node_id:{} root_slot:{:?}", + // vote_data.vote_data.node_pubkey, + // vote_data.vote_data.root_slot, + // ); + if vote_data.vote_data.root_slot.is_none() { + log::info!("Update vote account:{vote_account_pk} with None root slot."); + } + + if voteacc.vote_data.root_slot.is_none() { + log::info!( + "Update vote account:{vote_account_pk} that were having None root slot." + ); + } + + *voteacc = Arc::new(vote_data); + } + } + // If value doesn't exist yet, then insert a new value of 1 + std::collections::hash_map::Entry::Vacant(vacant) => { + log::info!( + "New Vote added for: {vote_account_pk} node_id:{}, root slot:{:?}", + vote_data.vote_data.node_pubkey, + vote_data.vote_data.root_slot, + ); + vacant.insert(Arc::new(vote_data)); + } + }; + } +} + +pub fn merge_program_account_in_vote_map( + vote_map: &mut VoteMap, + pa_list: Vec<(Pubkey, Account)>, + last_update_slot: Slot, +) { + pa_list + .into_iter() + .filter_map( + |(pk, account)| match VoteState::deserialize(&account.data) { + Ok(vote) => Some((pk, vote)), + Err(err) => { + log::warn!("Error during vote account data deserialisation:{err}"); + None + } + }, + ) + .for_each(|(pk, vote)| { + //log::info!("Vote init {pk} :{vote:?}"); + let vote = StoredVote { + pubkey: pk, + vote_data: vote, + last_update_slot, + write_version: 0, + }; + VoteStore::vote_map_insert_vote(vote_map, pk, vote); + }); } From b6cef88fa94867743f3dfa0067a036cc4e41ac8e Mon Sep 17 00:00:00 2001 From: musitdev Date: Thu, 12 Oct 2023 11:38:10 +0200 Subject: [PATCH 07/56] remove old comment --- stake_vote/src/leader_schedule.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stake_vote/src/leader_schedule.rs b/stake_vote/src/leader_schedule.rs index e3a6cac9..6058e6c9 100644 --- a/stake_vote/src/leader_schedule.rs +++ b/stake_vote/src/leader_schedule.rs @@ -106,8 +106,8 @@ fn process_leadershedule_event( VoteStore::take_votestore(votestore), ) { (Ok((stake_map, mut stake_history)), Ok(vote_map)) => { - //For test TODO put in extract and restore process to avoid to clone. log::info!("LeaderScheduleEvent::CalculateScedule"); + //do the calculus in a blocking task. let jh = tokio::task::spawn_blocking({ move || { let next_epoch = current_epoch + 1; From 952b0c866c5f1931d8a94b8ff07ff25e86c3eaf2 Mon Sep 17 00:00:00 2001 From: musitdev Date: Wed, 18 Oct 2023 09:34:16 +0200 Subject: [PATCH 08/56] add stake and schedule bootstrap --- stake_vote/src/bootstrap.rs | 304 +++++++++++++++++++++++++++--- stake_vote/src/leader_schedule.rs | 39 ++-- stake_vote/src/lib.rs | 54 +++++- stake_vote/src/stake.rs | 6 +- stake_vote/src/utils.rs | 51 +++++ stake_vote/src/vote.rs | 4 - tests/client.test.ts | 10 + 7 files changed, 422 insertions(+), 46 deletions(-) diff --git a/stake_vote/src/bootstrap.rs b/stake_vote/src/bootstrap.rs index 9323c8a8..fadab96f 100644 --- a/stake_vote/src/bootstrap.rs +++ b/stake_vote/src/bootstrap.rs @@ -1,13 +1,29 @@ use crate::epoch::ScheduleEpochData; +use crate::stake::StakeMap; +use crate::stake::StakeStore; +use crate::vote::VoteMap; +use crate::vote::VoteStore; +use anyhow::bail; +use futures_util::stream::FuturesUnordered; +use solana_client::client_error::ClientError; +use solana_client::rpc_client::RpcClient; use solana_lite_rpc_core::stores::data_cache::DataCache; use solana_lite_rpc_core::structures::leaderschedule::CalculatedSchedule; -use solana_rpc_client::nonblocking::rpc_client::RpcClient; -use std::sync::Arc; +use solana_lite_rpc_core::structures::leaderschedule::LeaderScheduleData; +use solana_sdk::account::Account; +use solana_sdk::commitment_config::CommitmentConfig; +use solana_sdk::pubkey::Pubkey; +use solana_sdk::stake_history::StakeHistory; +use std::time::Duration; +use tokio::task::JoinHandle; -pub async fn bootstrap_process_data( - data_cache: &DataCache, - rpc_client: Arc, -) -> (ScheduleEpochData, BootstrapData) { +//File where the Vote and stake use to calculate the leader schedule at epoch are stored. +//Use to bootstrap current and next epoch leader schedule. +//TODO to be removed with inter RPC bootstrap and snapshot read. +pub const CURRENT_EPOCH_VOTE_STAKES_FILE: &str = "current_vote_stakes.json"; +pub const NEXT_EPOCH_VOTE_STAKES_FILE: &str = "next_vote_stakes.json"; + +pub async fn bootstrap_scheduleepoch_data(data_cache: &DataCache) -> ScheduleEpochData { let new_rate_activation_epoch = solana_sdk::feature_set::FeatureSet::default() .new_warmup_cooldown_rate_epoch(data_cache.epoch_data.get_epoch_schedule()); @@ -22,25 +38,269 @@ pub async fn bootstrap_process_data( new_rate_activation_epoch, }; - //Init bootstrap process - let bootstrap_data = crate::bootstrap::BootstrapData { - done: false, - sleep_time: 1, - rpc_client, - }; - (current_schedule_epoch, bootstrap_data) + current_schedule_epoch +} + +/* +Bootstrap state changes + + InitBootstrap + | + |Fetch accounts| + + | | + Error BootstrapAccountsFetched(account list) + | | + |Exit| |Extract stores| + | | + Error StoreExtracted(account list, stores) + | | + | Wait(1s)| |Merge accounts in store| + | | | + BootstrapAccountsFetched(account list) Error AccountsMerged(stores) + | | + |Log and skip| |Merges store| + |Account | | | + Error End + | + |never occurs restart| + | + InitBootstrap +*/ + +pub fn run_bootstrap_events( + event: BootstrapEvent, + bootstrap_tasks: &mut FuturesUnordered>, + stakestore: &mut StakeStore, + votestore: &mut VoteStore, +) -> anyhow::Result> { + let result = process_bootstrap_event(event, stakestore, votestore); + match result { + BootsrapProcessResult::TaskHandle(jh) => { + bootstrap_tasks.push(jh); + Ok(None) + } + BootsrapProcessResult::Event(event) => { + run_bootstrap_events(event, bootstrap_tasks, stakestore, votestore) + } + BootsrapProcessResult::End => Ok(Some(true)), + BootsrapProcessResult::Error(err) => bail!(err), + } +} + +pub enum BootstrapEvent { + InitBootstrap { + sleep_time: u64, + rpc_url: String, + }, + BootstrapAccountsFetched( + Vec<(Pubkey, Account)>, + Vec<(Pubkey, Account)>, + Account, + String, + ), + StoreExtracted( + StakeMap, + VoteMap, + Vec<(Pubkey, Account)>, + Vec<(Pubkey, Account)>, + Account, + String, + ), + AccountsMerged(StakeMap, Option, VoteMap, String), + Exit, +} + +enum BootsrapProcessResult { + TaskHandle(JoinHandle), + Event(BootstrapEvent), + Error(String), + End, +} + +fn process_bootstrap_event( + event: BootstrapEvent, + stakestore: &mut StakeStore, + votestore: &mut VoteStore, +) -> BootsrapProcessResult { + match event { + BootstrapEvent::InitBootstrap { + sleep_time, + rpc_url, + } => { + let jh = tokio::task::spawn_blocking(move || { + log::info!("BootstrapEvent::InitBootstrap RECV"); + if sleep_time > 0 { + std::thread::sleep(Duration::from_secs(sleep_time)); + } + match crate::bootstrap::bootstrap_accounts(rpc_url.clone()) { + Ok((stakes, votes, history)) => { + BootstrapEvent::BootstrapAccountsFetched(stakes, votes, history, rpc_url) + } + Err(err) => { + log::warn!( + "Bootstrap account error during fetching accounts err:{err}. Exit" + ); + BootstrapEvent::Exit + } + } + }); + BootsrapProcessResult::TaskHandle(jh) + } + BootstrapEvent::BootstrapAccountsFetched(stakes, votes, history, rpc_url) => { + log::info!("BootstrapEvent::BootstrapAccountsFetched RECV"); + match ( + StakeStore::take_stakestore(stakestore), + VoteStore::take_votestore(votestore), + ) { + (Ok((stake_map, _)), Ok(vote_map)) => { + BootsrapProcessResult::Event(BootstrapEvent::StoreExtracted( + stake_map, vote_map, stakes, votes, history, rpc_url, + )) + } + _ => { + let jh = tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(1)).await; + BootstrapEvent::BootstrapAccountsFetched(stakes, votes, history, rpc_url) + }); + BootsrapProcessResult::TaskHandle(jh) + } + } + } + BootstrapEvent::StoreExtracted( + mut stake_map, + mut vote_map, + stakes, + votes, + history, + rpc_url, + ) => { + log::info!("BootstrapEvent::StoreExtracted RECV"); + + let stake_history = crate::account::read_historystake_from_account(history); + if stake_history.is_none() { + return BootsrapProcessResult::Error( + "Bootstrap error, can't read stake history from account data.".to_string(), + ); + } + + //merge new PA with stake map and vote map in a specific task + let jh = tokio::task::spawn_blocking({ + move || { + //update pa_list to set slot update to start epoq one. + crate::stake::merge_program_account_in_strake_map( + &mut stake_map, + stakes, + 0, //with RPC no way to know the slot of the account update. Set to 0. + ); + crate::vote::merge_program_account_in_vote_map( + &mut vote_map, + votes, + 0, //with RPC no way to know the slot of the account update. Set to 0. + ); + + BootstrapEvent::AccountsMerged(stake_map, stake_history, vote_map, rpc_url) + } + }); + BootsrapProcessResult::TaskHandle(jh) + } + BootstrapEvent::AccountsMerged(stake_map, stake_history, vote_map, rpc_url) => { + log::info!("BootstrapEvent::AccountsMerged RECV"); + match ( + StakeStore::merge_stakestore(stakestore, stake_map, stake_history), + VoteStore::merge_votestore(votestore, vote_map), + ) { + (Ok(()), Ok(())) => BootsrapProcessResult::End, + _ => { + //TODO remove this error using type state + log::warn!("BootstrapEvent::AccountsMerged merge stake or vote fail, non extracted stake/vote map err, restart bootstrap"); + BootsrapProcessResult::Event(BootstrapEvent::InitBootstrap { + sleep_time: 10, + rpc_url, + }) + } + } + } + BootstrapEvent::Exit => panic!("Bootstrap account can't be done exit"), + } +} + +fn bootstrap_accounts( + rpc_url: String, +) -> Result<(Vec<(Pubkey, Account)>, Vec<(Pubkey, Account)>, Account), ClientError> { + get_stake_account(rpc_url) + .and_then(|(stakes, rpc_url)| { + get_vote_account(rpc_url).map(|(votes, rpc_url)| (stakes, votes, rpc_url)) + }) + .and_then(|(stakes, votes, rpc_url)| { + get_stakehistory_account(rpc_url).map(|history| (stakes, votes, history)) + }) +} + +fn get_stake_account(rpc_url: String) -> Result<(Vec<(Pubkey, Account)>, String), ClientError> { + log::info!("TaskToExec RpcGetStakeAccount start"); + let rpc_client = RpcClient::new_with_timeout_and_commitment( + rpc_url.clone(), + Duration::from_secs(600), + CommitmentConfig::finalized(), + ); + let res_stake = rpc_client.get_program_accounts(&solana_sdk::stake::program::id()); + log::info!("TaskToExec RpcGetStakeAccount END"); + res_stake.map(|stake| (stake, rpc_url)) } -pub fn bootstrap_leader_schedule( - current_file_patch: &str, - next_file_patch: &str, +fn get_vote_account(rpc_url: String) -> Result<(Vec<(Pubkey, Account)>, String), ClientError> { + log::info!("TaskToExec RpcGetVoteAccount start"); + let rpc_client = RpcClient::new_with_timeout_and_commitment( + rpc_url.clone(), + Duration::from_secs(600), + CommitmentConfig::finalized(), + ); + let res_vote = rpc_client.get_program_accounts(&solana_sdk::vote::program::id()); + log::info!("TaskToExec RpcGetVoteAccount END"); + res_vote.map(|votes| (votes, rpc_url)) +} + +pub fn get_stakehistory_account(rpc_url: String) -> Result { + log::info!("TaskToExec RpcGetStakeHistory start"); + let rpc_client = RpcClient::new_with_timeout_and_commitment( + rpc_url, + Duration::from_secs(600), + CommitmentConfig::finalized(), + ); + let res_stake = rpc_client.get_account(&solana_sdk::sysvar::stake_history::id()); + log::info!("TaskToExec RpcGetStakeHistory END",); + res_stake +} + +pub fn bootstrap_current_leader_schedule( slots_in_epoch: u64, ) -> anyhow::Result { - todo!(); -} + let (current_epoch, current_stakes) = + crate::utils::read_schedule_vote_stakes(CURRENT_EPOCH_VOTE_STAKES_FILE)?; + let (next_epoch, next_stakes) = + crate::utils::read_schedule_vote_stakes(NEXT_EPOCH_VOTE_STAKES_FILE)?; + + //calcualte leader schedule for all vote stakes. + let current_schedule = crate::leader_schedule::calculate_leader_schedule( + ¤t_stakes, + current_epoch, + slots_in_epoch, + ); + let next_schedule = + crate::leader_schedule::calculate_leader_schedule(&next_stakes, next_epoch, slots_in_epoch); -pub struct BootstrapData { - pub done: bool, - pub sleep_time: u64, - pub rpc_client: Arc, + Ok(CalculatedSchedule { + current: Some(LeaderScheduleData { + schedule: current_schedule, + //TODO use epoch stake for get_vote_accounts + epoch: current_epoch, + }), + next: Some(LeaderScheduleData { + schedule: next_schedule, + //TODO use epoch stake for get_vote_accounts + // vote_stakes: next_stakes.stake_vote_map, + epoch: next_epoch, + }), + }) } diff --git a/stake_vote/src/leader_schedule.rs b/stake_vote/src/leader_schedule.rs index 6058e6c9..21355cf4 100644 --- a/stake_vote/src/leader_schedule.rs +++ b/stake_vote/src/leader_schedule.rs @@ -4,23 +4,13 @@ use crate::vote::{VoteMap, VoteStore}; use futures::stream::FuturesUnordered; use itertools::Itertools; use serde::{Deserialize, Serialize}; -use solana_client::rpc_client::RpcClient; use solana_ledger::leader_schedule::LeaderSchedule; -use solana_program::sysvar::epoch_schedule::EpochSchedule; use solana_sdk::clock::NUM_CONSECUTIVE_LEADER_SLOTS; -use solana_sdk::commitment_config::CommitmentConfig; -use solana_sdk::feature_set::FeatureSet; use solana_sdk::pubkey::Pubkey; use solana_sdk::stake::state::StakeActivationStatus; use solana_sdk::stake_history::StakeHistory; -use std::collections::BTreeMap; use std::collections::HashMap; -use std::fs::File; -use std::io::Write; -use std::str::FromStr; use std::sync::Arc; -use std::time::Duration; -use std::time::{SystemTime, UNIX_EPOCH}; use tokio::task::JoinHandle; #[derive(Debug)] @@ -125,6 +115,32 @@ fn process_leadershedule_event( next_epoch, slots_in_epoch, ); + + if std::path::Path::new(crate::bootstrap::NEXT_EPOCH_VOTE_STAKES_FILE) + .exists() + { + if let Err(err) = std::fs::rename( + crate::bootstrap::NEXT_EPOCH_VOTE_STAKES_FILE, + crate::bootstrap::CURRENT_EPOCH_VOTE_STAKES_FILE, + ) { + log::error!( + "Fail to rename current leader schedule on disk because :{err}" + ); + } + } + + //save new vote stake in a file for bootstrap. + if let Err(err) = crate::utils::save_schedule_vote_stakes( + crate::bootstrap::NEXT_EPOCH_VOTE_STAKES_FILE, + &epoch_vote_stakes, + next_epoch, + ) { + log::error!( + "Error during saving the new leader schedule of epoch:{} in a file error:{err}", + next_epoch + ); + } + log::info!("End calculate leader schedule"); LeaderScheduleEvent::MergeStoreAndSaveSchedule( stake_map, @@ -187,6 +203,7 @@ fn calculate_epoch_stakes( mut stake_history: Option<&mut StakeHistory>, new_rate_activation_epoch: Option, ) -> HashMap)> { + //code taken from Solana code: runtime::stakes::activate_epoch function //update stake history with current end epoch stake values. let stake_history_entry = stake_map @@ -240,7 +257,7 @@ fn calculate_epoch_stakes( //Copied from leader_schedule_utils.rs // Mostly cribbed from leader_schedule_utils -fn calculate_leader_schedule( +pub fn calculate_leader_schedule( stake_vote_map: &HashMap)>, epoch: u64, slots_in_epoch: u64, diff --git a/stake_vote/src/lib.rs b/stake_vote/src/lib.rs index dbd1821e..452447b5 100644 --- a/stake_vote/src/lib.rs +++ b/stake_vote/src/lib.rs @@ -1,4 +1,5 @@ use crate::account::AccountPretty; +use crate::bootstrap::BootstrapEvent; use crate::leader_schedule::LeaderScheduleGeneratedData; use futures::Stream; use futures_util::stream::FuturesUnordered; @@ -36,7 +37,9 @@ pub async fn start_stakes_and_votes_loop( rpc_client: Arc, grpc_url: String, ) -> anyhow::Result> { + log::info!("Start Stake and Vote loop."); let mut account_gyzer_stream = subscribe_geyzer(grpc_url).await?; + log::info!("Stake and Vote geyzer subscription done."); let jh = tokio::spawn(async move { //Stake account management struct let mut stakestore = stake::StakeStore::new(STAKESTORE_INITIAL_CAPACITY); @@ -45,19 +48,41 @@ pub async fn start_stakes_and_votes_loop( let mut votestore = vote::VoteStore::new(VOTESTORE_INITIAL_CAPACITY); //Init bootstrap process - let (mut current_schedule_epoch, bootstrap_data) = - crate::bootstrap::bootstrap_process_data(&data_cache, rpc_client.clone()).await; + let mut current_schedule_epoch = + crate::bootstrap::bootstrap_scheduleepoch_data(&data_cache).await; + + match crate::bootstrap::bootstrap_current_leader_schedule( + current_schedule_epoch.slots_in_epoch, + ) { + Ok(current_schedule_data) => { + data_cache.leader_schedule = Arc::new(current_schedule_data) + } + Err(err) => { + log::warn!("Error during current leader schedule bootstrap from files:{err}") + } + } //future execution collection. let mut spawned_leader_schedule_task = FuturesUnordered::new(); + let mut spawned_bootstrap_task = FuturesUnordered::new(); + let jh = tokio::spawn(async move { + BootstrapEvent::InitBootstrap { + sleep_time: 1, + rpc_url: rpc_client.url(), + } + }); + spawned_bootstrap_task.push(jh); + + let mut bootstrap_done = false; loop { tokio::select! { //manage confirm new slot notification to detect epoch change. Ok(_) = slot_notification.recv() => { + //log::info!("Stake and Vote receive a slot."); let new_slot = crate::utils::get_current_confirmed_slot(&data_cache).await; let schedule_event = current_schedule_epoch.process_new_confirmed_slot(new_slot, &data_cache).await; - if bootstrap_data.done { + if bootstrap_done { if let Some(init_event) = schedule_event { crate::leader_schedule::run_leader_schedule_events( init_event, @@ -79,11 +104,16 @@ pub async fn start_stakes_and_votes_loop( Ok(msg) => { match msg.update_oneof { Some(UpdateOneof::Account(account)) => { + // log::info!("Stake and Vote geyzer receive an account:{}.", + // account.account.clone().map(|a| + // solana_sdk::pubkey::Pubkey::try_from(a.pubkey).map(|k| k.to_string()) + // .unwrap_or("bad pubkey".to_string()).to_string()) + // .unwrap_or("no content".to_string()) + // ); //store new account stake. let current_slot = crate::utils::get_current_confirmed_slot(&data_cache).await; if let Some(account) = AccountPretty::new_from_geyzer(account, current_slot) { - //log::trace!("Geyser receive new account"); match account.owner { solana_sdk::stake::program::ID => { log::info!("Geyser notif stake account:{}", account); @@ -96,7 +126,7 @@ pub async fn start_stakes_and_votes_loop( } } solana_sdk::vote::program::ID => { - // Generatea lot of logs. log::info!("Geyser notif VOTE account:{}", account); + //log::info!("Geyser notif VOTE account:{}", account); let account_pubkey = account.pubkey; //process vote accout notification if let Err(err) = votestore.add_vote(account, current_schedule_epoch.last_slot_in_epoch) { @@ -109,6 +139,9 @@ pub async fn start_stakes_and_votes_loop( } } Some(UpdateOneof::Ping(_)) => log::trace!("UpdateOneof::Ping"), + Some(UpdateOneof::Slot(slot)) => { + log::trace!("Receive slot slot: {slot:?}"); + } bad_msg => { log::info!("Geyser stream unexpected message received:{:?}", bad_msg); } @@ -128,6 +161,14 @@ pub async fn start_stakes_and_votes_loop( } } } + //manage bootstrap event + Some(Ok(event)) = spawned_bootstrap_task.next() => { + match crate::bootstrap::run_bootstrap_events(event, &mut spawned_bootstrap_task, &mut stakestore, &mut votestore) { + Ok(Some(boot_res))=> bootstrap_done = boot_res, + Ok(None) => (), + Err(err) => log::error!("Stake / Vote Account bootstrap fail because '{err}'"), + } + } //Manage leader schedule generation process Some(Ok(event)) = spawned_leader_schedule_task.next() => { let new_leader_schedule = crate::leader_schedule::run_leader_schedule_events( @@ -190,7 +231,8 @@ async fn subscribe_geyzer( let confirmed_stream = client .subscribe_once( - slots.clone(), + Default::default(), //slots + //slots.clone(), accounts.clone(), //accounts Default::default(), //tx Default::default(), //entry diff --git a/stake_vote/src/stake.rs b/stake_vote/src/stake.rs index 428157b2..def92225 100644 --- a/stake_vote/src/stake.rs +++ b/stake_vote/src/stake.rs @@ -65,9 +65,9 @@ impl StakeStore { } } - pub fn get_stake_history(&self) -> Option { - self.stakes.content.1.clone() - } + // pub fn get_stake_history(&self) -> Option { + // self.stakes.content.1.clone() + // } pub fn notify_stake_change( &mut self, diff --git a/stake_vote/src/utils.rs b/stake_vote/src/utils.rs index faf36858..297a8b5b 100644 --- a/stake_vote/src/utils.rs +++ b/stake_vote/src/utils.rs @@ -1,4 +1,6 @@ +use crate::vote::StoredVote; use anyhow::bail; +use serde::{Deserialize, Serialize}; use solana_lite_rpc_core::stores::block_information_store::BlockInformation; use solana_lite_rpc_core::stores::data_cache::DataCache; use solana_lite_rpc_core::structures::epoch::Epoch as LiteRpcEpoch; @@ -6,6 +8,10 @@ use solana_sdk::commitment_config::CommitmentConfig; use solana_sdk::pubkey::Pubkey; use std::collections::HashMap; use std::default::Default; +use std::fs::File; +use std::io::Write; +use std::str::FromStr; +use std::sync::Arc; pub async fn get_current_confirmed_slot(data_cache: &DataCache) -> u64 { let commitment = CommitmentConfig::confirmed(); @@ -21,6 +27,51 @@ pub async fn get_current_epoch(data_cache: &DataCache) -> LiteRpcEpoch { data_cache.get_current_epoch(commitment).await } +//Read save epoch vote stake to bootstrap current leader shedule and get_vote_account. +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +struct StringSavedStake { + epoch: u64, + stake_vote_map: HashMap)>, +} + +pub fn read_schedule_vote_stakes( + file_path: &str, +) -> anyhow::Result<(u64, HashMap)>)> { + let content = std::fs::read_to_string(file_path)?; + let stakes_str: StringSavedStake = serde_json::from_str(&content)?; + //convert to EpochStake because json hashmap parser can only have String key. + let ret_stakes = stakes_str + .stake_vote_map + .into_iter() + .map(|(pk, st)| (Pubkey::from_str(&pk).unwrap(), (st.0, st.1))) + .collect(); + Ok((stakes_str.epoch, ret_stakes)) +} + +pub fn save_schedule_vote_stakes( + base_file_path: &str, + stake_vote_map: &HashMap)>, + epoch: u64, +) -> anyhow::Result<()> { + //save new schedule for restart. + //need to convert hahsmap key to String because json aloow only string + //key for dictionnary. + //it's better to use json because the file can use to very some stake by hand. + //in the end it will be removed with the bootstrap process. + let save_stakes = StringSavedStake { + epoch, + stake_vote_map: stake_vote_map + .iter() + .map(|(pk, st)| (pk.to_string(), (st.0, Arc::clone(&st.1)))) + .collect(), + }; + let serialized_stakes = serde_json::to_string(&save_stakes).unwrap(); + let mut file = File::create(base_file_path).unwrap(); + file.write_all(serialized_stakes.as_bytes()).unwrap(); + file.flush().unwrap(); + Ok(()) +} + //Takable struct code pub trait TakableContent: Default { fn add_value(&mut self, val: T); diff --git a/stake_vote/src/vote.rs b/stake_vote/src/vote.rs index a3444fd9..661e34ff 100644 --- a/stake_vote/src/vote.rs +++ b/stake_vote/src/vote.rs @@ -74,10 +74,6 @@ impl VoteStore { crate::utils::merge(&mut votestore.votes, vote_map) } - fn insert_vote(&mut self, vote_account: Pubkey, vote_data: StoredVote) { - Self::vote_map_insert_vote(&mut self.votes.content, vote_account, vote_data); - } - fn remove_from_store(&mut self, account_pk: &Pubkey, update_slot: Slot) { if self .votes diff --git a/tests/client.test.ts b/tests/client.test.ts index 9d994cb4..cfbae8e8 100644 --- a/tests/client.test.ts +++ b/tests/client.test.ts @@ -78,3 +78,13 @@ test('get epoch info', async () => { }); + +test('get leader schedule', async () => { + { + const leaderSchedule = await connection.getLeaderSchedule(); + expect(Object.keys(leaderSchedule).length > 0); + } +}); + + + From 85b3eb3faf0e885dcb67a6c48b1e0d2b432edd18 Mon Sep 17 00:00:00 2001 From: musitdev Date: Wed, 18 Oct 2023 09:58:16 +0200 Subject: [PATCH 09/56] correct clippy warning --- core/src/structures/leaderschedule.rs | 2 +- stake_vote/src/account.rs | 6 ++---- stake_vote/src/bootstrap.rs | 8 ++++---- stake_vote/src/epoch.rs | 2 +- stake_vote/src/leader_schedule.rs | 1 + stake_vote/src/utils.rs | 1 + 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/core/src/structures/leaderschedule.rs b/core/src/structures/leaderschedule.rs index 05086b42..e0233614 100644 --- a/core/src/structures/leaderschedule.rs +++ b/core/src/structures/leaderschedule.rs @@ -16,7 +16,7 @@ impl CalculatedSchedule { commitment: Option, data_cache: &DataCache, ) -> Option>> { - let commitment = commitment.unwrap_or_else(|| CommitmentConfig::confirmed()); + let commitment = commitment.unwrap_or_else(CommitmentConfig::confirmed); let slot = match slot { Some(slot) => slot, None => { diff --git a/stake_vote/src/account.rs b/stake_vote/src/account.rs index 9e96c708..c099c0df 100644 --- a/stake_vote/src/account.rs +++ b/stake_vote/src/account.rs @@ -57,7 +57,7 @@ impl AccountPretty { } pub fn read_stake(&self) -> anyhow::Result> { - read_stake_from_account_data(&mut self.data.as_slice()) + read_stake_from_account_data(self.data.as_slice()) } pub fn read_vote(&self) -> anyhow::Result { @@ -74,9 +74,7 @@ impl std::fmt::Display for AccountPretty { write!( f, "{} at slot:{} lpt:{}", - self.pubkey.to_string(), - self.slot, - self.lamports + self.pubkey, self.slot, self.lamports ) } } diff --git a/stake_vote/src/bootstrap.rs b/stake_vote/src/bootstrap.rs index fadab96f..67040999 100644 --- a/stake_vote/src/bootstrap.rs +++ b/stake_vote/src/bootstrap.rs @@ -28,7 +28,7 @@ pub async fn bootstrap_scheduleepoch_data(data_cache: &DataCache) -> ScheduleEpo .new_warmup_cooldown_rate_epoch(data_cache.epoch_data.get_epoch_schedule()); let bootstrap_epoch = crate::utils::get_current_epoch(data_cache).await; - let current_schedule_epoch = ScheduleEpochData { + ScheduleEpochData { current_epoch: bootstrap_epoch.epoch, slots_in_epoch: bootstrap_epoch.slots_in_epoch, last_slot_in_epoch: data_cache @@ -36,9 +36,7 @@ pub async fn bootstrap_scheduleepoch_data(data_cache: &DataCache) -> ScheduleEpo .get_last_slot_in_epoch(bootstrap_epoch.epoch), current_confirmed_slot: bootstrap_epoch.absolute_slot, new_rate_activation_epoch, - }; - - current_schedule_epoch + } } /* @@ -111,6 +109,7 @@ pub enum BootstrapEvent { Exit, } +#[allow(clippy::large_enum_variant)] //214 byte large and only use during bootstrap. enum BootsrapProcessResult { TaskHandle(JoinHandle), Event(BootstrapEvent), @@ -225,6 +224,7 @@ fn process_bootstrap_event( } } +#[allow(clippy::type_complexity)] fn bootstrap_accounts( rpc_url: String, ) -> Result<(Vec<(Pubkey, Account)>, Vec<(Pubkey, Account)>, Account), ClientError> { diff --git a/stake_vote/src/epoch.rs b/stake_vote/src/epoch.rs index 4d6d8e5c..ab3622d0 100644 --- a/stake_vote/src/epoch.rs +++ b/stake_vote/src/epoch.rs @@ -47,7 +47,7 @@ impl ScheduleEpochData { Some(crate::leader_schedule::LeaderScheduleEvent::Init( self.current_epoch, self.slots_in_epoch, - self.new_rate_activation_epoch.clone(), + self.new_rate_activation_epoch, )) } else { None diff --git a/stake_vote/src/leader_schedule.rs b/stake_vote/src/leader_schedule.rs index 21355cf4..c79174a1 100644 --- a/stake_vote/src/leader_schedule.rs +++ b/stake_vote/src/leader_schedule.rs @@ -46,6 +46,7 @@ InitLeaderscedule MergeStore(stakes, votes, schedule) InitLeaderscedule */ +#[allow(clippy::large_enum_variant)] //256 byte large and only use during schedule calculus. pub enum LeaderScheduleEvent { Init(u64, u64, Option), MergeStoreAndSaveSchedule( diff --git a/stake_vote/src/utils.rs b/stake_vote/src/utils.rs index 297a8b5b..f48a3e66 100644 --- a/stake_vote/src/utils.rs +++ b/stake_vote/src/utils.rs @@ -34,6 +34,7 @@ struct StringSavedStake { stake_vote_map: HashMap)>, } +#[allow(clippy::type_complexity)] pub fn read_schedule_vote_stakes( file_path: &str, ) -> anyhow::Result<(u64, HashMap)>)> { From 68c341a2dc5317438e408657148fc5d0686a5bb4 Mon Sep 17 00:00:00 2001 From: musitdev Date: Wed, 18 Oct 2023 13:17:04 +0200 Subject: [PATCH 10/56] update stake history sync algo --- stake_vote/src/leader_schedule.rs | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/stake_vote/src/leader_schedule.rs b/stake_vote/src/leader_schedule.rs index c79174a1..4d2f4922 100644 --- a/stake_vote/src/leader_schedule.rs +++ b/stake_vote/src/leader_schedule.rs @@ -91,7 +91,7 @@ fn process_leadershedule_event( votestore: &mut VoteStore, ) -> LeaderScheduleResult { match event { - LeaderScheduleEvent::Init(current_epoch, slots_in_epoch, new_rate_activation_epoch) => { + LeaderScheduleEvent::Init(new_epoch, slots_in_epoch, new_rate_activation_epoch) => { match ( StakeStore::take_stakestore(stakestore), VoteStore::take_votestore(votestore), @@ -101,16 +101,15 @@ fn process_leadershedule_event( //do the calculus in a blocking task. let jh = tokio::task::spawn_blocking({ move || { - let next_epoch = current_epoch + 1; let epoch_vote_stakes = calculate_epoch_stakes( &stake_map, &vote_map, - current_epoch, - next_epoch, + new_epoch, stake_history.as_mut(), new_rate_activation_epoch, ); + let next_epoch = new_epoch + 1; let leader_schedule = calculate_leader_schedule( &epoch_vote_stakes, next_epoch, @@ -151,7 +150,7 @@ fn process_leadershedule_event( vote_stakes: epoch_vote_stakes, epoch: next_epoch, }, - (current_epoch, slots_in_epoch, new_rate_activation_epoch), + (new_epoch, slots_in_epoch, new_rate_activation_epoch), stake_history, ) } @@ -161,7 +160,7 @@ fn process_leadershedule_event( _ => { log::error!("Create leadershedule init event error during extract store"); LeaderScheduleResult::Event(LeaderScheduleEvent::Init( - current_epoch, + new_epoch, slots_in_epoch, new_rate_activation_epoch, )) @@ -172,7 +171,7 @@ fn process_leadershedule_event( stake_map, vote_map, schedule_data, - (current_epoch, slots_in_epoch, epoch_schedule), + (new_epoch, slots_in_epoch, epoch_schedule), stake_history, ) => { log::info!("LeaderScheduleEvent::MergeStoreAndSaveSchedule RECV"); @@ -186,7 +185,7 @@ fn process_leadershedule_event( //TODO remove this error using type state log::warn!("LeaderScheduleEvent::MergeStoreAndSaveSchedule merge stake or vote fail, -restart Schedule"); LeaderScheduleResult::Event(LeaderScheduleEvent::Init( - current_epoch, + new_epoch, slots_in_epoch, epoch_schedule, )) @@ -199,30 +198,32 @@ fn process_leadershedule_event( fn calculate_epoch_stakes( stake_map: &StakeMap, vote_map: &VoteMap, - current_epoch: u64, - next_epoch: u64, + new_epoch: u64, mut stake_history: Option<&mut StakeHistory>, new_rate_activation_epoch: Option, ) -> HashMap)> { //code taken from Solana code: runtime::stakes::activate_epoch function //update stake history with current end epoch stake values. + //stake history is added for the ended epoch using all stakes at the end of the epoch. + let ended_epoch = new_epoch - 1; let stake_history_entry = stake_map .values() .fold(StakeActivationStatus::default(), |acc, stake_account| { let delegation = stake_account.stake; acc + delegation.stake_activating_and_deactivating( - current_epoch, + ended_epoch, stake_history.as_deref(), new_rate_activation_epoch, ) }); match stake_history { - Some(ref mut stake_history) => stake_history.add(current_epoch, stake_history_entry), + Some(ref mut stake_history) => stake_history.add(ended_epoch, stake_history_entry), None => log::warn!("Vote stake calculus without Stake History"), }; - //calculate schedule stakes. + //calculate schedule stakes at beginning of new epoch. + //Next epoch schedule use the stake at the beginning of last epoch. let delegated_stakes: HashMap = stake_map .values() @@ -230,7 +231,7 @@ fn calculate_epoch_stakes( let delegation = stake_account.stake; let entry = delegated_stakes.entry(delegation.voter_pubkey).or_default(); *entry += delegation.stake( - next_epoch, + new_epoch, stake_history.as_deref(), new_rate_activation_epoch, ); From 7fba1071654ea229191707f8ad336b123205fcb1 Mon Sep 17 00:00:00 2001 From: musitdev Date: Wed, 18 Oct 2023 13:28:30 +0200 Subject: [PATCH 11/56] update nodeid list generation to have several vote account per node --- stake_vote/src/leader_schedule.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/stake_vote/src/leader_schedule.rs b/stake_vote/src/leader_schedule.rs index 4d2f4922..92a4af8b 100644 --- a/stake_vote/src/leader_schedule.rs +++ b/stake_vote/src/leader_schedule.rs @@ -264,11 +264,16 @@ pub fn calculate_leader_schedule( epoch: u64, slots_in_epoch: u64, ) -> HashMap> { - let mut stakes: Vec<(Pubkey, u64)> = stake_vote_map + let stakes_map: HashMap = stake_vote_map .iter() .filter_map(|(_, (stake, vote_account))| { - (*stake > 0).then_some((vote_account.vote_data.node_pubkey, *stake)) + (*stake != 0u64).then_some((vote_account.vote_data.node_pubkey, *stake)) }) + .into_grouping_map() + .aggregate(|acc, _node_pubkey, stake| Some(acc.unwrap_or_default() + stake)); + let mut stakes: Vec<(Pubkey, u64)> = stakes_map + .into_iter() + .map(|(key, stake)| (key, stake)) .collect(); let mut seed = [0u8; 32]; From b3317667a6552ce386b31b8c8b0c7d395d5f7004 Mon Sep 17 00:00:00 2001 From: musitdev Date: Wed, 25 Oct 2023 19:26:59 +0200 Subject: [PATCH 12/56] first impl for RPC get_vote_accounts --- core/src/structures/leaderschedule.rs | 35 +++- docs/rpcv2.md | 1 + lite-rpc/src/bridge.rs | 18 ++ lite-rpc/src/rpc.rs | 16 +- stake_vote/src/bootstrap.rs | 111 +++++++---- stake_vote/src/leader_schedule.rs | 69 ++++--- stake_vote/src/lib.rs | 127 +++++++++++-- stake_vote/src/stake.rs | 119 ++++++------ stake_vote/src/utils.rs | 263 ++++++++++++++++++-------- stake_vote/src/vote.rs | 170 +++++++++++++++-- yarn.lock | 107 ++++++----- 11 files changed, 753 insertions(+), 283 deletions(-) diff --git a/core/src/structures/leaderschedule.rs b/core/src/structures/leaderschedule.rs index e0233614..20bff03a 100644 --- a/core/src/structures/leaderschedule.rs +++ b/core/src/structures/leaderschedule.rs @@ -1,7 +1,37 @@ use crate::stores::block_information_store::BlockInformation; use crate::stores::data_cache::DataCache; +use solana_rpc_client_api::config::RpcGetVoteAccountsConfig; use solana_sdk::commitment_config::CommitmentConfig; +use solana_sdk::pubkey::ParsePubkeyError; +use solana_sdk::pubkey::Pubkey; use std::collections::HashMap; +use std::str::FromStr; + +#[derive(Clone)] +pub struct GetVoteAccountsConfig { + pub vote_pubkey: Option, + pub commitment: Option, + pub keep_unstaked_delinquents: Option, + pub delinquent_slot_distance: Option, +} + +impl TryFrom for GetVoteAccountsConfig { + type Error = ParsePubkeyError; + + fn try_from(config: RpcGetVoteAccountsConfig) -> Result { + let vote_pubkey = config + .vote_pubkey + .as_ref() + .map(|pk| Pubkey::from_str(pk)) + .transpose()?; + Ok(GetVoteAccountsConfig { + vote_pubkey, + commitment: config.commitment, + keep_unstaked_delinquents: config.keep_unstaked_delinquents, + delinquent_slot_distance: config.delinquent_slot_distance, + }) + } +} #[derive(Clone, Default)] pub struct CalculatedSchedule { @@ -31,7 +61,7 @@ impl CalculatedSchedule { let get_schedule = |schedule_data: Option<&LeaderScheduleData>| { schedule_data.and_then(|current| { - (current.epoch == epoch.epoch).then_some(current.schedule.clone()) + (current.epoch == epoch.epoch).then_some(current.schedule_by_node.clone()) }) }; get_schedule(self.current.as_ref()).or_else(|| get_schedule(self.next.as_ref())) @@ -40,6 +70,7 @@ impl CalculatedSchedule { #[derive(Clone)] pub struct LeaderScheduleData { - pub schedule: HashMap>, + pub schedule_by_node: HashMap>, + pub schedule_by_slot: Vec, pub epoch: u64, } diff --git a/docs/rpcv2.md b/docs/rpcv2.md index e51f5302..176e2cf0 100644 --- a/docs/rpcv2.md +++ b/docs/rpcv2.md @@ -55,6 +55,7 @@ Method calls: ##### Cluster info Domain - [getclusternodes](https://docs.solana.com/api/http#getclusternodes) not in geyser plugin can be get from gossip. Try to update gyser first. + ##### Validator Domain - [getslot](https://docs.solana.com/api/http#getslot) Need top add 2 new commitment level for first shred seen and half confirm (1/3 of the stake has voted on the block) - [getBlockHeight](https://docs.solana.com/api/http#getblockheight) diff --git a/lite-rpc/src/bridge.rs b/lite-rpc/src/bridge.rs index de36bd6c..01ff2a9e 100644 --- a/lite-rpc/src/bridge.rs +++ b/lite-rpc/src/bridge.rs @@ -3,6 +3,8 @@ use crate::{ jsonrpsee_subscrption_handler_sink::JsonRpseeSubscriptionHandlerSink, rpc::LiteRpcServer, }; +use solana_rpc_client_api::config::RpcGetVoteAccountsConfig; +use solana_rpc_client_api::response::RpcVoteAccountStatus; use solana_sdk::epoch_info::EpochInfo; use std::collections::HashMap; @@ -485,4 +487,20 @@ impl LiteRpcServer for LiteBridge { .await; Ok(schedule) } + async fn get_slot_leaders( + &self, + start_slot: u64, + limit: u64, + ) -> crate::rpc::Result> { + todo!() + } + + async fn get_vote_accounts( + &self, + config: Option, + ) -> crate::rpc::Result { + let config: GetVoteAccountsConfig = + GetVoteAccountsConfig::try_from(config.unwrap_or_default())?; + todo!(); + } } diff --git a/lite-rpc/src/rpc.rs b/lite-rpc/src/rpc.rs index 81843f0b..2861586c 100644 --- a/lite-rpc/src/rpc.rs +++ b/lite-rpc/src/rpc.rs @@ -1,6 +1,7 @@ use crate::configs::{IsBlockHashValidConfig, SendTransactionConfig}; use jsonrpsee::core::SubscriptionResult; use jsonrpsee::proc_macros::rpc; +use solana_rpc_client_api::config::RpcGetVoteAccountsConfig; use solana_rpc_client_api::config::{ RpcBlockConfig, RpcBlockSubscribeConfig, RpcBlockSubscribeFilter, RpcBlocksConfigWrapper, RpcContextConfig, RpcEncodingConfigWrapper, RpcEpochConfig, RpcGetVoteAccountsConfig, @@ -13,9 +14,9 @@ use solana_rpc_client_api::response::{ RpcContactInfo, RpcLeaderSchedule, RpcPerfSample, RpcPrioritizationFee, RpcVersionInfo, RpcVoteAccountStatus, }; - use solana_sdk::commitment_config::CommitmentConfig; use solana_sdk::epoch_info::EpochInfo; +use solana_sdk::pubkey::Pubkey; use solana_sdk::slot_history::Slot; use solana_transaction_status::{TransactionStatus, UiConfirmedBlock}; use std::collections::HashMap; @@ -238,4 +239,17 @@ pub trait LiteRpc { slot: Option, config: Option, ) -> crate::rpc::Result>>>; + + #[method(name = "getSlotLeaders")] + async fn get_slot_leaders( + &self, + start_slot: u64, + limit: u64, + ) -> crate::rpc::Result>; + + #[method(name = "getVoteAccounts")] + async fn get_vote_accounts( + &self, + config: Option, + ) -> crate::rpc::Result; } diff --git a/stake_vote/src/bootstrap.rs b/stake_vote/src/bootstrap.rs index 67040999..7971c200 100644 --- a/stake_vote/src/bootstrap.rs +++ b/stake_vote/src/bootstrap.rs @@ -1,9 +1,13 @@ use crate::epoch::ScheduleEpochData; +use crate::leader_schedule::LeaderScheduleGeneratedData; use crate::stake::StakeMap; use crate::stake::StakeStore; +use crate::utils::{Takable, TakeResult}; +use crate::vote::EpochVoteStakes; use crate::vote::VoteMap; use crate::vote::VoteStore; use anyhow::bail; +use futures::future::join_all; use futures_util::stream::FuturesUnordered; use solana_client::client_error::ClientError; use solana_client::rpc_client::RpcClient; @@ -71,17 +75,22 @@ pub fn run_bootstrap_events( bootstrap_tasks: &mut FuturesUnordered>, stakestore: &mut StakeStore, votestore: &mut VoteStore, -) -> anyhow::Result> { - let result = process_bootstrap_event(event, stakestore, votestore); + slots_in_epoch: u64, +) -> anyhow::Result>> { + let result = process_bootstrap_event(event, stakestore, votestore, slots_in_epoch); match result { BootsrapProcessResult::TaskHandle(jh) => { bootstrap_tasks.push(jh); Ok(None) } - BootsrapProcessResult::Event(event) => { - run_bootstrap_events(event, bootstrap_tasks, stakestore, votestore) - } - BootsrapProcessResult::End => Ok(Some(true)), + BootsrapProcessResult::Event(event) => run_bootstrap_events( + event, + bootstrap_tasks, + stakestore, + votestore, + slots_in_epoch, + ), + BootsrapProcessResult::End(leader_schedule_result) => Ok(Some(leader_schedule_result)), BootsrapProcessResult::Error(err) => bail!(err), } } @@ -105,7 +114,13 @@ pub enum BootstrapEvent { Account, String, ), - AccountsMerged(StakeMap, Option, VoteMap, String), + AccountsMerged( + StakeMap, + Option, + VoteMap, + String, + anyhow::Result, + ), Exit, } @@ -114,13 +129,14 @@ enum BootsrapProcessResult { TaskHandle(JoinHandle), Event(BootstrapEvent), Error(String), - End, + End(anyhow::Result), } fn process_bootstrap_event( event: BootstrapEvent, stakestore: &mut StakeStore, votestore: &mut VoteStore, + slots_in_epoch: u64, ) -> BootsrapProcessResult { match event { BootstrapEvent::InitBootstrap { @@ -148,24 +164,30 @@ fn process_bootstrap_event( } BootstrapEvent::BootstrapAccountsFetched(stakes, votes, history, rpc_url) => { log::info!("BootstrapEvent::BootstrapAccountsFetched RECV"); - match ( - StakeStore::take_stakestore(stakestore), - VoteStore::take_votestore(votestore), - ) { - (Ok((stake_map, _)), Ok(vote_map)) => { + match (&mut stakestore.stakes, &mut votestore.votes).take() { + TakeResult::Map(((stake_map, _), vote_map)) => { BootsrapProcessResult::Event(BootstrapEvent::StoreExtracted( stake_map, vote_map, stakes, votes, history, rpc_url, )) } - _ => { - let jh = tokio::spawn(async move { - tokio::time::sleep(Duration::from_secs(1)).await; - BootstrapEvent::BootstrapAccountsFetched(stakes, votes, history, rpc_url) + TakeResult::Taken(stake_notify) => { + let notif_jh = tokio::spawn({ + async move { + let notifs = stake_notify + .iter() + .map(|n| n.notified()) + .collect::>(); + join_all(notifs).await; + BootstrapEvent::BootstrapAccountsFetched( + stakes, votes, history, rpc_url, + ) + } }); - BootsrapProcessResult::TaskHandle(jh) + BootsrapProcessResult::TaskHandle(notif_jh) } } } + BootstrapEvent::StoreExtracted( mut stake_map, mut vote_map, @@ -198,18 +220,33 @@ fn process_bootstrap_event( 0, //with RPC no way to know the slot of the account update. Set to 0. ); - BootstrapEvent::AccountsMerged(stake_map, stake_history, vote_map, rpc_url) + let leader_schedule_result = bootstrap_current_leader_schedule(slots_in_epoch); + + BootstrapEvent::AccountsMerged( + stake_map, + stake_history, + vote_map, + rpc_url, + leader_schedule_result, + ) } }); BootsrapProcessResult::TaskHandle(jh) } - BootstrapEvent::AccountsMerged(stake_map, stake_history, vote_map, rpc_url) => { + BootstrapEvent::AccountsMerged( + stake_map, + stake_history, + vote_map, + rpc_url, + leader_schedule_result, + ) => { log::info!("BootstrapEvent::AccountsMerged RECV"); + match ( - StakeStore::merge_stakestore(stakestore, stake_map, stake_history), - VoteStore::merge_votestore(votestore, vote_map), + stakestore.stakes.merge((stake_map, stake_history)), + votestore.votes.merge(vote_map), ) { - (Ok(()), Ok(())) => BootsrapProcessResult::End, + (Ok(()), Ok(())) => BootsrapProcessResult::End(leader_schedule_result), _ => { //TODO remove this error using type state log::warn!("BootstrapEvent::AccountsMerged merge stake or vote fail, non extracted stake/vote map err, restart bootstrap"); @@ -273,33 +310,41 @@ pub fn get_stakehistory_account(rpc_url: String) -> Result res_stake } +// pub struct BootstrapScheduleResult { +// schedule: CalculatedSchedule, +// vote_stakes: Vec, +// } + pub fn bootstrap_current_leader_schedule( slots_in_epoch: u64, ) -> anyhow::Result { - let (current_epoch, current_stakes) = + let (current_epoch, current_epoch_stakes) = crate::utils::read_schedule_vote_stakes(CURRENT_EPOCH_VOTE_STAKES_FILE)?; - let (next_epoch, next_stakes) = + let (next_epoch, next_epoch_stakes) = crate::utils::read_schedule_vote_stakes(NEXT_EPOCH_VOTE_STAKES_FILE)?; //calcualte leader schedule for all vote stakes. let current_schedule = crate::leader_schedule::calculate_leader_schedule( - ¤t_stakes, + ¤t_epoch_stakes, current_epoch, slots_in_epoch, ); - let next_schedule = - crate::leader_schedule::calculate_leader_schedule(&next_stakes, next_epoch, slots_in_epoch); + + let next_schedule = crate::leader_schedule::calculate_leader_schedule( + &next_epoch_stakes, + next_epoch, + slots_in_epoch, + ); Ok(CalculatedSchedule { current: Some(LeaderScheduleData { - schedule: current_schedule, - //TODO use epoch stake for get_vote_accounts + schedule_by_node: LeaderScheduleGeneratedData::get_schedule_by_nodes(¤t_schedule), + schedule_by_slot: current_schedule.get_slot_leaders().to_vec(), epoch: current_epoch, }), next: Some(LeaderScheduleData { - schedule: next_schedule, - //TODO use epoch stake for get_vote_accounts - // vote_stakes: next_stakes.stake_vote_map, + schedule_by_node: LeaderScheduleGeneratedData::get_schedule_by_nodes(&next_schedule), + schedule_by_slot: next_schedule.get_slot_leaders().to_vec(), epoch: next_epoch, }), }) diff --git a/stake_vote/src/leader_schedule.rs b/stake_vote/src/leader_schedule.rs index 92a4af8b..4887b979 100644 --- a/stake_vote/src/leader_schedule.rs +++ b/stake_vote/src/leader_schedule.rs @@ -1,6 +1,8 @@ use crate::stake::{StakeMap, StakeStore}; +use crate::utils::{Takable, TakeResult}; use crate::vote::StoredVote; use crate::vote::{VoteMap, VoteStore}; +use futures::future::join_all; use futures::stream::FuturesUnordered; use itertools::Itertools; use serde::{Deserialize, Serialize}; @@ -15,11 +17,24 @@ use tokio::task::JoinHandle; #[derive(Debug)] pub struct LeaderScheduleGeneratedData { - pub schedule: HashMap>, - pub vote_stakes: HashMap)>, + pub schedule: LeaderSchedule, + pub epoch_vote_stakes: HashMap)>, pub epoch: u64, } +impl LeaderScheduleGeneratedData { + pub fn get_schedule_by_nodes(schedule: &LeaderSchedule) -> HashMap> { + schedule + .get_slot_leaders() + .iter() + .enumerate() + .map(|(i, pk)| (pk.to_string(), i)) + .into_group_map() + .into_iter() + .collect() + } +} + #[derive(Debug, Serialize, Deserialize)] pub struct EpochStake { epoch: u64, @@ -92,11 +107,8 @@ fn process_leadershedule_event( ) -> LeaderScheduleResult { match event { LeaderScheduleEvent::Init(new_epoch, slots_in_epoch, new_rate_activation_epoch) => { - match ( - StakeStore::take_stakestore(stakestore), - VoteStore::take_votestore(votestore), - ) { - (Ok((stake_map, mut stake_history)), Ok(vote_map)) => { + match (&mut stakestore.stakes, &mut votestore.votes).take() { + TakeResult::Map(((stake_map, mut stake_history), vote_map)) => { log::info!("LeaderScheduleEvent::CalculateScedule"); //do the calculus in a blocking task. let jh = tokio::task::spawn_blocking({ @@ -142,12 +154,13 @@ fn process_leadershedule_event( } log::info!("End calculate leader schedule"); + LeaderScheduleEvent::MergeStoreAndSaveSchedule( stake_map, vote_map, LeaderScheduleGeneratedData { schedule: leader_schedule, - vote_stakes: epoch_vote_stakes, + epoch_vote_stakes, epoch: next_epoch, }, (new_epoch, slots_in_epoch, new_rate_activation_epoch), @@ -157,13 +170,22 @@ fn process_leadershedule_event( }); LeaderScheduleResult::TaskHandle(jh) } - _ => { - log::error!("Create leadershedule init event error during extract store"); - LeaderScheduleResult::Event(LeaderScheduleEvent::Init( - new_epoch, - slots_in_epoch, - new_rate_activation_epoch, - )) + TakeResult::Taken(stake_notify) => { + let notif_jh = tokio::spawn({ + async move { + let notifs = stake_notify + .iter() + .map(|n| n.notified()) + .collect::>(); + join_all(notifs).await; + LeaderScheduleEvent::Init( + new_epoch, + slots_in_epoch, + new_rate_activation_epoch, + ) + } + }); + LeaderScheduleResult::TaskHandle(notif_jh) } } } @@ -176,8 +198,8 @@ fn process_leadershedule_event( ) => { log::info!("LeaderScheduleEvent::MergeStoreAndSaveSchedule RECV"); match ( - StakeStore::merge_stakestore(stakestore, stake_map, stake_history), - VoteStore::merge_votestore(votestore, vote_map), + stakestore.stakes.merge((stake_map, stake_history)), + votestore.votes.merge(vote_map), ) { (Ok(()), Ok(())) => LeaderScheduleResult::End(schedule_data), _ => { @@ -263,7 +285,7 @@ pub fn calculate_leader_schedule( stake_vote_map: &HashMap)>, epoch: u64, slots_in_epoch: u64, -) -> HashMap> { +) -> LeaderSchedule { let stakes_map: HashMap = stake_vote_map .iter() .filter_map(|(_, (stake, vote_account))| { @@ -281,16 +303,7 @@ pub fn calculate_leader_schedule( sort_stakes(&mut stakes); log::info!("calculate_leader_schedule stakes:{stakes:?} epoch:{epoch}"); let schedule = LeaderSchedule::new(&stakes, seed, slots_in_epoch, NUM_CONSECUTIVE_LEADER_SLOTS); - - let slot_schedule = schedule - .get_slot_leaders() - .iter() - .enumerate() - .map(|(i, pk)| (pk.to_string(), i)) - .into_group_map() - .into_iter() - .collect(); - slot_schedule + schedule } // Cribbed from leader_schedule_utils diff --git a/stake_vote/src/lib.rs b/stake_vote/src/lib.rs index 452447b5..296b2a55 100644 --- a/stake_vote/src/lib.rs +++ b/stake_vote/src/lib.rs @@ -1,15 +1,19 @@ use crate::account::AccountPretty; use crate::bootstrap::BootstrapEvent; use crate::leader_schedule::LeaderScheduleGeneratedData; +use crate::utils::{Takable, TakeResult}; use futures::Stream; use futures_util::stream::FuturesUnordered; use futures_util::StreamExt; use solana_lite_rpc_core::stores::data_cache::DataCache; +use solana_lite_rpc_core::structures::leaderschedule::GetVoteAccountsConfig; use solana_lite_rpc_core::structures::leaderschedule::LeaderScheduleData; use solana_lite_rpc_core::types::SlotStream; use solana_rpc_client::nonblocking::rpc_client::RpcClient; +use solana_rpc_client_api::response::RpcVoteAccountStatus; use std::collections::HashMap; use std::sync::Arc; +use tokio::sync::mpsc::Receiver; use yellowstone_grpc_client::GeyserGrpcClient; use yellowstone_grpc_proto::geyser::CommitmentLevel; use yellowstone_grpc_proto::prelude::SubscribeRequestFilterAccounts; @@ -34,6 +38,10 @@ type Slot = u64; pub async fn start_stakes_and_votes_loop( mut data_cache: DataCache, mut slot_notification: SlotStream, + mut vote_account_rpc_request: Receiver<( + GetVoteAccountsConfig, + tokio::sync::oneshot::Sender, + )>, rpc_client: Arc, grpc_url: String, ) -> anyhow::Result> { @@ -51,20 +59,11 @@ pub async fn start_stakes_and_votes_loop( let mut current_schedule_epoch = crate::bootstrap::bootstrap_scheduleepoch_data(&data_cache).await; - match crate::bootstrap::bootstrap_current_leader_schedule( - current_schedule_epoch.slots_in_epoch, - ) { - Ok(current_schedule_data) => { - data_cache.leader_schedule = Arc::new(current_schedule_data) - } - Err(err) => { - log::warn!("Error during current leader schedule bootstrap from files:{err}") - } - } - //future execution collection. let mut spawned_leader_schedule_task = FuturesUnordered::new(); let mut spawned_bootstrap_task = FuturesUnordered::new(); + let mut rpc_notify_task = FuturesUnordered::new(); + let mut rpc_exec_task = FuturesUnordered::new(); let jh = tokio::spawn(async move { BootstrapEvent::InitBootstrap { sleep_time: 1, @@ -74,6 +73,7 @@ pub async fn start_stakes_and_votes_loop( spawned_bootstrap_task.push(jh); let mut bootstrap_done = false; + let mut pending_rpc_request = vec![]; loop { tokio::select! { @@ -93,6 +93,84 @@ pub async fn start_stakes_and_votes_loop( } } } + Some((config, return_channel)) = vote_account_rpc_request.recv() => { + pending_rpc_request.push(return_channel); + let current_slot = crate::utils::get_current_confirmed_slot(&data_cache).await; + let vote_accounts = votestore.vote_stakes_for_epoch(0); //TODO define epoch storage. + match votestore.votes.take() { + TakeResult::Map(votes) => { + let jh = tokio::task::spawn_blocking({ + move || { + let rpc_vote_accounts = crate::vote::get_rpc_vote_accounts_info( + current_slot, + &votes, + &vote_accounts.as_ref().unwrap().vote_stakes, //TODO put in take. + config, + ); + (votes, vote_accounts, rpc_vote_accounts) + } + }); + rpc_exec_task.push(jh); + } + TakeResult::Taken(mut stake_notify) => { + let notif_jh = tokio::spawn({ + async move { + stake_notify.pop().unwrap().notified().await; + (current_slot, vote_accounts, config) + } + }); + rpc_notify_task.push(notif_jh); + } + } + } + //manage rpc waiting request notification. + Some(Ok((votes, vote_accounts, rpc_vote_accounts))) = rpc_exec_task.next() => { + if let Err(err) = votestore.votes.merge(votes) { + log::info!("Error during RPC get vote account merge:{err}"); + } + + //avoid clone on the first request + //TODO change the logic use take less one. + if pending_rpc_request.len() == 1 { + if let Err(_) = pending_rpc_request.pop().unwrap().send(rpc_vote_accounts.clone()) { + log::error!("Vote accounts RPC channel send closed."); + } + } else { + for return_channel in pending_rpc_request.drain(..) { + if let Err(_) = return_channel.send(rpc_vote_accounts.clone()) { + log::error!("Vote accounts RPC channel send closed."); + } + } + } + } + //manage rpc waiting request notification. + Some(Ok((current_slot, vote_accounts, config))) = rpc_notify_task.next() => { + match votestore.votes.take() { + TakeResult::Map(votes) => { + let jh = tokio::task::spawn_blocking({ + move || { + let rpc_vote_accounts = crate::vote::get_rpc_vote_accounts_info( + current_slot, + &votes, + &vote_accounts.as_ref().unwrap().vote_stakes, //TODO put in take. + config, + ); + (votes, vote_accounts, rpc_vote_accounts) + } + }); + rpc_exec_task.push(jh); + } + TakeResult::Taken(mut stake_notify) => { + let notif_jh = tokio::spawn({ + async move { + stake_notify.pop().unwrap().notified().await; + (current_slot, vote_accounts, config) + } + }); + rpc_notify_task.push(notif_jh); + } + } + } //manage geyser account notification //Geyser delete account notification patch must be installed on the validator. //see https://github.com/solana-labs/solana/pull/33292 @@ -129,7 +207,7 @@ pub async fn start_stakes_and_votes_loop( //log::info!("Geyser notif VOTE account:{}", account); let account_pubkey = account.pubkey; //process vote accout notification - if let Err(err) = votestore.add_vote(account, current_schedule_epoch.last_slot_in_epoch) { + if let Err(err) = votestore.notify_vote_change(account, current_schedule_epoch.last_slot_in_epoch) { log::warn!("Can't add new stake from account data err:{} account:{}", err, account_pubkey); continue; } @@ -163,8 +241,22 @@ pub async fn start_stakes_and_votes_loop( } //manage bootstrap event Some(Ok(event)) = spawned_bootstrap_task.next() => { - match crate::bootstrap::run_bootstrap_events(event, &mut spawned_bootstrap_task, &mut stakestore, &mut votestore) { - Ok(Some(boot_res))=> bootstrap_done = boot_res, + match crate::bootstrap::run_bootstrap_events(event, &mut spawned_bootstrap_task, &mut stakestore, &mut votestore, current_schedule_epoch.slots_in_epoch) { + Ok(Some(boot_res))=> { + + match boot_res { + Ok(current_schedule_data) => { + //let data_schedule = Arc::make_mut(&mut data_cache.leader_schedule); + + data_cache.leader_schedule = Arc::new(current_schedule_data); + bootstrap_done = true; + } + Err(err) => { + log::warn!("Error during current leader schedule bootstrap from files:{err}") + } + } + + }, Ok(None) => (), Err(err) => log::error!("Stake / Vote Account bootstrap fail because '{err}'"), } @@ -183,10 +275,11 @@ pub async fn start_stakes_and_votes_loop( data_schedule.current = data_schedule.next.take(); match new_leader_schedule { //TODO use vote_stakes for vote accounts RPC call. - Some(LeaderScheduleGeneratedData{schedule, vote_stakes, epoch}) => { + Some(schedule_data) => { let new_schedule_data = LeaderScheduleData{ - schedule, - epoch + schedule_by_node: LeaderScheduleGeneratedData::get_schedule_by_nodes(&schedule_data.schedule), + schedule_by_slot: schedule_data.schedule.get_slot_leaders().to_vec(), + epoch: schedule_data.epoch }; data_schedule.next = Some(new_schedule_data); } diff --git a/stake_vote/src/stake.rs b/stake_vote/src/stake.rs index def92225..9ca93e53 100644 --- a/stake_vote/src/stake.rs +++ b/stake_vote/src/stake.rs @@ -1,5 +1,8 @@ +use crate::utils::Takable; use crate::utils::TakableContent; use crate::utils::TakableMap; +use crate::utils::TakeResult; +use crate::utils::UpdateAction; use crate::AccountPretty; use crate::Slot; use anyhow::bail; @@ -13,30 +16,30 @@ use std::collections::HashMap; pub type StakeMap = HashMap; type StakeContent = (StakeMap, Option); -#[derive(Debug, Default)] -pub enum StakeAction { - Notify { - stake: StoredStake, - }, - Remove(Pubkey, Slot), - // Merge { - // source_account: Pubkey, - // destination_account: Pubkey, - // update_slot: Slot, - // }, - #[default] - None, -} - -impl StakeAction { - fn get_update_slot(&self) -> u64 { - match self { - StakeAction::Notify { stake } => stake.last_update_slot, - StakeAction::Remove(_, slot) => *slot, - StakeAction::None => 0, - } - } -} +// #[derive(Debug, Default)] +// pub enum StakeAction { +// Notify { +// stake: StoredStake, +// }, +// Remove(Pubkey, Slot), +// // Merge { +// // source_account: Pubkey, +// // destination_account: Pubkey, +// // update_slot: Slot, +// // }, +// #[default] +// None, +// } + +// impl StakeAction { +// fn get_update_slot(&self) -> u64 { +// match self { +// StakeAction::Notify { stake } => stake.last_update_slot, +// StakeAction::Remove(_, slot) => *slot, +// StakeAction::None => 0, +// } +// } +// } #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct StoredStake { @@ -47,15 +50,15 @@ pub struct StoredStake { pub write_version: u64, } -impl TakableContent for StakeContent { - fn add_value(&mut self, val: StakeAction) { +impl TakableContent for StakeContent { + fn add_value(&mut self, val: UpdateAction) { StakeStore::process_stake_action(&mut self.0, val); } } #[derive(Debug, Default)] pub struct StakeStore { - stakes: TakableMap, + pub stakes: TakableMap, } impl StakeStore { @@ -76,9 +79,9 @@ impl StakeStore { ) -> anyhow::Result<()> { //if lamport == 0 the account has been removed. if account.lamports == 0 { - self.notify_stake_action( - StakeAction::Remove(account.pubkey, account.slot), - current_end_epoch_slot, + self.stakes.add_value( + UpdateAction::Remove(account.pubkey, account.slot), + account.slot <= current_end_epoch_slot, ); } else { let Ok(delegated_stake_opt) = account.read_stake() else { @@ -94,26 +97,36 @@ impl StakeStore { write_version: account.write_version, }; - self.notify_stake_action(StakeAction::Notify { stake }, current_end_epoch_slot); + let action_update_slot = stake.last_update_slot; + self.stakes.add_value( + UpdateAction::Notify(action_update_slot, stake), + action_update_slot <= current_end_epoch_slot, + ); } } Ok(()) } - pub fn notify_stake_action(&mut self, action: StakeAction, current_end_epoch_slot: Slot) { - let action_update_slot = action.get_update_slot(); - self.stakes - .add_value(action, action_update_slot <= current_end_epoch_slot); - } + //helper method to extract and merge stakes. + // pub fn take_stakestore(&mut self) -> TakeResult { + // self.stakes.take() + // } + + // pub fn merge_stakestore( + // &mut self, + // stake_map: StakeMap, + // stake_hisotry: Option, + // ) -> anyhow::Result<()> { + // self.stakes.merge((stake_map, stake_hisotry)) + // } - fn process_stake_action(stakes: &mut StakeMap, action: StakeAction) { + fn process_stake_action(stakes: &mut StakeMap, action: UpdateAction) { match action { - StakeAction::Notify { stake } => { + UpdateAction::Notify(_, stake) => { Self::notify_stake(stakes, stake); } - StakeAction::Remove(account_pk, slot) => Self::remove_stake(stakes, &account_pk, slot), - StakeAction::None => (), + UpdateAction::Remove(account_pk, slot) => Self::remove_stake(stakes, &account_pk, slot), } } fn notify_stake(map: &mut StakeMap, stake: StoredStake) { @@ -151,20 +164,20 @@ impl StakeStore { } } - //helper method to extract and merge stakes. - pub fn take_stakestore( - stakestore: &mut StakeStore, - ) -> anyhow::Result<(StakeMap, Option)> { - crate::utils::take(&mut stakestore.stakes) - } + // //helper method to extract and merge stakes. + // pub fn take_stakestore( + // stakestore: &mut StakeStore, + // ) -> anyhow::Result<(StakeMap, Option)> { + // crate::utils::take(&mut stakestore.stakes) + // } - pub fn merge_stakestore( - stakestore: &mut StakeStore, - stake_map: StakeMap, - stake_history: Option, - ) -> anyhow::Result<()> { - crate::utils::merge(&mut stakestore.stakes, (stake_map, stake_history)) - } + // pub fn merge_stakestore( + // stakestore: &mut StakeStore, + // stake_map: StakeMap, + // stake_history: Option, + // ) -> anyhow::Result<()> { + // crate::utils::merge(&mut stakestore.stakes, (stake_map, stake_history)) + // } } pub fn merge_program_account_in_strake_map( diff --git a/stake_vote/src/utils.rs b/stake_vote/src/utils.rs index f48a3e66..7bceeb3d 100644 --- a/stake_vote/src/utils.rs +++ b/stake_vote/src/utils.rs @@ -1,4 +1,5 @@ use crate::vote::StoredVote; +use crate::Slot; use anyhow::bail; use serde::{Deserialize, Serialize}; use solana_lite_rpc_core::stores::block_information_store::BlockInformation; @@ -12,6 +13,7 @@ use std::fs::File; use std::io::Write; use std::str::FromStr; use std::sync::Arc; +use tokio::sync::Notify; pub async fn get_current_confirmed_slot(data_cache: &DataCache) -> u64 { let commitment = CommitmentConfig::confirmed(); @@ -73,9 +75,120 @@ pub fn save_schedule_vote_stakes( Ok(()) } +#[derive(Debug)] +pub enum UpdateAction { + Notify(Slot, Account), + Remove(Pubkey, Slot), +} + +// impl UpdateAction { +// pub fn get_update_slot(&self) -> u64 { +// match self { +// UpdateAction::Notify(slot, _) | UpdateAction::Remove(_, slot) => *slot, +// } +// } +// } + +pub enum TakeResult { + //Vec because can wait on several collection to be merged + Taken(Vec>), + Map(C), +} + +impl TakeResult { + pub fn and_then(self, action: TakeResult) -> TakeResult<(C1, C2)> { + match (self, action) { + (TakeResult::Taken(mut notif1), TakeResult::Taken(mut notif2)) => { + notif1.append(&mut notif2); + TakeResult::Taken(notif1) + } + (TakeResult::Map(content1), TakeResult::Map(content2)) => { + TakeResult::Map((content1, content2)) + } + _ => unreachable!("Bad take result association."), //TODO add mix result. + } + } + + // pub fn get_content(self) -> Option { + // match self { + // TakeResult::Taken(_) => None, + // TakeResult::Map(content) => Some(content), + // } + // } +} + //Takable struct code pub trait TakableContent: Default { - fn add_value(&mut self, val: T); + fn add_value(&mut self, val: UpdateAction); +} + +//Takable struct code +pub trait Takable { + fn take(self) -> TakeResult; + fn merge(self, content: C) -> anyhow::Result<()>; + fn is_taken(&self) -> bool; +} + +impl<'a, T, C: TakableContent> Takable for &'a mut TakableMap { + fn take(self) -> TakeResult { + match self.content.take() { + Some(content) => TakeResult::Map(content), + None => TakeResult::Taken(vec![Arc::clone(&self.notifier)]), + } + } + + fn merge(mut self, mut content: C) -> anyhow::Result<()> { + if self.content.is_none() { + //apply stake added during extraction. + for val in self.updates.drain(..) { + content.add_value(val); + } + self.content = Some(content); + self.notifier.notify_waiters(); + Ok(()) + } else { + bail!("TakableMap with a existing content".to_string()) + } + } + + fn is_taken(&self) -> bool { + self.content.is_none() + } +} + +impl<'a, T1, T2, C1: TakableContent, C2: TakableContent> Takable<(C1, C2)> + for (&'a mut TakableMap, &'a mut TakableMap) +{ + fn take(self) -> TakeResult<(C1, C2)> { + let first = self.0; + let second = self.1; + + match (first.is_taken(), second.is_taken()) { + (true, true) | (false, false) => first.take().and_then(second.take()), + (true, false) => { + match first.take() { + TakeResult::Taken(notif) => TakeResult::Taken(notif), + TakeResult::Map(_) => unreachable!(), //tested before. + } + } + (false, true) => { + match second.take() { + TakeResult::Taken(notif) => TakeResult::Taken(notif), + TakeResult::Map(_) => unreachable!(), //tested before. + } + } + } + } + + fn merge(self, content: (C1, C2)) -> anyhow::Result<()> { + self.0 + .merge(content.0) + .and_then(|_| self.1.merge(content.1)) + } + + fn is_taken(&self) -> bool { + self.0.is_taken() && self.1.is_taken() + } } ///A struct that hold a collection call content that can be taken during some time and merged after. @@ -83,84 +196,60 @@ pub trait TakableContent: Default { ///It allow to process struct content while allowing to still update it without lock. #[derive(Default, Debug)] pub struct TakableMap> { - pub content: C, - pub updates: Vec, - taken: bool, + pub content: Option, + pub updates: Vec>, + notifier: Arc, } impl + Default> TakableMap { pub fn new(content: C) -> Self { TakableMap { - content, + content: Some(content), updates: vec![], - taken: false, + notifier: Arc::new(Notify::new()), } } //add a value to the content if not taken or put it in the update waiting list. //Use force_in_update to force the insert in update waiting list. - pub fn add_value(&mut self, val: T, force_in_update: bool) { + pub fn add_value(&mut self, val: UpdateAction, force_in_update: bool) { //during extract push the new update or //don't insert now account change that has been done in next epoch. //put in update pool to be merged next epoch change. - match self.taken || force_in_update { + match self.content.is_none() || force_in_update { true => self.updates.push(val), - false => self.content.add_value(val), - } - } - - pub fn take(self) -> (Self, C) { - let takenmap = TakableMap { - content: C::default(), - updates: self.updates, - taken: true, - }; - (takenmap, self.content) - } - - pub fn merge(self, content: C) -> Self { - let mut mergedstore = TakableMap { - content, - updates: vec![], - taken: false, - }; - - //apply stake added during extraction. - for val in self.updates { - mergedstore.content.add_value(val); + false => { + let content = self.content.as_mut().unwrap(); //unwrap tested + content.add_value(val); + } } - mergedstore - } - - pub fn is_taken(&self) -> bool { - self.taken } } -pub fn take + Default>( - map: &mut TakableMap, -) -> anyhow::Result { - if map.is_taken() { - bail!("TakableMap already taken. Try later"); - } - let new_store = std::mem::take(map); - let (new_store, content) = new_store.take(); - *map = new_store; - Ok(content) -} +// pub fn take + Default>( +// map: &mut TakableMap, +// ) -> anyhow::Result { +// if map.is_taken() { +// bail!("TakableMap already taken. Try later"); +// } +// let new_store = std::mem::take(map); +// let (new_store, content) = new_store.take(); +// *map = new_store; +// Ok(content) +// } -pub fn merge + Default>( - map: &mut TakableMap, - content: C, -) -> anyhow::Result<()> { - if !map.is_taken() { - bail!("TakableMap merge of non taken map. Try later"); - } - let new_store = std::mem::take(map); - let new_store = new_store.merge(content); - *map = new_store; - Ok(()) -} +// pub fn merge + Default>( +// map: &mut TakableMap, +// content: C, +// ) -> anyhow::Result<()> { +// if !map.is_taken() { +// bail!("TakableMap merge of non taken map. Try later"); +// } +// let new_store = std::mem::take(map); +// let new_store = new_store.merge(content); +// *map = new_store; +// Ok(()) +// } #[cfg(test)] mod tests { @@ -169,37 +258,55 @@ mod tests { #[test] fn test_takable_struct() { impl TakableContent for Vec { - fn add_value(&mut self, val: u64) { - self.push(val) + fn add_value(&mut self, val: UpdateAction) { + match val { + UpdateAction::Notify(account, _) => self.push(account), + UpdateAction::Remove(_, _) => (), + UpdateAction::None => (), + } } } let content: Vec = vec![]; let mut takable = TakableMap::new(content); - takable.add_value(23, false); - assert_eq!(takable.content.len(), 1); + takable.add_value(UpdateAction::Notify(23, 0), false); + assert_eq!(takable.content.as_ref().unwrap().len(), 1); - takable.add_value(24, true); - assert_eq!(takable.content.len(), 1); + takable.add_value(UpdateAction::Notify(24, 0), true); + assert_eq!(takable.content.as_ref().unwrap().len(), 1); assert_eq!(takable.updates.len(), 1); - let content = take(&mut takable).unwrap(); - assert_eq!(content.len(), 1); - assert_eq!(takable.content.len(), 0); + let take_content = (&mut takable).take(); + assert_take_content_map(&take_content, 1); assert_eq!(takable.updates.len(), 1); - let err_content = take(&mut takable); - assert!(err_content.is_err()); - assert_eq!(content.len(), 1); - assert_eq!(takable.content.len(), 0); + let take_content = (&mut takable).take(); + assert_take_content_taken(&take_content); + assert!(takable.content.is_none()); assert_eq!(takable.updates.len(), 1); - takable.add_value(25, false); - assert_eq!(takable.content.len(), 0); + takable.add_value(UpdateAction::Notify(25, 0), false); assert_eq!(takable.updates.len(), 2); - merge(&mut takable, content).unwrap(); - assert_eq!(takable.content.len(), 3); + let content = match take_content { + TakeResult::Taken(_) => panic!("not a content"), + TakeResult::Map(content) => content, + }; + takable.merge(content); + assert_eq!(takable.content.as_ref().unwrap().len(), 3); assert_eq!(takable.updates.len(), 0); - let err = merge(&mut takable, vec![]); - assert!(err.is_err()); + //merge(&mut takable, vec![]); + //assert!(err.is_err()); + } + + fn assert_take_content_map(take_content: &TakeResult>, len: usize) { + match take_content { + TakeResult::Taken(_) => assert!(false), + TakeResult::Map(content) => assert_eq!(content.len(), len), + } + } + fn assert_take_content_taken(take_content: &TakeResult>) { + match take_content { + TakeResult::Taken(_) => (), + TakeResult::Map(_) => assert!(false), + } } } diff --git a/stake_vote/src/vote.rs b/stake_vote/src/vote.rs index 661e34ff..4d47db37 100644 --- a/stake_vote/src/vote.rs +++ b/stake_vote/src/vote.rs @@ -1,9 +1,14 @@ use crate::utils::TakableContent; use crate::utils::TakableMap; +use crate::utils::UpdateAction; use crate::AccountPretty; use crate::Slot; use anyhow::bail; use serde::{Deserialize, Serialize}; +use solana_lite_rpc_core::structures::leaderschedule::GetVoteAccountsConfig; +use solana_rpc_client_api::request::MAX_RPC_VOTE_ACCOUNT_INFO_EPOCH_CREDITS_HISTORY; +use solana_rpc_client_api::response::RpcVoteAccountInfo; +use solana_rpc_client_api::response::RpcVoteAccountStatus; use solana_sdk::account::Account; use solana_sdk::pubkey::Pubkey; use solana_sdk::vote::state::VoteState; @@ -12,9 +17,15 @@ use std::sync::Arc; pub type VoteMap = HashMap>; +#[derive(Debug, Clone)] +pub struct EpochVoteStakes { + pub vote_stakes: HashMap)>, + pub epoch: u64, +} + impl TakableContent for VoteMap { - fn add_value(&mut self, val: StoredVote) { - VoteStore::vote_map_insert_vote(self, val.pubkey, val); + fn add_value(&mut self, val: UpdateAction) { + VoteStore::process_vote_action(self, val); } } @@ -26,29 +37,84 @@ pub struct StoredVote { pub write_version: u64, } +impl StoredVote { + pub fn convert_to_rpc_vote_account_info( + &self, + activated_stake: u64, + epoch_vote_account: bool, + ) -> RpcVoteAccountInfo { + let last_vote = self + .vote_data + .votes + .iter() + .last() + .map(|vote| vote.slot()) + .unwrap_or_default(); + + RpcVoteAccountInfo { + vote_pubkey: self.pubkey.to_string(), + node_pubkey: self.vote_data.node_pubkey.to_string(), + activated_stake, + commission: self.vote_data.commission, + epoch_vote_account, + epoch_credits: self.vote_data.epoch_credits.clone(), + last_vote, + root_slot: self.vote_data.root_slot.unwrap_or_default(), + } + } +} + #[derive(Default)] pub struct VoteStore { - votes: TakableMap, + pub votes: TakableMap, + epoch_vote_stake_map: HashMap, } impl VoteStore { pub fn new(capacity: usize) -> Self { VoteStore { votes: TakableMap::new(HashMap::with_capacity(capacity)), + epoch_vote_stake_map: HashMap::new(), } } - pub fn add_vote( + + pub fn add_epoch_vote_stake(&mut self, stakes: EpochVoteStakes) { + self.epoch_vote_stake_map.insert(stakes.epoch, stakes); + } + + pub fn vote_stakes_for_epoch(&self, epoch: u64) -> Option { + self.epoch_vote_stake_map.get(&epoch).cloned() + } + + pub fn notify_vote_change( &mut self, new_account: AccountPretty, current_end_epoch_slot: Slot, ) -> anyhow::Result<()> { if new_account.lamports == 0 { - self.remove_from_store(&new_account.pubkey, new_account.slot); + //self.remove_from_store(&new_account.pubkey, new_account.slot); + self.votes.add_value( + UpdateAction::Remove(new_account.pubkey, new_account.slot), + new_account.slot <= current_end_epoch_slot, + ); } else { - let Ok(vote_data) = new_account.read_vote() else { + let Ok(mut vote_data) = new_account.read_vote() else { bail!("Can't read Vote from account data"); }; + //remove unnecessary entry. See Solana code rpc::rpc::get_vote_accounts + let epoch_credits = vote_data.epoch_credits(); + vote_data.epoch_credits = + if epoch_credits.len() > MAX_RPC_VOTE_ACCOUNT_INFO_EPOCH_CREDITS_HISTORY { + epoch_credits + .iter() + .skip(epoch_credits.len() - MAX_RPC_VOTE_ACCOUNT_INFO_EPOCH_CREDITS_HISTORY) + .cloned() + .collect() + } else { + epoch_credits.clone() + }; + //log::info!("add_vote {} :{vote_data:?}", new_account.pubkey); let new_voteacc = StoredVote { @@ -59,35 +125,49 @@ impl VoteStore { }; let action_update_slot = new_voteacc.last_update_slot; - self.votes - .add_value(new_voteacc, action_update_slot <= current_end_epoch_slot); + self.votes.add_value( + UpdateAction::Notify(action_update_slot, new_voteacc), + action_update_slot <= current_end_epoch_slot, + ); } Ok(()) } - //helper method to extract and merge stakes. - pub fn take_votestore(votestore: &mut VoteStore) -> anyhow::Result { - crate::utils::take(&mut votestore.votes) - } - pub fn merge_votestore(votestore: &mut VoteStore, vote_map: VoteMap) -> anyhow::Result<()> { - crate::utils::merge(&mut votestore.votes, vote_map) + fn process_vote_action(votes: &mut VoteMap, action: UpdateAction) { + match action { + UpdateAction::Notify(_, vote) => { + Self::vote_map_insert_vote(votes, vote); + } + UpdateAction::Remove(account_pk, slot) => { + Self::remove_from_store(votes, &account_pk, slot) + } + } } - fn remove_from_store(&mut self, account_pk: &Pubkey, update_slot: Slot) { - if self - .votes - .content + // //helper method to extract and merge stakes. + // pub fn take_votestore(&mut self) -> TakeResult { + // self.votes.take() + // } + + // pub fn merge_votestore(&mut self, vote_map: VoteMap) -> anyhow::Result<()> { + // self.votes.merge(vote_map) + // } + + fn remove_from_store(votes: &mut VoteMap, account_pk: &Pubkey, update_slot: Slot) { + //TODO use action. + if votes .get(account_pk) .map(|vote| vote.last_update_slot <= update_slot) .unwrap_or(true) { log::info!("Vote remove_from_store for {}", account_pk.to_string()); - self.votes.content.remove(account_pk); + votes.remove(account_pk); } } - fn vote_map_insert_vote(map: &mut VoteMap, vote_account_pk: Pubkey, vote_data: StoredVote) { + fn vote_map_insert_vote(map: &mut VoteMap, vote_data: StoredVote) { + let vote_account_pk = vote_data.pubkey; match map.entry(vote_account_pk) { std::collections::hash_map::Entry::Occupied(occupied) => { let voteacc = occupied.into_mut(); // <-- get mut reference to existing value @@ -147,6 +227,54 @@ pub fn merge_program_account_in_vote_map( last_update_slot, write_version: 0, }; - VoteStore::vote_map_insert_vote(vote_map, pk, vote); + VoteStore::vote_map_insert_vote(vote_map, vote); }); } + +//TODO put in config instead of const. +// Validators that are this number of slots behind are considered delinquent +pub const DELINQUENT_VALIDATOR_SLOT_DISTANCE: u64 = 128; +pub fn get_rpc_vote_accounts_info( + current_slot: Slot, + votes: &VoteMap, + vote_accounts: &HashMap)>, + config: GetVoteAccountsConfig, +) -> RpcVoteAccountStatus { + //TODO + //manage + + //From Solana rpc::rpc::metaz::get_vote_accounts() code. + let (current_vote_accounts, delinquent_vote_accounts): ( + Vec, + Vec, + ) = votes + .values() + .map(|vote| { + let (stake, epoch_vote_account) = vote_accounts + .get(&vote.pubkey) + .map(|(stake, _)| (*stake, true)) + .unwrap_or((0, false)); + vote.convert_to_rpc_vote_account_info(stake, epoch_vote_account) + }) + .partition(|vote_account_info| { + if current_slot >= DELINQUENT_VALIDATOR_SLOT_DISTANCE { + vote_account_info.last_vote > current_slot - DELINQUENT_VALIDATOR_SLOT_DISTANCE + } else { + vote_account_info.last_vote > 0 + } + }); + let keep_unstaked_delinquents = config.keep_unstaked_delinquents.unwrap_or_default(); + let delinquent_vote_accounts = if !keep_unstaked_delinquents { + delinquent_vote_accounts + .into_iter() + .filter(|vote_account_info| vote_account_info.activated_stake > 0) + .collect::>() + } else { + delinquent_vote_accounts + }; + + RpcVoteAccountStatus { + current: current_vote_accounts, + delinquent: delinquent_vote_accounts, + } +} diff --git a/yarn.lock b/yarn.lock index deb7ba81..6f339c5e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -22,7 +22,7 @@ resolved "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.20.10.tgz" integrity sha512-sEnuDPpOJR/fcafHMjpcpGN5M2jbUGUHwmuWKM/YdPzeEDJg8bgmbcWQFUfE32MQjti1koACvoPVsDe8Uq+idg== -"@babel/core@^7.11.6", "@babel/core@^7.12.3": +"@babel/core@^7.0.0", "@babel/core@^7.0.0-0", "@babel/core@^7.11.6", "@babel/core@^7.12.3", "@babel/core@^7.8.0", "@babel/core@>=7.0.0-beta.0 <8": version "7.20.12" resolved "https://registry.npmjs.org/@babel/core/-/core-7.20.12.tgz" integrity sha512-XsMfHovsUYHFMdrIHkZphTN/2Hzzi78R08NuHfDBehym2VsPDL6Zn/JAD/JQdnRvbSsbQc4mVaU1m6JgtTEElg== @@ -501,7 +501,7 @@ slash "^3.0.0" write-file-atomic "^4.0.1" -"@jest/types@^29.3.1": +"@jest/types@^29.0.0", "@jest/types@^29.3.1": version "29.3.1" resolved "https://registry.npmjs.org/@jest/types/-/types-29.3.1.tgz" integrity sha512-d0S0jmmTpjnhCmNpApgX3jrUZgZ22ivKJRvL2lli5hpCRoNnp1f85r2/wpKfXuYu8E7Jjh1hGfhPyup1NM5AmA== @@ -540,7 +540,7 @@ resolved "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz" integrity sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw== -"@jridgewell/sourcemap-codec@1.4.14", "@jridgewell/sourcemap-codec@^1.4.10": +"@jridgewell/sourcemap-codec@^1.4.10", "@jridgewell/sourcemap-codec@1.4.14": version "1.4.14" resolved "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz" integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw== @@ -613,7 +613,7 @@ "@solana/buffer-layout-utils" "^0.2.0" buffer "^6.0.3" -"@solana/web3.js@^1.32.0", "@solana/web3.js@^1.73.0": +"@solana/web3.js@^1.32.0", "@solana/web3.js@^1.47.4", "@solana/web3.js@^1.73.0": version "1.73.0" resolved "https://registry.npmjs.org/@solana/web3.js/-/web3.js-1.73.0.tgz" integrity sha512-YrgX3Py7ylh8NYkbanoINUPCj//bWUjYZ5/WPy9nQ9SK3Cl7QWCR+NmbDjmC/fTspZGR+VO9LTQslM++jr5PRw== @@ -743,14 +743,6 @@ dependencies: "@types/yargs-parser" "*" -JSONStream@^1.3.5: - version "1.3.5" - resolved "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz" - integrity sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ== - dependencies: - jsonparse "^1.2.0" - through ">=2.2.7 <3" - agentkeepalive@^4.2.1: version "4.2.1" resolved "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.2.1.tgz" @@ -806,7 +798,7 @@ argparse@^1.0.7: dependencies: sprintf-js "~1.0.2" -babel-jest@^29.3.1: +babel-jest@^29.0.0, babel-jest@^29.3.1: version "29.3.1" resolved "https://registry.npmjs.org/babel-jest/-/babel-jest-29.3.1.tgz" integrity sha512-aard+xnMoxgjwV70t0L6wkW/3HQQtV+O0PEimxKgzNqCJnbYmroPojdP2tqKSOAt8QAKV/uSZU8851M7B5+fcA== @@ -931,7 +923,7 @@ braces@^3.0.2: dependencies: fill-range "^7.0.1" -browserslist@^4.21.3: +browserslist@^4.21.3, "browserslist@>= 4.21.0": version "4.21.4" resolved "https://registry.npmjs.org/browserslist/-/browserslist-4.21.4.tgz" integrity sha512-CBHJJdDmgjl3daYjN5Cp5kbTf1mUhZoS+beLklHIvkOWscs83YAhLlF3Wsh/lciQYAcbBJgTOD44VtG31ZM4Hw== @@ -967,14 +959,6 @@ buffer-from@^1.0.0: resolved "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz" integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== -buffer@6.0.1: - version "6.0.1" - resolved "https://registry.npmjs.org/buffer/-/buffer-6.0.1.tgz" - integrity sha512-rVAXBwEcEoYtxnHSO5iWyhzV/O1WMtkUYWlfdLS7FjU4PnSJJHEfHXi/uHPI5EwltmOA794gN3bm3/pzuctWjQ== - dependencies: - base64-js "^1.3.1" - ieee754 "^1.2.1" - buffer@^6.0.3, buffer@~6.0.3: version "6.0.3" resolved "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz" @@ -983,6 +967,14 @@ buffer@^6.0.3, buffer@~6.0.3: base64-js "^1.3.1" ieee754 "^1.2.1" +buffer@6.0.1: + version "6.0.1" + resolved "https://registry.npmjs.org/buffer/-/buffer-6.0.1.tgz" + integrity sha512-rVAXBwEcEoYtxnHSO5iWyhzV/O1WMtkUYWlfdLS7FjU4PnSJJHEfHXi/uHPI5EwltmOA794gN3bm3/pzuctWjQ== + dependencies: + base64-js "^1.3.1" + ieee754 "^1.2.1" + bufferutil@^4.0.1: version "4.0.7" resolved "https://registry.npmjs.org/bufferutil/-/bufferutil-4.0.7.tgz" @@ -1075,16 +1067,16 @@ color-convert@^2.0.1: dependencies: color-name "~1.1.4" -color-name@1.1.3: - version "1.1.3" - resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz" - integrity sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw== - color-name@~1.1.4: version "1.1.4" resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== +color-name@1.1.3: + version "1.1.3" + resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz" + integrity sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw== + commander@^2.20.3: version "2.20.3" resolved "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz" @@ -1095,7 +1087,12 @@ concat-map@0.0.1: resolved "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg== -convert-source-map@^1.6.0, convert-source-map@^1.7.0: +convert-source-map@^1.6.0: + version "1.9.0" + resolved "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz" + integrity sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A== + +convert-source-map@^1.7.0: version "1.9.0" resolved "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz" integrity sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A== @@ -1116,7 +1113,7 @@ cross-spawn@^7.0.3: crypto@^1.0.1: version "1.0.1" - resolved "https://registry.yarnpkg.com/crypto/-/crypto-1.0.1.tgz#2af1b7cad8175d24c8a1b0778255794a21803037" + resolved "https://registry.npmjs.org/crypto/-/crypto-1.0.1.tgz" integrity sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig== debug@^4.1.0, debug@^4.1.1: @@ -1251,7 +1248,7 @@ eyes@^0.1.8: resolved "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz" integrity sha512-GipyPsXO1anza0AOZdy69Im7hGFCNB7Y/NGjDlZGJ3GJJLtwNSb2vrzYrTYJRrRloVx7pl+bhUaTB8yiccPvFQ== -fast-json-stable-stringify@2.x, fast-json-stable-stringify@^2.1.0: +fast-json-stable-stringify@^2.1.0, fast-json-stable-stringify@2.x: version "2.1.0" resolved "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz" integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== @@ -1293,11 +1290,6 @@ fs.realpath@^1.0.0: resolved "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz" integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== -fsevents@^2.3.2: - version "2.3.2" - resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" - integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== - function-bind@^1.1.1: version "1.1.1" resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz" @@ -1502,13 +1494,13 @@ jayson@^3.4.4: "@types/connect" "^3.4.33" "@types/node" "^12.12.54" "@types/ws" "^7.4.4" - JSONStream "^1.3.5" commander "^2.20.3" delay "^5.0.0" es6-promisify "^5.0.0" eyes "^0.1.8" isomorphic-ws "^4.0.1" json-stringify-safe "^5.0.1" + JSONStream "^1.3.5" lodash "^4.17.20" uuid "^8.3.2" ws "^7.4.5" @@ -1716,7 +1708,7 @@ jest-resolve-dependencies@^29.3.1: jest-regex-util "^29.2.0" jest-snapshot "^29.3.1" -jest-resolve@^29.3.1: +jest-resolve@*, jest-resolve@^29.3.1: version "29.3.1" resolved "https://registry.npmjs.org/jest-resolve/-/jest-resolve-29.3.1.tgz" integrity sha512-amXJgH/Ng712w3Uz5gqzFBBjxV8WFLSmNjoreBGMqxgCz5cH7swmBZzgBaCIOsvb0NbpJ0vgaSFdJqMdT+rADw== @@ -1864,7 +1856,7 @@ jest-worker@^29.3.1: merge-stream "^2.0.0" supports-color "^8.0.0" -jest@^29.3.1: +jest@^29.0.0, jest@^29.3.1: version "29.3.1" resolved "https://registry.npmjs.org/jest/-/jest-29.3.1.tgz" integrity sha512-6iWfL5DTT0Np6UYs/y5Niu7WIfNv/wRTtN5RSXt2DIEft3dx3zPuw/3WJQBCJfmEzvDiEKwoqMbGD9n49+qLSA== @@ -1912,6 +1904,14 @@ jsonparse@^1.2.0: resolved "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz" integrity sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg== +JSONStream@^1.3.5: + version "1.3.5" + resolved "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz" + integrity sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ== + dependencies: + jsonparse "^1.2.0" + through ">=2.2.7 <3" + kleur@^3.0.3: version "3.0.3" resolved "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz" @@ -2002,7 +2002,7 @@ minimatch@^3.0.4, minimatch@^3.1.1: dependencies: brace-expansion "^1.1.7" -ms@2.1.2, ms@^2.0.0: +ms@^2.0.0, ms@2.1.2: version "2.1.2" resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== @@ -2214,17 +2214,24 @@ safe-buffer@^5.0.1: resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== -semver@7.x, semver@^7.3.5: +semver@^6.0.0, semver@^6.3.0: + version "6.3.0" + resolved "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz" + integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== + +semver@^7.3.5: version "7.3.8" resolved "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz" integrity sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A== dependencies: lru-cache "^6.0.0" -semver@^6.0.0, semver@^6.3.0: - version "6.3.0" - resolved "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz" - integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== +semver@7.x: + version "7.3.8" + resolved "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz" + integrity sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A== + dependencies: + lru-cache "^6.0.0" shebang-command@^2.0.0: version "2.0.0" @@ -2413,10 +2420,10 @@ type-fest@^0.21.3: resolved "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz" integrity sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w== -typescript@^4.9.4: - version "4.9.4" - resolved "https://registry.npmjs.org/typescript/-/typescript-4.9.4.tgz" - integrity sha512-Uz+dTXYzxXXbsFpM86Wh3dKCxrQqUcVMxwU54orwlJjOpO3ao8L7j5lH+dWfTwgCwIuM9GQ2kvVotzYJMXTBZg== +typescript@^4.9.5, typescript@>=4.3: + version "4.9.5" + resolved "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz" + integrity sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g== update-browserslist-db@^1.0.9: version "1.0.10" @@ -2426,7 +2433,7 @@ update-browserslist-db@^1.0.9: escalade "^3.1.1" picocolors "^1.0.0" -utf-8-validate@^5.0.2: +utf-8-validate@^5.0.2, utf-8-validate@>=5.0.2: version "5.0.10" resolved "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-5.0.10.tgz" integrity sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ== @@ -2496,7 +2503,7 @@ write-file-atomic@^4.0.1: imurmurhash "^0.1.4" signal-exit "^3.0.7" -ws@^7.4.5: +ws@*, ws@^7.4.5: version "7.5.9" resolved "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz" integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q== From 8a0b7ccae3d736c3ddd27850ae26d8c752c24d2e Mon Sep 17 00:00:00 2001 From: musitdev Date: Thu, 26 Oct 2023 19:37:56 +0200 Subject: [PATCH 13/56] update take/merge algo --- stake_vote/src/bootstrap.rs | 86 +++++++++++++++++------- stake_vote/src/leader_schedule.rs | 19 ++++-- stake_vote/src/lib.rs | 89 ++++--------------------- stake_vote/src/rpcrequest.rs | 107 ++++++++++++++++++++++++++++++ stake_vote/src/stake.rs | 2 - stake_vote/src/utils.rs | 79 +++++++++++++++++++++- stake_vote/src/vote.rs | 40 +++++++---- 7 files changed, 298 insertions(+), 124 deletions(-) create mode 100644 stake_vote/src/rpcrequest.rs diff --git a/stake_vote/src/bootstrap.rs b/stake_vote/src/bootstrap.rs index 7971c200..447a3618 100644 --- a/stake_vote/src/bootstrap.rs +++ b/stake_vote/src/bootstrap.rs @@ -4,6 +4,7 @@ use crate::stake::StakeMap; use crate::stake::StakeStore; use crate::utils::{Takable, TakeResult}; use crate::vote::EpochVoteStakes; +use crate::vote::EpochVoteStakesCache; use crate::vote::VoteMap; use crate::vote::VoteStore; use anyhow::bail; @@ -109,6 +110,7 @@ pub enum BootstrapEvent { StoreExtracted( StakeMap, VoteMap, + EpochVoteStakesCache, Vec<(Pubkey, Account)>, Vec<(Pubkey, Account)>, Account, @@ -118,6 +120,7 @@ pub enum BootstrapEvent { StakeMap, Option, VoteMap, + EpochVoteStakesCache, String, anyhow::Result, ), @@ -148,7 +151,7 @@ fn process_bootstrap_event( if sleep_time > 0 { std::thread::sleep(Duration::from_secs(sleep_time)); } - match crate::bootstrap::bootstrap_accounts(rpc_url.clone()) { + match bootstrap_accounts(rpc_url.clone()) { Ok((stakes, votes, history)) => { BootstrapEvent::BootstrapAccountsFetched(stakes, votes, history, rpc_url) } @@ -165,9 +168,15 @@ fn process_bootstrap_event( BootstrapEvent::BootstrapAccountsFetched(stakes, votes, history, rpc_url) => { log::info!("BootstrapEvent::BootstrapAccountsFetched RECV"); match (&mut stakestore.stakes, &mut votestore.votes).take() { - TakeResult::Map(((stake_map, _), vote_map)) => { + TakeResult::Map(((stake_map, _), (vote_map, epoch_cache))) => { BootsrapProcessResult::Event(BootstrapEvent::StoreExtracted( - stake_map, vote_map, stakes, votes, history, rpc_url, + stake_map, + vote_map, + epoch_cache, + stakes, + votes, + history, + rpc_url, )) } TakeResult::Taken(stake_notify) => { @@ -191,6 +200,7 @@ fn process_bootstrap_event( BootstrapEvent::StoreExtracted( mut stake_map, mut vote_map, + mut epoch_cache, stakes, votes, history, @@ -220,15 +230,28 @@ fn process_bootstrap_event( 0, //with RPC no way to know the slot of the account update. Set to 0. ); - let leader_schedule_result = bootstrap_current_leader_schedule(slots_in_epoch); - - BootstrapEvent::AccountsMerged( - stake_map, - stake_history, - vote_map, - rpc_url, - leader_schedule_result, - ) + match bootstrap_current_leader_schedule(slots_in_epoch) { + Ok((leader_schedule, current_epoch_stakes, next_epoch_stakes)) => { + epoch_cache.add_stakes_for_epoch(current_epoch_stakes); + epoch_cache.add_stakes_for_epoch(next_epoch_stakes); + BootstrapEvent::AccountsMerged( + stake_map, + stake_history, + vote_map, + epoch_cache, + rpc_url, + Ok(leader_schedule), + ) + } + Err(err) => BootstrapEvent::AccountsMerged( + stake_map, + stake_history, + vote_map, + epoch_cache, + rpc_url, + Err(err), + ), + } } }); BootsrapProcessResult::TaskHandle(jh) @@ -237,6 +260,7 @@ fn process_bootstrap_event( stake_map, stake_history, vote_map, + epoch_cache, rpc_url, leader_schedule_result, ) => { @@ -244,7 +268,7 @@ fn process_bootstrap_event( match ( stakestore.stakes.merge((stake_map, stake_history)), - votestore.votes.merge(vote_map), + votestore.votes.merge((vote_map, epoch_cache)), ) { (Ok(()), Ok(())) => BootsrapProcessResult::End(leader_schedule_result), _ => { @@ -317,7 +341,7 @@ pub fn get_stakehistory_account(rpc_url: String) -> Result pub fn bootstrap_current_leader_schedule( slots_in_epoch: u64, -) -> anyhow::Result { +) -> anyhow::Result<(CalculatedSchedule, EpochVoteStakes, EpochVoteStakes)> { let (current_epoch, current_epoch_stakes) = crate::utils::read_schedule_vote_stakes(CURRENT_EPOCH_VOTE_STAKES_FILE)?; let (next_epoch, next_epoch_stakes) = @@ -336,16 +360,30 @@ pub fn bootstrap_current_leader_schedule( slots_in_epoch, ); - Ok(CalculatedSchedule { - current: Some(LeaderScheduleData { - schedule_by_node: LeaderScheduleGeneratedData::get_schedule_by_nodes(¤t_schedule), - schedule_by_slot: current_schedule.get_slot_leaders().to_vec(), + Ok(( + CalculatedSchedule { + current: Some(LeaderScheduleData { + schedule_by_node: LeaderScheduleGeneratedData::get_schedule_by_nodes( + ¤t_schedule, + ), + schedule_by_slot: current_schedule.get_slot_leaders().to_vec(), + epoch: current_epoch, + }), + next: Some(LeaderScheduleData { + schedule_by_node: LeaderScheduleGeneratedData::get_schedule_by_nodes( + &next_schedule, + ), + schedule_by_slot: next_schedule.get_slot_leaders().to_vec(), + epoch: next_epoch, + }), + }, + EpochVoteStakes { epoch: current_epoch, - }), - next: Some(LeaderScheduleData { - schedule_by_node: LeaderScheduleGeneratedData::get_schedule_by_nodes(&next_schedule), - schedule_by_slot: next_schedule.get_slot_leaders().to_vec(), + vote_stakes: current_epoch_stakes, + }, + EpochVoteStakes { epoch: next_epoch, - }), - }) + vote_stakes: next_epoch_stakes, + }, + )) } diff --git a/stake_vote/src/leader_schedule.rs b/stake_vote/src/leader_schedule.rs index 4887b979..ca78f83b 100644 --- a/stake_vote/src/leader_schedule.rs +++ b/stake_vote/src/leader_schedule.rs @@ -1,5 +1,7 @@ use crate::stake::{StakeMap, StakeStore}; use crate::utils::{Takable, TakeResult}; +use crate::vote::EpochVoteStakes; +use crate::vote::EpochVoteStakesCache; use crate::vote::StoredVote; use crate::vote::{VoteMap, VoteStore}; use futures::future::join_all; @@ -18,7 +20,6 @@ use tokio::task::JoinHandle; #[derive(Debug)] pub struct LeaderScheduleGeneratedData { pub schedule: LeaderSchedule, - pub epoch_vote_stakes: HashMap)>, pub epoch: u64, } @@ -67,6 +68,7 @@ pub enum LeaderScheduleEvent { MergeStoreAndSaveSchedule( StakeMap, VoteMap, + EpochVoteStakesCache, LeaderScheduleGeneratedData, (u64, u64, Option), Option, @@ -108,7 +110,7 @@ fn process_leadershedule_event( match event { LeaderScheduleEvent::Init(new_epoch, slots_in_epoch, new_rate_activation_epoch) => { match (&mut stakestore.stakes, &mut votestore.votes).take() { - TakeResult::Map(((stake_map, mut stake_history), vote_map)) => { + TakeResult::Map(((stake_map, mut stake_history), (vote_map, mut epoch_cache))) => { log::info!("LeaderScheduleEvent::CalculateScedule"); //do the calculus in a blocking task. let jh = tokio::task::spawn_blocking({ @@ -153,14 +155,19 @@ fn process_leadershedule_event( ); } + epoch_cache.add_stakes_for_epoch(EpochVoteStakes { + epoch: new_epoch, + vote_stakes: epoch_vote_stakes, + }); + log::info!("End calculate leader schedule"); LeaderScheduleEvent::MergeStoreAndSaveSchedule( stake_map, vote_map, + epoch_cache, LeaderScheduleGeneratedData { schedule: leader_schedule, - epoch_vote_stakes, epoch: next_epoch, }, (new_epoch, slots_in_epoch, new_rate_activation_epoch), @@ -192,6 +199,7 @@ fn process_leadershedule_event( LeaderScheduleEvent::MergeStoreAndSaveSchedule( stake_map, vote_map, + epoch_cache, schedule_data, (new_epoch, slots_in_epoch, epoch_schedule), stake_history, @@ -199,7 +207,7 @@ fn process_leadershedule_event( log::info!("LeaderScheduleEvent::MergeStoreAndSaveSchedule RECV"); match ( stakestore.stakes.merge((stake_map, stake_history)), - votestore.votes.merge(vote_map), + votestore.votes.merge((vote_map, epoch_cache)), ) { (Ok(()), Ok(())) => LeaderScheduleResult::End(schedule_data), _ => { @@ -302,8 +310,7 @@ pub fn calculate_leader_schedule( seed[0..8].copy_from_slice(&epoch.to_le_bytes()); sort_stakes(&mut stakes); log::info!("calculate_leader_schedule stakes:{stakes:?} epoch:{epoch}"); - let schedule = LeaderSchedule::new(&stakes, seed, slots_in_epoch, NUM_CONSECUTIVE_LEADER_SLOTS); - schedule + LeaderSchedule::new(&stakes, seed, slots_in_epoch, NUM_CONSECUTIVE_LEADER_SLOTS) } // Cribbed from leader_schedule_utils diff --git a/stake_vote/src/lib.rs b/stake_vote/src/lib.rs index 296b2a55..6ca1654c 100644 --- a/stake_vote/src/lib.rs +++ b/stake_vote/src/lib.rs @@ -1,7 +1,6 @@ use crate::account::AccountPretty; use crate::bootstrap::BootstrapEvent; use crate::leader_schedule::LeaderScheduleGeneratedData; -use crate::utils::{Takable, TakeResult}; use futures::Stream; use futures_util::stream::FuturesUnordered; use futures_util::StreamExt; @@ -26,6 +25,7 @@ mod account; mod bootstrap; mod epoch; mod leader_schedule; +mod rpcrequest; mod stake; mod utils; mod vote; @@ -62,8 +62,6 @@ pub async fn start_stakes_and_votes_loop( //future execution collection. let mut spawned_leader_schedule_task = FuturesUnordered::new(); let mut spawned_bootstrap_task = FuturesUnordered::new(); - let mut rpc_notify_task = FuturesUnordered::new(); - let mut rpc_exec_task = FuturesUnordered::new(); let jh = tokio::spawn(async move { BootstrapEvent::InitBootstrap { sleep_time: 1, @@ -72,8 +70,9 @@ pub async fn start_stakes_and_votes_loop( }); spawned_bootstrap_task.push(jh); + let mut rpc_request_processor = crate::rpcrequest::RpcRequestData::new(); + let mut bootstrap_done = false; - let mut pending_rpc_request = vec![]; loop { tokio::select! { @@ -94,82 +93,21 @@ pub async fn start_stakes_and_votes_loop( } } Some((config, return_channel)) = vote_account_rpc_request.recv() => { - pending_rpc_request.push(return_channel); let current_slot = crate::utils::get_current_confirmed_slot(&data_cache).await; - let vote_accounts = votestore.vote_stakes_for_epoch(0); //TODO define epoch storage. - match votestore.votes.take() { - TakeResult::Map(votes) => { - let jh = tokio::task::spawn_blocking({ - move || { - let rpc_vote_accounts = crate::vote::get_rpc_vote_accounts_info( - current_slot, - &votes, - &vote_accounts.as_ref().unwrap().vote_stakes, //TODO put in take. - config, - ); - (votes, vote_accounts, rpc_vote_accounts) - } - }); - rpc_exec_task.push(jh); - } - TakeResult::Taken(mut stake_notify) => { - let notif_jh = tokio::spawn({ - async move { - stake_notify.pop().unwrap().notified().await; - (current_slot, vote_accounts, config) - } - }); - rpc_notify_task.push(notif_jh); - } - } + rpc_request_processor.process_get_vote_accounts(current_slot, config, return_channel, &mut votestore).await; } //manage rpc waiting request notification. - Some(Ok((votes, vote_accounts, rpc_vote_accounts))) = rpc_exec_task.next() => { - if let Err(err) = votestore.votes.merge(votes) { - log::info!("Error during RPC get vote account merge:{err}"); - } - - //avoid clone on the first request - //TODO change the logic use take less one. - if pending_rpc_request.len() == 1 { - if let Err(_) = pending_rpc_request.pop().unwrap().send(rpc_vote_accounts.clone()) { - log::error!("Vote accounts RPC channel send closed."); - } - } else { - for return_channel in pending_rpc_request.drain(..) { - if let Err(_) = return_channel.send(rpc_vote_accounts.clone()) { - log::error!("Vote accounts RPC channel send closed."); - } - } - } + Some(Ok((votes, vote_accounts, rpc_vote_accounts))) = rpc_request_processor.rpc_exec_task.next() => { + rpc_request_processor.notify_end_rpc_get_vote_accounts( + votes, + vote_accounts, + rpc_vote_accounts, + &mut votestore, + ).await; } //manage rpc waiting request notification. - Some(Ok((current_slot, vote_accounts, config))) = rpc_notify_task.next() => { - match votestore.votes.take() { - TakeResult::Map(votes) => { - let jh = tokio::task::spawn_blocking({ - move || { - let rpc_vote_accounts = crate::vote::get_rpc_vote_accounts_info( - current_slot, - &votes, - &vote_accounts.as_ref().unwrap().vote_stakes, //TODO put in take. - config, - ); - (votes, vote_accounts, rpc_vote_accounts) - } - }); - rpc_exec_task.push(jh); - } - TakeResult::Taken(mut stake_notify) => { - let notif_jh = tokio::spawn({ - async move { - stake_notify.pop().unwrap().notified().await; - (current_slot, vote_accounts, config) - } - }); - rpc_notify_task.push(notif_jh); - } - } + Some(Ok((current_slot, config))) = rpc_request_processor.rpc_notify_task.next() => { + rpc_request_processor.take_vote_accounts_and_process(&mut votestore, current_slot, config).await; } //manage geyser account notification //Geyser delete account notification patch must be installed on the validator. @@ -274,7 +212,6 @@ pub async fn start_stakes_and_votes_loop( let data_schedule = Arc::make_mut(&mut data_cache.leader_schedule); data_schedule.current = data_schedule.next.take(); match new_leader_schedule { - //TODO use vote_stakes for vote accounts RPC call. Some(schedule_data) => { let new_schedule_data = LeaderScheduleData{ schedule_by_node: LeaderScheduleGeneratedData::get_schedule_by_nodes(&schedule_data.schedule), diff --git a/stake_vote/src/rpcrequest.rs b/stake_vote/src/rpcrequest.rs new file mode 100644 index 00000000..cc752e5e --- /dev/null +++ b/stake_vote/src/rpcrequest.rs @@ -0,0 +1,107 @@ +use crate::utils::wait_for_merge_or_get_content; +use crate::utils::Takable; +use crate::vote::EpochVoteStakesCache; +use crate::vote::VoteMap; +use crate::vote::VoteStore; +use crate::Slot; +use futures_util::stream::FuturesUnordered; +use solana_lite_rpc_core::structures::leaderschedule::GetVoteAccountsConfig; +use solana_rpc_client_api::response::RpcVoteAccountStatus; +use tokio::sync::oneshot; +use tokio::task::JoinHandle; + +pub struct RpcRequestData { + pub rpc_notify_task: FuturesUnordered>, + pub rpc_exec_task: + FuturesUnordered>, + pending_rpc_request: Option>>, +} + +impl RpcRequestData { + pub fn new() -> Self { + RpcRequestData { + rpc_notify_task: FuturesUnordered::new(), + rpc_exec_task: FuturesUnordered::new(), + pending_rpc_request: None, + } + } + + pub async fn process_get_vote_accounts( + &mut self, + current_slot: Slot, + config: GetVoteAccountsConfig, + return_channel: oneshot::Sender, + votestore: &mut VoteStore, + ) { + match self.pending_rpc_request { + Some(ref mut pending) => pending.push(return_channel), + None => { + self.pending_rpc_request = Some(vec![return_channel]); + } + } + self.take_vote_accounts_and_process(votestore, current_slot, config) + .await; + } + pub async fn notify_end_rpc_get_vote_accounts( + &mut self, + votes: VoteMap, + vote_accounts: EpochVoteStakesCache, + rpc_vote_accounts: RpcVoteAccountStatus, + votestore: &mut VoteStore, + ) { + if let Err(err) = votestore.votes.merge((votes, vote_accounts)) { + log::info!("Error during RPC get vote account merge:{err}"); + } + + //avoid clone on the first request + if let Some(ref mut pending_rpc_request) = self.pending_rpc_request { + for return_channel in pending_rpc_request.drain(0..pending_rpc_request.len() - 1) { + if return_channel.send(rpc_vote_accounts.clone()).is_err() { + log::error!("Vote accounts RPC channel send closed."); + } + } + if pending_rpc_request + .pop() + .unwrap() + .send(rpc_vote_accounts) + .is_err() + { + log::error!("Vote accounts RPC channel send closed."); + } + } + } + + pub async fn take_vote_accounts_and_process( + &mut self, + votestore: &mut VoteStore, + current_slot: Slot, + config: GetVoteAccountsConfig, + ) { + if let Some(((votes, vote_accounts), (current_slot, config))) = + wait_for_merge_or_get_content( + &mut votestore.votes, + (current_slot, config), + &mut self.rpc_notify_task, + ) + .await + { + let jh = tokio::task::spawn_blocking({ + move || { + let rpc_vote_accounts = crate::vote::get_rpc_vote_accounts_info( + current_slot, + &votes, + //TODO manage missing epoch and return error. + &vote_accounts + .vote_stakes_for_epoch(0) + .as_ref() + .unwrap() + .vote_stakes, + config, + ); + (votes, vote_accounts, rpc_vote_accounts) + } + }); + self.rpc_exec_task.push(jh); + } + } +} diff --git a/stake_vote/src/stake.rs b/stake_vote/src/stake.rs index 9ca93e53..2846221c 100644 --- a/stake_vote/src/stake.rs +++ b/stake_vote/src/stake.rs @@ -1,7 +1,5 @@ -use crate::utils::Takable; use crate::utils::TakableContent; use crate::utils::TakableMap; -use crate::utils::TakeResult; use crate::utils::UpdateAction; use crate::AccountPretty; use crate::Slot; diff --git a/stake_vote/src/utils.rs b/stake_vote/src/utils.rs index 7bceeb3d..ea67253e 100644 --- a/stake_vote/src/utils.rs +++ b/stake_vote/src/utils.rs @@ -1,6 +1,8 @@ use crate::vote::StoredVote; use crate::Slot; use anyhow::bail; +use futures_util::future::join_all; +use futures_util::stream::FuturesUnordered; use serde::{Deserialize, Serialize}; use solana_lite_rpc_core::stores::block_information_store::BlockInformation; use solana_lite_rpc_core::stores::data_cache::DataCache; @@ -14,6 +16,7 @@ use std::io::Write; use std::str::FromStr; use std::sync::Arc; use tokio::sync::Notify; +use tokio::task::JoinHandle; pub async fn get_current_confirmed_slot(data_cache: &DataCache) -> u64 { let commitment = CommitmentConfig::confirmed(); @@ -191,6 +194,55 @@ impl<'a, T1, T2, C1: TakableContent, C2: TakableContent> Takable<(C1, C2 } } +// pub async fn take_or_wait(takeMap: impl Takable, f: F) -> Option +// where +// F: FnOnce(()) -> Fut + std::marker::Send + 'static, +// Fut: Future + std::marker::Send, +// Notif: std::marker::Send + 'static, +// { +// match takeMap.take() { +// TakeResult::Map(content) => Some(content), +// TakeResult::Taken(stake_notify) => { +// let notif_jh = tokio::spawn({ +// async move { +// let notifs = stake_notify +// .iter() +// .map(|n| n.notified()) +// .collect::>(); +// join_all(notifs).await; +// f(()).await +// } +// }); +// // waiter_futures.push(notif_jh); +// None +// } +// } +// } + +pub async fn wait_for_merge_or_get_content( + take_map: impl Takable, + notify_content: NotifyContent, + waiter_futures: &mut FuturesUnordered>, +) -> Option<(C, NotifyContent)> { + match take_map.take() { + TakeResult::Map(content) => Some((content, notify_content)), + TakeResult::Taken(stake_notify) => { + let notif_jh = tokio::spawn({ + async move { + let notifs = stake_notify + .iter() + .map(|n| n.notified()) + .collect::>(); + join_all(notifs).await; + notify_content + } + }); + waiter_futures.push(notif_jh); + None + } + } +} + ///A struct that hold a collection call content that can be taken during some time and merged after. ///During the time the content is taken, new added values are cached and added to the content after the merge. ///It allow to process struct content while allowing to still update it without lock. @@ -224,6 +276,30 @@ impl + Default> TakableMap { } } } + + // pub async fn wait_for_merge_or_get_content( + // &mut self, + // notify_content: Notify, + // waiter_futures: &mut FuturesUnordered>, + // ) -> Option<(C, Notify)> { + // match self.take() { + // TakeResult::Map(content) => Some((content, notify_content)), + // TakeResult::Taken(stake_notify) => { + // let notif_jh = tokio::spawn({ + // async move { + // let notifs = stake_notify + // .iter() + // .map(|n| n.notified()) + // .collect::>(); + // join_all(notifs).await; + // notify_content + // } + // }); + // waiter_futures.push(notif_jh); + // None + // } + // } + // } } // pub fn take + Default>( @@ -262,7 +338,6 @@ mod tests { match val { UpdateAction::Notify(account, _) => self.push(account), UpdateAction::Remove(_, _) => (), - UpdateAction::None => (), } } } @@ -289,7 +364,7 @@ mod tests { TakeResult::Taken(_) => panic!("not a content"), TakeResult::Map(content) => content, }; - takable.merge(content); + takable.merge(content).unwrap(); assert_eq!(takable.content.as_ref().unwrap().len(), 3); assert_eq!(takable.updates.len(), 0); diff --git a/stake_vote/src/vote.rs b/stake_vote/src/vote.rs index 4d47db37..00a5a543 100644 --- a/stake_vote/src/vote.rs +++ b/stake_vote/src/vote.rs @@ -16,6 +16,7 @@ use std::collections::HashMap; use std::sync::Arc; pub type VoteMap = HashMap>; +pub type VoteContent = (VoteMap, EpochVoteStakesCache); #[derive(Debug, Clone)] pub struct EpochVoteStakes { @@ -23,9 +24,27 @@ pub struct EpochVoteStakes { pub epoch: u64, } -impl TakableContent for VoteMap { +//TODO define the cache invalidation. +#[derive(Default)] +pub struct EpochVoteStakesCache { + pub cache: HashMap, +} + +impl EpochVoteStakesCache { + pub fn vote_stakes_for_epoch(&self, epoch: u64) -> Option { + self.cache.get(&epoch).cloned() + } + + pub fn add_stakes_for_epoch(&mut self, vote_stakes: EpochVoteStakes) { + if self.cache.insert(vote_stakes.epoch, vote_stakes).is_some() { + log::warn!("Override existing vote stake epoch cache for epoch:"); + } + } +} + +impl TakableContent for VoteContent { fn add_value(&mut self, val: UpdateAction) { - VoteStore::process_vote_action(self, val); + VoteStore::process_vote_action(&mut self.0, val); } } @@ -66,26 +85,19 @@ impl StoredVote { #[derive(Default)] pub struct VoteStore { - pub votes: TakableMap, - epoch_vote_stake_map: HashMap, + pub votes: TakableMap, } impl VoteStore { pub fn new(capacity: usize) -> Self { VoteStore { - votes: TakableMap::new(HashMap::with_capacity(capacity)), - epoch_vote_stake_map: HashMap::new(), + votes: TakableMap::new(( + HashMap::with_capacity(capacity), + EpochVoteStakesCache::default(), + )), } } - pub fn add_epoch_vote_stake(&mut self, stakes: EpochVoteStakes) { - self.epoch_vote_stake_map.insert(stakes.epoch, stakes); - } - - pub fn vote_stakes_for_epoch(&self, epoch: u64) -> Option { - self.epoch_vote_stake_map.get(&epoch).cloned() - } - pub fn notify_vote_change( &mut self, new_account: AccountPretty, From 69754a019a0acc739e326a2e39a214ca19f41144 Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 6 Nov 2023 13:35:58 +0100 Subject: [PATCH 14/56] start get_leader_slot, update for get_vote_account --- core/src/solana_utils.rs | 12 ++++++++ core/src/structures/leaderschedule.rs | 8 +++--- lite-rpc/src/bridge.rs | 23 +++++++++++++-- lite-rpc/src/main.rs | 5 +++- stake_vote/src/leader_schedule.rs | 9 ++++++ stake_vote/src/lib.rs | 41 +++++++++------------------ stake_vote/src/utils.rs | 9 ------ 7 files changed, 64 insertions(+), 43 deletions(-) diff --git a/core/src/solana_utils.rs b/core/src/solana_utils.rs index 8b658231..8449e985 100644 --- a/core/src/solana_utils.rs +++ b/core/src/solana_utils.rs @@ -1,3 +1,6 @@ +use crate::stores::data_cache::DataCache; +use solana_sdk::commitment_config::CommitmentConfig; +use crate::stores::block_information_store::BlockInformation; use serde::Serialize; use solana_sdk::hash::Hash; use solana_sdk::signature::Signature; @@ -32,3 +35,12 @@ impl SerializableTransaction for VersionedTransaction { self.uses_durable_nonce() } } + +pub async fn get_current_confirmed_slot(data_cache: &DataCache) -> u64 { + let commitment = CommitmentConfig::confirmed(); + let BlockInformation { slot, .. } = data_cache + .block_information_store + .get_latest_block(commitment) + .await; + slot +} diff --git a/core/src/structures/leaderschedule.rs b/core/src/structures/leaderschedule.rs index 20bff03a..2d55f7d2 100644 --- a/core/src/structures/leaderschedule.rs +++ b/core/src/structures/leaderschedule.rs @@ -7,7 +7,7 @@ use solana_sdk::pubkey::Pubkey; use std::collections::HashMap; use std::str::FromStr; -#[derive(Clone)] +#[derive(Clone, Default)] pub struct GetVoteAccountsConfig { pub vote_pubkey: Option, pub commitment: Option, @@ -33,7 +33,7 @@ impl TryFrom for GetVoteAccountsConfig { } } -#[derive(Clone, Default)] +#[derive(Clone, Default, Debug)] pub struct CalculatedSchedule { pub current: Option, pub next: Option, @@ -46,7 +46,7 @@ impl CalculatedSchedule { commitment: Option, data_cache: &DataCache, ) -> Option>> { - let commitment = commitment.unwrap_or_else(CommitmentConfig::confirmed); + let commitment = commitment.unwrap_or_else(CommitmentConfig::default); let slot = match slot { Some(slot) => slot, None => { @@ -68,7 +68,7 @@ impl CalculatedSchedule { } } -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct LeaderScheduleData { pub schedule_by_node: HashMap>, pub schedule_by_slot: Vec, diff --git a/lite-rpc/src/bridge.rs b/lite-rpc/src/bridge.rs index 01ff2a9e..985a3a74 100644 --- a/lite-rpc/src/bridge.rs +++ b/lite-rpc/src/bridge.rs @@ -3,6 +3,7 @@ use crate::{ jsonrpsee_subscrption_handler_sink::JsonRpseeSubscriptionHandlerSink, rpc::LiteRpcServer, }; +use solana_lite_rpc_core::structures::leaderschedule::GetVoteAccountsConfig; use solana_rpc_client_api::config::RpcGetVoteAccountsConfig; use solana_rpc_client_api::response::RpcVoteAccountStatus; use solana_sdk::epoch_info::EpochInfo; @@ -40,6 +41,7 @@ use solana_sdk::{commitment_config::CommitmentConfig, pubkey::Pubkey, slot_histo use solana_transaction_status::{TransactionStatus, UiConfirmedBlock}; use std::{str::FromStr, sync::Arc}; use tokio::net::ToSocketAddrs; +use tokio::sync::oneshot; lazy_static::lazy_static! { static ref RPC_SEND_TX: IntCounter = @@ -65,6 +67,10 @@ pub struct LiteBridge { rpc_client: Arc, transaction_service: TransactionService, history: History, + state_vote_sendder: tokio::sync::mpsc::Sender<( + GetVoteAccountsConfig, + tokio::sync::oneshot::Sender, + )>, } impl LiteBridge { @@ -73,12 +79,17 @@ impl LiteBridge { data_cache: DataCache, transaction_service: TransactionService, history: History, + state_vote_sendder: tokio::sync::mpsc::Sender<( + GetVoteAccountsConfig, + oneshot::Sender, + )>, ) -> Self { Self { rpc_client, data_cache, transaction_service, history, + state_vote_sendder, } } @@ -500,7 +511,15 @@ impl LiteRpcServer for LiteBridge { config: Option, ) -> crate::rpc::Result { let config: GetVoteAccountsConfig = - GetVoteAccountsConfig::try_from(config.unwrap_or_default())?; - todo!(); + GetVoteAccountsConfig::try_from(config.unwrap_or_default()).unwrap_or_default(); + let (tx, rx) = oneshot::channel(); + if let Err(err) = self.state_vote_sendder.send((config, tx)).await { + return Err(jsonrpsee::core::Error::Custom(format!( + "error during query processing:{err}", + ))); + } + rx.await.map_err(|err| { + jsonrpsee::core::Error::Custom(format!("error during query processing:{err}")) + }) } } diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index af9ca54e..8ad16905 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -198,7 +198,7 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: prometheus_addr, data_cache: data_cache.clone(), }; - let leader_schedule = Arc::new(JsonRpcLeaderGetter::new(rpc_client.clone(), 1024, 128)); + let leader_schedule = Arc::new(JsonRpcLeaderGetter::new(rpc_client.clone(), 10, 128)); let tpu_service: TpuService = TpuService::new( tpu_config, @@ -221,9 +221,11 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: ); //start stake vote and leader schedule. + let (rpc_stakes_send, rpc_stakes_recv) = mpsc::channel(1000); let stake_vote_jh = solana_lite_rpc_stakevote::start_stakes_and_votes_loop( data_cache.clone(), slot_notifier.resubscribe(), + rpc_stakes_recv, Arc::clone(&rpc_client), grpc_addr, ) @@ -243,6 +245,7 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: data_cache.clone(), transaction_service, history, + rpc_stakes_send, ) .start(lite_rpc_http_addr, lite_rpc_ws_addr), ); diff --git a/stake_vote/src/leader_schedule.rs b/stake_vote/src/leader_schedule.rs index ca78f83b..704090e4 100644 --- a/stake_vote/src/leader_schedule.rs +++ b/stake_vote/src/leader_schedule.rs @@ -1,3 +1,4 @@ +use solana_lite_rpc_core::structures::leaderschedule::LeaderScheduleData; use crate::stake::{StakeMap, StakeStore}; use crate::utils::{Takable, TakeResult}; use crate::vote::EpochVoteStakes; @@ -20,6 +21,7 @@ use tokio::task::JoinHandle; #[derive(Debug)] pub struct LeaderScheduleGeneratedData { pub schedule: LeaderSchedule, + pub rpc_data: LeaderScheduleData, pub epoch: u64, } @@ -162,12 +164,19 @@ fn process_leadershedule_event( log::info!("End calculate leader schedule"); + let rpc_data = LeaderScheduleData{ + schedule_by_node: LeaderScheduleGeneratedData::get_schedule_by_nodes(&leader_schedule), + schedule_by_slot: leader_schedule.get_slot_leaders().to_vec(), + epoch: next_epoch, + }; + LeaderScheduleEvent::MergeStoreAndSaveSchedule( stake_map, vote_map, epoch_cache, LeaderScheduleGeneratedData { schedule: leader_schedule, + rpc_data, epoch: next_epoch, }, (new_epoch, slots_in_epoch, new_rate_activation_epoch), diff --git a/stake_vote/src/lib.rs b/stake_vote/src/lib.rs index 6ca1654c..c5343af4 100644 --- a/stake_vote/src/lib.rs +++ b/stake_vote/src/lib.rs @@ -1,12 +1,10 @@ use crate::account::AccountPretty; use crate::bootstrap::BootstrapEvent; -use crate::leader_schedule::LeaderScheduleGeneratedData; use futures::Stream; use futures_util::stream::FuturesUnordered; use futures_util::StreamExt; use solana_lite_rpc_core::stores::data_cache::DataCache; use solana_lite_rpc_core::structures::leaderschedule::GetVoteAccountsConfig; -use solana_lite_rpc_core::structures::leaderschedule::LeaderScheduleData; use solana_lite_rpc_core::types::SlotStream; use solana_rpc_client::nonblocking::rpc_client::RpcClient; use solana_rpc_client_api::response::RpcVoteAccountStatus; @@ -78,8 +76,8 @@ pub async fn start_stakes_and_votes_loop( tokio::select! { //manage confirm new slot notification to detect epoch change. Ok(_) = slot_notification.recv() => { - //log::info!("Stake and Vote receive a slot."); - let new_slot = crate::utils::get_current_confirmed_slot(&data_cache).await; + log::info!("Stake and Vote receive a slot."); + let new_slot = solana_lite_rpc_core::solana_utils::get_current_confirmed_slot(&data_cache).await; let schedule_event = current_schedule_epoch.process_new_confirmed_slot(new_slot, &data_cache).await; if bootstrap_done { if let Some(init_event) = schedule_event { @@ -93,7 +91,7 @@ pub async fn start_stakes_and_votes_loop( } } Some((config, return_channel)) = vote_account_rpc_request.recv() => { - let current_slot = crate::utils::get_current_confirmed_slot(&data_cache).await; + let current_slot = solana_lite_rpc_core::solana_utils::get_current_confirmed_slot(&data_cache).await; rpc_request_processor.process_get_vote_accounts(current_slot, config, return_channel, &mut votestore).await; } //manage rpc waiting request notification. @@ -127,7 +125,7 @@ pub async fn start_stakes_and_votes_loop( // .unwrap_or("no content".to_string()) // ); //store new account stake. - let current_slot = crate::utils::get_current_confirmed_slot(&data_cache).await; + let current_slot = solana_lite_rpc_core::solana_utils::get_current_confirmed_slot(&data_cache).await; if let Some(account) = AccountPretty::new_from_geyzer(account, current_slot) { match account.owner { @@ -181,18 +179,17 @@ pub async fn start_stakes_and_votes_loop( Some(Ok(event)) = spawned_bootstrap_task.next() => { match crate::bootstrap::run_bootstrap_events(event, &mut spawned_bootstrap_task, &mut stakestore, &mut votestore, current_schedule_epoch.slots_in_epoch) { Ok(Some(boot_res))=> { - match boot_res { Ok(current_schedule_data) => { //let data_schedule = Arc::make_mut(&mut data_cache.leader_schedule); - data_cache.leader_schedule = Arc::new(current_schedule_data); - bootstrap_done = true; } Err(err) => { log::warn!("Error during current leader schedule bootstrap from files:{err}") } } + log::info!("Bootstrap done."); + bootstrap_done = true; }, Ok(None) => (), @@ -207,24 +204,14 @@ pub async fn start_stakes_and_votes_loop( &mut stakestore, &mut votestore, ); - //clone old schedule values is there's other use. - //only done once epoch. Avoid to use a Mutex. - let data_schedule = Arc::make_mut(&mut data_cache.leader_schedule); - data_schedule.current = data_schedule.next.take(); - match new_leader_schedule { - Some(schedule_data) => { - let new_schedule_data = LeaderScheduleData{ - schedule_by_node: LeaderScheduleGeneratedData::get_schedule_by_nodes(&schedule_data.schedule), - schedule_by_slot: schedule_data.schedule.get_slot_leaders().to_vec(), - epoch: schedule_data.epoch - }; - data_schedule.next = Some(new_schedule_data); - } - None => { - log::warn!("Error during schedule calculus. No schedule for this epoch."); - data_schedule.next = None; - } - }; + if let Some(new_leader_schedule) = new_leader_schedule { + //clone old schedule values is there's other use. + //only done once epoch. Avoid to use a Mutex. + let data_schedule = Arc::make_mut(&mut data_cache.leader_schedule); + data_schedule.current = data_schedule.next.take(); + data_schedule.next = Some(new_leader_schedule.rpc_data); + } + } } } diff --git a/stake_vote/src/utils.rs b/stake_vote/src/utils.rs index ea67253e..eacd85cb 100644 --- a/stake_vote/src/utils.rs +++ b/stake_vote/src/utils.rs @@ -18,15 +18,6 @@ use std::sync::Arc; use tokio::sync::Notify; use tokio::task::JoinHandle; -pub async fn get_current_confirmed_slot(data_cache: &DataCache) -> u64 { - let commitment = CommitmentConfig::confirmed(); - let BlockInformation { slot, .. } = data_cache - .block_information_store - .get_latest_block(commitment) - .await; - slot -} - pub async fn get_current_epoch(data_cache: &DataCache) -> LiteRpcEpoch { let commitment = CommitmentConfig::confirmed(); data_cache.get_current_epoch(commitment).await From cce787fb05de77503810f168792d7a60f5583f73 Mon Sep 17 00:00:00 2001 From: musitdev Date: Tue, 7 Nov 2023 22:17:31 +0100 Subject: [PATCH 15/56] update with stake history geyser notification --- stake_vote/src/account.rs | 11 ++-- stake_vote/src/bootstrap.rs | 23 ++++---- stake_vote/src/epoch.rs | 56 +++++++++++++++++-- stake_vote/src/leader_schedule.rs | 62 +++++++++------------ stake_vote/src/lib.rs | 24 ++++++++- stake_vote/src/stake.rs | 67 ++--------------------- stake_vote/src/utils.rs | 90 ------------------------------- stake_vote/src/vote.rs | 12 ----- 8 files changed, 119 insertions(+), 226 deletions(-) diff --git a/stake_vote/src/account.rs b/stake_vote/src/account.rs index c099c0df..3e8f4874 100644 --- a/stake_vote/src/account.rs +++ b/stake_vote/src/account.rs @@ -1,7 +1,5 @@ use anyhow::bail; use borsh::BorshDeserialize; -use solana_sdk::account::Account; -use solana_sdk::account::AccountSharedData; use solana_sdk::pubkey::Pubkey; use solana_sdk::stake::state::Delegation; use solana_sdk::stake::state::StakeState; @@ -60,6 +58,10 @@ impl AccountPretty { read_stake_from_account_data(self.data.as_slice()) } + pub fn read_stake_history(&self) -> Option { + read_historystake_from_account(self.data.as_slice()) + } + pub fn read_vote(&self) -> anyhow::Result { if self.data.is_empty() { log::warn!("Vote account with empty data. Can't read vote."); @@ -92,6 +94,7 @@ pub fn read_stake_from_account_data(mut data: &[u8]) -> anyhow::Result Option { - solana_sdk::account::from_account::(&AccountSharedData::from(account)) +pub fn read_historystake_from_account(account_data: &[u8]) -> Option { + //solana_sdk::account::from_account::(&AccountSharedData::from(account)) + bincode::deserialize(account_data).ok() } diff --git a/stake_vote/src/bootstrap.rs b/stake_vote/src/bootstrap.rs index 447a3618..00f813cd 100644 --- a/stake_vote/src/bootstrap.rs +++ b/stake_vote/src/bootstrap.rs @@ -18,7 +18,6 @@ use solana_lite_rpc_core::structures::leaderschedule::LeaderScheduleData; use solana_sdk::account::Account; use solana_sdk::commitment_config::CommitmentConfig; use solana_sdk::pubkey::Pubkey; -use solana_sdk::stake_history::StakeHistory; use std::time::Duration; use tokio::task::JoinHandle; @@ -33,15 +32,15 @@ pub async fn bootstrap_scheduleepoch_data(data_cache: &DataCache) -> ScheduleEpo .new_warmup_cooldown_rate_epoch(data_cache.epoch_data.get_epoch_schedule()); let bootstrap_epoch = crate::utils::get_current_epoch(data_cache).await; - ScheduleEpochData { - current_epoch: bootstrap_epoch.epoch, - slots_in_epoch: bootstrap_epoch.slots_in_epoch, - last_slot_in_epoch: data_cache + ScheduleEpochData::new( + bootstrap_epoch.epoch, + bootstrap_epoch.slots_in_epoch, + data_cache .epoch_data .get_last_slot_in_epoch(bootstrap_epoch.epoch), - current_confirmed_slot: bootstrap_epoch.absolute_slot, + bootstrap_epoch.absolute_slot, new_rate_activation_epoch, - } + ) } /* @@ -118,7 +117,6 @@ pub enum BootstrapEvent { ), AccountsMerged( StakeMap, - Option, VoteMap, EpochVoteStakesCache, String, @@ -168,7 +166,7 @@ fn process_bootstrap_event( BootstrapEvent::BootstrapAccountsFetched(stakes, votes, history, rpc_url) => { log::info!("BootstrapEvent::BootstrapAccountsFetched RECV"); match (&mut stakestore.stakes, &mut votestore.votes).take() { - TakeResult::Map(((stake_map, _), (vote_map, epoch_cache))) => { + TakeResult::Map((stake_map, (vote_map, epoch_cache))) => { BootsrapProcessResult::Event(BootstrapEvent::StoreExtracted( stake_map, vote_map, @@ -208,7 +206,7 @@ fn process_bootstrap_event( ) => { log::info!("BootstrapEvent::StoreExtracted RECV"); - let stake_history = crate::account::read_historystake_from_account(history); + let stake_history = crate::account::read_historystake_from_account(&history.data); if stake_history.is_none() { return BootsrapProcessResult::Error( "Bootstrap error, can't read stake history from account data.".to_string(), @@ -236,7 +234,6 @@ fn process_bootstrap_event( epoch_cache.add_stakes_for_epoch(next_epoch_stakes); BootstrapEvent::AccountsMerged( stake_map, - stake_history, vote_map, epoch_cache, rpc_url, @@ -245,7 +242,6 @@ fn process_bootstrap_event( } Err(err) => BootstrapEvent::AccountsMerged( stake_map, - stake_history, vote_map, epoch_cache, rpc_url, @@ -258,7 +254,6 @@ fn process_bootstrap_event( } BootstrapEvent::AccountsMerged( stake_map, - stake_history, vote_map, epoch_cache, rpc_url, @@ -267,7 +262,7 @@ fn process_bootstrap_event( log::info!("BootstrapEvent::AccountsMerged RECV"); match ( - stakestore.stakes.merge((stake_map, stake_history)), + stakestore.stakes.merge(stake_map), votestore.votes.merge((vote_map, epoch_cache)), ) { (Ok(()), Ok(())) => BootsrapProcessResult::End(leader_schedule_result), diff --git a/stake_vote/src/epoch.rs b/stake_vote/src/epoch.rs index ab3622d0..1bc57ab9 100644 --- a/stake_vote/src/epoch.rs +++ b/stake_vote/src/epoch.rs @@ -1,17 +1,43 @@ use crate::leader_schedule::LeaderScheduleEvent; use serde::{Deserialize, Serialize}; use solana_lite_rpc_core::stores::data_cache::DataCache; +use solana_sdk::stake_history::StakeHistory; -#[derive(Debug, Default, Copy, Clone, PartialOrd, PartialEq, Eq, Ord, Serialize, Deserialize)] +//#[derive(Debug, Default, Copy, Clone, PartialOrd, PartialEq, Eq, Ord, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] pub struct ScheduleEpochData { pub current_epoch: u64, pub slots_in_epoch: u64, pub last_slot_in_epoch: u64, pub current_confirmed_slot: u64, pub new_rate_activation_epoch: Option, + //to start a new epoch and schedule, the new stake history + //Must be notified and the end epoch slot notfied. + //these field store each event. + //If they're defined an new epoch and leader schedule can append. + new_stake_history: Option, + next_epoch_change: Option<(u64, u64)>, } impl ScheduleEpochData { + pub fn new( + current_epoch: u64, + slots_in_epoch: u64, + last_slot_in_epoch: u64, + current_confirmed_slot: u64, + new_rate_activation_epoch: Option, + ) -> Self { + ScheduleEpochData { + current_epoch, + slots_in_epoch, + last_slot_in_epoch, + current_confirmed_slot, + new_rate_activation_epoch, + new_stake_history: None, + next_epoch_change: None, + } + } + pub async fn process_new_confirmed_slot( &mut self, new_slot: u64, @@ -26,6 +52,14 @@ impl ScheduleEpochData { } } + pub fn set_epoch_stake_history( + &mut self, + history: StakeHistory, + ) -> Option { + self.new_stake_history = Some(history); + self.verify_epoch_change() + } + async fn manage_change_epoch(&mut self, data_cache: &DataCache) -> Option { //execute leaderschedule calculus at the last slot of the current epoch. //account change of the slot has been send at confirmed slot. @@ -37,17 +71,33 @@ impl ScheduleEpochData { let next_epoch = data_cache .epoch_data .get_epoch_at_slot(self.last_slot_in_epoch + 1); - self.current_epoch = next_epoch.epoch; - self.last_slot_in_epoch = data_cache + let last_slot_in_epoch = data_cache .epoch_data .get_last_slot_in_epoch(next_epoch.epoch); + //start leader schedule calculus + //at current epoch change the schedule is calculated for the next epoch. + self.next_epoch_change = Some((next_epoch.epoch, last_slot_in_epoch)); + self.verify_epoch_change() + } else { + None + } + } + + fn verify_epoch_change(&mut self) -> Option { + if self.new_stake_history.is_some() && self.next_epoch_change.is_some() { + log::info!("Change epoch at slot:{}", self.current_confirmed_slot); + let (next_epoch, last_slot_in_epoch) = self.next_epoch_change.take().unwrap(); //unwrap tested before. + self.current_epoch = next_epoch; + self.last_slot_in_epoch = last_slot_in_epoch; + //start leader schedule calculus //at current epoch change the schedule is calculated for the next epoch. Some(crate::leader_schedule::LeaderScheduleEvent::Init( self.current_epoch, self.slots_in_epoch, self.new_rate_activation_epoch, + self.new_stake_history.take().unwrap(), //unwrap tested before )) } else { None diff --git a/stake_vote/src/leader_schedule.rs b/stake_vote/src/leader_schedule.rs index 704090e4..3fe756dc 100644 --- a/stake_vote/src/leader_schedule.rs +++ b/stake_vote/src/leader_schedule.rs @@ -1,4 +1,3 @@ -use solana_lite_rpc_core::structures::leaderschedule::LeaderScheduleData; use crate::stake::{StakeMap, StakeStore}; use crate::utils::{Takable, TakeResult}; use crate::vote::EpochVoteStakes; @@ -10,9 +9,9 @@ use futures::stream::FuturesUnordered; use itertools::Itertools; use serde::{Deserialize, Serialize}; use solana_ledger::leader_schedule::LeaderSchedule; +use solana_lite_rpc_core::structures::leaderschedule::LeaderScheduleData; use solana_sdk::clock::NUM_CONSECUTIVE_LEADER_SLOTS; use solana_sdk::pubkey::Pubkey; -use solana_sdk::stake::state::StakeActivationStatus; use solana_sdk::stake_history::StakeHistory; use std::collections::HashMap; use std::sync::Arc; @@ -66,14 +65,14 @@ InitLeaderscedule MergeStore(stakes, votes, schedule) #[allow(clippy::large_enum_variant)] //256 byte large and only use during schedule calculus. pub enum LeaderScheduleEvent { - Init(u64, u64, Option), + Init(u64, u64, Option, StakeHistory), MergeStoreAndSaveSchedule( StakeMap, VoteMap, EpochVoteStakesCache, LeaderScheduleGeneratedData, (u64, u64, Option), - Option, + StakeHistory, ), } @@ -110,9 +109,14 @@ fn process_leadershedule_event( votestore: &mut VoteStore, ) -> LeaderScheduleResult { match event { - LeaderScheduleEvent::Init(new_epoch, slots_in_epoch, new_rate_activation_epoch) => { + LeaderScheduleEvent::Init( + new_epoch, + slots_in_epoch, + new_rate_activation_epoch, + stake_history, + ) => { match (&mut stakestore.stakes, &mut votestore.votes).take() { - TakeResult::Map(((stake_map, mut stake_history), (vote_map, mut epoch_cache))) => { + TakeResult::Map((stake_map, (vote_map, mut epoch_cache))) => { log::info!("LeaderScheduleEvent::CalculateScedule"); //do the calculus in a blocking task. let jh = tokio::task::spawn_blocking({ @@ -121,7 +125,7 @@ fn process_leadershedule_event( &stake_map, &vote_map, new_epoch, - stake_history.as_mut(), + &stake_history, new_rate_activation_epoch, ); @@ -164,10 +168,13 @@ fn process_leadershedule_event( log::info!("End calculate leader schedule"); - let rpc_data = LeaderScheduleData{ - schedule_by_node: LeaderScheduleGeneratedData::get_schedule_by_nodes(&leader_schedule), - schedule_by_slot: leader_schedule.get_slot_leaders().to_vec(), - epoch: next_epoch, + let rpc_data = LeaderScheduleData { + schedule_by_node: + LeaderScheduleGeneratedData::get_schedule_by_nodes( + &leader_schedule, + ), + schedule_by_slot: leader_schedule.get_slot_leaders().to_vec(), + epoch: next_epoch, }; LeaderScheduleEvent::MergeStoreAndSaveSchedule( @@ -198,6 +205,7 @@ fn process_leadershedule_event( new_epoch, slots_in_epoch, new_rate_activation_epoch, + stake_history, ) } }); @@ -215,7 +223,7 @@ fn process_leadershedule_event( ) => { log::info!("LeaderScheduleEvent::MergeStoreAndSaveSchedule RECV"); match ( - stakestore.stakes.merge((stake_map, stake_history)), + stakestore.stakes.merge(stake_map), votestore.votes.merge((vote_map, epoch_cache)), ) { (Ok(()), Ok(())) => LeaderScheduleResult::End(schedule_data), @@ -227,6 +235,7 @@ fn process_leadershedule_event( new_epoch, slots_in_epoch, epoch_schedule, + stake_history, )) } } @@ -238,29 +247,9 @@ fn calculate_epoch_stakes( stake_map: &StakeMap, vote_map: &VoteMap, new_epoch: u64, - mut stake_history: Option<&mut StakeHistory>, + stake_history: &StakeHistory, new_rate_activation_epoch: Option, ) -> HashMap)> { - //code taken from Solana code: runtime::stakes::activate_epoch function - //update stake history with current end epoch stake values. - //stake history is added for the ended epoch using all stakes at the end of the epoch. - let ended_epoch = new_epoch - 1; - let stake_history_entry = - stake_map - .values() - .fold(StakeActivationStatus::default(), |acc, stake_account| { - let delegation = stake_account.stake; - acc + delegation.stake_activating_and_deactivating( - ended_epoch, - stake_history.as_deref(), - new_rate_activation_epoch, - ) - }); - match stake_history { - Some(ref mut stake_history) => stake_history.add(ended_epoch, stake_history_entry), - None => log::warn!("Vote stake calculus without Stake History"), - }; - //calculate schedule stakes at beginning of new epoch. //Next epoch schedule use the stake at the beginning of last epoch. let delegated_stakes: HashMap = @@ -269,11 +258,8 @@ fn calculate_epoch_stakes( .fold(HashMap::default(), |mut delegated_stakes, stake_account| { let delegation = stake_account.stake; let entry = delegated_stakes.entry(delegation.voter_pubkey).or_default(); - *entry += delegation.stake( - new_epoch, - stake_history.as_deref(), - new_rate_activation_epoch, - ); + *entry += + delegation.stake(new_epoch, Some(stake_history), new_rate_activation_epoch); delegated_stakes }); diff --git a/stake_vote/src/lib.rs b/stake_vote/src/lib.rs index c5343af4..c717860b 100644 --- a/stake_vote/src/lib.rs +++ b/stake_vote/src/lib.rs @@ -148,6 +148,26 @@ pub async fn start_stakes_and_votes_loop( continue; } } + //stake history account must be updated before new leader schedule calculus. + solana_sdk::sysvar::stake_history::ID => { + log::info!("Geyser notifstake_history"); + match account.read_stake_history() { + Some(stake_history) => { + let schedule_event = current_schedule_epoch.set_epoch_stake_history(stake_history); + if bootstrap_done { + if let Some(init_event) = schedule_event { + crate::leader_schedule::run_leader_schedule_events( + init_event, + &mut spawned_leader_schedule_task, + &mut stakestore, + &mut votestore, + ); + } + } + } + None => log::error!("Bootstrap error, can't read stake history from geyzer account data."), + } + } _ => log::warn!("receive an account notification from a unknown owner:{account:?}"), } } @@ -209,7 +229,7 @@ pub async fn start_stakes_and_votes_loop( //only done once epoch. Avoid to use a Mutex. let data_schedule = Arc::make_mut(&mut data_cache.leader_schedule); data_schedule.current = data_schedule.next.take(); - data_schedule.next = Some(new_leader_schedule.rpc_data); + data_schedule.next = Some(new_leader_schedule.rpc_data); } } @@ -233,7 +253,7 @@ async fn subscribe_geyzer( accounts.insert( "client".to_owned(), SubscribeRequestFilterAccounts { - account: vec![], + account: vec![solana_sdk::sysvar::stake_history::ID.to_string()], owner: vec![ solana_sdk::stake::program::ID.to_string(), solana_sdk::vote::program::ID.to_string(), diff --git a/stake_vote/src/stake.rs b/stake_vote/src/stake.rs index 2846221c..359226ba 100644 --- a/stake_vote/src/stake.rs +++ b/stake_vote/src/stake.rs @@ -8,36 +8,9 @@ use serde::{Deserialize, Serialize}; use solana_sdk::account::Account; use solana_sdk::pubkey::Pubkey; use solana_sdk::stake::state::Delegation; -use solana_sdk::stake_history::StakeHistory; use std::collections::HashMap; pub type StakeMap = HashMap; -type StakeContent = (StakeMap, Option); - -// #[derive(Debug, Default)] -// pub enum StakeAction { -// Notify { -// stake: StoredStake, -// }, -// Remove(Pubkey, Slot), -// // Merge { -// // source_account: Pubkey, -// // destination_account: Pubkey, -// // update_slot: Slot, -// // }, -// #[default] -// None, -// } - -// impl StakeAction { -// fn get_update_slot(&self) -> u64 { -// match self { -// StakeAction::Notify { stake } => stake.last_update_slot, -// StakeAction::Remove(_, slot) => *slot, -// StakeAction::None => 0, -// } -// } -// } #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct StoredStake { @@ -48,28 +21,24 @@ pub struct StoredStake { pub write_version: u64, } -impl TakableContent for StakeContent { +impl TakableContent for StakeMap { fn add_value(&mut self, val: UpdateAction) { - StakeStore::process_stake_action(&mut self.0, val); + StakeStore::process_stake_action(self, val); } } #[derive(Debug, Default)] pub struct StakeStore { - pub stakes: TakableMap, + pub stakes: TakableMap, } impl StakeStore { pub fn new(capacity: usize) -> Self { StakeStore { - stakes: TakableMap::new((HashMap::with_capacity(capacity), None)), + stakes: TakableMap::new(HashMap::with_capacity(capacity)), } } - // pub fn get_stake_history(&self) -> Option { - // self.stakes.content.1.clone() - // } - pub fn notify_stake_change( &mut self, account: AccountPretty, @@ -106,19 +75,6 @@ impl StakeStore { Ok(()) } - //helper method to extract and merge stakes. - // pub fn take_stakestore(&mut self) -> TakeResult { - // self.stakes.take() - // } - - // pub fn merge_stakestore( - // &mut self, - // stake_map: StakeMap, - // stake_hisotry: Option, - // ) -> anyhow::Result<()> { - // self.stakes.merge((stake_map, stake_hisotry)) - // } - fn process_stake_action(stakes: &mut StakeMap, action: UpdateAction) { match action { UpdateAction::Notify(_, stake) => { @@ -161,21 +117,6 @@ impl StakeStore { stakes.remove(account_pk); } } - - // //helper method to extract and merge stakes. - // pub fn take_stakestore( - // stakestore: &mut StakeStore, - // ) -> anyhow::Result<(StakeMap, Option)> { - // crate::utils::take(&mut stakestore.stakes) - // } - - // pub fn merge_stakestore( - // stakestore: &mut StakeStore, - // stake_map: StakeMap, - // stake_history: Option, - // ) -> anyhow::Result<()> { - // crate::utils::merge(&mut stakestore.stakes, (stake_map, stake_history)) - // } } pub fn merge_program_account_in_strake_map( diff --git a/stake_vote/src/utils.rs b/stake_vote/src/utils.rs index eacd85cb..94553c85 100644 --- a/stake_vote/src/utils.rs +++ b/stake_vote/src/utils.rs @@ -4,7 +4,6 @@ use anyhow::bail; use futures_util::future::join_all; use futures_util::stream::FuturesUnordered; use serde::{Deserialize, Serialize}; -use solana_lite_rpc_core::stores::block_information_store::BlockInformation; use solana_lite_rpc_core::stores::data_cache::DataCache; use solana_lite_rpc_core::structures::epoch::Epoch as LiteRpcEpoch; use solana_sdk::commitment_config::CommitmentConfig; @@ -75,14 +74,6 @@ pub enum UpdateAction { Remove(Pubkey, Slot), } -// impl UpdateAction { -// pub fn get_update_slot(&self) -> u64 { -// match self { -// UpdateAction::Notify(slot, _) | UpdateAction::Remove(_, slot) => *slot, -// } -// } -// } - pub enum TakeResult { //Vec because can wait on several collection to be merged Taken(Vec>), @@ -102,13 +93,6 @@ impl TakeResult { _ => unreachable!("Bad take result association."), //TODO add mix result. } } - - // pub fn get_content(self) -> Option { - // match self { - // TakeResult::Taken(_) => None, - // TakeResult::Map(content) => Some(content), - // } - // } } //Takable struct code @@ -185,31 +169,6 @@ impl<'a, T1, T2, C1: TakableContent, C2: TakableContent> Takable<(C1, C2 } } -// pub async fn take_or_wait(takeMap: impl Takable, f: F) -> Option -// where -// F: FnOnce(()) -> Fut + std::marker::Send + 'static, -// Fut: Future + std::marker::Send, -// Notif: std::marker::Send + 'static, -// { -// match takeMap.take() { -// TakeResult::Map(content) => Some(content), -// TakeResult::Taken(stake_notify) => { -// let notif_jh = tokio::spawn({ -// async move { -// let notifs = stake_notify -// .iter() -// .map(|n| n.notified()) -// .collect::>(); -// join_all(notifs).await; -// f(()).await -// } -// }); -// // waiter_futures.push(notif_jh); -// None -// } -// } -// } - pub async fn wait_for_merge_or_get_content( take_map: impl Takable, notify_content: NotifyContent, @@ -267,57 +226,8 @@ impl + Default> TakableMap { } } } - - // pub async fn wait_for_merge_or_get_content( - // &mut self, - // notify_content: Notify, - // waiter_futures: &mut FuturesUnordered>, - // ) -> Option<(C, Notify)> { - // match self.take() { - // TakeResult::Map(content) => Some((content, notify_content)), - // TakeResult::Taken(stake_notify) => { - // let notif_jh = tokio::spawn({ - // async move { - // let notifs = stake_notify - // .iter() - // .map(|n| n.notified()) - // .collect::>(); - // join_all(notifs).await; - // notify_content - // } - // }); - // waiter_futures.push(notif_jh); - // None - // } - // } - // } } -// pub fn take + Default>( -// map: &mut TakableMap, -// ) -> anyhow::Result { -// if map.is_taken() { -// bail!("TakableMap already taken. Try later"); -// } -// let new_store = std::mem::take(map); -// let (new_store, content) = new_store.take(); -// *map = new_store; -// Ok(content) -// } - -// pub fn merge + Default>( -// map: &mut TakableMap, -// content: C, -// ) -> anyhow::Result<()> { -// if !map.is_taken() { -// bail!("TakableMap merge of non taken map. Try later"); -// } -// let new_store = std::mem::take(map); -// let new_store = new_store.merge(content); -// *map = new_store; -// Ok(()) -// } - #[cfg(test)] mod tests { use super::*; diff --git a/stake_vote/src/vote.rs b/stake_vote/src/vote.rs index 00a5a543..e119e61c 100644 --- a/stake_vote/src/vote.rs +++ b/stake_vote/src/vote.rs @@ -157,15 +157,6 @@ impl VoteStore { } } - // //helper method to extract and merge stakes. - // pub fn take_votestore(&mut self) -> TakeResult { - // self.votes.take() - // } - - // pub fn merge_votestore(&mut self, vote_map: VoteMap) -> anyhow::Result<()> { - // self.votes.merge(vote_map) - // } - fn remove_from_store(votes: &mut VoteMap, account_pk: &Pubkey, update_slot: Slot) { //TODO use action. if votes @@ -252,9 +243,6 @@ pub fn get_rpc_vote_accounts_info( vote_accounts: &HashMap)>, config: GetVoteAccountsConfig, ) -> RpcVoteAccountStatus { - //TODO - //manage - //From Solana rpc::rpc::metaz::get_vote_accounts() code. let (current_vote_accounts, delinquent_vote_accounts): ( Vec, From b02ee5f45f14ee4d4a2ffdfa37b616f89439f561 Mon Sep 17 00:00:00 2001 From: musitdev Date: Wed, 8 Nov 2023 17:08:47 +0100 Subject: [PATCH 16/56] correct geyser stake history subscription --- core/src/structures/leaderschedule.rs | 40 ++++++++++ stake_vote/src/epoch.rs | 7 +- stake_vote/src/lib.rs | 111 ++++++++++++++++++-------- 3 files changed, 124 insertions(+), 34 deletions(-) diff --git a/core/src/structures/leaderschedule.rs b/core/src/structures/leaderschedule.rs index 2d55f7d2..fafc50fd 100644 --- a/core/src/structures/leaderschedule.rs +++ b/core/src/structures/leaderschedule.rs @@ -66,6 +66,46 @@ impl CalculatedSchedule { }; get_schedule(self.current.as_ref()).or_else(|| get_schedule(self.next.as_ref())) } + + // pub async fn get_slot_leaders(&self, start_slot: Slot, limit: u64) -> Result> { + // debug!( + // "get_slot_leaders rpc request received (start: {} limit: {})", + // start_slot, limit + // ); + + // let limit = limit as usize; + // if limit > MAX_GET_SLOT_LEADERS { + // return Err(Error::invalid_params(format!( + // "Invalid limit; max {MAX_GET_SLOT_LEADERS}" + // ))); + // } + // let bank = self.bank(commitment); + + // let (mut epoch, mut slot_index) = + // bank.epoch_schedule().get_epoch_and_slot_index(start_slot); + + // let mut slot_leaders = Vec::with_capacity(limit); + // while slot_leaders.len() < limit { + // if let Some(leader_schedule) = + // self.leader_schedule_cache.get_epoch_leader_schedule(epoch) + // { + // slot_leaders.extend( + // leader_schedule + // .get_slot_leaders() + // .iter() + // .skip(slot_index as usize) + // .take(limit.saturating_sub(slot_leaders.len())), + // ); + // } else { + // return Err(Error::invalid_params(format!( + // "Invalid slot range: leader schedule for epoch {epoch} is unavailable" + // ))); + // } + + // epoch += 1; + // slot_index = 0; + // } + // } } #[derive(Clone, Debug)] diff --git a/stake_vote/src/epoch.rs b/stake_vote/src/epoch.rs index 1bc57ab9..ac185bfb 100644 --- a/stake_vote/src/epoch.rs +++ b/stake_vote/src/epoch.rs @@ -56,6 +56,7 @@ impl ScheduleEpochData { &mut self, history: StakeHistory, ) -> Option { + log::info!("set_epoch_stake_history"); self.new_stake_history = Some(history); self.verify_epoch_change() } @@ -67,7 +68,11 @@ impl ScheduleEpochData { //to avoid to delay too much the schedule, start the calculus at the end of the epoch. //the first epoch slot arrive very late cause of the stake account notification from the validator. if self.current_confirmed_slot >= self.last_slot_in_epoch { - log::info!("Change epoch at slot:{}", self.current_confirmed_slot); + log::info!( + "manage_change_epoch at slot:{} last_slot_in_epoch:{}", + self.current_confirmed_slot, + self.last_slot_in_epoch + ); let next_epoch = data_cache .epoch_data .get_epoch_at_slot(self.last_slot_in_epoch + 1); diff --git a/stake_vote/src/lib.rs b/stake_vote/src/lib.rs index c717860b..25999cdc 100644 --- a/stake_vote/src/lib.rs +++ b/stake_vote/src/lib.rs @@ -8,6 +8,7 @@ use solana_lite_rpc_core::structures::leaderschedule::GetVoteAccountsConfig; use solana_lite_rpc_core::types::SlotStream; use solana_rpc_client::nonblocking::rpc_client::RpcClient; use solana_rpc_client_api::response::RpcVoteAccountStatus; +use solana_sdk::pubkey::Pubkey; use std::collections::HashMap; use std::sync::Arc; use tokio::sync::mpsc::Receiver; @@ -44,7 +45,8 @@ pub async fn start_stakes_and_votes_loop( grpc_url: String, ) -> anyhow::Result> { log::info!("Start Stake and Vote loop."); - let mut account_gyzer_stream = subscribe_geyzer(grpc_url).await?; + let mut stake_vote_geyzer_stream = subscribe_geyzer_stake_vote_owner(grpc_url.clone()).await?; + let mut stake_history_geyzer_stream = subscribe_geyzer_stake_history(grpc_url).await?; log::info!("Stake and Vote geyzer subscription done."); let jh = tokio::spawn(async move { //Stake account management struct @@ -76,7 +78,7 @@ pub async fn start_stakes_and_votes_loop( tokio::select! { //manage confirm new slot notification to detect epoch change. Ok(_) = slot_notification.recv() => { - log::info!("Stake and Vote receive a slot."); + //log::info!("Stake and Vote receive a slot."); let new_slot = solana_lite_rpc_core::solana_utils::get_current_confirmed_slot(&data_cache).await; let schedule_event = current_schedule_epoch.process_new_confirmed_slot(new_slot, &data_cache).await; if bootstrap_done { @@ -107,10 +109,46 @@ pub async fn start_stakes_and_votes_loop( Some(Ok((current_slot, config))) = rpc_request_processor.rpc_notify_task.next() => { rpc_request_processor.take_vote_accounts_and_process(&mut votestore, current_slot, config).await; } + //manage geyser stake_history notification + ret = stake_history_geyzer_stream.next() => { + match ret { + Some(Ok(msg)) => { + if let Some(UpdateOneof::Account(account)) = msg.update_oneof { + if let Some(account) = account.account { + let acc_id = Pubkey::try_from(account.pubkey).expect("valid pubkey"); + if acc_id == solana_sdk::sysvar::stake_history::ID { + log::info!("Geyser notifstake_history"); + match crate::account::read_historystake_from_account(account.data.as_slice()) { + Some(stake_history) => { + let schedule_event = current_schedule_epoch.set_epoch_stake_history(stake_history); + if bootstrap_done { + if let Some(init_event) = schedule_event { + crate::leader_schedule::run_leader_schedule_events( + init_event, + &mut spawned_leader_schedule_task, + &mut stakestore, + &mut votestore, + ); + } + } + } + None => log::error!("Bootstrap error, can't read stake history from geyzer account data."), + } + } + } + } + }, + None | Some(Err(_)) => { + //TODO Restart geyser connection and the bootstrap. + log::error!("The stake_history geyser stream close or in error try to reconnect and resynchronize."); + break; + } + } + } //manage geyser account notification //Geyser delete account notification patch must be installed on the validator. //see https://github.com/solana-labs/solana/pull/33292 - ret = account_gyzer_stream.next() => { + ret = stake_vote_geyzer_stream.next() => { match ret { Some(message) => { //process the message @@ -148,26 +186,6 @@ pub async fn start_stakes_and_votes_loop( continue; } } - //stake history account must be updated before new leader schedule calculus. - solana_sdk::sysvar::stake_history::ID => { - log::info!("Geyser notifstake_history"); - match account.read_stake_history() { - Some(stake_history) => { - let schedule_event = current_schedule_epoch.set_epoch_stake_history(stake_history); - if bootstrap_done { - if let Some(init_event) = schedule_event { - crate::leader_schedule::run_leader_schedule_events( - init_event, - &mut spawned_leader_schedule_task, - &mut stakestore, - &mut votestore, - ); - } - } - } - None => log::error!("Bootstrap error, can't read stake history from geyzer account data."), - } - } _ => log::warn!("receive an account notification from a unknown owner:{account:?}"), } } @@ -209,6 +227,8 @@ pub async fn start_stakes_and_votes_loop( } } log::info!("Bootstrap done."); + //update current epoch to manage epoch change during bootstrap. + current_schedule_epoch = crate::bootstrap::bootstrap_scheduleepoch_data(&data_cache).await; bootstrap_done = true; }, @@ -240,20 +260,17 @@ pub async fn start_stakes_and_votes_loop( } //subscribe Geyser grpc -async fn subscribe_geyzer( +async fn subscribe_geyzer_stake_vote_owner( grpc_url: String, ) -> anyhow::Result>> { let mut client = GeyserGrpcClient::connect(grpc_url, None::<&'static str>, None)?; - //slot subscription - let mut slots = HashMap::new(); - slots.insert("client".to_string(), SubscribeRequestFilterSlots {}); //account subscription let mut accounts: HashMap = HashMap::new(); accounts.insert( - "client".to_owned(), + "stake_vote".to_owned(), SubscribeRequestFilterAccounts { - account: vec![solana_sdk::sysvar::stake_history::ID.to_string()], + account: vec![], owner: vec![ solana_sdk::stake::program::ID.to_string(), solana_sdk::vote::program::ID.to_string(), @@ -262,14 +279,42 @@ async fn subscribe_geyzer( }, ); - //block Meta subscription filter - let mut blocks_meta = HashMap::new(); - blocks_meta.insert("client".to_string(), SubscribeRequestFilterBlocksMeta {}); + let confirmed_stream = client + .subscribe_once( + Default::default(), //slots + accounts.clone(), //accounts + Default::default(), //tx + Default::default(), //entry + Default::default(), //full block + Default::default(), //block meta + Some(CommitmentLevel::Confirmed), + vec![], + ) + .await?; + + Ok(confirmed_stream) +} + +//subscribe Geyser grpc +async fn subscribe_geyzer_stake_history( + grpc_url: String, +) -> anyhow::Result>> { + let mut client = GeyserGrpcClient::connect(grpc_url, None::<&'static str>, None)?; + + //account subscription + let mut accounts: HashMap = HashMap::new(); + accounts.insert( + "stake_history".to_owned(), + SubscribeRequestFilterAccounts { + account: vec![solana_sdk::sysvar::stake_history::ID.to_string()], + owner: vec![], + filters: vec![], + }, + ); let confirmed_stream = client .subscribe_once( Default::default(), //slots - //slots.clone(), accounts.clone(), //accounts Default::default(), //tx Default::default(), //entry From 32789c9a110e1bcf85ad08fb104a764796f70f29 Mon Sep 17 00:00:00 2001 From: musitdev Date: Wed, 8 Nov 2023 18:08:23 +0100 Subject: [PATCH 17/56] remove stake log --- stake_vote/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stake_vote/src/lib.rs b/stake_vote/src/lib.rs index 25999cdc..1f52c00a 100644 --- a/stake_vote/src/lib.rs +++ b/stake_vote/src/lib.rs @@ -168,7 +168,7 @@ pub async fn start_stakes_and_votes_loop( if let Some(account) = AccountPretty::new_from_geyzer(account, current_slot) { match account.owner { solana_sdk::stake::program::ID => { - log::info!("Geyser notif stake account:{}", account); + log::trace!("Geyser notif stake account:{}", account); if let Err(err) = stakestore.notify_stake_change( account, current_schedule_epoch.last_slot_in_epoch, From 901b76c4e0a65dc8b76f65c2a7146ae71e95d4de Mon Sep 17 00:00:00 2001 From: musitdev Date: Wed, 8 Nov 2023 18:20:31 +0100 Subject: [PATCH 18/56] pass fmt and clippy --- core/src/solana_utils.rs | 4 ++-- stake_vote/src/account.rs | 6 +++--- stake_vote/src/lib.rs | 3 +-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/core/src/solana_utils.rs b/core/src/solana_utils.rs index 8449e985..d94da64d 100644 --- a/core/src/solana_utils.rs +++ b/core/src/solana_utils.rs @@ -1,7 +1,7 @@ -use crate::stores::data_cache::DataCache; -use solana_sdk::commitment_config::CommitmentConfig; use crate::stores::block_information_store::BlockInformation; +use crate::stores::data_cache::DataCache; use serde::Serialize; +use solana_sdk::commitment_config::CommitmentConfig; use solana_sdk::hash::Hash; use solana_sdk::signature::Signature; use solana_sdk::transaction::{uses_durable_nonce, Transaction, VersionedTransaction}; diff --git a/stake_vote/src/account.rs b/stake_vote/src/account.rs index 3e8f4874..dabdb961 100644 --- a/stake_vote/src/account.rs +++ b/stake_vote/src/account.rs @@ -58,9 +58,9 @@ impl AccountPretty { read_stake_from_account_data(self.data.as_slice()) } - pub fn read_stake_history(&self) -> Option { - read_historystake_from_account(self.data.as_slice()) - } + // pub fn read_stake_history(&self) -> Option { + // read_historystake_from_account(self.data.as_slice()) + // } pub fn read_vote(&self) -> anyhow::Result { if self.data.is_empty() { diff --git a/stake_vote/src/lib.rs b/stake_vote/src/lib.rs index 1f52c00a..55bca52a 100644 --- a/stake_vote/src/lib.rs +++ b/stake_vote/src/lib.rs @@ -14,10 +14,9 @@ use std::sync::Arc; use tokio::sync::mpsc::Receiver; use yellowstone_grpc_client::GeyserGrpcClient; use yellowstone_grpc_proto::geyser::CommitmentLevel; +use yellowstone_grpc_proto::prelude::subscribe_update::UpdateOneof; use yellowstone_grpc_proto::prelude::SubscribeRequestFilterAccounts; -use yellowstone_grpc_proto::prelude::SubscribeRequestFilterBlocksMeta; use yellowstone_grpc_proto::prelude::SubscribeUpdate; -use yellowstone_grpc_proto::prelude::{subscribe_update::UpdateOneof, SubscribeRequestFilterSlots}; use yellowstone_grpc_proto::tonic::Status; mod account; From 5c64a151c21a4b8fbab218eb4ca667d237619d92 Mon Sep 17 00:00:00 2001 From: musitdev Date: Thu, 9 Nov 2023 16:37:37 +0100 Subject: [PATCH 19/56] test getvoteaccount. Correct some bugs --- lite-rpc/src/bridge.rs | 1 + stake_vote/src/leader_schedule.rs | 1 - stake_vote/src/lib.rs | 16 ++++++--- stake_vote/src/rpcrequest.rs | 59 +++++++++++++++++++------------ stake_vote/src/utils.rs | 30 +++++++++++----- stake_vote/src/vote.rs | 5 +-- 6 files changed, 73 insertions(+), 39 deletions(-) diff --git a/lite-rpc/src/bridge.rs b/lite-rpc/src/bridge.rs index 985a3a74..6ae03eec 100644 --- a/lite-rpc/src/bridge.rs +++ b/lite-rpc/src/bridge.rs @@ -490,6 +490,7 @@ impl LiteRpcServer for LiteBridge { slot: Option, config: Option, ) -> crate::rpc::Result>>> { + log::warn!("receive get_leader_schedule rpc call"); //TODO verify leader identity. let schedule = self .data_cache diff --git a/stake_vote/src/leader_schedule.rs b/stake_vote/src/leader_schedule.rs index 3fe756dc..04e391f4 100644 --- a/stake_vote/src/leader_schedule.rs +++ b/stake_vote/src/leader_schedule.rs @@ -304,7 +304,6 @@ pub fn calculate_leader_schedule( let mut seed = [0u8; 32]; seed[0..8].copy_from_slice(&epoch.to_le_bytes()); sort_stakes(&mut stakes); - log::info!("calculate_leader_schedule stakes:{stakes:?} epoch:{epoch}"); LeaderSchedule::new(&stakes, seed, slots_in_epoch, NUM_CONSECUTIVE_LEADER_SLOTS) } diff --git a/stake_vote/src/lib.rs b/stake_vote/src/lib.rs index 55bca52a..e22012f3 100644 --- a/stake_vote/src/lib.rs +++ b/stake_vote/src/lib.rs @@ -3,11 +3,13 @@ use crate::bootstrap::BootstrapEvent; use futures::Stream; use futures_util::stream::FuturesUnordered; use futures_util::StreamExt; +use solana_lite_rpc_core::stores::block_information_store::BlockInformation; use solana_lite_rpc_core::stores::data_cache::DataCache; use solana_lite_rpc_core::structures::leaderschedule::GetVoteAccountsConfig; use solana_lite_rpc_core::types::SlotStream; use solana_rpc_client::nonblocking::rpc_client::RpcClient; use solana_rpc_client_api::response::RpcVoteAccountStatus; +use solana_sdk::commitment_config::CommitmentConfig; use solana_sdk::pubkey::Pubkey; use std::collections::HashMap; use std::sync::Arc; @@ -92,8 +94,14 @@ pub async fn start_stakes_and_votes_loop( } } Some((config, return_channel)) = vote_account_rpc_request.recv() => { - let current_slot = solana_lite_rpc_core::solana_utils::get_current_confirmed_slot(&data_cache).await; - rpc_request_processor.process_get_vote_accounts(current_slot, config, return_channel, &mut votestore).await; + let commitment = config.commitment.unwrap_or(CommitmentConfig::confirmed()); + let BlockInformation { slot, .. } = data_cache + .block_information_store + .get_latest_block(commitment) + .await; + + let current_epoch = data_cache.get_current_epoch(commitment).await; + rpc_request_processor.process_get_vote_accounts(slot, current_epoch.epoch, config, return_channel, &mut votestore).await; } //manage rpc waiting request notification. Some(Ok((votes, vote_accounts, rpc_vote_accounts))) = rpc_request_processor.rpc_exec_task.next() => { @@ -105,8 +113,8 @@ pub async fn start_stakes_and_votes_loop( ).await; } //manage rpc waiting request notification. - Some(Ok((current_slot, config))) = rpc_request_processor.rpc_notify_task.next() => { - rpc_request_processor.take_vote_accounts_and_process(&mut votestore, current_slot, config).await; + Some(Ok((current_slot, epoch, config))) = rpc_request_processor.rpc_notify_task.next() => { + rpc_request_processor.take_vote_accounts_and_process(&mut votestore, current_slot, epoch, config).await; } //manage geyser stake_history notification ret = stake_history_geyzer_stream.next() => { diff --git a/stake_vote/src/rpcrequest.rs b/stake_vote/src/rpcrequest.rs index cc752e5e..e79b1b2a 100644 --- a/stake_vote/src/rpcrequest.rs +++ b/stake_vote/src/rpcrequest.rs @@ -11,7 +11,7 @@ use tokio::sync::oneshot; use tokio::task::JoinHandle; pub struct RpcRequestData { - pub rpc_notify_task: FuturesUnordered>, + pub rpc_notify_task: FuturesUnordered>, pub rpc_exec_task: FuturesUnordered>, pending_rpc_request: Option>>, @@ -29,6 +29,7 @@ impl RpcRequestData { pub async fn process_get_vote_accounts( &mut self, current_slot: Slot, + epoch: u64, config: GetVoteAccountsConfig, return_channel: oneshot::Sender, votestore: &mut VoteStore, @@ -39,7 +40,7 @@ impl RpcRequestData { self.pending_rpc_request = Some(vec![return_channel]); } } - self.take_vote_accounts_and_process(votestore, current_slot, config) + self.take_vote_accounts_and_process(votestore, current_slot, epoch, config) .await; } pub async fn notify_end_rpc_get_vote_accounts( @@ -50,14 +51,15 @@ impl RpcRequestData { votestore: &mut VoteStore, ) { if let Err(err) = votestore.votes.merge((votes, vote_accounts)) { - log::info!("Error during RPC get vote account merge:{err}"); + log::error!("Error during RPC get vote account merge:{err}"); } - //avoid clone on the first request - if let Some(ref mut pending_rpc_request) = self.pending_rpc_request { - for return_channel in pending_rpc_request.drain(0..pending_rpc_request.len() - 1) { - if return_channel.send(rpc_vote_accounts.clone()).is_err() { - log::error!("Vote accounts RPC channel send closed."); + if let Some(mut pending_rpc_request) = self.pending_rpc_request.take() { + if pending_rpc_request.len() > 1 { + for return_channel in pending_rpc_request.drain(0..pending_rpc_request.len() - 1) { + if return_channel.send(rpc_vote_accounts.clone()).is_err() { + log::error!("Vote accounts RPC channel send closed."); + } } } if pending_rpc_request @@ -75,30 +77,41 @@ impl RpcRequestData { &mut self, votestore: &mut VoteStore, current_slot: Slot, + epoch: u64, config: GetVoteAccountsConfig, ) { - if let Some(((votes, vote_accounts), (current_slot, config))) = + if let Some(((votes, vote_accounts), (current_slot, epoch, config))) = wait_for_merge_or_get_content( &mut votestore.votes, - (current_slot, config), + (current_slot, epoch, config), &mut self.rpc_notify_task, ) .await { + //validate that we have the epoch. + let jh = tokio::task::spawn_blocking({ - move || { - let rpc_vote_accounts = crate::vote::get_rpc_vote_accounts_info( - current_slot, - &votes, - //TODO manage missing epoch and return error. - &vote_accounts - .vote_stakes_for_epoch(0) - .as_ref() - .unwrap() - .vote_stakes, - config, - ); - (votes, vote_accounts, rpc_vote_accounts) + move || match vote_accounts.vote_stakes_for_epoch(epoch) { + Some(stakes) => { + let rpc_vote_accounts = crate::vote::get_rpc_vote_accounts_info( + current_slot, + &votes, + &stakes.vote_stakes, + config, + ); + (votes, vote_accounts, rpc_vote_accounts) + } + None => { + log::warn!("Get vote account for epoch:{epoch}. No data available"); + ( + votes, + vote_accounts, + RpcVoteAccountStatus { + current: vec![], + delinquent: vec![], + }, + ) + } } }); self.rpc_exec_task.push(jh); diff --git a/stake_vote/src/utils.rs b/stake_vote/src/utils.rs index 94553c85..831236e7 100644 --- a/stake_vote/src/utils.rs +++ b/stake_vote/src/utils.rs @@ -122,7 +122,7 @@ impl<'a, T, C: TakableContent> Takable for &'a mut TakableMap { content.add_value(val); } self.content = Some(content); - self.notifier.notify_waiters(); + self.notifier.notify_one(); Ok(()) } else { bail!("TakableMap with a existing content".to_string()) @@ -232,8 +232,8 @@ impl + Default> TakableMap { mod tests { use super::*; - #[test] - fn test_takable_struct() { + #[tokio::test] + async fn test_takable_struct() { impl TakableContent for Vec { fn add_value(&mut self, val: UpdateAction) { match val { @@ -254,23 +254,35 @@ mod tests { let take_content = (&mut takable).take(); assert_take_content_map(&take_content, 1); + let content = match take_content { + TakeResult::Taken(_) => panic!("not a content"), + TakeResult::Map(content) => content, + }; assert_eq!(takable.updates.len(), 1); let take_content = (&mut takable).take(); assert_take_content_taken(&take_content); + let notifier = match take_content { + TakeResult::Taken(notifier) => notifier, + TakeResult::Map(_) => panic!("not a notifier"), + }; + assert_eq!(notifier.len(), 1); + let notif_jh = tokio::spawn(async move { + notifier[0].as_ref().notified().await; + }); + assert!(takable.content.is_none()); assert_eq!(takable.updates.len(), 1); takable.add_value(UpdateAction::Notify(25, 0), false); assert_eq!(takable.updates.len(), 2); - let content = match take_content { - TakeResult::Taken(_) => panic!("not a content"), - TakeResult::Map(content) => content, - }; takable.merge(content).unwrap(); assert_eq!(takable.content.as_ref().unwrap().len(), 3); assert_eq!(takable.updates.len(), 0); - //merge(&mut takable, vec![]); - //assert!(err.is_err()); + //wait for notifier + if let Err(_) = tokio::time::timeout(std::time::Duration::from_millis(1000), notif_jh).await + { + assert!(false, "take notifier timeout"); + } } fn assert_take_content_map(take_content: &TakeResult>, len: usize) { diff --git a/stake_vote/src/vote.rs b/stake_vote/src/vote.rs index e119e61c..08f22161 100644 --- a/stake_vote/src/vote.rs +++ b/stake_vote/src/vote.rs @@ -31,11 +31,12 @@ pub struct EpochVoteStakesCache { } impl EpochVoteStakesCache { - pub fn vote_stakes_for_epoch(&self, epoch: u64) -> Option { - self.cache.get(&epoch).cloned() + pub fn vote_stakes_for_epoch(&self, epoch: u64) -> Option<&EpochVoteStakes> { + self.cache.get(&epoch) } pub fn add_stakes_for_epoch(&mut self, vote_stakes: EpochVoteStakes) { + log::info!("add_stakes_for_epoch :{}", vote_stakes.epoch); if self.cache.insert(vote_stakes.epoch, vote_stakes).is_some() { log::warn!("Override existing vote stake epoch cache for epoch:"); } From 93ab67db188c839053422119ea2d7268243770c9 Mon Sep 17 00:00:00 2001 From: musitdev Date: Sat, 11 Nov 2023 18:42:03 +0100 Subject: [PATCH 20/56] test leader and vote RPC calls --- core/src/stores/data_cache.rs | 5 +- core/src/structures/leaderschedule.rs | 103 +++++++++++++++++--------- lite-rpc/src/bridge.rs | 15 +++- lite-rpc/src/main.rs | 19 ++--- stake_vote/src/lib.rs | 9 ++- stake_vote/src/vote.rs | 11 ++- 6 files changed, 104 insertions(+), 58 deletions(-) diff --git a/core/src/stores/data_cache.rs b/core/src/stores/data_cache.rs index 975f5b74..6ea29e88 100644 --- a/core/src/stores/data_cache.rs +++ b/core/src/stores/data_cache.rs @@ -4,6 +4,7 @@ use solana_sdk::hash::Hash; use solana_sdk::slot_history::Slot; use solana_sdk::{commitment_config::CommitmentConfig, pubkey::Pubkey}; use std::sync::{atomic::AtomicU64, Arc}; +use tokio::sync::RwLock; use crate::{ stores::{ @@ -37,7 +38,7 @@ pub struct DataCache { pub identity_stakes: IdentityStakes, pub cluster_info: ClusterInfo, pub epoch_data: EpochCache, - pub leader_schedule: Arc, + pub leader_schedule: Arc>, } impl DataCache { @@ -88,7 +89,7 @@ impl DataCache { store: Arc::new(DashMap::new()), }, epoch_data: EpochCache::new_for_tests(), - leader_schedule: Arc::new(CalculatedSchedule::default()), + leader_schedule: Arc::new(RwLock::new(CalculatedSchedule::default())), } } } diff --git a/core/src/structures/leaderschedule.rs b/core/src/structures/leaderschedule.rs index fafc50fd..68bb79e2 100644 --- a/core/src/structures/leaderschedule.rs +++ b/core/src/structures/leaderschedule.rs @@ -4,6 +4,8 @@ use solana_rpc_client_api::config::RpcGetVoteAccountsConfig; use solana_sdk::commitment_config::CommitmentConfig; use solana_sdk::pubkey::ParsePubkeyError; use solana_sdk::pubkey::Pubkey; +use solana_sdk::slot_history::Slot; +use solana_sdk::sysvar::epoch_schedule::EpochSchedule; use std::collections::HashMap; use std::str::FromStr; @@ -46,7 +48,13 @@ impl CalculatedSchedule { commitment: Option, data_cache: &DataCache, ) -> Option>> { - let commitment = commitment.unwrap_or_else(CommitmentConfig::default); + log::info!( + "get_leader_schedule_for_slot current:{:?} next:{:?} ", + self.current.clone().unwrap_or_default(), + self.next.clone().unwrap_or_default() + ); + + let commitment = commitment.unwrap_or_default(); let slot = match slot { Some(slot) => slot, None => { @@ -67,48 +75,69 @@ impl CalculatedSchedule { get_schedule(self.current.as_ref()).or_else(|| get_schedule(self.next.as_ref())) } - // pub async fn get_slot_leaders(&self, start_slot: Slot, limit: u64) -> Result> { - // debug!( - // "get_slot_leaders rpc request received (start: {} limit: {})", - // start_slot, limit - // ); + pub async fn get_slot_leaders( + &self, + start_slot: Slot, + limit: u64, + epock_schedule: &EpochSchedule, + ) -> Result, String> { + log::debug!( + "get_slot_leaders rpc request received (start: {} limit: {})", + start_slot, + limit + ); + pub const MAX_GET_SLOT_LEADERS: usize = + solana_rpc_client_api::request::MAX_GET_SLOT_LEADERS; + + let mut limit = limit as usize; + if limit > MAX_GET_SLOT_LEADERS { + return Err(format!( + "Invalid Params: Invalid limit; max {MAX_GET_SLOT_LEADERS}" + )); + } - // let limit = limit as usize; - // if limit > MAX_GET_SLOT_LEADERS { - // return Err(Error::invalid_params(format!( - // "Invalid limit; max {MAX_GET_SLOT_LEADERS}" - // ))); - // } - // let bank = self.bank(commitment); + let (epoch, slot_index) = epock_schedule.get_epoch_and_slot_index(start_slot); + let mut slot_leaders = Vec::with_capacity(limit); - // let (mut epoch, mut slot_index) = - // bank.epoch_schedule().get_epoch_and_slot_index(start_slot); + let mut extend_slot_from_epoch = |leader_schedule: &[Pubkey], slot_index: usize| { + let take = limit.saturating_sub(slot_leaders.len()); + slot_leaders.extend(leader_schedule.iter().skip(slot_index).take(take)); + limit -= slot_leaders.len(); + }; - // let mut slot_leaders = Vec::with_capacity(limit); - // while slot_leaders.len() < limit { - // if let Some(leader_schedule) = - // self.leader_schedule_cache.get_epoch_leader_schedule(epoch) - // { - // slot_leaders.extend( - // leader_schedule - // .get_slot_leaders() - // .iter() - // .skip(slot_index as usize) - // .take(limit.saturating_sub(slot_leaders.len())), - // ); - // } else { - // return Err(Error::invalid_params(format!( - // "Invalid slot range: leader schedule for epoch {epoch} is unavailable" - // ))); - // } + // log::info!( + // "get_slot_leaders epoch:{epoch} current:{:?} next:{:?} ", + // self.current.clone().unwrap_or_default(), + // self.next.clone().unwrap_or_default() + // ); - // epoch += 1; - // slot_index = 0; - // } - // } + //TODO manage more leader schedule data in storage. + //Here only search on current and next epoch + let res = [ + (&self.current, slot_index as usize, epoch), + (&self.next, slot_index as usize, epoch), + (&self.next, 0, epoch + 1), + ] + .into_iter() + .filter_map(|(epoch_data, slot_index, epoch)| { + epoch_data.as_ref().and_then(|epoch_data| { + (epoch_data.epoch == epoch).then_some((epoch_data, slot_index)) + }) + }) + .map(|(epoch_data, slot_index)| { + extend_slot_from_epoch(&epoch_data.schedule_by_slot, slot_index); + }) + .collect::>(); + match res.is_empty() { + true => Err(format!( + "Invalid Params: Invalid slot range: leader schedule for epoch {epoch} is unavailable" + )), + false => Ok(slot_leaders), + } + } } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub struct LeaderScheduleData { pub schedule_by_node: HashMap>, pub schedule_by_slot: Vec, diff --git a/lite-rpc/src/bridge.rs b/lite-rpc/src/bridge.rs index 6ae03eec..9127d796 100644 --- a/lite-rpc/src/bridge.rs +++ b/lite-rpc/src/bridge.rs @@ -490,11 +490,12 @@ impl LiteRpcServer for LiteBridge { slot: Option, config: Option, ) -> crate::rpc::Result>>> { - log::warn!("receive get_leader_schedule rpc call"); //TODO verify leader identity. let schedule = self .data_cache .leader_schedule + .read() + .await .get_leader_schedule_for_slot(slot, config.and_then(|c| c.commitment), &self.data_cache) .await; Ok(schedule) @@ -504,7 +505,17 @@ impl LiteRpcServer for LiteBridge { start_slot: u64, limit: u64, ) -> crate::rpc::Result> { - todo!() + let epock_schedule = self.data_cache.epoch_data.get_epoch_schedule(); + + self.data_cache + .leader_schedule + .read() + .await + .get_slot_leaders(start_slot, limit, epock_schedule) + .await + .map_err(|err| { + jsonrpsee::core::Error::Custom(format!("error during query processing:{err}")) + }) } async fn get_vote_accounts( diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index 8ad16905..eca75f41 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -1,18 +1,14 @@ pub mod rpc_tester; -use solana_lite_rpc_core::structures::leaderschedule::CalculatedSchedule; -use std::time::Duration; - +use crate::rpc_tester::RpcTester; +use crate::rpc_tester::RpcTester; use anyhow::bail; use dashmap::DashMap; -use lite_rpc::bridge::LiteBridge; use lite_rpc::cli::Config; use lite_rpc::postgres_logger::PostgresLogger; use lite_rpc::service_spawner::ServiceSpawner; -use lite_rpc::DEFAULT_MAX_NUMBER_OF_TXS_IN_QUEUE; -use solana_lite_rpc_history::postgres::postgres_config::PostgresSessionConfig; - -use crate::rpc_tester::RpcTester; +use lite_rpc::{bridge::LiteBridge, cli::Args}; +use lite_rpc::{DEFAULT_MAX_NUMBER_OF_TXS_IN_QUEUE, GRPC_VERSION, NB_SLOTS_TRANSACTIONS_TO_CACHE}; use log::info; use solana_lite_rpc_cluster_endpoints::endpoint_stremers::EndpointStreaming; use solana_lite_rpc_cluster_endpoints::grpc_subscription::create_grpc_subscription; @@ -30,6 +26,7 @@ use solana_lite_rpc_core::stores::{ subscription_store::SubscriptionStore, tx_store::TxStore, }; +use solana_lite_rpc_core::structures::leaderschedule::CalculatedSchedule; use solana_lite_rpc_core::structures::{ epoch::EpochCache, identity_stakes::IdentityStakes, notifications::NotificationSender, produced_block::ProducedBlock, @@ -38,6 +35,8 @@ use solana_lite_rpc_core::types::BlockStream; use solana_lite_rpc_core::AnyhowJoinHandle; use solana_lite_rpc_history::block_stores::inmemory_block_store::InmemoryBlockStore; use solana_lite_rpc_history::history::History; +use solana_lite_rpc_history::postgres::postgres_config::PostgresSessionConfig; +use solana_lite_rpc_history::postgres::postgres_config::PostgresSessionConfig; use solana_lite_rpc_history::postgres::postgres_session::PostgresSessionCache; use solana_lite_rpc_services::data_caching_service::DataCachingService; use solana_lite_rpc_services::tpu_utils::tpu_connection_path::TpuConnectionPath; @@ -50,7 +49,9 @@ use solana_sdk::signature::Keypair; use solana_sdk::signer::Signer; use std::net::{SocketAddr, ToSocketAddrs}; use std::sync::Arc; +use std::time::Duration; use tokio::sync::mpsc; +use tokio::sync::RwLock; async fn get_latest_block( mut block_stream: BlockStream, @@ -160,7 +161,7 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: store: Arc::new(DashMap::new()), }, epoch_data, - leader_schedule: Arc::new(CalculatedSchedule::default()), + leader_schedule: Arc::new(RwLock::new(CalculatedSchedule::default())), }; let lata_cache_service = DataCachingService { diff --git a/stake_vote/src/lib.rs b/stake_vote/src/lib.rs index e22012f3..92d6b2fa 100644 --- a/stake_vote/src/lib.rs +++ b/stake_vote/src/lib.rs @@ -36,7 +36,7 @@ const VOTESTORE_INITIAL_CAPACITY: usize = 600000; type Slot = u64; pub async fn start_stakes_and_votes_loop( - mut data_cache: DataCache, + data_cache: DataCache, mut slot_notification: SlotStream, mut vote_account_rpc_request: Receiver<( GetVoteAccountsConfig, @@ -226,8 +226,8 @@ pub async fn start_stakes_and_votes_loop( Ok(Some(boot_res))=> { match boot_res { Ok(current_schedule_data) => { - //let data_schedule = Arc::make_mut(&mut data_cache.leader_schedule); - data_cache.leader_schedule = Arc::new(current_schedule_data); + let mut data_schedule = data_cache.leader_schedule.write().await; + *data_schedule = current_schedule_data; } Err(err) => { log::warn!("Error during current leader schedule bootstrap from files:{err}") @@ -254,7 +254,8 @@ pub async fn start_stakes_and_votes_loop( if let Some(new_leader_schedule) = new_leader_schedule { //clone old schedule values is there's other use. //only done once epoch. Avoid to use a Mutex. - let data_schedule = Arc::make_mut(&mut data_cache.leader_schedule); + log::info!("End leader schedule calculus for epoch:{}", new_leader_schedule.epoch); + let mut data_schedule = data_cache.leader_schedule.write().await; data_schedule.current = data_schedule.next.take(); data_schedule.next = Some(new_leader_schedule.rpc_data); } diff --git a/stake_vote/src/vote.rs b/stake_vote/src/vote.rs index 08f22161..5dd948c6 100644 --- a/stake_vote/src/vote.rs +++ b/stake_vote/src/vote.rs @@ -235,15 +235,18 @@ pub fn merge_program_account_in_vote_map( }); } -//TODO put in config instead of const. // Validators that are this number of slots behind are considered delinquent -pub const DELINQUENT_VALIDATOR_SLOT_DISTANCE: u64 = 128; pub fn get_rpc_vote_accounts_info( current_slot: Slot, votes: &VoteMap, vote_accounts: &HashMap)>, config: GetVoteAccountsConfig, ) -> RpcVoteAccountStatus { + pub const DELINQUENT_VALIDATOR_SLOT_DISTANCE: u64 = + solana_rpc_client_api::request::DELINQUENT_VALIDATOR_SLOT_DISTANCE; + let delinquent_validator_slot_distance = config + .delinquent_slot_distance + .unwrap_or(DELINQUENT_VALIDATOR_SLOT_DISTANCE); //From Solana rpc::rpc::metaz::get_vote_accounts() code. let (current_vote_accounts, delinquent_vote_accounts): ( Vec, @@ -258,8 +261,8 @@ pub fn get_rpc_vote_accounts_info( vote.convert_to_rpc_vote_account_info(stake, epoch_vote_account) }) .partition(|vote_account_info| { - if current_slot >= DELINQUENT_VALIDATOR_SLOT_DISTANCE { - vote_account_info.last_vote > current_slot - DELINQUENT_VALIDATOR_SLOT_DISTANCE + if current_slot >= delinquent_validator_slot_distance { + vote_account_info.last_vote > current_slot - delinquent_validator_slot_distance } else { vote_account_info.last_vote > 0 } From bff31e08af9128d054ff67a326eb9ef67cbb8470 Mon Sep 17 00:00:00 2001 From: musitdev Date: Tue, 21 Nov 2023 14:27:55 +0100 Subject: [PATCH 21/56] correct logic error that was storing all received account update --- stake_vote/src/stake.rs | 4 ++-- stake_vote/src/utils.rs | 1 + stake_vote/src/vote.rs | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/stake_vote/src/stake.rs b/stake_vote/src/stake.rs index 359226ba..454ebb4a 100644 --- a/stake_vote/src/stake.rs +++ b/stake_vote/src/stake.rs @@ -48,7 +48,7 @@ impl StakeStore { if account.lamports == 0 { self.stakes.add_value( UpdateAction::Remove(account.pubkey, account.slot), - account.slot <= current_end_epoch_slot, + account.slot > current_end_epoch_slot, ); } else { let Ok(delegated_stake_opt) = account.read_stake() else { @@ -67,7 +67,7 @@ impl StakeStore { let action_update_slot = stake.last_update_slot; self.stakes.add_value( UpdateAction::Notify(action_update_slot, stake), - action_update_slot <= current_end_epoch_slot, + action_update_slot > current_end_epoch_slot, ); } } diff --git a/stake_vote/src/utils.rs b/stake_vote/src/utils.rs index 831236e7..a26ac4eb 100644 --- a/stake_vote/src/utils.rs +++ b/stake_vote/src/utils.rs @@ -218,6 +218,7 @@ impl + Default> TakableMap { //during extract push the new update or //don't insert now account change that has been done in next epoch. //put in update pool to be merged next epoch change. + //log::info!("tm u:{} c:{} f:{}", self.updates.len(), self.content.is_none(), force_in_update); match self.content.is_none() || force_in_update { true => self.updates.push(val), false => { diff --git a/stake_vote/src/vote.rs b/stake_vote/src/vote.rs index 5dd948c6..be2e0d0d 100644 --- a/stake_vote/src/vote.rs +++ b/stake_vote/src/vote.rs @@ -108,7 +108,7 @@ impl VoteStore { //self.remove_from_store(&new_account.pubkey, new_account.slot); self.votes.add_value( UpdateAction::Remove(new_account.pubkey, new_account.slot), - new_account.slot <= current_end_epoch_slot, + new_account.slot > current_end_epoch_slot, ); } else { let Ok(mut vote_data) = new_account.read_vote() else { @@ -140,7 +140,7 @@ impl VoteStore { let action_update_slot = new_voteacc.last_update_slot; self.votes.add_value( UpdateAction::Notify(action_update_slot, new_voteacc), - action_update_slot <= current_end_epoch_slot, + action_update_slot > current_end_epoch_slot, ); } From e5d4942ddb69286be123784be1fd5db1fbcdf25b Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 27 Nov 2023 14:56:35 +0100 Subject: [PATCH 22/56] add account update at epoch --- stake_vote/src/leader_schedule.rs | 1 + stake_vote/src/lib.rs | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/stake_vote/src/leader_schedule.rs b/stake_vote/src/leader_schedule.rs index 04e391f4..3fe756dc 100644 --- a/stake_vote/src/leader_schedule.rs +++ b/stake_vote/src/leader_schedule.rs @@ -304,6 +304,7 @@ pub fn calculate_leader_schedule( let mut seed = [0u8; 32]; seed[0..8].copy_from_slice(&epoch.to_le_bytes()); sort_stakes(&mut stakes); + log::info!("calculate_leader_schedule stakes:{stakes:?} epoch:{epoch}"); LeaderSchedule::new(&stakes, seed, slots_in_epoch, NUM_CONSECUTIVE_LEADER_SLOTS) } diff --git a/stake_vote/src/lib.rs b/stake_vote/src/lib.rs index 92d6b2fa..867633fb 100644 --- a/stake_vote/src/lib.rs +++ b/stake_vote/src/lib.rs @@ -75,6 +75,10 @@ pub async fn start_stakes_and_votes_loop( let mut bootstrap_done = false; + //for test to count the number of account notified at epoch change. + let mut account_update_notification = None; + let mut epoch_wait_account_notification_task = FuturesUnordered::new(); + loop { tokio::select! { //manage confirm new slot notification to detect epoch change. @@ -90,9 +94,22 @@ pub async fn start_stakes_and_votes_loop( &mut stakestore, &mut votestore, ); + + //for test to count the number of account notified at epoch change. + account_update_notification = Some(0); + let jh = tokio::spawn(async move { + //sleep 3 minutes and count the number of account notification. + tokio::time::sleep(tokio::time::Duration::from_secs(180)).await; + }); + epoch_wait_account_notification_task.push(jh); + } } } + Some(Ok(())) = epoch_wait_account_notification_task.next() => { + log::info!("Epoch change account count:{} during 3mn", account_update_notification.as_ref().unwrap_or(&0)); + account_update_notification = None; + } Some((config, return_channel)) = vote_account_rpc_request.recv() => { let commitment = config.commitment.unwrap_or(CommitmentConfig::confirmed()); let BlockInformation { slot, .. } = data_cache @@ -176,6 +193,9 @@ pub async fn start_stakes_and_votes_loop( match account.owner { solana_sdk::stake::program::ID => { log::trace!("Geyser notif stake account:{}", account); + if let Some(ref mut counter) = account_update_notification { + *counter +=1; + } if let Err(err) = stakestore.notify_stake_change( account, current_schedule_epoch.last_slot_in_epoch, From eadff4fce3cabe3ff2d65d95a76aa5cdfd710695 Mon Sep 17 00:00:00 2001 From: musitdev Date: Tue, 28 Nov 2023 09:19:32 +0100 Subject: [PATCH 23/56] set inital schedule parameters --- lite-rpc/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index eca75f41..940b8148 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -199,7 +199,7 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: prometheus_addr, data_cache: data_cache.clone(), }; - let leader_schedule = Arc::new(JsonRpcLeaderGetter::new(rpc_client.clone(), 10, 128)); + let leader_schedule = Arc::new(JsonRpcLeaderGetter::new(rpc_client.clone(), 1024, 128)); let tpu_service: TpuService = TpuService::new( tpu_config, From 079a25b0717c3ae280b4a5a489d8141793277cd9 Mon Sep 17 00:00:00 2001 From: aniketfuryrocks Date: Tue, 28 Nov 2023 13:01:18 +0530 Subject: [PATCH 24/56] -D warning --- .github/workflows/build_test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 86cf4945..013dd44f 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -47,5 +47,10 @@ jobs: run: | cargo build --locked --workspace --all-targets + - name: Run fmt+clippy + run: | + cargo fmt --all --check + cargo clippy --locked --workspace --all-targets -D warnings + - name: Run Tests run: RUST_LOG=info cargo test From 16e8c53fad5d13a3d49bd66d254b08058d8c27b5 Mon Sep 17 00:00:00 2001 From: musitdev Date: Tue, 28 Nov 2023 11:24:25 +0100 Subject: [PATCH 25/56] change some ogs to trace and set default gprc add to localhost --- lite-rpc/src/lib.rs | 2 +- stake_vote/src/stake.rs | 4 ++-- stake_vote/src/vote.rs | 18 +++++++++--------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lite-rpc/src/lib.rs b/lite-rpc/src/lib.rs index f5c8da9a..ccf1825c 100644 --- a/lite-rpc/src/lib.rs +++ b/lite-rpc/src/lib.rs @@ -33,7 +33,7 @@ pub const DEFAULT_TRANSACTION_CONFIRMATION_STATUS: TransactionConfirmationStatus TransactionConfirmationStatus::Finalized; #[from_env] -pub const DEFAULT_GRPC_ADDR: &str = "http://127.0.0.0:10000"; +pub const DEFAULT_GRPC_ADDR: &str = "http://localhost:10000"; #[from_env] pub const GRPC_VERSION: &str = "1.16.1"; diff --git a/stake_vote/src/stake.rs b/stake_vote/src/stake.rs index 454ebb4a..8d712984 100644 --- a/stake_vote/src/stake.rs +++ b/stake_vote/src/stake.rs @@ -92,13 +92,13 @@ impl StakeStore { //doesn't erase new state with an old one. Can arrive during bootstrapping. //several instructions can be done in the same slot. if strstake.last_update_slot <= stake.last_update_slot { - log::info!("stake_map_notify_stake Stake store updated stake: {} old_stake:{strstake:?} stake:{stake:?}", stake.pubkey); + log::trace!("stake_map_notify_stake Stake store updated stake: {} old_stake:{strstake:?} stake:{stake:?}", stake.pubkey); *strstake = stake; } } // If value doesn't exist yet, then insert a new value of 1 std::collections::hash_map::Entry::Vacant(vacant) => { - log::info!( + log::trace!( "stake_map_notify_stake Stake store insert stake: {} stake:{stake:?}", stake.pubkey ); diff --git a/stake_vote/src/vote.rs b/stake_vote/src/vote.rs index be2e0d0d..4d44cf94 100644 --- a/stake_vote/src/vote.rs +++ b/stake_vote/src/vote.rs @@ -181,22 +181,22 @@ impl VoteStore { // vote_data.vote_data.node_pubkey, // vote_data.vote_data.root_slot, // ); - if vote_data.vote_data.root_slot.is_none() { - log::info!("Update vote account:{vote_account_pk} with None root slot."); - } + // if vote_data.vote_data.root_slot.is_none() { + // log::info!("Update vote account:{vote_account_pk} with None root slot."); + // } - if voteacc.vote_data.root_slot.is_none() { - log::info!( - "Update vote account:{vote_account_pk} that were having None root slot." - ); - } + // if voteacc.vote_data.root_slot.is_none() { + // log::info!( + // "Update vote account:{vote_account_pk} that were having None root slot." + // ); + // } *voteacc = Arc::new(vote_data); } } // If value doesn't exist yet, then insert a new value of 1 std::collections::hash_map::Entry::Vacant(vacant) => { - log::info!( + log::trace!( "New Vote added for: {vote_account_pk} node_id:{}, root slot:{:?}", vote_data.vote_data.node_pubkey, vote_data.vote_data.root_slot, From ecffad0a7395eb27c85dc8d3c7e83bc3c01e21f7 Mon Sep 17 00:00:00 2001 From: musitdev Date: Tue, 28 Nov 2023 12:07:21 +0100 Subject: [PATCH 26/56] change some logs --- stake_vote/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stake_vote/src/lib.rs b/stake_vote/src/lib.rs index 867633fb..dbf053fd 100644 --- a/stake_vote/src/lib.rs +++ b/stake_vote/src/lib.rs @@ -45,7 +45,7 @@ pub async fn start_stakes_and_votes_loop( rpc_client: Arc, grpc_url: String, ) -> anyhow::Result> { - log::info!("Start Stake and Vote loop."); + log::info!("Start Stake and Vote loop on :{grpc_url}."); let mut stake_vote_geyzer_stream = subscribe_geyzer_stake_vote_owner(grpc_url.clone()).await?; let mut stake_history_geyzer_stream = subscribe_geyzer_stake_history(grpc_url).await?; log::info!("Stake and Vote geyzer subscription done."); From faa5d1a12c6fae6c30fff0ccd931d75a0088ff1b Mon Sep 17 00:00:00 2001 From: musitdev Date: Tue, 28 Nov 2023 19:22:00 +0100 Subject: [PATCH 27/56] rebase from main --- lite-rpc/src/bridge.rs | 23 +++-------------------- lite-rpc/src/rpc.rs | 28 +++++----------------------- stake_vote/src/lib.rs | 2 ++ 3 files changed, 10 insertions(+), 43 deletions(-) diff --git a/lite-rpc/src/bridge.rs b/lite-rpc/src/bridge.rs index 9127d796..7fafe7ec 100644 --- a/lite-rpc/src/bridge.rs +++ b/lite-rpc/src/bridge.rs @@ -4,8 +4,6 @@ use crate::{ rpc::LiteRpcServer, }; use solana_lite_rpc_core::structures::leaderschedule::GetVoteAccountsConfig; -use solana_rpc_client_api::config::RpcGetVoteAccountsConfig; -use solana_rpc_client_api::response::RpcVoteAccountStatus; use solana_sdk::epoch_info::EpochInfo; use std::collections::HashMap; @@ -26,15 +24,15 @@ use solana_rpc_client::nonblocking::rpc_client::RpcClient; use solana_rpc_client_api::{ config::{ RpcBlockConfig, RpcBlockSubscribeConfig, RpcBlockSubscribeFilter, RpcBlocksConfigWrapper, - RpcContextConfig, RpcEncodingConfigWrapper, RpcEpochConfig, RpcGetVoteAccountsConfig, + RpcContextConfig, RpcEncodingConfigWrapper, RpcGetVoteAccountsConfig, RpcLeaderScheduleConfig, RpcProgramAccountsConfig, RpcRequestAirdropConfig, RpcSignatureStatusConfig, RpcSignatureSubscribeConfig, RpcSignaturesForAddressConfig, RpcTransactionLogsConfig, RpcTransactionLogsFilter, }, response::{ Response as RpcResponse, RpcBlockhash, RpcConfirmedTransactionStatusWithSignature, - RpcContactInfo, RpcLeaderSchedule, RpcPerfSample, RpcPrioritizationFee, RpcResponseContext, - RpcVersionInfo, RpcVoteAccountStatus, + RpcContactInfo, RpcPerfSample, RpcPrioritizationFee, RpcResponseContext, RpcVersionInfo, + RpcVoteAccountStatus, }, }; use solana_sdk::{commitment_config::CommitmentConfig, pubkey::Pubkey, slot_history::Slot}; @@ -281,21 +279,6 @@ impl LiteRpcServer for LiteBridge { Ok(epoch_info) } - async fn get_leader_schedule( - &self, - _slot: Option, - _config: Option>, - ) -> crate::rpc::Result> { - todo!() - } - - async fn get_vote_accounts( - &self, - _config: Option, - ) -> crate::rpc::Result { - todo!() - } - async fn get_recent_performance_samples( &self, _limit: Option, diff --git a/lite-rpc/src/rpc.rs b/lite-rpc/src/rpc.rs index 2861586c..15971781 100644 --- a/lite-rpc/src/rpc.rs +++ b/lite-rpc/src/rpc.rs @@ -1,18 +1,16 @@ use crate::configs::{IsBlockHashValidConfig, SendTransactionConfig}; use jsonrpsee::core::SubscriptionResult; use jsonrpsee::proc_macros::rpc; -use solana_rpc_client_api::config::RpcGetVoteAccountsConfig; use solana_rpc_client_api::config::{ RpcBlockConfig, RpcBlockSubscribeConfig, RpcBlockSubscribeFilter, RpcBlocksConfigWrapper, - RpcContextConfig, RpcEncodingConfigWrapper, RpcEpochConfig, RpcGetVoteAccountsConfig, - RpcLeaderScheduleConfig, RpcProgramAccountsConfig, RpcRequestAirdropConfig, - RpcSignatureStatusConfig, RpcSignatureSubscribeConfig, RpcSignaturesForAddressConfig, - RpcTransactionLogsConfig, RpcTransactionLogsFilter, + RpcContextConfig, RpcEncodingConfigWrapper, RpcGetVoteAccountsConfig, RpcLeaderScheduleConfig, + RpcProgramAccountsConfig, RpcRequestAirdropConfig, RpcSignatureStatusConfig, + RpcSignatureSubscribeConfig, RpcSignaturesForAddressConfig, RpcTransactionLogsConfig, + RpcTransactionLogsFilter, }; use solana_rpc_client_api::response::{ Response as RpcResponse, RpcBlockhash, RpcConfirmedTransactionStatusWithSignature, - RpcContactInfo, RpcLeaderSchedule, RpcPerfSample, RpcPrioritizationFee, RpcVersionInfo, - RpcVoteAccountStatus, + RpcContactInfo, RpcPerfSample, RpcPrioritizationFee, RpcVersionInfo, RpcVoteAccountStatus, }; use solana_sdk::commitment_config::CommitmentConfig; use solana_sdk::epoch_info::EpochInfo; @@ -112,22 +110,6 @@ pub trait LiteRpc { // block: u64, // ) -> Result>; - #[method(name = "getEpochInfo")] - async fn get_epoch_info(&self, config: Option) -> Result; - - #[method(name = "getLeaderSchedule")] - async fn get_leader_schedule( - &self, - slot: Option, - config: Option>, - ) -> Result>; - - #[method(name = "getVoteAccounts")] - async fn get_vote_accounts( - &self, - config: Option, - ) -> Result; - #[method(name = "getRecentPerformanceSamples")] async fn get_recent_performance_samples( &self, diff --git a/stake_vote/src/lib.rs b/stake_vote/src/lib.rs index dbf053fd..5f1b6b3d 100644 --- a/stake_vote/src/lib.rs +++ b/stake_vote/src/lib.rs @@ -317,6 +317,7 @@ async fn subscribe_geyzer_stake_vote_owner( Default::default(), //block meta Some(CommitmentLevel::Confirmed), vec![], + None, ) .await?; @@ -350,6 +351,7 @@ async fn subscribe_geyzer_stake_history( Default::default(), //block meta Some(CommitmentLevel::Confirmed), vec![], + None, ) .await?; From 5a24c2bb4d2af4cd0606bceca13d4e6830fc0ee9 Mon Sep 17 00:00:00 2001 From: musitdev Date: Wed, 27 Dec 2023 14:28:10 +0100 Subject: [PATCH 28/56] set stake_vote optional and integrate leader schedule and vote in send Tx --- cluster-endpoints/src/grpc_leaders_getter.rs | 74 +++++++++++++++++++ cluster-endpoints/src/lib.rs | 1 + .../src/rpc_polling/poll_slots.rs | 2 +- config.example.json | 1 + config.json | 16 ++++ lite-rpc/src/bridge.rs | 43 +++++++---- lite-rpc/src/cli.rs | 3 +- lite-rpc/src/main.rs | 71 +++++++++++++----- stake_vote/src/bootstrap.rs | 28 ++++++- stake_vote/src/lib.rs | 7 +- stake_vote/src/vote.rs | 14 ++++ 11 files changed, 220 insertions(+), 40 deletions(-) create mode 100644 cluster-endpoints/src/grpc_leaders_getter.rs create mode 100644 config.json diff --git a/cluster-endpoints/src/grpc_leaders_getter.rs b/cluster-endpoints/src/grpc_leaders_getter.rs new file mode 100644 index 00000000..2bb3afc2 --- /dev/null +++ b/cluster-endpoints/src/grpc_leaders_getter.rs @@ -0,0 +1,74 @@ +use anyhow::{bail, Error}; +use async_trait::async_trait; +use solana_lite_rpc_core::structures::epoch::EpochCache; +use solana_lite_rpc_core::structures::leaderschedule::CalculatedSchedule; +use solana_lite_rpc_core::{ + structures::leader_data::LeaderData, traits::leaders_fetcher_interface::LeaderFetcherInterface, +}; +use std::sync::Arc; +use tokio::sync::RwLock; + +// Stores leaders for slots from older to newer in leader schedule +// regularly removed old leaders and adds new ones +pub struct GrpcLeaderGetter { + epoch_data: EpochCache, + leader_schedule: Arc>, +} + +impl GrpcLeaderGetter { + pub fn new(leader_schedule: Arc>, epoch_data: EpochCache) -> Self { + Self { + leader_schedule, + epoch_data, + } + } +} + +#[async_trait] +impl LeaderFetcherInterface for GrpcLeaderGetter { + async fn get_slot_leaders( + &self, + from: solana_sdk::slot_history::Slot, + to: solana_sdk::slot_history::Slot, + ) -> anyhow::Result> { + //get epoch of from/to slot to see if they're in the current stored epoch. + let from_epoch = self.epoch_data.get_epoch_at_slot(from).epoch; + let to_epoch = self.epoch_data.get_epoch_at_slot(to).epoch; + let leader_schedule_data = self.leader_schedule.read().await; + let current_epoch = leader_schedule_data + .current + .as_ref() + .map(|e| e.epoch) + .unwrap_or(from_epoch); + let next_epoch = leader_schedule_data + .current + .as_ref() + .map(|e| e.epoch) + .unwrap_or(to_epoch); + if from > to { + bail!("invalid arguments for get_slot_leaders"); + } + if from_epoch < current_epoch || from_epoch > next_epoch { + bail!("invalid arguments for get_slot_leaders"); + } + if to_epoch < current_epoch || to_epoch > next_epoch { + bail!("invalid arguments for get_slot_leaders"); + } + + let limit = to - from; + + let schedule = leader_schedule_data + .get_slot_leaders(from, limit, self.epoch_data.get_epoch_schedule()) + .await + .map_err(Error::msg)?; + + Ok(schedule + .into_iter() + .enumerate() + .map(|(index, pubkey)| LeaderData { + leader_slot: from + index as u64, + pubkey, + }) + .collect()) + } +} diff --git a/cluster-endpoints/src/lib.rs b/cluster-endpoints/src/lib.rs index 23060022..9d65054a 100644 --- a/cluster-endpoints/src/lib.rs +++ b/cluster-endpoints/src/lib.rs @@ -1,5 +1,6 @@ pub mod endpoint_stremers; pub mod grpc_inspect; +pub mod grpc_leaders_getter; pub mod grpc_multiplex; pub mod grpc_stream_utils; pub mod grpc_subscription; diff --git a/cluster-endpoints/src/rpc_polling/poll_slots.rs b/cluster-endpoints/src/rpc_polling/poll_slots.rs index d8ffc84e..337f2e3c 100644 --- a/cluster-endpoints/src/rpc_polling/poll_slots.rs +++ b/cluster-endpoints/src/rpc_polling/poll_slots.rs @@ -5,7 +5,7 @@ use solana_client::nonblocking::rpc_client::RpcClient; use solana_lite_rpc_core::{structures::slot_notification::SlotNotification, AnyhowJoinHandle}; use solana_sdk::{commitment_config::CommitmentConfig, slot_history::Slot}; use tokio::sync::broadcast::Sender; -const AVERAGE_SLOT_CHANGE_TIME: Duration = Duration::from_millis(400); +const AVERAGE_SLOT_CHANGE_TIME: Duration = Duration::from_millis(800); pub async fn poll_commitment_slots( rpc_client: Arc, diff --git a/config.example.json b/config.example.json index 665d2a91..701c96a4 100644 --- a/config.example.json +++ b/config.example.json @@ -10,6 +10,7 @@ "transaction_retry_after_secs": 3, "quic_proxy_addr": null, "use_grpc": false, + "activate_leader_schedule": false, "grpc_addr": "http://127.0.0.0:10000", "grpc_x_token": null, "postgres": { diff --git a/config.json b/config.json new file mode 100644 index 00000000..164e273e --- /dev/null +++ b/config.json @@ -0,0 +1,16 @@ +{ + "rpc_addr": "http://localhost:8899", + "ws_addr": "ws://localhost:8900", + "lite_rpc_http_addr": "[::]:8890", + "lite_rpc_ws_addr": "[::]:8891", + "fanout_size": 18, + "identity_keypair": null, + "prometheus_addr": "[::]:9091", + "maximum_retries_per_tx": 40, + "transaction_retry_after_secs": 3, + "quic_proxy_addr": null, + "use_grpc": false, + "activate_leader_schedule": false, + "grpc_addr": "http://127.0.0.0:10000", + "grpc_x_token": null +} diff --git a/lite-rpc/src/bridge.rs b/lite-rpc/src/bridge.rs index 7fafe7ec..39ec6549 100644 --- a/lite-rpc/src/bridge.rs +++ b/lite-rpc/src/bridge.rs @@ -65,10 +65,12 @@ pub struct LiteBridge { rpc_client: Arc, transaction_service: TransactionService, history: History, - state_vote_sendder: tokio::sync::mpsc::Sender<( - GetVoteAccountsConfig, - tokio::sync::oneshot::Sender, - )>, + state_vote_sendder: Option< + tokio::sync::mpsc::Sender<( + GetVoteAccountsConfig, + tokio::sync::oneshot::Sender, + )>, + >, } impl LiteBridge { @@ -77,10 +79,12 @@ impl LiteBridge { data_cache: DataCache, transaction_service: TransactionService, history: History, - state_vote_sendder: tokio::sync::mpsc::Sender<( - GetVoteAccountsConfig, - oneshot::Sender, - )>, + state_vote_sendder: Option< + tokio::sync::mpsc::Sender<( + GetVoteAccountsConfig, + oneshot::Sender, + )>, + >, ) -> Self { Self { rpc_client, @@ -507,14 +511,21 @@ impl LiteRpcServer for LiteBridge { ) -> crate::rpc::Result { let config: GetVoteAccountsConfig = GetVoteAccountsConfig::try_from(config.unwrap_or_default()).unwrap_or_default(); - let (tx, rx) = oneshot::channel(); - if let Err(err) = self.state_vote_sendder.send((config, tx)).await { - return Err(jsonrpsee::core::Error::Custom(format!( - "error during query processing:{err}", - ))); + if let Some(state_vote_sendder) = &self.state_vote_sendder { + let (tx, rx) = oneshot::channel(); + if let Err(err) = state_vote_sendder.send((config, tx)).await { + return Err(jsonrpsee::core::Error::Custom(format!( + "error during query processing:{err}", + ))); + } + rx.await.map_err(|err| { + jsonrpsee::core::Error::Custom(format!("error during query processing:{err}")) + }) + } else { + self.rpc_client + .get_vote_accounts() + .await + .map_err(|err| (jsonrpsee::core::Error::Custom(err.to_string()))) } - rx.await.map_err(|err| { - jsonrpsee::core::Error::Custom(format!("error during query processing:{err}")) - }) } } diff --git a/lite-rpc/src/cli.rs b/lite-rpc/src/cli.rs index c01e5d62..7ec41607 100644 --- a/lite-rpc/src/cli.rs +++ b/lite-rpc/src/cli.rs @@ -42,7 +42,8 @@ pub struct Config { pub quic_proxy_addr: Option, #[serde(default)] pub use_grpc: bool, - + #[serde(default)] + pub activate_leader_schedule: bool, #[serde(default = "Config::default_grpc_addr")] pub grpc_addr: String, #[serde(default)] diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index 940b8148..0e469c0c 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -11,6 +11,7 @@ use lite_rpc::{bridge::LiteBridge, cli::Args}; use lite_rpc::{DEFAULT_MAX_NUMBER_OF_TXS_IN_QUEUE, GRPC_VERSION, NB_SLOTS_TRANSACTIONS_TO_CACHE}; use log::info; use solana_lite_rpc_cluster_endpoints::endpoint_stremers::EndpointStreaming; +use solana_lite_rpc_cluster_endpoints::grpc_leaders_getter::GrpcLeaderGetter; use solana_lite_rpc_cluster_endpoints::grpc_subscription::create_grpc_subscription; use solana_lite_rpc_cluster_endpoints::grpc_subscription_autoreconnect::{ GrpcConnectionTimeouts, GrpcSourceConfig, @@ -31,6 +32,7 @@ use solana_lite_rpc_core::structures::{ epoch::EpochCache, identity_stakes::IdentityStakes, notifications::NotificationSender, produced_block::ProducedBlock, }; +use solana_lite_rpc_core::traits::leaders_fetcher_interface::LeaderFetcherInterface; use solana_lite_rpc_core::types::BlockStream; use solana_lite_rpc_core::AnyhowJoinHandle; use solana_lite_rpc_history::block_stores::inmemory_block_store::InmemoryBlockStore; @@ -99,6 +101,9 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: transaction_retry_after_secs, quic_proxy_addr, use_grpc, + activate_leader_schedule, + grpc_addr, + grpc_x_token, .. } = args; @@ -147,6 +152,7 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: info!("Got finalized block: {:?}", finalized_block.slot); let epoch_data = EpochCache::bootstrap_epoch(&rpc_client).await?; + let slots_per_epoch = epoch_data.get_epoch_schedule().slots_per_epoch; let block_information_store = BlockInformationStore::new(BlockInformation::from_block(&finalized_block)); @@ -164,13 +170,13 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: leader_schedule: Arc::new(RwLock::new(CalculatedSchedule::default())), }; - let lata_cache_service = DataCachingService { + let data_cache_service = DataCachingService { data_cache: data_cache.clone(), clean_duration: Duration::from_secs(120), }; // to avoid laggin we resubscribe to block notification - let data_caching_service = lata_cache_service.listen( + let data_caching_service = data_cache_service.listen( blocks_notifier.resubscribe(), slot_notifier.resubscribe(), cluster_info_notifier, @@ -199,7 +205,52 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: prometheus_addr, data_cache: data_cache.clone(), }; - let leader_schedule = Arc::new(JsonRpcLeaderGetter::new(rpc_client.clone(), 1024, 128)); + + //init grpc leader schedule and vote account is configured. + let (leader_schedule, rpc_stakes_send): (Arc, Option<_>) = + if use_grpc && activate_leader_schedule { + //init leader schedule grpc process. + //1) get stored schedule and stakes + if let Some((leader_schedule, vote_stakes)) = + solana_lite_rpc_stakevote::bootstrap_leaderschedule_from_files(slots_per_epoch) + { + data_cache + .identity_stakes + .update_stakes_for_identity(vote_stakes) + .await; + let mut data_schedule = data_cache.leader_schedule.write().await; + *data_schedule = leader_schedule; + } + //2) start stake vote and leader schedule. + let (rpc_stakes_send, rpc_stakes_recv) = mpsc::channel(1000); + let stake_vote_jh = solana_lite_rpc_stakevote::start_stakes_and_votes_loop( + data_cache.clone(), + slot_notifier.resubscribe(), + rpc_stakes_recv, + Arc::clone(&rpc_client), + grpc_addr, + ) + .await?; + + // + tokio::spawn(async move { + let err = stake_vote_jh.await; + log::error!("Vote and stake Services exit with error: {err:?}"); + }); + + ( + Arc::new(GrpcLeaderGetter::new( + Arc::clone(&data_cache.leader_schedule), + data_cache.epoch_data.clone(), + )), + Some(rpc_stakes_send), + ) + } else { + ( + Arc::new(JsonRpcLeaderGetter::new(rpc_client.clone(), 1024, 128)), + None, + ) + }; let tpu_service: TpuService = TpuService::new( tpu_config, @@ -221,17 +272,6 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: slot_notifier.resubscribe(), ); - //start stake vote and leader schedule. - let (rpc_stakes_send, rpc_stakes_recv) = mpsc::channel(1000); - let stake_vote_jh = solana_lite_rpc_stakevote::start_stakes_and_votes_loop( - data_cache.clone(), - slot_notifier.resubscribe(), - rpc_stakes_recv, - Arc::clone(&rpc_client), - grpc_addr, - ) - .await?; - drop(slot_notifier); let support_service = tokio::spawn(async move { spawner.spawn_support_services().await }); @@ -251,9 +291,6 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: .start(lite_rpc_http_addr, lite_rpc_ws_addr), ); tokio::select! { - res = stake_vote_jh => { - anyhow::bail!("Stakes Votes Services {res:?}") - } res = tx_service_jh => { anyhow::bail!("Tx Services {res:?}") } diff --git a/stake_vote/src/bootstrap.rs b/stake_vote/src/bootstrap.rs index 00f813cd..6931c1f0 100644 --- a/stake_vote/src/bootstrap.rs +++ b/stake_vote/src/bootstrap.rs @@ -12,6 +12,7 @@ use futures::future::join_all; use futures_util::stream::FuturesUnordered; use solana_client::client_error::ClientError; use solana_client::rpc_client::RpcClient; +use solana_client::rpc_response::RpcVoteAccountStatus; use solana_lite_rpc_core::stores::data_cache::DataCache; use solana_lite_rpc_core::structures::leaderschedule::CalculatedSchedule; use solana_lite_rpc_core::structures::leaderschedule::LeaderScheduleData; @@ -43,6 +44,21 @@ pub async fn bootstrap_scheduleepoch_data(data_cache: &DataCache) -> ScheduleEpo ) } +// Return the current and next epoxh leader schedule and the current epoch stakes of vote accounts +// if the corresponding files exist. +pub fn bootstrap_leaderschedule_from_files( + slots_in_epoch: u64, +) -> Option<(CalculatedSchedule, RpcVoteAccountStatus)> { + bootstrap_current_leader_schedule(slots_in_epoch) + .map(|(leader_schedule, current_epoch_stakes, _)| { + let vote_acccounts = crate::vote::get_rpc_vote_account_info_from_current_epoch_stakes( + ¤t_epoch_stakes, + ); + (leader_schedule, vote_acccounts) + }) + .ok() +} + /* Bootstrap state changes @@ -76,7 +92,7 @@ pub fn run_bootstrap_events( stakestore: &mut StakeStore, votestore: &mut VoteStore, slots_in_epoch: u64, -) -> anyhow::Result>> { +) -> anyhow::Result>> { let result = process_bootstrap_event(event, stakestore, votestore, slots_in_epoch); match result { BootsrapProcessResult::TaskHandle(jh) => { @@ -120,7 +136,7 @@ pub enum BootstrapEvent { VoteMap, EpochVoteStakesCache, String, - anyhow::Result, + anyhow::Result<(CalculatedSchedule, RpcVoteAccountStatus)>, ), Exit, } @@ -130,7 +146,7 @@ enum BootsrapProcessResult { TaskHandle(JoinHandle), Event(BootstrapEvent), Error(String), - End(anyhow::Result), + End(anyhow::Result<(CalculatedSchedule, RpcVoteAccountStatus)>), } fn process_bootstrap_event( @@ -230,6 +246,10 @@ fn process_bootstrap_event( match bootstrap_current_leader_schedule(slots_in_epoch) { Ok((leader_schedule, current_epoch_stakes, next_epoch_stakes)) => { + let vote_acccounts = + crate::vote::get_rpc_vote_account_info_from_current_epoch_stakes( + ¤t_epoch_stakes, + ); epoch_cache.add_stakes_for_epoch(current_epoch_stakes); epoch_cache.add_stakes_for_epoch(next_epoch_stakes); BootstrapEvent::AccountsMerged( @@ -237,7 +257,7 @@ fn process_bootstrap_event( vote_map, epoch_cache, rpc_url, - Ok(leader_schedule), + Ok((leader_schedule, vote_acccounts)), ) } Err(err) => BootstrapEvent::AccountsMerged( diff --git a/stake_vote/src/lib.rs b/stake_vote/src/lib.rs index 5f1b6b3d..8e589a1b 100644 --- a/stake_vote/src/lib.rs +++ b/stake_vote/src/lib.rs @@ -30,6 +30,8 @@ mod stake; mod utils; mod vote; +pub use bootstrap::bootstrap_leaderschedule_from_files; + const STAKESTORE_INITIAL_CAPACITY: usize = 600000; const VOTESTORE_INITIAL_CAPACITY: usize = 600000; @@ -245,7 +247,10 @@ pub async fn start_stakes_and_votes_loop( match crate::bootstrap::run_bootstrap_events(event, &mut spawned_bootstrap_task, &mut stakestore, &mut votestore, current_schedule_epoch.slots_in_epoch) { Ok(Some(boot_res))=> { match boot_res { - Ok(current_schedule_data) => { + Ok((current_schedule_data, vote_stakes)) => { + data_cache + .identity_stakes + .update_stakes_for_identity(vote_stakes).await; let mut data_schedule = data_cache.leader_schedule.write().await; *data_schedule = current_schedule_data; } diff --git a/stake_vote/src/vote.rs b/stake_vote/src/vote.rs index 4d44cf94..b9d305aa 100644 --- a/stake_vote/src/vote.rs +++ b/stake_vote/src/vote.rs @@ -282,3 +282,17 @@ pub fn get_rpc_vote_accounts_info( delinquent: delinquent_vote_accounts, } } + +pub fn get_rpc_vote_account_info_from_current_epoch_stakes( + current_epoch_stakes: &EpochVoteStakes, +) -> RpcVoteAccountStatus { + let current_vote_accounts: Vec = current_epoch_stakes + .vote_stakes + .values() + .map(|(stake, vote)| vote.convert_to_rpc_vote_account_info(*stake, true)) + .collect(); + RpcVoteAccountStatus { + current: current_vote_accounts, + delinquent: vec![], //no info about delinquent at startup. + } +} From df8b041051182e1cd955c83a784c97348dc87a08 Mon Sep 17 00:00:00 2001 From: musitdev Date: Wed, 27 Dec 2023 14:43:20 +0100 Subject: [PATCH 29/56] remove test config --- config.json | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 config.json diff --git a/config.json b/config.json deleted file mode 100644 index 164e273e..00000000 --- a/config.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "rpc_addr": "http://localhost:8899", - "ws_addr": "ws://localhost:8900", - "lite_rpc_http_addr": "[::]:8890", - "lite_rpc_ws_addr": "[::]:8891", - "fanout_size": 18, - "identity_keypair": null, - "prometheus_addr": "[::]:9091", - "maximum_retries_per_tx": 40, - "transaction_retry_after_secs": 3, - "quic_proxy_addr": null, - "use_grpc": false, - "activate_leader_schedule": false, - "grpc_addr": "http://127.0.0.0:10000", - "grpc_x_token": null -} From dbc152bb64660d406dfd9a7590443beb2c111d80 Mon Sep 17 00:00:00 2001 From: musitdev Date: Wed, 3 Jan 2024 16:50:16 +0100 Subject: [PATCH 30/56] rebase from master --- Cargo.lock | 696 ++++++++++++++++++++-------------------- lite-rpc/src/main.rs | 18 +- stake_vote/src/utils.rs | 2 +- 3 files changed, 351 insertions(+), 365 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0f469b76..c9980ce9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,9 +76,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" dependencies = [ "cfg-if", "getrandom 0.2.11", @@ -143,9 +143,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" +checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" dependencies = [ "anstyle", "anstyle-parse", @@ -163,37 +163,37 @@ checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] name = "anstyle-parse" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.1" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" [[package]] name = "arc-swap" @@ -255,7 +255,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ed4aa4fe255d0bc6d79373f7e31d2ea147bcf486cba1be5ba7ea85abdb92348" dependencies = [ - "quote 1.0.33", + "quote 1.0.35", "syn 1.0.109", ] @@ -267,8 +267,8 @@ checksum = "7abe79b0e4288889c4574159ab790824d0033b9fdcb2a112a3182fac2e514565" dependencies = [ "num-bigint 0.4.4", "num-traits 0.2.17", - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "syn 1.0.109", ] @@ -303,8 +303,8 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae3281bc6d0fd7e549af32b52511e1302185bd688fd3359fa36423346ff682ea" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "syn 1.0.109", ] @@ -364,8 +364,8 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "syn 1.0.109", "synstructure", ] @@ -376,8 +376,8 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "syn 1.0.109", ] @@ -447,20 +447,20 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.39", + "proc-macro2 1.0.74", + "quote 1.0.35", + "syn 2.0.46", ] [[package]] name = "async-trait" -version = "0.1.74" +version = "0.1.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.39", + "proc-macro2 1.0.74", + "quote 1.0.35", + "syn 2.0.46", ] [[package]] @@ -602,7 +602,7 @@ version = "0.2.3" dependencies = [ "anyhow", "bincode", - "clap 4.4.10", + "clap 4.4.12", "csv", "dashmap 5.5.3", "dirs", @@ -642,13 +642,13 @@ dependencies = [ "lazy_static", "lazycell", "peeking_take_while", - "prettyplease 0.2.15", - "proc-macro2 1.0.70", - "quote 1.0.33", + "prettyplease 0.2.16", + "proc-macro2 1.0.74", + "quote 1.0.35", "regex", "rustc-hash", "shlex", - "syn 2.0.39", + "syn 2.0.46", ] [[package]] @@ -740,7 +740,7 @@ dependencies = [ "borsh-derive-internal 0.9.3", "borsh-schema-derive-internal 0.9.3", "proc-macro-crate 0.1.5", - "proc-macro2 1.0.70", + "proc-macro2 1.0.74", "syn 1.0.109", ] @@ -753,7 +753,7 @@ dependencies = [ "borsh-derive-internal 0.10.3", "borsh-schema-derive-internal 0.10.3", "proc-macro-crate 0.1.5", - "proc-macro2 1.0.70", + "proc-macro2 1.0.74", "syn 1.0.109", ] @@ -763,8 +763,8 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5449c28a7b352f2d1e592a8a28bf139bc71afb0764a14f3c02500935d8c44065" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "syn 1.0.109", ] @@ -774,8 +774,8 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afb438156919598d2c7bad7e1c0adf3d26ed3840dbc010db1a882a65583ca2fb" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "syn 1.0.109", ] @@ -785,8 +785,8 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdbd5696d8bfa21d53d9fe39a714a18538bad11492a42d066dbbc395fb1951c0" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "syn 1.0.109", ] @@ -796,8 +796,8 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "634205cc43f74a1b9046ef87c4540ebda95696ec0f315024860cad7c5b0f5ccd" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "syn 1.0.109", ] @@ -830,9 +830,9 @@ checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" [[package]] name = "bstr" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c" +checksum = "c48f0051a4b4c5e0b6d365cd04af53aeaa209e3cc15ec2cdb69e73cc87fbd0dc" dependencies = [ "memchr", "serde", @@ -869,9 +869,9 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "965ab7eb5f8f97d2a083c799f3a1b994fc397b2fe2da5d1da1626ce15a39f2b1" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.39", + "proc-macro2 1.0.74", + "quote 1.0.35", + "syn 2.0.46", ] [[package]] @@ -977,9 +977,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" +checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" dependencies = [ "glob", "libc", @@ -1019,9 +1019,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.10" +version = "4.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fffed7514f420abec6d183b1d3acfd9099c79c3a10a06ade4f8203f1411272" +checksum = "dcfab8ba68f3668e89f6ff60f5b205cea56aa7b769451a59f34b8682f51c056d" dependencies = [ "clap_builder", "clap_derive", @@ -1029,9 +1029,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.9" +version = "4.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63361bae7eef3771745f02d8d892bec2fee5f6e34af316ba556e7f97a7069ff1" +checksum = "fb7fb5e4e979aec3be7791562fcba452f94ad85e954da024396433e0e25a79e9" dependencies = [ "anstream", "anstyle", @@ -1046,9 +1046,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck", - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.39", + "proc-macro2 1.0.74", + "quote 1.0.35", + "syn 2.0.46", ] [[package]] @@ -1087,9 +1087,9 @@ dependencies = [ [[package]] name = "concurrent-queue" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f057a694a54f12365049b0958a1685bb52d567f5593b355fbf685838e873d400" +checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" dependencies = [ "crossbeam-utils", ] @@ -1148,8 +1148,8 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a4f51209740b5e1589e702b3044cdd4562cef41b6da404904192ffffb852d62" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "syn 1.0.109", ] @@ -1167,9 +1167,9 @@ checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ "core-foundation-sys", "libc", @@ -1177,9 +1177,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "countmap" @@ -1210,9 +1210,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.8" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +checksum = "82a9b73a36529d9c47029b9fb3a6f0ea3cc916a261195352ba19e770fc1748b2" dependencies = [ "cfg-if", "crossbeam-utils", @@ -1220,9 +1220,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751" dependencies = [ "cfg-if", "crossbeam-epoch", @@ -1231,22 +1231,20 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.15" +version = "0.9.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +checksum = "0e3681d554572a651dda4186cd47240627c3d0114d45a95f6ad27f2f22e7548d" dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "memoffset 0.9.0", - "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" dependencies = [ "cfg-if", ] @@ -1339,10 +1337,10 @@ checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "strsim 0.10.0", - "syn 2.0.39", + "syn 2.0.46", ] [[package]] @@ -1352,8 +1350,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core", - "quote 1.0.33", - "syn 2.0.39", + "quote 1.0.35", + "syn 2.0.46", ] [[package]] @@ -1411,9 +1409,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ "powerfmt", ] @@ -1430,8 +1428,8 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "syn 1.0.109", ] @@ -1442,8 +1440,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case", - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "rustc_version", "syn 1.0.109", ] @@ -1482,9 +1480,9 @@ dependencies = [ [[package]] name = "dir-diff" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2860407d7d7e2e004bb2128510ad9e8d669e76fa005ccf567977b5d71b8b4a0b" +checksum = "a7ad16bf5f84253b50d6557681c58c3ab67c47c77d39fed9aeb56e947290bd10" dependencies = [ "walkdir", ] @@ -1516,9 +1514,9 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.39", + "proc-macro2 1.0.74", + "quote 1.0.35", + "syn 2.0.46", ] [[package]] @@ -1627,9 +1625,9 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.39", + "proc-macro2 1.0.74", + "quote 1.0.35", + "syn 2.0.46", ] [[package]] @@ -1696,14 +1694,14 @@ checksum = "835a3dc7d1ec9e75e2b5fb4ba75396837112d2060b03f7d43bc1897c7f7211da" [[package]] name = "filetime" -version = "0.2.22" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" +checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.3.5", - "windows-sys 0.48.0", + "redox_syscall 0.4.1", + "windows-sys 0.52.0", ] [[package]] @@ -1766,9 +1764,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -1781,9 +1779,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -1791,15 +1789,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -1808,32 +1806,32 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-macro" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.39", + "proc-macro2 1.0.74", + "quote 1.0.35", + "syn 2.0.46", ] [[package]] name = "futures-sink" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-timer" @@ -1847,9 +1845,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -1912,9 +1910,9 @@ dependencies = [ [[package]] name = "geyser-grpc-connector" -version = "0.7.0+yellowstone.1.11" +version = "0.7.1+yellowstone.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94e6b95f0c048f9062479859095cf4fd28977c8dddc897360c6429ae392589f" +checksum = "f11c9bfa33ff6c5cf688e53df9b2de84b47f932527445bc6e33649c22a7791f7" dependencies = [ "anyhow", "async-stream", @@ -1922,7 +1920,7 @@ dependencies = [ "bincode", "derive_more", "futures", - "itertools", + "itertools 0.10.5", "log", "merge-streams", "solana-sdk", @@ -2085,7 +2083,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ - "ahash 0.8.6", + "ahash 0.8.7", ] [[package]] @@ -2177,11 +2175,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.5" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2197,9 +2195,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", "http", @@ -2226,9 +2224,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" dependencies = [ "bytes", "futures-channel", @@ -2241,7 +2239,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2 0.5.5", "tokio", "tower-service", "tracing", @@ -2276,7 +2274,7 @@ dependencies = [ "http", "hyper", "log", - "rustls 0.21.9", + "rustls 0.21.10", "rustls-native-certs", "tokio", "tokio-rustls 0.24.1", @@ -2309,9 +2307,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.58" +version = "0.1.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -2442,9 +2440,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "jobserver" @@ -2573,8 +2571,8 @@ checksum = "d814a21d9a819f8de1a41b819a263ffd68e4bb5f043d936db1c49b54684bde0a" dependencies = [ "heck", "proc-macro-crate 1.3.1", - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "syn 1.0.109", ] @@ -2658,18 +2656,18 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.150" +version = "0.2.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" [[package]] name = "libloading" -version = "0.7.4" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +checksum = "c571b676ddfc9a8c12f1f3d3085a7b163966a8fd8098a90640953ce5f6170161" dependencies = [ "cfg-if", - "winapi", + "windows-sys 0.48.0", ] [[package]] @@ -2765,9 +2763,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" +checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" [[package]] name = "lite-rpc" @@ -2782,7 +2780,7 @@ dependencies = [ "bs58", "bytes", "chrono", - "clap 4.4.10", + "clap 4.4.12", "const_env", "dashmap 5.5.3", "dotenv", @@ -2890,9 +2888,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "memmap2" @@ -2966,9 +2964,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" dependencies = [ "libc", "wasi 0.11.0+wasi-snapshot-preview1", @@ -2991,8 +2989,8 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a7d5f7076603ebc68de2dc6a650ec331a062a13abaa346975be747bbfa4b789" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "syn 1.0.109", ] @@ -3105,8 +3103,8 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "syn 1.0.109", ] @@ -3116,9 +3114,9 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfb77679af88f8b125209d354a202862602672222e7f2313fdd6dc349bad4712" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.39", + "proc-macro2 1.0.74", + "quote 1.0.35", + "syn 2.0.46", ] [[package]] @@ -3207,9 +3205,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96667db765a921f7b295ffee8b60472b686a51d4f21c2ee4ffdb94c7013b65a6" dependencies = [ "proc-macro-crate 1.3.1", - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.39", + "proc-macro2 1.0.74", + "quote 1.0.35", + "syn 2.0.46", ] [[package]] @@ -3218,10 +3216,10 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c11e44798ad209ccdd91fc192f0526a369a01234f7373e1b141c96d7cee4f0e" dependencies = [ - "proc-macro-crate 2.0.0", - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.39", + "proc-macro-crate 2.0.1", + "proc-macro2 1.0.74", + "quote 1.0.35", + "syn 2.0.46", ] [[package]] @@ -3232,9 +3230,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] @@ -3250,9 +3248,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "opaque-debug" @@ -3262,9 +3260,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.60" +version = "0.10.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800" +checksum = "8cde4d2d9200ad5909f8dac647e29482e07c3a35de8a13fce7c9c7747ad9f671" dependencies = [ "bitflags 2.4.1", "cfg-if", @@ -3281,9 +3279,9 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.39", + "proc-macro2 1.0.74", + "quote 1.0.35", + "syn 2.0.46", ] [[package]] @@ -3294,18 +3292,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "300.1.6+3.1.4" +version = "300.2.1+3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439fac53e092cd7442a3660c85dde4643ab3b5bd39040912388dcdabf6b88085" +checksum = "3fe476c29791a5ca0d1273c697e96085bbabbbea2ef7afd5617e78a4b40332d3" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.96" +version = "0.9.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f" +checksum = "c1665caf8ab2dc9aef43d1c0023bd904633a6a05cb30b0ad59bec2ae986e57a7" dependencies = [ "cc", "libc", @@ -3344,8 +3342,8 @@ checksum = "5f7d21ccd03305a674437ee1248f3ab5d4b1db095cf1caf49f1713ddf61956b7" dependencies = [ "Inflector", "proc-macro-error", - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "syn 1.0.109", ] @@ -3500,9 +3498,9 @@ version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.39", + "proc-macro2 1.0.74", + "quote 1.0.35", + "syn 2.0.46", ] [[package]] @@ -3530,9 +3528,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" [[package]] name = "plain" @@ -3554,9 +3552,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" [[package]] name = "postgres-native-tls" @@ -3619,18 +3617,18 @@ version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" dependencies = [ - "proc-macro2 1.0.70", + "proc-macro2 1.0.74", "syn 1.0.109", ] [[package]] name = "prettyplease" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" +checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" dependencies = [ - "proc-macro2 1.0.70", - "syn 2.0.39", + "proc-macro2 1.0.74", + "syn 2.0.46", ] [[package]] @@ -3654,11 +3652,12 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8366a6159044a37876a2b9817124296703c586a5c92e2c53751fa06d8d43e8" +checksum = "97dc5fea232fc28d2f597b37c4876b348a40e33f3b02cc975c8d006d78d94b1a" dependencies = [ - "toml_edit 0.20.7", + "toml_datetime", + "toml_edit 0.20.2", ] [[package]] @@ -3668,8 +3667,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "syn 1.0.109", "version_check", ] @@ -3680,8 +3679,8 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "version_check", ] @@ -3696,9 +3695,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.70" +version = "1.0.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +checksum = "2de98502f212cfcea8d0bb305bd0f49d7ebdd75b64ba0a68f937d888f4e0d6db" dependencies = [ "unicode-ident", ] @@ -3773,11 +3772,11 @@ dependencies = [ "multimap", "once_cell", "petgraph", - "prettyplease 0.2.15", + "prettyplease 0.2.16", "prost 0.12.3", "prost-types 0.12.3", "regex", - "syn 2.0.39", + "syn 2.0.46", "tempfile", "which", ] @@ -3790,8 +3789,8 @@ checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" dependencies = [ "anyhow", "itertools 0.10.5", - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "syn 1.0.109", ] @@ -3803,9 +3802,9 @@ checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" dependencies = [ "anyhow", "itertools 0.11.0", - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.39", + "proc-macro2 1.0.74", + "quote 1.0.35", + "syn 2.0.46", ] [[package]] @@ -3911,11 +3910,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ - "proc-macro2 1.0.70", + "proc-macro2 1.0.74", ] [[package]] @@ -4051,15 +4050,6 @@ dependencies = [ "bitflags 1.3.2", ] -[[package]] -name = "redox_syscall" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_syscall" version = "0.4.1" @@ -4141,9 +4131,9 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" -version = "0.11.22" +version = "0.11.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" dependencies = [ "async-compression", "base64 0.21.5", @@ -4165,7 +4155,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.9", + "rustls 0.21.10", "rustls-pemfile", "serde", "serde_json", @@ -4201,9 +4191,9 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.5" +version = "0.17.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" +checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" dependencies = [ "cc", "getrandom 0.2.11", @@ -4276,15 +4266,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.25" +version = "0.38.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" +checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" dependencies = [ "bitflags 2.4.1", "errno", "libc", "linux-raw-sys", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -4301,12 +4291,12 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.9" +version = "0.21.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" +checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", - "ring 0.17.5", + "ring 0.17.7", "rustls-webpki 0.101.7", "sct", ] @@ -4348,7 +4338,7 @@ version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.17.5", + "ring 0.17.7", "untrusted 0.9.0", ] @@ -4360,9 +4350,9 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" [[package]] name = "same-file" @@ -4375,11 +4365,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -4403,9 +4393,9 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.39", + "proc-macro2 1.0.74", + "quote 1.0.35", + "syn 2.0.46", ] [[package]] @@ -4414,7 +4404,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.17.5", + "ring 0.17.7", "untrusted 0.9.0", ] @@ -4443,9 +4433,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" +checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" [[package]] name = "send_wrapper" @@ -4455,38 +4445,38 @@ checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" [[package]] name = "serde" -version = "1.0.193" +version = "1.0.194" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +checksum = "0b114498256798c94a0689e1a15fec6005dee8ac1f41de56404b67afc2a4b773" dependencies = [ "serde_derive", ] [[package]] name = "serde_bytes" -version = "0.11.12" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab33ec92f677585af6d88c65593ae2375adde54efdbf16d597f2cbc7a6d368ff" +checksum = "8b8497c313fd43ab992087548117643f6fcd935cbf36f176ffda0aacf9591734" dependencies = [ "serde", ] [[package]] name = "serde_derive" -version = "1.0.193" +version = "1.0.194" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +checksum = "a3385e45322e8f9931410f01b3031ec534c3947d0e94c18049af4d9f9907d4e0" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.39", + "proc-macro2 1.0.74", + "quote 1.0.35", + "syn 2.0.46", ] [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.110" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "6fbd975230bada99c8bb618e0c365c2eefa219158d5c6c29610fd09ff1833257" dependencies = [ "itoa", "ryu", @@ -4522,9 +4512,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "881b6f881b17d13214e5d494c939ebab463d01264ce1811e9d4ac3a882e7695f" dependencies = [ "darling", - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.39", + "proc-macro2 1.0.74", + "quote 1.0.35", + "syn 2.0.46", ] [[package]] @@ -4940,7 +4930,7 @@ version = "1.16.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d266bf0311bb403d31206aa2904b8741f57c7f5e27580b6810ad5e22fc7c3282" dependencies = [ - "ahash 0.8.6", + "ahash 0.8.7", "blake3", "block-buffer 0.10.4", "bs58", @@ -4973,10 +4963,10 @@ version = "1.16.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dfe18c5155015dcb494c6de84a03b725fcf90ec2006a047769018b94c2cf0de" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "rustc_version", - "syn 2.0.39", + "syn 2.0.46", ] [[package]] @@ -5055,12 +5045,8 @@ dependencies = [ "dashmap 5.5.3", "derive_more", "futures", -<<<<<<< HEAD "geyser-grpc-connector", - "itertools", -======= "itertools 0.10.5", ->>>>>>> a084068 (Stake and vote management. Integration of the main loop in current architecture) "lazy_static", "log", "merge-streams", @@ -5153,7 +5139,7 @@ dependencies = [ "bs58", "bytes", "chrono", - "clap 4.4.10", + "clap 4.4.12", "dashmap 5.5.3", "dotenv", "futures", @@ -5191,7 +5177,7 @@ dependencies = [ "bs58", "bytes", "chrono", - "clap 4.4.10", + "clap 4.4.12", "countmap", "crossbeam-channel", "dashmap 5.5.3", @@ -5376,7 +5362,7 @@ version = "1.16.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f900c1015844087cd4f10ba9d2d26a9859f2f5ca07427865cc74942595abc0a7" dependencies = [ - "ahash 0.8.6", + "ahash 0.8.7", "bincode", "bv", "caps", @@ -5752,10 +5738,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e560806a3859717eb2220b26e2cd68bb757b63affa3e79c3f1d8d853b5ee78f" dependencies = [ "bs58", - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "rustversion", - "syn 2.0.39", + "syn 2.0.46", ] [[package]] @@ -6108,9 +6094,9 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fadbefec4f3c678215ca72bd71862697bb06b41fd77c0088902dd3203354387b" dependencies = [ - "quote 1.0.33", + "quote 1.0.35", "spl-discriminator-syn", - "syn 2.0.39", + "syn 2.0.46", ] [[package]] @@ -6119,10 +6105,10 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e5f2044ca42c8938d54d1255ce599c79a1ffd86b677dfab695caa20f9ffc3f2" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "sha2 0.10.8", - "syn 2.0.39", + "syn 2.0.46", "thiserror", ] @@ -6176,10 +6162,10 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5269c8e868da17b6552ef35a51355a017bd8e0eae269c201fef830d35fa52c" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "sha2 0.10.8", - "syn 2.0.39", + "syn 2.0.46", ] [[package]] @@ -6321,8 +6307,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ "heck", - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "rustversion", "syn 1.0.109", ] @@ -6356,19 +6342,19 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.39" +version = "2.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +checksum = "89456b690ff72fddcecf231caedbe615c59480c93358a93dfae7fc29e3ebbf0e" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "unicode-ident", ] @@ -6384,8 +6370,8 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", + "proc-macro2 1.0.74", + "quote 1.0.35", "syn 1.0.109", "unicode-xid 0.2.4", ] @@ -6424,15 +6410,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.8.1" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" dependencies = [ "cfg-if", "fastrand", "redox_syscall 0.4.1", "rustix", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -6461,22 +6447,22 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.50" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.50" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.39", + "proc-macro2 1.0.74", + "quote 1.0.35", + "syn 2.0.46", ] [[package]] @@ -6491,9 +6477,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" +checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" dependencies = [ "deranged", "itoa", @@ -6511,9 +6497,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" dependencies = [ "time-core", ] @@ -6554,9 +6540,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.34.0" +version = "1.35.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" +checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" dependencies = [ "backtrace", "bytes", @@ -6587,9 +6573,9 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.39", + "proc-macro2 1.0.74", + "quote 1.0.35", + "syn 2.0.46", ] [[package]] @@ -6645,7 +6631,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.9", + "rustls 0.21.10", "tokio", ] @@ -6702,9 +6688,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.5" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" [[package]] name = "toml_edit" @@ -6719,9 +6705,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.20.7" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" +checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338" dependencies = [ "indexmap 2.1.0", "toml_datetime", @@ -6782,7 +6768,7 @@ dependencies = [ "percent-encoding", "pin-project", "prost 0.12.3", - "rustls 0.21.9", + "rustls 0.21.10", "rustls-native-certs", "rustls-pemfile", "tokio", @@ -6801,9 +6787,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5bf5e9b9c0f7e0a7c027dcfaba7b2c60816c7049171f679d99ee2ff65d0de8c4" dependencies = [ "prettyplease 0.1.25", - "proc-macro2 1.0.70", + "proc-macro2 1.0.74", "prost-build 0.11.9", - "quote 1.0.33", + "quote 1.0.35", "syn 1.0.109", ] @@ -6813,11 +6799,11 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d021fc044c18582b9a2408cd0dd05b1596e3ecdb5c4df822bb0183545683889" dependencies = [ - "prettyplease 0.2.15", - "proc-macro2 1.0.70", + "prettyplease 0.2.16", + "proc-macro2 1.0.74", "prost-build 0.12.3", - "quote 1.0.33", - "syn 2.0.39", + "quote 1.0.35", + "syn 2.0.46", ] [[package]] @@ -6883,9 +6869,9 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.39", + "proc-macro2 1.0.74", + "quote 1.0.35", + "syn 2.0.46", ] [[package]] @@ -6945,9 +6931,9 @@ checksum = "0de5f738ceab88e2491a94ddc33c3feeadfa95fedc60363ef110845df12f3878" [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" @@ -6979,9 +6965,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" [[package]] name = "unicode-ident" @@ -7160,9 +7146,9 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.39", + "proc-macro2 1.0.74", + "quote 1.0.35", + "syn 2.0.46", "wasm-bindgen-shared", ] @@ -7184,7 +7170,7 @@ version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" dependencies = [ - "quote 1.0.33", + "quote 1.0.35", "wasm-bindgen-macro-support", ] @@ -7194,9 +7180,9 @@ version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.39", + "proc-macro2 1.0.74", + "quote 1.0.35", + "syn 2.0.46", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -7223,7 +7209,7 @@ version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" dependencies = [ - "ring 0.17.5", + "ring 0.17.7", "untrusted 0.9.0", ] @@ -7306,11 +7292,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.51.1" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.48.5", + "windows-targets 0.52.0", ] [[package]] @@ -7528,9 +7514,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.19" +version = "0.5.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b" +checksum = "97a4882e6b134d6c28953a387571f1acdd3496830d5e36c5e3a1075580ea641c" dependencies = [ "memchr", ] @@ -7565,11 +7551,13 @@ dependencies = [ [[package]] name = "xattr" -version = "1.0.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4686009f71ff3e5c4dbcf1a282d0a44db3f021ba69350cd42086b3e5f1c6985" +checksum = "914566e6413e7fa959cc394fb30e563ba80f3541fbd40816d4c05a0fc3f2a0f1" dependencies = [ "libc", + "linux-raw-sys", + "rustix", ] [[package]] @@ -7615,22 +7603,22 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.26" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e97e415490559a91254a2979b4829267a57d2fcd741a98eee8b722fb57289aa0" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.26" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.39", + "proc-macro2 1.0.74", + "quote 1.0.35", + "syn 2.0.46", ] [[package]] @@ -7648,9 +7636,9 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.39", + "proc-macro2 1.0.74", + "quote 1.0.35", + "syn 2.0.46", ] [[package]] diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index 0e469c0c..f7b3f4e1 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -1,14 +1,13 @@ pub mod rpc_tester; -use crate::rpc_tester::RpcTester; use crate::rpc_tester::RpcTester; use anyhow::bail; use dashmap::DashMap; +use lite_rpc::bridge::LiteBridge; use lite_rpc::cli::Config; use lite_rpc::postgres_logger::PostgresLogger; use lite_rpc::service_spawner::ServiceSpawner; -use lite_rpc::{bridge::LiteBridge, cli::Args}; -use lite_rpc::{DEFAULT_MAX_NUMBER_OF_TXS_IN_QUEUE, GRPC_VERSION, NB_SLOTS_TRANSACTIONS_TO_CACHE}; +use lite_rpc::DEFAULT_MAX_NUMBER_OF_TXS_IN_QUEUE; use log::info; use solana_lite_rpc_cluster_endpoints::endpoint_stremers::EndpointStreaming; use solana_lite_rpc_cluster_endpoints::grpc_leaders_getter::GrpcLeaderGetter; @@ -38,7 +37,6 @@ use solana_lite_rpc_core::AnyhowJoinHandle; use solana_lite_rpc_history::block_stores::inmemory_block_store::InmemoryBlockStore; use solana_lite_rpc_history::history::History; use solana_lite_rpc_history::postgres::postgres_config::PostgresSessionConfig; -use solana_lite_rpc_history::postgres::postgres_config::PostgresSessionConfig; use solana_lite_rpc_history::postgres::postgres_session::PostgresSessionCache; use solana_lite_rpc_services::data_caching_service::DataCachingService; use solana_lite_rpc_services::tpu_utils::tpu_connection_path::TpuConnectionPath; @@ -90,6 +88,7 @@ pub async fn start_postgres( pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow::Result<()> { let grpc_sources = args.get_grpc_sources(); + log::info!("grpc_sources:{grpc_sources:?}"); let Config { lite_rpc_ws_addr, lite_rpc_http_addr, @@ -103,7 +102,6 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: use_grpc, activate_leader_schedule, grpc_addr, - grpc_x_token, .. } = args; @@ -139,7 +137,7 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: info!("Creating RPC poll subscription..."); create_json_rpc_polling_subscription(rpc_client.clone())? }; - + log::info!("ici1"); let EndpointStreaming { blocks_notifier, cluster_info_notifier, @@ -183,7 +181,7 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: vote_account_notifier, ); drop(blocks_notifier); - + log::info!("ici1"); let (notification_channel, postgres) = start_postgres(postgres).await?; let tpu_config = TpuServiceConfig { @@ -205,7 +203,7 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: prometheus_addr, data_cache: data_cache.clone(), }; - + log::info!("ici2"); //init grpc leader schedule and vote account is configured. let (leader_schedule, rpc_stakes_send): (Arc, Option<_>) = if use_grpc && activate_leader_schedule { @@ -247,11 +245,11 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: ) } else { ( - Arc::new(JsonRpcLeaderGetter::new(rpc_client.clone(), 1024, 128)), + Arc::new(JsonRpcLeaderGetter::new(rpc_client.clone(), 40, 12)), None, ) }; - + log::info!("ici4"); let tpu_service: TpuService = TpuService::new( tpu_config, validator_identity, diff --git a/stake_vote/src/utils.rs b/stake_vote/src/utils.rs index a26ac4eb..65cba17e 100644 --- a/stake_vote/src/utils.rs +++ b/stake_vote/src/utils.rs @@ -115,7 +115,7 @@ impl<'a, T, C: TakableContent> Takable for &'a mut TakableMap { } } - fn merge(mut self, mut content: C) -> anyhow::Result<()> { + fn merge(self, mut content: C) -> anyhow::Result<()> { if self.content.is_none() { //apply stake added during extraction. for val in self.updates.drain(..) { From 2e7d86b8968ec0107e9c7145cc410f531137cd2a Mon Sep 17 00:00:00 2001 From: musitdev Date: Thu, 4 Jan 2024 09:25:46 +0100 Subject: [PATCH 31/56] rm trace comment --- lite-rpc/src/main.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index f7b3f4e1..0201cc39 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -133,6 +133,12 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: }) .collect(), )? + + // create_grpc_subscription( + // rpc_client.clone(), + // grpc_addr.clone(), + // GRPC_VERSION.to_string(), + // )? } else { info!("Creating RPC poll subscription..."); create_json_rpc_polling_subscription(rpc_client.clone())? @@ -181,7 +187,7 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: vote_account_notifier, ); drop(blocks_notifier); - log::info!("ici1"); + let (notification_channel, postgres) = start_postgres(postgres).await?; let tpu_config = TpuServiceConfig { From 77853a34ca2b45e75e6048f4ce0e9e514b32baab Mon Sep 17 00:00:00 2001 From: musitdev Date: Thu, 4 Jan 2024 09:56:55 +0100 Subject: [PATCH 32/56] rm trace comment --- lite-rpc/src/main.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index 0201cc39..80439247 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -143,7 +143,6 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: info!("Creating RPC poll subscription..."); create_json_rpc_polling_subscription(rpc_client.clone())? }; - log::info!("ici1"); let EndpointStreaming { blocks_notifier, cluster_info_notifier, @@ -209,7 +208,6 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: prometheus_addr, data_cache: data_cache.clone(), }; - log::info!("ici2"); //init grpc leader schedule and vote account is configured. let (leader_schedule, rpc_stakes_send): (Arc, Option<_>) = if use_grpc && activate_leader_schedule { @@ -255,7 +253,6 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: None, ) }; - log::info!("ici4"); let tpu_service: TpuService = TpuService::new( tpu_config, validator_identity, From 772e2c99092313e39b32cd2ecc23d9f482d9c27c Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 4 Jan 2024 11:12:45 +0100 Subject: [PATCH 33/56] reset values for leader schedule cache --- lite-rpc/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index 80439247..a1d2de5b 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -249,7 +249,7 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: ) } else { ( - Arc::new(JsonRpcLeaderGetter::new(rpc_client.clone(), 40, 12)), + Arc::new(JsonRpcLeaderGetter::new(rpc_client.clone(), 1024, 128)), None, ) }; From 402ebd69b15da0894234d2744d112c36cb1c3aca Mon Sep 17 00:00:00 2001 From: musitdev Date: Thu, 4 Jan 2024 13:36:13 +0100 Subject: [PATCH 34/56] correct from PR remarks --- .../src/rpc_polling/poll_slots.rs | 2 +- config.example.json | 2 +- core/src/structures/leaderschedule.rs | 2 +- lite-rpc/src/cli.rs | 2 +- lite-rpc/src/main.rs | 4 ++-- stake_vote/src/account.rs | 8 +++---- stake_vote/src/bootstrap.rs | 2 +- stake_vote/src/lib.rs | 24 +++++++++---------- 8 files changed, 23 insertions(+), 23 deletions(-) diff --git a/cluster-endpoints/src/rpc_polling/poll_slots.rs b/cluster-endpoints/src/rpc_polling/poll_slots.rs index 337f2e3c..d8ffc84e 100644 --- a/cluster-endpoints/src/rpc_polling/poll_slots.rs +++ b/cluster-endpoints/src/rpc_polling/poll_slots.rs @@ -5,7 +5,7 @@ use solana_client::nonblocking::rpc_client::RpcClient; use solana_lite_rpc_core::{structures::slot_notification::SlotNotification, AnyhowJoinHandle}; use solana_sdk::{commitment_config::CommitmentConfig, slot_history::Slot}; use tokio::sync::broadcast::Sender; -const AVERAGE_SLOT_CHANGE_TIME: Duration = Duration::from_millis(800); +const AVERAGE_SLOT_CHANGE_TIME: Duration = Duration::from_millis(400); pub async fn poll_commitment_slots( rpc_client: Arc, diff --git a/config.example.json b/config.example.json index 701c96a4..320fbefd 100644 --- a/config.example.json +++ b/config.example.json @@ -10,7 +10,7 @@ "transaction_retry_after_secs": 3, "quic_proxy_addr": null, "use_grpc": false, - "activate_leader_schedule": false, + "calculate_leader_schedule_form_geyser": false, "grpc_addr": "http://127.0.0.0:10000", "grpc_x_token": null, "postgres": { diff --git a/core/src/structures/leaderschedule.rs b/core/src/structures/leaderschedule.rs index 68bb79e2..88aef66a 100644 --- a/core/src/structures/leaderschedule.rs +++ b/core/src/structures/leaderschedule.rs @@ -48,7 +48,7 @@ impl CalculatedSchedule { commitment: Option, data_cache: &DataCache, ) -> Option>> { - log::info!( + log::debug!( "get_leader_schedule_for_slot current:{:?} next:{:?} ", self.current.clone().unwrap_or_default(), self.next.clone().unwrap_or_default() diff --git a/lite-rpc/src/cli.rs b/lite-rpc/src/cli.rs index 7ec41607..5d398929 100644 --- a/lite-rpc/src/cli.rs +++ b/lite-rpc/src/cli.rs @@ -43,7 +43,7 @@ pub struct Config { #[serde(default)] pub use_grpc: bool, #[serde(default)] - pub activate_leader_schedule: bool, + pub calculate_leader_schedule_form_geyser: bool, #[serde(default = "Config::default_grpc_addr")] pub grpc_addr: String, #[serde(default)] diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index a1d2de5b..979b9231 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -100,7 +100,7 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: transaction_retry_after_secs, quic_proxy_addr, use_grpc, - activate_leader_schedule, + calculate_leader_schedule_form_geyser, grpc_addr, .. } = args; @@ -210,7 +210,7 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: }; //init grpc leader schedule and vote account is configured. let (leader_schedule, rpc_stakes_send): (Arc, Option<_>) = - if use_grpc && activate_leader_schedule { + if use_grpc && calculate_leader_schedule_form_geyser { //init leader schedule grpc process. //1) get stored schedule and stakes if let Some((leader_schedule, vote_stakes)) = diff --git a/stake_vote/src/account.rs b/stake_vote/src/account.rs index dabdb961..ab1a03fc 100644 --- a/stake_vote/src/account.rs +++ b/stake_vote/src/account.rs @@ -23,14 +23,14 @@ pub struct AccountPretty { } impl AccountPretty { - pub fn new_from_geyzer( + pub fn new_from_geyser( geyser_account: SubscribeUpdateAccount, current_slot: u64, ) -> Option { let Some(inner_account) = geyser_account.account else { - log::warn!("Receive a SubscribeUpdateAccount without account."); - return None; - }; + log::warn!("Receive a SubscribeUpdateAccount without account."); + return None; + }; if geyser_account.slot != current_slot { log::trace!( diff --git a/stake_vote/src/bootstrap.rs b/stake_vote/src/bootstrap.rs index 6931c1f0..6a426ca4 100644 --- a/stake_vote/src/bootstrap.rs +++ b/stake_vote/src/bootstrap.rs @@ -28,7 +28,7 @@ use tokio::task::JoinHandle; pub const CURRENT_EPOCH_VOTE_STAKES_FILE: &str = "current_vote_stakes.json"; pub const NEXT_EPOCH_VOTE_STAKES_FILE: &str = "next_vote_stakes.json"; -pub async fn bootstrap_scheduleepoch_data(data_cache: &DataCache) -> ScheduleEpochData { +pub async fn bootstrap_schedule_epoch_data(data_cache: &DataCache) -> ScheduleEpochData { let new_rate_activation_epoch = solana_sdk::feature_set::FeatureSet::default() .new_warmup_cooldown_rate_epoch(data_cache.epoch_data.get_epoch_schedule()); diff --git a/stake_vote/src/lib.rs b/stake_vote/src/lib.rs index 8e589a1b..912c6250 100644 --- a/stake_vote/src/lib.rs +++ b/stake_vote/src/lib.rs @@ -48,9 +48,9 @@ pub async fn start_stakes_and_votes_loop( grpc_url: String, ) -> anyhow::Result> { log::info!("Start Stake and Vote loop on :{grpc_url}."); - let mut stake_vote_geyzer_stream = subscribe_geyzer_stake_vote_owner(grpc_url.clone()).await?; - let mut stake_history_geyzer_stream = subscribe_geyzer_stake_history(grpc_url).await?; - log::info!("Stake and Vote geyzer subscription done."); + let mut stake_vote_geyser_stream = subscribe_geyser_stake_vote_owner(grpc_url.clone()).await?; + let mut stake_history_geyser_stream = subscribe_geyser_stake_history(grpc_url).await?; + log::info!("Stake and Vote geyser subscription done."); let jh = tokio::spawn(async move { //Stake account management struct let mut stakestore = stake::StakeStore::new(STAKESTORE_INITIAL_CAPACITY); @@ -60,7 +60,7 @@ pub async fn start_stakes_and_votes_loop( //Init bootstrap process let mut current_schedule_epoch = - crate::bootstrap::bootstrap_scheduleepoch_data(&data_cache).await; + crate::bootstrap::bootstrap_schedule_epoch_data(&data_cache).await; //future execution collection. let mut spawned_leader_schedule_task = FuturesUnordered::new(); @@ -136,7 +136,7 @@ pub async fn start_stakes_and_votes_loop( rpc_request_processor.take_vote_accounts_and_process(&mut votestore, current_slot, epoch, config).await; } //manage geyser stake_history notification - ret = stake_history_geyzer_stream.next() => { + ret = stake_history_geyser_stream.next() => { match ret { Some(Ok(msg)) => { if let Some(UpdateOneof::Account(account)) = msg.update_oneof { @@ -158,7 +158,7 @@ pub async fn start_stakes_and_votes_loop( } } } - None => log::error!("Bootstrap error, can't read stake history from geyzer account data."), + None => log::error!("Bootstrap error, can't read stake history from geyser account data."), } } } @@ -174,7 +174,7 @@ pub async fn start_stakes_and_votes_loop( //manage geyser account notification //Geyser delete account notification patch must be installed on the validator. //see https://github.com/solana-labs/solana/pull/33292 - ret = stake_vote_geyzer_stream.next() => { + ret = stake_vote_geyser_stream.next() => { match ret { Some(message) => { //process the message @@ -182,7 +182,7 @@ pub async fn start_stakes_and_votes_loop( Ok(msg) => { match msg.update_oneof { Some(UpdateOneof::Account(account)) => { - // log::info!("Stake and Vote geyzer receive an account:{}.", + // log::info!("Stake and Vote geyser receive an account:{}.", // account.account.clone().map(|a| // solana_sdk::pubkey::Pubkey::try_from(a.pubkey).map(|k| k.to_string()) // .unwrap_or("bad pubkey".to_string()).to_string()) @@ -191,7 +191,7 @@ pub async fn start_stakes_and_votes_loop( //store new account stake. let current_slot = solana_lite_rpc_core::solana_utils::get_current_confirmed_slot(&data_cache).await; - if let Some(account) = AccountPretty::new_from_geyzer(account, current_slot) { + if let Some(account) = AccountPretty::new_from_geyser(account, current_slot) { match account.owner { solana_sdk::stake::program::ID => { log::trace!("Geyser notif stake account:{}", account); @@ -260,7 +260,7 @@ pub async fn start_stakes_and_votes_loop( } log::info!("Bootstrap done."); //update current epoch to manage epoch change during bootstrap. - current_schedule_epoch = crate::bootstrap::bootstrap_scheduleepoch_data(&data_cache).await; + current_schedule_epoch = crate::bootstrap::bootstrap_schedule_epoch_data(&data_cache).await; bootstrap_done = true; }, @@ -293,7 +293,7 @@ pub async fn start_stakes_and_votes_loop( } //subscribe Geyser grpc -async fn subscribe_geyzer_stake_vote_owner( +async fn subscribe_geyser_stake_vote_owner( grpc_url: String, ) -> anyhow::Result>> { let mut client = GeyserGrpcClient::connect(grpc_url, None::<&'static str>, None)?; @@ -330,7 +330,7 @@ async fn subscribe_geyzer_stake_vote_owner( } //subscribe Geyser grpc -async fn subscribe_geyzer_stake_history( +async fn subscribe_geyser_stake_history( grpc_url: String, ) -> anyhow::Result>> { let mut client = GeyserGrpcClient::connect(grpc_url, None::<&'static str>, None)?; From 21a8f12ed42e8be3f5a22e73a5d47a7cd639efc7 Mon Sep 17 00:00:00 2001 From: musitdev Date: Thu, 4 Jan 2024 14:05:11 +0100 Subject: [PATCH 35/56] correct clippy errors --- Cargo.toml | 2 ++ stake_vote/src/utils.rs | 10 ++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 56b79b16..095e9253 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,6 @@ [workspace] +resolver = "2" + members = [ "core", "services", diff --git a/stake_vote/src/utils.rs b/stake_vote/src/utils.rs index 65cba17e..bf8d5806 100644 --- a/stake_vote/src/utils.rs +++ b/stake_vote/src/utils.rs @@ -280,22 +280,24 @@ mod tests { assert_eq!(takable.updates.len(), 0); //wait for notifier - if let Err(_) = tokio::time::timeout(std::time::Duration::from_millis(1000), notif_jh).await + if tokio::time::timeout(std::time::Duration::from_millis(1000), notif_jh) + .await + .is_err() { - assert!(false, "take notifier timeout"); + panic!("take notifier timeout"); } } fn assert_take_content_map(take_content: &TakeResult>, len: usize) { match take_content { - TakeResult::Taken(_) => assert!(false), + TakeResult::Taken(_) => unreachable!(), TakeResult::Map(content) => assert_eq!(content.len(), len), } } fn assert_take_content_taken(take_content: &TakeResult>) { match take_content { TakeResult::Taken(_) => (), - TakeResult::Map(_) => assert!(false), + TakeResult::Map(_) => unreachable!(), } } } From 4fb8df5b7beee066220789967be27ed9ff822ff8 Mon Sep 17 00:00:00 2001 From: musitdev Date: Thu, 4 Jan 2024 14:44:07 +0100 Subject: [PATCH 36/56] correct CI --- .github/workflows/build_test.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 013dd44f..1de7bb4d 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -46,11 +46,6 @@ jobs: - name: Early Build run: | cargo build --locked --workspace --all-targets - - - name: Run fmt+clippy - run: | - cargo fmt --all --check - cargo clippy --locked --workspace --all-targets -D warnings - name: Run Tests run: RUST_LOG=info cargo test From 032232f6f382e63a11214f6f5f6d212eef5116aa Mon Sep 17 00:00:00 2001 From: musitdev Date: Fri, 5 Jan 2024 15:58:25 +0100 Subject: [PATCH 37/56] add rpc leader shedule loading if the bootstrap file is not present --- cluster-endpoints/src/grpc_leaders_getter.rs | 19 ++- core/src/structures/epoch.rs | 15 +- lite-rpc/src/main.rs | 25 ++-- stake_vote/src/bootstrap.rs | 145 ++++++++++++++++--- stake_vote/src/epoch.rs | 4 +- stake_vote/src/leader_schedule.rs | 5 +- stake_vote/src/lib.rs | 48 +++++- stake_vote/src/vote.rs | 6 +- 8 files changed, 212 insertions(+), 55 deletions(-) diff --git a/cluster-endpoints/src/grpc_leaders_getter.rs b/cluster-endpoints/src/grpc_leaders_getter.rs index 2bb3afc2..da317985 100644 --- a/cluster-endpoints/src/grpc_leaders_getter.rs +++ b/cluster-endpoints/src/grpc_leaders_getter.rs @@ -44,15 +44,26 @@ impl LeaderFetcherInterface for GrpcLeaderGetter { .current .as_ref() .map(|e| e.epoch) - .unwrap_or(to_epoch); + .unwrap_or(to_epoch) + + 1; if from > to { - bail!("invalid arguments for get_slot_leaders"); + bail!( + "invalid arguments for get_slot_leaders: from:{from} to:{to} from:{from} > to:{to}" + ); } if from_epoch < current_epoch || from_epoch > next_epoch { - bail!("invalid arguments for get_slot_leaders"); + bail!( + "invalid arguments for get_slot_leaders: from:{from} to:{to} \ + from_epoch:{from_epoch} < current_epoch:{current_epoch} \ + || from_epoch > next_epoch:{next_epoch}" + ); } if to_epoch < current_epoch || to_epoch > next_epoch { - bail!("invalid arguments for get_slot_leaders"); + bail!( + "invalid arguments for get_slot_leaders: from:{from} to:{to} \ + to_epoch:{to_epoch} < current_epoch:{current_epoch} \ + || to_epoch:{to_epoch} > next_epoch:{next_epoch}" + ); } let limit = to - from; diff --git a/core/src/structures/epoch.rs b/core/src/structures/epoch.rs index 15cb3a1d..c0ea1572 100644 --- a/core/src/structures/epoch.rs +++ b/core/src/structures/epoch.rs @@ -60,7 +60,9 @@ impl EpochCache { self.epoch_schedule.get_last_slot_in_epoch(epoch) } - pub async fn bootstrap_epoch(rpc_client: &RpcClient) -> anyhow::Result { + pub async fn bootstrap_epoch( + rpc_client: &RpcClient, + ) -> anyhow::Result<(EpochCache, EpochInfo)> { let res_epoch = rpc_client .get_account(&solana_sdk::sysvar::epoch_schedule::id()) .await?; @@ -72,9 +74,14 @@ impl EpochCache { bail!("Error during bootstrap epoch. SysvarAccountType::EpochSchedule can't be deserilized. Epoch can't be calculated."); }; - Ok(EpochCache { - epoch_schedule: Arc::new(epoch_schedule), - }) + let epoch_info = rpc_client.get_epoch_info().await?; + + Ok(( + EpochCache { + epoch_schedule: Arc::new(epoch_schedule), + }, + epoch_info, + )) } } diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index 979b9231..2d14bad4 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -154,8 +154,7 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: get_latest_block(blocks_notifier.resubscribe(), CommitmentConfig::finalized()).await; info!("Got finalized block: {:?}", finalized_block.slot); - let epoch_data = EpochCache::bootstrap_epoch(&rpc_client).await?; - let slots_per_epoch = epoch_data.get_epoch_schedule().slots_per_epoch; + let (epoch_data, current_epoch_info) = EpochCache::bootstrap_epoch(&rpc_client).await?; let block_information_store = BlockInformationStore::new(BlockInformation::from_block(&finalized_block)); @@ -212,17 +211,15 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: let (leader_schedule, rpc_stakes_send): (Arc, Option<_>) = if use_grpc && calculate_leader_schedule_form_geyser { //init leader schedule grpc process. - //1) get stored schedule and stakes - if let Some((leader_schedule, vote_stakes)) = - solana_lite_rpc_stakevote::bootstrap_leaderschedule_from_files(slots_per_epoch) - { - data_cache - .identity_stakes - .update_stakes_for_identity(vote_stakes) - .await; - let mut data_schedule = data_cache.leader_schedule.write().await; - *data_schedule = leader_schedule; - } + + //1) get stored leader schedule and stakes (or via RPC if not present) + solana_lite_rpc_stakevote::bootstrat_literpc_leader_schedule( + rpc_client.url(), + &data_cache, + current_epoch_info.epoch, + ) + .await; + //2) start stake vote and leader schedule. let (rpc_stakes_send, rpc_stakes_recv) = mpsc::channel(1000); let stake_vote_jh = solana_lite_rpc_stakevote::start_stakes_and_votes_loop( @@ -249,7 +246,7 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: ) } else { ( - Arc::new(JsonRpcLeaderGetter::new(rpc_client.clone(), 1024, 128)), + Arc::new(JsonRpcLeaderGetter::new(rpc_client.clone(), 40, 12)), None, ) }; diff --git a/stake_vote/src/bootstrap.rs b/stake_vote/src/bootstrap.rs index 6a426ca4..cc749dbc 100644 --- a/stake_vote/src/bootstrap.rs +++ b/stake_vote/src/bootstrap.rs @@ -11,14 +11,19 @@ use anyhow::bail; use futures::future::join_all; use futures_util::stream::FuturesUnordered; use solana_client::client_error::ClientError; +use solana_client::client_error::ClientErrorKind; use solana_client::rpc_client::RpcClient; use solana_client::rpc_response::RpcVoteAccountStatus; use solana_lite_rpc_core::stores::data_cache::DataCache; use solana_lite_rpc_core::structures::leaderschedule::CalculatedSchedule; use solana_lite_rpc_core::structures::leaderschedule::LeaderScheduleData; +use solana_program::slot_history::Slot; use solana_sdk::account::Account; use solana_sdk::commitment_config::CommitmentConfig; +use solana_sdk::epoch_info::EpochInfo; use solana_sdk::pubkey::Pubkey; +use solana_sdk::sysvar::epoch_schedule::EpochSchedule; +use std::collections::HashMap; use std::time::Duration; use tokio::task::JoinHandle; @@ -47,9 +52,10 @@ pub async fn bootstrap_schedule_epoch_data(data_cache: &DataCache) -> ScheduleEp // Return the current and next epoxh leader schedule and the current epoch stakes of vote accounts // if the corresponding files exist. pub fn bootstrap_leaderschedule_from_files( + current_epoch_of_loading: u64, slots_in_epoch: u64, ) -> Option<(CalculatedSchedule, RpcVoteAccountStatus)> { - bootstrap_current_leader_schedule(slots_in_epoch) + bootstrap_current_leader_schedule(slots_in_epoch, current_epoch_of_loading) .map(|(leader_schedule, current_epoch_stakes, _)| { let vote_acccounts = crate::vote::get_rpc_vote_account_info_from_current_epoch_stakes( ¤t_epoch_stakes, @@ -59,6 +65,55 @@ pub fn bootstrap_leaderschedule_from_files( .ok() } +// Return the current or next epoch leader schedule using the RPC calls. +pub fn bootstrap_leaderschedule_from_rpc( + rpc_url: String, + epoch_schedule: &EpochSchedule, +) -> Result { + let current_epoch = get_rpc_epoch_info(rpc_url.clone())?; + let current_schedule_by_node = + get_rpc_leader_schedule(rpc_url.clone(), None)?.ok_or(ClientError { + request: None, + kind: ClientErrorKind::Custom("RPC return no leader schedule".to_string()), + })?; + + let first_epoch_slot = epoch_schedule.get_first_slot_in_epoch(current_epoch.epoch); + let current_schedule_by_slot = get_rpc_slot_leaders( + rpc_url.clone(), + first_epoch_slot, + epoch_schedule.slots_per_epoch, + )?; + + //get next epoch rpc schedule + let next_epoch = current_epoch.epoch + 1; + let next_first_epoch_slot = epoch_schedule.get_first_slot_in_epoch(next_epoch); + let next_schedule_by_node = + get_rpc_leader_schedule(rpc_url.clone(), Some(next_first_epoch_slot))?.ok_or( + ClientError { + request: None, + kind: ClientErrorKind::Custom("RPC return no leader schedule".to_string()), + }, + )?; + let next_schedule_by_slot = get_rpc_slot_leaders( + rpc_url, + next_first_epoch_slot, + epoch_schedule.slots_per_epoch, + )?; + + Ok(CalculatedSchedule { + current: Some(LeaderScheduleData { + schedule_by_node: current_schedule_by_node.clone(), + schedule_by_slot: current_schedule_by_slot.clone(), + epoch: current_epoch.epoch, + }), + next: Some(LeaderScheduleData { + schedule_by_node: next_schedule_by_node, + schedule_by_slot: next_schedule_by_slot, + epoch: current_epoch.epoch + 1, + }), + }) +} + /* Bootstrap state changes @@ -92,8 +147,15 @@ pub fn run_bootstrap_events( stakestore: &mut StakeStore, votestore: &mut VoteStore, slots_in_epoch: u64, + current_epoch_of_loading: u64, ) -> anyhow::Result>> { - let result = process_bootstrap_event(event, stakestore, votestore, slots_in_epoch); + let result = process_bootstrap_event( + event, + stakestore, + votestore, + slots_in_epoch, + current_epoch_of_loading, + ); match result { BootsrapProcessResult::TaskHandle(jh) => { bootstrap_tasks.push(jh); @@ -105,6 +167,7 @@ pub fn run_bootstrap_events( stakestore, votestore, slots_in_epoch, + current_epoch_of_loading, ), BootsrapProcessResult::End(leader_schedule_result) => Ok(Some(leader_schedule_result)), BootsrapProcessResult::Error(err) => bail!(err), @@ -154,6 +217,7 @@ fn process_bootstrap_event( stakestore: &mut StakeStore, votestore: &mut VoteStore, slots_in_epoch: u64, + current_epoch_of_loading: u64, ) -> BootsrapProcessResult { match event { BootstrapEvent::InitBootstrap { @@ -161,7 +225,6 @@ fn process_bootstrap_event( rpc_url, } => { let jh = tokio::task::spawn_blocking(move || { - log::info!("BootstrapEvent::InitBootstrap RECV"); if sleep_time > 0 { std::thread::sleep(Duration::from_secs(sleep_time)); } @@ -180,7 +243,6 @@ fn process_bootstrap_event( BootsrapProcessResult::TaskHandle(jh) } BootstrapEvent::BootstrapAccountsFetched(stakes, votes, history, rpc_url) => { - log::info!("BootstrapEvent::BootstrapAccountsFetched RECV"); match (&mut stakestore.stakes, &mut votestore.votes).take() { TakeResult::Map((stake_map, (vote_map, epoch_cache))) => { BootsrapProcessResult::Event(BootstrapEvent::StoreExtracted( @@ -220,8 +282,6 @@ fn process_bootstrap_event( history, rpc_url, ) => { - log::info!("BootstrapEvent::StoreExtracted RECV"); - let stake_history = crate::account::read_historystake_from_account(&history.data); if stake_history.is_none() { return BootsrapProcessResult::Error( @@ -244,7 +304,10 @@ fn process_bootstrap_event( 0, //with RPC no way to know the slot of the account update. Set to 0. ); - match bootstrap_current_leader_schedule(slots_in_epoch) { + match bootstrap_current_leader_schedule( + current_epoch_of_loading, + slots_in_epoch, + ) { Ok((leader_schedule, current_epoch_stakes, next_epoch_stakes)) => { let vote_acccounts = crate::vote::get_rpc_vote_account_info_from_current_epoch_stakes( @@ -279,8 +342,6 @@ fn process_bootstrap_event( rpc_url, leader_schedule_result, ) => { - log::info!("BootstrapEvent::AccountsMerged RECV"); - match ( stakestore.stakes.merge(stake_map), votestore.votes.merge((vote_map, epoch_cache)), @@ -314,39 +375,68 @@ fn bootstrap_accounts( } fn get_stake_account(rpc_url: String) -> Result<(Vec<(Pubkey, Account)>, String), ClientError> { - log::info!("TaskToExec RpcGetStakeAccount start"); let rpc_client = RpcClient::new_with_timeout_and_commitment( rpc_url.clone(), Duration::from_secs(600), CommitmentConfig::finalized(), ); - let res_stake = rpc_client.get_program_accounts(&solana_sdk::stake::program::id()); - log::info!("TaskToExec RpcGetStakeAccount END"); - res_stake.map(|stake| (stake, rpc_url)) + rpc_client + .get_program_accounts(&solana_sdk::stake::program::id()) + .map(|stake| (stake, rpc_url)) } fn get_vote_account(rpc_url: String) -> Result<(Vec<(Pubkey, Account)>, String), ClientError> { - log::info!("TaskToExec RpcGetVoteAccount start"); let rpc_client = RpcClient::new_with_timeout_and_commitment( rpc_url.clone(), Duration::from_secs(600), CommitmentConfig::finalized(), ); - let res_vote = rpc_client.get_program_accounts(&solana_sdk::vote::program::id()); - log::info!("TaskToExec RpcGetVoteAccount END"); - res_vote.map(|votes| (votes, rpc_url)) + rpc_client + .get_program_accounts(&solana_sdk::vote::program::id()) + .map(|votes| (votes, rpc_url)) } pub fn get_stakehistory_account(rpc_url: String) -> Result { - log::info!("TaskToExec RpcGetStakeHistory start"); let rpc_client = RpcClient::new_with_timeout_and_commitment( rpc_url, Duration::from_secs(600), CommitmentConfig::finalized(), ); - let res_stake = rpc_client.get_account(&solana_sdk::sysvar::stake_history::id()); - log::info!("TaskToExec RpcGetStakeHistory END",); - res_stake + rpc_client.get_account(&solana_sdk::sysvar::stake_history::id()) +} + +fn get_rpc_epoch_info(rpc_url: String) -> Result { + let rpc_client = RpcClient::new_with_timeout_and_commitment( + rpc_url.clone(), + Duration::from_secs(600), + CommitmentConfig::finalized(), + ); + rpc_client.get_epoch_info() +} + +fn get_rpc_leader_schedule( + rpc_url: String, + slot: Option, +) -> Result>>, ClientError> { + let rpc_client = RpcClient::new_with_timeout_and_commitment( + rpc_url.clone(), + Duration::from_secs(600), + CommitmentConfig::finalized(), + ); + rpc_client.get_leader_schedule(slot) +} + +fn get_rpc_slot_leaders( + rpc_url: String, + start_slot: u64, + limit: u64, +) -> Result, ClientError> { + let rpc_client = RpcClient::new_with_timeout_and_commitment( + rpc_url.clone(), + Duration::from_secs(600), + CommitmentConfig::finalized(), + ); + rpc_client.get_slot_leaders(start_slot, limit) } // pub struct BootstrapScheduleResult { @@ -355,6 +445,7 @@ pub fn get_stakehistory_account(rpc_url: String) -> Result // } pub fn bootstrap_current_leader_schedule( + current_epoch_of_loading: u64, slots_in_epoch: u64, ) -> anyhow::Result<(CalculatedSchedule, EpochVoteStakes, EpochVoteStakes)> { let (current_epoch, current_epoch_stakes) = @@ -362,6 +453,18 @@ pub fn bootstrap_current_leader_schedule( let (next_epoch, next_epoch_stakes) = crate::utils::read_schedule_vote_stakes(NEXT_EPOCH_VOTE_STAKES_FILE)?; + //verify that the current loaded epoch correspond to the current epoch slot + if current_epoch_of_loading != current_epoch { + return Err(ClientError { + request: None, + kind: ClientErrorKind::Custom( + "Current epoch bootstrap file doesn't correspond to the validator current epoch." + .to_string(), + ), + } + .into()); + } + //calcualte leader schedule for all vote stakes. let current_schedule = crate::leader_schedule::calculate_leader_schedule( ¤t_epoch_stakes, diff --git a/stake_vote/src/epoch.rs b/stake_vote/src/epoch.rs index ac185bfb..0ee271d8 100644 --- a/stake_vote/src/epoch.rs +++ b/stake_vote/src/epoch.rs @@ -56,7 +56,7 @@ impl ScheduleEpochData { &mut self, history: StakeHistory, ) -> Option { - log::info!("set_epoch_stake_history"); + log::debug!("set_epoch_stake_history"); self.new_stake_history = Some(history); self.verify_epoch_change() } @@ -68,7 +68,7 @@ impl ScheduleEpochData { //to avoid to delay too much the schedule, start the calculus at the end of the epoch. //the first epoch slot arrive very late cause of the stake account notification from the validator. if self.current_confirmed_slot >= self.last_slot_in_epoch { - log::info!( + log::debug!( "manage_change_epoch at slot:{} last_slot_in_epoch:{}", self.current_confirmed_slot, self.last_slot_in_epoch diff --git a/stake_vote/src/leader_schedule.rs b/stake_vote/src/leader_schedule.rs index 3fe756dc..7a674a69 100644 --- a/stake_vote/src/leader_schedule.rs +++ b/stake_vote/src/leader_schedule.rs @@ -117,7 +117,7 @@ fn process_leadershedule_event( ) => { match (&mut stakestore.stakes, &mut votestore.votes).take() { TakeResult::Map((stake_map, (vote_map, mut epoch_cache))) => { - log::info!("LeaderScheduleEvent::CalculateScedule"); + log::info!("Start calculate leader schedule"); //do the calculus in a blocking task. let jh = tokio::task::spawn_blocking({ move || { @@ -221,7 +221,6 @@ fn process_leadershedule_event( (new_epoch, slots_in_epoch, epoch_schedule), stake_history, ) => { - log::info!("LeaderScheduleEvent::MergeStoreAndSaveSchedule RECV"); match ( stakestore.stakes.merge(stake_map), votestore.votes.merge((vote_map, epoch_cache)), @@ -304,7 +303,7 @@ pub fn calculate_leader_schedule( let mut seed = [0u8; 32]; seed[0..8].copy_from_slice(&epoch.to_le_bytes()); sort_stakes(&mut stakes); - log::info!("calculate_leader_schedule stakes:{stakes:?} epoch:{epoch}"); + //log::info!("calculate_leader_schedule stakes:{stakes:?} epoch:{epoch}"); LeaderSchedule::new(&stakes, seed, slots_in_epoch, NUM_CONSECUTIVE_LEADER_SLOTS) } diff --git a/stake_vote/src/lib.rs b/stake_vote/src/lib.rs index 912c6250..b73eaf22 100644 --- a/stake_vote/src/lib.rs +++ b/stake_vote/src/lib.rs @@ -30,13 +30,55 @@ mod stake; mod utils; mod vote; -pub use bootstrap::bootstrap_leaderschedule_from_files; +// pub use bootstrap::{bootstrap_leaderschedule_from_files, bootstrap_leaderschedule_from_rpc}; const STAKESTORE_INITIAL_CAPACITY: usize = 600000; const VOTESTORE_INITIAL_CAPACITY: usize = 600000; type Slot = u64; +pub async fn bootstrat_literpc_leader_schedule( + rpc_url: String, + data_cache: &DataCache, + current_epoch_of_loading: u64, +) { + //init leader schedule grpc process. + //1) get stored schedule and stakes + let slots_per_epoch = data_cache.epoch_data.get_epoch_schedule().slots_per_epoch; + match crate::bootstrap::bootstrap_leaderschedule_from_files( + current_epoch_of_loading, + slots_per_epoch, + ) { + Some((leader_schedule, vote_stakes)) => { + data_cache + .identity_stakes + .update_stakes_for_identity(vote_stakes) + .await; + let mut data_schedule = data_cache.leader_schedule.write().await; + *data_schedule = leader_schedule; + } + None => { + log::info!("Leader schedule bootstrap file not found. Try to boot from rpc."); + match crate::bootstrap::bootstrap_leaderschedule_from_rpc( + rpc_url, + data_cache.epoch_data.get_epoch_schedule(), + ) { + Ok(leader_schedule) => { + log::info!("Leader schedule bootstrap from rpc done.",); + let mut data_schedule = data_cache.leader_schedule.write().await; + *data_schedule = leader_schedule; + } + Err(err) => { + log::warn!( + "An error occurs during bootstrap of the leader schedule using rpc:{err}" + ); + log::warn!("No schedule has been loaded"); + } + } + } + } +} + pub async fn start_stakes_and_votes_loop( data_cache: DataCache, mut slot_notification: SlotStream, @@ -143,7 +185,7 @@ pub async fn start_stakes_and_votes_loop( if let Some(account) = account.account { let acc_id = Pubkey::try_from(account.pubkey).expect("valid pubkey"); if acc_id == solana_sdk::sysvar::stake_history::ID { - log::info!("Geyser notifstake_history"); + log::debug!("Geyser notifstake_history"); match crate::account::read_historystake_from_account(account.data.as_slice()) { Some(stake_history) => { let schedule_event = current_schedule_epoch.set_epoch_stake_history(stake_history); @@ -244,7 +286,7 @@ pub async fn start_stakes_and_votes_loop( } //manage bootstrap event Some(Ok(event)) = spawned_bootstrap_task.next() => { - match crate::bootstrap::run_bootstrap_events(event, &mut spawned_bootstrap_task, &mut stakestore, &mut votestore, current_schedule_epoch.slots_in_epoch) { + match crate::bootstrap::run_bootstrap_events(event, &mut spawned_bootstrap_task, &mut stakestore, &mut votestore, current_schedule_epoch.slots_in_epoch, current_schedule_epoch.current_epoch) { Ok(Some(boot_res))=> { match boot_res { Ok((current_schedule_data, vote_stakes)) => { diff --git a/stake_vote/src/vote.rs b/stake_vote/src/vote.rs index b9d305aa..a837cd61 100644 --- a/stake_vote/src/vote.rs +++ b/stake_vote/src/vote.rs @@ -36,10 +36,8 @@ impl EpochVoteStakesCache { } pub fn add_stakes_for_epoch(&mut self, vote_stakes: EpochVoteStakes) { - log::info!("add_stakes_for_epoch :{}", vote_stakes.epoch); - if self.cache.insert(vote_stakes.epoch, vote_stakes).is_some() { - log::warn!("Override existing vote stake epoch cache for epoch:"); - } + log::debug!("add_stakes_for_epoch :{}", vote_stakes.epoch); + self.cache.insert(vote_stakes.epoch, vote_stakes); } } From 29478ad262b1dc37a29cd5a2cc9f1d55f5f91fb8 Mon Sep 17 00:00:00 2001 From: musitdev Date: Sat, 6 Jan 2024 16:59:59 +0100 Subject: [PATCH 38/56] change slot leader calculus. Remove rpc call --- stake_vote/src/bootstrap.rs | 43 ++++++++++++++----------------- stake_vote/src/leader_schedule.rs | 16 ++++++++++++ 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/stake_vote/src/bootstrap.rs b/stake_vote/src/bootstrap.rs index cc749dbc..3d73894b 100644 --- a/stake_vote/src/bootstrap.rs +++ b/stake_vote/src/bootstrap.rs @@ -77,12 +77,15 @@ pub fn bootstrap_leaderschedule_from_rpc( kind: ClientErrorKind::Custom("RPC return no leader schedule".to_string()), })?; - let first_epoch_slot = epoch_schedule.get_first_slot_in_epoch(current_epoch.epoch); - let current_schedule_by_slot = get_rpc_slot_leaders( - rpc_url.clone(), - first_epoch_slot, - epoch_schedule.slots_per_epoch, - )?; + //Calculate the slot leaders by from the node schedule because RPC call get_slot_leaders is limited to 5000 slots. + let current_schedule_by_slot = + crate::leader_schedule::calculate_slot_leaders_from_schedule(¤t_schedule_by_node) + .map_err(|err| ClientError { + request: None, + kind: ClientErrorKind::Custom(format!( + "Leader schedule from RPC can't generate slot leaders because:{err}" + )), + })?; //get next epoch rpc schedule let next_epoch = current_epoch.epoch + 1; @@ -94,11 +97,16 @@ pub fn bootstrap_leaderschedule_from_rpc( kind: ClientErrorKind::Custom("RPC return no leader schedule".to_string()), }, )?; - let next_schedule_by_slot = get_rpc_slot_leaders( - rpc_url, - next_first_epoch_slot, - epoch_schedule.slots_per_epoch, - )?; + + //Calculate the slot leaders by from the node schedule because RPC call get_slot_leaders is limited to 5000 slots. + let next_schedule_by_slot = + crate::leader_schedule::calculate_slot_leaders_from_schedule(&next_schedule_by_node) + .map_err(|err| ClientError { + request: None, + kind: ClientErrorKind::Custom(format!( + "Leader schedule from RPC can't generate slot leaders because:{err}" + )), + })?; Ok(CalculatedSchedule { current: Some(LeaderScheduleData { @@ -426,19 +434,6 @@ fn get_rpc_leader_schedule( rpc_client.get_leader_schedule(slot) } -fn get_rpc_slot_leaders( - rpc_url: String, - start_slot: u64, - limit: u64, -) -> Result, ClientError> { - let rpc_client = RpcClient::new_with_timeout_and_commitment( - rpc_url.clone(), - Duration::from_secs(600), - CommitmentConfig::finalized(), - ); - rpc_client.get_slot_leaders(start_slot, limit) -} - // pub struct BootstrapScheduleResult { // schedule: CalculatedSchedule, // vote_stakes: Vec, diff --git a/stake_vote/src/leader_schedule.rs b/stake_vote/src/leader_schedule.rs index 7a674a69..7d217c24 100644 --- a/stake_vote/src/leader_schedule.rs +++ b/stake_vote/src/leader_schedule.rs @@ -13,7 +13,9 @@ use solana_lite_rpc_core::structures::leaderschedule::LeaderScheduleData; use solana_sdk::clock::NUM_CONSECUTIVE_LEADER_SLOTS; use solana_sdk::pubkey::Pubkey; use solana_sdk::stake_history::StakeHistory; +use std::collections::BTreeMap; use std::collections::HashMap; +use std::str::FromStr; use std::sync::Arc; use tokio::task::JoinHandle; @@ -307,6 +309,20 @@ pub fn calculate_leader_schedule( LeaderSchedule::new(&stakes, seed, slots_in_epoch, NUM_CONSECUTIVE_LEADER_SLOTS) } +pub fn calculate_slot_leaders_from_schedule( + leader_scheudle: &HashMap>, +) -> Result, String> { + let mut slot_leaders_map = BTreeMap::new(); + for (pk, index_list) in leader_scheudle { + for index in index_list { + let pubkey = Pubkey::from_str(pk) + .map_err(|err| format!("Pubkey from leader schedule not a plublic key:{err}"))?; + slot_leaders_map.insert(index, pubkey); + } + } + Ok(slot_leaders_map.into_values().collect()) +} + // Cribbed from leader_schedule_utils fn sort_stakes(stakes: &mut Vec<(Pubkey, u64)>) { // Sort first by stake. If stakes are the same, sort by pubkey to ensure a From ef5716932fabe25d94e8123afe12abbf26e9a570 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Sun, 7 Jan 2024 20:08:20 +0100 Subject: [PATCH 39/56] fix: panic on geyser close, multiplex bug https://github.com/blockworks-foundation/geyser-grpc-connector/issues/3 --- Cargo.lock | 4 ++-- cluster-endpoints/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c9980ce9..2a055d8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1910,9 +1910,9 @@ dependencies = [ [[package]] name = "geyser-grpc-connector" -version = "0.7.1+yellowstone.1.11" +version = "0.7.2+yellowstone.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f11c9bfa33ff6c5cf688e53df9b2de84b47f932527445bc6e33649c22a7791f7" +checksum = "5ff13bf8aff2952263d1f70096c8f92a174194fa07e1ce1d1b51f59d78cae87e" dependencies = [ "anyhow", "async-stream", diff --git a/cluster-endpoints/Cargo.toml b/cluster-endpoints/Cargo.toml index b105dfeb..b31e9a2a 100644 --- a/cluster-endpoints/Cargo.toml +++ b/cluster-endpoints/Cargo.toml @@ -10,7 +10,7 @@ license = "AGPL" [dependencies] #geyser-grpc-connector = { path = "../../geyser-grpc-connector" } #geyser-grpc-connector = { tag = "v0.5.0+yellowstone.1.11+solana.1.16.17", git = "https://github.com/blockworks-foundation/geyser-grpc-connector.git" } -geyser-grpc-connector = "0.7.0+yellowstone.1.11" +geyser-grpc-connector = "0.7.2+yellowstone.1.11" solana-sdk = { workspace = true } solana-rpc-client-api = { workspace = true } From e42db51d2108a75126917ce8964f21a02a5c690e Mon Sep 17 00:00:00 2001 From: godmodegalactus Date: Mon, 8 Jan 2024 12:11:48 +0100 Subject: [PATCH 40/56] Reduce network by only fetching block meta instead of whole block --- cluster-endpoints/src/grpc_multiplex.rs | 68 ++++++++++++++++++++----- 1 file changed, 56 insertions(+), 12 deletions(-) diff --git a/cluster-endpoints/src/grpc_multiplex.rs b/cluster-endpoints/src/grpc_multiplex.rs index 0f8884f8..0ddc4775 100644 --- a/cluster-endpoints/src/grpc_multiplex.rs +++ b/cluster-endpoints/src/grpc_multiplex.rs @@ -1,5 +1,6 @@ use crate::grpc_stream_utils::channelize_stream; use crate::grpc_subscription::map_block_update; +use futures::StreamExt; use geyser_grpc_connector::grpc_subscription_autoreconnect::{ create_geyser_reconnecting_stream, GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig, }; @@ -7,7 +8,6 @@ use geyser_grpc_connector::grpcmultiplex_fastestwins::{ create_multiplexed_stream, FromYellowstoneExtractor, }; use log::info; -use merge_streams::MergeStreams; use solana_lite_rpc_core::structures::produced_block::ProducedBlock; use solana_lite_rpc_core::structures::slot_notification::SlotNotification; use solana_lite_rpc_core::AnyhowJoinHandle; @@ -36,6 +36,21 @@ impl FromYellowstoneExtractor for BlockExtractor { } } +struct BlockHashExtractor(CommitmentConfig); + +impl FromYellowstoneExtractor for BlockHashExtractor { + type Target = String; + fn map_yellowstone_update(&self, update: SubscribeUpdate) -> Option<(u64, String)> { + match update.update_oneof { + Some(UpdateOneof::Block(block)) => Some((block.slot, block.blockhash)), + Some(UpdateOneof::BlockMeta(block_meta)) => { + Some((block_meta.slot, block_meta.blockhash)) + } + _ => None, + } + } +} + pub fn create_grpc_multiplex_blocks_subscription( grpc_sources: Vec, ) -> (Receiver, AnyhowJoinHandle) { @@ -68,28 +83,57 @@ pub fn create_grpc_multiplex_blocks_subscription( create_multiplexed_stream(streams, BlockExtractor(commitment_config)) }; - let multiplex_stream_finalized = { + let (sender, multiplexed_merged_blocks) = + tokio::sync::broadcast::channel::(1000); + + let meta_stream_finalized = { let commitment_config = CommitmentConfig::finalized(); let mut streams = Vec::new(); for grpc_source in &grpc_sources { let stream = create_geyser_reconnecting_stream( grpc_source.clone(), - GeyserFilter(commitment_config).blocks_and_txs(), + GeyserFilter(commitment_config).blocks_meta(), ); streams.push(stream); } - - create_multiplexed_stream(streams, BlockExtractor(commitment_config)) + create_multiplexed_stream(streams, BlockHashExtractor(commitment_config)) + }; + let jh_channelizer = { + // spawn merged + tokio::task::spawn(async move { + let mut map_of_confimed_blocks = HashMap::::new(); + let mut multiplex_stream_confirmed = std::pin::pin!(multiplex_stream_confirmed); + let mut meta_stream_finalized = std::pin::pin!(meta_stream_finalized); + let sender = sender; + loop { + tokio::select! { + confirmed_block = multiplex_stream_confirmed.next() => { + if let Some(confirmed_block) = confirmed_block { + if let Err(e) = sender.send(confirmed_block.clone()) { + panic!("Confirmed block stream send gave error {e:?}"); + } + map_of_confimed_blocks.insert(confirmed_block.blockhash.clone(), confirmed_block); + } else { + panic!("Confirmed stream broke"); + } + }, + meta_finalized = meta_stream_finalized.next() => { + if let Some(blockhash) = meta_finalized { + if let Some(mut finalized_block) = map_of_confimed_blocks.remove(&blockhash) { + finalized_block.commitment_config = CommitmentConfig::finalized(); + if let Err(e) = sender.send(finalized_block.clone()) { + panic!("Finalized block stream send gave error {e:?}"); + } + } + } + } + } + } + }) }; - let merged_stream_confirmed_finalize = - (multiplex_stream_confirmed, multiplex_stream_finalized).merge(); - - let (multiplexed_finalized_blocks, jh_channelizer) = - channelize_stream(merged_stream_confirmed_finalize); - - (multiplexed_finalized_blocks, jh_channelizer) + (multiplexed_merged_blocks, jh_channelizer) } struct SlotExtractor {} From 21d0cee3324b949e4e2a212e4ce11a009dd75d2a Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 8 Jan 2024 13:19:23 +0100 Subject: [PATCH 41/56] set initial rpc schedule parameters --- lite-rpc/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index 2d14bad4..8d4c3d8c 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -246,7 +246,7 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: ) } else { ( - Arc::new(JsonRpcLeaderGetter::new(rpc_client.clone(), 40, 12)), + Arc::new(JsonRpcLeaderGetter::new(rpc_client.clone(), 1024, 128)), None, ) }; From 9ef773c129173b02562621da6b7628b761383ad8 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 8 Jan 2024 14:14:59 +0100 Subject: [PATCH 42/56] refactoring --- cluster-endpoints/src/grpc_multiplex.rs | 88 ++++++++++++++----------- core/src/structures/produced_block.rs | 8 +++ 2 files changed, 58 insertions(+), 38 deletions(-) diff --git a/cluster-endpoints/src/grpc_multiplex.rs b/cluster-endpoints/src/grpc_multiplex.rs index 0ddc4775..1bcfe811 100644 --- a/cluster-endpoints/src/grpc_multiplex.rs +++ b/cluster-endpoints/src/grpc_multiplex.rs @@ -2,12 +2,12 @@ use crate::grpc_stream_utils::channelize_stream; use crate::grpc_subscription::map_block_update; use futures::StreamExt; use geyser_grpc_connector::grpc_subscription_autoreconnect::{ - create_geyser_reconnecting_stream, GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig, + create_geyser_reconnecting_stream, GeyserFilter, GrpcSourceConfig, }; use geyser_grpc_connector::grpcmultiplex_fastestwins::{ create_multiplexed_stream, FromYellowstoneExtractor, }; -use log::info; +use log::{debug, info, trace}; use solana_lite_rpc_core::structures::produced_block::ProducedBlock; use solana_lite_rpc_core::structures::slot_notification::SlotNotification; use solana_lite_rpc_core::AnyhowJoinHandle; @@ -36,13 +36,12 @@ impl FromYellowstoneExtractor for BlockExtractor { } } -struct BlockHashExtractor(CommitmentConfig); +struct BlockMetaHashExtractor(CommitmentConfig); -impl FromYellowstoneExtractor for BlockHashExtractor { +impl FromYellowstoneExtractor for BlockMetaHashExtractor { type Target = String; fn map_yellowstone_update(&self, update: SubscribeUpdate) -> Option<(u64, String)> { match update.update_oneof { - Some(UpdateOneof::Block(block)) => Some((block.slot, block.blockhash)), Some(UpdateOneof::BlockMeta(block_meta)) => { Some((block_meta.slot, block_meta.blockhash)) } @@ -51,6 +50,7 @@ impl FromYellowstoneExtractor for BlockHashExtractor { } } +/// connect to multiple grpc sources to consume confirmed blocks and block status update pub fn create_grpc_multiplex_blocks_subscription( grpc_sources: Vec, ) -> (Receiver, AnyhowJoinHandle) { @@ -62,13 +62,7 @@ pub fn create_grpc_multiplex_blocks_subscription( info!("- connection to {}", grpc_source); } - let _timeouts = GrpcConnectionTimeouts { - connect_timeout: Duration::from_secs(5), - request_timeout: Duration::from_secs(5), - subscribe_timeout: Duration::from_secs(5), - }; - - let multiplex_stream_confirmed = { + let confirmed_blocks_stream = { let commitment_config = CommitmentConfig::confirmed(); let mut streams = Vec::new(); @@ -83,10 +77,7 @@ pub fn create_grpc_multiplex_blocks_subscription( create_multiplexed_stream(streams, BlockExtractor(commitment_config)) }; - let (sender, multiplexed_merged_blocks) = - tokio::sync::broadcast::channel::(1000); - - let meta_stream_finalized = { + let finalized_blockmeta_stream = { let commitment_config = CommitmentConfig::finalized(); let mut streams = Vec::new(); @@ -97,35 +88,56 @@ pub fn create_grpc_multiplex_blocks_subscription( ); streams.push(stream); } - create_multiplexed_stream(streams, BlockHashExtractor(commitment_config)) + create_multiplexed_stream(streams, BlockMetaHashExtractor(commitment_config)) }; - let jh_channelizer = { - // spawn merged + + // return value is the broadcast receiver + let (producedblock_sender, blocks_output_stream) = + tokio::sync::broadcast::channel::(1000); + + let jh_block_emitter_task = { tokio::task::spawn(async move { - let mut map_of_confimed_blocks = HashMap::::new(); - let mut multiplex_stream_confirmed = std::pin::pin!(multiplex_stream_confirmed); - let mut meta_stream_finalized = std::pin::pin!(meta_stream_finalized); - let sender = sender; + // by blockhash + let mut recent_confirmed_blocks = HashMap::::new(); + let mut confirmed_blocks_stream = std::pin::pin!(confirmed_blocks_stream); + let mut finalized_blockmeta_stream = std::pin::pin!(finalized_blockmeta_stream); + + let sender = producedblock_sender; + let mut cleanup_tick = tokio::time::interval(Duration::from_secs(5)); + let mut current_slot: Slot = 0; loop { tokio::select! { - confirmed_block = multiplex_stream_confirmed.next() => { - if let Some(confirmed_block) = confirmed_block { - if let Err(e) = sender.send(confirmed_block.clone()) { - panic!("Confirmed block stream send gave error {e:?}"); + confirmed_block = confirmed_blocks_stream.next() => { + let confirmed_block = confirmed_block.expect("confirmed block from stream"); + current_slot = confirmed_block.slot; + trace!("got confirmed block {} with blockhash {}", + confirmed_block.slot, confirmed_block.blockhash.clone()); + if let Err(e) = sender.send(confirmed_block.clone()) { + panic!("Confirmed block stream send gave error {e:?}"); + } + recent_confirmed_blocks.insert(confirmed_block.blockhash.clone(), confirmed_block); + }, + meta_finalized = finalized_blockmeta_stream.next() => { + let blockhash = meta_finalized.expect("finalized block meta from stream"); + if let Some(cached_confirmed_block) = recent_confirmed_blocks.remove(&blockhash) { + let finalized_block = cached_confirmed_block.to_finalized_block(); + debug!("got finalized blockmeta {} with blockhash {}", + finalized_block.slot, finalized_block.blockhash.clone()); + if let Err(e) = sender.send(finalized_block) { + panic!("Finalized block stream send gave error {e:?}"); } - map_of_confimed_blocks.insert(confirmed_block.blockhash.clone(), confirmed_block); } else { - panic!("Confirmed stream broke"); + debug!("finalized block meta received for blockhash {} which was never seen or already emitted", blockhash); } }, - meta_finalized = meta_stream_finalized.next() => { - if let Some(blockhash) = meta_finalized { - if let Some(mut finalized_block) = map_of_confimed_blocks.remove(&blockhash) { - finalized_block.commitment_config = CommitmentConfig::finalized(); - if let Err(e) = sender.send(finalized_block.clone()) { - panic!("Finalized block stream send gave error {e:?}"); - } - } + _ = cleanup_tick.tick() => { + let size_before = recent_confirmed_blocks.len(); + recent_confirmed_blocks.retain(|_blockhash, block| { + block.slot > current_slot - 100 // must be greater than finalized slot distance (31) + }); + let cleaned = size_before - recent_confirmed_blocks.len(); + if cleaned > 0 { + debug!("cleaned {} confirmed blocks from cache", cleaned); } } } @@ -133,7 +145,7 @@ pub fn create_grpc_multiplex_blocks_subscription( }) }; - (multiplexed_merged_blocks, jh_channelizer) + (blocks_output_stream, jh_block_emitter_task) } struct SlotExtractor {} diff --git a/core/src/structures/produced_block.rs b/core/src/structures/produced_block.rs index 5c53568a..d8fae66c 100644 --- a/core/src/structures/produced_block.rs +++ b/core/src/structures/produced_block.rs @@ -165,4 +165,12 @@ impl ProducedBlock { rewards, } } + + /// moving commitment level to finalized + pub fn to_finalized_block(&self) -> Self { + ProducedBlock { + commitment_config: CommitmentConfig::finalized(), + ..self.clone() + } + } } From e5e754555392829588a6a8449a01eb8df0e51c5f Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 8 Jan 2024 14:15:42 +0100 Subject: [PATCH 43/56] ->32 --- cluster-endpoints/src/grpc_multiplex.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-endpoints/src/grpc_multiplex.rs b/cluster-endpoints/src/grpc_multiplex.rs index 1bcfe811..30b018e8 100644 --- a/cluster-endpoints/src/grpc_multiplex.rs +++ b/cluster-endpoints/src/grpc_multiplex.rs @@ -133,7 +133,7 @@ pub fn create_grpc_multiplex_blocks_subscription( _ = cleanup_tick.tick() => { let size_before = recent_confirmed_blocks.len(); recent_confirmed_blocks.retain(|_blockhash, block| { - block.slot > current_slot - 100 // must be greater than finalized slot distance (31) + block.slot > current_slot - 100 // must be greater than finalized slot distance (32) }); let cleaned = size_before - recent_confirmed_blocks.len(); if cleaned > 0 { From dc0c60ff05f142d066b0c77b932d2564d0c3de52 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 8 Jan 2024 14:18:43 +0100 Subject: [PATCH 44/56] rename count --- cluster-endpoints/src/grpc_multiplex.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster-endpoints/src/grpc_multiplex.rs b/cluster-endpoints/src/grpc_multiplex.rs index 30b018e8..d234d255 100644 --- a/cluster-endpoints/src/grpc_multiplex.rs +++ b/cluster-endpoints/src/grpc_multiplex.rs @@ -135,9 +135,9 @@ pub fn create_grpc_multiplex_blocks_subscription( recent_confirmed_blocks.retain(|_blockhash, block| { block.slot > current_slot - 100 // must be greater than finalized slot distance (32) }); - let cleaned = size_before - recent_confirmed_blocks.len(); - if cleaned > 0 { - debug!("cleaned {} confirmed blocks from cache", cleaned); + let cnt_cleaned = size_before - recent_confirmed_blocks.len(); + if cnt_cleaned > 0 { + debug!("cleaned {} confirmed blocks from cache", cnt_cleaned); } } } From 8996a974248e4c9f9e10bf13a043a21529b5d01c Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 8 Jan 2024 14:21:05 +0100 Subject: [PATCH 45/56] change .send error handling on broadcast channel --- cluster-endpoints/src/grpc_multiplex.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cluster-endpoints/src/grpc_multiplex.rs b/cluster-endpoints/src/grpc_multiplex.rs index d234d255..a557f1db 100644 --- a/cluster-endpoints/src/grpc_multiplex.rs +++ b/cluster-endpoints/src/grpc_multiplex.rs @@ -7,7 +7,7 @@ use geyser_grpc_connector::grpc_subscription_autoreconnect::{ use geyser_grpc_connector::grpcmultiplex_fastestwins::{ create_multiplexed_stream, FromYellowstoneExtractor, }; -use log::{debug, info, trace}; +use log::{debug, info, trace, warn}; use solana_lite_rpc_core::structures::produced_block::ProducedBlock; use solana_lite_rpc_core::structures::slot_notification::SlotNotification; use solana_lite_rpc_core::AnyhowJoinHandle; @@ -113,7 +113,8 @@ pub fn create_grpc_multiplex_blocks_subscription( trace!("got confirmed block {} with blockhash {}", confirmed_block.slot, confirmed_block.blockhash.clone()); if let Err(e) = sender.send(confirmed_block.clone()) { - panic!("Confirmed block stream send gave error {e:?}"); + warn!("Confirmed block channel has no receivers {e:?}"); + continue } recent_confirmed_blocks.insert(confirmed_block.blockhash.clone(), confirmed_block); }, @@ -124,7 +125,8 @@ pub fn create_grpc_multiplex_blocks_subscription( debug!("got finalized blockmeta {} with blockhash {}", finalized_block.slot, finalized_block.blockhash.clone()); if let Err(e) = sender.send(finalized_block) { - panic!("Finalized block stream send gave error {e:?}"); + warn!("Finalized block channel has no receivers {e:?}"); + continue; } } else { debug!("finalized block meta received for blockhash {} which was never seen or already emitted", blockhash); From 079ff113c4ea948b83ceb120f586c50be09036d8 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 8 Jan 2024 14:51:32 +0100 Subject: [PATCH 46/56] fix cleanup logic --- cluster-endpoints/src/grpc_multiplex.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster-endpoints/src/grpc_multiplex.rs b/cluster-endpoints/src/grpc_multiplex.rs index a557f1db..faeeb53e 100644 --- a/cluster-endpoints/src/grpc_multiplex.rs +++ b/cluster-endpoints/src/grpc_multiplex.rs @@ -104,12 +104,11 @@ pub fn create_grpc_multiplex_blocks_subscription( let sender = producedblock_sender; let mut cleanup_tick = tokio::time::interval(Duration::from_secs(5)); - let mut current_slot: Slot = 0; + let mut last_finalized_slot: Slot = 0; loop { tokio::select! { confirmed_block = confirmed_blocks_stream.next() => { let confirmed_block = confirmed_block.expect("confirmed block from stream"); - current_slot = confirmed_block.slot; trace!("got confirmed block {} with blockhash {}", confirmed_block.slot, confirmed_block.blockhash.clone()); if let Err(e) = sender.send(confirmed_block.clone()) { @@ -122,6 +121,7 @@ pub fn create_grpc_multiplex_blocks_subscription( let blockhash = meta_finalized.expect("finalized block meta from stream"); if let Some(cached_confirmed_block) = recent_confirmed_blocks.remove(&blockhash) { let finalized_block = cached_confirmed_block.to_finalized_block(); + last_finalized_slot = finalized_block.slot; debug!("got finalized blockmeta {} with blockhash {}", finalized_block.slot, finalized_block.blockhash.clone()); if let Err(e) = sender.send(finalized_block) { @@ -135,7 +135,7 @@ pub fn create_grpc_multiplex_blocks_subscription( _ = cleanup_tick.tick() => { let size_before = recent_confirmed_blocks.len(); recent_confirmed_blocks.retain(|_blockhash, block| { - block.slot > current_slot - 100 // must be greater than finalized slot distance (32) + last_finalized_slot == 0 || block.slot > last_finalized_slot - 100 }); let cnt_cleaned = size_before - recent_confirmed_blocks.len(); if cnt_cleaned > 0 { From f910587553880ca254ba3f5dc473632a2499263e Mon Sep 17 00:00:00 2001 From: Groovie | Mango <95291500+grooviegermanikus@users.noreply.github.com> Date: Thu, 11 Jan 2024 14:16:32 +0100 Subject: [PATCH 47/56] fix prioritization_fees overflow (#245) * fix prioritization_fees overflow * suggest inline * Cargo fmt --------- Co-authored-by: galactus <96341601+godmodegalactus@users.noreply.github.com> Co-authored-by: godmodegalactus --- core/src/structures/produced_block.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/core/src/structures/produced_block.rs b/core/src/structures/produced_block.rs index d8fae66c..91a287e3 100644 --- a/core/src/structures/produced_block.rs +++ b/core/src/structures/produced_block.rs @@ -122,7 +122,7 @@ impl ProducedBlock { if let Some((units, additional_fee)) = legacy_compute_budget { cu_requested = Some(units); if additional_fee > 0 { - prioritization_fees = Some(((units * 1000) / additional_fee).into()) + prioritization_fees = Some(calc_prioritization_fees(units, additional_fee)) } }; @@ -174,3 +174,18 @@ impl ProducedBlock { } } } + +#[inline] +fn calc_prioritization_fees(units: u32, additional_fee: u32) -> u64 { + (units as u64 * 1000) / additional_fee as u64 +} + +#[test] +fn overflow_u32() { + // value high enough to overflow u32 if multiplied by 1000 + let units: u32 = 4_000_000_000; + let additional_fee: u32 = 100; + let prioritization_fees: u64 = calc_prioritization_fees(units, additional_fee); + + assert_eq!(40_000_000_000, prioritization_fees); +} From d91e2e93167e7cfcc18e2f614bfd59949051e00a Mon Sep 17 00:00:00 2001 From: galactus <96341601+godmodegalactus@users.noreply.github.com> Date: Thu, 11 Jan 2024 14:37:06 +0100 Subject: [PATCH 48/56] disable geyser stakes and votes for refactoring (#268) --- Cargo.lock | 1 - lite-rpc/Cargo.toml | 1 - lite-rpc/src/bridge.rs | 36 ++---------------------------- lite-rpc/src/main.rs | 50 ++---------------------------------------- 4 files changed, 4 insertions(+), 84 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2a055d8e..9ea313c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2798,7 +2798,6 @@ dependencies = [ "solana-lite-rpc-core", "solana-lite-rpc-history", "solana-lite-rpc-services", - "solana-lite-rpc-stakevote", "solana-rpc-client", "solana-rpc-client-api", "solana-sdk", diff --git a/lite-rpc/Cargo.toml b/lite-rpc/Cargo.toml index 6d4defa7..192b6d9a 100644 --- a/lite-rpc/Cargo.toml +++ b/lite-rpc/Cargo.toml @@ -46,7 +46,6 @@ solana-lite-rpc-core = { workspace = true } solana-lite-rpc-services = { workspace = true } solana-lite-rpc-cluster-endpoints = { workspace = true } solana-lite-rpc-history = { workspace = true } -solana-lite-rpc-stakevote = { workspace = true } [dev-dependencies] bench = { path = "../bench" } diff --git a/lite-rpc/src/bridge.rs b/lite-rpc/src/bridge.rs index 39ec6549..f91bd591 100644 --- a/lite-rpc/src/bridge.rs +++ b/lite-rpc/src/bridge.rs @@ -3,7 +3,6 @@ use crate::{ jsonrpsee_subscrption_handler_sink::JsonRpseeSubscriptionHandlerSink, rpc::LiteRpcServer, }; -use solana_lite_rpc_core::structures::leaderschedule::GetVoteAccountsConfig; use solana_sdk::epoch_info::EpochInfo; use std::collections::HashMap; @@ -39,7 +38,6 @@ use solana_sdk::{commitment_config::CommitmentConfig, pubkey::Pubkey, slot_histo use solana_transaction_status::{TransactionStatus, UiConfirmedBlock}; use std::{str::FromStr, sync::Arc}; use tokio::net::ToSocketAddrs; -use tokio::sync::oneshot; lazy_static::lazy_static! { static ref RPC_SEND_TX: IntCounter = @@ -65,12 +63,6 @@ pub struct LiteBridge { rpc_client: Arc, transaction_service: TransactionService, history: History, - state_vote_sendder: Option< - tokio::sync::mpsc::Sender<( - GetVoteAccountsConfig, - tokio::sync::oneshot::Sender, - )>, - >, } impl LiteBridge { @@ -79,19 +71,12 @@ impl LiteBridge { data_cache: DataCache, transaction_service: TransactionService, history: History, - state_vote_sendder: Option< - tokio::sync::mpsc::Sender<( - GetVoteAccountsConfig, - oneshot::Sender, - )>, - >, ) -> Self { Self { rpc_client, data_cache, transaction_service, history, - state_vote_sendder, } } @@ -507,25 +492,8 @@ impl LiteRpcServer for LiteBridge { async fn get_vote_accounts( &self, - config: Option, + _config: Option, ) -> crate::rpc::Result { - let config: GetVoteAccountsConfig = - GetVoteAccountsConfig::try_from(config.unwrap_or_default()).unwrap_or_default(); - if let Some(state_vote_sendder) = &self.state_vote_sendder { - let (tx, rx) = oneshot::channel(); - if let Err(err) = state_vote_sendder.send((config, tx)).await { - return Err(jsonrpsee::core::Error::Custom(format!( - "error during query processing:{err}", - ))); - } - rx.await.map_err(|err| { - jsonrpsee::core::Error::Custom(format!("error during query processing:{err}")) - }) - } else { - self.rpc_client - .get_vote_accounts() - .await - .map_err(|err| (jsonrpsee::core::Error::Custom(err.to_string()))) - } + todo!() } } diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index 8d4c3d8c..dc4580c8 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -10,7 +10,6 @@ use lite_rpc::service_spawner::ServiceSpawner; use lite_rpc::DEFAULT_MAX_NUMBER_OF_TXS_IN_QUEUE; use log::info; use solana_lite_rpc_cluster_endpoints::endpoint_stremers::EndpointStreaming; -use solana_lite_rpc_cluster_endpoints::grpc_leaders_getter::GrpcLeaderGetter; use solana_lite_rpc_cluster_endpoints::grpc_subscription::create_grpc_subscription; use solana_lite_rpc_cluster_endpoints::grpc_subscription_autoreconnect::{ GrpcConnectionTimeouts, GrpcSourceConfig, @@ -31,7 +30,6 @@ use solana_lite_rpc_core::structures::{ epoch::EpochCache, identity_stakes::IdentityStakes, notifications::NotificationSender, produced_block::ProducedBlock, }; -use solana_lite_rpc_core::traits::leaders_fetcher_interface::LeaderFetcherInterface; use solana_lite_rpc_core::types::BlockStream; use solana_lite_rpc_core::AnyhowJoinHandle; use solana_lite_rpc_history::block_stores::inmemory_block_store::InmemoryBlockStore; @@ -100,8 +98,6 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: transaction_retry_after_secs, quic_proxy_addr, use_grpc, - calculate_leader_schedule_form_geyser, - grpc_addr, .. } = args; @@ -154,7 +150,7 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: get_latest_block(blocks_notifier.resubscribe(), CommitmentConfig::finalized()).await; info!("Got finalized block: {:?}", finalized_block.slot); - let (epoch_data, current_epoch_info) = EpochCache::bootstrap_epoch(&rpc_client).await?; + let (epoch_data, _current_epoch_info) = EpochCache::bootstrap_epoch(&rpc_client).await?; let block_information_store = BlockInformationStore::new(BlockInformation::from_block(&finalized_block)); @@ -208,48 +204,7 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: data_cache: data_cache.clone(), }; //init grpc leader schedule and vote account is configured. - let (leader_schedule, rpc_stakes_send): (Arc, Option<_>) = - if use_grpc && calculate_leader_schedule_form_geyser { - //init leader schedule grpc process. - - //1) get stored leader schedule and stakes (or via RPC if not present) - solana_lite_rpc_stakevote::bootstrat_literpc_leader_schedule( - rpc_client.url(), - &data_cache, - current_epoch_info.epoch, - ) - .await; - - //2) start stake vote and leader schedule. - let (rpc_stakes_send, rpc_stakes_recv) = mpsc::channel(1000); - let stake_vote_jh = solana_lite_rpc_stakevote::start_stakes_and_votes_loop( - data_cache.clone(), - slot_notifier.resubscribe(), - rpc_stakes_recv, - Arc::clone(&rpc_client), - grpc_addr, - ) - .await?; - - // - tokio::spawn(async move { - let err = stake_vote_jh.await; - log::error!("Vote and stake Services exit with error: {err:?}"); - }); - - ( - Arc::new(GrpcLeaderGetter::new( - Arc::clone(&data_cache.leader_schedule), - data_cache.epoch_data.clone(), - )), - Some(rpc_stakes_send), - ) - } else { - ( - Arc::new(JsonRpcLeaderGetter::new(rpc_client.clone(), 1024, 128)), - None, - ) - }; + let leader_schedule = Arc::new(JsonRpcLeaderGetter::new(rpc_client.clone(), 1024, 128)); let tpu_service: TpuService = TpuService::new( tpu_config, validator_identity, @@ -284,7 +239,6 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: data_cache.clone(), transaction_service, history, - rpc_stakes_send, ) .start(lite_rpc_http_addr, lite_rpc_ws_addr), ); From 3f69de5247a318e9fb92090dca7aa0f90c6610aa Mon Sep 17 00:00:00 2001 From: godmodegalactus Date: Fri, 12 Jan 2024 14:21:54 +0100 Subject: [PATCH 49/56] Resarting geyser slot and block stream in case they disconnect --- cluster-endpoints/src/grpc_multiplex.rs | 239 ++++++++++++++---------- 1 file changed, 139 insertions(+), 100 deletions(-) diff --git a/cluster-endpoints/src/grpc_multiplex.rs b/cluster-endpoints/src/grpc_multiplex.rs index faeeb53e..f0a819f2 100644 --- a/cluster-endpoints/src/grpc_multiplex.rs +++ b/cluster-endpoints/src/grpc_multiplex.rs @@ -1,4 +1,3 @@ -use crate::grpc_stream_utils::channelize_stream; use crate::grpc_subscription::map_block_update; use futures::StreamExt; use geyser_grpc_connector::grpc_subscription_autoreconnect::{ @@ -62,84 +61,98 @@ pub fn create_grpc_multiplex_blocks_subscription( info!("- connection to {}", grpc_source); } - let confirmed_blocks_stream = { - let commitment_config = CommitmentConfig::confirmed(); - - let mut streams = Vec::new(); - for grpc_source in &grpc_sources { - let stream = create_geyser_reconnecting_stream( - grpc_source.clone(), - GeyserFilter(commitment_config).blocks_and_txs(), - ); - streams.push(stream); - } - - create_multiplexed_stream(streams, BlockExtractor(commitment_config)) - }; - - let finalized_blockmeta_stream = { - let commitment_config = CommitmentConfig::finalized(); - - let mut streams = Vec::new(); - for grpc_source in &grpc_sources { - let stream = create_geyser_reconnecting_stream( - grpc_source.clone(), - GeyserFilter(commitment_config).blocks_meta(), - ); - streams.push(stream); - } - create_multiplexed_stream(streams, BlockMetaHashExtractor(commitment_config)) - }; - // return value is the broadcast receiver let (producedblock_sender, blocks_output_stream) = tokio::sync::broadcast::channel::(1000); let jh_block_emitter_task = { tokio::task::spawn(async move { - // by blockhash - let mut recent_confirmed_blocks = HashMap::::new(); - let mut confirmed_blocks_stream = std::pin::pin!(confirmed_blocks_stream); - let mut finalized_blockmeta_stream = std::pin::pin!(finalized_blockmeta_stream); - - let sender = producedblock_sender; - let mut cleanup_tick = tokio::time::interval(Duration::from_secs(5)); - let mut last_finalized_slot: Slot = 0; loop { - tokio::select! { - confirmed_block = confirmed_blocks_stream.next() => { - let confirmed_block = confirmed_block.expect("confirmed block from stream"); - trace!("got confirmed block {} with blockhash {}", - confirmed_block.slot, confirmed_block.blockhash.clone()); - if let Err(e) = sender.send(confirmed_block.clone()) { - warn!("Confirmed block channel has no receivers {e:?}"); - continue - } - recent_confirmed_blocks.insert(confirmed_block.blockhash.clone(), confirmed_block); - }, - meta_finalized = finalized_blockmeta_stream.next() => { - let blockhash = meta_finalized.expect("finalized block meta from stream"); - if let Some(cached_confirmed_block) = recent_confirmed_blocks.remove(&blockhash) { - let finalized_block = cached_confirmed_block.to_finalized_block(); - last_finalized_slot = finalized_block.slot; - debug!("got finalized blockmeta {} with blockhash {}", - finalized_block.slot, finalized_block.blockhash.clone()); - if let Err(e) = sender.send(finalized_block) { - warn!("Finalized block channel has no receivers {e:?}"); - continue; + let confirmed_blocks_stream = { + let commitment_config = CommitmentConfig::confirmed(); + + let mut streams = Vec::new(); + for grpc_source in &grpc_sources { + let stream = create_geyser_reconnecting_stream( + grpc_source.clone(), + GeyserFilter(commitment_config).blocks_and_txs(), + ); + streams.push(stream); + } + + create_multiplexed_stream(streams, BlockExtractor(commitment_config)) + }; + + let finalized_blockmeta_stream = { + let commitment_config = CommitmentConfig::finalized(); + + let mut streams = Vec::new(); + for grpc_source in &grpc_sources { + let stream = create_geyser_reconnecting_stream( + grpc_source.clone(), + GeyserFilter(commitment_config).blocks_meta(), + ); + streams.push(stream); + } + create_multiplexed_stream(streams, BlockMetaHashExtractor(commitment_config)) + }; + + // by blockhash + let mut recent_confirmed_blocks = HashMap::::new(); + let mut confirmed_blocks_stream = std::pin::pin!(confirmed_blocks_stream); + let mut finalized_blockmeta_stream = std::pin::pin!(finalized_blockmeta_stream); + + let mut cleanup_tick = tokio::time::interval(Duration::from_secs(5)); + let mut last_finalized_slot: Slot = 0; + let mut cleanup_without_recv_blocks: u8 = 0; + let mut cleanup_without_recv_blocks_meta: u8 = 0; + const MAX_ALLOWED_CLEANUP_WITHOUT_RECV : u8 = 12; // 12*5 = 60s without recving data + loop { + tokio::select! { + confirmed_block = confirmed_blocks_stream.next() => { + cleanup_without_recv_blocks = 0; + + let confirmed_block = confirmed_block.expect("confirmed block from stream"); + trace!("got confirmed block {} with blockhash {}", + confirmed_block.slot, confirmed_block.blockhash.clone()); + if let Err(e) = producedblock_sender.send(confirmed_block.clone()) { + warn!("Confirmed block channel has no receivers {e:?}"); + continue + } + recent_confirmed_blocks.insert(confirmed_block.blockhash.clone(), confirmed_block); + }, + meta_finalized = finalized_blockmeta_stream.next() => { + cleanup_without_recv_blocks_meta = 0; + let blockhash = meta_finalized.expect("finalized block meta from stream"); + if let Some(cached_confirmed_block) = recent_confirmed_blocks.remove(&blockhash) { + let finalized_block = cached_confirmed_block.to_finalized_block(); + last_finalized_slot = finalized_block.slot; + debug!("got finalized blockmeta {} with blockhash {}", + finalized_block.slot, finalized_block.blockhash.clone()); + if let Err(e) = producedblock_sender.send(finalized_block) { + warn!("Finalized block channel has no receivers {e:?}"); + continue; + } + } else { + debug!("finalized block meta received for blockhash {} which was never seen or already emitted", blockhash); + } + }, + _ = cleanup_tick.tick() => { + if cleanup_without_recv_blocks_meta > MAX_ALLOWED_CLEANUP_WITHOUT_RECV || + cleanup_without_recv_blocks > MAX_ALLOWED_CLEANUP_WITHOUT_RECV { + log::error!("block or block meta stream stopped restaring blocks"); + break; + } + cleanup_without_recv_blocks += 1; + cleanup_without_recv_blocks_meta += 1; + let size_before = recent_confirmed_blocks.len(); + recent_confirmed_blocks.retain(|_blockhash, block| { + last_finalized_slot == 0 || block.slot > last_finalized_slot - 100 + }); + let cnt_cleaned = size_before - recent_confirmed_blocks.len(); + if cnt_cleaned > 0 { + debug!("cleaned {} confirmed blocks from cache", cnt_cleaned); } - } else { - debug!("finalized block meta received for blockhash {} which was never seen or already emitted", blockhash); - } - }, - _ = cleanup_tick.tick() => { - let size_before = recent_confirmed_blocks.len(); - recent_confirmed_blocks.retain(|_blockhash, block| { - last_finalized_slot == 0 || block.slot > last_finalized_slot - 100 - }); - let cnt_cleaned = size_before - recent_confirmed_blocks.len(); - if cnt_cleaned > 0 { - debug!("cleaned {} confirmed blocks from cache", cnt_cleaned); } } } @@ -179,37 +192,63 @@ pub fn create_grpc_multiplex_slots_subscription( info!("- connection to {}", grpc_source); } - let multiplex_stream = { - let mut streams = Vec::new(); - for grpc_source in &grpc_sources { - let mut slots = HashMap::new(); - slots.insert( - "client".to_string(), - SubscribeRequestFilterSlots { - filter_by_commitment: Some(true), - }, - ); - - let filter = SubscribeRequest { - slots, - accounts: Default::default(), - transactions: HashMap::new(), - entry: Default::default(), - blocks: HashMap::new(), - blocks_meta: HashMap::new(), - commitment: Some(yellowstone_grpc_proto::geyser::CommitmentLevel::Processed as i32), - accounts_data_slice: Default::default(), - ping: None, + let (multiplexed_messages_sender, multiplexed_messages) = tokio::sync::broadcast::channel(1000); + + let jh = tokio::spawn(async move { + loop { + let multiplex_stream = { + let mut streams = Vec::new(); + for grpc_source in &grpc_sources { + let mut slots = HashMap::new(); + slots.insert( + "client".to_string(), + SubscribeRequestFilterSlots { + filter_by_commitment: Some(true), + }, + ); + + let filter = SubscribeRequest { + slots, + accounts: Default::default(), + transactions: HashMap::new(), + entry: Default::default(), + blocks: HashMap::new(), + blocks_meta: HashMap::new(), + commitment: Some(yellowstone_grpc_proto::geyser::CommitmentLevel::Processed as i32), + accounts_data_slice: Default::default(), + ping: None, + }; + + let stream = create_geyser_reconnecting_stream(grpc_source.clone(), filter); + streams.push(stream); + } + + create_multiplexed_stream(streams, SlotExtractor {}) }; - let stream = create_geyser_reconnecting_stream(grpc_source.clone(), filter); - streams.push(stream); + let mut multiplex_stream = std::pin::pin!(multiplex_stream); + loop { + tokio::select! { + slot_data = multiplex_stream.next() => { + if let Some(slot_data) = slot_data { + match multiplexed_messages_sender.send(slot_data) { + Ok(receivers) => { + trace!("sent data to {} receivers", receivers); + } + Err(send_error) => log::error!("Get error while sending on slot channel {}", send_error), + }; + } else { + debug!("Slot stream send None type"); + } + }, + _ = tokio::time::sleep(Duration::from_secs(30)) => { + log::error!("Slots timedout restarting subscription"); + break; + } + } + } } + }); - create_multiplexed_stream(streams, SlotExtractor {}) - }; - - let (multiplexed_stream, jh_channelizer) = channelize_stream(multiplex_stream); - - (multiplexed_stream, jh_channelizer) + (multiplexed_messages, jh) } From 24989505832af57f7d13e6f9c9b98e3257d16ec5 Mon Sep 17 00:00:00 2001 From: godmodegalactus Date: Fri, 12 Jan 2024 14:33:00 +0100 Subject: [PATCH 50/56] Reconnecing to block stream and slot stream incase of error --- .env.example | 2 +- core/src/quic_connection_utils.rs | 4 ++-- lite-rpc/src/bridge.rs | 3 +-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.env.example b/.env.example index d777fe6e..2f0ac619 100644 --- a/.env.example +++ b/.env.example @@ -11,7 +11,7 @@ WS_ADDR=ws://0.0.0.0:8900 # PROMETHEUS_ADDR=your_prometheus_address_here ## Fanout size and retries configuration -FANOUT_SIZE=18 +FANOUT_SIZE=32 MAX_RETRIES=40 RETRY_TIMEOUT=3 diff --git a/core/src/quic_connection_utils.rs b/core/src/quic_connection_utils.rs index 4f3c7400..a5c40b5a 100644 --- a/core/src/quic_connection_utils.rs +++ b/core/src/quic_connection_utils.rs @@ -158,7 +158,7 @@ impl QuicConnectionUtils { } } Err(_) => { - warn!("timeout while writing transaction for {}", identity); + log::debug!("timeout while writing transaction for {}", identity); return Err(QuicConnectionError::TimeOut); } } @@ -177,7 +177,7 @@ impl QuicConnectionUtils { } } Err(_) => { - warn!("timeout while finishing transaction for {}", identity); + log::debug!("timeout while finishing transaction for {}", identity); return Err(QuicConnectionError::TimeOut); } } diff --git a/lite-rpc/src/bridge.rs b/lite-rpc/src/bridge.rs index f91bd591..e301feb6 100644 --- a/lite-rpc/src/bridge.rs +++ b/lite-rpc/src/bridge.rs @@ -12,7 +12,6 @@ use solana_lite_rpc_services::{ use anyhow::Context; use jsonrpsee::{core::SubscriptionResult, server::ServerBuilder, PendingSubscriptionSink}; -use log::info; use prometheus::{opts, register_int_counter, IntCounter}; use solana_lite_rpc_core::{ stores::{block_information_store::BlockInformation, data_cache::DataCache, tx_store::TxProps}, @@ -207,7 +206,7 @@ impl LiteRpcServer for LiteBridge { .get_latest_block(commitment_config) .await; - info!("glb {blockhash} {slot} {block_height}"); + log::trace!("glb {blockhash} {slot} {block_height}"); Ok(RpcResponse { context: RpcResponseContext { From c3da6ad1b6dedca66d881c12fce9fce4585087e5 Mon Sep 17 00:00:00 2001 From: godmodegalactus Date: Fri, 12 Jan 2024 14:40:48 +0100 Subject: [PATCH 51/56] cargo fmt and clippy issues --- cluster-endpoints/src/grpc_multiplex.rs | 22 ++++++++++++---------- core/src/quic_connection_utils.rs | 2 +- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/cluster-endpoints/src/grpc_multiplex.rs b/cluster-endpoints/src/grpc_multiplex.rs index f0a819f2..11d78a5e 100644 --- a/cluster-endpoints/src/grpc_multiplex.rs +++ b/cluster-endpoints/src/grpc_multiplex.rs @@ -70,7 +70,7 @@ pub fn create_grpc_multiplex_blocks_subscription( loop { let confirmed_blocks_stream = { let commitment_config = CommitmentConfig::confirmed(); - + let mut streams = Vec::new(); for grpc_source in &grpc_sources { let stream = create_geyser_reconnecting_stream( @@ -79,13 +79,13 @@ pub fn create_grpc_multiplex_blocks_subscription( ); streams.push(stream); } - + create_multiplexed_stream(streams, BlockExtractor(commitment_config)) }; - + let finalized_blockmeta_stream = { let commitment_config = CommitmentConfig::finalized(); - + let mut streams = Vec::new(); for grpc_source in &grpc_sources { let stream = create_geyser_reconnecting_stream( @@ -106,7 +106,7 @@ pub fn create_grpc_multiplex_blocks_subscription( let mut last_finalized_slot: Slot = 0; let mut cleanup_without_recv_blocks: u8 = 0; let mut cleanup_without_recv_blocks_meta: u8 = 0; - const MAX_ALLOWED_CLEANUP_WITHOUT_RECV : u8 = 12; // 12*5 = 60s without recving data + const MAX_ALLOWED_CLEANUP_WITHOUT_RECV: u8 = 12; // 12*5 = 60s without recving data loop { tokio::select! { confirmed_block = confirmed_blocks_stream.next() => { @@ -206,7 +206,7 @@ pub fn create_grpc_multiplex_slots_subscription( filter_by_commitment: Some(true), }, ); - + let filter = SubscribeRequest { slots, accounts: Default::default(), @@ -214,15 +214,17 @@ pub fn create_grpc_multiplex_slots_subscription( entry: Default::default(), blocks: HashMap::new(), blocks_meta: HashMap::new(), - commitment: Some(yellowstone_grpc_proto::geyser::CommitmentLevel::Processed as i32), + commitment: Some( + yellowstone_grpc_proto::geyser::CommitmentLevel::Processed as i32, + ), accounts_data_slice: Default::default(), ping: None, }; - + let stream = create_geyser_reconnecting_stream(grpc_source.clone(), filter); streams.push(stream); } - + create_multiplexed_stream(streams, SlotExtractor {}) }; @@ -230,7 +232,7 @@ pub fn create_grpc_multiplex_slots_subscription( loop { tokio::select! { slot_data = multiplex_stream.next() => { - if let Some(slot_data) = slot_data { + if let Some(slot_data) = slot_data { match multiplexed_messages_sender.send(slot_data) { Ok(receivers) => { trace!("sent data to {} receivers", receivers); diff --git a/core/src/quic_connection_utils.rs b/core/src/quic_connection_utils.rs index a5c40b5a..e64a2330 100644 --- a/core/src/quic_connection_utils.rs +++ b/core/src/quic_connection_utils.rs @@ -1,4 +1,4 @@ -use log::{info, trace, warn}; +use log::{info, trace}; use quinn::{ ClientConfig, Connection, ConnectionError, Endpoint, EndpointConfig, IdleTimeout, SendStream, TokioRuntime, TransportConfig, From 9e1c36771a51fb060a54f7a3de88cdc2f25165d7 Mon Sep 17 00:00:00 2001 From: godmodegalactus Date: Fri, 12 Jan 2024 15:57:09 +0100 Subject: [PATCH 52/56] Fix CI --- .github/workflows/clippy_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clippy_test.yml b/.github/workflows/clippy_test.yml index 368c0636..c00e6a21 100644 --- a/.github/workflows/clippy_test.yml +++ b/.github/workflows/clippy_test.yml @@ -48,5 +48,5 @@ jobs: - name: Run fmt+clippy run: | - cargo fmt --all --check - cargo clippy --locked --workspace --all-targets -- -D warnings + cargo +nightly fmt --all --check + cargo +nightly clippy --locked --workspace --all-targets -- -D warnings From 1923cb9fefb62706abad1647fcbf37b3960ddf52 Mon Sep 17 00:00:00 2001 From: godmodegalactus Date: Fri, 12 Jan 2024 16:12:20 +0100 Subject: [PATCH 53/56] Fix clippy --- bench/src/helpers.rs | 4 ++-- stake_vote/src/leader_schedule.rs | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/bench/src/helpers.rs b/bench/src/helpers.rs index f35e88bb..e3469b19 100644 --- a/bench/src/helpers.rs +++ b/bench/src/helpers.rs @@ -146,7 +146,7 @@ fn transaction_size_small() { let seed = 42; let random_strings = BenchHelper::generate_random_strings(1, Some(seed), 10); - let rand_string = random_strings.get(0).unwrap(); + let rand_string = random_strings.first().unwrap(); let tx = BenchHelper::create_memo_tx_small(rand_string, &payer_keypair, blockhash); assert_eq!(bincode::serialized_size(&tx).unwrap(), 179); @@ -161,7 +161,7 @@ fn transaction_size_large() { let seed = 42; let random_strings = BenchHelper::generate_random_strings(1, Some(seed), 240); - let rand_string = random_strings.get(0).unwrap(); + let rand_string = random_strings.first().unwrap(); let tx = BenchHelper::create_memo_tx_large(rand_string, &payer_keypair, blockhash); assert_eq!(bincode::serialized_size(&tx).unwrap(), 1186); diff --git a/stake_vote/src/leader_schedule.rs b/stake_vote/src/leader_schedule.rs index 7d217c24..59ff130d 100644 --- a/stake_vote/src/leader_schedule.rs +++ b/stake_vote/src/leader_schedule.rs @@ -299,7 +299,6 @@ pub fn calculate_leader_schedule( .aggregate(|acc, _node_pubkey, stake| Some(acc.unwrap_or_default() + stake)); let mut stakes: Vec<(Pubkey, u64)> = stakes_map .into_iter() - .map(|(key, stake)| (key, stake)) .collect(); let mut seed = [0u8; 32]; From 1acc71d51dbf6e446d56c2e48a89cadc9a2f7746 Mon Sep 17 00:00:00 2001 From: godmodegalactus Date: Fri, 12 Jan 2024 16:19:00 +0100 Subject: [PATCH 54/56] Fix fmt --- stake_vote/src/leader_schedule.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/stake_vote/src/leader_schedule.rs b/stake_vote/src/leader_schedule.rs index 59ff130d..a8da85c9 100644 --- a/stake_vote/src/leader_schedule.rs +++ b/stake_vote/src/leader_schedule.rs @@ -297,9 +297,7 @@ pub fn calculate_leader_schedule( }) .into_grouping_map() .aggregate(|acc, _node_pubkey, stake| Some(acc.unwrap_or_default() + stake)); - let mut stakes: Vec<(Pubkey, u64)> = stakes_map - .into_iter() - .collect(); + let mut stakes: Vec<(Pubkey, u64)> = stakes_map.into_iter().collect(); let mut seed = [0u8; 32]; seed[0..8].copy_from_slice(&epoch.to_le_bytes()); From 78e1bb2b797b1d372880a2ce5688daf7c2f859d6 Mon Sep 17 00:00:00 2001 From: godmodegalactus Date: Fri, 12 Jan 2024 16:39:19 +0100 Subject: [PATCH 55/56] More clippy changes --- .github/workflows/clippy_test.yml | 6 +++--- cluster-endpoints/src/grpc_subscription.rs | 16 ++++------------ core/src/stores/block_information_store.rs | 4 +--- run_clippy_fmt.sh | 2 ++ 4 files changed, 10 insertions(+), 18 deletions(-) create mode 100644 run_clippy_fmt.sh diff --git a/.github/workflows/clippy_test.yml b/.github/workflows/clippy_test.yml index c00e6a21..f2f4bc29 100644 --- a/.github/workflows/clippy_test.yml +++ b/.github/workflows/clippy_test.yml @@ -28,7 +28,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: # use toolchain version from rust-toolchain.toml - toolchain: nightly + toolchain: +nightly-2024-01-05 components: rustfmt, clippy cache: true # avoid the default "-D warnings" which thrashes cache @@ -48,5 +48,5 @@ jobs: - name: Run fmt+clippy run: | - cargo +nightly fmt --all --check - cargo +nightly clippy --locked --workspace --all-targets -- -D warnings + cargo +nightly-2024-01-05 fmt --all --check + cargo +nightly-2024-01-05 clippy --locked --workspace --all-targets -- -D warnings diff --git a/cluster-endpoints/src/grpc_subscription.rs b/cluster-endpoints/src/grpc_subscription.rs index 13da5faa..1a6593aa 100644 --- a/cluster-endpoints/src/grpc_subscription.rs +++ b/cluster-endpoints/src/grpc_subscription.rs @@ -47,21 +47,13 @@ pub fn map_block_update( .transactions .into_iter() .filter_map(|tx| { - let Some(meta) = tx.meta else { - return None; - }; + let meta = tx.meta?; - let Some(transaction) = tx.transaction else { - return None; - }; + let transaction = tx.transaction?; - let Some(message) = transaction.message else { - return None; - }; + let message = transaction.message?; - let Some(header) = message.header else { - return None; - }; + let header = message.header?; let signatures = transaction .signatures diff --git a/core/src/stores/block_information_store.rs b/core/src/stores/block_information_store.rs index 762d429a..610cbe08 100644 --- a/core/src/stores/block_information_store.rs +++ b/core/src/stores/block_information_store.rs @@ -58,9 +58,7 @@ impl BlockInformationStore { } pub fn get_block_info(&self, blockhash: &str) -> Option { - let Some(info) = self.blocks.get(blockhash) else { - return None; - }; + let info = self.blocks.get(blockhash)?; Some(info.value().to_owned()) } diff --git a/run_clippy_fmt.sh b/run_clippy_fmt.sh new file mode 100644 index 00000000..ed379a8b --- /dev/null +++ b/run_clippy_fmt.sh @@ -0,0 +1,2 @@ +cargo +nightly-2024-01-05 fmt --all +cargo +nightly-2024-01-05 clippy --locked --workspace --all-targets -- -D warnings \ No newline at end of file From 338b20f6875c7138bdcc9d6913a168a33d9e74b8 Mon Sep 17 00:00:00 2001 From: godmodegalactus Date: Fri, 12 Jan 2024 17:47:13 +0100 Subject: [PATCH 56/56] minor changes --- .github/workflows/clippy_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clippy_test.yml b/.github/workflows/clippy_test.yml index f2f4bc29..f267d618 100644 --- a/.github/workflows/clippy_test.yml +++ b/.github/workflows/clippy_test.yml @@ -28,7 +28,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: # use toolchain version from rust-toolchain.toml - toolchain: +nightly-2024-01-05 + toolchain: nightly-2024-01-05 components: rustfmt, clippy cache: true # avoid the default "-D warnings" which thrashes cache