From bc048541fc785c97873e7048fcbc4dbd567f6cee Mon Sep 17 00:00:00 2001 From: Jeff Biseda Date: Tue, 19 Sep 2023 17:19:23 -0700 Subject: [PATCH] Warn if vaildator appears to be repairing excessively --- banking-bench/src/main.rs | 7 +- core/benches/banking_stage.rs | 7 +- core/src/accounts_hash_verifier.rs | 7 +- core/src/banking_stage.rs | 8 +- core/src/cluster_slots_service.rs | 7 +- core/src/repair/ancestor_hashes_service.rs | 2 + core/src/repair/repair_service.rs | 7 +- core/src/repair/serve_repair.rs | 7 +- core/src/replay_stage.rs | 1 + core/src/tvu.rs | 8 +- core/src/validator.rs | 3 +- core/src/window_service.rs | 20 +++- core/tests/epoch_accounts_hash.rs | 1 + core/tests/snapshots.rs | 8 +- gossip/src/cluster_info.rs | 109 ++++++++++++++++-- gossip/src/duplicate_shred_listener.rs | 1 + gossip/src/gossip_service.rs | 9 +- gossip/tests/gossip.rs | 2 + ledger-tool/src/ledger_utils.rs | 1 + ledger/src/blockstore.rs | 25 +++- ledger/src/blockstore_metrics.rs | 67 ++++++++++- ledger/src/slot_stats.rs | 5 +- rpc/src/cluster_tpu_info.rs | 1 + rpc/src/rpc.rs | 22 +++- rpc/src/rpc_health.rs | 40 ++----- rpc/src/rpc_service.rs | 21 ++-- turbine/benches/cluster_info.rs | 1 + turbine/benches/retransmit_stage.rs | 7 +- turbine/src/broadcast_stage.rs | 1 + .../broadcast_fake_shreds_run.rs | 7 +- .../broadcast_stage/standard_broadcast_run.rs | 1 + turbine/src/cluster_nodes.rs | 7 +- validator/src/admin_rpc_service.rs | 1 + validator/src/bootstrap.rs | 7 +- 34 files changed, 348 insertions(+), 80 deletions(-) diff --git a/banking-bench/src/main.rs b/banking-bench/src/main.rs index bb5149f47c85b9..a99a207fbba367 100644 --- a/banking-bench/src/main.rs +++ b/banking-bench/src/main.rs @@ -436,7 +436,12 @@ fn main() { let cluster_info = { let keypair = Arc::new(Keypair::new()); let node = Node::new_localhost_with_pubkey(&keypair.pubkey()); - ClusterInfo::new(node.info, keypair, SocketAddrSpace::Unspecified) + ClusterInfo::new( + node.info, + keypair, + SocketAddrSpace::Unspecified, + /*known_validators*/ None, + ) }; let cluster_info = Arc::new(cluster_info); let tpu_disable_quic = matches.is_present("tpu_disable_quic"); diff --git a/core/benches/banking_stage.rs b/core/benches/banking_stage.rs index 6219f4abb9a265..5d278cc80ccec7 100644 --- a/core/benches/banking_stage.rs +++ b/core/benches/banking_stage.rs @@ -288,7 +288,12 @@ fn bench_banking(bencher: &mut Bencher, tx_type: TransactionType) { let cluster_info = { let keypair = Arc::new(Keypair::new()); let node = Node::new_localhost_with_pubkey(&keypair.pubkey()); - ClusterInfo::new(node.info, keypair, SocketAddrSpace::Unspecified) + ClusterInfo::new( + node.info, + keypair, + SocketAddrSpace::Unspecified, + /*known_validators*/ None, + ) }; let cluster_info = Arc::new(cluster_info); let (s, _r) = unbounded(); diff --git a/core/src/accounts_hash_verifier.rs b/core/src/accounts_hash_verifier.rs index cb87cdc513a90c..4cf7229b97e28e 100644 --- a/core/src/accounts_hash_verifier.rs +++ b/core/src/accounts_hash_verifier.rs @@ -601,7 +601,12 @@ mod tests { fn new_test_cluster_info() -> ClusterInfo { let keypair = Arc::new(Keypair::new()); let contact_info = ContactInfo::new_localhost(&keypair.pubkey(), timestamp()); - ClusterInfo::new(contact_info, keypair, SocketAddrSpace::Unspecified) + ClusterInfo::new( + contact_info, + keypair, + SocketAddrSpace::Unspecified, + /*known_validators*/ None, + ) } #[test] diff --git a/core/src/banking_stage.rs b/core/src/banking_stage.rs index 2dfb1e32b1d688..0d01a7690405ea 100644 --- a/core/src/banking_stage.rs +++ b/core/src/banking_stage.rs @@ -648,8 +648,12 @@ mod tests { pub(crate) fn new_test_cluster_info(keypair: Option>) -> (Node, ClusterInfo) { let keypair = keypair.unwrap_or_else(|| Arc::new(Keypair::new())); let node = Node::new_localhost_with_pubkey(&keypair.pubkey()); - let cluster_info = - ClusterInfo::new(node.info.clone(), keypair, SocketAddrSpace::Unspecified); + let cluster_info = ClusterInfo::new( + node.info.clone(), + keypair, + SocketAddrSpace::Unspecified, + /*known_validators*/ None, + ); (node, cluster_info) } diff --git a/core/src/cluster_slots_service.rs b/core/src/cluster_slots_service.rs index 7f6226e87e468f..4b64af820bba42 100644 --- a/core/src/cluster_slots_service.rs +++ b/core/src/cluster_slots_service.rs @@ -192,7 +192,12 @@ mod test { let keypair = Arc::new(Keypair::new()); let pubkey = keypair.pubkey(); let node_info = Node::new_localhost_with_pubkey(&pubkey); - let cluster_info = ClusterInfo::new(node_info.info, keypair, SocketAddrSpace::Unspecified); + let cluster_info = ClusterInfo::new( + node_info.info, + keypair, + SocketAddrSpace::Unspecified, + /*known_validators*/ None, + ); ClusterSlotsService::update_lowest_slot(5, &cluster_info); cluster_info.flush_push_queue(); let lowest = { diff --git a/core/src/repair/ancestor_hashes_service.rs b/core/src/repair/ancestor_hashes_service.rs index 3214c89e14ea15..b039cd66fdcbc1 100644 --- a/core/src/repair/ancestor_hashes_service.rs +++ b/core/src/repair/ancestor_hashes_service.rs @@ -1256,6 +1256,7 @@ mod test { responder_node.info.clone(), Arc::new(keypair), SocketAddrSpace::Unspecified, + /*known_validators*/ None, ); let responder_serve_repair = ServeRepair::new( Arc::new(cluster_info), @@ -1350,6 +1351,7 @@ mod test { Node::new_localhost_with_pubkey(&keypair.pubkey()).info, Arc::new(keypair), SocketAddrSpace::Unspecified, + /*known_validators*/ None, )); let repair_whitelist = Arc::new(RwLock::new(HashSet::default())); let requester_serve_repair = ServeRepair::new( diff --git a/core/src/repair/repair_service.rs b/core/src/repair/repair_service.rs index c7cfab03f8f8ed..db6cdb46c9030a 100644 --- a/core/src/repair/repair_service.rs +++ b/core/src/repair/repair_service.rs @@ -879,7 +879,12 @@ mod test { fn new_test_cluster_info() -> ClusterInfo { let keypair = Arc::new(Keypair::new()); let contact_info = ContactInfo::new_localhost(&keypair.pubkey(), timestamp()); - ClusterInfo::new(contact_info, keypair, SocketAddrSpace::Unspecified) + ClusterInfo::new( + contact_info, + keypair, + SocketAddrSpace::Unspecified, + /*known_validators*/ None, + ) } #[test] diff --git a/core/src/repair/serve_repair.rs b/core/src/repair/serve_repair.rs index 8ab42c28829f1d..9741f4b8b078f5 100644 --- a/core/src/repair/serve_repair.rs +++ b/core/src/repair/serve_repair.rs @@ -1969,7 +1969,12 @@ mod tests { fn new_test_cluster_info() -> ClusterInfo { let keypair = Arc::new(Keypair::new()); let contact_info = ContactInfo::new_localhost(&keypair.pubkey(), timestamp()); - ClusterInfo::new(contact_info, keypair, SocketAddrSpace::Unspecified) + ClusterInfo::new( + contact_info, + keypair, + SocketAddrSpace::Unspecified, + /*known_validators*/ None, + ) } #[test] diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs index 0fec5020d6dcb9..8d630866a56834 100644 --- a/core/src/replay_stage.rs +++ b/core/src/replay_stage.rs @@ -4110,6 +4110,7 @@ pub(crate) mod tests { Node::new_localhost_with_pubkey(&my_pubkey).info, Arc::new(my_keypairs.node_keypair.insecure_clone()), SocketAddrSpace::Unspecified, + /*known_validators*/ None, ); assert_eq!(my_pubkey, cluster_info.id()); diff --git a/core/src/tvu.rs b/core/src/tvu.rs index 0b8358863fbceb..86413cd42ab84d 100644 --- a/core/src/tvu.rs +++ b/core/src/tvu.rs @@ -410,8 +410,12 @@ pub mod tests { let (repair_quic_endpoint_sender, _repair_quic_endpoint_receiver) = tokio::sync::mpsc::channel(/*buffer:*/ 128); //start cluster_info1 - let cluster_info1 = - ClusterInfo::new(target1.info.clone(), keypair, SocketAddrSpace::Unspecified); + let cluster_info1 = ClusterInfo::new( + target1.info.clone(), + keypair, + SocketAddrSpace::Unspecified, + /*known_validators*/ None, + ); cluster_info1.insert_info(leader.info); let cref1 = Arc::new(cluster_info1); diff --git a/core/src/validator.rs b/core/src/validator.rs index a0c39da764239b..711482e345c8ba 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -725,6 +725,7 @@ impl Validator { node.info.clone(), identity_keypair.clone(), socket_addr_space, + config.known_validators.clone(), ); cluster_info.set_contact_debug_interval(config.contact_debug_interval); cluster_info.set_entrypoints(cluster_entrypoints); @@ -971,7 +972,6 @@ impl Validator { ledger_path, config.validator_exit.clone(), exit.clone(), - config.known_validators.clone(), rpc_override_health_check.clone(), startup_verification_complete, optimistically_confirmed_bank.clone(), @@ -2576,6 +2576,7 @@ mod tests { ContactInfo::new_localhost(&node_keypair.pubkey(), timestamp()), node_keypair, SocketAddrSpace::Unspecified, + /*known_validators*/ None, ); let (genesis_config, _mint_keypair) = create_genesis_config(1); diff --git a/core/src/window_service.rs b/core/src/window_service.rs index a68a20e2078471..46f79733d5448d 100644 --- a/core/src/window_service.rs +++ b/core/src/window_service.rs @@ -1,6 +1,7 @@ //! `window_service` handles the data plane incoming shreds, storing them in //! blockstore and retransmitting where required //! + use { crate::{ cluster_info_vote_listener::VerifiedVoteReceiver, @@ -21,6 +22,7 @@ use { solana_gossip::cluster_info::ClusterInfo, solana_ledger::{ blockstore::{Blockstore, BlockstoreInsertionMetrics, PossibleDuplicateShred}, + blockstore_metrics::ExcessiveRepairContext, leader_schedule_cache::LeaderScheduleCache, shred::{self, Nonce, ReedSolomonCache, Shred}, }, @@ -236,6 +238,7 @@ fn run_insert( handle_duplicate: F, metrics: &mut BlockstoreInsertionMetrics, ws_metrics: &mut WindowServiceMetrics, + excessive_repair_context: &mut ExcessiveRepairContext, completed_data_sets_sender: &CompletedDataSetsSender, retransmit_sender: &Sender>, outstanding_requests: &RwLock, @@ -303,6 +306,7 @@ where &handle_duplicate, reed_solomon_cache, metrics, + excessive_repair_context, )?; completed_data_sets_sender.try_send(completed_data_sets)?; @@ -361,7 +365,7 @@ impl WindowService { let (duplicate_sender, duplicate_receiver) = unbounded(); let t_check_duplicate = Self::start_check_duplicate_thread( - cluster_info, + cluster_info.clone(), exit.clone(), blockstore.clone(), duplicate_receiver, @@ -369,6 +373,7 @@ impl WindowService { ); let t_insert = Self::start_window_insert_thread( + cluster_info, exit, blockstore, leader_schedule_cache, @@ -416,6 +421,7 @@ impl WindowService { } fn start_window_insert_thread( + cluster_info: Arc, exit: Arc, blockstore: Arc, leader_schedule_cache: Arc, @@ -442,6 +448,8 @@ impl WindowService { }; let mut metrics = BlockstoreInsertionMetrics::default(); let mut ws_metrics = WindowServiceMetrics::default(); + let mut excessive_repair_context = ExcessiveRepairContext::default(); + let mut last_print = Instant::now(); while !exit.load(Ordering::Relaxed) { if let Err(e) = run_insert( @@ -452,6 +460,7 @@ impl WindowService { handle_duplicate, &mut metrics, &mut ws_metrics, + &mut excessive_repair_context, &completed_data_sets_sender, &retransmit_sender, &outstanding_requests, @@ -464,6 +473,12 @@ impl WindowService { } if last_print.elapsed().as_secs() > 2 { + excessive_repair_context.report_excessive_repairs(|| { + ( + cluster_info.get_my_max_slot(), + cluster_info.get_known_validators_max_slot(), + ) + }); metrics.report_metrics("blockstore-insert-shreds"); metrics = BlockstoreInsertionMetrics::default(); ws_metrics.report_metrics("recv-window-insert-shreds"); @@ -581,6 +596,7 @@ mod test { contact_info, Arc::new(keypair), SocketAddrSpace::Unspecified, + /*known_validators*/ None, ); run_check_duplicate( &cluster_info, @@ -615,6 +631,7 @@ mod test { contact_info, Arc::new(keypair), SocketAddrSpace::Unspecified, + /*known_validators*/ None, )); // Start duplicate thread receiving and inserting duplicates @@ -650,6 +667,7 @@ mod test { &handle_duplicate, &ReedSolomonCache::default(), &mut BlockstoreInsertionMetrics::default(), + &mut ExcessiveRepairContext::default(), ) .unwrap(); diff --git a/core/tests/epoch_accounts_hash.rs b/core/tests/epoch_accounts_hash.rs index 1f6eb702769d3e..a40b6f1571b574 100755 --- a/core/tests/epoch_accounts_hash.rs +++ b/core/tests/epoch_accounts_hash.rs @@ -130,6 +130,7 @@ impl TestEnvironment { ContactInfo::new_localhost(&node_id.pubkey(), timestamp()), Arc::clone(&node_id), SocketAddrSpace::Unspecified, + /*known_validators*/ None, )); let pruned_banks_receiver = diff --git a/core/tests/snapshots.rs b/core/tests/snapshots.rs index b61e84a90810c9..6039079c5b8d95 100644 --- a/core/tests/snapshots.rs +++ b/core/tests/snapshots.rs @@ -505,7 +505,12 @@ fn test_concurrent_snapshot_packaging( timestamp(), // wallclock 0u16, // shred_version ); - ClusterInfo::new(contact_info, keypair, SocketAddrSpace::Unspecified) + ClusterInfo::new( + contact_info, + keypair, + SocketAddrSpace::Unspecified, + /*known_validators*/ None, + ) }); let (snapshot_package_sender, snapshot_package_receiver) = crossbeam_channel::unbounded(); @@ -955,6 +960,7 @@ fn test_snapshots_with_background_services( ContactInfo::new_localhost(&node_keypair.pubkey(), timestamp()), node_keypair, SocketAddrSpace::Unspecified, + /*known_validators*/ None, )); let (pruned_banks_sender, pruned_banks_receiver) = unbounded(); diff --git a/gossip/src/cluster_info.rs b/gossip/src/cluster_info.rs index b0b99b1c02dca4..0799d39535f381 100644 --- a/gossip/src/cluster_info.rs +++ b/gossip/src/cluster_info.rs @@ -178,6 +178,7 @@ pub struct ClusterInfo { instance: RwLock, contact_info_path: PathBuf, socket_addr_space: SocketAddrSpace, + known_validators: Option>, } #[derive(Clone, Debug, Default, Deserialize, Serialize, AbiExample)] @@ -411,6 +412,7 @@ impl ClusterInfo { contact_info: ContactInfo, keypair: Arc, socket_addr_space: SocketAddrSpace, + known_validators: Option>, ) -> Self { assert_eq!(contact_info.pubkey(), &keypair.pubkey()); let id = *contact_info.pubkey(); @@ -433,6 +435,7 @@ impl ClusterInfo { contact_info_path: PathBuf::default(), contact_save_interval: 0, // disabled socket_addr_space, + known_validators, }; me.insert_self(); me.push_self(); @@ -447,6 +450,14 @@ impl ClusterInfo { &self.socket_addr_space } + pub fn known_validators(&self) -> &Option> { + &self.known_validators + } + + pub fn set_known_validators(&mut self, known_validators: Option>) { + self.known_validators = known_validators; + } + fn push_self(&self) { let now = timestamp(); let node = { @@ -2770,6 +2781,29 @@ impl ClusterInfo { (contact_info, gossip_socket, None) } + + fn get_node_max_slot(&self, node: &Pubkey) -> Option { + self.get_accounts_hash_for_node(node, |hashes| { + hashes + .iter() + .max_by(|a, b| a.0.cmp(&b.0)) + .map(|slot_hash| slot_hash.0) + }) + .flatten() + } + + pub fn get_my_max_slot(&self) -> Option { + self.get_node_max_slot(&self.id()) + } + + pub fn get_known_validators_max_slot(&self) -> Option { + self.known_validators.as_ref().and_then(|known_validators| { + known_validators + .iter() + .filter_map(|known_validator| self.get_node_max_slot(known_validator)) + .max() + }) + } } // Returns root bank's epoch duration. Falls back on @@ -3227,7 +3261,12 @@ mod tests { let cluster_info = Arc::new({ let keypair = Arc::new(Keypair::new()); let node = Node::new_localhost_with_pubkey(&keypair.pubkey()); - ClusterInfo::new(node.info, keypair, SocketAddrSpace::Unspecified) + ClusterInfo::new( + node.info, + keypair, + SocketAddrSpace::Unspecified, + /*known_validators*/ None, + ) }); let entrypoint_pubkey = solana_sdk::pubkey::new_rand(); let data = test_crds_values(entrypoint_pubkey); @@ -3291,6 +3330,7 @@ mod tests { ContactInfo::new_localhost(&this_node.pubkey(), timestamp()), this_node.clone(), SocketAddrSpace::Unspecified, + /*known_validators*/ None, ); let remote_nodes: Vec<(Keypair, SocketAddr)> = repeat_with(|| new_rand_remote_node(&mut rng)) @@ -3346,6 +3386,7 @@ mod tests { ContactInfo::new_localhost(&this_node.pubkey(), timestamp()), this_node.clone(), SocketAddrSpace::Unspecified, + /*known_validators*/ None, ); let remote_nodes: Vec<(Keypair, SocketAddr)> = repeat_with(|| new_rand_remote_node(&mut rng)) @@ -3567,7 +3608,12 @@ mod tests { let cluster_info = Arc::new({ let keypair = Arc::new(Keypair::new()); let node = Node::new_localhost_with_pubkey(&keypair.pubkey()); - ClusterInfo::new(node.info, keypair, SocketAddrSpace::Unspecified) + ClusterInfo::new( + node.info, + keypair, + SocketAddrSpace::Unspecified, + /*known_validators*/ None, + ) }); cluster_info.insert_info(spy); cluster_info.gossip.refresh_push_active_set( @@ -3597,7 +3643,12 @@ mod tests { fn test_cluster_info_new() { let keypair = Arc::new(Keypair::new()); let d = ContactInfo::new_localhost(&keypair.pubkey(), timestamp()); - let cluster_info = ClusterInfo::new(d.clone(), keypair, SocketAddrSpace::Unspecified); + let cluster_info = ClusterInfo::new( + d.clone(), + keypair, + SocketAddrSpace::Unspecified, + /*known_validators*/ None, + ); assert_eq!(d.pubkey(), &cluster_info.id()); } @@ -3605,7 +3656,12 @@ mod tests { fn insert_info_test() { let keypair = Arc::new(Keypair::new()); let d = ContactInfo::new_localhost(&keypair.pubkey(), timestamp()); - let cluster_info = ClusterInfo::new(d, keypair, SocketAddrSpace::Unspecified); + let cluster_info = ClusterInfo::new( + d, + keypair, + SocketAddrSpace::Unspecified, + /*known_validators*/ None, + ); let d = ContactInfo::new_localhost(&solana_sdk::pubkey::new_rand(), timestamp()); let label = CrdsValueLabel::LegacyContactInfo(*d.pubkey()); cluster_info.insert_info(d); @@ -3696,6 +3752,7 @@ mod tests { contact_info, Arc::new(keypair), SocketAddrSpace::Unspecified, + /*known_validators*/ None, ); let stakes = HashMap::::default(); cluster_info.ping_cache.lock().unwrap().mock_pong( @@ -3749,7 +3806,12 @@ mod tests { fn test_refresh_vote() { let keypair = Arc::new(Keypair::new()); let contact_info = ContactInfo::new_localhost(&keypair.pubkey(), 0); - let cluster_info = ClusterInfo::new(contact_info, keypair, SocketAddrSpace::Unspecified); + let cluster_info = ClusterInfo::new( + contact_info, + keypair, + SocketAddrSpace::Unspecified, + /*known_validators*/ None, + ); // Construct and push a vote for some other slot let unrefresh_slot = 5; @@ -3837,8 +3899,12 @@ mod tests { fn test_push_vote() { let keypair = Arc::new(Keypair::new()); let contact_info = ContactInfo::new_localhost(&keypair.pubkey(), 0); - let cluster_info = - ClusterInfo::new(contact_info, keypair.clone(), SocketAddrSpace::Unspecified); + let cluster_info = ClusterInfo::new( + contact_info, + keypair.clone(), + SocketAddrSpace::Unspecified, + /*known_validators*/ None, + ); // make sure empty crds is handled correctly let mut cursor = Cursor::default(); @@ -3906,7 +3972,12 @@ mod tests { }; let keypair = Arc::new(Keypair::new()); let contact_info = ContactInfo::new_localhost(&keypair.pubkey(), 0); - let cluster_info = ClusterInfo::new(contact_info, keypair, SocketAddrSpace::Unspecified); + let cluster_info = ClusterInfo::new( + contact_info, + keypair, + SocketAddrSpace::Unspecified, + /*known_validators*/ None, + ); let mut tower = Vec::new(); for k in 0..MAX_LOCKOUT_HISTORY { let slot = k as Slot; @@ -3955,7 +4026,12 @@ mod tests { fn test_push_epoch_slots() { let keypair = Arc::new(Keypair::new()); let contact_info = ContactInfo::new_localhost(&keypair.pubkey(), 0); - let cluster_info = ClusterInfo::new(contact_info, keypair, SocketAddrSpace::Unspecified); + let cluster_info = ClusterInfo::new( + contact_info, + keypair, + SocketAddrSpace::Unspecified, + /*known_validators*/ None, + ); let slots = cluster_info.get_epoch_slots(&mut Cursor::default()); assert!(slots.is_empty()); cluster_info.push_epoch_slots(&[0]); @@ -4012,6 +4088,7 @@ mod tests { ContactInfo::new_localhost(&node_keypair.pubkey(), timestamp()), node_keypair, SocketAddrSpace::Unspecified, + /*known_validators*/ None, ); let entrypoint_pubkey = solana_sdk::pubkey::new_rand(); let entrypoint = LegacyContactInfo::new_localhost(&entrypoint_pubkey, timestamp()); @@ -4167,7 +4244,12 @@ mod tests { fn test_tvu_peers_and_stakes() { let keypair = Arc::new(Keypair::new()); let d = ContactInfo::new_localhost(&keypair.pubkey(), timestamp()); - let cluster_info = ClusterInfo::new(d.clone(), keypair, SocketAddrSpace::Unspecified); + let cluster_info = ClusterInfo::new( + d.clone(), + keypair, + SocketAddrSpace::Unspecified, + /*known_validators*/ None, + ); let mut stakes = HashMap::new(); // no stake @@ -4209,6 +4291,7 @@ mod tests { ContactInfo::new_localhost(&node_keypair.pubkey(), timestamp()), node_keypair, SocketAddrSpace::Unspecified, + /*known_validators*/ None, ); let entrypoint_pubkey = solana_sdk::pubkey::new_rand(); let mut entrypoint = ContactInfo::new_localhost(&entrypoint_pubkey, timestamp()); @@ -4271,6 +4354,7 @@ mod tests { ContactInfo::new_localhost(&node_keypair.pubkey(), timestamp()), node_keypair, SocketAddrSpace::Unspecified, + /*known_validators*/ None, ); for i in 0..10 { // make these invalid for the upcoming repair request @@ -4342,6 +4426,7 @@ mod tests { ContactInfo::new_localhost(&node_keypair.pubkey(), timestamp()), node_keypair, SocketAddrSpace::Unspecified, + /*known_validators*/ None, ); //random should be hard to compress let mut rng = rand::thread_rng(); @@ -4394,6 +4479,7 @@ mod tests { ContactInfo::new_localhost(&node_keypair.pubkey(), timestamp()), node_keypair, SocketAddrSpace::Unspecified, + /*known_validators*/ None, )); assert_eq!(cluster_info.my_shred_version(), 0); @@ -4486,6 +4572,7 @@ mod tests { }, node_keypair, SocketAddrSpace::Unspecified, + /*known_validators*/ None, )); assert_eq!(cluster_info.my_shred_version(), 2); @@ -4525,6 +4612,7 @@ mod tests { node.info, Arc::new(Keypair::new()), SocketAddrSpace::Unspecified, + /*known_validators*/ None, )); let entrypoint_pubkey = solana_sdk::pubkey::new_rand(); let entrypoint = LegacyContactInfo::new_localhost(&entrypoint_pubkey, timestamp()); @@ -4575,6 +4663,7 @@ mod tests { node.info, host1_key.clone(), SocketAddrSpace::Unspecified, + /*known_validators*/ None, )); let mut cursor = Cursor::default(); assert!(cluster_info.get_duplicate_shreds(&mut cursor).is_empty()); diff --git a/gossip/src/duplicate_shred_listener.rs b/gossip/src/duplicate_shred_listener.rs index 3ac347fde6055c..e05e8151aa2f42 100644 --- a/gossip/src/duplicate_shred_listener.rs +++ b/gossip/src/duplicate_shred_listener.rs @@ -105,6 +105,7 @@ mod tests { node.info, host1_key, SocketAddrSpace::Unspecified, + /*known_validators*/ None, )); let exit = Arc::new(AtomicBool::new(false)); let count = Arc::new(AtomicU32::new(0)); diff --git a/gossip/src/gossip_service.rs b/gossip/src/gossip_service.rs index b587a5e0672421..b7f1eba9de45d7 100644 --- a/gossip/src/gossip_service.rs +++ b/gossip/src/gossip_service.rs @@ -311,7 +311,12 @@ pub fn make_gossip_node( } else { ClusterInfo::spy_node(keypair.pubkey(), shred_version) }; - let cluster_info = ClusterInfo::new(node, Arc::new(keypair), socket_addr_space); + let cluster_info = ClusterInfo::new( + node, + Arc::new(keypair), + socket_addr_space, + /*known_validators*/ None, + ); if let Some(entrypoint) = entrypoint { cluster_info.set_entrypoint(ContactInfo::new_gossip_entry_point(entrypoint)); } @@ -349,6 +354,7 @@ mod tests { tn.info.clone(), Arc::new(Keypair::new()), SocketAddrSpace::Unspecified, + /*known_validators*/ None, ); let c = Arc::new(cluster_info); let d = GossipService::new( @@ -377,6 +383,7 @@ mod tests { contact_info, Arc::new(keypair), SocketAddrSpace::Unspecified, + /*known_validators*/ None, ); cluster_info.insert_info(peer0_info.clone()); cluster_info.insert_info(peer1_info); diff --git a/gossip/tests/gossip.rs b/gossip/tests/gossip.rs index 9240c2b3ef9145..f3bb24e93ca13d 100644 --- a/gossip/tests/gossip.rs +++ b/gossip/tests/gossip.rs @@ -42,6 +42,7 @@ fn test_node(exit: Arc) -> (Arc, GossipService, UdpSock test_node.info.clone(), keypair, SocketAddrSpace::Unspecified, + /*known_validators*/ None, )); let gossip_service = GossipService::new( &cluster_info, @@ -70,6 +71,7 @@ fn test_node_with_bank( test_node.info.clone(), node_keypair, SocketAddrSpace::Unspecified, + /*known_validators*/ None, )); let gossip_service = GossipService::new( &cluster_info, diff --git a/ledger-tool/src/ledger_utils.rs b/ledger-tool/src/ledger_utils.rs index 6514312bc5d43d..c670de13cc8cae 100644 --- a/ledger-tool/src/ledger_utils.rs +++ b/ledger-tool/src/ledger_utils.rs @@ -262,6 +262,7 @@ pub fn load_and_process_ledger( ContactInfo::new_localhost(&node_id.pubkey(), timestamp()), Arc::clone(&node_id), SocketAddrSpace::Unspecified, + /*known_validators*/ None, )); let (accounts_package_sender, accounts_package_receiver) = crossbeam_channel::unbounded(); let accounts_hash_verifier = AccountsHashVerifier::new( diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index b4426aa3678501..46e55296ecc8e2 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -10,6 +10,7 @@ use { WriteBatch, }, blockstore_meta::*, + blockstore_metrics::ExcessiveRepairContext, blockstore_options::{ AccessType, BlockstoreOptions, LedgerColumnOptions, BLOCKSTORE_DIRECTORY_ROCKS_FIFO, BLOCKSTORE_DIRECTORY_ROCKS_LEVEL, @@ -842,6 +843,7 @@ impl Blockstore { retransmit_sender: Option<&Sender>>>, reed_solomon_cache: &ReedSolomonCache, metrics: &mut BlockstoreInsertionMetrics, + excessive_repair_context: &mut ExcessiveRepairContext, ) -> Result { assert_eq!(shreds.len(), is_repaired.len()); let mut total_start = Measure::start("Total elapsed"); @@ -882,6 +884,7 @@ impl Blockstore { &mut duplicate_shreds, leader_schedule, shred_source, + excessive_repair_context, ) { Err(InsertDataShredError::Exists) => { if is_repaired { @@ -918,6 +921,7 @@ impl Blockstore { is_trusted, shred_source, metrics, + excessive_repair_context, ); } }; @@ -965,6 +969,7 @@ impl Blockstore { &mut duplicate_shreds, leader_schedule, ShredSource::Recovered, + excessive_repair_context, ) { Err(InsertDataShredError::Exists) => { metrics.num_recovered_exists += 1; @@ -1052,6 +1057,7 @@ impl Blockstore { }) } + #[allow(clippy::too_many_arguments)] pub fn insert_shreds_handle_duplicate( &self, shreds: Vec, @@ -1062,6 +1068,7 @@ impl Blockstore { handle_duplicate: &F, reed_solomon_cache: &ReedSolomonCache, metrics: &mut BlockstoreInsertionMetrics, + excessive_repair_context: &mut ExcessiveRepairContext, ) -> Result> where F: Fn(PossibleDuplicateShred), @@ -1077,6 +1084,7 @@ impl Blockstore { retransmit_sender, reed_solomon_cache, metrics, + excessive_repair_context, )?; for shred in duplicate_shreds { @@ -1169,6 +1177,7 @@ impl Blockstore { None, // retransmit-sender &ReedSolomonCache::default(), &mut BlockstoreInsertionMetrics::default(), + &mut ExcessiveRepairContext::default(), )?; Ok(insert_results.completed_data_set_infos) } @@ -1186,6 +1195,7 @@ impl Blockstore { is_trusted: bool, shred_source: ShredSource, metrics: &mut BlockstoreInsertionMetrics, + excessive_repair_context: &mut ExcessiveRepairContext, ) -> bool { let slot = shred.slot(); let shred_index = u64::from(shred.index()); @@ -1263,8 +1273,13 @@ impl Blockstore { return false; } - self.slots_stats - .record_shred(shred.slot(), shred.fec_set_index(), shred_source, None); + self.slots_stats.record_shred( + shred.slot(), + shred.fec_set_index(), + shred_source, + None, + excessive_repair_context, + ); // insert coding shred into rocks let result = self @@ -1363,6 +1378,7 @@ impl Blockstore { duplicate_shreds: &mut Vec, leader_schedule: Option<&LeaderScheduleCache>, shred_source: ShredSource, + excessive_repair_context: &mut ExcessiveRepairContext, ) -> std::result::Result, InsertDataShredError> { let slot = shred.slot(); let shred_index = u64::from(shred.index()); @@ -1420,6 +1436,7 @@ impl Blockstore { &shred, write_batch, shred_source, + excessive_repair_context, )?; just_inserted_shreds.insert(shred.id(), shred); index_meta_working_set_entry.did_insert_occur = true; @@ -1611,6 +1628,7 @@ impl Blockstore { shred: &Shred, write_batch: &mut WriteBatch, shred_source: ShredSource, + excessive_repair_context: &mut ExcessiveRepairContext, ) -> Result> { let slot = shred.slot(); let index = u64::from(shred.index()); @@ -1669,6 +1687,7 @@ impl Blockstore { shred.fec_set_index(), shred_source, Some(slot_meta), + excessive_repair_context, ); // slot is full, send slot full timing to poh_timing_report service. @@ -6726,6 +6745,7 @@ pub mod tests { false, ShredSource::Turbine, &mut BlockstoreInsertionMetrics::default(), + &mut ExcessiveRepairContext::default(), )); // insert again fails on dupe @@ -6741,6 +6761,7 @@ pub mod tests { false, ShredSource::Turbine, &mut BlockstoreInsertionMetrics::default(), + &mut ExcessiveRepairContext::default(), )); assert_eq!( duplicate_shreds, diff --git a/ledger/src/blockstore_metrics.rs b/ledger/src/blockstore_metrics.rs index 46cceb55b36a7c..95ed7297e8a257 100644 --- a/ledger/src/blockstore_metrics.rs +++ b/ledger/src/blockstore_metrics.rs @@ -5,15 +5,80 @@ use { PerfContext, }, solana_metrics::datapoint_info, - solana_sdk::timing::timestamp, + solana_sdk::{slot_history::Slot, timing::timestamp}, std::{ cell::RefCell, + collections::VecDeque, fmt::Debug, sync::atomic::{AtomicU64, AtomicUsize, Ordering}, time::{Duration, Instant}, }, }; +// Max number of samples to maintain +const EXCESSIVE_REPAIR_SAMPLES_MAX: usize = 1_000; +// Percent of samples which will trigger an excessive repair warning +const EXCESSIVE_REPAIR_SAMPLES_THRESHOLD: f32 = 0.5; +// Slot distance to use for comparison with known validators +const EXCESSIVE_REPAIR_SLOT_DISTANCE: u64 = 150; +// Percent of shreds repaired to indicate excessively repaired slot +const EXCESSIVE_REPAIR_SLOT_THRESHOLD: f32 = 0.5; + +#[derive(Default)] +pub struct ExcessiveRepairContext { + // Record excessive repair status of slots as they are completed as 'full' + excessive_repairs: VecDeque, + caught_up: bool, +} + +impl ExcessiveRepairContext { + pub fn record_slot(&mut self, num_shreds: u64, num_repairs: usize) { + let slot_excessive_threshold = num_shreds as f32 * EXCESSIVE_REPAIR_SLOT_THRESHOLD; + let excessive_repairs = num_repairs as f32 > slot_excessive_threshold; + self.excessive_repairs.push_back(excessive_repairs); + if self.excessive_repairs.len() > EXCESSIVE_REPAIR_SAMPLES_MAX { + self.excessive_repairs.pop_front(); + } + } + + pub fn report_excessive_repairs( + &mut self, + get_cluster_max_slots: impl FnOnce() -> ( + /*my_max_slot*/ Option, + /*known_validators_max_slot*/ Option, + ), + ) { + // Only check when we have a full sample set + if self.excessive_repairs.len() < EXCESSIVE_REPAIR_SAMPLES_MAX { + return; + } + let excessive_repair_count = self.excessive_repairs.iter().filter(|x| **x).count(); + let threshold_count = + self.excessive_repairs.len() as f32 * EXCESSIVE_REPAIR_SAMPLES_THRESHOLD; + if excessive_repair_count as f32 > threshold_count { + // We're repairing many slots, check if we think we're caught up + if let (Some(my_max_slot), Some(known_validators_max_slot)) = get_cluster_max_slots() { + if my_max_slot + > known_validators_max_slot.saturating_sub(EXCESSIVE_REPAIR_SLOT_DISTANCE) + { + // Avoid warning the first time the validator catches up after repair + if !self.caught_up { + self.caught_up = true; + } else { + warn!( + "This node is caught up but is relying heavily on repair. {excessive_repair_count} of the \ + last {EXCESSIVE_REPAIR_SAMPLES_MAX} slots where mostly repaired." + ); + } + self.excessive_repairs.clear(); + } else { + self.caught_up = false; + } + } + } + } +} + #[derive(Default)] pub struct BlockstoreInsertionMetrics { pub insert_lock_elapsed_us: u64, diff --git a/ledger/src/slot_stats.rs b/ledger/src/slot_stats.rs index 9033c3d1600f89..015e6dacac1f57 100644 --- a/ledger/src/slot_stats.rs +++ b/ledger/src/slot_stats.rs @@ -1,5 +1,5 @@ use { - crate::blockstore_meta::SlotMeta, + crate::{blockstore_meta::SlotMeta, blockstore_metrics::ExcessiveRepairContext}, bitflags::bitflags, lru::LruCache, solana_sdk::clock::Slot, @@ -96,6 +96,7 @@ impl SlotsStats { fec_set_index: u32, source: ShredSource, slot_meta: Option<&SlotMeta>, + excessive_repair_context: &mut ExcessiveRepairContext, ) { let mut slot_full_reporting_info = None; let mut stats = self.stats.lock().unwrap(); @@ -118,6 +119,8 @@ impl SlotsStats { slot_full_reporting_info = Some((slot_stats.num_repaired, slot_stats.num_recovered)); } + excessive_repair_context + .record_slot(slot_stats.last_index, slot_stats.num_repaired); } } drop(stats); diff --git a/rpc/src/cluster_tpu_info.rs b/rpc/src/cluster_tpu_info.rs index 5a692944fa6faf..939a967413367f 100644 --- a/rpc/src/cluster_tpu_info.rs +++ b/rpc/src/cluster_tpu_info.rs @@ -165,6 +165,7 @@ mod test { ContactInfo::new_localhost(&node_keypair.pubkey(), timestamp()), node_keypair, SocketAddrSpace::Unspecified, + /*known_validators*/ None, )); let validator0_socket = ( diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index 997102b3e6564c..265bf19b805b9f 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -355,7 +355,12 @@ impl JsonRpcRequestProcessor { &keypair.pubkey(), solana_sdk::timing::timestamp(), // wallclock ); - ClusterInfo::new(contact_info, keypair, socket_addr_space) + ClusterInfo::new( + contact_info, + keypair, + socket_addr_space, + /*known_validators*/ None, + ) }); let tpu_address = cluster_info .my_contact_info() @@ -387,7 +392,6 @@ impl JsonRpcRequestProcessor { validator_exit: create_validator_exit(exit.clone()), health: Arc::new(RpcHealth::new( cluster_info.clone(), - None, 0, exit, Arc::clone(bank.get_startup_verification_complete()), @@ -4705,7 +4709,12 @@ pub mod tests { &keypair.pubkey(), solana_sdk::timing::timestamp(), // wallclock ); - ClusterInfo::new(contact_info, keypair, SocketAddrSpace::Unspecified) + ClusterInfo::new( + contact_info, + keypair, + SocketAddrSpace::Unspecified, + /*known_validators*/ None, + ) } fn create_test_request(method: &str, params: Option) -> serde_json::Value { @@ -6412,7 +6421,12 @@ pub mod tests { &keypair.pubkey(), &socketaddr!(Ipv4Addr::LOCALHOST, 1234), ); - ClusterInfo::new(contact_info, keypair, SocketAddrSpace::Unspecified) + ClusterInfo::new( + contact_info, + keypair, + SocketAddrSpace::Unspecified, + /*known_validators*/ None, + ) }); let connection_cache = Arc::new(ConnectionCache::new("connection_cache_test")); let tpu_address = cluster_info diff --git a/rpc/src/rpc_health.rs b/rpc/src/rpc_health.rs index 022b2e03d1eae2..37f893d79beeb4 100644 --- a/rpc/src/rpc_health.rs +++ b/rpc/src/rpc_health.rs @@ -1,12 +1,9 @@ use { solana_gossip::cluster_info::ClusterInfo, - solana_sdk::{clock::Slot, pubkey::Pubkey}, - std::{ - collections::HashSet, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, + solana_sdk::clock::Slot, + std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, }, }; @@ -19,7 +16,6 @@ pub enum RpcHealthStatus { pub struct RpcHealth { cluster_info: Arc, - known_validators: Option>, health_check_slot_distance: u64, override_health_check: Arc, startup_verification_complete: Arc, @@ -30,14 +26,12 @@ pub struct RpcHealth { impl RpcHealth { pub fn new( cluster_info: Arc, - known_validators: Option>, health_check_slot_distance: u64, override_health_check: Arc, startup_verification_complete: Arc, ) -> Self { Self { cluster_info, - known_validators, health_check_slot_distance, override_health_check, startup_verification_complete, @@ -60,29 +54,10 @@ impl RpcHealth { if self.override_health_check.load(Ordering::Relaxed) { RpcHealthStatus::Ok - } else if let Some(known_validators) = &self.known_validators { + } else if self.cluster_info.known_validators().is_some() { match ( - self.cluster_info - .get_accounts_hash_for_node(&self.cluster_info.id(), |hashes| { - hashes - .iter() - .max_by(|a, b| a.0.cmp(&b.0)) - .map(|slot_hash| slot_hash.0) - }) - .flatten(), - known_validators - .iter() - .filter_map(|known_validator| { - self.cluster_info - .get_accounts_hash_for_node(known_validator, |hashes| { - hashes - .iter() - .max_by(|a, b| a.0.cmp(&b.0)) - .map(|slot_hash| slot_hash.0) - }) - .flatten() - }) - .max(), + self.cluster_info.get_my_max_slot(), + self.cluster_info.get_known_validators_max_slot(), ) { ( Some(latest_account_hash_slot), @@ -131,7 +106,6 @@ impl RpcHealth { use crate::rpc::tests::new_test_cluster_info; Arc::new(Self::new( Arc::new(new_test_cluster_info()), - None, 42, Arc::new(AtomicBool::new(false)), Arc::new(AtomicBool::new(true)), diff --git a/rpc/src/rpc_service.rs b/rpc/src/rpc_service.rs index 4fdde57c31a9fe..64d265bf8f88f6 100644 --- a/rpc/src/rpc_service.rs +++ b/rpc/src/rpc_service.rs @@ -37,12 +37,11 @@ use { }, solana_sdk::{ exit::Exit, genesis_config::DEFAULT_GENESIS_DOWNLOAD_PATH, hash::Hash, - native_token::lamports_to_sol, pubkey::Pubkey, + native_token::lamports_to_sol, }, solana_send_transaction_service::send_transaction_service::{self, SendTransactionService}, solana_storage_bigtable::CredentialType, std::{ - collections::HashSet, net::SocketAddr, path::{Path, PathBuf}, sync::{ @@ -350,7 +349,6 @@ impl JsonRpcService { ledger_path: &Path, validator_exit: Arc>, exit: Arc, - known_validators: Option>, override_health_check: Arc, startup_verification_complete: Arc, optimistically_confirmed_bank: Arc>, @@ -369,7 +367,6 @@ impl JsonRpcService { let health = Arc::new(RpcHealth::new( cluster_info.clone(), - known_validators, config.health_check_slot_distance, override_health_check, startup_verification_complete, @@ -601,6 +598,7 @@ mod tests { signature::Signer, }, std::{ + collections::HashSet, io::Write, net::{IpAddr, Ipv4Addr}, }, @@ -643,7 +641,6 @@ mod tests { &PathBuf::from("farf"), validator_exit, exit, - None, Arc::new(AtomicBool::new(false)), Arc::new(AtomicBool::new(true)), optimistically_confirmed_bank, @@ -897,19 +894,21 @@ mod tests { #[test] fn test_health_check_with_known_validators() { - let cluster_info = Arc::new(new_test_cluster_info()); - let health_check_slot_distance = 123; - let override_health_check = Arc::new(AtomicBool::new(false)); - let startup_verification_complete = Arc::new(AtomicBool::new(true)); - let known_validators = vec![ + let known_validators = [ solana_sdk::pubkey::new_rand(), solana_sdk::pubkey::new_rand(), solana_sdk::pubkey::new_rand(), ]; + let mut cluster_info = new_test_cluster_info(); + cluster_info + .set_known_validators(Some(HashSet::from_iter(known_validators.iter().copied()))); + let cluster_info = Arc::new(cluster_info); + let health_check_slot_distance = 123; + let override_health_check = Arc::new(AtomicBool::new(false)); + let startup_verification_complete = Arc::new(AtomicBool::new(true)); let health = Arc::new(RpcHealth::new( cluster_info.clone(), - Some(known_validators.clone().into_iter().collect()), health_check_slot_distance, override_health_check.clone(), startup_verification_complete, diff --git a/turbine/benches/cluster_info.rs b/turbine/benches/cluster_info.rs index 954e32903c3e2f..1d21df80af3188 100644 --- a/turbine/benches/cluster_info.rs +++ b/turbine/benches/cluster_info.rs @@ -45,6 +45,7 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) { leader_info.info, leader_keypair, SocketAddrSpace::Unspecified, + /*known_validators*/ None, ); let socket = UdpSocket::bind("0.0.0.0:0").unwrap(); let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000); diff --git a/turbine/benches/retransmit_stage.rs b/turbine/benches/retransmit_stage.rs index b0dd67db8225ec..2a7054d230b06f 100644 --- a/turbine/benches/retransmit_stage.rs +++ b/turbine/benches/retransmit_stage.rs @@ -53,7 +53,12 @@ fn bench_retransmitter(bencher: &mut Bencher) { let cluster_info = { let keypair = Arc::new(Keypair::new()); let node = Node::new_localhost_with_pubkey(&keypair.pubkey()); - ClusterInfo::new(node.info, keypair, SocketAddrSpace::Unspecified) + ClusterInfo::new( + node.info, + keypair, + SocketAddrSpace::Unspecified, + /*known_validators*/ None, + ) }; const NUM_PEERS: usize = 4; let peer_sockets: Vec<_> = repeat_with(|| { diff --git a/turbine/src/broadcast_stage.rs b/turbine/src/broadcast_stage.rs index 07be0d0bfd6daa..b55b71d07b7b7b 100644 --- a/turbine/src/broadcast_stage.rs +++ b/turbine/src/broadcast_stage.rs @@ -660,6 +660,7 @@ pub mod test { leader_info.info.clone(), leader_keypair, SocketAddrSpace::Unspecified, + /*known_validators*/ None, ); cluster_info.insert_info(broadcast_buddy.info); let cluster_info = Arc::new(cluster_info); diff --git a/turbine/src/broadcast_stage/broadcast_fake_shreds_run.rs b/turbine/src/broadcast_stage/broadcast_fake_shreds_run.rs index 1464d46493d730..f4867a03b5f050 100644 --- a/turbine/src/broadcast_stage/broadcast_fake_shreds_run.rs +++ b/turbine/src/broadcast_stage/broadcast_fake_shreds_run.rs @@ -173,7 +173,12 @@ mod tests { let cluster = { let keypair = Arc::new(Keypair::new()); let contact_info = ContactInfo::new_localhost(&keypair.pubkey(), 0); - ClusterInfo::new(contact_info, keypair, SocketAddrSpace::Unspecified) + ClusterInfo::new( + contact_info, + keypair, + SocketAddrSpace::Unspecified, + /*known_validators*/ None, + ) }; for k in 1..5 { cluster.insert_info(ContactInfo::new_with_socketaddr( diff --git a/turbine/src/broadcast_stage/standard_broadcast_run.rs b/turbine/src/broadcast_stage/standard_broadcast_run.rs index 031e72012340e7..8fd5ec6ea5bff8 100644 --- a/turbine/src/broadcast_stage/standard_broadcast_run.rs +++ b/turbine/src/broadcast_stage/standard_broadcast_run.rs @@ -538,6 +538,7 @@ mod test { leader_info.info, leader_keypair.clone(), SocketAddrSpace::Unspecified, + /*known_validators*/ None, )); let socket = UdpSocket::bind("0.0.0.0:0").unwrap(); let mut genesis_config = create_genesis_config(10_000).genesis_config; diff --git a/turbine/src/cluster_nodes.rs b/turbine/src/cluster_nodes.rs index 57676a34b75eff..a97057cd9bd596 100644 --- a/turbine/src/cluster_nodes.rs +++ b/turbine/src/cluster_nodes.rs @@ -459,7 +459,12 @@ pub fn make_test_cluster( .collect(); // Add some staked nodes with no contact-info. stakes.extend(repeat_with(|| (Pubkey::new_unique(), rng.gen_range(0..20))).take(100)); - let cluster_info = ClusterInfo::new(this_node, keypair, SocketAddrSpace::Unspecified); + let cluster_info = ClusterInfo::new( + this_node, + keypair, + SocketAddrSpace::Unspecified, + /*known_validators*/ None, + ); let nodes: Vec<_> = nodes .iter() .map(LegacyContactInfo::try_from) diff --git a/validator/src/admin_rpc_service.rs b/validator/src/admin_rpc_service.rs index e10ab05ea01577..421eab3eb0c370 100644 --- a/validator/src/admin_rpc_service.rs +++ b/validator/src/admin_rpc_service.rs @@ -867,6 +867,7 @@ mod tests { ), keypair, SocketAddrSpace::Unspecified, + /*known_validators*/ None, )); let exit = Arc::new(AtomicBool::new(false)); let validator_exit = create_validator_exit(exit); diff --git a/validator/src/bootstrap.rs b/validator/src/bootstrap.rs index 88a45fdad50635..e4d4f1c4810160 100644 --- a/validator/src/bootstrap.rs +++ b/validator/src/bootstrap.rs @@ -155,7 +155,12 @@ fn start_gossip_node( *gossip_addr, expected_shred_version.unwrap_or(0), ); - let mut cluster_info = ClusterInfo::new(contact_info, identity_keypair, socket_addr_space); + let mut cluster_info = ClusterInfo::new( + contact_info, + identity_keypair, + socket_addr_space, + /*known_validators*/ None, + ); cluster_info.set_entrypoints(cluster_entrypoints.to_vec()); cluster_info.restore_contact_info(ledger_path, 0); let cluster_info = Arc::new(cluster_info);