diff --git a/Cargo.lock b/Cargo.lock index affd1b7b..5af13eb2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6112,6 +6112,7 @@ dependencies = [ "solana-metrics", "solana-program", "solana-sdk", + "solana-transaction-status", "spl-pod", "spl-stake-pool", "thiserror", diff --git a/docker-compose.yaml b/docker-compose.yaml index d6ae3273..9ff0e144 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -35,4 +35,34 @@ services: - GOSSIP_ENTRYPOINT=${GOSSIP_ENTRYPOINT} volumes: - ./credentials:/credentials - restart: on-failure:5 \ No newline at end of file + restart: on-failure:5 + + metrics-only: + build: + context: . + target: validator-history + container_name: metrics-only + environment: + - RUST_LOG=${RUST_LOG:-info} + - SOLANA_METRICS_CONFIG=${SOLANA_METRICS_CONFIG} + - JSON_RPC_URL=${JSON_RPC_URL} + - CLUSTER=${CLUSTER} + - KEYPAIR=${KEYPAIR} + - VALIDATOR_HISTORY_PROGRAM_ID=${VALIDATOR_HISTORY_PROGRAM_ID} + - TIP_DISTRIBUTION_PROGRAM_ID=${TIP_DISTRIBUTION_PROGRAM_ID} + - STEWARD_PROGRAM_ID=${STEWARD_PROGRAM_ID} + - STEWARD_CONFIG=${STEWARD_CONFIG} + - METRICS_INTERVAL=${METRICS_INTERVAL} + - STEWARD_INTERVAL=1000000000000 + - VALIDATOR_HISTORY_INTERVAL=1000000000000 + - RUN_CLUSTER_HISTORY=false + - RUN_COPY_VOTE_ACCOUNTS=false + - RUN_MEV_COMMISSION=false + - RUN_MEV_EARNED=false + - RUN_STEWARD=false + - RUN_STAKE_UPLOAD=false + - RUN_GOSSIP_UPLOAD=false + - RUN_EMIT_METRICS=true + volumes: + - ./credentials:/credentials + restart: on-failure:5 diff --git a/keepers/validator-keeper/src/entries/crank_steward.rs b/keepers/validator-keeper/src/entries/crank_steward.rs index 9c37cc18..d5f19ee5 100644 --- a/keepers/validator-keeper/src/entries/crank_steward.rs +++ b/keepers/validator-keeper/src/entries/crank_steward.rs @@ -9,6 +9,7 @@ use solana_program::instruction::Instruction; use solana_sdk::stake::instruction::deactivate_delinquent_stake; use solana_sdk::stake::state::StakeStateV2; +use solana_sdk::transaction::Transaction; use solana_sdk::vote::state::VoteState; use solana_sdk::{pubkey::Pubkey, signature::Keypair, stake, system_program}; use spl_stake_pool::instruction::{ @@ -122,31 +123,33 @@ pub fn _get_update_stake_pool_ixs( .get(&validator_info.vote_account_address) .expect("Stake account not found"); - let should_deactivate = if raw_vote_account.is_none() || raw_stake_account.is_none() { - true - } else { - let stake_account = - StakeStateV2::deserialize(&mut raw_stake_account.clone().unwrap().data.as_slice()) - .expect("Could not deserialize stake account"); + let should_deactivate = match (raw_vote_account, raw_stake_account) { + (None, Some(_)) => true, + (Some(raw_vote_account), Some(raw_stake_account)) => { + let stake_account = + StakeStateV2::deserialize(&mut raw_stake_account.data.as_slice()) + .expect("Could not deserialize stake account"); - let vote_account = VoteState::deserialize(&raw_vote_account.clone().unwrap().data) - .expect("Could not deserialize vote account"); + let vote_account = VoteState::deserialize(&raw_vote_account.data) + .expect("Could not deserialize vote account"); - let latest_epoch = vote_account.epoch_credits.iter().last().unwrap().0; + let latest_epoch = vote_account.epoch_credits.iter().last().unwrap().0; - match stake_account { - StakeStateV2::Stake(_meta, stake, _stake_flags) => { - if stake.delegation.deactivation_epoch != std::u64::MAX { + match stake_account { + StakeStateV2::Stake(_meta, stake, _stake_flags) => { + if stake.delegation.deactivation_epoch != std::u64::MAX { + false + } else { + latest_epoch <= epoch - 5 + } + } + _ => { + println!("🔶 Error: Stake account is not StakeStateV2::Stake"); false - } else { - latest_epoch <= epoch - 5 } } - _ => { - println!("🔶 Error: Stake account is not StakeStateV2::Stake"); - false - } } + (_, None) => false, }; if should_deactivate { @@ -217,6 +220,13 @@ async fn _update_pool( // TODO fix println!("Deactivating Delinquent"); + // for ix in deactivate_delinquent_ixs { + // let tx = Transaction::new_signed_with_payer(&[ix], Some(&payer.pubkey()), &[&payer]); + // let tx = client + // .send_and_confirm_transaction_with_spinner_and_config(&tx) + // .await?; + // stats.add_tx(&tx); + // } let deactivate_txs_to_run = package_instructions( &deactivate_delinquent_ixs, 1, @@ -493,11 +503,7 @@ async fn _handle_delinquent_validators( let bad_vote_accounts = checks .iter() .filter_map(|(vote_account, check)| { - if !check.has_history - || !check.has_stake_account - || check.is_deactivated - || !check.has_vote_account - { + if !check.has_history || check.is_deactivated || !check.has_vote_account { Some(*vote_account) } else { None diff --git a/keepers/validator-keeper/src/main.rs b/keepers/validator-keeper/src/main.rs index 84b2db11..5bb4a2b8 100644 --- a/keepers/validator-keeper/src/main.rs +++ b/keepers/validator-keeper/src/main.rs @@ -249,7 +249,6 @@ async fn run_keeper(keeper_config: KeeperConfig) { // ---------------------- EMIT --------------------------------- if should_fire(tick, metrics_interval) { - info!("Emitting metrics..."); keeper_state.set_runs_errors_and_txs_for_epoch(operations::metrics_emit::fire( &keeper_config, &keeper_state, @@ -257,6 +256,7 @@ async fn run_keeper(keeper_config: KeeperConfig) { } if should_emit(tick, &intervals) { + info!("Emitting metrics..."); keeper_state.emit(); KeeperOperations::emit( @@ -332,6 +332,8 @@ async fn main() { no_pack: args.no_pack, pay_for_new_accounts: args.pay_for_new_accounts, cool_down_range: args.cool_down_range, + tx_retry_count: args.tx_retry_count, + tx_confirmation_seconds: args.tx_confirmation_seconds, }; run_keeper(config).await; diff --git a/keepers/validator-keeper/src/operations/cluster_history.rs b/keepers/validator-keeper/src/operations/cluster_history.rs index dc4dfe21..e7dde1b6 100644 --- a/keepers/validator-keeper/src/operations/cluster_history.rs +++ b/keepers/validator-keeper/src/operations/cluster_history.rs @@ -40,8 +40,18 @@ async fn _process( keypair: &Arc, program_id: &Pubkey, priority_fee_in_microlamports: u64, + retry_count: u16, + confirmation_time: u64, ) -> Result { - update_cluster_info(client, keypair, program_id, priority_fee_in_microlamports).await + update_cluster_info( + client, + keypair, + program_id, + priority_fee_in_microlamports, + retry_count, + confirmation_time, + ) + .await } pub async fn fire( @@ -52,6 +62,8 @@ pub async fn fire( let keypair = &keeper_config.keypair; let program_id = &keeper_config.validator_history_program_id; let priority_fee_in_microlamports = keeper_config.priority_fee_in_microlamports; + let retry_count = keeper_config.tx_retry_count; + let confirmation_time = keeper_config.tx_confirmation_seconds; let operation = _get_operation(); let epoch_info = &keeper_state.epoch_info; @@ -63,7 +75,16 @@ pub async fn fire( _should_run(epoch_info, runs_for_epoch) && check_flag(keeper_config.run_flags, operation); if should_run { - match _process(client, keypair, program_id, priority_fee_in_microlamports).await { + match _process( + client, + keypair, + program_id, + priority_fee_in_microlamports, + retry_count, + confirmation_time, + ) + .await + { Ok(stats) => { for message in stats.results.iter() { if let Err(e) = message { @@ -126,6 +147,8 @@ pub async fn update_cluster_info( keypair: &Arc, program_id: &Pubkey, priority_fee_in_microlamports: u64, + retry_count: u16, + confirmation_time: u64, ) -> Result { let ixs = get_update_cluster_info_instructions( program_id, @@ -133,5 +156,5 @@ pub async fn update_cluster_info( priority_fee_in_microlamports, ); - submit_transactions(client, vec![ixs], keypair).await + submit_transactions(client, vec![ixs], keypair, retry_count, confirmation_time).await } diff --git a/keepers/validator-keeper/src/operations/gossip_upload.rs b/keepers/validator-keeper/src/operations/gossip_upload.rs index c8466d06..70fbc2eb 100644 --- a/keepers/validator-keeper/src/operations/gossip_upload.rs +++ b/keepers/validator-keeper/src/operations/gossip_upload.rs @@ -44,7 +44,6 @@ fn _should_run(epoch_info: &EpochInfo, runs_for_epoch: u64) -> bool { || (epoch_info.slot_index > epoch_info.slots_in_epoch / 2 && runs_for_epoch < 2) || (epoch_info.slot_index > epoch_info.slots_in_epoch * 9 / 10 && runs_for_epoch < 3) } - async fn _process( client: &Arc, keypair: &Arc, @@ -52,6 +51,8 @@ async fn _process( priority_fee_in_microlamports: u64, entrypoint: &SocketAddr, keeper_state: &KeeperState, + retry_count: u16, + confirmation_time: u64, ) -> Result> { upload_gossip_values( client, @@ -60,6 +61,8 @@ async fn _process( priority_fee_in_microlamports, entrypoint, keeper_state, + retry_count, + confirmation_time, ) .await } @@ -76,6 +79,8 @@ pub async fn fire( .expect("Entry point not set"); let priority_fee_in_microlamports = keeper_config.priority_fee_in_microlamports; + let retry_count = keeper_config.tx_retry_count; + let confirmation_time = keeper_config.tx_confirmation_seconds; let operation = _get_operation(); let (mut runs_for_epoch, mut errors_for_epoch, mut txs_for_epoch) = @@ -92,6 +97,8 @@ pub async fn fire( priority_fee_in_microlamports, entrypoint, keeper_state, + retry_count, + confirmation_time, ) .await { @@ -255,6 +262,8 @@ pub async fn upload_gossip_values( priority_fee_in_microlamports: u64, entrypoint: &SocketAddr, keeper_state: &KeeperState, + retry_count: u16, + confirmation_time: u64, ) -> Result> { let vote_accounts = keeper_state.vote_account_map.values().collect::>(); let validator_history_map = &keeper_state.validator_history_map; @@ -314,7 +323,14 @@ pub async fn upload_gossip_values( .map(|entry| entry.build_update_tx(priority_fee_in_microlamports)) .collect::>(); - let submit_result = submit_transactions(client, update_transactions, keypair).await; + let submit_result = submit_transactions( + client, + update_transactions, + keypair, + retry_count, + confirmation_time, + ) + .await; submit_result.map_err(|e| e.into()) } diff --git a/keepers/validator-keeper/src/operations/metrics_emit.rs b/keepers/validator-keeper/src/operations/metrics_emit.rs index da09fbc1..ef162e4d 100644 --- a/keepers/validator-keeper/src/operations/metrics_emit.rs +++ b/keepers/validator-keeper/src/operations/metrics_emit.rs @@ -115,8 +115,8 @@ pub fn emit_validator_history_metrics( let mut cluster_history_blocks: i64 = 0; let cluster_history_entry = cluster_history.history.last(); if let Some(cluster_history) = cluster_history_entry { - // Looking for previous epoch to be updated - if cluster_history.epoch as u64 == epoch_info.epoch - 1 { + // Looking for current epoch to be updated, implies previous is complete as well + if cluster_history.epoch as u64 == epoch_info.epoch { cluster_history_blocks = 1; } } diff --git a/keepers/validator-keeper/src/operations/mev_commission.rs b/keepers/validator-keeper/src/operations/mev_commission.rs index 6af16494..7319d268 100644 --- a/keepers/validator-keeper/src/operations/mev_commission.rs +++ b/keepers/validator-keeper/src/operations/mev_commission.rs @@ -38,6 +38,8 @@ async fn _process( program_id: &Pubkey, tip_distribution_program_id: &Pubkey, keeper_state: &KeeperState, + retry_count: u16, + confirmation_time: u64, priority_fee_in_microlamports: u64, no_pack: bool, ) -> Result { @@ -47,6 +49,8 @@ async fn _process( program_id, tip_distribution_program_id, keeper_state, + retry_count, + confirmation_time, priority_fee_in_microlamports, no_pack, ) @@ -76,6 +80,8 @@ pub async fn fire( program_id, tip_distribution_program_id, keeper_state, + keeper_config.tx_retry_count, + keeper_config.tx_confirmation_seconds, priority_fee_in_microlamports, keeper_config.no_pack, ) @@ -112,6 +118,8 @@ pub async fn update_mev_commission( program_id: &Pubkey, tip_distribution_program_id: &Pubkey, keeper_state: &KeeperState, + retry_count: u16, + confirmation_time: u64, priority_fee_in_microlamports: u64, no_pack: bool, ) -> Result { @@ -148,6 +156,8 @@ pub async fn update_mev_commission( update_instructions, keypair, priority_fee_in_microlamports, + retry_count, + confirmation_time, None, no_pack, ) diff --git a/keepers/validator-keeper/src/operations/mev_earned.rs b/keepers/validator-keeper/src/operations/mev_earned.rs index 0773c23e..5e200c11 100644 --- a/keepers/validator-keeper/src/operations/mev_earned.rs +++ b/keepers/validator-keeper/src/operations/mev_earned.rs @@ -40,6 +40,8 @@ async fn _process( program_id: &Pubkey, tip_distribution_program_id: &Pubkey, priority_fee_in_microlamports: u64, + retry_count: u16, + confirmation_time: u64, keeper_state: &KeeperState, no_pack: bool, ) -> Result { @@ -48,6 +50,8 @@ async fn _process( keypair, program_id, priority_fee_in_microlamports, + retry_count, + confirmation_time, tip_distribution_program_id, keeper_state, no_pack, @@ -78,6 +82,8 @@ pub async fn fire( program_id, tip_distribution_program_id, priority_fee_in_microlamports, + keeper_config.tx_retry_count, + keeper_config.tx_confirmation_seconds, keeper_state, keeper_config.no_pack, ) @@ -113,6 +119,8 @@ pub async fn update_mev_earned( keypair: &Arc, program_id: &Pubkey, priority_fee_in_microlamports: u64, + retry_count: u16, + confirmation_time: u64, tip_distribution_program_id: &Pubkey, keeper_state: &KeeperState, no_pack: bool, @@ -165,6 +173,8 @@ pub async fn update_mev_earned( update_instructions, keypair, priority_fee_in_microlamports, + retry_count, + confirmation_time, None, no_pack, ) diff --git a/keepers/validator-keeper/src/operations/stake_upload.rs b/keepers/validator-keeper/src/operations/stake_upload.rs index a4a2bf40..093af351 100644 --- a/keepers/validator-keeper/src/operations/stake_upload.rs +++ b/keepers/validator-keeper/src/operations/stake_upload.rs @@ -39,6 +39,8 @@ async fn _process( keypair: &Arc, program_id: &Pubkey, priority_fee_in_microlamports: u64, + retry_count: u16, + confirmation_time: u64, keeper_state: &KeeperState, no_pack: bool, ) -> Result { @@ -47,6 +49,8 @@ async fn _process( keypair, program_id, priority_fee_in_microlamports, + retry_count, + confirmation_time, keeper_state, no_pack, ) @@ -61,6 +65,8 @@ pub async fn fire( let keypair = &keeper_config.keypair; let program_id = &keeper_config.validator_history_program_id; let priority_fee_in_microlamports = keeper_config.priority_fee_in_microlamports; + let retry_count = keeper_config.tx_retry_count; + let confirmation_time = keeper_config.tx_confirmation_seconds; let operation = _get_operation(); let (mut runs_for_epoch, mut errors_for_epoch, mut txs_for_epoch) = @@ -75,6 +81,8 @@ pub async fn fire( keypair, program_id, priority_fee_in_microlamports, + retry_count, + confirmation_time, keeper_state, keeper_config.no_pack, ) @@ -110,6 +118,8 @@ pub async fn update_stake_history( keypair: &Arc, program_id: &Pubkey, priority_fee_in_microlamports: u64, + retry_count: u16, + confirmation_time: u64, keeper_state: &KeeperState, no_pack: bool, ) -> Result { @@ -164,6 +174,8 @@ pub async fn update_stake_history( update_instructions, keypair, priority_fee_in_microlamports, + retry_count, + confirmation_time, None, no_pack, ) diff --git a/keepers/validator-keeper/src/operations/vote_account.rs b/keepers/validator-keeper/src/operations/vote_account.rs index 2ea4fe75..0fffe47d 100644 --- a/keepers/validator-keeper/src/operations/vote_account.rs +++ b/keepers/validator-keeper/src/operations/vote_account.rs @@ -41,6 +41,8 @@ async fn _process( keypair: &Arc, program_id: &Pubkey, priority_fee_in_microlamports: u64, + retry_count: u16, + confirmation_time: u64, keeper_state: &KeeperState, no_pack: bool, ) -> Result { @@ -49,6 +51,8 @@ async fn _process( keypair, program_id, priority_fee_in_microlamports, + retry_count, + confirmation_time, keeper_state, no_pack, ) @@ -63,6 +67,8 @@ pub async fn fire( let keypair = &keeper_config.keypair; let program_id = &keeper_config.validator_history_program_id; let priority_fee_in_microlamports = keeper_config.priority_fee_in_microlamports; + let retry_count = keeper_config.tx_retry_count; + let confirmation_time = keeper_config.tx_confirmation_seconds; let operation = _get_operation(); let epoch_info = &keeper_state.epoch_info; @@ -82,6 +88,8 @@ pub async fn fire( keypair, program_id, priority_fee_in_microlamports, + retry_count, + confirmation_time, keeper_state, keeper_config.no_pack, ) @@ -121,6 +129,8 @@ pub async fn update_vote_accounts( keypair: &Arc, program_id: &Pubkey, priority_fee_in_microlamports: u64, + retry_count: u16, + confirmation_time: u64, keeper_state: &KeeperState, no_pack: bool, ) -> Result { @@ -157,6 +167,8 @@ pub async fn update_vote_accounts( update_instructions, keypair, priority_fee_in_microlamports, + retry_count, + confirmation_time, Some(300_000), no_pack, ) diff --git a/keepers/validator-keeper/src/state/keeper_config.rs b/keepers/validator-keeper/src/state/keeper_config.rs index 8f41f367..022ef25e 100644 --- a/keepers/validator-keeper/src/state/keeper_config.rs +++ b/keepers/validator-keeper/src/state/keeper_config.rs @@ -14,6 +14,8 @@ pub struct KeeperConfig { pub steward_program_id: Pubkey, pub steward_config: Pubkey, pub priority_fee_in_microlamports: u64, + pub tx_retry_count: u16, + pub tx_confirmation_seconds: u64, pub oracle_authority_keypair: Option>, pub gossip_entrypoint: Option, pub validator_history_interval: u64, @@ -94,6 +96,12 @@ pub struct Args { #[arg(long, env, default_value = "20000")] pub priority_fees: u64, + #[arg(long, env, default_value = "50")] + pub tx_retry_count: u16, + + #[arg(long, env, default_value = "30")] + pub tx_confirmation_seconds: u64, + /// Cluster to specify #[arg(long, env, default_value_t = Cluster::Mainnet)] pub cluster: Cluster, @@ -167,6 +175,8 @@ impl fmt::Display for Args { Steward Interval: {} seconds\n\ Metrics Interval: {} seconds\n\ Priority Fees: {} microlamports\n\ + Retry Count: {}\n\ + Confirmation Seconds: {}\n\ Cluster: {:?}\n\ Run Cluster History: {}\n\ Run Copy Vote Accounts: {}\n\ @@ -193,6 +203,8 @@ impl fmt::Display for Args { self.steward_interval, self.metrics_interval, self.priority_fees, + self.tx_retry_count, + self.tx_confirmation_seconds, self.cluster, self.run_cluster_history, self.run_copy_vote_accounts, diff --git a/keepers/validator-keeper/src/state/update_state.rs b/keepers/validator-keeper/src/state/update_state.rs index 1a8fe9d0..bcd22cff 100644 --- a/keepers/validator-keeper/src/state/update_state.rs +++ b/keepers/validator-keeper/src/state/update_state.rs @@ -81,9 +81,15 @@ pub async fn create_missing_accounts( let mut created_accounts_for_epoch = vec![]; // Create Missing Accounts - let new_validator_history_accounts = - create_missing_validator_history_accounts(client, keypair, program_id, keeper_state) - .await?; + let new_validator_history_accounts = create_missing_validator_history_accounts( + client, + keypair, + program_id, + keeper_state, + keeper_config.tx_retry_count, + keeper_config.tx_confirmation_seconds, + ) + .await?; created_accounts_for_epoch.push(( KeeperCreates::CreateValidatorHistory, new_validator_history_accounts, @@ -286,6 +292,8 @@ async fn create_missing_validator_history_accounts( keypair: &Arc, program_id: &Pubkey, keeper_state: &KeeperState, + retry_count: u16, + confirmation_time: u64, ) -> Result> { let vote_accounts = &keeper_state .vote_account_map @@ -319,7 +327,14 @@ async fn create_missing_validator_history_accounts( let accounts_created = create_transactions.len(); - submit_transactions(client, create_transactions, keypair).await?; + submit_transactions( + client, + create_transactions, + keypair, + retry_count, + confirmation_time, + ) + .await?; Ok(accounts_created) } diff --git a/sdk/Cargo.toml b/sdk/Cargo.toml index ca19901f..975197fd 100644 --- a/sdk/Cargo.toml +++ b/sdk/Cargo.toml @@ -23,6 +23,7 @@ solana-client = "1.18" solana-metrics = "1.18" solana-program = "1.18" solana-sdk = "1.18" +solana-transaction-status = "1.18" spl-pod = "0.1.0" spl-stake-pool = { features = ["no-entrypoint"], version = "1.0.0" } thiserror = "1.0.37" diff --git a/sdk/src/utils/transactions.rs b/sdk/src/utils/transactions.rs index fccd3a8c..13d1167a 100644 --- a/sdk/src/utils/transactions.rs +++ b/sdk/src/utils/transactions.rs @@ -4,7 +4,9 @@ use std::vec; use std::{collections::HashMap, sync::Arc, time::Duration}; use log::*; -use solana_client::rpc_response::{Response, RpcSimulateTransactionResult, RpcVoteAccountInfo}; +use solana_client::rpc_response::{ + Response, RpcResult, RpcSimulateTransactionResult, RpcVoteAccountInfo, +}; use solana_client::{client_error::ClientError, nonblocking::rpc_client::RpcClient}; use solana_metrics::datapoint_error; use solana_program::hash::Hash; @@ -17,6 +19,7 @@ use solana_sdk::{ instruction::Instruction, packet::Packet, pubkey::Pubkey, signature::Keypair, signature::Signature, signer::Signer, transaction::Transaction, }; +use solana_transaction_status::TransactionStatus; use tokio::task; use tokio::time::sleep; @@ -197,6 +200,19 @@ pub async fn get_vote_accounts_with_retry( } } +pub async fn get_signature_statuses_with_retry( + client: &RpcClient, + signatures: &[Signature], +) -> RpcResult>> { + for _ in 1..4 { + if let Ok(result) = client.get_signature_statuses(signatures).await { + return Ok(result); + } + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + } + client.get_signature_statuses(signatures).await +} + async fn find_ix_per_tx( client: &Arc, instruction: &Instruction, @@ -264,14 +280,17 @@ async fn parallel_confirm_transactions( let confirmation_futures: Vec<_> = signatures_to_confirm .chunks(SIG_STATUS_BATCH_SIZE) .map(|sig_batch| async move { - match client.get_signature_statuses(sig_batch).await { + match get_signature_statuses_with_retry(client, sig_batch).await { Ok(sig_batch_response) => sig_batch_response .value .iter() .enumerate() .map(|(i, sig_status)| (sig_batch[i], sig_status.clone())) .collect::>(), - Err(_) => vec![], + Err(e) => { + info!("Failed getting signature statuses: {}", e); + vec![] + } } }) .collect(); @@ -643,6 +662,8 @@ pub async fn submit_transactions( client: &Arc, transactions: Vec>, keypair: &Arc, + retry_count: u16, + confirmation_time: u64, ) -> Result { let mut stats = SubmitStats::default(); let tx_slice = transactions @@ -650,7 +671,9 @@ pub async fn submit_transactions( .map(|t| t.as_slice()) .collect::>(); - match parallel_execute_transactions(client, &tx_slice, keypair, 100, 20).await { + match parallel_execute_transactions(client, &tx_slice, keypair, retry_count, confirmation_time) + .await + { Ok(results) => { stats.successes = results.iter().filter(|&tx| tx.is_ok()).count() as u64; stats.errors = results.len() as u64 - stats.successes; @@ -666,6 +689,8 @@ pub async fn submit_instructions( instructions: Vec, keypair: &Arc, priority_fee_in_microlamports: u64, + retry_count: u16, + confirmation_time: u64, max_cu_per_tx: Option, no_pack: bool, ) -> Result { @@ -674,8 +699,8 @@ pub async fn submit_instructions( client, &instructions, keypair, - 100, - 20, + retry_count, + confirmation_time, priority_fee_in_microlamports, max_cu_per_tx, no_pack, diff --git a/utils/steward-cli/src/commands/actions/auto_remove_validator_from_pool.rs b/utils/steward-cli/src/commands/actions/auto_remove_validator_from_pool.rs index 6a30bd9b..a0900d42 100644 --- a/utils/steward-cli/src/commands/actions/auto_remove_validator_from_pool.rs +++ b/utils/steward-cli/src/commands/actions/auto_remove_validator_from_pool.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use anchor_lang::{InstructionData, ToAccountMetas}; use anyhow::Result; -use solana_client::nonblocking::rpc_client::RpcClient; +use solana_client::{nonblocking::rpc_client::RpcClient, rpc_config::RpcSendTransactionConfig}; use solana_program::instruction::Instruction; use spl_stake_pool::{find_stake_program_address, find_transient_stake_program_address}; use stakenet_sdk::utils::{ @@ -13,8 +13,8 @@ use stakenet_sdk::utils::{ use validator_history::id as validator_history_id; use solana_sdk::{ - pubkey::Pubkey, signature::read_keypair_file, signer::Signer, stake, system_program, - transaction::Transaction, + commitment_config::CommitmentConfig, pubkey::Pubkey, signature::read_keypair_file, + signer::Signer, stake, system_program, transaction::Transaction, }; use crate::commands::command_args::AutoRemoveValidatorFromPool; @@ -109,7 +109,14 @@ pub async fn command_auto_remove_validator_from_pool( print_base58_tx(&configured_ix) } else { let signature = client - .send_and_confirm_transaction_with_spinner(&transaction) + .send_and_confirm_transaction_with_spinner_and_config( + &transaction, + CommitmentConfig::default(), + RpcSendTransactionConfig { + skip_preflight: true, + ..Default::default() + }, + ) .await?; println!("Signature: {}", signature); diff --git a/utils/steward-cli/src/commands/actions/instant_remove_validator.rs b/utils/steward-cli/src/commands/actions/instant_remove_validator.rs new file mode 100644 index 00000000..8c82e53b --- /dev/null +++ b/utils/steward-cli/src/commands/actions/instant_remove_validator.rs @@ -0,0 +1,87 @@ +use std::sync::Arc; + +use anchor_lang::{InstructionData, ToAccountMetas}; +use anyhow::Result; +use solana_client::{nonblocking::rpc_client::RpcClient, rpc_config::RpcSendTransactionConfig}; +use solana_program::instruction::Instruction; + +use solana_sdk::{ + commitment_config::CommitmentConfig, pubkey::Pubkey, signature::read_keypair_file, + signer::Signer, transaction::Transaction, +}; + +use crate::commands::command_args::InstantRemoveValidator; +use stakenet_sdk::utils::{ + accounts::get_all_steward_accounts, + transactions::{configure_instruction, print_base58_tx}, +}; + +pub async fn command_instant_remove_validator( + args: InstantRemoveValidator, + client: &Arc, + program_id: Pubkey, +) -> Result<()> { + let payer = read_keypair_file(args.permissionless_parameters.payer_keypair_path) + .expect("Failed reading keypair file (Payer)"); + + let steward_config = args.permissionless_parameters.steward_config; + + let steward_accounts = get_all_steward_accounts(client, &program_id, &steward_config).await?; + + let ix = Instruction { + program_id, + accounts: jito_steward::accounts::InstantRemoveValidator { + config: args.permissionless_parameters.steward_config, + state_account: steward_accounts.state_address, + validator_list: steward_accounts.validator_list_address, + stake_pool: steward_accounts.stake_pool_address, + } + .to_account_metas(None), + data: jito_steward::instruction::InstantRemoveValidator { + validator_index_to_remove: args.validator_index_to_remove, + } + .data(), + }; + + let blockhash = client.get_latest_blockhash().await?; + + let configured_ix = configure_instruction( + &[ix], + args.permissionless_parameters + .transaction_parameters + .priority_fee, + args.permissionless_parameters + .transaction_parameters + .compute_limit, + args.permissionless_parameters + .transaction_parameters + .heap_size, + ); + + let transaction = Transaction::new_signed_with_payer( + &configured_ix, + Some(&payer.pubkey()), + &[&payer], + blockhash, + ); + + if args + .permissionless_parameters + .transaction_parameters + .print_tx + { + print_base58_tx(&configured_ix) + } else { + let signature = client + .send_and_confirm_transaction_with_spinner_and_config( + &transaction, + CommitmentConfig::default(), + RpcSendTransactionConfig::default(), + ) + .await?; + + println!("Signature: {}", signature); + } + + Ok(()) +} diff --git a/utils/steward-cli/src/commands/actions/mod.rs b/utils/steward-cli/src/commands/actions/mod.rs index f95d730f..eebaeee1 100644 --- a/utils/steward-cli/src/commands/actions/mod.rs +++ b/utils/steward-cli/src/commands/actions/mod.rs @@ -2,6 +2,7 @@ pub mod add_to_blacklist; pub mod auto_add_validator_from_pool; pub mod auto_remove_validator_from_pool; pub mod close_steward; +pub mod instant_remove_validator; pub mod manually_copy_all_vote_accounts; pub mod manually_copy_vote_accounts; pub mod manually_remove_validator; @@ -15,3 +16,4 @@ pub mod revert_staker; pub mod set_staker; pub mod update_authority; pub mod update_config; +pub mod update_validator_list_balance; diff --git a/utils/steward-cli/src/commands/actions/remove_bad_validators.rs b/utils/steward-cli/src/commands/actions/remove_bad_validators.rs index fb60aec1..6a8fc88c 100644 --- a/utils/steward-cli/src/commands/actions/remove_bad_validators.rs +++ b/utils/steward-cli/src/commands/actions/remove_bad_validators.rs @@ -153,7 +153,7 @@ pub async fn command_remove_bad_validators( } else { println!("Submitting {} instructions", ixs_to_run.len()); - let submit_stats = submit_transactions(client, txs_to_run, &arc_payer).await?; + let submit_stats = submit_transactions(client, txs_to_run, &arc_payer, 20, 30).await?; println!("Submit stats: {:?}", submit_stats); } diff --git a/utils/steward-cli/src/commands/actions/update_validator_list_balance.rs b/utils/steward-cli/src/commands/actions/update_validator_list_balance.rs new file mode 100644 index 00000000..cde6f77a --- /dev/null +++ b/utils/steward-cli/src/commands/actions/update_validator_list_balance.rs @@ -0,0 +1,65 @@ +use std::sync::Arc; + +use solana_client::{nonblocking::rpc_client::RpcClient, rpc_config::RpcSendTransactionConfig}; +use solana_sdk::{ + commitment_config::CommitmentConfig, pubkey::Pubkey, signature::read_keypair_file, + signer::Signer, transaction::Transaction, +}; +use spl_stake_pool::instruction::update_validator_list_balance; +use stakenet_sdk::utils::accounts::get_all_steward_accounts; + +use crate::commands::command_args::UpdateValidatorListBalance; + +pub async fn command_update_validator_list_balance( + client: &Arc, + args: UpdateValidatorListBalance, + program_id: Pubkey, +) -> Result<(), anyhow::Error> { + let steward_config = args.permissionless_parameters.steward_config; + let payer = Arc::new( + read_keypair_file(args.permissionless_parameters.payer_keypair_path) + .expect("Failed reading keypair file ( Payer )"), + ); + + let all_steward_accounts = + get_all_steward_accounts(client, &program_id, &steward_config).await?; + + let stake_pool = all_steward_accounts.stake_pool_address; + let validator_list = all_steward_accounts.validator_list_address; + + let target_vote_account = all_steward_accounts.validator_list_account.validators + [args.validator_list_index as usize] + .vote_account_address; + + let instruction = update_validator_list_balance( + &spl_stake_pool::id(), + &stake_pool, + &all_steward_accounts.stake_pool_withdraw_authority, + &validator_list, + &all_steward_accounts.reserve_stake_address, + &all_steward_accounts.validator_list_account, + &[target_vote_account], + args.validator_list_index, + false, + ); + + let recent_blockhash = client.get_latest_blockhash().await?; + let transaction = Transaction::new_signed_with_payer( + &[instruction], + Some(&payer.pubkey()), + &[&*payer], + recent_blockhash, + ); + + let signature = client + .send_and_confirm_transaction_with_spinner_and_config( + &transaction, + CommitmentConfig::confirmed(), + RpcSendTransactionConfig::default(), + ) + .await?; + + println!("Transaction signature: {}", signature); + + Ok(()) +} diff --git a/utils/steward-cli/src/commands/command_args.rs b/utils/steward-cli/src/commands/command_args.rs index 23b0bea4..88d3ef48 100644 --- a/utils/steward-cli/src/commands/command_args.rs +++ b/utils/steward-cli/src/commands/command_args.rs @@ -220,6 +220,8 @@ pub enum Commands { ManuallyRemoveValidator(ManuallyRemoveValidator), AutoRemoveValidatorFromPool(AutoRemoveValidatorFromPool), AutoAddValidatorFromPool(AutoAddValidatorFromPool), + InstantRemoveValidator(InstantRemoveValidator), + UpdateValidatorListBalance(UpdateValidatorListBalance), // Cranks CrankSteward(CrankSteward), @@ -435,6 +437,17 @@ pub struct ManuallyRemoveValidator { pub validator_index_to_remove: u64, } +#[derive(Parser)] +#[command(about = "Instantly removes validator from pool")] +pub struct InstantRemoveValidator { + #[command(flatten)] + pub permissionless_parameters: PermissionlessParameters, + + /// Validator index of validator list to remove + #[arg(long, env)] + pub validator_index_to_remove: u64, +} + #[derive(Parser)] #[command(about = "Removes bad validators from the pool")] pub struct RemoveBadValidators { @@ -464,6 +477,20 @@ pub struct AutoAddValidatorFromPool { pub vote_account: Pubkey, } +#[derive(Parser)] +#[command(about = "Updates validator list balance (spl_stake_pool command) for a single validator")] +pub struct UpdateValidatorListBalance { + #[command(flatten)] + pub permissionless_parameters: PermissionlessParameters, + + /// Validator index in the validator list + #[arg(long, env)] + pub validator_list_index: u32, + + #[arg(long, env, default_value_t = false)] + pub no_merge: bool, +} + // ---------- CRANKS ------------ #[derive(Parser)] diff --git a/utils/steward-cli/src/commands/cranks/compute_score.rs b/utils/steward-cli/src/commands/cranks/compute_score.rs index 947343e7..003b3cc6 100644 --- a/utils/steward-cli/src/commands/cranks/compute_score.rs +++ b/utils/steward-cli/src/commands/cranks/compute_score.rs @@ -32,17 +32,6 @@ pub async fn command_crank_compute_score( let steward_accounts = get_all_steward_accounts(client, &program_id, &steward_config).await?; - match steward_accounts.state_account.state.state_tag { - StewardStateEnum::ComputeScores => { /* Continue */ } - _ => { - println!( - "State account is not in ComputeScores state: {}", - steward_accounts.state_account.state.state_tag - ); - return Ok(()); - } - } - let validators_to_run = (0..steward_accounts.state_account.state.num_pool_validators) .filter_map(|validator_index| { let has_been_scored = steward_accounts diff --git a/utils/steward-cli/src/commands/info/view_state.rs b/utils/steward-cli/src/commands/info/view_state.rs index c6a955f3..cf8cf09c 100644 --- a/utils/steward-cli/src/commands/info/view_state.rs +++ b/utils/steward-cli/src/commands/info/view_state.rs @@ -1,6 +1,8 @@ use anchor_lang::AccountDeserialize; use anyhow::Result; -use jito_steward::{utils::ValidatorList, Config, StewardStateAccount}; +use jito_steward::{ + constants::LAMPORT_BALANCE_DEFAULT, utils::ValidatorList, Config, StewardStateAccount, +}; use solana_client::nonblocking::rpc_client::RpcClient; use solana_sdk::{account::Account, pubkey::Pubkey}; use spl_stake_pool::{ @@ -298,36 +300,60 @@ fn _print_verbose_state( formatted_string += &format!("Stake Account: {:?}\n", stake_address); formatted_string += &format!("Transient Stake Account: {:?}\n", transient_stake_address); formatted_string += &format!( - "Validator Lamports: {:?}\n", - steward_state_account + "Internal Validator Lamports: {}\n", + match steward_state_account .state .validator_lamport_balances .get(index) + { + Some(&LAMPORT_BALANCE_DEFAULT) | None => "Unset".to_string(), + Some(&lamports) => lamports.to_string(), + } ); - formatted_string += &format!("Index: {:?}\n", index); + formatted_string += &format!("Index: {}\n", index); formatted_string += &format!( - "Marked for removal: {:?}\n", - steward_state_account.state.validators_to_remove.get(index) + "Marked for removal: {}\n", + steward_state_account + .state + .validators_to_remove + .get(index) + .unwrap_or_default() ); formatted_string += &format!( - "Is Instant Unstake: {:?}\n", - steward_state_account.state.instant_unstake.get(index) + "Marked for immediate removal: {}\n", + steward_state_account + .state + .validators_for_immediate_removal + .get(index) + .unwrap_or_default() ); formatted_string += &format!( - "Score: {:?}\n", - steward_state_account.state.scores.get(index) + "Is Instant Unstake: {}\n", + steward_state_account + .state + .instant_unstake + .get(index) + .unwrap_or_default() + ); + formatted_string += &format!( + "Score: {}\n", + steward_state_account.state.scores.get(index).unwrap_or(&0) ); formatted_string += &format!( - "Yield Score: {:?}\n", - steward_state_account.state.yield_scores.get(index) + "Yield Score: {}\n", + steward_state_account + .state + .yield_scores + .get(index) + .unwrap_or(&0) ); formatted_string += &format!("Score Index: {:?}\n", score_index); formatted_string += &format!("Yield Score Index: {:?}\n", yield_score_index); if let Some(history_info) = history_info { formatted_string += &format!( - "\nValidator History Index: {:?}\n", + "\nValidator History Index: {}\n", format!("{:?}", history_info.index) ); diff --git a/utils/steward-cli/src/main.rs b/utils/steward-cli/src/main.rs index 2337f165..ebae522f 100644 --- a/utils/steward-cli/src/main.rs +++ b/utils/steward-cli/src/main.rs @@ -6,6 +6,7 @@ use commands::{ auto_add_validator_from_pool::command_auto_add_validator_from_pool, auto_remove_validator_from_pool::command_auto_remove_validator_from_pool, close_steward::command_close_steward, + instant_remove_validator::command_instant_remove_validator, manually_copy_all_vote_accounts::command_manually_copy_all_vote_accounts, manually_copy_vote_accounts::command_manually_copy_vote_account, manually_remove_validator::command_manually_remove_validator, pause::command_pause, @@ -15,6 +16,7 @@ use commands::{ resume::command_resume, revert_staker::command_revert_staker, set_staker::command_set_staker, update_authority::command_update_authority, update_config::command_update_config, + update_validator_list_balance::command_update_validator_list_balance, }, command_args::{Args, Commands}, cranks::{ @@ -82,6 +84,9 @@ async fn main() -> Result<()> { Commands::ManuallyCopyAllVoteAccounts(args) => { command_manually_copy_all_vote_accounts(args, &client, program_id).await } + Commands::InstantRemoveValidator(args) => { + command_instant_remove_validator(args, &client, program_id).await + } Commands::AutoRemoveValidatorFromPool(args) => { command_auto_remove_validator_from_pool(args, &client, program_id).await } @@ -95,6 +100,9 @@ async fn main() -> Result<()> { Commands::RemoveFromBlacklist(args) => { command_remove_from_blacklist(args, &client, program_id).await } + Commands::UpdateValidatorListBalance(args) => { + command_update_validator_list_balance(&client, args, program_id).await + } // --- Cranks --- Commands::CrankSteward(args) => command_crank_steward(args, &client, program_id).await,