Tweak keeper for alerts (#74)

docker-compose.yaml

@@ -35,4 +35,34 @@ services:
       - GOSSIP_ENTRYPOINT=${GOSSIP_ENTRYPOINT}
     volumes:
       - ./credentials:/credentials
-    restart: on-failure:5
+    restart: on-failure:5
+
+  metrics-only:
+    build:
+      context: .
+      target: validator-history
+    container_name: metrics-only
+    environment:
+      - RUST_LOG=${RUST_LOG:-info}
+      - SOLANA_METRICS_CONFIG=${SOLANA_METRICS_CONFIG}
+      - JSON_RPC_URL=${JSON_RPC_URL}
+      - CLUSTER=${CLUSTER}
+      - KEYPAIR=${KEYPAIR}
+      - VALIDATOR_HISTORY_PROGRAM_ID=${VALIDATOR_HISTORY_PROGRAM_ID}
+      - TIP_DISTRIBUTION_PROGRAM_ID=${TIP_DISTRIBUTION_PROGRAM_ID}
+      - STEWARD_PROGRAM_ID=${STEWARD_PROGRAM_ID}
+      - STEWARD_CONFIG=${STEWARD_CONFIG}
+      - METRICS_INTERVAL=${METRICS_INTERVAL}
+      - STEWARD_INTERVAL=1000000000000
+      - VALIDATOR_HISTORY_INTERVAL=1000000000000
+      - RUN_CLUSTER_HISTORY=false
+      - RUN_COPY_VOTE_ACCOUNTS=false
+      - RUN_MEV_COMMISSION=false
+      - RUN_MEV_EARNED=false
+      - RUN_STEWARD=false
+      - RUN_STAKE_UPLOAD=false
+      - RUN_GOSSIP_UPLOAD=false
+      - RUN_EMIT_METRICS=true
+    volumes:
+      - ./credentials:/credentials
+    restart: on-failure:5

keepers/validator-keeper/src/entries/crank_steward.rs

@@ -9,6 +9,7 @@ use solana_program::instruction::Instruction;
 
 use solana_sdk::stake::instruction::deactivate_delinquent_stake;
 use solana_sdk::stake::state::StakeStateV2;
+use solana_sdk::transaction::Transaction;
 use solana_sdk::vote::state::VoteState;
 use solana_sdk::{pubkey::Pubkey, signature::Keypair, stake, system_program};
 use spl_stake_pool::instruction::{
@@ -122,31 +123,33 @@ pub fn _get_update_stake_pool_ixs(
             .get(&validator_info.vote_account_address)
             .expect("Stake account not found");
 
-        let should_deactivate = if raw_vote_account.is_none() || raw_stake_account.is_none() {
-            true
-        } else {
-            let stake_account =
-                StakeStateV2::deserialize(&mut raw_stake_account.clone().unwrap().data.as_slice())
-                    .expect("Could not deserialize stake account");
+        let should_deactivate = match (raw_vote_account, raw_stake_account) {
+            (None, Some(_)) => true,
+            (Some(raw_vote_account), Some(raw_stake_account)) => {
+                let stake_account =
+                    StakeStateV2::deserialize(&mut raw_stake_account.data.as_slice())
+                        .expect("Could not deserialize stake account");
 
-            let vote_account = VoteState::deserialize(&raw_vote_account.clone().unwrap().data)
-                .expect("Could not deserialize vote account");
+                let vote_account = VoteState::deserialize(&raw_vote_account.data)
+                    .expect("Could not deserialize vote account");
 
-            let latest_epoch = vote_account.epoch_credits.iter().last().unwrap().0;
+                let latest_epoch = vote_account.epoch_credits.iter().last().unwrap().0;
 
-            match stake_account {
-                StakeStateV2::Stake(_meta, stake, _stake_flags) => {
-                    if stake.delegation.deactivation_epoch != std::u64::MAX {
+                match stake_account {
+                    StakeStateV2::Stake(_meta, stake, _stake_flags) => {
+                        if stake.delegation.deactivation_epoch != std::u64::MAX {
+                            false
+                        } else {
+                            latest_epoch <= epoch - 5
+                        }
+                    }
+                    _ => {
+                        println!("🔶 Error: Stake account is not StakeStateV2::Stake");
                         false
-                    } else {
-                        latest_epoch <= epoch - 5
                     }
                 }
-                _ => {
-                    println!("🔶 Error: Stake account is not StakeStateV2::Stake");
-                    false
-                }
             }
+            (_, None) => false,
         };
 
         if should_deactivate {
@@ -217,6 +220,13 @@ async fn _update_pool(
 
     // TODO fix
     println!("Deactivating Delinquent");
+    // for ix in deactivate_delinquent_ixs {
+    //     let tx = Transaction::new_signed_with_payer(&[ix], Some(&payer.pubkey()), &[&payer]);
+    //     let tx = client
+    //         .send_and_confirm_transaction_with_spinner_and_config(&tx)
+    //         .await?;
+    //     stats.add_tx(&tx);
+    // }
     let deactivate_txs_to_run = package_instructions(
         &deactivate_delinquent_ixs,
         1,
@@ -493,11 +503,7 @@ async fn _handle_delinquent_validators(
     let bad_vote_accounts = checks
         .iter()
         .filter_map(|(vote_account, check)| {
-            if !check.has_history
-                || !check.has_stake_account
-                || check.is_deactivated
-                || !check.has_vote_account
-            {
+            if !check.has_history || check.is_deactivated || !check.has_vote_account {
                 Some(*vote_account)
             } else {
                 None

keepers/validator-keeper/src/main.rs

@@ -249,14 +249,14 @@ async fn run_keeper(keeper_config: KeeperConfig) {
         // ---------------------- EMIT ---------------------------------
 
         if should_fire(tick, metrics_interval) {
-            info!("Emitting metrics...");
             keeper_state.set_runs_errors_and_txs_for_epoch(operations::metrics_emit::fire(
                 &keeper_config,
                 &keeper_state,
             ));
         }
 
         if should_emit(tick, &intervals) {
+            info!("Emitting metrics...");
             keeper_state.emit();
 
             KeeperOperations::emit(
@@ -332,6 +332,8 @@ async fn main() {
         no_pack: args.no_pack,
         pay_for_new_accounts: args.pay_for_new_accounts,
         cool_down_range: args.cool_down_range,
+        tx_retry_count: args.tx_retry_count,
+        tx_confirmation_seconds: args.tx_confirmation_seconds,
     };
 
     run_keeper(config).await;

keepers/validator-keeper/src/operations/cluster_history.rs

@@ -40,8 +40,18 @@ async fn _process(
     keypair: &Arc<Keypair>,
     program_id: &Pubkey,
     priority_fee_in_microlamports: u64,
+    retry_count: u16,
+    confirmation_time: u64,
 ) -> Result<SubmitStats, JitoTransactionExecutionError> {
-    update_cluster_info(client, keypair, program_id, priority_fee_in_microlamports).await
+    update_cluster_info(
+        client,
+        keypair,
+        program_id,
+        priority_fee_in_microlamports,
+        retry_count,
+        confirmation_time,
+    )
+    .await
 }
 
 pub async fn fire(
@@ -52,6 +62,8 @@ pub async fn fire(
     let keypair = &keeper_config.keypair;
     let program_id = &keeper_config.validator_history_program_id;
     let priority_fee_in_microlamports = keeper_config.priority_fee_in_microlamports;
+    let retry_count = keeper_config.tx_retry_count;
+    let confirmation_time = keeper_config.tx_confirmation_seconds;
 
     let operation = _get_operation();
     let epoch_info = &keeper_state.epoch_info;
@@ -63,7 +75,16 @@ pub async fn fire(
         _should_run(epoch_info, runs_for_epoch) && check_flag(keeper_config.run_flags, operation);
 
     if should_run {
-        match _process(client, keypair, program_id, priority_fee_in_microlamports).await {
+        match _process(
+            client,
+            keypair,
+            program_id,
+            priority_fee_in_microlamports,
+            retry_count,
+            confirmation_time,
+        )
+        .await
+        {
             Ok(stats) => {
                 for message in stats.results.iter() {
                     if let Err(e) = message {
@@ -126,12 +147,14 @@ pub async fn update_cluster_info(
     keypair: &Arc<Keypair>,
     program_id: &Pubkey,
     priority_fee_in_microlamports: u64,
+    retry_count: u16,
+    confirmation_time: u64,
 ) -> Result<SubmitStats, JitoTransactionExecutionError> {
     let ixs = get_update_cluster_info_instructions(
         program_id,
         &keypair.pubkey(),
         priority_fee_in_microlamports,
     );
 
-    submit_transactions(client, vec![ixs], keypair).await
+    submit_transactions(client, vec![ixs], keypair, retry_count, confirmation_time).await
 }

keepers/validator-keeper/src/operations/gossip_upload.rs

@@ -44,14 +44,15 @@ fn _should_run(epoch_info: &EpochInfo, runs_for_epoch: u64) -> bool {
         || (epoch_info.slot_index > epoch_info.slots_in_epoch / 2 && runs_for_epoch < 2)
         || (epoch_info.slot_index > epoch_info.slots_in_epoch * 9 / 10 && runs_for_epoch < 3)
 }
-
 async fn _process(
     client: &Arc<RpcClient>,
     keypair: &Arc<Keypair>,
     program_id: &Pubkey,
     priority_fee_in_microlamports: u64,
     entrypoint: &SocketAddr,
     keeper_state: &KeeperState,
+    retry_count: u16,
+    confirmation_time: u64,
 ) -> Result<SubmitStats, Box<dyn std::error::Error>> {
     upload_gossip_values(
         client,
@@ -60,6 +61,8 @@ async fn _process(
         priority_fee_in_microlamports,
         entrypoint,
         keeper_state,
+        retry_count,
+        confirmation_time,
     )
     .await
 }
@@ -76,6 +79,8 @@ pub async fn fire(
         .expect("Entry point not set");
 
     let priority_fee_in_microlamports = keeper_config.priority_fee_in_microlamports;
+    let retry_count = keeper_config.tx_retry_count;
+    let confirmation_time = keeper_config.tx_confirmation_seconds;
 
     let operation = _get_operation();
     let (mut runs_for_epoch, mut errors_for_epoch, mut txs_for_epoch) =
@@ -92,6 +97,8 @@ pub async fn fire(
             priority_fee_in_microlamports,
             entrypoint,
             keeper_state,
+            retry_count,
+            confirmation_time,
         )
         .await
         {
@@ -255,6 +262,8 @@ pub async fn upload_gossip_values(
     priority_fee_in_microlamports: u64,
     entrypoint: &SocketAddr,
     keeper_state: &KeeperState,
+    retry_count: u16,
+    confirmation_time: u64,
 ) -> Result<SubmitStats, Box<dyn std::error::Error>> {
     let vote_accounts = keeper_state.vote_account_map.values().collect::<Vec<_>>();
     let validator_history_map = &keeper_state.validator_history_map;
@@ -314,7 +323,14 @@ pub async fn upload_gossip_values(
         .map(|entry| entry.build_update_tx(priority_fee_in_microlamports))
         .collect::<Vec<_>>();
 
-    let submit_result = submit_transactions(client, update_transactions, keypair).await;
+    let submit_result = submit_transactions(
+        client,
+        update_transactions,
+        keypair,
+        retry_count,
+        confirmation_time,
+    )
+    .await;
 
     submit_result.map_err(|e| e.into())
 }

keepers/validator-keeper/src/operations/metrics_emit.rs

@@ -115,8 +115,8 @@ pub fn emit_validator_history_metrics(
     let mut cluster_history_blocks: i64 = 0;
     let cluster_history_entry = cluster_history.history.last();
     if let Some(cluster_history) = cluster_history_entry {
-        // Looking for previous epoch to be updated
-        if cluster_history.epoch as u64 == epoch_info.epoch - 1 {
+        // Looking for current epoch to be updated, implies previous is complete as well
+        if cluster_history.epoch as u64 == epoch_info.epoch {
             cluster_history_blocks = 1;
         }
     }

keepers/validator-keeper/src/operations/mev_commission.rs

@@ -38,6 +38,8 @@ async fn _process(
     program_id: &Pubkey,
     tip_distribution_program_id: &Pubkey,
     keeper_state: &KeeperState,
+    retry_count: u16,
+    confirmation_time: u64,
     priority_fee_in_microlamports: u64,
     no_pack: bool,
 ) -> Result<SubmitStats, JitoTransactionError> {
@@ -47,6 +49,8 @@ async fn _process(
         program_id,
         tip_distribution_program_id,
         keeper_state,
+        retry_count,
+        confirmation_time,
         priority_fee_in_microlamports,
         no_pack,
     )
@@ -76,6 +80,8 @@ pub async fn fire(
             program_id,
             tip_distribution_program_id,
             keeper_state,
+            keeper_config.tx_retry_count,
+            keeper_config.tx_confirmation_seconds,
             priority_fee_in_microlamports,
             keeper_config.no_pack,
         )
@@ -112,6 +118,8 @@ pub async fn update_mev_commission(
     program_id: &Pubkey,
     tip_distribution_program_id: &Pubkey,
     keeper_state: &KeeperState,
+    retry_count: u16,
+    confirmation_time: u64,
     priority_fee_in_microlamports: u64,
     no_pack: bool,
 ) -> Result<SubmitStats, JitoTransactionError> {
@@ -148,6 +156,8 @@ pub async fn update_mev_commission(
         update_instructions,
         keypair,
         priority_fee_in_microlamports,
+        retry_count,
+        confirmation_time,
         None,
         no_pack,
     )

keepers/validator-keeper/src/operations/mev_earned.rs

@@ -40,6 +40,8 @@ async fn _process(
     program_id: &Pubkey,
     tip_distribution_program_id: &Pubkey,
     priority_fee_in_microlamports: u64,
+    retry_count: u16,
+    confirmation_time: u64,
     keeper_state: &KeeperState,
     no_pack: bool,
 ) -> Result<SubmitStats, JitoTransactionError> {
@@ -48,6 +50,8 @@ async fn _process(
         keypair,
         program_id,
         priority_fee_in_microlamports,
+        retry_count,
+        confirmation_time,
         tip_distribution_program_id,
         keeper_state,
         no_pack,
@@ -78,6 +82,8 @@ pub async fn fire(
             program_id,
             tip_distribution_program_id,
             priority_fee_in_microlamports,
+            keeper_config.tx_retry_count,
+            keeper_config.tx_confirmation_seconds,
             keeper_state,
             keeper_config.no_pack,
         )
@@ -113,6 +119,8 @@ pub async fn update_mev_earned(
     keypair: &Arc<Keypair>,
     program_id: &Pubkey,
     priority_fee_in_microlamports: u64,
+    retry_count: u16,
+    confirmation_time: u64,
     tip_distribution_program_id: &Pubkey,
     keeper_state: &KeeperState,
     no_pack: bool,
@@ -165,6 +173,8 @@ pub async fn update_mev_earned(
         update_instructions,
         keypair,
         priority_fee_in_microlamports,
+        retry_count,
+        confirmation_time,
         None,
         no_pack,
     )