Tweak keeper for alerts

docker-compose.yaml

@@ -27,12 +27,40 @@ services:
       - METRICS_INTERVAL=${METRICS_INTERVAL}
       - RUN_STAKE_UPLOAD=${RUN_STAKE_UPLOAD}
       - RUN_GOSSIP_UPLOAD=${RUN_GOSSIP_UPLOAD}
-      - RUN_EMIT_METRICS=${RUN_EMIT_METRICS}
+      - RUN_EMIT_METRICS=false
       - FULL_STARTUP=${FULL_STARTUP}
       - NO_PACK=${NO_PACK}
       - PAY_FOR_NEW_ACCOUNTS=${PAY_FOR_NEW_ACCOUNTS}
       - COOL_DOWN_RANGE=${COOL_DOWN_RANGE}
       - GOSSIP_ENTRYPOINT=${GOSSIP_ENTRYPOINT}
     volumes:
       - ./credentials:/credentials
-    restart: on-failure:5
+    restart: on-failure:5
+
+  metrics-only:
+    build:
+      context: .
+      target: validator-history
+    container_name: metrics-only
+    environment:
+      - RUST_LOG=${RUST_LOG:-info}
+      - SOLANA_METRICS_CONFIG=${SOLANA_METRICS_CONFIG}
+      - JSON_RPC_URL=${JSON_RPC_URL}
+      - CLUSTER=${CLUSTER}
+      - KEYPAIR=${KEYPAIR}
+      - VALIDATOR_HISTORY_PROGRAM_ID=${VALIDATOR_HISTORY_PROGRAM_ID}
+      - TIP_DISTRIBUTION_PROGRAM_ID=${TIP_DISTRIBUTION_PROGRAM_ID}
+      - STEWARD_PROGRAM_ID=${STEWARD_PROGRAM_ID}
+      - STEWARD_CONFIG=${STEWARD_CONFIG}
+      - METRICS_INTERVAL=${METRICS_INTERVAL}
+      - RUN_CLUSTER_HISTORY=false
+      - RUN_COPY_VOTE_ACCOUNTS=false
+      - RUN_MEV_COMMISSION=false
+      - RUN_MEV_EARNED=false
+      - RUN_STEWARD=false
+      - RUN_STAKE_UPLOAD=false
+      - RUN_GOSSIP_UPLOAD=false
+      - RUN_EMIT_METRICS=true
+    volumes:
+      - ./credentials:/credentials
+    restart: on-failure:5

keepers/validator-keeper/src/entries/crank_steward.rs

@@ -122,31 +122,33 @@ pub fn _get_update_stake_pool_ixs(
             .get(&validator_info.vote_account_address)
             .expect("Stake account not found");
 
-        let should_deactivate = if raw_vote_account.is_none() || raw_stake_account.is_none() {
-            true
-        } else {
-            let stake_account =
-                StakeStateV2::deserialize(&mut raw_stake_account.clone().unwrap().data.as_slice())
-                    .expect("Could not deserialize stake account");
-
-            let vote_account = VoteState::deserialize(&raw_vote_account.clone().unwrap().data)
-                .expect("Could not deserialize vote account");
-
-            let latest_epoch = vote_account.epoch_credits.iter().last().unwrap().0;
-
-            match stake_account {
-                StakeStateV2::Stake(_meta, stake, _stake_flags) => {
-                    if stake.delegation.deactivation_epoch != std::u64::MAX {
+        let should_deactivate = match (raw_vote_account, raw_stake_account) {
+            (None, Some(_)) => true,
+            (Some(raw_vote_account), Some(raw_stake_account)) => {
+                let stake_account =
+                    StakeStateV2::deserialize(&mut raw_stake_account.data.as_slice())
+                        .expect("Could not deserialize stake account");
+
+                let vote_account = VoteState::deserialize(&raw_vote_account.data)
+                    .expect("Could not deserialize vote account");
+
+                let latest_epoch = vote_account.epoch_credits.iter().last().unwrap().0;
+
+                match stake_account {
+                    StakeStateV2::Stake(_meta, stake, _stake_flags) => {
+                        if stake.delegation.deactivation_epoch != std::u64::MAX {
+                            false
+                        } else {
+                            latest_epoch <= epoch - 5
+                        }
+                    }
+                    _ => {
+                        println!("🔶 Error: Stake account is not StakeStateV2::Stake");
                         false
-                    } else {
-                        latest_epoch <= epoch - 5
                     }
                 }
-                _ => {
-                    println!("🔶 Error: Stake account is not StakeStateV2::Stake");
-                    false
-                }
             }
+            (_, None) => false,
         };
 
         if should_deactivate {

keepers/validator-keeper/src/operations/metrics_emit.rs

@@ -115,8 +115,8 @@ pub fn emit_validator_history_metrics(
     let mut cluster_history_blocks: i64 = 0;
     let cluster_history_entry = cluster_history.history.last();
     if let Some(cluster_history) = cluster_history_entry {
-        // Looking for previous epoch to be updated
-        if cluster_history.epoch as u64 == epoch_info.epoch - 1 {
+        // Looking for current epoch to be updated, implies previous is complete as well
+        if cluster_history.epoch as u64 == epoch_info.epoch {
             cluster_history_blocks = 1;
         }
     }

sdk/Cargo.toml

@@ -23,6 +23,7 @@ solana-client = "1.18"
 solana-metrics = "1.18"
 solana-program = "1.18"
 solana-sdk = "1.18"
+solana-transaction-status = "1.18"
 spl-pod = "0.1.0"
 spl-stake-pool = { features = ["no-entrypoint"], version = "1.0.0" }
 thiserror = "1.0.37"

sdk/src/utils/transactions.rs

@@ -4,7 +4,9 @@ use std::vec;
 use std::{collections::HashMap, sync::Arc, time::Duration};
 
 use log::*;
-use solana_client::rpc_response::{Response, RpcSimulateTransactionResult, RpcVoteAccountInfo};
+use solana_client::rpc_response::{
+    Response, RpcResult, RpcSimulateTransactionResult, RpcVoteAccountInfo,
+};
 use solana_client::{client_error::ClientError, nonblocking::rpc_client::RpcClient};
 use solana_metrics::datapoint_error;
 use solana_program::hash::Hash;
@@ -17,6 +19,7 @@ use solana_sdk::{
     instruction::Instruction, packet::Packet, pubkey::Pubkey, signature::Keypair,
     signature::Signature, signer::Signer, transaction::Transaction,
 };
+use solana_transaction_status::TransactionStatus;
 use tokio::task;
 use tokio::time::sleep;
 
@@ -197,6 +200,19 @@ pub async fn get_vote_accounts_with_retry(
     }
 }
 
+pub async fn get_signature_statuses_with_retry(
+    client: &RpcClient,
+    signatures: &[Signature],
+) -> RpcResult<Vec<Option<TransactionStatus>>> {
+    for _ in 1..4 {
+        if let Ok(result) = client.get_signature_statuses(signatures).await {
+            return Ok(result);
+        }
+        tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
+    }
+    client.get_signature_statuses(signatures).await
+}
+
 async fn find_ix_per_tx(
     client: &Arc<RpcClient>,
     instruction: &Instruction,
@@ -264,14 +280,17 @@ async fn parallel_confirm_transactions(
     let confirmation_futures: Vec<_> = signatures_to_confirm
         .chunks(SIG_STATUS_BATCH_SIZE)
         .map(|sig_batch| async move {
-            match client.get_signature_statuses(sig_batch).await {
+            match get_signature_statuses_with_retry(client, sig_batch).await {
                 Ok(sig_batch_response) => sig_batch_response
                     .value
                     .iter()
                     .enumerate()
                     .map(|(i, sig_status)| (sig_batch[i], sig_status.clone()))
                     .collect::<Vec<_>>(),
-                Err(_) => vec![],
+                Err(e) => {
+                    info!("Failed getting signature statuses: {}", e);
+                    vec![]
+                }
             }
         })
         .collect();
@@ -650,7 +669,7 @@ pub async fn submit_transactions(
         .map(|t| t.as_slice())
         .collect::<Vec<_>>();
 
-    match parallel_execute_transactions(client, &tx_slice, keypair, 100, 20).await {
+    match parallel_execute_transactions(client, &tx_slice, keypair, 100, 30).await {
         Ok(results) => {
             stats.successes = results.iter().filter(|&tx| tx.is_ok()).count() as u64;
             stats.errors = results.len() as u64 - stats.successes;