Fixes: rebalance decrease amount calculation, validator_lamport_balan…

…ce initialization, tweaks to keeper (#72)

Changes:
* Fixes rebalance decrease: the stake pool's validator_list lamport
amount can get out of alignment with the associated stake account's
active lamports, resulting in more lamports being withdrawn from
validators than is possible. The problem is fixed by passing along the
true active stake amount into the decrease_validator_stake calculation
* Unblocks unstaking for stake deposits by fixing how
validator_lamport_balances are initialized. Currently, all
validator_lamport_balances start at 0, meaning all current lamports are
seen as stake deposits and tried to be unstaked until the cap is hit.
However, we can load each value in a `rebalance` instruction when there
are no activating or deactivating lamports and then continue with the
working process. Adds a sentinel value and checks to skip modifying it
when sentinel is present.
* Will deploy on test program
sssh4zkKhX8jXTNQz1xDHyGpygzgu2UhcRcUvZihBjP before deploying on mainnet
program

.github/workflows/build.yaml

@@ -129,6 +129,8 @@ jobs:
         with:
           submodules: recursive
       - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: nightly-2024-02-04
       - uses: actions/download-artifact@v4
         with:
           name: validator_history.so
@@ -138,18 +140,12 @@ jobs:
           name: jito_steward.so
           path: target/deploy/
       - name: cargo test
-        run: cargo test --package tests --all-features --color auto -- --skip steward::test_state_methods
+        run: cargo test --package tests --all-features --color auto
         shell: bash
         env:
           RUST_LOG: trace
           SBF_OUT_DIR: ${{ github.workspace }}/target/deploy
           RUST_MIN_STACK: 5000000
-      - name: cargo test steward::test_state_methods
-        run: cargo test --package tests --test mod steward::test_state_methods
-        shell: bash
-        env:
-          RUST_LOG: trace
-          RUST_MIN_STACK: 5000000
 
   # release only runs on tagged commits
   # it should wait for all the other steps to finish, to ensure releases are the highest quality

docker-compose.yaml

@@ -35,4 +35,34 @@ services:
       - GOSSIP_ENTRYPOINT=${GOSSIP_ENTRYPOINT}
     volumes:
       - ./credentials:/credentials
-    restart: on-failure:5
+    restart: on-failure:5
+
+  metrics-only:
+    build:
+      context: .
+      target: validator-history
+    container_name: metrics-only
+    environment:
+      - RUST_LOG=${RUST_LOG:-info}
+      - SOLANA_METRICS_CONFIG=${SOLANA_METRICS_CONFIG}
+      - JSON_RPC_URL=${JSON_RPC_URL}
+      - CLUSTER=${CLUSTER}
+      - KEYPAIR=${KEYPAIR}
+      - VALIDATOR_HISTORY_PROGRAM_ID=${VALIDATOR_HISTORY_PROGRAM_ID}
+      - TIP_DISTRIBUTION_PROGRAM_ID=${TIP_DISTRIBUTION_PROGRAM_ID}
+      - STEWARD_PROGRAM_ID=${STEWARD_PROGRAM_ID}
+      - STEWARD_CONFIG=${STEWARD_CONFIG}
+      - METRICS_INTERVAL=${METRICS_INTERVAL}
+      - STEWARD_INTERVAL=1000000000000
+      - VALIDATOR_HISTORY_INTERVAL=1000000000000
+      - RUN_CLUSTER_HISTORY=false
+      - RUN_COPY_VOTE_ACCOUNTS=false
+      - RUN_MEV_COMMISSION=false
+      - RUN_MEV_EARNED=false
+      - RUN_STEWARD=false
+      - RUN_STAKE_UPLOAD=false
+      - RUN_GOSSIP_UPLOAD=false
+      - RUN_EMIT_METRICS=true
+    volumes:
+      - ./credentials:/credentials
+    restart: on-failure:5

keepers/validator-keeper/src/entries/crank_steward.rs

@@ -122,31 +122,33 @@ pub fn _get_update_stake_pool_ixs(
             .get(&validator_info.vote_account_address)
             .expect("Stake account not found");
 
-        let should_deactivate = if raw_vote_account.is_none() || raw_stake_account.is_none() {
-            true
-        } else {
-            let stake_account =
-                StakeStateV2::deserialize(&mut raw_stake_account.clone().unwrap().data.as_slice())
-                    .expect("Could not deserialize stake account");
-
-            let vote_account = VoteState::deserialize(&raw_vote_account.clone().unwrap().data)
-                .expect("Could not deserialize vote account");
-
-            let latest_epoch = vote_account.epoch_credits.iter().last().unwrap().0;
-
-            match stake_account {
-                StakeStateV2::Stake(_meta, stake, _stake_flags) => {
-                    if stake.delegation.deactivation_epoch != std::u64::MAX {
+        let should_deactivate = match (raw_vote_account, raw_stake_account) {
+            (None, Some(_)) => true,
+            (Some(raw_vote_account), Some(raw_stake_account)) => {
+                let stake_account =
+                    StakeStateV2::deserialize(&mut raw_stake_account.data.as_slice())
+                        .expect("Could not deserialize stake account");
+
+                let vote_account = VoteState::deserialize(&raw_vote_account.data)
+                    .expect("Could not deserialize vote account");
+
+                let latest_epoch = vote_account.epoch_credits.iter().last().unwrap().0;
+
+                match stake_account {
+                    StakeStateV2::Stake(_meta, stake, _stake_flags) => {
+                        if stake.delegation.deactivation_epoch != std::u64::MAX {
+                            false
+                        } else {
+                            latest_epoch <= epoch - 5
+                        }
+                    }
+                    _ => {
+                        println!("🔶 Error: Stake account is not StakeStateV2::Stake");
                         false
-                    } else {
-                        latest_epoch <= epoch - 5
                     }
                 }
-                _ => {
-                    println!("🔶 Error: Stake account is not StakeStateV2::Stake");
-                    false
-                }
             }
+            (_, None) => false,
         };
 
         if should_deactivate {
@@ -217,6 +219,13 @@ async fn _update_pool(
 
     // TODO fix
     println!("Deactivating Delinquent");
+    // for ix in deactivate_delinquent_ixs {
+    //     let tx = Transaction::new_signed_with_payer(&[ix], Some(&payer.pubkey()), &[&payer]);
+    //     let tx = client
+    //         .send_and_confirm_transaction_with_spinner_and_config(&tx)
+    //         .await?;
+    //     stats.add_tx(&tx);
+    // }
     let deactivate_txs_to_run = package_instructions(
         &deactivate_delinquent_ixs,
         1,
@@ -493,11 +502,7 @@ async fn _handle_delinquent_validators(
     let bad_vote_accounts = checks
         .iter()
         .filter_map(|(vote_account, check)| {
-            if !check.has_history
-                || !check.has_stake_account
-                || check.is_deactivated
-                || !check.has_vote_account
-            {
+            if !check.has_history || check.is_deactivated || !check.has_vote_account {
                 Some(*vote_account)
             } else {
                 None

keepers/validator-keeper/src/main.rs

@@ -249,14 +249,14 @@ async fn run_keeper(keeper_config: KeeperConfig) {
         // ---------------------- EMIT ---------------------------------
 
         if should_fire(tick, metrics_interval) {
-            info!("Emitting metrics...");
             keeper_state.set_runs_errors_and_txs_for_epoch(operations::metrics_emit::fire(
                 &keeper_config,
                 &keeper_state,
             ));
         }
 
         if should_emit(tick, &intervals) {
+            info!("Emitting metrics...");
             keeper_state.emit();
 
             KeeperOperations::emit(
@@ -332,6 +332,8 @@ async fn main() {
         no_pack: args.no_pack,
         pay_for_new_accounts: args.pay_for_new_accounts,
         cool_down_range: args.cool_down_range,
+        tx_retry_count: args.tx_retry_count,
+        tx_confirmation_seconds: args.tx_confirmation_seconds,
     };
 
     run_keeper(config).await;

keepers/validator-keeper/src/operations/cluster_history.rs

@@ -40,8 +40,18 @@ async fn _process(
     keypair: &Arc<Keypair>,
     program_id: &Pubkey,
     priority_fee_in_microlamports: u64,
+    retry_count: u16,
+    confirmation_time: u64,
 ) -> Result<SubmitStats, JitoTransactionExecutionError> {
-    update_cluster_info(client, keypair, program_id, priority_fee_in_microlamports).await
+    update_cluster_info(
+        client,
+        keypair,
+        program_id,
+        priority_fee_in_microlamports,
+        retry_count,
+        confirmation_time,
+    )
+    .await
 }
 
 pub async fn fire(
@@ -52,6 +62,8 @@ pub async fn fire(
     let keypair = &keeper_config.keypair;
     let program_id = &keeper_config.validator_history_program_id;
     let priority_fee_in_microlamports = keeper_config.priority_fee_in_microlamports;
+    let retry_count = keeper_config.tx_retry_count;
+    let confirmation_time = keeper_config.tx_confirmation_seconds;
 
     let operation = _get_operation();
     let epoch_info = &keeper_state.epoch_info;
@@ -63,7 +75,16 @@ pub async fn fire(
         _should_run(epoch_info, runs_for_epoch) && check_flag(keeper_config.run_flags, operation);
 
     if should_run {
-        match _process(client, keypair, program_id, priority_fee_in_microlamports).await {
+        match _process(
+            client,
+            keypair,
+            program_id,
+            priority_fee_in_microlamports,
+            retry_count,
+            confirmation_time,
+        )
+        .await
+        {
             Ok(stats) => {
                 for message in stats.results.iter() {
                     if let Err(e) = message {
@@ -126,12 +147,14 @@ pub async fn update_cluster_info(
     keypair: &Arc<Keypair>,
     program_id: &Pubkey,
     priority_fee_in_microlamports: u64,
+    retry_count: u16,
+    confirmation_time: u64,
 ) -> Result<SubmitStats, JitoTransactionExecutionError> {
     let ixs = get_update_cluster_info_instructions(
         program_id,
         &keypair.pubkey(),
         priority_fee_in_microlamports,
     );
 
-    submit_transactions(client, vec![ixs], keypair).await
+    submit_transactions(client, vec![ixs], keypair, retry_count, confirmation_time).await
 }

keepers/validator-keeper/src/operations/gossip_upload.rs

@@ -45,13 +45,16 @@ fn _should_run(epoch_info: &EpochInfo, runs_for_epoch: u64) -> bool {
         || (epoch_info.slot_index > epoch_info.slots_in_epoch * 9 / 10 && runs_for_epoch < 3)
 }
 
+#[allow(clippy::too_many_arguments)]
 async fn _process(
     client: &Arc<RpcClient>,
     keypair: &Arc<Keypair>,
     program_id: &Pubkey,
     priority_fee_in_microlamports: u64,
     entrypoint: &SocketAddr,
     keeper_state: &KeeperState,
+    retry_count: u16,
+    confirmation_time: u64,
 ) -> Result<SubmitStats, Box<dyn std::error::Error>> {
     upload_gossip_values(
         client,
@@ -60,6 +63,8 @@ async fn _process(
         priority_fee_in_microlamports,
         entrypoint,
         keeper_state,
+        retry_count,
+        confirmation_time,
     )
     .await
 }
@@ -76,6 +81,8 @@ pub async fn fire(
         .expect("Entry point not set");
 
     let priority_fee_in_microlamports = keeper_config.priority_fee_in_microlamports;
+    let retry_count = keeper_config.tx_retry_count;
+    let confirmation_time = keeper_config.tx_confirmation_seconds;
 
     let operation = _get_operation();
     let (mut runs_for_epoch, mut errors_for_epoch, mut txs_for_epoch) =
@@ -92,6 +99,8 @@ pub async fn fire(
             priority_fee_in_microlamports,
             entrypoint,
             keeper_state,
+            retry_count,
+            confirmation_time,
         )
         .await
         {
@@ -248,13 +257,16 @@ fn build_gossip_entry(
     }
 }
 
+#[allow(clippy::too_many_arguments)]
 pub async fn upload_gossip_values(
     client: &Arc<RpcClient>,
     keypair: &Arc<Keypair>,
     program_id: &Pubkey,
     priority_fee_in_microlamports: u64,
     entrypoint: &SocketAddr,
     keeper_state: &KeeperState,
+    retry_count: u16,
+    confirmation_time: u64,
 ) -> Result<SubmitStats, Box<dyn std::error::Error>> {
     let vote_accounts = keeper_state.vote_account_map.values().collect::<Vec<_>>();
     let validator_history_map = &keeper_state.validator_history_map;
@@ -314,7 +326,14 @@ pub async fn upload_gossip_values(
         .map(|entry| entry.build_update_tx(priority_fee_in_microlamports))
         .collect::<Vec<_>>();
 
-    let submit_result = submit_transactions(client, update_transactions, keypair).await;
+    let submit_result = submit_transactions(
+        client,
+        update_transactions,
+        keypair,
+        retry_count,
+        confirmation_time,
+    )
+    .await;
 
     submit_result.map_err(|e| e.into())
 }

keepers/validator-keeper/src/operations/metrics_emit.rs

@@ -115,8 +115,8 @@ pub fn emit_validator_history_metrics(
     let mut cluster_history_blocks: i64 = 0;
     let cluster_history_entry = cluster_history.history.last();
     if let Some(cluster_history) = cluster_history_entry {
-        // Looking for previous epoch to be updated
-        if cluster_history.epoch as u64 == epoch_info.epoch - 1 {
+        // Looking for current epoch to be updated, implies previous is complete as well
+        if cluster_history.epoch as u64 == epoch_info.epoch {
             cluster_history_blocks = 1;
         }
     }

keepers/validator-keeper/src/operations/mev_commission.rs

@@ -32,12 +32,15 @@ fn _should_run() -> bool {
     true
 }
 
+#[allow(clippy::too_many_arguments)]
 async fn _process(
     client: &Arc<RpcClient>,
     keypair: &Arc<Keypair>,
     program_id: &Pubkey,
     tip_distribution_program_id: &Pubkey,
     keeper_state: &KeeperState,
+    retry_count: u16,
+    confirmation_time: u64,
     priority_fee_in_microlamports: u64,
     no_pack: bool,
 ) -> Result<SubmitStats, JitoTransactionError> {
@@ -47,6 +50,8 @@ async fn _process(
         program_id,
         tip_distribution_program_id,
         keeper_state,
+        retry_count,
+        confirmation_time,
         priority_fee_in_microlamports,
         no_pack,
     )
@@ -76,6 +81,8 @@ pub async fn fire(
             program_id,
             tip_distribution_program_id,
             keeper_state,
+            keeper_config.tx_retry_count,
+            keeper_config.tx_confirmation_seconds,
             priority_fee_in_microlamports,
             keeper_config.no_pack,
         )
@@ -106,12 +113,15 @@ pub async fn fire(
 
 // ----------------- OPERATION SPECIFIC FUNCTIONS -----------------
 
+#[allow(clippy::too_many_arguments)]
 pub async fn update_mev_commission(
     client: &Arc<RpcClient>,
     keypair: &Arc<Keypair>,
     program_id: &Pubkey,
     tip_distribution_program_id: &Pubkey,
     keeper_state: &KeeperState,
+    retry_count: u16,
+    confirmation_time: u64,
     priority_fee_in_microlamports: u64,
     no_pack: bool,
 ) -> Result<SubmitStats, JitoTransactionError> {
@@ -148,6 +158,8 @@ pub async fn update_mev_commission(
         update_instructions,
         keypair,
         priority_fee_in_microlamports,
+        retry_count,
+        confirmation_time,
         None,
         no_pack,
     )