Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Tweak keeper for alerts #74

Merged
merged 13 commits into from
Aug 7, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

32 changes: 31 additions & 1 deletion docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -35,4 +35,34 @@ services:
- GOSSIP_ENTRYPOINT=${GOSSIP_ENTRYPOINT}
volumes:
- ./credentials:/credentials
restart: on-failure:5
restart: on-failure:5

metrics-only:
build:
context: .
target: validator-history
container_name: metrics-only
environment:
- RUST_LOG=${RUST_LOG:-info}
- SOLANA_METRICS_CONFIG=${SOLANA_METRICS_CONFIG}
- JSON_RPC_URL=${JSON_RPC_URL}
- CLUSTER=${CLUSTER}
- KEYPAIR=${KEYPAIR}
- VALIDATOR_HISTORY_PROGRAM_ID=${VALIDATOR_HISTORY_PROGRAM_ID}
- TIP_DISTRIBUTION_PROGRAM_ID=${TIP_DISTRIBUTION_PROGRAM_ID}
- STEWARD_PROGRAM_ID=${STEWARD_PROGRAM_ID}
- STEWARD_CONFIG=${STEWARD_CONFIG}
- METRICS_INTERVAL=${METRICS_INTERVAL}
- STEWARD_INTERVAL=1000000000000
- VALIDATOR_HISTORY_INTERVAL=1000000000000
- RUN_CLUSTER_HISTORY=false
- RUN_COPY_VOTE_ACCOUNTS=false
- RUN_MEV_COMMISSION=false
- RUN_MEV_EARNED=false
- RUN_STEWARD=false
- RUN_STAKE_UPLOAD=false
- RUN_GOSSIP_UPLOAD=false
- RUN_EMIT_METRICS=true
volumes:
- ./credentials:/credentials
restart: on-failure:5
52 changes: 29 additions & 23 deletions keepers/validator-keeper/src/entries/crank_steward.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use solana_program::instruction::Instruction;

use solana_sdk::stake::instruction::deactivate_delinquent_stake;
use solana_sdk::stake::state::StakeStateV2;
use solana_sdk::transaction::Transaction;
use solana_sdk::vote::state::VoteState;
use solana_sdk::{pubkey::Pubkey, signature::Keypair, stake, system_program};
use spl_stake_pool::instruction::{
Expand Down Expand Up @@ -122,31 +123,33 @@ pub fn _get_update_stake_pool_ixs(
.get(&validator_info.vote_account_address)
.expect("Stake account not found");

let should_deactivate = if raw_vote_account.is_none() || raw_stake_account.is_none() {
true
} else {
let stake_account =
StakeStateV2::deserialize(&mut raw_stake_account.clone().unwrap().data.as_slice())
.expect("Could not deserialize stake account");
let should_deactivate = match (raw_vote_account, raw_stake_account) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is much better!

(None, Some(_)) => true,
(Some(raw_vote_account), Some(raw_stake_account)) => {
let stake_account =
StakeStateV2::deserialize(&mut raw_stake_account.data.as_slice())
.expect("Could not deserialize stake account");

let vote_account = VoteState::deserialize(&raw_vote_account.clone().unwrap().data)
.expect("Could not deserialize vote account");
let vote_account = VoteState::deserialize(&raw_vote_account.data)
.expect("Could not deserialize vote account");

let latest_epoch = vote_account.epoch_credits.iter().last().unwrap().0;
let latest_epoch = vote_account.epoch_credits.iter().last().unwrap().0;

match stake_account {
StakeStateV2::Stake(_meta, stake, _stake_flags) => {
if stake.delegation.deactivation_epoch != std::u64::MAX {
match stake_account {
StakeStateV2::Stake(_meta, stake, _stake_flags) => {
if stake.delegation.deactivation_epoch != std::u64::MAX {
false
} else {
latest_epoch <= epoch - 5
}
}
_ => {
println!("🔶 Error: Stake account is not StakeStateV2::Stake");
false
} else {
latest_epoch <= epoch - 5
}
}
_ => {
println!("🔶 Error: Stake account is not StakeStateV2::Stake");
false
}
}
(_, None) => false,
};

if should_deactivate {
Expand Down Expand Up @@ -217,6 +220,13 @@ async fn _update_pool(

// TODO fix
println!("Deactivating Delinquent");
// for ix in deactivate_delinquent_ixs {
// let tx = Transaction::new_signed_with_payer(&[ix], Some(&payer.pubkey()), &[&payer]);
// let tx = client
// .send_and_confirm_transaction_with_spinner_and_config(&tx)
// .await?;
// stats.add_tx(&tx);
// }
let deactivate_txs_to_run = package_instructions(
&deactivate_delinquent_ixs,
1,
Expand Down Expand Up @@ -493,11 +503,7 @@ async fn _handle_delinquent_validators(
let bad_vote_accounts = checks
.iter()
.filter_map(|(vote_account, check)| {
if !check.has_history
|| !check.has_stake_account
|| check.is_deactivated
|| !check.has_vote_account
{
if !check.has_history || check.is_deactivated || !check.has_vote_account {
Some(*vote_account)
} else {
None
Expand Down
4 changes: 3 additions & 1 deletion keepers/validator-keeper/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -249,14 +249,14 @@ async fn run_keeper(keeper_config: KeeperConfig) {
// ---------------------- EMIT ---------------------------------

if should_fire(tick, metrics_interval) {
info!("Emitting metrics...");
keeper_state.set_runs_errors_and_txs_for_epoch(operations::metrics_emit::fire(
&keeper_config,
&keeper_state,
));
}

if should_emit(tick, &intervals) {
info!("Emitting metrics...");
keeper_state.emit();

KeeperOperations::emit(
Expand Down Expand Up @@ -332,6 +332,8 @@ async fn main() {
no_pack: args.no_pack,
pay_for_new_accounts: args.pay_for_new_accounts,
cool_down_range: args.cool_down_range,
tx_retry_count: args.tx_retry_count,
tx_confirmation_seconds: args.tx_confirmation_seconds,
};

run_keeper(config).await;
Expand Down
29 changes: 26 additions & 3 deletions keepers/validator-keeper/src/operations/cluster_history.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,18 @@ async fn _process(
keypair: &Arc<Keypair>,
program_id: &Pubkey,
priority_fee_in_microlamports: u64,
retry_count: u16,
confirmation_time: u64,
) -> Result<SubmitStats, JitoTransactionExecutionError> {
update_cluster_info(client, keypair, program_id, priority_fee_in_microlamports).await
update_cluster_info(
client,
keypair,
program_id,
priority_fee_in_microlamports,
retry_count,
confirmation_time,
)
.await
}

pub async fn fire(
Expand All @@ -52,6 +62,8 @@ pub async fn fire(
let keypair = &keeper_config.keypair;
let program_id = &keeper_config.validator_history_program_id;
let priority_fee_in_microlamports = keeper_config.priority_fee_in_microlamports;
let retry_count = keeper_config.tx_retry_count;
let confirmation_time = keeper_config.tx_confirmation_seconds;

let operation = _get_operation();
let epoch_info = &keeper_state.epoch_info;
Expand All @@ -63,7 +75,16 @@ pub async fn fire(
_should_run(epoch_info, runs_for_epoch) && check_flag(keeper_config.run_flags, operation);

if should_run {
match _process(client, keypair, program_id, priority_fee_in_microlamports).await {
match _process(
client,
keypair,
program_id,
priority_fee_in_microlamports,
retry_count,
confirmation_time,
)
.await
{
Ok(stats) => {
for message in stats.results.iter() {
if let Err(e) = message {
Expand Down Expand Up @@ -126,12 +147,14 @@ pub async fn update_cluster_info(
keypair: &Arc<Keypair>,
program_id: &Pubkey,
priority_fee_in_microlamports: u64,
retry_count: u16,
confirmation_time: u64,
) -> Result<SubmitStats, JitoTransactionExecutionError> {
let ixs = get_update_cluster_info_instructions(
program_id,
&keypair.pubkey(),
priority_fee_in_microlamports,
);

submit_transactions(client, vec![ixs], keypair).await
submit_transactions(client, vec![ixs], keypair, retry_count, confirmation_time).await
}
20 changes: 18 additions & 2 deletions keepers/validator-keeper/src/operations/gossip_upload.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,15 @@ fn _should_run(epoch_info: &EpochInfo, runs_for_epoch: u64) -> bool {
|| (epoch_info.slot_index > epoch_info.slots_in_epoch / 2 && runs_for_epoch < 2)
|| (epoch_info.slot_index > epoch_info.slots_in_epoch * 9 / 10 && runs_for_epoch < 3)
}

async fn _process(
client: &Arc<RpcClient>,
keypair: &Arc<Keypair>,
program_id: &Pubkey,
priority_fee_in_microlamports: u64,
entrypoint: &SocketAddr,
keeper_state: &KeeperState,
retry_count: u16,
confirmation_time: u64,
) -> Result<SubmitStats, Box<dyn std::error::Error>> {
upload_gossip_values(
client,
Expand All @@ -60,6 +61,8 @@ async fn _process(
priority_fee_in_microlamports,
entrypoint,
keeper_state,
retry_count,
confirmation_time,
)
.await
}
Expand All @@ -76,6 +79,8 @@ pub async fn fire(
.expect("Entry point not set");

let priority_fee_in_microlamports = keeper_config.priority_fee_in_microlamports;
let retry_count = keeper_config.tx_retry_count;
let confirmation_time = keeper_config.tx_confirmation_seconds;

let operation = _get_operation();
let (mut runs_for_epoch, mut errors_for_epoch, mut txs_for_epoch) =
Expand All @@ -92,6 +97,8 @@ pub async fn fire(
priority_fee_in_microlamports,
entrypoint,
keeper_state,
retry_count,
confirmation_time,
)
.await
{
Expand Down Expand Up @@ -255,6 +262,8 @@ pub async fn upload_gossip_values(
priority_fee_in_microlamports: u64,
entrypoint: &SocketAddr,
keeper_state: &KeeperState,
retry_count: u16,
confirmation_time: u64,
) -> Result<SubmitStats, Box<dyn std::error::Error>> {
let vote_accounts = keeper_state.vote_account_map.values().collect::<Vec<_>>();
let validator_history_map = &keeper_state.validator_history_map;
Expand Down Expand Up @@ -314,7 +323,14 @@ pub async fn upload_gossip_values(
.map(|entry| entry.build_update_tx(priority_fee_in_microlamports))
.collect::<Vec<_>>();

let submit_result = submit_transactions(client, update_transactions, keypair).await;
let submit_result = submit_transactions(
client,
update_transactions,
keypair,
retry_count,
confirmation_time,
)
.await;

submit_result.map_err(|e| e.into())
}
Expand Down
4 changes: 2 additions & 2 deletions keepers/validator-keeper/src/operations/metrics_emit.rs
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,8 @@ pub fn emit_validator_history_metrics(
let mut cluster_history_blocks: i64 = 0;
let cluster_history_entry = cluster_history.history.last();
if let Some(cluster_history) = cluster_history_entry {
// Looking for previous epoch to be updated
if cluster_history.epoch as u64 == epoch_info.epoch - 1 {
// Looking for current epoch to be updated, implies previous is complete as well
if cluster_history.epoch as u64 == epoch_info.epoch {
cluster_history_blocks = 1;
}
}
Expand Down
10 changes: 10 additions & 0 deletions keepers/validator-keeper/src/operations/mev_commission.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ async fn _process(
program_id: &Pubkey,
tip_distribution_program_id: &Pubkey,
keeper_state: &KeeperState,
retry_count: u16,
confirmation_time: u64,
priority_fee_in_microlamports: u64,
no_pack: bool,
) -> Result<SubmitStats, JitoTransactionError> {
Expand All @@ -47,6 +49,8 @@ async fn _process(
program_id,
tip_distribution_program_id,
keeper_state,
retry_count,
confirmation_time,
priority_fee_in_microlamports,
no_pack,
)
Expand Down Expand Up @@ -76,6 +80,8 @@ pub async fn fire(
program_id,
tip_distribution_program_id,
keeper_state,
keeper_config.tx_retry_count,
keeper_config.tx_confirmation_seconds,
priority_fee_in_microlamports,
keeper_config.no_pack,
)
Expand Down Expand Up @@ -112,6 +118,8 @@ pub async fn update_mev_commission(
program_id: &Pubkey,
tip_distribution_program_id: &Pubkey,
keeper_state: &KeeperState,
retry_count: u16,
confirmation_time: u64,
priority_fee_in_microlamports: u64,
no_pack: bool,
) -> Result<SubmitStats, JitoTransactionError> {
Expand Down Expand Up @@ -148,6 +156,8 @@ pub async fn update_mev_commission(
update_instructions,
keypair,
priority_fee_in_microlamports,
retry_count,
confirmation_time,
None,
no_pack,
)
Expand Down
10 changes: 10 additions & 0 deletions keepers/validator-keeper/src/operations/mev_earned.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ async fn _process(
program_id: &Pubkey,
tip_distribution_program_id: &Pubkey,
priority_fee_in_microlamports: u64,
retry_count: u16,
confirmation_time: u64,
keeper_state: &KeeperState,
no_pack: bool,
) -> Result<SubmitStats, JitoTransactionError> {
Expand All @@ -48,6 +50,8 @@ async fn _process(
keypair,
program_id,
priority_fee_in_microlamports,
retry_count,
confirmation_time,
tip_distribution_program_id,
keeper_state,
no_pack,
Expand Down Expand Up @@ -78,6 +82,8 @@ pub async fn fire(
program_id,
tip_distribution_program_id,
priority_fee_in_microlamports,
keeper_config.tx_retry_count,
keeper_config.tx_confirmation_seconds,
keeper_state,
keeper_config.no_pack,
)
Expand Down Expand Up @@ -113,6 +119,8 @@ pub async fn update_mev_earned(
keypair: &Arc<Keypair>,
program_id: &Pubkey,
priority_fee_in_microlamports: u64,
retry_count: u16,
confirmation_time: u64,
tip_distribution_program_id: &Pubkey,
keeper_state: &KeeperState,
no_pack: bool,
Expand Down Expand Up @@ -165,6 +173,8 @@ pub async fn update_mev_earned(
update_instructions,
keypair,
priority_fee_in_microlamports,
retry_count,
confirmation_time,
None,
no_pack,
)
Expand Down
Loading