add files

tip-distributor/Cargo.toml

@@ -7,29 +7,32 @@ description = "Collection of binaries used to distribute MEV rewards to delegato
 
 [dependencies]
 anchor-lang = { workspace = true }
-clap = { version = "=4.1.11", features = ["derive", "env"] }
-env_logger = "0.9.0"
-futures = "0.3.21"
-im = "15.1.0"
-itertools = "0.10.3"
+async-channel = { workspace = true }
+clap = { version = "4.1.11", features = ["derive", "env"] }
+env_logger = { workspace = true }
+futures = { workspace = true }
+im = { workspace = true }
+itertools = { workspace = true }
 jito-tip-distribution = { workspace = true }
 jito-tip-payment = { workspace = true }
-log = "0.4.17"
-num-traits = "0.2.15"
-serde = "1.0.137"
-serde_json = "1.0.81"
+log = { workspace = true }
+num-traits = { workspace = true }
+rand = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
 solana-client = { workspace = true }
 solana-genesis-utils = { workspace = true }
 solana-ledger = { workspace = true }
 solana-merkle-tree = { workspace = true }
 solana-metrics = { workspace = true }
 solana-program = { workspace = true }
+solana-program-runtime = { workspace = true }
 solana-rpc-client-api = { workspace = true }
 solana-runtime = { workspace = true }
 solana-sdk = { workspace = true }
 solana-stake-program = { workspace = true }
 thiserror = { workspace = true }
-tokio = { workspace = true, features = ["rt-multi-thread", "macros", "sync", "time", "full"] }
+tokio = { workspace = true, features = ["rt-multi-thread", "macros", "sync", "time"] }
 
 [[bin]]
 name = "solana-stake-meta-generator"

tip-distributor/README.md

@@ -27,8 +27,15 @@ out into the PDA until some slot in epoch N + 1. Due to this we cannot rely on t
 in the PDAs. We use the bank solely to take a snapshot of delegations, but an RPC node to fetch the PDA lamports for more up-to-date data.
 
 ### merkle-root-generator
-This script accepts a path to the above JSON file as one of its arguments, and generates a merkle-root. It'll optionally upload the root
-on-chain if specified. Additionally, it'll spit the generated merkle trees out into a JSON file.
+This script accepts a path to the above JSON file as one of its arguments, and generates a merkle-root into a JSON file.
+
+### merkle-root-uploader
+Uploads the root on-chain.
+
+### claim-mev-tips
+This reads the file outputted by `merkle-root-generator` and finds all eligible accounts to receive mev tips. Transactions
+are created and sent to the RPC server.
+
 
 ## How it works?
 In order to use this library as the merkle root creator one must follow the following steps:
@@ -38,6 +45,8 @@ In order to use this library as the merkle root creator one must follow the foll
    1. The snapshot created at `${WHERE_TO_CREATE_SNAPSHOT}` will have the highest slot of `${YOUR_SLOT}`, assuming you downloaded the correct snapshot.
 4. Run `stake-meta-generator --ledger-path ${WHERE_TO_CREATE_SNAPSHOT} --tip-distribution-program-id ${PUBKEY} --out-path ${JSON_OUT_PATH} --snapshot-slot ${SLOT} --rpc-url ${URL}`
    1. Note: `${WHERE_TO_CREATE_SNAPSHOT}` must be the same in steps 3 & 4.
-5. Run `merkle-root-generator --path-to-my-keypair ${KEYPAIR_PATH} --stake-meta-coll-path ${STAKE_META_COLLECTION_JSON} --rpc-url ${URL} --upload-roots ${BOOL} --force-upload-root ${BOOL}`
+5. Run `merkle-root-generator --stake-meta-coll-path ${STAKE_META_COLLECTION_JSON} --rpc-url ${URL} --out-path ${MERKLE_ROOT_PATH}`
+6. Run `merkle-root-uploader --out-path ${MERKLE_ROOT_PATH} --keypair-path ${KEYPAIR_PATH} --rpc-url ${URL} --tip-distribution-program-id ${PROGRAM_ID}`
+7. Run `solana-claim-mev-tips --merkle-trees-path /solana/ledger/autosnapshot/merkle-tree-221615999.json --rpc-url ${URL} --tip-distribution-program-id ${PROGRAM_ID} --keypair-path ${KEYPAIR_PATH}`
 
 Voila!

tip-distributor/src/bin/claim-mev-tips.rs

@@ -3,9 +3,13 @@
 use {
     clap::Parser,
     log::*,
+    solana_metrics::{datapoint_error, datapoint_info},
     solana_sdk::pubkey::Pubkey,
     solana_tip_distributor::claim_mev_workflow::claim_mev_tips,
-    std::{path::PathBuf, str::FromStr},
+    std::{
+        path::PathBuf,
+        time::{Duration, Instant},
+    },
 };
 
 #[derive(Parser, Debug)]
@@ -16,37 +20,71 @@ struct Args {
     merkle_trees_path: PathBuf,
 
     /// RPC to send transactions through
-    #[arg(long, env)]
+    #[arg(long, env, default_value = "http://localhost:8899")]
     rpc_url: String,
 
     /// Tip distribution program ID
     #[arg(long, env)]
-    tip_distribution_program_id: String,
+    tip_distribution_program_id: Pubkey,
 
     /// Path to keypair
     #[arg(long, env)]
     keypair_path: PathBuf,
+
+    /// Number of unique connections to the RPC server for sending txns
+    #[arg(long, env, default_value_t = 100)]
+    rpc_send_connection_count: u64,
+
+    /// Rate-limits the maximum number of requests per RPC connection
+    #[arg(long, env, default_value_t = 200)]
+    max_concurrent_rpc_get_reqs: usize,
+
+    #[arg(long, env, default_value_t = 5)]
+    max_loop_retries: u64,
+
+    /// Limits how long before send loop runs before stopping. Defaults to 10 mins
+    #[arg(long, env, default_value_t = 10*60)]
+    max_loop_duration_secs: u64,
 }
 
-fn main() {
+#[tokio::main]
+async fn main() {
     env_logger::init();
-    info!("Starting to claim mev tips...");
-
     let args: Args = Args::parse();
+    info!("Starting to claim mev tips...");
+    let start = Instant::now();
 
-    let tip_distribution_program_id = Pubkey::from_str(&args.tip_distribution_program_id)
-        .expect("valid tip_distribution_program_id");
-
-    if let Err(e) = claim_mev_tips(
+    match claim_mev_tips(
         &args.merkle_trees_path,
-        &args.rpc_url,
-        &tip_distribution_program_id,
+        args.rpc_url,
+        args.rpc_send_connection_count,
+        args.max_concurrent_rpc_get_reqs,
+        &args.tip_distribution_program_id,
         &args.keypair_path,
-    ) {
-        panic!("error claiming mev tips: {:?}", e);
+        args.max_loop_retries,
+        Duration::from_secs(args.max_loop_duration_secs),
+    )
+    .await
+    {
+        Err(e) => datapoint_error!(
+            "claim_mev_workflow-claim_error",
+            ("error", 1, i64),
+            ("err_str", e.to_string(), String),
+            (
+                "merkle_trees_path",
+                args.merkle_trees_path.to_string_lossy(),
+                String
+            ),
+            ("latency_us", start.elapsed().as_micros(), i64),
+        ),
+        Ok(()) => datapoint_info!(
+            "claim_mev_workflow-claim_completion",
+            (
+                "merkle_trees_path",
+                args.merkle_trees_path.to_string_lossy(),
+                String
+            ),
+            ("latency_us", start.elapsed().as_micros(), i64),
+        ),
     }
-    info!(
-        "done claiming mev tips from file {:?}",
-        args.merkle_trees_path
-    );
 }

tip-distributor/src/bin/merkle-root-uploader.rs

@@ -1,9 +1,6 @@
 use {
-    clap::Parser,
-    log::info,
-    solana_sdk::pubkey::Pubkey,
-    solana_tip_distributor::merkle_root_upload_workflow::upload_merkle_root,
-    std::{path::PathBuf, str::FromStr},
+    clap::Parser, log::info, solana_sdk::pubkey::Pubkey,
+    solana_tip_distributor::merkle_root_upload_workflow::upload_merkle_root, std::path::PathBuf,
 };
 
 #[derive(Parser, Debug)]
@@ -23,23 +20,30 @@ struct Args {
 
     /// Tip distribution program ID
     #[arg(long, env)]
-    tip_distribution_program_id: String,
+    tip_distribution_program_id: Pubkey,
+
+    /// Rate-limits the maximum number of requests per RPC connection
+    #[arg(long, env, default_value_t = 100)]
+    max_concurrent_rpc_get_reqs: usize,
+
+    /// Number of transactions to send to RPC at a time.
+    #[arg(long, env, default_value_t = 64)]
+    txn_send_batch_size: usize,
 }
 
 fn main() {
     env_logger::init();
 
     let args: Args = Args::parse();
 
-    let tip_distribution_program_id = Pubkey::from_str(&args.tip_distribution_program_id)
-        .expect("valid tip_distribution_program_id");
-
     info!("starting merkle root uploader...");
     if let Err(e) = upload_merkle_root(
         &args.merkle_root_path,
         &args.keypair_path,
         &args.rpc_url,
-        &tip_distribution_program_id,
+        &args.tip_distribution_program_id,
+        args.max_concurrent_rpc_get_reqs,
+        args.txn_send_batch_size,
     ) {
         panic!("failed to upload merkle roots: {:?}", e);
     }

tip-distributor/src/bin/reclaim-rent.rs

@@ -8,7 +8,7 @@ use {
         commitment_config::CommitmentConfig, pubkey::Pubkey, signature::read_keypair_file,
     },
     solana_tip_distributor::reclaim_rent_workflow::reclaim_rent,
-    std::{path::PathBuf, str::FromStr, time::Duration},
+    std::{path::PathBuf, time::Duration},
     tokio::runtime::Runtime,
 };
 
@@ -22,7 +22,7 @@ struct Args {
     rpc_url: String,
 
     /// Tip distribution program ID.
-    #[arg(long, env, value_parser = Pubkey::from_str)]
+    #[arg(long, env)]
     tip_distribution_program_id: Pubkey,
 
     /// The keypair signing and paying for transactions.
@@ -33,6 +33,14 @@ struct Args {
     #[arg(long, env, default_value_t = 180)]
     rpc_timeout_secs: u64,
 
+    /// Rate-limits the maximum number of requests per RPC connection
+    #[arg(long, env, default_value_t = 100)]
+    max_concurrent_rpc_get_reqs: usize,
+
+    /// Number of transactions to send to RPC at a time.
+    #[arg(long, env, default_value_t = 64)]
+    txn_send_batch_size: usize,
+
     /// Specifies whether to reclaim rent on behalf of validators from respective TDAs.
     #[arg(long, env)]
     should_reclaim_tdas: bool,
@@ -41,7 +49,7 @@ struct Args {
 fn main() {
     env_logger::init();
 
-    info!("Starting to claim mev tips...");
+    info!("Starting to reclaim mev tips...");
     let args: Args = Args::parse();
 
     let runtime = Runtime::new().unwrap();
@@ -53,6 +61,8 @@ fn main() {
         ),
         args.tip_distribution_program_id,
         read_keypair_file(&args.keypair_path).expect("read keypair file"),
+        args.max_concurrent_rpc_get_reqs,
+        args.txn_send_batch_size,
         args.should_reclaim_tdas,
     )) {
         panic!("error reclaiming rent: {e:?}");