From c256b975916113c056259431a329dc46c092367d Mon Sep 17 00:00:00 2001 From: Lucas B Date: Thu, 25 Aug 2022 17:18:46 -0500 Subject: [PATCH] jito patch only reroute if relayer connected (#123) feat: add client tls config (#121) remove extra val (#129) fix clippy (#130) copy all binaries to docker-output (#131) Ledger tool halts at slot passed to create-snapshot (#118) update program submodule (#133) quick fix for tips and clearing old bundles (#135) update submodule to new program (#136) Improve stake-meta-generator usability (#134) pinning submodule head (#140) Use BundleAccountLocker when handling tip txs (#147) Add metrics for relayer + block engine proxy (#149) Build claim-mev in docker (#141) Rework bundle receiving and add metrics (#152) (#154) update submodule + dev files (#158) Deterministically find tip amounts, add meta to stake info, and cleanup pubkey/strings in MEV tips (#159) update jito-programs submodule (#160) Separate MEV tip related workflow (#161) Add block builder fee protos (#162) fix jito programs (#163) update submodule so autosnapshot exits out of ledger tool early (#164) Pipe through block builder fee (#167) pull in new snapshot code (#171) block builder bug (#172) Pull in new slack autosnapshot submodule (#174) sort stake meta json and use int math (#176) add accountsdb conn submod (#169) Update tip distribution parameters (#177) new submodules (#180) Add buildkite link for jito CI (#183) Fixed broken links to repositories (#184) Changed from ssh to https transfer for clone Seg/update submods (#187) fix tests (#190) rm geyser submod (#192) rm dangling geyser references (#193) fix syntax err (#195) use deterministic req ids in batch calls (#199) update jito-programs revert cargo update Cargo lock update with path fix fix cargo update autosnapshot with block lookback (#201) [JIT-460] When claiming mev tips, skip accounts that won't have min rent exempt amount after claiming (#203) Add logging for sol balance desired (#205) * add logging * add logging * update msg * tweak vars update submodule (#204) use efficient data structures when calling batch_simulate_bundles (#206) [JIT-504] Add low balance check in uploading merkle roots (#209) add config to simulate on top of working bank (#211) rm frozen bank check simulate_bundle rpc bugfixes (#214) rm frozen bank check in simulate_bundle rpc method [JIT-519] Store ClaimStatus address in merkle-root-json (#210) * add files * switch to include bump update submodule (#217) add amount filter (#218) update autosnapshot (#222) Print TX error in Bundles (#223) add new args to support single relayer and block-engine endpoints (#224) point to new jito-programs submod and invoke updated init tda instruction (#228) fix clippy errors (#230) fix validator start scripts (#232) Point README to gitbook (#237) use packaged cargo bin to build (#239) Add validator identity pubkey to StakeMeta (#226) The vote account associated with a validator is not a permanent link, so log the validator identity as well. bugfix: conditionally compile with debug flags (#240) Seg/tip distributor master (#242) * validate tree nodes * fix unit tests * pr feedback * bump jito-programs submod Simplify bootstrapping (#241) * startup without precompile * update spacing * use release mode * spacing fix validation rm validation skip Account for block builder fee when generating excess tip balance (#247) Improve docker caching delay constructing claim mev txs (#253) fix stake meta tests from bb fee (#254) fix tests Buffer bundles that exceed cost model (#225) * buffer bundles that exceed cost model clear qos failed bundles buffer if not leader soon (#260) update Cargo.lock to correct solana versions in jito-programs submodule (#265) fix simulate_bundle client and better error handling (#267) update submod (#272) Preallocate Bundle Cost (#238) fix Dockerfile (#278) Fix Tests (#279) Fix Tests (#281) * fix tests update jito-programs submod (#282) add reclaim rent workflow (#283) update jito-programs submod fix clippy errs rm wrong assertion and swap out file write fn call (#292) Remove security.md (#293) demote frequent relayer_stage-stream_error to warn (#275) account for case where TDA exists but not allocated (#295) implement better retries for tip-distributor workflows (#297) limit number of concurrent rpc calls (#298) Discard Empty Packet Batches (#299) Identity Hotswap (#290) small fixes (#305) Set backend config from admin rpc (#304) Admin Shred Receiver Change (#306) Seg/rm bundle UUID (#309) Fix github workflow to recursively clone (#327) Add recursive checkout for downstream-project-spl.yaml (#341) Use cluster info functions for tpu (#345) Use git rev-parse for git sha Remove blacklisted tx from message_hash_to_transaction (#374) Updates bootstrap and start scripts needed for local dev. (#384) Remove Deprecated Cli Args (#387) Master Rebase improve simulate_bundle errors and response (#404) derive Clone on accountoverrides (#416) Add upsert to AccountOverrides (#419) update jito-programs (#430) [JIT-1661] Faster Autosnapshot (#436) Reverts simulate_transaction result calls to upstream (#446) Don't unlock accounts in TransactionBatches used during simulation (#449) first pass at wiring up jito-plugin (#428) [JIT-1713] Fix bundle's blockspace preallocation (#489) [JIT-1708] Fix TOC TOU condition for relayer and block engine config (#491) [JIT-1710] - Optimize Bundle Consumer Checks (#490) Add Blockhash Metrics to Bundle Committer (#500) add priority fee ix to mev-claim (#520) Update Autosnapshot (#548) Run MEV claims + reclaiming rent-exempt amounts in parallel. (#582) Update CI (#584) - Add recursive submodule checkouts. - Re-add solana-secondary step Add more release fixes (#585) Fix more release urls (#588) [JIT-1812] Fix blocking mutexs (#495) [JIT-1711] Compare the unprocessed transaction storage BundleStorage against a constant instead of VecDeque::capacity() (#587) Automatically rebase Jito-Solana on a periodic basis. Send message on slack during any failures or success. Fix periodic rebase #594 Fixes the following bugs in the periodic rebase: Sends multiple messages on failure instead of one Cancels entire job if one branch fails Ignore buildkite curl errors for rebasing and try to keep curling until job times out (#597) Sleep longer waiting for buildkite to start (#598) correctly initialize account overrides (#595) Fix: Ensure set contact info to UDP port instead of QUIC (#603) Add fast replay branch to daily rebase (#607) take a snapshot of all bundle accounts before sim (#13) (#615) update jito-programs submodule Export agave binaries during docker build (BP #627) (#628) Backport #611 (#631) Publish releases to S3 and GCS (#633) (#634) Add packet flag for from staked sender (#655) Co-authored-by: Jed <4679729+jedleggett@users.noreply.github.com> Add bundle storage to new unprocessed transaction storage method Loosen tip requirement [v2.0] (#685) Add comments around ignoring the slot returned from ImmutableDeserializedPacket::build_sanitized_transaction Backport: Separate out broadcast + retransmit shredstream (#703) (#704) --- Cargo.lock | 8 +++--- Cargo.toml | 12 ++++----- bundle/src/bundle_execution.rs | 11 +++++--- ci/do-audit.sh | 11 ++++++++ core/src/bundle_stage/bundle_consumer.rs | 32 ++---------------------- core/src/tip_manager.rs | 9 +++++-- programs/sbf/Cargo.lock | 6 ++--- svm/src/account_loader.rs | 26 +++++++++++-------- transaction-status/src/lib.rs | 5 +++- 9 files changed, 60 insertions(+), 60 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 64135fd484..6c33455347 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3249,7 +3249,7 @@ dependencies = [ [[package]] name = "jito-protos" -version = "2.0.18" +version = "2.0.19" dependencies = [ "bytes", "prost", @@ -6085,7 +6085,7 @@ dependencies = [ [[package]] name = "solana-bundle" -version = "2.0.18" +version = "2.0.19" dependencies = [ "anchor-lang", "assert_matches", @@ -7592,7 +7592,7 @@ dependencies = [ [[package]] name = "solana-runtime-plugin" -version = "2.0.18" +version = "2.0.19" dependencies = [ "crossbeam-channel", "json5", @@ -7944,7 +7944,7 @@ dependencies = [ [[package]] name = "solana-tip-distributor" -version = "2.0.18" +version = "2.0.19" dependencies = [ "anchor-lang", "clap 4.3.21", diff --git a/Cargo.toml b/Cargo.toml index 1686a4e0dc..0e174fb422 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -248,13 +248,9 @@ jemallocator = { package = "tikv-jemallocator", version = "0.4.1", features = [ "unprefixed_malloc_on_supported_platforms", ] } js-sys = "0.3.69" -jito-protos = { path = "jito-protos", version = "=2.0.18" } -jito-tip-distribution = { path = "jito-programs/mev-programs/programs/tip-distribution", features = [ - "no-entrypoint", -] } -jito-tip-payment = { path = "jito-programs/mev-programs/programs/tip-payment", features = [ - "no-entrypoint", -] } +jito-protos = { path = "jito-protos", version = "=2.0.19" } +jito-tip-distribution = { path = "jito-programs/mev-programs/programs/tip-distribution", features = ["no-entrypoint"] } +jito-tip-payment = { path = "jito-programs/mev-programs/programs/tip-payment", features = ["no-entrypoint"] } json5 = "0.4.1" jsonrpc-core = "18.0.0" jsonrpc-core-client = "18.0.0" @@ -348,6 +344,7 @@ solana-bench-tps = { path = "bench-tps", version = "=2.0.19" } solana-bloom = { path = "bloom", version = "=2.0.19" } solana-bpf-loader-program = { path = "programs/bpf_loader", version = "=2.0.19" } solana-bucket-map = { path = "bucket_map", version = "=2.0.19" } +solana-bundle = { path = "bundle", version = "=2.0.19" } agave-cargo-registry = { path = "cargo-registry", version = "=2.0.19" } solana-clap-utils = { path = "clap-utils", version = "=2.0.19" } solana-clap-v3-utils = { path = "clap-v3-utils", version = "=2.0.19" } @@ -402,6 +399,7 @@ solana-rpc-client = { path = "rpc-client", version = "=2.0.19", default-features solana-rpc-client-api = { path = "rpc-client-api", version = "=2.0.19" } solana-rpc-client-nonce-utils = { path = "rpc-client-nonce-utils", version = "=2.0.19" } solana-runtime = { path = "runtime", version = "=2.0.19" } +solana-runtime-plugin = { path = "runtime-plugin", version = "=2.0.19" } solana-runtime-transaction = { path = "runtime-transaction", version = "=2.0.19" } solana-sdk = { path = "sdk", version = "=2.0.19" } solana-sdk-macro = { path = "sdk/macro", version = "=2.0.19" } diff --git a/bundle/src/bundle_execution.rs b/bundle/src/bundle_execution.rs index 6e396c8513..d78d9eff8e 100644 --- a/bundle/src/bundle_execution.rs +++ b/bundle/src/bundle_execution.rs @@ -10,7 +10,7 @@ use { transaction_batch::TransactionBatch, }, solana_sdk::{ - account::AccountSharedData, + account::{AccountSharedData, ReadableAccount}, bundle::SanitizedBundle, nonce::state::DurableNonce, pubkey::Pubkey, @@ -28,6 +28,7 @@ use { solana_transaction_status::{token_balances::TransactionTokenBalances, PreBalanceInfo}, std::{ cmp::{max, min}, + sync::Arc, time::{Duration, Instant}, }, thiserror::Error, @@ -224,7 +225,7 @@ pub fn check_bundle_execution_results<'a>( /// we should add in the BundleTransactionsOutput of A and C and return the error for B. #[allow(clippy::too_many_arguments)] pub fn load_and_execute_bundle<'a>( - bank: &Bank, + bank: &Arc, bundle: &'a SanitizedBundle, // Max blockhash age max_age: usize, @@ -450,7 +451,11 @@ pub fn load_and_execute_bundle<'a>( ) .0; for (pubkey, data) in accounts { - account_overrides.set_account(pubkey, Some(data.clone())); + if data.lamports() == 0 { + account_overrides.set_account(pubkey, Some(AccountSharedData::default())); + } else { + account_overrides.set_account(pubkey, Some(data.clone())); + } } saturating_add_assign!(metrics.cache_accounts_us, m.end_as_us()); diff --git a/ci/do-audit.sh b/ci/do-audit.sh index aca50d122f..0e63dcef22 100755 --- a/ci/do-audit.sh +++ b/ci/do-audit.sh @@ -34,6 +34,17 @@ cargo_audit_ignores=( # curve25519-dalek --ignore RUSTSEC-2024-0344 + # Crate: idna + # Version: 0.1.5 + # Title: `idna` accepts Punycode labels that do not produce any non-ASCII when decoded + # Date: 2024-12-09 + # ID: RUSTSEC-2024-0421 + # URL: https://rustsec.org/advisories/RUSTSEC-2024-0421 + # Solution: Upgrade to >=1.0.0 + # need to solve this depentant tree: + # jsonrpc-core-client v18.0.0 -> jsonrpc-client-transports v18.0.0 -> url v1.7.2 -> idna v0.1.5 + --ignore RUSTSEC-2024-0421 + # tonic # When using tonic::transport::Server there is a remote DoS attack that can cause # the server to exit cleanly on accepting a tcp/tls stream. diff --git a/core/src/bundle_stage/bundle_consumer.rs b/core/src/bundle_stage/bundle_consumer.rs index b8d71edaf6..32b9327081 100644 --- a/core/src/bundle_stage/bundle_consumer.rs +++ b/core/src/bundle_stage/bundle_consumer.rs @@ -11,7 +11,6 @@ use { bundle_stage_leader_metrics::BundleStageLeaderMetrics, committer::Committer, }, - consensus_cache_updater::ConsensusCacheUpdater, immutable_deserialized_bundle::ImmutableDeserializedBundle, proxy::block_engine_stage::BlockBuilderFeeInfo, tip_manager::TipManager, @@ -53,8 +52,6 @@ pub struct BundleConsumer { qos_service: QosService, log_messages_bytes_limit: Option, - consensus_cache_updater: ConsensusCacheUpdater, - tip_manager: TipManager, last_tip_update_slot: Slot, @@ -87,16 +84,16 @@ impl BundleConsumer { cluster_info: Arc, reserved_space: BundleReservedSpaceManager, ) -> Self { + let blacklisted_accounts = HashSet::from_iter([tip_manager.tip_payment_program_id()]); Self { committer, transaction_recorder, qos_service, log_messages_bytes_limit, - consensus_cache_updater: ConsensusCacheUpdater::default(), tip_manager, // MAX because sending tips during slot 0 in tests doesn't work last_tip_update_slot: u64::MAX, - blacklisted_accounts: HashSet::default(), + blacklisted_accounts, bundle_account_locker, block_builder_fee_info, max_bundle_retry_duration, @@ -120,15 +117,12 @@ impl BundleConsumer { // A bundle is not allowed to call the Tip Payment program in a bundle (or BankingStage). // - This is to avoid stealing of tips by malicious parties with bundles that crank the tip // payment program and set the tip receiver to themself. - // A bundle is not allowed to touch consensus-related accounts - // - This is to avoid stalling the voting BankingStage threads. pub fn consume_buffered_bundles( &mut self, bank_start: &BankStart, unprocessed_transaction_storage: &mut UnprocessedTransactionStorage, bundle_stage_leader_metrics: &mut BundleStageLeaderMetrics, ) { - self.maybe_update_blacklist(bank_start); self.reserved_space.tick(&bank_start.working_bank); let reached_end_of_slot = unprocessed_transaction_storage.process_bundles( @@ -164,28 +158,6 @@ impl BundleConsumer { } } - /// Blacklist is updated with the tip payment program + any consensus accounts. - fn maybe_update_blacklist(&mut self, bank_start: &BankStart) { - if self - .consensus_cache_updater - .maybe_update(&bank_start.working_bank) - { - self.blacklisted_accounts = self - .consensus_cache_updater - .consensus_accounts_cache() - .union(&HashSet::from_iter([self - .tip_manager - .tip_payment_program_id()])) - .cloned() - .collect(); - - debug!( - "updated blacklist with {} accounts", - self.blacklisted_accounts.len() - ); - } - } - #[allow(clippy::too_many_arguments)] fn do_process_bundles( bundle_account_locker: &BundleAccountLocker, diff --git a/core/src/tip_manager.rs b/core/src/tip_manager.rs index 397dfffea1..847d90321a 100644 --- a/core/src/tip_manager.rs +++ b/core/src/tip_manager.rs @@ -548,9 +548,14 @@ impl TipManager { None }; - let configured_tip_receiver = self.get_configured_tip_receiver(bank)?; + let tip_payment_config = self.get_tip_payment_config_account(bank)?; + let my_tip_receiver = self.get_my_tip_distribution_pda(bank.epoch()); - let maybe_change_tip_receiver_tx = if configured_tip_receiver != my_tip_receiver { + let maybe_change_tip_receiver_tx = if tip_payment_config.tip_receiver != my_tip_receiver + || tip_payment_config.block_builder != block_builder_fee_info.block_builder + || tip_payment_config.block_builder_commission_pct + != block_builder_fee_info.block_builder_commission + { debug!("change_tip_receiver=true"); Some(self.change_tip_receiver_and_block_builder_tx( &my_tip_receiver, diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index fb5ef32472..752b87dd09 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -2552,7 +2552,7 @@ dependencies = [ [[package]] name = "jito-protos" -version = "2.0.18" +version = "2.0.19" dependencies = [ "bytes", "prost", @@ -4955,7 +4955,7 @@ dependencies = [ [[package]] name = "solana-bundle" -version = "2.0.18" +version = "2.0.19" dependencies = [ "anchor-lang", "itertools 0.12.1", @@ -5936,7 +5936,7 @@ dependencies = [ [[package]] name = "solana-runtime-plugin" -version = "2.0.18" +version = "2.0.19" dependencies = [ "crossbeam-channel", "json5", diff --git a/svm/src/account_loader.rs b/svm/src/account_loader.rs index 8c58736aa0..00e89b3ebd 100644 --- a/svm/src/account_loader.rs +++ b/svm/src/account_loader.rs @@ -190,6 +190,15 @@ pub(crate) fn load_accounts( .collect() } +fn create_new_account() -> AccountSharedData { + let mut default_account = AccountSharedData::default(); + // All new accounts must be rent-exempt (enforced in Bank::execute_loaded_transaction). + // Currently, rent collection sets rent_epoch to u64::MAX, but initializing the account + // with this field already set would allow us to skip rent collection for these accounts. + default_account.set_rent_epoch(RENT_EXEMPT_RENT_EPOCH); + default_account +} + fn load_transaction_accounts( callbacks: &CB, message: &SanitizedMessage, @@ -241,11 +250,12 @@ fn load_transaction_accounts( } else if let Some(account_override) = account_overrides.and_then(|overrides| overrides.get(key)) { - info!( - "loaded account from cache key: {:?} override: {:?}", - key, account_override - ); - (account_override.data().len(), account_override.clone(), 0) + if account_override.lamports() == 0 { + let default_account = create_new_account(); + (default_account.data().len(), default_account, 0) + } else { + (account_override.data().len(), account_override.clone(), 0) + } } else if let Some(program) = (!disable_account_loader_special_case && !instruction_account && !message.is_writable(i)) @@ -276,11 +286,7 @@ fn load_transaction_accounts( }) .unwrap_or_else(|| { account_found = false; - let mut default_account = AccountSharedData::default(); - // All new accounts must be rent-exempt (enforced in Bank::execute_loaded_transaction). - // Currently, rent collection sets rent_epoch to u64::MAX, but initializing the account - // with this field already set would allow us to skip rent collection for these accounts. - default_account.set_rent_epoch(RENT_EXEMPT_RENT_EPOCH); + let default_account = create_new_account(); (default_account.data().len(), default_account, 0) }) }; diff --git a/transaction-status/src/lib.rs b/transaction-status/src/lib.rs index 27311f46e9..b3cce3c558 100644 --- a/transaction-status/src/lib.rs +++ b/transaction-status/src/lib.rs @@ -27,7 +27,10 @@ use { }, transaction_context::TransactionReturnData, }, - std::{collections::HashSet, fmt}, + std::{ + collections::{HashMap, HashSet}, + fmt, + }, thiserror::Error, };