v1.16: Adds --no-skip-initial-accounts-db-clean *hidden* CLI flag (…

…backport of #33664) (#33676)

* Adds `--no-skip-initial-accounts-db-clean` *hidden* CLI flag (#33664)

(cherry picked from commit 452fd5d)

# Conflicts:
#	runtime/src/bank/serde_snapshot.rs
#	runtime/src/snapshot_bank_utils.rs

* fix backport conflicts/issues

---------

Co-authored-by: Brooks <[email protected]>

core/src/validator.rs

@@ -236,6 +236,7 @@ pub struct ValidatorConfig {
     pub warp_slot: Option<Slot>,
     pub accounts_db_test_hash_calculation: bool,
     pub accounts_db_skip_shrink: bool,
+    pub accounts_db_force_initial_clean: bool,
     pub tpu_coalesce: Duration,
     pub staked_nodes_overrides: Arc<RwLock<HashMap<Pubkey, u64>>>,
     pub validator_exit: Arc<RwLock<Exit>>,
@@ -301,6 +302,7 @@ impl Default for ValidatorConfig {
             warp_slot: None,
             accounts_db_test_hash_calculation: false,
             accounts_db_skip_shrink: false,
+            accounts_db_force_initial_clean: false,
             tpu_coalesce: DEFAULT_TPU_COALESCE,
             staked_nodes_overrides: Arc::new(RwLock::new(HashMap::new())),
             validator_exit: Arc::new(RwLock::new(Exit::default())),
@@ -1625,6 +1627,7 @@ fn load_blockstore(
         shrink_ratio: config.accounts_shrink_ratio,
         accounts_db_test_hash_calculation: config.accounts_db_test_hash_calculation,
         accounts_db_skip_shrink: config.accounts_db_skip_shrink,
+        accounts_db_force_initial_clean: config.accounts_db_force_initial_clean,
         runtime_config: config.runtime_config.clone(),
         ..blockstore_processor::ProcessOptions::default()
     };

core/tests/epoch_accounts_hash.rs

@@ -452,6 +452,7 @@ fn test_snapshots_have_expected_epoch_accounts_hash() {
                 AccountShrinkThreshold::default(),
                 true,
                 true,
+                false,
                 true,
                 None,
                 None,

core/tests/snapshots.rs

@@ -177,6 +177,7 @@ fn restore_from_snapshot(
         check_hash_calculation,
         false,
         false,
+        false,
         Some(ACCOUNTS_DB_CONFIG_FOR_TESTING),
         None,
         &Arc::default(),
@@ -893,6 +894,7 @@ fn restore_from_snapshots_and_check_banks_are_equal(
         false,
         false,
         false,
+        false,
         Some(ACCOUNTS_DB_CONFIG_FOR_TESTING),
         None,
         &Arc::default(),
@@ -1112,6 +1114,7 @@ fn test_snapshots_with_background_services(
         false,
         false,
         false,
+        false,
         Some(ACCOUNTS_DB_CONFIG_FOR_TESTING),
         None,
         &exit,

ledger/src/bank_forks_utils.rs

@@ -225,6 +225,7 @@ fn bank_forks_from_snapshot(
             process_options.shrink_ratio,
             process_options.accounts_db_test_hash_calculation,
             process_options.accounts_db_skip_shrink,
+            process_options.accounts_db_force_initial_clean,
             process_options.verify_index,
             process_options.accounts_db_config.clone(),
             accounts_update_notifier,

ledger/src/blockstore_processor.rs

@@ -624,6 +624,7 @@ pub struct ProcessOptions {
     pub allow_dead_slots: bool,
     pub accounts_db_test_hash_calculation: bool,
     pub accounts_db_skip_shrink: bool,
+    pub accounts_db_force_initial_clean: bool,
     pub accounts_db_config: Option<AccountsDbConfig>,
     pub verify_index: bool,
     pub shrink_ratio: AccountShrinkThreshold,

local-cluster/src/validator_configs.rs

@@ -51,6 +51,7 @@ pub fn safe_clone_config(config: &ValidatorConfig) -> ValidatorConfig {
         warp_slot: config.warp_slot,
         accounts_db_test_hash_calculation: config.accounts_db_test_hash_calculation,
         accounts_db_skip_shrink: config.accounts_db_skip_shrink,
+        accounts_db_force_initial_clean: config.accounts_db_force_initial_clean,
         tpu_coalesce: config.tpu_coalesce,
         staked_nodes_overrides: config.staked_nodes_overrides.clone(),
         validator_exit: Arc::new(RwLock::new(Exit::default())),

runtime/src/bank.rs

@@ -6935,12 +6935,13 @@ impl Bank {
     pub fn verify_snapshot_bank(
         &self,
         test_hash_calculation: bool,
-        accounts_db_skip_shrink: bool,
+        skip_shrink: bool,
+        force_clean: bool,
         last_full_snapshot_slot: Slot,
         base: Option<(Slot, /*capitalization*/ u64)>,
     ) -> bool {
         let (_, clean_time_us) = measure_us!({
-            let should_clean = !accounts_db_skip_shrink && self.slot() > 0;
+            let should_clean = force_clean || (!skip_shrink && self.slot() > 0);
             if should_clean {
                 info!("Cleaning...");
                 // We cannot clean past the last full snapshot's slot because we are about to
@@ -6960,7 +6961,7 @@ impl Bank {
         });
 
         let (_, shrink_time_us) = measure_us!({
-            let should_shrink = !accounts_db_skip_shrink && self.slot() > 0;
+            let should_shrink = !skip_shrink && self.slot() > 0;
             if should_shrink {
                 info!("Shrinking...");
                 self.rc.accounts.accounts_db.shrink_all_slots(

runtime/src/bank/tests.rs

@@ -3726,11 +3726,11 @@ fn test_verify_snapshot_bank() {
     bank.freeze();
     add_root_and_flush_write_cache(&bank);
     bank.update_accounts_hash_for_tests();
-    assert!(bank.verify_snapshot_bank(true, false, bank.slot(), None));
+    assert!(bank.verify_snapshot_bank(true, false, false, bank.slot(), None));
 
     // tamper the bank after freeze!
     bank.increment_signature_count(1);
-    assert!(!bank.verify_snapshot_bank(true, false, bank.slot(), None));
+    assert!(!bank.verify_snapshot_bank(true, false, false, bank.slot(), None));
 }
 
 // Test that two bank forks with the same accounts should not hash to the same value.

runtime/src/serde_snapshot/tests.rs

@@ -630,6 +630,7 @@ fn test_extra_fields_full_snapshot_archive() {
         false,
         false,
         false,
+        false,
         Some(crate::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING),
         None,
         &Arc::default(),

runtime/src/snapshot_utils.rs

@@ -1535,6 +1535,7 @@ pub fn bank_from_snapshot_archives(
     shrink_ratio: AccountShrinkThreshold,
     test_hash_calculation: bool,
     accounts_db_skip_shrink: bool,
+    accounts_db_force_initial_clean: bool,
     verify_index: bool,
     accounts_db_config: Option<AccountsDbConfig>,
     accounts_update_notifier: Option<AccountsUpdateNotifier>,
@@ -1617,6 +1618,7 @@ pub fn bank_from_snapshot_archives(
     if !bank.verify_snapshot_bank(
         test_hash_calculation,
         accounts_db_skip_shrink || !full_snapshot_archive_info.is_remote(),
+        accounts_db_force_initial_clean,
         full_snapshot_archive_info.slot(),
         base,
     ) && limit_load_slot_count_from_snapshot.is_none()
@@ -1654,6 +1656,7 @@ pub fn bank_from_latest_snapshot_archives(
     shrink_ratio: AccountShrinkThreshold,
     test_hash_calculation: bool,
     accounts_db_skip_shrink: bool,
+    accounts_db_force_initial_clean: bool,
     verify_index: bool,
     accounts_db_config: Option<AccountsDbConfig>,
     accounts_update_notifier: Option<AccountsUpdateNotifier>,
@@ -1698,6 +1701,7 @@ pub fn bank_from_latest_snapshot_archives(
         shrink_ratio,
         test_hash_calculation,
         accounts_db_skip_shrink,
+        accounts_db_force_initial_clean,
         verify_index,
         accounts_db_config,
         accounts_update_notifier,
@@ -4400,6 +4404,7 @@ mod tests {
             false,
             false,
             false,
+            false,
             Some(ACCOUNTS_DB_CONFIG_FOR_TESTING),
             None,
             &Arc::default(),
@@ -4511,6 +4516,7 @@ mod tests {
             false,
             false,
             false,
+            false,
             Some(ACCOUNTS_DB_CONFIG_FOR_TESTING),
             None,
             &Arc::default(),
@@ -4642,6 +4648,7 @@ mod tests {
             false,
             false,
             false,
+            false,
             Some(ACCOUNTS_DB_CONFIG_FOR_TESTING),
             None,
             &Arc::default(),
@@ -4763,6 +4770,7 @@ mod tests {
             false,
             false,
             false,
+            false,
             Some(ACCOUNTS_DB_CONFIG_FOR_TESTING),
             None,
             &Arc::default(),
@@ -4900,6 +4908,7 @@ mod tests {
             false,
             false,
             false,
+            false,
             Some(ACCOUNTS_DB_CONFIG_FOR_TESTING),
             None,
             &Arc::default(),
@@ -4965,6 +4974,7 @@ mod tests {
             false,
             false,
             false,
+            false,
             Some(ACCOUNTS_DB_CONFIG_FOR_TESTING),
             None,
             &Arc::default(),
@@ -5533,6 +5543,7 @@ mod tests {
             false,
             false,
             false,
+            false,
             Some(ACCOUNTS_DB_CONFIG_FOR_TESTING),
             None,
             &Arc::default(),

validator/src/cli.rs

@@ -1197,6 +1197,13 @@ pub fn app<'a>(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> {
                 .help("Debug option to scan all append vecs and verify account index refcounts prior to clean")
                 .hidden(hidden_unless_forced())
         )
+        .arg(
+            Arg::with_name("no_skip_initial_accounts_db_clean")
+                .long("no-skip-initial-accounts-db-clean")
+                .help("Do not skip the initial cleaning of accounts when verifying snapshot bank")
+                .hidden(hidden_unless_forced())
+                .conflicts_with("accounts_db_skip_shrink")
+        )
         .arg(
             Arg::with_name("accounts_db_create_ancient_storage_packed")
                 .long("accounts-db-create-ancient-storage-packed")

validator/src/main.rs

@@ -1377,6 +1377,7 @@ pub fn main() {
         accounts_db_test_hash_calculation: matches.is_present("accounts_db_test_hash_calculation"),
         accounts_db_config,
         accounts_db_skip_shrink: true,
+        accounts_db_force_initial_clean: matches.is_present("no_skip_initial_accounts_db_clean"),
         tpu_coalesce,
         no_wait_for_vote_to_start_leader: matches.is_present("no_wait_for_vote_to_start_leader"),
         accounts_shrink_ratio,