Add BankWithScheduler for upcoming scheduler code (solana-labs#33704)

* Add BankWithScheduler for upcoming scheduler code

* Remove too confusing insert_without_scheduler()

* Add doc comment as a bonus

* Simplify BankForks::banks()

* Add derive(Debug) on BankWithScheduler

core/benches/banking_stage.rs

@@ -37,7 +37,8 @@ use {
     },
     solana_poh::poh_recorder::{create_test_recorder, WorkingBankEntry},
     solana_runtime::{
-        bank::Bank, bank_forks::BankForks, prioritization_fee_cache::PrioritizationFeeCache,
+        bank::Bank, bank_forks::BankForks, installed_scheduler_pool::BankWithScheduler,
+        prioritization_fee_cache::PrioritizationFeeCache,
     },
     solana_sdk::{
         genesis_config::GenesisConfig,
@@ -398,6 +399,7 @@ fn simulate_process_entries(
     num_accounts: usize,
 ) {
     let bank = Arc::new(Bank::new_for_benches(genesis_config));
+    let bank = BankWithScheduler::new_without_scheduler(bank);
 
     for i in 0..(num_accounts / 2) {
         bank.transfer(initial_lamports, mint_keypair, &keypairs[i * 2].pubkey())

core/src/replay_stage.rs

@@ -61,6 +61,7 @@ use {
         bank::{bank_hash_details, Bank, NewBankOptions},
         bank_forks::{BankForks, MAX_ROOT_DISTANCE_FOR_VOTE_ONLY},
         commitment::BlockCommitmentCache,
+        installed_scheduler_pool::BankWithScheduler,
         prioritization_fee_cache::PrioritizationFeeCache,
     },
     solana_sdk::{
@@ -1988,7 +1989,7 @@ impl ReplayStage {
 
     #[allow(clippy::too_many_arguments)]
     fn replay_blockstore_into_bank(
-        bank: &Arc<Bank>,
+        bank: &BankWithScheduler,
         blockstore: &Blockstore,
         replay_stats: &RwLock<ReplaySlotStats>,
         replay_progress: &RwLock<ConfirmationProgress>,
@@ -2599,7 +2600,11 @@ impl ReplayStage {
                         return replay_result;
                     }
 
-                    let bank = bank_forks.read().unwrap().get(bank_slot).unwrap();
+                    let bank = bank_forks
+                        .read()
+                        .unwrap()
+                        .get_with_scheduler(bank_slot)
+                        .unwrap();
                     let parent_slot = bank.parent_slot();
                     let (num_blocks_on_fork, num_dropped_blocks_on_fork) = {
                         let stats = progress_lock
@@ -2687,7 +2692,11 @@ impl ReplayStage {
             debug!("bank_slot {:?} is marked dead", bank_slot);
             replay_result.is_slot_dead = true;
         } else {
-            let bank = bank_forks.read().unwrap().get(bank_slot).unwrap();
+            let bank = bank_forks
+                .read()
+                .unwrap()
+                .get_with_scheduler(bank_slot)
+                .unwrap();
             let parent_slot = bank.parent_slot();
             let prev_leader_slot = progress.get_bank_prev_leader_slot(&bank);
             let (num_blocks_on_fork, num_dropped_blocks_on_fork) = {
@@ -2768,7 +2777,11 @@ impl ReplayStage {
             }
 
             let bank_slot = replay_result.bank_slot;
-            let bank = &bank_forks.read().unwrap().get(bank_slot).unwrap();
+            let bank = &bank_forks
+                .read()
+                .unwrap()
+                .get_with_scheduler(bank_slot)
+                .unwrap();
             if let Some(replay_result) = &replay_result.replay_result {
                 match replay_result {
                     Ok(replay_tx_count) => tx_count += replay_tx_count,
@@ -2826,7 +2839,9 @@ impl ReplayStage {
                 );
                 // report cost tracker stats
                 cost_update_sender
-                    .send(CostUpdate::FrozenBank { bank: bank.clone() })
+                    .send(CostUpdate::FrozenBank {
+                        bank: bank.clone_without_scheduler(),
+                    })
                     .unwrap_or_else(|err| {
                         warn!("cost_update_sender failed sending bank stats: {:?}", err)
                     });
@@ -2887,7 +2902,7 @@ impl ReplayStage {
                 if let Some(sender) = bank_notification_sender {
                     sender
                         .sender
-                        .send(BankNotification::Frozen(bank.clone()))
+                        .send(BankNotification::Frozen(bank.clone_without_scheduler()))
                         .unwrap_or_else(|err| warn!("bank_notification_sender failed: {:?}", err));
                 }
                 blockstore_processor::cache_block_meta(bank, cache_block_meta_sender);
@@ -4747,7 +4762,7 @@ pub(crate) mod tests {
             assert_eq!(bank0.tick_height(), bank0.max_tick_height());
             let bank1 = Bank::new_from_parent(bank0, &Pubkey::default(), 1);
             bank_forks.write().unwrap().insert(bank1);
-            let bank1 = bank_forks.read().unwrap().get(1).unwrap();
+            let bank1 = bank_forks.read().unwrap().get_with_scheduler(1).unwrap();
             let bank1_progress = progress
                 .entry(bank1.slot())
                 .or_insert_with(|| ForkProgress::new(bank1.last_blockhash(), None, None, 0, 0));

core/tests/epoch_accounts_hash.rs

@@ -599,18 +599,22 @@ fn test_epoch_accounts_hash_and_warping() {
     );
     // flush the write cache so warping can calculate the accounts hash from storages
     bank.force_flush_accounts_cache();
-    let bank = bank_forks.write().unwrap().insert(Bank::warp_from_parent(
-        bank,
-        &Pubkey::default(),
-        eah_stop_slot_in_next_epoch,
-        CalcAccountsHashDataSource::Storages,
-    ));
+    let bank = bank_forks
+        .write()
+        .unwrap()
+        .insert(Bank::warp_from_parent(
+            bank,
+            &Pubkey::default(),
+            eah_stop_slot_in_next_epoch,
+            CalcAccountsHashDataSource::Storages,
+        ))
+        .clone_without_scheduler();
     let slot = bank.slot().checked_add(1).unwrap();
-    let bank =
-        bank_forks
-            .write()
-            .unwrap()
-            .insert(Bank::new_from_parent(bank, &Pubkey::default(), slot));
+    let bank = bank_forks
+        .write()
+        .unwrap()
+        .insert(Bank::new_from_parent(bank, &Pubkey::default(), slot))
+        .clone_without_scheduler();
     bank_forks.write().unwrap().set_root(
         bank.slot(),
         &test_environment
@@ -634,18 +638,22 @@ fn test_epoch_accounts_hash_and_warping() {
         epoch_schedule.get_first_slot_in_epoch(bank.epoch() + 1) + eah_start_offset;
     // flush the write cache so warping can calculate the accounts hash from storages
     bank.force_flush_accounts_cache();
-    let bank = bank_forks.write().unwrap().insert(Bank::warp_from_parent(
-        bank,
-        &Pubkey::default(),
-        eah_start_slot_in_next_epoch,
-        CalcAccountsHashDataSource::Storages,
-    ));
+    let bank = bank_forks
+        .write()
+        .unwrap()
+        .insert(Bank::warp_from_parent(
+            bank,
+            &Pubkey::default(),
+            eah_start_slot_in_next_epoch,
+            CalcAccountsHashDataSource::Storages,
+        ))
+        .clone_without_scheduler();
     let slot = bank.slot().checked_add(1).unwrap();
-    let bank =
-        bank_forks
-            .write()
-            .unwrap()
-            .insert(Bank::new_from_parent(bank, &Pubkey::default(), slot));
+    let bank = bank_forks
+        .write()
+        .unwrap()
+        .insert(Bank::new_from_parent(bank, &Pubkey::default(), slot))
+        .clone_without_scheduler();
     bank_forks.write().unwrap().set_root(
         bank.slot(),
         &test_environment