From 7b89222fde87b1523e83a2f6432823351810b1e7 Mon Sep 17 00:00:00 2001 From: "Jeff Washington (jwash)" Date: Wed, 23 Mar 2022 11:53:37 -0500 Subject: [PATCH] don't start extra threads for shrink/clean/hash (#23858) --- runtime/src/accounts_db.rs | 11 +++++++---- runtime/src/accounts_index.rs | 4 ++-- runtime/src/accounts_index_storage.rs | 19 +++++++++++++++++-- runtime/src/bank.rs | 5 +++-- 4 files changed, 29 insertions(+), 10 deletions(-) diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs index 5402ff1b912e24..25e64efb3f6679 100644 --- a/runtime/src/accounts_db.rs +++ b/runtime/src/accounts_db.rs @@ -30,6 +30,7 @@ use { ScanResult, SlotList, SlotSlice, ZeroLamport, ACCOUNTS_INDEX_CONFIG_FOR_BENCHMARKS, ACCOUNTS_INDEX_CONFIG_FOR_TESTING, }, + accounts_index_storage::Startup, accounts_update_notifier_interface::AccountsUpdateNotifier, active_stats::{ActiveStatItem, ActiveStats}, ancestors::Ancestors, @@ -6856,7 +6857,8 @@ impl AccountsDb { let account = AccountSharedData::new(lamports, space, &owner); let added = AtomicUsize::default(); for pass in 0..=passes { - self.accounts_index.set_startup(true); + self.accounts_index + .set_startup(Startup::StartupWithExtraThreads); let roots_in_this_pass = roots .iter() .skip(pass * per_pass) @@ -6907,7 +6909,7 @@ impl AccountsDb { self.maybe_throttle_index_generation(); self.store_accounts_frozen((*slot, &add[..]), Some(&hashes[..]), None, None); }); - self.accounts_index.set_startup(false); + self.accounts_index.set_startup(Startup::Normal); } info!("added {} filler accounts", added.load(Ordering::Relaxed)); } @@ -6941,7 +6943,8 @@ impl AccountsDb { let passes = if verify { 2 } else { 1 }; for pass in 0..passes { if pass == 0 { - self.accounts_index.set_startup(true); + self.accounts_index + .set_startup(Startup::StartupWithExtraThreads); } let storage_info = StorageSizeAndCountMap::default(); let total_processed_slots_across_all_threads = AtomicU64::new(0); @@ -7079,7 +7082,7 @@ impl AccountsDb { if pass == 0 { // tell accounts index we are done adding the initial accounts at startup let mut m = Measure::start("accounts_index_idle_us"); - self.accounts_index.set_startup(false); + self.accounts_index.set_startup(Startup::Normal); m.stop(); index_flush_us = m.as_us(); } diff --git a/runtime/src/accounts_index.rs b/runtime/src/accounts_index.rs index bbcb1f47bf11c9..813fb2cd079c6f 100644 --- a/runtime/src/accounts_index.rs +++ b/runtime/src/accounts_index.rs @@ -1,6 +1,6 @@ use { crate::{ - accounts_index_storage::AccountsIndexStorage, + accounts_index_storage::{AccountsIndexStorage, Startup}, ancestors::Ancestors, bucket_map_holder::{Age, BucketMapHolder}, contains::Contains, @@ -1505,7 +1505,7 @@ impl AccountsIndex { iter.hold_range_in_memory(range, start_holding, thread_pool); } - pub fn set_startup(&self, value: bool) { + pub fn set_startup(&self, value: Startup) { self.storage.set_startup(value); } diff --git a/runtime/src/accounts_index_storage.rs b/runtime/src/accounts_index_storage.rs index b4c8d809ed9830..0da823c63f6f46 100644 --- a/runtime/src/accounts_index_storage.rs +++ b/runtime/src/accounts_index_storage.rs @@ -86,13 +86,28 @@ impl BgThreads { } } +/// modes the system can be in +pub enum Startup { + /// not startup, but steady state execution + Normal, + /// startup (not steady state execution) + /// requesting 'startup'-like behavior where in-mem acct idx items are flushed asap + Startup, + /// startup (not steady state execution) + /// but also requesting additional threads to be running to flush the acct idx to disk asap + /// The idea is that the best perf to ssds will be with multiple threads, + /// but during steady state, we can't allocate as many threads because we'd starve the rest of the system. + StartupWithExtraThreads, +} + impl AccountsIndexStorage { /// startup=true causes: /// in mem to act in a way that flushes to disk asap /// also creates some additional bg threads to facilitate flushing to disk asap /// startup=false is 'normal' operation - pub fn set_startup(&self, value: bool) { - if value { + pub fn set_startup(&self, startup: Startup) { + let value = !matches!(startup, Startup::Normal); + if matches!(startup, Startup::StartupWithExtraThreads) { // create some additional bg threads to help get things to the disk index asap *self.startup_worker_threads.lock().unwrap() = Some(BgThreads::new( &self.storage, diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 726de0ef0201f1..b0829160328ae6 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -43,6 +43,7 @@ use { ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS, ACCOUNTS_DB_CONFIG_FOR_TESTING, }, accounts_index::{AccountSecondaryIndexes, IndexKey, ScanConfig, ScanResult}, + accounts_index_storage::Startup, accounts_update_notifier_interface::AccountsUpdateNotifier, ancestors::{Ancestors, AncestorsForSerialization}, blockhash_queue::BlockhashQueue, @@ -6039,7 +6040,7 @@ impl Bank { .accounts .accounts_db .accounts_index - .set_startup(true); + .set_startup(Startup::Startup); let mut shrink_all_slots_time = Measure::start("shrink_all_slots"); if !accounts_db_skip_shrink && self.slot() > 0 { info!("shrinking.."); @@ -6055,7 +6056,7 @@ impl Bank { .accounts .accounts_db .accounts_index - .set_startup(false); + .set_startup(Startup::Normal); info!("verify_hash.."); let mut verify2_time = Measure::start("verify_hash");