Skip to content

Commit

Permalink
Support a custom path for the accounts hash cache (#33115)
Browse files Browse the repository at this point in the history
(cherry picked from commit d1b8499)

# Conflicts:
#	runtime/src/accounts_db.rs
  • Loading branch information
brooksprumo authored and mergify[bot] committed Sep 1, 2023
1 parent c976676 commit 4b56b04
Showing 1 changed file with 65 additions and 7 deletions.
72 changes: 65 additions & 7 deletions runtime/src/accounts_db.rs
Original file line number Diff line number Diff line change
Expand Up @@ -470,6 +470,10 @@ pub(crate) struct ShrinkCollect<'a, T: ShrinkCollectRefs<'a>> {

pub const ACCOUNTS_DB_CONFIG_FOR_TESTING: AccountsDbConfig = AccountsDbConfig {
index: Some(ACCOUNTS_INDEX_CONFIG_FOR_TESTING),
<<<<<<< HEAD:runtime/src/accounts_db.rs
=======
base_working_path: None,
>>>>>>> d1b849972f (Support a custom path for the accounts hash cache (#33115)):accounts-db/src/accounts_db.rs
accounts_hash_cache_path: None,
filler_accounts_config: FillerAccountsConfig::const_default(),
write_cache_limit_bytes: None,
Expand All @@ -481,6 +485,10 @@ pub const ACCOUNTS_DB_CONFIG_FOR_TESTING: AccountsDbConfig = AccountsDbConfig {
};
pub const ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS: AccountsDbConfig = AccountsDbConfig {
index: Some(ACCOUNTS_INDEX_CONFIG_FOR_BENCHMARKS),
<<<<<<< HEAD:runtime/src/accounts_db.rs
=======
base_working_path: None,
>>>>>>> d1b849972f (Support a custom path for the accounts hash cache (#33115)):accounts-db/src/accounts_db.rs
accounts_hash_cache_path: None,
filler_accounts_config: FillerAccountsConfig::const_default(),
write_cache_limit_bytes: None,
Expand Down Expand Up @@ -540,6 +548,11 @@ const ANCIENT_APPEND_VEC_DEFAULT_OFFSET: Option<i64> = Some(-10_000);
#[derive(Debug, Default, Clone)]
pub struct AccountsDbConfig {
pub index: Option<AccountsIndexConfig>,
<<<<<<< HEAD:runtime/src/accounts_db.rs
=======
/// Base directory for various necessary files
pub base_working_path: Option<PathBuf>,
>>>>>>> d1b849972f (Support a custom path for the accounts hash cache (#33115)):accounts-db/src/accounts_db.rs
pub accounts_hash_cache_path: Option<PathBuf>,
pub filler_accounts_config: FillerAccountsConfig,
pub write_cache_limit_bytes: Option<u64>,
Expand Down Expand Up @@ -1396,15 +1409,19 @@ pub struct AccountsDb {
/// Set of storage paths to pick from
pub(crate) paths: Vec<PathBuf>,

<<<<<<< HEAD:runtime/src/accounts_db.rs
=======
/// Base directory for various necessary files
base_working_path: PathBuf,
// used by tests - held until we are dropped
#[allow(dead_code)]
base_working_temp_dir: Option<TempDir>,

>>>>>>> d1b849972f (Support a custom path for the accounts hash cache (#33115)):accounts-db/src/accounts_db.rs
full_accounts_hash_cache_path: PathBuf,
incremental_accounts_hash_cache_path: PathBuf,
transient_accounts_hash_cache_path: PathBuf,

// used by tests
// holds this until we are dropped
#[allow(dead_code)]
temp_accounts_hash_cache_path: Option<TempDir>,

pub shrink_paths: RwLock<Option<Vec<PathBuf>>>,

/// Directory of paths this accounts_db needs to hold/remove
Expand Down Expand Up @@ -2351,19 +2368,28 @@ impl<'a> AppendVecScan for ScanState<'a> {
}

impl AccountsDb {
<<<<<<< HEAD:runtime/src/accounts_db.rs
pub const ACCOUNTS_HASH_CACHE_DIR: &str = "accounts_hash_cache";
=======
pub const DEFAULT_ACCOUNTS_HASH_CACHE_DIR: &'static str = "accounts_hash_cache";
>>>>>>> d1b849972f (Support a custom path for the accounts hash cache (#33115)):accounts-db/src/accounts_db.rs

pub fn default_for_tests() -> Self {
Self::default_with_accounts_index(AccountInfoAccountsIndex::default_for_tests(), None)
Self::default_with_accounts_index(AccountInfoAccountsIndex::default_for_tests(), None, None)
}

fn default_with_accounts_index(
accounts_index: AccountInfoAccountsIndex,
<<<<<<< HEAD:runtime/src/accounts_db.rs
=======
base_working_path: Option<PathBuf>,
>>>>>>> d1b849972f (Support a custom path for the accounts hash cache (#33115)):accounts-db/src/accounts_db.rs
accounts_hash_cache_path: Option<PathBuf>,
) -> Self {
let num_threads = get_thread_count();
const MAX_READ_ONLY_CACHE_DATA_SIZE: usize = 400_000_000; // 400M bytes

<<<<<<< HEAD:runtime/src/accounts_db.rs
let mut temp_accounts_hash_cache_path = None;
let accounts_hash_cache_path = accounts_hash_cache_path.unwrap_or_else(|| {
temp_accounts_hash_cache_path = Some(TempDir::new().unwrap());
Expand All @@ -2373,6 +2399,19 @@ impl AccountsDb {
.path()
.to_path_buf()
});
=======
let (base_working_path, base_working_temp_dir) =
if let Some(base_working_path) = base_working_path {
(base_working_path, None)
} else {
let base_working_temp_dir = TempDir::new().unwrap();
let base_working_path = base_working_temp_dir.path().to_path_buf();
(base_working_path, Some(base_working_temp_dir))
};
>>>>>>> d1b849972f (Support a custom path for the accounts hash cache (#33115)):accounts-db/src/accounts_db.rs

let accounts_hash_cache_path = accounts_hash_cache_path
.unwrap_or_else(|| base_working_path.join(Self::DEFAULT_ACCOUNTS_HASH_CACHE_DIR));

let mut bank_hash_stats = HashMap::new();
bank_hash_stats.insert(0, BankHashStats::default());
Expand Down Expand Up @@ -2402,10 +2441,14 @@ impl AccountsDb {
write_cache_limit_bytes: None,
write_version: AtomicU64::new(0),
paths: vec![],
<<<<<<< HEAD:runtime/src/accounts_db.rs
=======
base_working_path,
base_working_temp_dir,
>>>>>>> d1b849972f (Support a custom path for the accounts hash cache (#33115)):accounts-db/src/accounts_db.rs
full_accounts_hash_cache_path: accounts_hash_cache_path.join("full"),
incremental_accounts_hash_cache_path: accounts_hash_cache_path.join("incremental"),
transient_accounts_hash_cache_path: accounts_hash_cache_path.join("transient"),
temp_accounts_hash_cache_path,
shrink_paths: RwLock::new(None),
temp_paths: None,
file_size: DEFAULT_FILE_SIZE,
Expand Down Expand Up @@ -2485,8 +2528,15 @@ impl AccountsDb {
);
let accounts_hash_cache_path = accounts_db_config
.as_ref()
<<<<<<< HEAD:runtime/src/accounts_db.rs
.and_then(|x| x.accounts_hash_cache_path.clone());

=======
.and_then(|x| x.base_working_path.clone());
let accounts_hash_cache_path = accounts_db_config
.as_ref()
.and_then(|config| config.accounts_hash_cache_path.clone());
>>>>>>> d1b849972f (Support a custom path for the accounts hash cache (#33115)):accounts-db/src/accounts_db.rs
let filler_accounts_config = accounts_db_config
.as_ref()
.map(|config| config.filler_accounts_config)
Expand Down Expand Up @@ -2541,7 +2591,15 @@ impl AccountsDb {
.and_then(|x| x.write_cache_limit_bytes),
partitioned_epoch_rewards_config,
exhaustively_verify_refcounts,
<<<<<<< HEAD:runtime/src/accounts_db.rs
..Self::default_with_accounts_index(accounts_index, accounts_hash_cache_path)
=======
..Self::default_with_accounts_index(
accounts_index,
base_working_path,
accounts_hash_cache_path,
)
>>>>>>> d1b849972f (Support a custom path for the accounts hash cache (#33115)):accounts-db/src/accounts_db.rs
};
if paths_is_empty {
// Create a temporary set of accounts directories, used primarily
Expand Down

0 comments on commit 4b56b04

Please sign in to comment.