Skip to content

Commit

Permalink
[TieredStorage] WritableAccountsFileProvider
Browse files Browse the repository at this point in the history
  • Loading branch information
@yhchiang-sol
yhchiang-sol committed Mar 26, 2024
1 parent c867522 commit 14a2d10
Show file tree
Hide file tree
Showing 4 changed files with 58 additions and 31 deletions.
8 changes: 4 additions & 4 deletions accounts-db/src/account_storage.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,7 @@ impl Default for AccountStorageStatus {

#[cfg(test)]
pub(crate) mod tests {
use {super::*, std::path::Path};
use {super::*, crate::accounts_file::AppendVecProvider, std::path::Path};

#[test]
fn test_shrink_in_progress() {
Expand All @@ -292,13 +292,13 @@ pub(crate) mod tests {
let store_file_size = 4000;
let store_file_size2 = store_file_size * 2;
// 2 append vecs with same id, but different sizes
let entry = Arc::new(AccountStorageEntry::new(
let entry = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
slot,
id,
store_file_size,
));
let entry2 = Arc::new(AccountStorageEntry::new(
let entry2 = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
slot,
id,
Expand Down Expand Up @@ -348,7 +348,7 @@ pub(crate) mod tests {
// add a map store
let common_store_path = Path::new("");
let store_file_size = 4000;
Arc::new(AccountStorageEntry::new(
Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
slot,
id,
Expand Down
58 changes: 33 additions & 25 deletions accounts-db/src/accounts_db.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ use {
},
accounts_cache::{AccountsCache, CachedAccount, SlotCache},
accounts_file::{
AccountsFile, AccountsFileError, MatchAccountOwnerError, ALIGN_BOUNDARY_OFFSET,
AccountsFile, AccountsFileError, AccountsFileProvider, AppendVecProvider,
MatchAccountOwnerError, ALIGN_BOUNDARY_OFFSET,
},
accounts_hash::{
AccountHash, AccountsDeltaHash, AccountsHash, AccountsHashKind, AccountsHasher,
Expand All @@ -55,9 +56,7 @@ use {
ancient_append_vecs::{
get_ancient_append_vec_capacity, is_ancient, AccountsToStore, StorageSelector,
},
append_vec::{
aligned_stored_size, AppendVec, APPEND_VEC_MMAPPED_FILES_OPEN, STORE_META_OVERHEAD,
},
append_vec::{aligned_stored_size, APPEND_VEC_MMAPPED_FILES_OPEN, STORE_META_OVERHEAD},
cache_hash_data::{
CacheHashData, CacheHashDataFileReference, DeletionPolicy as CacheHashDeletionPolicy,
},
Expand Down Expand Up @@ -1031,10 +1030,15 @@ pub struct AccountStorageEntry {
}

impl AccountStorageEntry {
pub fn new(path: &Path, slot: Slot, id: AccountsFileId, file_size: u64) -> Self {
pub fn new<T: AccountsFileProvider>(
path: &Path,
slot: Slot,
id: AccountsFileId,
file_size: u64,
) -> Self {
let tail = AccountsFile::file_name(slot, id);
let path = Path::new(path).join(tail);
let accounts = AccountsFile::AppendVec(AppendVec::new(&path, true, file_size as usize));
let accounts = T::new_writable(&path, file_size);

Self {
id,
Expand Down Expand Up @@ -2514,7 +2518,7 @@ impl AccountsDb {
}

fn new_storage_entry(&self, slot: Slot, path: &Path, size: u64) -> AccountStorageEntry {
AccountStorageEntry::new(path, slot, self.next_id(), size)
AccountStorageEntry::new::<AppendVecProvider>(path, slot, self.next_id(), size)
}

pub fn expected_cluster_type(&self) -> ClusterType {
Expand Down Expand Up @@ -6330,7 +6334,7 @@ impl AccountsDb {
slot_stores: &HashMap<AccountsFileId, Arc<AccountStorageEntry>>,
) -> Arc<AccountStorageEntry> {
let size = slot_stores.values().map(|storage| storage.capacity()).sum();
let storage = AccountStorageEntry::new(path, slot, id, size);
let storage = AccountStorageEntry::new::<AppendVecProvider>(path, slot, id, size);

// get unique accounts, most recent version by write_version
let mut accum = HashMap::<Pubkey, StoredAccountMeta<'_>>::default();
Expand Down Expand Up @@ -9508,7 +9512,7 @@ pub mod tests {
accounts_hash::MERKLE_FANOUT,
accounts_index::{tests::*, AccountSecondaryIndexesIncludeExclude},
ancient_append_vecs,
append_vec::{test_utils::TempFile, AppendVecStoredAccountMeta},
append_vec::{test_utils::TempFile, AppendVec, AppendVecStoredAccountMeta},
cache_hash_data::CacheHashDataFile,
inline_spl_token,
},
Expand Down Expand Up @@ -10547,7 +10551,8 @@ pub mod tests {
let (_temp_dirs, paths) = get_temp_accounts_paths(1).unwrap();
let slot_expected: Slot = 0;
let size: usize = 123;
let data = AccountStorageEntry::new(&paths[0], slot_expected, 0, size as u64);
let data =
AccountStorageEntry::new::<AppendVecProvider>(&paths[0], slot_expected, 0, size as u64);

let arc = Arc::new(data);
let storages = vec![arc];
Expand Down Expand Up @@ -10598,7 +10603,8 @@ pub mod tests {
let (_temp_dirs, paths) = get_temp_accounts_paths(1).unwrap();
let slot_expected: Slot = 0;
let size: usize = 123;
let mut data = AccountStorageEntry::new(&paths[0], slot_expected, 0, size as u64);
let mut data =
AccountStorageEntry::new::<AppendVecProvider>(&paths[0], slot_expected, 0, size as u64);
let av = AccountsFile::AppendVec(AppendVec::new(&tf.path, true, 1024 * 1024));
data.accounts = av;

Expand Down Expand Up @@ -10714,7 +10720,8 @@ pub mod tests {
let (_temp_dirs, paths) = get_temp_accounts_paths(1).unwrap();
let slot_expected: Slot = 0;
let size: usize = 123;
let mut data = AccountStorageEntry::new(&paths[0], slot_expected, 0, size as u64);
let mut data =
AccountStorageEntry::new::<AppendVecProvider>(&paths[0], slot_expected, 0, size as u64);
let av = AccountsFile::AppendVec(AppendVec::new(&tf.path, true, 1024 * 1024));
data.accounts = av;

Expand Down Expand Up @@ -10793,7 +10800,8 @@ pub mod tests {
let (_temp_dirs, paths) = get_temp_accounts_paths(1).unwrap();
let file_size = account_data_size.unwrap_or(123) * 100 / fill_percentage;
let size_aligned: usize = aligned_stored_size(file_size as usize);
let mut data = AccountStorageEntry::new(&paths[0], slot, id, size_aligned as u64);
let mut data =
AccountStorageEntry::new::<AppendVecProvider>(&paths[0], slot, id, size_aligned as u64);
let av = AccountsFile::AppendVec(AppendVec::new(
&tf.path,
true,
Expand Down Expand Up @@ -12897,7 +12905,7 @@ pub mod tests {
let store_file_size = 2 * PAGE_SIZE;

let store1_id = 22;
let store1 = Arc::new(AccountStorageEntry::new(
let store1 = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
slot_id_1,
store1_id,
Expand All @@ -12910,7 +12918,7 @@ pub mod tests {
let slot_id_2 = 13;

let store2_id = 44;
let store2 = Arc::new(AccountStorageEntry::new(
let store2 = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
slot_id_2,
store2_id,
Expand All @@ -12926,7 +12934,7 @@ pub mod tests {

let slot_id_3 = 14;
let store3_id = 55;
let entry3 = Arc::new(AccountStorageEntry::new(
let entry3 = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
slot_id_3,
store3_id,
Expand Down Expand Up @@ -12970,7 +12978,7 @@ pub mod tests {
let store_file_size = 2 * PAGE_SIZE;

let store1_id = 22;
let store1 = Arc::new(AccountStorageEntry::new(
let store1 = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
slot_id_1,
store1_id,
Expand All @@ -12982,7 +12990,7 @@ pub mod tests {

let slot_id_2 = 13;
let store2_id = 44;
let store2 = Arc::new(AccountStorageEntry::new(
let store2 = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
slot_id_2,
store2_id,
Expand All @@ -12999,7 +13007,7 @@ pub mod tests {

let slot_id_3 = 14;
let store3_id = 55;
let entry3 = Arc::new(AccountStorageEntry::new(
let entry3 = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
slot_id_3,
store3_id,
Expand Down Expand Up @@ -13036,7 +13044,7 @@ pub mod tests {

let store_file_size = 4 * PAGE_SIZE;
let store1_id = 22;
let store1 = Arc::new(AccountStorageEntry::new(
let store1 = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
slot1,
store1_id,
Expand All @@ -13054,7 +13062,7 @@ pub mod tests {

let store2_id = 44;
let slot2 = 44;
let store2 = Arc::new(AccountStorageEntry::new(
let store2 = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
slot2,
store2_id,
Expand Down Expand Up @@ -15066,11 +15074,11 @@ pub mod tests {
#[test]
fn test_shrink_productive() {
solana_logger::setup();
let s1 = AccountStorageEntry::new(Path::new("."), 0, 0, 1024);
let s1 = AccountStorageEntry::new::<AppendVecProvider>(Path::new("."), 0, 0, 1024);
let store = Arc::new(s1);
assert!(!AccountsDb::is_shrinking_productive(0, &store));

let s1 = AccountStorageEntry::new(Path::new("."), 0, 0, PAGE_SIZE * 4);
let s1 = AccountStorageEntry::new::<AppendVecProvider>(Path::new("."), 0, 0, PAGE_SIZE * 4);
let store = Arc::new(s1);
store.add_account((3 * PAGE_SIZE as usize) - 1);
store.add_account(10);
Expand All @@ -15088,7 +15096,7 @@ pub mod tests {
let mut accounts = AccountsDb::new_single_for_tests();
let common_store_path = Path::new("");
let store_file_size = 2 * PAGE_SIZE;
let entry = Arc::new(AccountStorageEntry::new(
let entry = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
0,
1,
Expand Down Expand Up @@ -17424,7 +17432,7 @@ pub mod tests {
solana_logger::setup();
let common_store_path = Path::new("");
let store_file_size = 2 * PAGE_SIZE;
let entry = Arc::new(AccountStorageEntry::new(
let entry = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
0,
1,
Expand Down
18 changes: 18 additions & 0 deletions accounts-db/src/accounts_file.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,24 @@ impl<'a> Iterator for AccountsFileIter<'a> {
}
}

pub trait AccountsFileProvider {
fn new_writable(path: &Path, file_size: u64) -> AccountsFile;
}

pub struct AppendVecProvider;
impl AccountsFileProvider for AppendVecProvider {
fn new_writable(path: &Path, file_size: u64) -> AccountsFile {
AccountsFile::AppendVec(AppendVec::new(path, true, file_size as usize))
}
}

pub struct HotStorageProvider;
impl AccountsFileProvider for HotStorageProvider {
fn new_writable(path: &Path, _file_size: u64) -> AccountsFile {
AccountsFile::TieredStorage(TieredStorage::new_writable(path))
}
}

#[cfg(test)]
pub mod tests {
use crate::accounts_file::AccountsFile;
Expand Down
5 changes: 3 additions & 2 deletions accounts-db/src/sorted_storages.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ mod tests {
super::*,
crate::{
accounts_db::{AccountStorageEntry, AccountsFileId},
accounts_file::AccountsFile,
accounts_file::{AccountsFile, AppendVecProvider},
append_vec::AppendVec,
},
std::sync::Arc,
Expand Down Expand Up @@ -445,7 +445,8 @@ mod tests {
let (_temp_dirs, paths) = crate::accounts_db::get_temp_accounts_paths(1).unwrap();
let size: usize = 123;
let slot = 0;
let mut data = AccountStorageEntry::new(&paths[0], slot, id, size as u64);
let mut data =
AccountStorageEntry::new::<AppendVecProvider>(&paths[0], slot, id, size as u64);
let av = AccountsFile::AppendVec(AppendVec::new(&tf.path, true, 1024 * 1024));
data.accounts = av;

Expand Down

0 comments on commit 14a2d10

Please sign in to comment.