From 523ea252551ae102a10bf56d4f22aa8a34b2931c Mon Sep 17 00:00:00 2001 From: Yueh-Hsuan Chiang <93241502+yhchiang-sol@users.noreply.github.com> Date: Fri, 1 Dec 2023 10:39:15 -0800 Subject: [PATCH] [TieredStorage] Allow HotStorage to handle more account data (#34155) #### Problem After we have defined AccountOffset to be u32, it means the address space within one HotAccountsFile is up to 4GB. However, since the Accounts Blocks in a HotAccountsFile is 8-byte aligned, it has the opportunity to store more data by internally multiplying the AccountOffset by 8. #### Summary of Changes This PR allows a HotAccountsFile to store up to 32GB accounts data by using 8 x AccountOffset as its actual offset. #### Test Plan Updated existing unit-tests. --- accounts-db/src/tiered_storage/hot.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/accounts-db/src/tiered_storage/hot.rs b/accounts-db/src/tiered_storage/hot.rs index cd6d9ee2f7f275..a7722c22dac43d 100644 --- a/accounts-db/src/tiered_storage/hot.rs +++ b/accounts-db/src/tiered_storage/hot.rs @@ -36,6 +36,10 @@ const MAX_HOT_PADDING: u8 = 7; /// The maximum allowed value for the owner index of a hot account. const MAX_HOT_OWNER_OFFSET: OwnerOffset = OwnerOffset((1 << 29) - 1); +/// The multiplier for converting AccountOffset to the internal hot account +/// offset. This increases the maximum size of a hot accounts file. +const HOT_ACCOUNT_OFFSET_MULTIPLIER: usize = 8; + #[bitfield(bits = 32)] #[repr(C)] #[derive(Debug, Default, Copy, Clone, Eq, PartialEq)] @@ -228,7 +232,9 @@ impl HotStorageReader { &self, account_offset: AccountOffset, ) -> TieredStorageResult<&HotAccountMeta> { - let (meta, _) = get_type::(&self.mmap, account_offset.block as usize)?; + let internal_account_offset = account_offset.block as usize * HOT_ACCOUNT_OFFSET_MULTIPLIER; + + let (meta, _) = get_type::(&self.mmap, internal_account_offset)?; Ok(meta) } @@ -468,7 +474,10 @@ pub mod tests { .map(|meta| { let prev_offset = current_offset; current_offset += file.write_type(meta).unwrap() as u32; - AccountOffset { block: prev_offset } + assert_eq!(prev_offset % HOT_ACCOUNT_OFFSET_MULTIPLIER as u32, 0); + AccountOffset { + block: prev_offset / HOT_ACCOUNT_OFFSET_MULTIPLIER as u32, + } }) .collect(); // while the test only focuses on account metas, writing a footer