Skip to content

Commit

Permalink
Uses mmap for names in tiered storage (#34127)
Browse files Browse the repository at this point in the history
  • Loading branch information
brooksprumo authored Nov 16, 2023
1 parent 7d7d72e commit 07b0b9f
Show file tree
Hide file tree
Showing 3 changed files with 20 additions and 18 deletions.
16 changes: 9 additions & 7 deletions accounts-db/src/tiered_storage/footer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -204,11 +204,11 @@ impl TieredStorageFooter {
Ok(footer)
}

pub fn new_from_mmap(map: &Mmap) -> TieredStorageResult<&TieredStorageFooter> {
let offset = map.len().saturating_sub(FOOTER_TAIL_SIZE);
let (footer_size, offset) = get_type::<u64>(map, offset)?;
let (_footer_version, offset) = get_type::<u64>(map, offset)?;
let (magic_number, _offset) = get_type::<TieredStorageMagicNumber>(map, offset)?;
pub fn new_from_mmap(mmap: &Mmap) -> TieredStorageResult<&TieredStorageFooter> {
let offset = mmap.len().saturating_sub(FOOTER_TAIL_SIZE);
let (footer_size, offset) = get_type::<u64>(mmap, offset)?;
let (_footer_version, offset) = get_type::<u64>(mmap, offset)?;
let (magic_number, _offset) = get_type::<TieredStorageMagicNumber>(mmap, offset)?;

if *magic_number != TieredStorageMagicNumber::default() {
return Err(TieredStorageError::MagicNumberMismatch(
Expand All @@ -217,8 +217,10 @@ impl TieredStorageFooter {
));
}

let (footer, _offset) =
get_type::<TieredStorageFooter>(map, map.len().saturating_sub(*footer_size as usize))?;
let (footer, _offset) = get_type::<TieredStorageFooter>(
mmap,
mmap.len().saturating_sub(*footer_size as usize),
)?;

Ok(footer)
}
Expand Down
14 changes: 7 additions & 7 deletions accounts-db/src/tiered_storage/index.rs
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ impl IndexBlockFormat {
/// Returns the address of the account given the specified index.
pub fn get_account_address<'a>(
&self,
map: &'a Mmap,
mmap: &'a Mmap,
footer: &TieredStorageFooter,
index_offset: IndexOffset,
) -> TieredStorageResult<&'a Pubkey> {
Expand All @@ -87,14 +87,14 @@ impl IndexBlockFormat {
footer.index_block_offset as usize + std::mem::size_of::<Pubkey>() * index_offset.0
}
};
let (address, _) = get_type::<Pubkey>(map, account_offset)?;
let (address, _) = get_type::<Pubkey>(mmap, account_offset)?;
Ok(address)
}

/// Returns the offset to the account given the specified index.
pub fn get_account_offset(
&self,
map: &Mmap,
mmap: &Mmap,
footer: &TieredStorageFooter,
index_offset: IndexOffset,
) -> TieredStorageResult<AccountOffset> {
Expand All @@ -103,7 +103,7 @@ impl IndexBlockFormat {
let account_offset = footer.index_block_offset as usize
+ std::mem::size_of::<Pubkey>() * footer.account_entry_count as usize
+ index_offset.0 * std::mem::size_of::<u64>();
let (account_block_offset, _) = get_type(map, account_offset)?;
let (account_block_offset, _) = get_type(mmap, account_offset)?;
Ok(AccountOffset {
block: *account_block_offset,
})
Expand Down Expand Up @@ -160,14 +160,14 @@ mod tests {
.create(false)
.open(&path)
.unwrap();
let map = unsafe { MmapOptions::new().map(&file).unwrap() };
let mmap = unsafe { MmapOptions::new().map(&file).unwrap() };
for (i, index_entry) in index_entries.iter().enumerate() {
let account_offset = indexer
.get_account_offset(&map, &footer, IndexOffset(i))
.get_account_offset(&mmap, &footer, IndexOffset(i))
.unwrap();
assert_eq!(index_entry.block_offset, account_offset.block as u64);
let address = indexer
.get_account_address(&map, &footer, IndexOffset(i))
.get_account_address(&mmap, &footer, IndexOffset(i))
.unwrap();
assert_eq!(index_entry.address, address);
}
Expand Down
8 changes: 4 additions & 4 deletions accounts-db/src/tiered_storage/mmap_utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,21 +16,21 @@ pub fn get_type<T>(map: &Mmap, offset: usize) -> IoResult<(&T, usize)> {
/// doesn't overrun the internal buffer. Otherwise return an Error.
/// Also return the offset of the first byte after the requested data that
/// falls on a 64-byte boundary.
pub fn get_slice(map: &Mmap, offset: usize, size: usize) -> IoResult<(&[u8], usize)> {
pub fn get_slice(mmap: &Mmap, offset: usize, size: usize) -> IoResult<(&[u8], usize)> {
let (next, overflow) = offset.overflowing_add(size);
if overflow || next > map.len() {
if overflow || next > mmap.len() {
error!(
"Requested offset {} and size {} while mmap only has length {}",
offset,
size,
map.len()
mmap.len()
);
return Err(std::io::Error::new(
std::io::ErrorKind::AddrNotAvailable,
"Requested offset and data length exceeds the mmap slice",
));
}
let data = &map[offset..next];
let data = &mmap[offset..next];
let next = u64_align!(next);
let ptr = data.as_ptr();

Expand Down

0 comments on commit 07b0b9f

Please sign in to comment.