Skip to content

Commit

Permalink
Supports archiving account storage files that are backed by Mmap or F…
Browse files Browse the repository at this point in the history
  • Loading branch information
brooksprumo authored May 17, 2024
1 parent 4f6dd3d commit 20ee70c
Show file tree
Hide file tree
Showing 2 changed files with 43 additions and 12 deletions.
27 changes: 27 additions & 0 deletions accounts-db/src/accounts_file.rs
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,24 @@ impl AccountsFile {
.data_for_archive(),
}
}

/// Returns the way to access this accounts file when archiving
pub fn internals_for_archive(&self) -> InternalsForArchive {
// Figure out if the backing access to this accounts file is Mmap or File I/O.
// For now, let `can_append()` fill the gap.
if self.can_append() {
match self {
Self::AppendVec(av) => InternalsForArchive::Mmap(av.data_for_archive()),
Self::TieredStorage(ts) => InternalsForArchive::Mmap(
ts.reader()
.expect("must be a reader when archiving")
.data_for_archive(),
),
}
} else {
InternalsForArchive::FileIo(self.path())
}
}
}

/// An enum that creates AccountsFile instance with the specified format.
Expand All @@ -310,6 +328,15 @@ impl AccountsFileProvider {
}
}

/// The access method to use when archiving an AccountsFile
#[derive(Debug)]
pub enum InternalsForArchive<'a> {
/// Accessing the internals is done via Mmap
Mmap(&'a [u8]),
/// Accessing the internals is done via File I/O
FileIo(&'a Path),
}

/// Information after storing accounts
#[derive(Debug)]
pub struct StoredAccountsInfo {
Expand Down
28 changes: 16 additions & 12 deletions runtime/src/snapshot_utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ use {
solana_accounts_db::{
account_storage::AccountStorageMap,
accounts_db::{AccountStorageEntry, AtomicAccountsFileId},
accounts_file::{AccountsFile, AccountsFileError, StorageAccess},
accounts_file::{AccountsFile, AccountsFileError, InternalsForArchive, StorageAccess},
hardened_unpack::{self, ParallelSelector, UnpackError},
shared_buffer_reader::{SharedBuffer, SharedBufferReader},
utils::{move_and_async_delete_path, ACCOUNTS_RUN_DIR, ACCOUNTS_SNAPSHOT_DIR},
Expand Down Expand Up @@ -823,17 +823,21 @@ pub fn archive_snapshot_package(
storage.slot(),
storage.append_vec_id(),
));
let mut header = tar::Header::new_gnu();
header.set_path(path_in_archive).map_err(|err| {
E::ArchiveAccountStorageFile(err, storage.path().to_path_buf())
})?;
header.set_size(storage.capacity());
header.set_cksum();
archive
.append(&header, storage.accounts.data_for_archive())
.map_err(|err| {
E::ArchiveAccountStorageFile(err, storage.path().to_path_buf())
})?;
match storage.accounts.internals_for_archive() {
InternalsForArchive::Mmap(data) => {
let mut header = tar::Header::new_gnu();
header.set_path(path_in_archive).map_err(|err| {
E::ArchiveAccountStorageFile(err, storage.path().to_path_buf())
})?;
header.set_size(storage.capacity());
header.set_cksum();
archive.append(&header, data)
}
InternalsForArchive::FileIo(path) => {
archive.append_path_with_name(path, path_in_archive)
}
}
.map_err(|err| E::ArchiveAccountStorageFile(err, storage.path().to_path_buf()))?;
}

archive.into_inner().map_err(E::FinishArchive)?;
Expand Down

0 comments on commit 20ee70c

Please sign in to comment.