Skip to content

Commit

Permalink
Add snapshot_utils::bank_from_latest_snapshot_archives()
Browse files Browse the repository at this point in the history 
While reviewing PR solana-labs#18565, as issue was brought up to refactor some code
around verifying the bank after rebuilding from snapshots.  A new
top-level function has been added to get the latest snapshot archives
and load the bank then verify.  Additionally, new tests have been
written and existing tests have been updated to use this new function.

Fixes solana-labs#18973

While resolving the issue, it became clear there was some additional
low-hanging fruit this change enabled.  Specifically, the functions
`bank_to_xxx_snapshot_archive()` now return their respective
`SnapshotArchiveInfo`.  And on the flip side,
`bank_from_snapshot_archives()` now takes `SnapshotArchiveInfo`s instead
of separate paths and archive formats.  This bundling simplifies bank
rebuilding.
  • Loading branch information
@brooksprumo
brooksprumo committed Jul 29, 2021
1 parent ba0f6dc commit 63014a8
Show file tree
Hide file tree
Showing 5 changed files with 357 additions and 211 deletions.
32 changes: 18 additions & 14 deletions core/src/validator.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1198,20 +1198,24 @@ fn new_banks_from_ledger(
);
leader_schedule_cache.set_root(&bank_forks.root_bank());

let archive_file = solana_runtime::snapshot_utils::bank_to_full_snapshot_archive(
ledger_path,
&bank_forks.root_bank(),
None,
&snapshot_config.snapshot_package_output_path,
snapshot_config.archive_format,
Some(bank_forks.root_bank().get_thread_pool()),
snapshot_config.maximum_snapshots_to_retain,
)
.unwrap_or_else(|err| {
error!("Unable to create snapshot: {}", err);
abort();
});
info!("created snapshot: {}", archive_file.display());
let full_snapshot_archive_info =
solana_runtime::snapshot_utils::bank_to_full_snapshot_archive(
ledger_path,
&bank_forks.root_bank(),
None,
&snapshot_config.snapshot_package_output_path,
snapshot_config.archive_format,
Some(bank_forks.root_bank().get_thread_pool()),
snapshot_config.maximum_snapshots_to_retain,
)
.unwrap_or_else(|err| {
error!("Unable to create snapshot: {}", err);
abort();
});
info!(
"created snapshot: {}",
full_snapshot_archive_info.path().display()
);
}

let tower = post_process_restored_tower(
Expand Down
81 changes: 19 additions & 62 deletions core/tests/snapshots.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,16 @@ mod tests {

let old_last_bank = old_bank_forks.get(old_last_slot).unwrap();

let check_hash_calculation = false;
let full_snapshot_archive_path = snapshot_utils::build_full_snapshot_archive_path(
snapshot_package_output_path.to_path_buf(),
old_last_bank.slot(),
&old_last_bank.get_accounts_hash(),
ArchiveFormat::TarBzip2,
);
let full_snapshot_archive_info =
snapshot_utils::FullSnapshotArchiveInfo::new_from_path(full_snapshot_archive_path)
.unwrap();

let (deserialized_bank, _timing) = snapshot_utils::bank_from_snapshot_archives(
account_paths,
&[],
Expand All @@ -167,22 +176,16 @@ mod tests {
.as_ref()
.unwrap()
.snapshot_path,
snapshot_utils::build_full_snapshot_archive_path(
snapshot_package_output_path.to_path_buf(),
old_last_bank.slot(),
&old_last_bank.get_accounts_hash(),
ArchiveFormat::TarBzip2,
),
&full_snapshot_archive_info,
None,
ArchiveFormat::TarBzip2,
old_genesis_config,
None,
None,
AccountSecondaryIndexes::default(),
false,
None,
accounts_db::AccountShrinkThreshold::default(),
check_hash_calculation,
false,
false,
)
.unwrap();
Expand Down Expand Up @@ -713,9 +716,8 @@ mod tests {
)
.unwrap();

restore_from_incremental_snapshot_and_check_banks_are_equal(
restore_from_snapshots_and_check_banks_are_equal(
&bank,
last_full_snapshot_slot.unwrap(),
&snapshot_test_config.snapshot_config,
snapshot_test_config.accounts_dir.path().to_path_buf(),
&snapshot_test_config.genesis_config_info.genesis_config,
Expand Down Expand Up @@ -780,57 +782,17 @@ mod tests {
Ok(())
}

fn restore_from_incremental_snapshot_and_check_banks_are_equal(
fn restore_from_snapshots_and_check_banks_are_equal(
bank: &Bank,
last_full_snapshot_slot: Slot,
snapshot_config: &SnapshotConfig,
accounts_dir: PathBuf,
genesis_config: &GenesisConfig,
) -> snapshot_utils::Result<()> {
let (
full_snapshot_archive_slot,
(incremental_snapshot_archive_base_slot, incremental_snapshot_archive_slot),
deserialized_bank,
) = restore_from_incremental_snapshot(snapshot_config, accounts_dir, genesis_config)?;

assert_eq!(
full_snapshot_archive_slot,
incremental_snapshot_archive_base_slot
);
assert_eq!(full_snapshot_archive_slot, last_full_snapshot_slot);
assert_eq!(incremental_snapshot_archive_slot, bank.slot(),);
assert_eq!(*bank, deserialized_bank);

Ok(())
}

fn restore_from_incremental_snapshot(
snapshot_config: &SnapshotConfig,
accounts_dir: PathBuf,
genesis_config: &GenesisConfig,
) -> snapshot_utils::Result<(Slot, (Slot, Slot), Bank)> {
let full_snapshot_archive_info = snapshot_utils::get_highest_full_snapshot_archive_info(
let (deserialized_bank, _) = snapshot_utils::bank_from_latest_snapshot_archives(
&snapshot_config.snapshot_path,
&snapshot_config.snapshot_package_output_path,
)
.ok_or_else(|| Error::new(ErrorKind::Other, "no full snapshot"))?;

let incremental_snapshot_archive_info =
snapshot_utils::get_highest_incremental_snapshot_archive_info(
&snapshot_config.snapshot_package_output_path,
*full_snapshot_archive_info.slot(),
)
.ok_or_else(|| Error::new(ErrorKind::Other, "no incremental snapshot"))?;

info!("Restoring bank from full snapshot slot: {}, and incremental snapshot slot: {} (with base slot: {})",
full_snapshot_archive_info.slot(), incremental_snapshot_archive_info.slot(), incremental_snapshot_archive_info.base_slot());

let (deserialized_bank, _) = snapshot_utils::bank_from_snapshot_archives(
&[accounts_dir],
&[],
&snapshot_config.snapshot_path,
full_snapshot_archive_info.path(),
Some(incremental_snapshot_archive_info.path()),
snapshot_config.archive_format,
genesis_config,
None,
None,
Expand All @@ -842,13 +804,8 @@ mod tests {
false,
)?;

Ok((
*full_snapshot_archive_info.slot(),
(
*incremental_snapshot_archive_info.base_slot(),
*incremental_snapshot_archive_info.slot(),
),
deserialized_bank,
))
assert_eq!(bank, &deserialized_bank);

Ok(())
}
}
4 changes: 2 additions & 2 deletions ledger-tool/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2198,7 +2198,7 @@ fn main() {
bank.slot(),
);

let archive_file = snapshot_utils::bank_to_full_snapshot_archive(
let full_snapshot_archive_info = snapshot_utils::bank_to_full_snapshot_archive(
ledger_path,
&bank,
Some(snapshot_version),
Expand All @@ -2216,7 +2216,7 @@ fn main() {
"Successfully created snapshot for slot {}, hash {}: {}",
bank.slot(),
bank.hash(),
archive_file.display(),
full_snapshot_archive_info.path().display(),
);
println!(
"Shred version: {}",
Expand Down
55 changes: 12 additions & 43 deletions ledger/src/bank_forks_utils.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,7 @@ use crate::{
};
use log::*;
use solana_entry::entry::VerifyRecyclers;
use solana_runtime::{
bank_forks::BankForks,
snapshot_config::SnapshotConfig,
snapshot_utils::{self, FullSnapshotArchiveInfo},
};
use solana_runtime::{bank_forks::BankForks, snapshot_config::SnapshotConfig, snapshot_utils};
use solana_sdk::{clock::Slot, genesis_config::GenesisConfig, hash::Hash};
use std::{fs, path::PathBuf, process, result};

Expand Down Expand Up @@ -44,20 +40,18 @@ pub fn load(
transaction_status_sender: Option<&TransactionStatusSender>,
cache_block_meta_sender: Option<&CacheBlockMetaSender>,
) -> LoadResult {
if let Some(snapshot_config) = snapshot_config.as_ref() {
if let Some(snapshot_config) = snapshot_config {
info!(
"Initializing snapshot path: {:?}",
snapshot_config.snapshot_path
"Initializing snapshot path: {}",
snapshot_config.snapshot_path.display()
);
let _ = fs::remove_dir_all(&snapshot_config.snapshot_path);
fs::create_dir_all(&snapshot_config.snapshot_path)
.expect("Couldn't create snapshot directory");

if let Some(full_snapshot_archive_info) =
snapshot_utils::get_highest_full_snapshot_archive_info(
&snapshot_config.snapshot_package_output_path,
)
{
if let Some(_) = snapshot_utils::get_highest_full_snapshot_archive_info(
&snapshot_config.snapshot_package_output_path,
) {
return load_from_snapshot(
genesis_config,
blockstore,
Expand All @@ -67,7 +61,6 @@ pub fn load(
process_options,
transaction_status_sender,
cache_block_meta_sender,
&full_snapshot_archive_info,
);
} else {
info!("No snapshot package available; will load from genesis");
Expand Down Expand Up @@ -115,26 +108,18 @@ fn load_from_snapshot(
process_options: ProcessOptions,
transaction_status_sender: Option<&TransactionStatusSender>,
cache_block_meta_sender: Option<&CacheBlockMetaSender>,
full_snapshot_archive_info: &FullSnapshotArchiveInfo,
) -> LoadResult {
info!(
"Loading snapshot package: {:?}",
full_snapshot_archive_info.path()
);

// Fail hard here if snapshot fails to load, don't silently continue
if account_paths.is_empty() {
error!("Account paths not present when booting from snapshot");
process::exit(1);
}

let (deserialized_bank, timings) = snapshot_utils::bank_from_snapshot_archives(
let (deserialized_bank, timings) = snapshot_utils::bank_from_latest_snapshot_archives(
&snapshot_config.snapshot_path,
&snapshot_config.snapshot_package_output_path,
&account_paths,
&process_options.frozen_accounts,
&snapshot_config.snapshot_path,
full_snapshot_archive_info.path(),
None,
*full_snapshot_archive_info.archive_format(),
genesis_config,
process_options.debug_keys.clone(),
Some(&crate::builtins::get(process_options.bpf_jit)),
Expand All @@ -146,30 +131,14 @@ fn load_from_snapshot(
process_options.verify_index,
)
.expect("Load from snapshot failed");
if let Some(shrink_paths) = shrink_paths {
deserialized_bank.set_shrink_paths(shrink_paths);
}

let deserialized_bank_slot_and_hash = (
deserialized_bank.slot(),
deserialized_bank.get_accounts_hash(),
);

if deserialized_bank_slot_and_hash
!= (
*full_snapshot_archive_info.slot(),
*full_snapshot_archive_info.hash(),
)
{
error!(
"Snapshot has mismatch:\narchive: {:?}\ndeserialized: {:?}",
(
full_snapshot_archive_info.slot(),
full_snapshot_archive_info.hash()
),
deserialized_bank_slot_and_hash
);
process::exit(1);
if let Some(shrink_paths) = shrink_paths {
deserialized_bank.set_shrink_paths(shrink_paths);
}

to_loadresult(
Expand Down
Loading

0 comments on commit 63014a8

Please sign in to comment.