Skip to content

Commit

Permalink
Add test_bank_forks_incremental_snapshot()
Browse files Browse the repository at this point in the history
  • Loading branch information
@brooksprumo
brooksprumo committed Jul 20, 2021
1 parent 9081086 commit d13bf96
Show file tree
Hide file tree
Showing 2 changed files with 279 additions and 19 deletions.
290 changes: 275 additions & 15 deletions core/tests/snapshots.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@ macro_rules! DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS {
fn test_slots_to_snapshot() {
run_test_slots_to_snapshot(SNAPSHOT_VERSION, CLUSTER_TYPE)
}

#[test]
fn test_bank_forks_incremental_snapshot_n() {
run_test_bank_forks_incremental_snapshot_n(SNAPSHOT_VERSION, CLUSTER_TYPE)
}
}
};
}
Expand All @@ -39,6 +44,7 @@ mod tests {
use crossbeam_channel::unbounded;
use fs_extra::dir::CopyOptions;
use itertools::Itertools;
use log::info;
use solana_core::snapshot_packager_service::{PendingSnapshotPackage, SnapshotPackagerService};
use solana_gossip::{cluster_info::ClusterInfo, contact_info::ContactInfo};
use solana_runtime::{
Expand All @@ -55,14 +61,15 @@ mod tests {
use solana_sdk::{
clock::Slot,
genesis_config::{ClusterType, GenesisConfig},
hash::hashv,
hash::{hashv, Hash},
pubkey::Pubkey,
signature::{Keypair, Signer},
system_transaction,
};
use std::{
collections::HashSet,
fs,
io::{Error, ErrorKind},
path::PathBuf,
sync::{
atomic::{AtomicBool, Ordering},
Expand All @@ -80,8 +87,8 @@ mod tests {

struct SnapshotTestConfig {
accounts_dir: TempDir,
snapshot_dir: TempDir,
_snapshot_output_path: TempDir,
snapshots_dir: TempDir,
_snapshot_archives_dir: TempDir,
snapshot_config: SnapshotConfig,
bank_forks: BankForks,
genesis_config_info: GenesisConfigInfo,
Expand All @@ -91,11 +98,12 @@ mod tests {
fn new(
snapshot_version: SnapshotVersion,
cluster_type: ClusterType,
snapshot_interval_slots: u64,
accounts_hash_interval_slots: Slot,
snapshot_interval_slots: Slot,
) -> SnapshotTestConfig {
let accounts_dir = TempDir::new().unwrap();
let snapshot_dir = TempDir::new().unwrap();
let snapshot_output_path = TempDir::new().unwrap();
let snapshots_dir = TempDir::new().unwrap();
let snapshot_archives_dir = TempDir::new().unwrap();
let mut genesis_config_info = create_genesis_config(10_000);
genesis_config_info.genesis_config.cluster_type = cluster_type;
let bank0 = Bank::new_with_paths(
Expand All @@ -111,21 +119,21 @@ mod tests {
);
bank0.freeze();
let mut bank_forks = BankForks::new(bank0);
bank_forks.accounts_hash_interval_slots = snapshot_interval_slots;
bank_forks.accounts_hash_interval_slots = accounts_hash_interval_slots;

let snapshot_config = SnapshotConfig {
snapshot_interval_slots,
snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()),
snapshot_path: PathBuf::from(snapshot_dir.path()),
snapshot_package_output_path: snapshot_archives_dir.path().to_path_buf(),
snapshot_path: snapshots_dir.path().to_path_buf(),
archive_format: ArchiveFormat::TarBzip2,
snapshot_version,
maximum_snapshots_to_retain: DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
};
bank_forks.set_snapshot_config(Some(snapshot_config.clone()));
SnapshotTestConfig {
accounts_dir,
snapshot_dir,
_snapshot_output_path: snapshot_output_path,
snapshots_dir,
_snapshot_archives_dir: snapshot_archives_dir,
snapshot_config,
bank_forks,
genesis_config_info,
Expand Down Expand Up @@ -204,7 +212,12 @@ mod tests {
{
solana_logger::setup();
// Set up snapshotting config
let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, cluster_type, 1);
let mut snapshot_test_config = SnapshotTestConfig::new(
snapshot_version,
cluster_type,
set_root_interval,
set_root_interval,
);

let bank_forks = &mut snapshot_test_config.bank_forks;
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
Expand Down Expand Up @@ -312,10 +325,11 @@ mod tests {
solana_logger::setup();

// Set up snapshotting config
let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, cluster_type, 1);
let mut snapshot_test_config =
SnapshotTestConfig::new(snapshot_version, cluster_type, 1, 1);

let bank_forks = &mut snapshot_test_config.bank_forks;
let snapshots_dir = &snapshot_test_config.snapshot_dir;
let snapshots_dir = &snapshot_test_config.snapshots_dir;
let snapshot_config = &snapshot_test_config.snapshot_config;
let snapshot_path = &snapshot_config.snapshot_path;
let snapshot_package_output_path = &snapshot_config.snapshot_package_output_path;
Expand Down Expand Up @@ -531,7 +545,8 @@ mod tests {
let mut snapshot_test_config = SnapshotTestConfig::new(
snapshot_version,
cluster_type,
(*add_root_interval * num_set_roots * 2) as u64,
(*add_root_interval * num_set_roots * 2) as Slot,
(*add_root_interval * num_set_roots * 2) as Slot,
);
let mut current_bank = snapshot_test_config.bank_forks[0].clone();
let request_sender = AbsRequestSender::new(Some(snapshot_sender));
Expand Down Expand Up @@ -600,4 +615,249 @@ mod tests {
);
}
}

fn run_test_bank_forks_incremental_snapshot_n(
snapshot_version: SnapshotVersion,
cluster_type: ClusterType,
) {
solana_logger::setup();

const SET_ROOT_INTERVAL: Slot = 2;
const INCREMENTAL_SNAPSHOT_INTERVAL_SLOTS: Slot = SET_ROOT_INTERVAL * 2;
const FULL_SNAPSHOT_INTERVAL_SLOTS: Slot = INCREMENTAL_SNAPSHOT_INTERVAL_SLOTS * 5;
const LAST_SLOT: Slot = FULL_SNAPSHOT_INTERVAL_SLOTS * 2 - 1;

info!("Running bank forks incremental snapshot test, full snapshot interval: {} slots, incremental snapshot interval: {} slots, last slot: {}, set root interval: {} slots",
FULL_SNAPSHOT_INTERVAL_SLOTS, INCREMENTAL_SNAPSHOT_INTERVAL_SLOTS, LAST_SLOT, SET_ROOT_INTERVAL);

let mut snapshot_test_config = SnapshotTestConfig::new(
snapshot_version,
cluster_type,
SET_ROOT_INTERVAL,
FULL_SNAPSHOT_INTERVAL_SLOTS,
);

let bank_forks = &mut snapshot_test_config.bank_forks;
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;

let (snapshot_request_sender, snapshot_request_receiver) = unbounded();
let (accounts_package_sender, _accounts_package_receiver) = channel();
let request_sender = AbsRequestSender::new(Some(snapshot_request_sender));
let snapshot_request_handler = SnapshotRequestHandler {
snapshot_config: snapshot_test_config.snapshot_config.clone(),
snapshot_request_receiver,
accounts_package_sender,
};

let mut last_full_snapshot_slot = None;
for slot in 1..=LAST_SLOT {
// Make a new bank and perform some transactions
let bank = {
let bank = Bank::new_from_parent(&bank_forks[slot - 1], &Pubkey::default(), slot);

let key = Keypair::new().pubkey();
let tx = system_transaction::transfer(mint_keypair, &key, 1, bank.last_blockhash());
assert_eq!(bank.process_transaction(&tx), Ok(()));

let key = Keypair::new().pubkey();
let tx = system_transaction::transfer(mint_keypair, &key, 0, bank.last_blockhash());
assert_eq!(bank.process_transaction(&tx), Ok(()));

while !bank.is_complete() {
bank.register_tick(&Hash::new_unique());
}

bank_forks.insert(bank)
};

// Set root to make sure we don't end up with too many account storage entries
// and to allow snapshotting of bank and the purging logic on status_cache to
// kick in
if slot % SET_ROOT_INTERVAL == 0 {
// set_root sends a snapshot request
bank_forks.set_root(bank.slot(), &request_sender, None);
bank.update_accounts_hash();
snapshot_request_handler.handle_snapshot_requests(false, false, false, 0);
}

// Since AccountsBackgroundService isn't running, manually make a full snapshot archive
// at the right interval
if slot % FULL_SNAPSHOT_INTERVAL_SLOTS == 0 {
make_full_snapshot_archive(&bank, &snapshot_test_config.snapshot_config).unwrap();
last_full_snapshot_slot = Some(slot);
}
// Similarly, make an incremental snapshot archive at the right interval, but only if
// there's been at least one full snapshot first, and a full snapshot wasn't already
// taken at this slot.
//
// Then, after making an incremental snapshot, restore the bank and verify it is correct
else if slot % INCREMENTAL_SNAPSHOT_INTERVAL_SLOTS == 0
&& last_full_snapshot_slot.is_some()
&& slot != last_full_snapshot_slot.unwrap()
{
make_incremental_snapshot_archive(
&bank,
last_full_snapshot_slot.unwrap(),
&snapshot_test_config.snapshot_config,
)
.unwrap();

restore_from_incremental_snapshot_and_check_banks_are_equal(
&bank,
last_full_snapshot_slot.unwrap(),
&snapshot_test_config.snapshot_config,
snapshot_test_config.accounts_dir.path().to_path_buf(),
&snapshot_test_config.genesis_config_info.genesis_config,
)
.unwrap();
}
}
}

fn make_full_snapshot_archive(
bank: &Bank,
snapshot_config: &SnapshotConfig,
) -> snapshot_utils::Result<()> {
let slot = bank.slot();
info!("Making full snapshot archive from bank at slot: {}", slot);
let slot_snapshot_path =
snapshot_utils::get_slot_snapshot_paths(&snapshot_config.snapshot_path)
.into_iter()
.find(|elem| elem.slot == slot)
.ok_or_else(|| {
Error::new(ErrorKind::Other, "did not find snapshot with this path")
})?;
let snapshot_package = snapshot_utils::package_snapshot(
bank,
&slot_snapshot_path,
&snapshot_config.snapshot_path,
bank.src.slot_deltas(&bank.src.roots()),
&snapshot_config.snapshot_package_output_path,
bank.get_snapshot_storages(),
snapshot_config.archive_format,
snapshot_config.snapshot_version,
None,
)?;
let snapshot_package = snapshot_utils::process_accounts_package_pre(
snapshot_package,
Some(bank.get_thread_pool()),
);
snapshot_utils::archive_snapshot_package(
&snapshot_package,
snapshot_config.maximum_snapshots_to_retain,
)
}

fn make_incremental_snapshot_archive(
bank: &Bank,
incremental_snapshot_base_slot: Slot,
snapshot_config: &SnapshotConfig,
) -> snapshot_utils::Result<()> {
let slot = bank.slot();
info!(
"Making incremental snapshot archive from bank at slot: {}, and base slot: {}",
slot, incremental_snapshot_base_slot,
);
let slot_snapshot_path =
snapshot_utils::get_slot_snapshot_paths(&snapshot_config.snapshot_path)
.into_iter()
.find(|elem| elem.slot == slot)
.ok_or_else(|| {
Error::new(ErrorKind::Other, "did not find snapshot with this path")
})?;
let incremental_snapshot_package = snapshot_utils::package_incremental_snapshot(
bank,
incremental_snapshot_base_slot,
&slot_snapshot_path,
&snapshot_config.snapshot_path,
bank.src.slot_deltas(&bank.src.roots()),
&snapshot_config.snapshot_package_output_path,
bank.get_incremental_snapshot_storages(incremental_snapshot_base_slot),
snapshot_config.archive_format,
snapshot_config.snapshot_version,
None,
)?;
let incremental_snapshot_package =
snapshot_utils::process_accounts_package_pre_for_incremental_snapshot(
incremental_snapshot_package,
Some(bank.get_thread_pool()),
incremental_snapshot_base_slot,
);
snapshot_utils::archive_snapshot_package(
&incremental_snapshot_package,
snapshot_config.maximum_snapshots_to_retain,
)
}

fn restore_from_incremental_snapshot_and_check_banks_are_equal(
bank: &Bank,
last_full_snapshot_slot: Slot,
snapshot_config: &SnapshotConfig,
accounts_dir: PathBuf,
genesis_config: &GenesisConfig,
) -> snapshot_utils::Result<()> {
let (
full_snapshot_archive_slot,
(incremental_snapshot_archive_base_slot, incremental_snapshot_archive_slot),
deserialized_bank,
) = restore_from_incremental_snapshot(snapshot_config, accounts_dir, genesis_config)?;

assert_eq!(
full_snapshot_archive_slot,
incremental_snapshot_archive_base_slot
);
assert_eq!(full_snapshot_archive_slot, last_full_snapshot_slot);
assert_eq!(incremental_snapshot_archive_slot, bank.slot(),);
assert_eq!(*bank, deserialized_bank);

Ok(())
}

fn restore_from_incremental_snapshot(
snapshot_config: &SnapshotConfig,
accounts_dir: PathBuf,
genesis_config: &GenesisConfig,
) -> snapshot_utils::Result<(Slot, (Slot, Slot), Bank)> {
let full_snapshot_archive_info = snapshot_utils::get_highest_snapshot_archive_info(
&snapshot_config.snapshot_package_output_path,
)
.ok_or_else(|| Error::new(ErrorKind::Other, "no full snapshot"))?;

let incremental_snapshot_archive_info =
snapshot_utils::get_highest_incremental_snapshot_archive_info(
&snapshot_config.snapshot_package_output_path,
full_snapshot_archive_info.slot,
)
.ok_or_else(|| Error::new(ErrorKind::Other, "no incremental snapshot"))?;

info!("Restoring bank from full snapshot slot: {}, and incremental snapshot slot: {} (with base slot: {})",
full_snapshot_archive_info.slot, incremental_snapshot_archive_info.inner.slot, incremental_snapshot_archive_info.base_slot);

let (deserialized_bank, _) = snapshot_utils::bank_from_snapshot_archives(
&[accounts_dir],
&[],
&snapshot_config.snapshot_path,
&full_snapshot_archive_info.path,
Some(&incremental_snapshot_archive_info.inner.path),
snapshot_config.archive_format,
genesis_config,
None,
None,
AccountSecondaryIndexes::default(),
false,
None,
accounts_db::AccountShrinkThreshold::default(),
false,
false,
)?;

Ok((
full_snapshot_archive_info.slot,
(
incremental_snapshot_archive_info.base_slot,
incremental_snapshot_archive_info.inner.slot,
),
deserialized_bank,
))
}
}
Loading

0 comments on commit d13bf96

Please sign in to comment.