diff --git a/core/tests/snapshots.rs b/core/tests/snapshots.rs index 0f4aa8c61c1f8e..05b00b261887ba 100644 --- a/core/tests/snapshots.rs +++ b/core/tests/snapshots.rs @@ -164,6 +164,7 @@ mod tests { None, AccountSecondaryIndexes::default(), false, + None, ) .unwrap(); diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index f2671284ee8c6d..bd6e84ed3ad969 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -771,6 +771,12 @@ fn main() { .validator(is_slot) .takes_value(true) .help("Halt processing at the given slot"); + let limit_load_slot_count_from_snapshot_arg = Arg::with_name("limit_load_slot_count_from_snapshot") + .long("limit-load-slot-count-from-snapshot") + .value_name("SLOT") + .validator(is_slot) + .takes_value(true) + .help("For debugging and profiling with large snapshots, artificially limit how many slots are loaded from a snapshot."); let hard_forks_arg = Arg::with_name("hard_forks") .long("hard-fork") .value_name("SLOT") @@ -1040,6 +1046,7 @@ fn main() { .arg(&no_snapshot_arg) .arg(&account_paths_arg) .arg(&halt_at_slot_arg) + .arg(&limit_load_slot_count_from_snapshot_arg) .arg(&hard_forks_arg) .arg(&no_accounts_db_caching_arg) .arg(&accounts_db_test_hash_calculation_arg) @@ -1744,6 +1751,12 @@ fn main() { poh_verify: !arg_matches.is_present("skip_poh_verify"), bpf_jit: !matches.is_present("no_bpf_jit"), accounts_db_caching_enabled: !arg_matches.is_present("no_accounts_db_caching"), + limit_load_slot_count_from_snapshot: value_t!( + arg_matches, + "limit_load_slot_count_from_snapshot", + usize + ) + .ok(), allow_dead_slots: arg_matches.is_present("allow_dead_slots"), accounts_db_test_hash_calculation: arg_matches .is_present("accounts_db_test_hash_calculation"), diff --git a/ledger/src/bank_forks_utils.rs b/ledger/src/bank_forks_utils.rs index ed5c7de54f200e..747bfe05dc4b81 100644 --- a/ledger/src/bank_forks_utils.rs +++ b/ledger/src/bank_forks_utils.rs @@ -71,6 +71,7 @@ pub fn load( Some(&crate::builtins::get(process_options.bpf_jit)), process_options.account_indexes.clone(), process_options.accounts_db_caching_enabled, + process_options.limit_load_slot_count_from_snapshot, ) .expect("Load from snapshot failed"); if let Some(shrink_paths) = shrink_paths { diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index bf204eef0b56c8..7e7fe6434bae49 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -368,6 +368,7 @@ pub struct ProcessOptions { pub debug_keys: Option>>, pub account_indexes: AccountSecondaryIndexes, pub accounts_db_caching_enabled: bool, + pub limit_load_slot_count_from_snapshot: Option, pub allow_dead_slots: bool, pub accounts_db_test_hash_calculation: bool, } diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs index 973cee1a43bc0b..a613d13d9915ad 100644 --- a/runtime/src/accounts_db.rs +++ b/runtime/src/accounts_db.rs @@ -5123,12 +5123,15 @@ impl AccountsDb { } #[allow(clippy::needless_collect)] - pub fn generate_index(&self) { + pub fn generate_index(&self, limit_load_slot_count_from_snapshot: Option) { type AccountsMap<'a> = HashMap)>; let mut slots = self.storage.all_slots(); #[allow(clippy::stable_sort_primitive)] slots.sort(); + if let Some(limit) = limit_load_slot_count_from_snapshot { + slots.truncate(limit); // get rid of the newer slots and keep just the older + } let total_processed_slots_across_all_threads = AtomicU64::new(0); let outer_slots_len = slots.len(); let chunk_size = (outer_slots_len / 7) + 1; // approximately 400k slots in a snapshot diff --git a/runtime/src/serde_snapshot.rs b/runtime/src/serde_snapshot.rs index 7b68c887d3053c..33ca72178ec26b 100644 --- a/runtime/src/serde_snapshot.rs +++ b/runtime/src/serde_snapshot.rs @@ -135,6 +135,7 @@ pub(crate) fn bank_from_stream( additional_builtins: Option<&Builtins>, account_indexes: AccountSecondaryIndexes, caching_enabled: bool, + limit_load_slot_count_from_snapshot: Option, ) -> std::result::Result where R: Read, @@ -154,6 +155,7 @@ where additional_builtins, account_indexes, caching_enabled, + limit_load_slot_count_from_snapshot, )?; Ok(bank) }}; @@ -243,6 +245,7 @@ fn reconstruct_bank_from_fields( additional_builtins: Option<&Builtins>, account_indexes: AccountSecondaryIndexes, caching_enabled: bool, + limit_load_slot_count_from_snapshot: Option, ) -> Result where E: SerializableStorage, @@ -254,6 +257,7 @@ where &genesis_config.cluster_type, account_indexes, caching_enabled, + limit_load_slot_count_from_snapshot, )?; accounts_db.freeze_accounts( &Ancestors::from(&bank_fields.ancestors), @@ -279,6 +283,7 @@ fn reconstruct_accountsdb_from_fields( cluster_type: &ClusterType, account_indexes: AccountSecondaryIndexes, caching_enabled: bool, + limit_load_slot_count_from_snapshot: Option, ) -> Result where E: SerializableStorage, @@ -371,6 +376,6 @@ where accounts_db .write_version .fetch_add(version, Ordering::Relaxed); - accounts_db.generate_index(); + accounts_db.generate_index(limit_load_slot_count_from_snapshot); Ok(accounts_db) } diff --git a/runtime/src/serde_snapshot/tests.rs b/runtime/src/serde_snapshot/tests.rs index f0604302dbd235..4971ee2d0a7702 100644 --- a/runtime/src/serde_snapshot/tests.rs +++ b/runtime/src/serde_snapshot/tests.rs @@ -73,6 +73,7 @@ where &ClusterType::Development, AccountSecondaryIndexes::default(), false, + None, ) } @@ -226,6 +227,7 @@ fn test_bank_serialize_style(serde_style: SerdeStyle) { None, AccountSecondaryIndexes::default(), false, + None, ) .unwrap(); dbank.src = ref_sc; diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs index 2913364c967bca..d59eb54c4f3de0 100644 --- a/runtime/src/snapshot_utils.rs +++ b/runtime/src/snapshot_utils.rs @@ -603,6 +603,7 @@ pub fn bank_from_archive>( additional_builtins: Option<&Builtins>, account_indexes: AccountSecondaryIndexes, accounts_db_caching_enabled: bool, + limit_load_slot_count_from_snapshot: Option, ) -> Result { let unpack_dir = tempfile::Builder::new() .prefix(TMP_SNAPSHOT_PREFIX) @@ -633,6 +634,7 @@ pub fn bank_from_archive>( additional_builtins, account_indexes, accounts_db_caching_enabled, + limit_load_slot_count_from_snapshot, )?; if !bank.verify_snapshot_bank() { @@ -791,6 +793,7 @@ fn rebuild_bank_from_snapshots( additional_builtins: Option<&Builtins>, account_indexes: AccountSecondaryIndexes, accounts_db_caching_enabled: bool, + limit_load_slot_count_from_snapshot: Option, ) -> Result { info!("snapshot version: {}", snapshot_version); @@ -826,6 +829,7 @@ fn rebuild_bank_from_snapshots( additional_builtins, account_indexes, accounts_db_caching_enabled, + limit_load_slot_count_from_snapshot, ), }?) })?;