Add snapshot_utils::bank_from_latest_snapshot_archives()

core/src/accounts_hash_verifier.rs

@@ -10,6 +10,7 @@ use solana_gossip::cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES};
 use solana_runtime::{
     accounts_db,
     snapshot_package::{AccountsPackage, AccountsPackagePre, AccountsPackageReceiver},
+    snapshot_utils::SnapshotArchiveInfo,
 };
 use solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey};
 use std::collections::{HashMap, HashSet};
@@ -125,19 +126,19 @@ impl AccountsHashVerifier {
         fault_injection_rate_slots: u64,
         snapshot_interval_slots: u64,
     ) {
-        let hash = accounts_package.hash;
+        let hash = *accounts_package.hash();
         if fault_injection_rate_slots != 0
-            && accounts_package.slot % fault_injection_rate_slots == 0
+            && accounts_package.slot() % fault_injection_rate_slots == 0
         {
             // For testing, publish an invalid hash to gossip.
             use rand::{thread_rng, Rng};
             use solana_sdk::hash::extend_and_hash;
-            warn!("inserting fault at slot: {}", accounts_package.slot);
+            warn!("inserting fault at slot: {}", accounts_package.slot());
             let rand = thread_rng().gen_range(0, 10);
             let hash = extend_and_hash(&hash, &[rand]);
-            hashes.push((accounts_package.slot, hash));
+            hashes.push((accounts_package.slot(), hash));
         } else {
-            hashes.push((accounts_package.slot, hash));
+            hashes.push((accounts_package.slot(), hash));
         }
 
         while hashes.len() > MAX_SNAPSHOT_HASHES {
@@ -281,18 +282,26 @@ mod tests {
         let exit = Arc::new(AtomicBool::new(false));
         let mut hashes = vec![];
         for i in 0..MAX_SNAPSHOT_HASHES + 1 {
+            let slot = 100 + i as u64;
+            let block_height = 100 + i as u64;
+            let slot_deltas = vec![];
             let snapshot_links = TempDir::new().unwrap();
-            let accounts_package = AccountsPackage {
-                hash: hash(&[i as u8]),
-                block_height: 100 + i as u64,
-                slot: 100 + i as u64,
-                slot_deltas: vec![],
+            let storages = vec![];
+            let snapshot_archive_path = PathBuf::from(".");
+            let hash = hash(&[i as u8]);
+            let archive_format = ArchiveFormat::TarBzip2;
+            let snapshot_version = SnapshotVersion::default();
+            let accounts_package = AccountsPackage::new(
+                slot,
+                block_height,
+                slot_deltas,
                 snapshot_links,
-                tar_output_file: PathBuf::from("."),
-                storages: vec![],
-                archive_format: ArchiveFormat::TarBzip2,
-                snapshot_version: SnapshotVersion::default(),
-            };
+                storages,
+                snapshot_archive_path,
+                hash,
+                archive_format,
+                snapshot_version,
+            );
 
             AccountsHashVerifier::process_accounts_package(
                 accounts_package,

core/src/snapshot_packager_service.rs

@@ -1,5 +1,8 @@
 use solana_gossip::cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES};
-use solana_runtime::{snapshot_package::AccountsPackage, snapshot_utils};
+use solana_runtime::{
+    snapshot_package::AccountsPackage,
+    snapshot_utils::{self, SnapshotArchiveInfo},
+};
 use solana_sdk::{clock::Slot, hash::Hash};
 use std::{
     sync::{
@@ -48,7 +51,7 @@ impl SnapshotPackagerService {
                         ) {
                             warn!("Failed to create snapshot archive: {}", err);
                         } else {
-                            hashes.push((snapshot_package.slot, snapshot_package.hash));
+                            hashes.push((snapshot_package.slot(), *snapshot_package.hash()));
                             while hashes.len() > MAX_SNAPSHOT_HASHES {
                                 hashes.remove(0);
                             }

core/src/validator.rs

@@ -61,6 +61,7 @@ use solana_runtime::{
     commitment::BlockCommitmentCache,
     hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
     snapshot_config::SnapshotConfig,
+    snapshot_utils::{self, SnapshotArchiveInfo},
 };
 use solana_sdk::{
     clock::Slot,
@@ -1194,7 +1195,7 @@ fn new_banks_from_ledger(
         );
         leader_schedule_cache.set_root(&bank_forks.root_bank());
 
-        let archive_file = solana_runtime::snapshot_utils::bank_to_full_snapshot_archive(
+        let full_snapshot_archive_info = snapshot_utils::bank_to_full_snapshot_archive(
             ledger_path,
             &bank_forks.root_bank(),
             None,
@@ -1207,7 +1208,10 @@ fn new_banks_from_ledger(
             error!("Unable to create snapshot: {}", err);
             abort();
         });
-        info!("created snapshot: {}", archive_file.display());
+        info!(
+            "created snapshot: {}",
+            full_snapshot_archive_info.path().display()
+        );
     }
 
     let tower = post_process_restored_tower(

core/tests/snapshots.rs

@@ -161,6 +161,16 @@ mod tests {
         let old_last_bank = old_bank_forks.get(old_last_slot).unwrap();
 
         let check_hash_calculation = false;
+        let full_snapshot_archive_path = snapshot_utils::build_full_snapshot_archive_path(
+            snapshot_package_output_path.to_path_buf(),
+            old_last_bank.slot(),
+            &old_last_bank.get_accounts_hash(),
+            ArchiveFormat::TarBzip2,
+        );
+        let full_snapshot_archive_info =
+            snapshot_utils::FullSnapshotArchiveInfo::new_from_path(full_snapshot_archive_path)
+                .unwrap();
+
         let (deserialized_bank, _timing) = snapshot_utils::bank_from_snapshot_archives(
             account_paths,
             &[],
@@ -169,14 +179,8 @@ mod tests {
                 .as_ref()
                 .unwrap()
                 .snapshot_path,
-            snapshot_utils::build_full_snapshot_archive_path(
-                snapshot_package_output_path.to_path_buf(),
-                old_last_bank.slot(),
-                &old_last_bank.get_accounts_hash(),
-                ArchiveFormat::TarBzip2,
-            ),
+            &full_snapshot_archive_info,
             None,
-            ArchiveFormat::TarBzip2,
             old_genesis_config,
             None,
             None,
@@ -716,9 +720,8 @@ mod tests {
                 )
                 .unwrap();
 
-                restore_from_incremental_snapshot_and_check_banks_are_equal(
+                restore_from_snapshots_and_check_banks_are_equal(
                     &bank,
-                    last_full_snapshot_slot.unwrap(),
                     &snapshot_test_config.snapshot_config,
                     snapshot_test_config.accounts_dir.path().to_path_buf(),
                     &snapshot_test_config.genesis_config_info.genesis_config,
@@ -783,57 +786,17 @@ mod tests {
         Ok(())
     }
 
-    fn restore_from_incremental_snapshot_and_check_banks_are_equal(
+    fn restore_from_snapshots_and_check_banks_are_equal(
         bank: &Bank,
-        last_full_snapshot_slot: Slot,
         snapshot_config: &SnapshotConfig,
         accounts_dir: PathBuf,
         genesis_config: &GenesisConfig,
     ) -> snapshot_utils::Result<()> {
-        let (
-            full_snapshot_archive_slot,
-            (incremental_snapshot_archive_base_slot, incremental_snapshot_archive_slot),
-            deserialized_bank,
-        ) = restore_from_incremental_snapshot(snapshot_config, accounts_dir, genesis_config)?;
-
-        assert_eq!(
-            full_snapshot_archive_slot,
-            incremental_snapshot_archive_base_slot
-        );
-        assert_eq!(full_snapshot_archive_slot, last_full_snapshot_slot);
-        assert_eq!(incremental_snapshot_archive_slot, bank.slot(),);
-        assert_eq!(*bank, deserialized_bank);
-
-        Ok(())
-    }
-
-    fn restore_from_incremental_snapshot(
-        snapshot_config: &SnapshotConfig,
-        accounts_dir: PathBuf,
-        genesis_config: &GenesisConfig,
-    ) -> snapshot_utils::Result<(Slot, (Slot, Slot), Bank)> {
-        let full_snapshot_archive_info = snapshot_utils::get_highest_full_snapshot_archive_info(
+        let (deserialized_bank, _) = snapshot_utils::bank_from_latest_snapshot_archives(
+            &snapshot_config.snapshot_path,
             &snapshot_config.snapshot_package_output_path,
-        )
-        .ok_or_else(|| Error::new(ErrorKind::Other, "no full snapshot"))?;
-
-        let incremental_snapshot_archive_info =
-            snapshot_utils::get_highest_incremental_snapshot_archive_info(
-                &snapshot_config.snapshot_package_output_path,
-                *full_snapshot_archive_info.slot(),
-            )
-            .ok_or_else(|| Error::new(ErrorKind::Other, "no incremental snapshot"))?;
-
-        info!("Restoring bank from full snapshot slot: {}, and incremental snapshot slot: {} (with base slot: {})",
-        full_snapshot_archive_info.slot(), incremental_snapshot_archive_info.slot(), incremental_snapshot_archive_info.base_slot());
-
-        let (deserialized_bank, _) = snapshot_utils::bank_from_snapshot_archives(
             &[accounts_dir],
             &[],
-            &snapshot_config.snapshot_path,
-            full_snapshot_archive_info.path(),
-            Some(incremental_snapshot_archive_info.path()),
-            snapshot_config.archive_format,
             genesis_config,
             None,
             None,
@@ -846,13 +809,8 @@ mod tests {
             false,
         )?;
 
-        Ok((
-            *full_snapshot_archive_info.slot(),
-            (
-                *incremental_snapshot_archive_info.base_slot(),
-                *incremental_snapshot_archive_info.slot(),
-            ),
-            deserialized_bank,
-        ))
+        assert_eq!(bank, &deserialized_bank);
+
+        Ok(())
     }
 }

ledger-tool/src/main.rs

@@ -31,7 +31,8 @@ use solana_runtime::{
     hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
     snapshot_config::SnapshotConfig,
     snapshot_utils::{
-        self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
+        self, ArchiveFormat, SnapshotArchiveInfo, SnapshotVersion,
+        DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
     },
 };
 use solana_sdk::{
@@ -2199,7 +2200,7 @@ fn main() {
                         bank.slot(),
                     );
 
-                    let archive_file = snapshot_utils::bank_to_full_snapshot_archive(
+                    let full_snapshot_archive_info = snapshot_utils::bank_to_full_snapshot_archive(
                         ledger_path,
                         &bank,
                         Some(snapshot_version),
@@ -2217,7 +2218,7 @@ fn main() {
                         "Successfully created snapshot for slot {}, hash {}: {}",
                         bank.slot(),
                         bank.hash(),
-                        archive_file.display(),
+                        full_snapshot_archive_info.path().display(),
                     );
                     println!(
                         "Shred version: {}",

ledger/src/bank_forks_utils.rs

@@ -8,11 +8,7 @@ use crate::{
 };
 use log::*;
 use solana_entry::entry::VerifyRecyclers;
-use solana_runtime::{
-    bank_forks::BankForks,
-    snapshot_config::SnapshotConfig,
-    snapshot_utils::{self, FullSnapshotArchiveInfo},
-};
+use solana_runtime::{bank_forks::BankForks, snapshot_config::SnapshotConfig, snapshot_utils};
 use solana_sdk::{clock::Slot, genesis_config::GenesisConfig, hash::Hash};
 use std::{fs, path::PathBuf, process, result};
 
@@ -44,19 +40,19 @@ pub fn load(
     transaction_status_sender: Option<&TransactionStatusSender>,
     cache_block_meta_sender: Option<&CacheBlockMetaSender>,
 ) -> LoadResult {
-    if let Some(snapshot_config) = snapshot_config.as_ref() {
+    if let Some(snapshot_config) = snapshot_config {
         info!(
-            "Initializing snapshot path: {:?}",
-            snapshot_config.snapshot_path
+            "Initializing snapshot path: {}",
+            snapshot_config.snapshot_path.display()
         );
         let _ = fs::remove_dir_all(&snapshot_config.snapshot_path);
         fs::create_dir_all(&snapshot_config.snapshot_path)
             .expect("Couldn't create snapshot directory");
 
-        if let Some(full_snapshot_archive_info) =
-            snapshot_utils::get_highest_full_snapshot_archive_info(
-                &snapshot_config.snapshot_package_output_path,
-            )
+        if snapshot_utils::get_highest_full_snapshot_archive_info(
+            &snapshot_config.snapshot_package_output_path,
+        )
+        .is_some()
         {
             return load_from_snapshot(
                 genesis_config,
@@ -67,7 +63,6 @@ pub fn load(
                 process_options,
                 transaction_status_sender,
                 cache_block_meta_sender,
-                &full_snapshot_archive_info,
             );
         } else {
             info!("No snapshot package available; will load from genesis");
@@ -115,26 +110,18 @@ fn load_from_snapshot(
     process_options: ProcessOptions,
     transaction_status_sender: Option<&TransactionStatusSender>,
     cache_block_meta_sender: Option<&CacheBlockMetaSender>,
-    full_snapshot_archive_info: &FullSnapshotArchiveInfo,
 ) -> LoadResult {
-    info!(
-        "Loading snapshot package: {:?}",
-        full_snapshot_archive_info.path()
-    );
-
     // Fail hard here if snapshot fails to load, don't silently continue
     if account_paths.is_empty() {
         error!("Account paths not present when booting from snapshot");
         process::exit(1);
     }
 
-    let (deserialized_bank, timings) = snapshot_utils::bank_from_snapshot_archives(
+    let (deserialized_bank, timings) = snapshot_utils::bank_from_latest_snapshot_archives(
+        &snapshot_config.snapshot_path,
+        &snapshot_config.snapshot_package_output_path,
         &account_paths,
         &process_options.frozen_accounts,
-        &snapshot_config.snapshot_path,
-        full_snapshot_archive_info.path(),
-        None,
-        *full_snapshot_archive_info.archive_format(),
         genesis_config,
         process_options.debug_keys.clone(),
         Some(&crate::builtins::get(process_options.bpf_jit)),
@@ -147,30 +134,14 @@ fn load_from_snapshot(
         process_options.verify_index,
     )
     .expect("Load from snapshot failed");
-    if let Some(shrink_paths) = shrink_paths {
-        deserialized_bank.set_shrink_paths(shrink_paths);
-    }
 
     let deserialized_bank_slot_and_hash = (
         deserialized_bank.slot(),
         deserialized_bank.get_accounts_hash(),
     );
 
-    if deserialized_bank_slot_and_hash
-        != (
-            *full_snapshot_archive_info.slot(),
-            *full_snapshot_archive_info.hash(),
-        )
-    {
-        error!(
-            "Snapshot has mismatch:\narchive: {:?}\ndeserialized: {:?}",
-            (
-                full_snapshot_archive_info.slot(),
-                full_snapshot_archive_info.hash()
-            ),
-            deserialized_bank_slot_and_hash
-        );
-        process::exit(1);
+    if let Some(shrink_paths) = shrink_paths {
+        deserialized_bank.set_shrink_paths(shrink_paths);
     }
 
     to_loadresult(