Construct the initial bank state from the snapshot/ and accounts file…

core/src/snapshot_packager_service.rs

@@ -326,7 +326,6 @@ mod tests {
                 archive_format,
             },
             block_height: slot,
-            slot_deltas: vec![],
             snapshot_links: link_snapshots_dir,
             snapshot_storages: vec![storage_entries],
             snapshot_version: SnapshotVersion::default(),

core/src/validator.rs

@@ -86,7 +86,7 @@ use {
         snapshot_config::SnapshotConfig,
         snapshot_hash::StartingSnapshotHashes,
         snapshot_package::PendingSnapshotPackage,
-        snapshot_utils,
+        snapshot_utils::{self, move_and_async_delete_path},
     },
     solana_sdk::{
         clock::Slot,
@@ -2002,94 +2002,15 @@ fn get_stake_percent_in_gossip(bank: &Bank, cluster_info: &ClusterInfo, log: boo
     online_stake_percentage as u64
 }
 
-/// Delete directories/files asynchronously to avoid blocking on it.
-/// Fist, in sync context, rename the original path to *_deleted,
-/// then spawn a thread to delete the renamed path.
-/// If the process is killed and the deleting process is not done,
-/// the leftover path will be deleted in the next process life, so
-/// there is no file space leaking.
-pub fn move_and_async_delete_path(path: impl AsRef<Path> + Copy) {
-    let mut path_delete = PathBuf::new();
-    path_delete.push(path);
-    path_delete.set_file_name(format!(
-        "{}{}",
-        path_delete.file_name().unwrap().to_str().unwrap(),
-        "_to_be_deleted"
-    ));
-
-    if path_delete.exists() {
-        std::fs::remove_dir_all(&path_delete).unwrap();
-    }
-
-    if !path.as_ref().exists() {
-        return;
-    }
-
-    if let Err(err) = std::fs::rename(path, &path_delete) {
-        warn!(
-            "Path renaming failed: {}.  Falling back to rm_dir in sync mode",
-            err.to_string()
-        );
-        delete_contents_of_path(path);
-        return;
-    }
-
-    Builder::new()
-        .name("solDeletePath".to_string())
-        .spawn(move || {
-            std::fs::remove_dir_all(path_delete).unwrap();
-        })
-        .unwrap();
-}
-
-/// Delete the files and subdirectories in a directory.
-/// This is useful if the process does not have permission
-/// to delete the top level directory it might be able to
-/// delete the contents of that directory.
-fn delete_contents_of_path(path: impl AsRef<Path> + Copy) {
-    if let Ok(dir_entries) = std::fs::read_dir(path) {
-        for entry in dir_entries.flatten() {
-            let sub_path = entry.path();
-            let metadata = match entry.metadata() {
-                Ok(metadata) => metadata,
-                Err(err) => {
-                    warn!(
-                        "Failed to get metadata for {}. Error: {}",
-                        sub_path.display(),
-                        err.to_string()
-                    );
-                    break;
-                }
-            };
-            if metadata.is_dir() {
-                if let Err(err) = std::fs::remove_dir_all(&sub_path) {
-                    warn!(
-                        "Failed to remove sub directory {}.  Error: {}",
-                        sub_path.display(),
-                        err.to_string()
-                    );
-                }
-            } else if metadata.is_file() {
-                if let Err(err) = std::fs::remove_file(&sub_path) {
-                    warn!(
-                        "Failed to remove file {}.  Error: {}",
-                        sub_path.display(),
-                        err.to_string()
-                    );
-                }
-            }
-        }
-    } else {
-        warn!(
-            "Failed to read the sub paths of {}",
-            path.as_ref().display()
-        );
-    }
-}
-
 fn cleanup_accounts_paths(config: &ValidatorConfig) {
     for accounts_path in &config.account_paths {
         move_and_async_delete_path(accounts_path);
+        // Let the empty top accounts/ dir exist.  It was created at very early
+        // stage.  It should not disappear after clearing its content.
+        if std::fs::metadata(accounts_path).is_err() {
+            // not the /mnt/account case
+            std::fs::create_dir_all(accounts_path).unwrap();
+        }
     }
     if let Some(ref shrink_paths) = config.account_shrink_paths {
         for accounts_path in shrink_paths {

core/tests/snapshots.rs

@@ -1,7 +1,6 @@
 #![allow(clippy::integer_arithmetic)]
 
 use {
-    bincode::serialize_into,
     crossbeam_channel::unbounded,
     fs_extra::dir::CopyOptions,
     itertools::Itertools,
@@ -16,10 +15,10 @@ use {
             AbsRequestHandlers, AbsRequestSender, AccountsBackgroundService,
             PrunedBanksRequestHandler, SnapshotRequestHandler,
         },
-        accounts_db::{self, ACCOUNTS_DB_CONFIG_FOR_TESTING},
+        accounts_db::{self, SnapshotStorages, ACCOUNTS_DB_CONFIG_FOR_TESTING},
         accounts_hash::AccountsHash,
         accounts_index::AccountSecondaryIndexes,
-        bank::{Bank, BankSlotDelta},
+        bank::Bank,
         bank_forks::BankForks,
         epoch_accounts_hash::EpochAccountsHash,
         genesis_utils::{create_genesis_config_with_leader, GenesisConfigInfo},
@@ -51,7 +50,7 @@ use {
     },
     solana_streamer::socket::SocketAddrSpace,
     std::{
-        collections::HashSet,
+        collections::{HashSet, VecDeque},
         fs,
         io::{Error, ErrorKind},
         path::PathBuf,
@@ -240,6 +239,7 @@ fn run_bank_forks_snapshot_n<F>(
         snapshot_request_receiver,
         accounts_package_sender,
     };
+    let mut snapshot_slot_storages: VecDeque<SnapshotStorages> = VecDeque::new();
     for slot in 1..=last_slot {
         let mut bank = Bank::new_from_parent(&bank_forks[slot - 1], &Pubkey::default(), slot);
         f(&mut bank, mint_keypair);
@@ -251,7 +251,12 @@ fn run_bank_forks_snapshot_n<F>(
             // set_root should send a snapshot request
             bank_forks.set_root(bank.slot(), &request_sender, None);
             bank.update_accounts_hash_for_tests();
-            snapshot_request_handler.handle_snapshot_requests(false, 0, &mut None);
+            snapshot_request_handler.handle_snapshot_requests(
+                false,
+                0,
+                &mut None,
+                &mut snapshot_slot_storages,
+            );
         }
     }
 
@@ -261,13 +266,11 @@ fn run_bank_forks_snapshot_n<F>(
     let bank_snapshots_dir = &snapshot_config.bank_snapshots_dir;
     let last_bank_snapshot_info = snapshot_utils::get_highest_bank_snapshot_pre(bank_snapshots_dir)
         .expect("no bank snapshots found in path");
-    let slot_deltas = last_bank.status_cache.read().unwrap().root_slot_deltas();
     let accounts_package = AccountsPackage::new_for_snapshot(
         AccountsPackageType::Snapshot(SnapshotType::FullSnapshot),
         &last_bank,
         &last_bank_snapshot_info,
         bank_snapshots_dir,
-        slot_deltas,
         &snapshot_config.full_snapshot_archives_dir,
         &snapshot_config.incremental_snapshot_archives_dir,
         last_bank.get_snapshot_storages(None),
@@ -366,8 +369,15 @@ fn test_concurrent_snapshot_packaging(
     // Take snapshot of zeroth bank
     let bank0 = bank_forks.get(0).unwrap();
     let storages = bank0.get_snapshot_storages(None);
-    snapshot_utils::add_bank_snapshot(bank_snapshots_dir, &bank0, &storages, snapshot_version)
-        .unwrap();
+    let slot_deltas = bank0.status_cache.read().unwrap().root_slot_deltas();
+    snapshot_utils::add_bank_snapshot(
+        bank_snapshots_dir,
+        &bank0,
+        &storages,
+        snapshot_version,
+        slot_deltas,
+    )
+    .unwrap();
 
     // Set up snapshotting channels
     let (real_accounts_package_sender, real_accounts_package_receiver) =
@@ -411,19 +421,20 @@ fn test_concurrent_snapshot_packaging(
         };
 
         let snapshot_storages = bank.get_snapshot_storages(None);
+        let slot_deltas = bank.status_cache.read().unwrap().root_slot_deltas();
         let bank_snapshot_info = snapshot_utils::add_bank_snapshot(
             bank_snapshots_dir,
             &bank,
             &snapshot_storages,
             snapshot_config.snapshot_version,
+            slot_deltas,
         )
         .unwrap();
         let accounts_package = AccountsPackage::new_for_snapshot(
             AccountsPackageType::Snapshot(SnapshotType::FullSnapshot),
             &bank,
             &bank_snapshot_info,
             bank_snapshots_dir,
-            vec![],
             full_snapshot_archives_dir,
             incremental_snapshot_archives_dir,
             snapshot_storages,
@@ -554,20 +565,6 @@ fn test_concurrent_snapshot_packaging(
 
     // Check the archive we cached the state for earlier was generated correctly
 
-    // before we compare, stick an empty status_cache in this dir so that the package comparison works
-    // This is needed since the status_cache is added by the packager and is not collected from
-    // the source dir for snapshots
-    snapshot_utils::serialize_snapshot_data_file(
-        &saved_snapshots_dir
-            .path()
-            .join(snapshot_utils::SNAPSHOT_STATUS_CACHE_FILENAME),
-        |stream| {
-            serialize_into(stream, &[] as &[BankSlotDelta])?;
-            Ok(())
-        },
-    )
-    .unwrap();
-
     // files were saved off before we reserialized the bank in the hacked up accounts_hash_verifier stand-in.
     solana_runtime::serde_snapshot::reserialize_bank_with_new_accounts_hash(
         saved_snapshots_dir.path(),
@@ -757,6 +754,7 @@ fn test_bank_forks_incremental_snapshot(
     };
 
     let mut last_full_snapshot_slot = None;
+    let mut snapshot_slot_storages: VecDeque<SnapshotStorages> = VecDeque::new();
     for slot in 1..=LAST_SLOT {
         // Make a new bank and perform some transactions
         let bank = {
@@ -788,6 +786,7 @@ fn test_bank_forks_incremental_snapshot(
                 false,
                 0,
                 &mut last_full_snapshot_slot,
+                &mut snapshot_slot_storages,
             );
         }
 

ledger-tool/src/main.rs

@@ -24,10 +24,7 @@ use {
         },
     },
     solana_cli_output::{CliAccount, CliAccountNewConfig, OutputFormat},
-    solana_core::{
-        system_monitor_service::{SystemMonitorService, SystemMonitorStatsReportConfig},
-        validator::move_and_async_delete_path,
-    },
+    solana_core::system_monitor_service::{SystemMonitorService, SystemMonitorStatsReportConfig},
     solana_entry::entry::Entry,
     solana_geyser_plugin_manager::geyser_plugin_service::GeyserPluginService,
     solana_ledger::{
@@ -63,8 +60,8 @@ use {
         snapshot_hash::StartingSnapshotHashes,
         snapshot_minimizer::SnapshotMinimizer,
         snapshot_utils::{
-            self, ArchiveFormat, SnapshotVersion, DEFAULT_ARCHIVE_COMPRESSION,
-            SUPPORTED_ARCHIVE_COMPRESSION,
+            self, move_and_async_delete_path, ArchiveFormat, SnapshotFrom, SnapshotVersion,
+            DEFAULT_ARCHIVE_COMPRESSION, SUPPORTED_ARCHIVE_COMPRESSION,
         },
     },
     solana_sdk::{
@@ -1028,6 +1025,7 @@ fn load_bank_forks(
     snapshot_archive_path: Option<PathBuf>,
     incremental_snapshot_archive_path: Option<PathBuf>,
 ) -> Result<(Arc<RwLock<BankForks>>, Option<StartingSnapshotHashes>), BlockstoreProcessorError> {
+    let arg_snapshot_from_file = arg_matches.is_present("snapshot_from_file");
     let bank_snapshots_dir = blockstore
         .ledger_path()
         .join(if blockstore.is_primary_access() {
@@ -1056,12 +1054,23 @@ fn load_bank_forks(
             starting_slot = std::cmp::max(full_snapshot_slot, incremental_snapshot_slot);
         }
 
-        Some(SnapshotConfig {
+        let archive_config = SnapshotConfig {
             full_snapshot_archives_dir,
             incremental_snapshot_archives_dir,
             bank_snapshots_dir,
             ..SnapshotConfig::new_load_only()
-        })
+        };
+        if !arg_snapshot_from_file {
+            Some(archive_config)
+        } else {
+            // Even in the from_file case, still get all the archive config info because they are
+            // needed if the fallback happens.
+            Some(SnapshotConfig {
+                snapshot_from: Some(SnapshotFrom::File),
+                archive_format: ArchiveFormat::None,
+                ..archive_config
+            })
+        }
     };
 
     if let Some(halt_slot) = process_options.halt_at_slot {
@@ -1101,6 +1110,11 @@ fn load_bank_forks(
         vec![non_primary_accounts_path]
     };
 
+    // Ensure the accounts paths exist.
+    for account_path in &account_paths {
+        std::fs::create_dir_all(account_path).unwrap_or_default();
+    }
+
     let mut accounts_update_notifier = Option::<AccountsUpdateNotifier>::default();
     if arg_matches.is_present("geyser_plugin_config") {
         let geyser_config_files = values_t_or_exit!(arg_matches, "geyser_plugin_config", String)
@@ -1304,6 +1318,10 @@ fn main() {
         .long("no-snapshot")
         .takes_value(false)
         .help("Do not start from a local snapshot if present");
+    let snapshot_from_file = Arg::with_name("snapshot_from_file")
+        .long("snapshot-from-file")
+        .takes_value(false)
+        .help("Construct initial bank state from files instead of archives");
     let no_bpf_jit_arg = Arg::with_name("no_bpf_jit")
         .long("no-bpf-jit")
         .takes_value(false)
@@ -1765,6 +1783,7 @@ fn main() {
             SubCommand::with_name("verify")
             .about("Verify the ledger")
             .arg(&no_snapshot_arg)
+            .arg(&snapshot_from_file)
             .arg(&account_paths_arg)
             .arg(&accounts_index_path_arg)
             .arg(&halt_at_slot_arg)
@@ -1805,6 +1824,7 @@ fn main() {
             SubCommand::with_name("graph")
             .about("Create a Graphviz rendering of the ledger")
             .arg(&no_snapshot_arg)
+            .arg(&snapshot_from_file)
             .arg(&account_paths_arg)
             .arg(&halt_at_slot_arg)
             .arg(&hard_forks_arg)
@@ -1834,6 +1854,7 @@ fn main() {
             SubCommand::with_name("create-snapshot")
             .about("Create a new ledger snapshot")
             .arg(&no_snapshot_arg)
+            .arg(&snapshot_from_file)
             .arg(&account_paths_arg)
             .arg(&skip_rewrites_arg)
             .arg(&accounts_db_skip_initial_hash_calc_arg)