Add remove_incomplete_bank_snapshot_dir

jeffwashington · Apr 10, 2023 · aaafef7 · aaafef7
1 parent 9a7b6ab
commit aaafef7
Show file tree

Hide file tree

Showing 3 changed files with 82 additions and 9 deletions.
diff --git a/core/src/validator.rs b/core/src/validator.rs
@@ -90,7 +90,7 @@ use {
         snapshot_archive_info::SnapshotArchiveInfoGetter,
         snapshot_config::SnapshotConfig,
         snapshot_hash::StartingSnapshotHashes,
-        snapshot_utils::{self, clean_orphaned_account_snapshot_dirs, move_and_async_delete_path},
+        snapshot_utils::{self, move_and_async_delete_path},
     },
     solana_sdk::{
         clock::Slot,
@@ -553,8 +553,17 @@ impl Validator {
         start.stop();
         info!("done. {}", start);
 
+        info!("Cleaning up incomplete snapshot dirs..");
+        if let Err(e) = snapshot_utils::remove_incomplete_bank_snapshot_dir(
+            &config.snapshot_config.bank_snapshots_dir,
+        ) {
+            return Err(format!(
+                "Failed to clean up incomplete snapshot dirs: {e:?}"
+            ));
+        }
+
         info!("Cleaning orphaned account snapshot directories..");
-        if let Err(e) = clean_orphaned_account_snapshot_dirs(
+        if let Err(e) = snapshot_utils::clean_orphaned_account_snapshot_dirs(
             &config.snapshot_config.bank_snapshots_dir,
             &config.account_snapshot_paths,
         ) {

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
@@ -74,9 +74,9 @@ use {
         snapshot_hash::StartingSnapshotHashes,
         snapshot_minimizer::SnapshotMinimizer,
         snapshot_utils::{
-            self, clean_orphaned_account_snapshot_dirs, create_all_accounts_run_and_snapshot_dirs,
-            move_and_async_delete_path, ArchiveFormat, SnapshotVersion,
-            DEFAULT_ARCHIVE_COMPRESSION, SUPPORTED_ARCHIVE_COMPRESSION,
+            self, create_all_accounts_run_and_snapshot_dirs, move_and_async_delete_path,
+            ArchiveFormat, SnapshotVersion, DEFAULT_ARCHIVE_COMPRESSION,
+            SUPPORTED_ARCHIVE_COMPRESSION,
         },
     },
     solana_sdk::{
@@ -1205,14 +1205,19 @@ fn load_bank_forks(
     });
     measure.stop();
     info!("done. {}", measure);
-
+    info!("Cleaning up incomplete snapshot dirs..");
+    if let Err(e) = snapshot_utils::remove_incomplete_bank_snapshot_dir(&bank_snapshots_dir) {
+        eprintln!("Failed to clean up incomplete snapshot dirs: {e:?}");
+        exit(1);
+    }
     info!(
         "Cleaning contents of account snapshot paths: {:?}",
         account_snapshot_paths
     );
-    if let Err(e) =
-        clean_orphaned_account_snapshot_dirs(&bank_snapshots_dir, &account_snapshot_paths)
-    {
+    if let Err(e) = snapshot_utils::clean_orphaned_account_snapshot_dirs(
+        &bank_snapshots_dir,
+        &account_snapshot_paths,
+    ) {
         eprintln!("Failed to clean orphaned account snapshot dirs.  Error: {e:?}");
         exit(1);
     }

diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs
@@ -788,6 +788,65 @@ pub fn get_bank_snapshots(bank_snapshots_dir: impl AsRef<Path>) -> Vec<BankSnaps
     bank_snapshots
 }
 
+/// Remove any incomplete bank snapshot directories.  Creating the bank snapshot directory takes
+/// multiple steps.  If the process dies in the middle, the directory may be left incomplete, not
+/// good to use for constructing a bank.  Remove such directories to avoid booting from incomplete
+/// snapshot dirs.
+pub fn remove_incomplete_bank_snapshot_dir(bank_snapshots_dir: impl AsRef<Path>) -> Result<()> {
+    let entries = fs::read_dir(&bank_snapshots_dir).map_err(|err| {
+        SnapshotError::IoWithSourceAndFile(
+            err,
+            "Unable to read bank snapshots directory",
+            bank_snapshots_dir.as_ref().to_path_buf(),
+        )
+    })?;
+
+    let mut final_result = Ok(());
+
+    for entry in entries {
+        let entry = match entry {
+            Ok(entry) => entry,
+            Err(_) => continue,
+        };
+
+        if !entry.path().is_dir() {
+            continue;
+        }
+
+        let slot = match entry.path().file_name().and_then(|file_name| {
+            file_name
+                .to_str()
+                .and_then(|file_name| file_name.parse::<Slot>().ok())
+        }) {
+            Some(slot) => slot,
+            None => continue,
+        };
+
+        let bank_snapshot_dir = get_bank_snapshots_dir(&bank_snapshots_dir, slot);
+        let file_complete = bank_snapshot_dir.join(SNAPSHOT_STATE_COMPLETE_FILENAME);
+        if file_complete.exists() {
+            continue;
+        }
+
+        match fs::remove_dir_all(bank_snapshot_dir) {
+            Ok(()) => {}
+            Err(err) => {
+                let snapshot_error = SnapshotError::IoWithSourceAndFile(
+                    err,
+                    "Unable to remove bank snapshots directory",
+                    bank_snapshots_dir.as_ref().to_path_buf(),
+                );
+
+                if final_result.is_ok() {
+                    final_result = Err(snapshot_error);
+                }
+            }
+        }
+    }
+
+    final_result
+}
+
 /// Get the bank snapshots in a directory
 ///
 /// This function retains only the bank snapshots of type BankSnapshotType::Pre