Add maximum_incremental_snapshot_archives_to_retain

Fixes solana-labs#18639

core/src/accounts_hash_verifier.rs

@@ -343,6 +343,7 @@ mod tests {
             archive_format: ArchiveFormat::Tar,
             snapshot_version: SnapshotVersion::default(),
             maximum_full_snapshot_archives_to_retain: usize::MAX,
+            maximum_incremental_snapshot_archives_to_retain: usize::MAX,
         };
         for i in 0..MAX_SNAPSHOT_HASHES + 1 {
             let accounts_package = AccountsPackage {

core/src/snapshot_packager_service.rs

@@ -1,7 +1,7 @@
 use solana_gossip::cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES};
 use solana_runtime::{
-    snapshot_archive_info::SnapshotArchiveInfoGetter, snapshot_package::PendingSnapshotPackage,
-    snapshot_utils,
+    snapshot_archive_info::SnapshotArchiveInfoGetter, snapshot_config::SnapshotConfig,
+    snapshot_package::PendingSnapshotPackage, snapshot_utils,
 };
 use solana_sdk::{clock::Slot, hash::Hash};
 use std::{
@@ -23,7 +23,7 @@ impl SnapshotPackagerService {
         starting_snapshot_hash: Option<(Slot, Hash)>,
         exit: &Arc<AtomicBool>,
         cluster_info: &Arc<ClusterInfo>,
-        maximum_snapshots_to_retain: usize,
+        snapshot_config: SnapshotConfig,
     ) -> Self {
         let exit = exit.clone();
         let cluster_info = cluster_info.clone();
@@ -53,7 +53,8 @@ impl SnapshotPackagerService {
                     // last_full_snapshot_slot that requires this archive call to succeed.
                     snapshot_utils::archive_snapshot_package(
                         &snapshot_package,
-                        maximum_snapshots_to_retain,
+                        snapshot_config.maximum_full_snapshot_archives_to_retain,
+                        snapshot_config.maximum_incremental_snapshot_archives_to_retain,
                     )
                     .expect("failed to archive snapshot package");
 
@@ -190,6 +191,7 @@ mod tests {
         snapshot_utils::archive_snapshot_package(
             &snapshot_package,
             snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
+            snapshot_utils::DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
         )
         .unwrap();
 

core/src/test_validator.rs

@@ -18,6 +18,7 @@ use {
         snapshot_config::SnapshotConfig,
         snapshot_utils::{
             ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
+            DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
         },
     },
     solana_sdk::{
@@ -529,6 +530,8 @@ impl TestValidator {
                 snapshot_version: SnapshotVersion::default(),
                 maximum_full_snapshot_archives_to_retain:
                     DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
+                maximum_incremental_snapshot_archives_to_retain:
+                    DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
             }),
             enforce_ulimit_nofile: false,
             warp_slot: config.warp_slot,

core/src/validator.rs

@@ -651,7 +651,7 @@ impl Validator {
                     snapshot_hash,
                     &exit,
                     &cluster_info,
-                    snapshot_config.maximum_full_snapshot_archives_to_retain,
+                    snapshot_config.clone(),
                 );
                 (
                     Some(snapshot_packager_service),
@@ -1238,6 +1238,7 @@ fn new_banks_from_ledger(
             &snapshot_config.snapshot_archives_dir,
             snapshot_config.archive_format,
             snapshot_config.maximum_full_snapshot_archives_to_retain,
+            snapshot_config.maximum_incremental_snapshot_archives_to_retain,
         )
         .unwrap_or_else(|err| {
             error!("Unable to create snapshot: {}", err);

core/tests/snapshots.rs

@@ -71,6 +71,7 @@ mod tests {
         },
         snapshot_utils::{
             self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
+            DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
         },
         status_cache::MAX_CACHE_ENTRIES,
     };
@@ -149,6 +150,8 @@ mod tests {
                 snapshot_version,
                 maximum_full_snapshot_archives_to_retain:
                     DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
+                maximum_incremental_snapshot_archives_to_retain:
+                    DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
             };
             bank_forks.set_snapshot_config(Some(snapshot_config.clone()));
             SnapshotTestConfig {
@@ -296,7 +299,8 @@ mod tests {
         let snapshot_package = SnapshotPackage::from(accounts_package);
         snapshot_utils::archive_snapshot_package(
             &snapshot_package,
-            DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
+            snapshot_config.maximum_full_snapshot_archives_to_retain,
+            snapshot_config.maximum_incremental_snapshot_archives_to_retain,
         )
         .unwrap();
 
@@ -384,7 +388,7 @@ mod tests {
         let saved_slot = 4;
         let mut saved_archive_path = None;
 
-        for forks in 0..snapshot_utils::MAX_BANK_SNAPSHOTS + 2 {
+        for forks in 0..snapshot_utils::MAX_BANK_SNAPSHOTS_TO_RETAIN + 2 {
             let bank = Bank::new_from_parent(
                 &bank_forks[forks as u64],
                 &Pubkey::default(),
@@ -480,7 +484,7 @@ mod tests {
         assert!(bank_snapshots
             .into_iter()
             .map(|path| path.slot)
-            .eq(3..=snapshot_utils::MAX_BANK_SNAPSHOTS as u64 + 2));
+            .eq(3..=snapshot_utils::MAX_BANK_SNAPSHOTS_TO_RETAIN as u64 + 2));
 
         // Create a SnapshotPackagerService to create tarballs from all the pending
         // SnapshotPackage's on the channel. By the time this service starts, we have already
@@ -502,7 +506,7 @@ mod tests {
             None,
             &exit,
             &cluster_info,
-            DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
+            snapshot_config.clone(),
         );
 
         let _package_receiver = std::thread::Builder::new()
@@ -773,6 +777,7 @@ mod tests {
             snapshot_config.archive_format,
             snapshot_config.snapshot_version,
             snapshot_config.maximum_full_snapshot_archives_to_retain,
+            snapshot_config.maximum_incremental_snapshot_archives_to_retain,
         )?;
 
         Ok(())
@@ -809,6 +814,7 @@ mod tests {
             snapshot_config.archive_format,
             snapshot_config.snapshot_version,
             snapshot_config.maximum_full_snapshot_archives_to_retain,
+            snapshot_config.maximum_incremental_snapshot_archives_to_retain,
         )?;
 
         Ok(())
@@ -923,9 +929,7 @@ mod tests {
             None,
             &exit,
             &cluster_info,
-            snapshot_test_config
-                .snapshot_config
-                .maximum_full_snapshot_archives_to_retain,
+            snapshot_test_config.snapshot_config.clone(),
         );
 
         let accounts_hash_verifier = AccountsHashVerifier::new(

download-utils/src/lib.rs

@@ -249,10 +249,15 @@ pub fn download_snapshot<'a, 'b>(
     snapshot_archives_dir: &Path,
     desired_snapshot_hash: (Slot, Hash),
     use_progress_bar: bool,
-    maximum_snapshots_to_retain: usize,
+    maximum_full_snapshot_archives_to_retain: usize,
+    maximum_incremental_snapshot_archives_to_retain: usize,
     progress_notify_callback: &'a mut DownloadProgressCallbackOption<'b>,
 ) -> Result<(), String> {
-    snapshot_utils::purge_old_snapshot_archives(snapshot_archives_dir, maximum_snapshots_to_retain);
+    snapshot_utils::purge_old_snapshot_archives(
+        snapshot_archives_dir,
+        maximum_full_snapshot_archives_to_retain,
+        maximum_incremental_snapshot_archives_to_retain,
+    );
 
     for compression in &[
         ArchiveFormat::TarZstd,

ledger-tool/src/main.rs

@@ -34,6 +34,7 @@ use solana_runtime::{
     snapshot_config::SnapshotConfig,
     snapshot_utils::{
         self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
+        DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
     },
 };
 use solana_sdk::{
@@ -719,6 +720,8 @@ fn load_bank_forks(
             archive_format: ArchiveFormat::TarBzip2,
             snapshot_version: SnapshotVersion::default(),
             maximum_full_snapshot_archives_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
+            maximum_incremental_snapshot_archives_to_retain:
+                DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
         })
     };
     let account_paths = if let Some(account_paths) = arg_matches.value_of("account_paths") {
@@ -2012,7 +2015,7 @@ fn main() {
                         })
                     });
 
-            let maximum_snapshots_to_retain =
+            let maximum_full_snapshot_archives_to_retain =
                 value_t_or_exit!(arg_matches, "maximum_snapshots_to_retain", usize);
             let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
             let blockstore = open_blockstore(
@@ -2229,7 +2232,8 @@ fn main() {
                         Some(snapshot_version),
                         output_directory,
                         ArchiveFormat::TarZstd,
-                        maximum_snapshots_to_retain,
+                        maximum_full_snapshot_archives_to_retain,
+                        snapshot_utils::DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
                     )
                     .unwrap_or_else(|err| {
                         eprintln!("Unable to create snapshot: {}", err);

local-cluster/tests/local_cluster.rs

@@ -1708,6 +1708,7 @@ fn test_snapshot_download() {
         archive_snapshot_hash,
         false,
         snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
+        snapshot_utils::DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
         &mut None,
     )
     .unwrap();
@@ -3536,6 +3537,8 @@ fn setup_snapshot_validator_config(
         snapshot_version: snapshot_utils::SnapshotVersion::default(),
         maximum_full_snapshot_archives_to_retain:
             snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
+        maximum_incremental_snapshot_archives_to_retain:
+            snapshot_utils::DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
     };
 
     // Create the account paths

replica-node/src/replica_node.rs

@@ -95,6 +95,7 @@ fn initialize_from_snapshot(
         replica_config.snapshot_info,
         false,
         snapshot_config.maximum_full_snapshot_archives_to_retain,
+        snapshot_config.maximum_incremental_snapshot_archives_to_retain,
         &mut None,
     )
     .unwrap();
@@ -270,6 +271,8 @@ impl ReplicaNode {
             snapshot_version: snapshot_utils::SnapshotVersion::default(),
             maximum_full_snapshot_archives_to_retain:
                 snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
+            maximum_incremental_snapshot_archives_to_retain:
+                snapshot_utils::DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
         };
 
         let bank_info =

replica-node/tests/local_replica.rs

@@ -129,6 +129,8 @@ fn setup_snapshot_validator_config(
         snapshot_version: snapshot_utils::SnapshotVersion::default(),
         maximum_full_snapshot_archives_to_retain:
             snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
+        maximum_incremental_snapshot_archives_to_retain:
+            snapshot_utils::DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
     };
 
     // Create the account paths

rpc/src/rpc_service.rs

@@ -506,6 +506,7 @@ mod tests {
             bank::Bank,
             snapshot_utils::{
                 ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
+                DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
             },
         },
         solana_sdk::{
@@ -623,6 +624,8 @@ mod tests {
                 snapshot_version: SnapshotVersion::default(),
                 maximum_full_snapshot_archives_to_retain:
                     DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
+                maximum_incremental_snapshot_archives_to_retain:
+                    DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
             }),
             bank_forks,
             RpcHealth::stub(),

runtime/src/snapshot_config.rs

@@ -26,4 +26,8 @@ pub struct SnapshotConfig {
 
     /// Maximum number of full snapshot archives to retain
     pub maximum_full_snapshot_archives_to_retain: usize,
+
+    /// Maximum number of incremental snapshot archives to retain
+    /// NOTE: Incremental snapshots will only be kept for the latest full snapshot
+    pub maximum_incremental_snapshot_archives_to_retain: usize,
 }