[storage] rename target_snapshot_size (#4615)

Co-authored-by: aldenhu <[email protected]>

api/test-context/src/test_context.rs

@@ -8,8 +8,8 @@ use aptos_api_types::{
     X_APTOS_LEDGER_TIMESTAMP, X_APTOS_LEDGER_VERSION,
 };
 use aptos_config::config::{
-    NodeConfig, RocksdbConfigs, DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
-    NO_OP_STORAGE_PRUNER_CONFIG, TARGET_SNAPSHOT_SIZE,
+    NodeConfig, RocksdbConfigs, BUFFERED_STATE_TARGET_ITEMS,
+    DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD, NO_OP_STORAGE_PRUNER_CONFIG,
 };
 use aptos_crypto::{hash::HashValue, SigningKey};
 use aptos_mempool::mocks::MockSharedMempool;
@@ -113,7 +113,7 @@ pub fn new_test_context(test_name: String, use_db_with_indexer: bool) -> TestCon
                 NO_OP_STORAGE_PRUNER_CONFIG, /* pruner */
                 RocksdbConfigs::default(),
                 false, /* indexer */
-                TARGET_SNAPSHOT_SIZE,
+                BUFFERED_STATE_TARGET_ITEMS,
                 DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
             )
             .unwrap(),

aptos-move/aptos-validator-interface/src/storage_interface.rs

@@ -4,8 +4,8 @@
 use crate::AptosValidatorInterface;
 use anyhow::{anyhow, Result};
 use aptos_config::config::{
-    RocksdbConfigs, DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD, NO_OP_STORAGE_PRUNER_CONFIG,
-    TARGET_SNAPSHOT_SIZE,
+    RocksdbConfigs, BUFFERED_STATE_TARGET_ITEMS, DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
+    NO_OP_STORAGE_PRUNER_CONFIG,
 };
 use aptos_types::{
     account_address::AccountAddress,
@@ -29,7 +29,7 @@ impl DBDebuggerInterface {
             NO_OP_STORAGE_PRUNER_CONFIG,
             RocksdbConfigs::default(),
             false,
-            TARGET_SNAPSHOT_SIZE,
+            BUFFERED_STATE_TARGET_ITEMS,
             DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
         )?)))
     }

aptos-node/src/lib.rs

@@ -590,7 +590,7 @@ pub fn setup_environment(
             node_config.storage.storage_pruner_config,
             node_config.storage.rocksdb_configs,
             node_config.storage.enable_indexer,
-            node_config.storage.target_snapshot_size,
+            node_config.storage.buffered_state_target_items,
             node_config.storage.max_num_nodes_per_lru_cache_shard,
         )
         .map_err(|err| anyhow!("DB failed to open {}", err))?,

config/src/config/storage_config.rs

@@ -11,7 +11,7 @@ use std::{
 // Lru cache will consume about 2G RAM based on this default value.
 pub const DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD: usize = 1 << 13;
 
-pub const TARGET_SNAPSHOT_SIZE: usize = 100_000;
+pub const BUFFERED_STATE_TARGET_ITEMS: usize = 100_000;
 
 /// Port selected RocksDB options for tuning underlying rocksdb instance of AptosDB.
 /// see <https://github.com/facebook/rocksdb/blob/master/include/rocksdb/options.h>
@@ -77,8 +77,13 @@ pub struct StorageConfig {
     pub storage_pruner_config: PrunerConfig,
     #[serde(skip)]
     data_dir: PathBuf,
-    /// The threshold that determine whether a snapshot should be committed to state merkle db.
-    pub target_snapshot_size: usize,
+    /// AptosDB persists the state authentication structure off the critical path
+    /// of transaction execution and batch up recent changes for performance. Once
+    /// the number of buffered state updates exceeds this config, a dump of all
+    /// buffered values into a snapshot is triggered. (Alternatively, if too many
+    /// transactions have been processed since last dump, a new dump is processed
+    /// as well.)
+    pub buffered_state_target_items: usize,
     /// The max # of nodes for a lru cache shard.
     pub max_num_nodes_per_lru_cache_shard: usize,
     /// Rocksdb-specific configurations
@@ -234,7 +239,7 @@ impl Default for StorageConfig {
             data_dir: PathBuf::from("/opt/aptos/data"),
             rocksdb_configs: RocksdbConfigs::default(),
             enable_indexer: false,
-            target_snapshot_size: TARGET_SNAPSHOT_SIZE,
+            buffered_state_target_items: BUFFERED_STATE_TARGET_ITEMS,
             max_num_nodes_per_lru_cache_shard: DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
         }
     }

crates/aptos-genesis/src/lib.rs

@@ -13,8 +13,8 @@ pub mod test_utils;
 
 use crate::{builder::GenesisConfiguration, config::ValidatorConfiguration};
 use aptos_config::config::{
-    RocksdbConfigs, DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD, NO_OP_STORAGE_PRUNER_CONFIG,
-    TARGET_SNAPSHOT_SIZE,
+    RocksdbConfigs, BUFFERED_STATE_TARGET_ITEMS, DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
+    NO_OP_STORAGE_PRUNER_CONFIG,
 };
 use aptos_crypto::ed25519::Ed25519PublicKey;
 use aptos_temppath::TempPath;
@@ -137,7 +137,7 @@ impl GenesisInfo {
             NO_OP_STORAGE_PRUNER_CONFIG,
             RocksdbConfigs::default(),
             false,
-            TARGET_SNAPSHOT_SIZE,
+            BUFFERED_STATE_TARGET_ITEMS,
             DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
         )?;
         let db_rw = DbReaderWriter::new(aptosdb);

crates/aptos-genesis/src/mainnet.rs

@@ -3,8 +3,8 @@
 
 use crate::{builder::GenesisConfiguration, config::ValidatorConfiguration};
 use aptos_config::config::{
-    RocksdbConfigs, DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD, NO_OP_STORAGE_PRUNER_CONFIG,
-    TARGET_SNAPSHOT_SIZE,
+    RocksdbConfigs, BUFFERED_STATE_TARGET_ITEMS, DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
+    NO_OP_STORAGE_PRUNER_CONFIG,
 };
 use aptos_temppath::TempPath;
 use aptos_types::{chain_id::ChainId, transaction::Transaction, waypoint::Waypoint};
@@ -124,7 +124,7 @@ impl MainnetGenesisInfo {
             NO_OP_STORAGE_PRUNER_CONFIG,
             RocksdbConfigs::default(),
             false,
-            TARGET_SNAPSHOT_SIZE,
+            BUFFERED_STATE_TARGET_ITEMS,
             DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
         )?;
         let db_rw = DbReaderWriter::new(aptosdb);

execution/db-bootstrapper/src/bin/db-bootstrapper.rs

@@ -3,8 +3,8 @@
 
 use anyhow::{ensure, format_err, Context, Result};
 use aptos_config::config::{
-    RocksdbConfigs, DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD, NO_OP_STORAGE_PRUNER_CONFIG,
-    TARGET_SNAPSHOT_SIZE,
+    RocksdbConfigs, BUFFERED_STATE_TARGET_ITEMS, DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
+    NO_OP_STORAGE_PRUNER_CONFIG,
 };
 use aptos_types::{transaction::Transaction, waypoint::Waypoint};
 use aptos_vm::AptosVM;
@@ -55,7 +55,7 @@ fn main() -> Result<()> {
         NO_OP_STORAGE_PRUNER_CONFIG, /* pruner */
         RocksdbConfigs::default(),
         false, /* indexer */
-        TARGET_SNAPSHOT_SIZE,
+        BUFFERED_STATE_TARGET_ITEMS,
         DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
     )
     .expect("Failed to open DB.");

execution/executor-benchmark/src/db_generator.rs

@@ -8,7 +8,7 @@ use aptos_config::{
 };
 
 use aptos_config::config::{
-    PrunerConfig, DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD, TARGET_SNAPSHOT_SIZE,
+    PrunerConfig, BUFFERED_STATE_TARGET_ITEMS, DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
 };
 use aptos_vm::AptosVM;
 use aptosdb::AptosDB;
@@ -62,7 +62,7 @@ fn bootstrap_with_genesis(db_dir: impl AsRef<Path>) {
             NO_OP_STORAGE_PRUNER_CONFIG,
             rocksdb_configs,
             false, /* indexer */
-            TARGET_SNAPSHOT_SIZE,
+            BUFFERED_STATE_TARGET_ITEMS,
             DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
         )
         .expect("DB should open."),

execution/executor-benchmark/src/lib.rs

@@ -13,8 +13,8 @@ use crate::{
     transaction_generator::TransactionGenerator,
 };
 use aptos_config::config::{
-    NodeConfig, PrunerConfig, RocksdbConfigs, DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
-    NO_OP_STORAGE_PRUNER_CONFIG, TARGET_SNAPSHOT_SIZE,
+    NodeConfig, PrunerConfig, RocksdbConfigs, BUFFERED_STATE_TARGET_ITEMS,
+    DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD, NO_OP_STORAGE_PRUNER_CONFIG,
 };
 use aptos_jellyfish_merkle::metrics::{
     APTOS_JELLYFISH_INTERNAL_ENCODED_BYTES, APTOS_JELLYFISH_LEAF_ENCODED_BYTES,
@@ -35,7 +35,7 @@ pub fn init_db_and_executor(config: &NodeConfig) -> (DbReaderWriter, BlockExecut
             config.storage.storage_pruner_config,
             RocksdbConfigs::default(),
             false,
-            config.storage.target_snapshot_size,
+            config.storage.buffered_state_target_items,
             config.storage.max_num_nodes_per_lru_cache_shard,
         )
         .expect("DB should open."),
@@ -59,7 +59,7 @@ fn create_checkpoint(source_dir: impl AsRef<Path>, checkpoint_dir: impl AsRef<Pa
         NO_OP_STORAGE_PRUNER_CONFIG, /* pruner */
         RocksdbConfigs::default(),
         false,
-        TARGET_SNAPSHOT_SIZE,
+        BUFFERED_STATE_TARGET_ITEMS,
         DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
     )
     .expect("db open failure.")

state-sync/state-sync-v2/state-sync-driver/src/tests/driver_factory.rs

@@ -4,8 +4,8 @@
 use crate::{driver_factory::DriverFactory, metadata_storage::PersistentMetadataStorage};
 use aptos_config::{
     config::{
-        RocksdbConfigs, DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD, NO_OP_STORAGE_PRUNER_CONFIG,
-        TARGET_SNAPSHOT_SIZE,
+        RocksdbConfigs, BUFFERED_STATE_TARGET_ITEMS, DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
+        NO_OP_STORAGE_PRUNER_CONFIG,
     },
     utils::get_genesis_txn,
 };
@@ -39,7 +39,7 @@ fn test_new_initialized_configs() {
         NO_OP_STORAGE_PRUNER_CONFIG,
         RocksdbConfigs::default(),
         false,
-        TARGET_SNAPSHOT_SIZE,
+        BUFFERED_STATE_TARGET_ITEMS,
         DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
     )
     .unwrap();

storage/README.md

@@ -77,8 +77,10 @@ storage:
   # AptosDB persists the state authentication structure off the critical path
   # of transaction execution and batch up recent changes for performance. Once
   # the number of buffered state updates exceeds this config, a dump of all
-  # buffered values into a snapshot is triggered.
-  target_snapshot_size: 100000
+  # buffered values into a snapshot is triggered. (Alternatively, if too many
+  # transactions have been processed since last dump, a new dump is processed
+  # as well.)
+  buffered_state_target_items: 100000
   # Determines the maximum memory consumption by the JMT node cache. Larger cache
   # helps with performance but consumes a lot of memory and can compete with
   # the filesystem cache.

storage/aptosdb/src/aptosdb_test.rs

@@ -12,7 +12,8 @@ use crate::{
 };
 use aptos_config::config::{
     EpochSnapshotPrunerConfig, LedgerPrunerConfig, PrunerConfig, RocksdbConfigs,
-    StateMerklePrunerConfig, DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD, TARGET_SNAPSHOT_SIZE,
+    StateMerklePrunerConfig, BUFFERED_STATE_TARGET_ITEMS,
+    DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
 };
 use aptos_crypto::{hash::CryptoHash, HashValue};
 use aptos_temppath::TempPath;
@@ -205,7 +206,7 @@ pub fn test_state_merkle_pruning_impl(
         },
         RocksdbConfigs::default(),
         false, /* enable_indexer */
-        TARGET_SNAPSHOT_SIZE,
+        BUFFERED_STATE_TARGET_ITEMS,
         DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
     )
     .unwrap();

storage/aptosdb/src/lib.rs

@@ -57,7 +57,8 @@ use anyhow::{bail, ensure, Result};
 #[cfg(any(test, feature = "fuzzing"))]
 use aptos_config::config::DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD;
 use aptos_config::config::{
-    PrunerConfig, RocksdbConfig, RocksdbConfigs, NO_OP_STORAGE_PRUNER_CONFIG, TARGET_SNAPSHOT_SIZE,
+    PrunerConfig, RocksdbConfig, RocksdbConfigs, BUFFERED_STATE_TARGET_ITEMS,
+    NO_OP_STORAGE_PRUNER_CONFIG,
 };
 
 use aptos_crypto::hash::HashValue;
@@ -262,7 +263,7 @@ impl AptosDB {
         ledger_rocksdb: DB,
         state_merkle_rocksdb: DB,
         pruner_config: PrunerConfig,
-        target_snapshot_size: usize,
+        buffered_state_target_items: usize,
         max_nodes_per_lru_cache_shard: usize,
         hack_for_tests: bool,
     ) -> Self {
@@ -281,7 +282,7 @@ impl AptosDB {
             Arc::clone(&arc_state_merkle_rocksdb),
             state_pruner,
             epoch_snapshot_pruner,
-            target_snapshot_size,
+            buffered_state_target_items,
             max_nodes_per_lru_cache_shard,
             hack_for_tests,
         ));
@@ -314,7 +315,7 @@ impl AptosDB {
         pruner_config: PrunerConfig,
         rocksdb_configs: RocksdbConfigs,
         enable_indexer: bool,
-        target_snapshot_size: usize,
+        buffered_state_target_items: usize,
         max_num_nodes_per_lru_cache_shard: usize,
     ) -> Result<Self> {
         ensure!(
@@ -362,7 +363,7 @@ impl AptosDB {
             ledger_db,
             state_merkle_db,
             pruner_config,
-            target_snapshot_size,
+            buffered_state_target_items,
             max_num_nodes_per_lru_cache_shard,
             readonly,
         );
@@ -455,7 +456,7 @@ impl AptosDB {
                 state_merkle_db_column_families(),
             )?,
             NO_OP_STORAGE_PRUNER_CONFIG,
-            TARGET_SNAPSHOT_SIZE,
+            BUFFERED_STATE_TARGET_ITEMS,
             0,
             true,
         ))
@@ -465,7 +466,7 @@ impl AptosDB {
     fn new_without_pruner<P: AsRef<Path> + Clone>(
         db_root_path: P,
         readonly: bool,
-        target_snapshot_size: usize,
+        buffered_state_target_items: usize,
         max_num_nodes_per_lru_cache_shard: usize,
         enable_indexer: bool,
     ) -> Self {
@@ -475,7 +476,7 @@ impl AptosDB {
             NO_OP_STORAGE_PRUNER_CONFIG, /* pruner */
             RocksdbConfigs::default(),
             enable_indexer,
-            target_snapshot_size,
+            buffered_state_target_items,
             max_num_nodes_per_lru_cache_shard,
         )
         .expect("Unable to open AptosDB")
@@ -487,7 +488,7 @@ impl AptosDB {
         Self::new_without_pruner(
             db_root_path,
             false,
-            TARGET_SNAPSHOT_SIZE,
+            BUFFERED_STATE_TARGET_ITEMS,
             DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
             false,
         )
@@ -496,7 +497,7 @@ impl AptosDB {
     /// This opens db in non-readonly mode, without the pruner and cache.
     #[cfg(any(test, feature = "fuzzing"))]
     pub fn new_for_test_no_cache<P: AsRef<Path> + Clone>(db_root_path: P) -> Self {
-        Self::new_without_pruner(db_root_path, false, TARGET_SNAPSHOT_SIZE, 0, false)
+        Self::new_without_pruner(db_root_path, false, BUFFERED_STATE_TARGET_ITEMS, 0, false)
     }
 
     /// This opens db in non-readonly mode, without the pruner, and with the indexer
@@ -505,22 +506,22 @@ impl AptosDB {
         Self::new_without_pruner(
             db_root_path,
             false,
-            TARGET_SNAPSHOT_SIZE,
+            BUFFERED_STATE_TARGET_ITEMS,
             DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
             true,
         )
     }
 
     /// This opens db in non-readonly mode, without the pruner.
     #[cfg(any(test, feature = "fuzzing"))]
-    pub fn new_for_test_with_target_snapshot_size<P: AsRef<Path> + Clone>(
+    pub fn new_for_test_with_buffered_state_target_items<P: AsRef<Path> + Clone>(
         db_root_path: P,
-        target_snapshot_size: usize,
+        buffered_state_target_items: usize,
     ) -> Self {
         Self::new_without_pruner(
             db_root_path,
             false,
-            target_snapshot_size,
+            buffered_state_target_items,
             DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
             false,
         )
@@ -532,7 +533,7 @@ impl AptosDB {
         Self::new_without_pruner(
             db_root_path,
             true,
-            TARGET_SNAPSHOT_SIZE,
+            BUFFERED_STATE_TARGET_ITEMS,
             DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
             false,
         )