diff --git a/Cargo.lock b/Cargo.lock index e2d879d537..aa05af941f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3715,6 +3715,7 @@ dependencies = [ "serde_json", "sha2", "structopt", + "tari_storage", "tari_test_utils", "tempfile", "toml 0.5.6", diff --git a/applications/tari_base_node/src/builder.rs b/applications/tari_base_node/src/builder.rs index fe18a87a43..5bc1b39670 100644 --- a/applications/tari_base_node/src/builder.rs +++ b/applications/tari_base_node/src/builder.rs @@ -21,7 +21,7 @@ // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. use crate::miner; -use futures::{channel::mpsc, future}; +use futures::future; use log::*; use std::{ fs, @@ -334,7 +334,7 @@ where B: 'static } /// Sets up and initializes the base node, creating the context and database -/// ## Paramters +/// ## Parameters /// `config` - The configuration for the base node /// `node_identity` - The node identity information of the base node /// `wallet_node_identity` - The node identity information of the base node's wallet @@ -367,7 +367,8 @@ pub async fn configure_and_initialize_node( NodeContainer::Memory(ctx) }, DatabaseType::LMDB(p) => { - let backend = create_lmdb_database(&p, MmrCacheConfig::default()).map_err(|e| e.to_string())?; + let backend = create_lmdb_database(&p, config.db_config.clone(), MmrCacheConfig::default()) + .map_err(|e| e.to_string())?; let ctx = build_node_context( backend, network, @@ -383,8 +384,9 @@ pub async fn configure_and_initialize_node( Ok(result) } -/// Constructs the base node context, this includes settin up the consensus manager, mempool, base node, wallet, miner -/// and state machine ## Paramters +/// Constructs the base node context, this includes setting up the consensus manager, mempool, base node, wallet, miner +/// and state machine +/// ## Parameters /// `backend` - Backend interface /// `network` - The NetworkType (rincewind, mainnet, local) /// `base_node_identity` - The node identity information of the base node diff --git a/base_layer/core/src/chain_storage/error.rs b/base_layer/core/src/chain_storage/error.rs index 36d2ce5d04..171a0e86bf 100644 --- a/base_layer/core/src/chain_storage/error.rs +++ b/base_layer/core/src/chain_storage/error.rs @@ -25,6 +25,7 @@ use crate::{ validation::ValidationError, }; use tari_mmr::{error::MerkleMountainRangeError, MerkleProofError}; +use tari_storage::lmdb_store::LMDBError; use thiserror::Error; use tokio::task; @@ -85,6 +86,11 @@ pub enum ChainStorageError { OutOfRange, #[error("Value not found: {0}")] LmdbValueNotFound(lmdb_zero::Error), + #[error("LMDB error: {source}")] + LmdbError { + #[from] + source: LMDBError, + }, } impl From for ChainStorageError { diff --git a/base_layer/core/src/chain_storage/lmdb_db/lmdb.rs b/base_layer/core/src/chain_storage/lmdb_db/lmdb.rs index 33334c54de..d634dc345a 100644 --- a/base_layer/core/src/chain_storage/lmdb_db/lmdb.rs +++ b/base_layer/core/src/chain_storage/lmdb_db/lmdb.rs @@ -45,21 +45,19 @@ pub fn serialize(data: &T) -> Result, ChainStorageError> where T: Serialize { let size = bincode::serialized_size(&data).map_err(|e| ChainStorageError::AccessError(e.to_string()))?; let mut buf = Vec::with_capacity(size as usize); - bincode::serialize_into(&mut buf, data) - .or_else(|e| { - error!(target: LOG_TARGET, "Could not serialize lmdb: {:?}", e); - Err(e) - }) - .map_err(|e| ChainStorageError::AccessError(e.to_string()))?; + bincode::serialize_into(&mut buf, data).map_err(|e| { + error!(target: LOG_TARGET, "Could not serialize lmdb: {:?}", e); + ChainStorageError::AccessError(e.to_string()) + })?; Ok(buf) } pub fn deserialize(buf_bytes: &[u8]) -> Result where T: DeserializeOwned { bincode::deserialize(buf_bytes) - .or_else(|e| { + .map_err(|e| { error!(target: LOG_TARGET, "Could not deserialize lmdb: {:?}", e); - Err(e) + e }) .map_err(|e| error::Error::ValRejected(e.to_string())) } diff --git a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs index 15a26fa8c8..3e792f36e2 100644 --- a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs +++ b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs @@ -90,7 +90,7 @@ use tari_mmr::{ MmrCacheConfig, MutableMmr, }; -use tari_storage::lmdb_store::{db, LMDBBuilder, LMDBStore}; +use tari_storage::lmdb_store::{db, LMDBBuilder, LMDBConfig, LMDBStore}; type DatabaseRef = Arc>; @@ -101,6 +101,7 @@ pub struct LMDBDatabase where D: Digest { env: Arc, + env_config: LMDBConfig, metadata_db: DatabaseRef, mem_metadata: ChainMetadata, // Memory copy of stored metadata headers_db: DatabaseRef, @@ -164,6 +165,7 @@ where D: Digest + Send + Sync }, range_proof_checkpoints, env, + env_config: store.env_config(), is_mem_metadata_dirty: false, }) } @@ -311,7 +313,7 @@ where D: Digest + Send + Sync fn op_insert(&mut self, txn: &WriteTransaction<'_>, kv_pair: &DbKeyValuePair) -> Result<(), ChainStorageError> { match kv_pair { DbKeyValuePair::Metadata(k, v) => { - lmdb_replace(&txn, &self.metadata_db, &(k.clone() as u32), &v)?; + lmdb_replace(&txn, &self.metadata_db, &(*k as u32), &v)?; self.is_mem_metadata_dirty = true; }, DbKeyValuePair::BlockHeader(k, v) => { @@ -509,6 +511,7 @@ where D: Digest + Send + Sync pub fn create_lmdb_database>( path: P, + config: LMDBConfig, mmr_cache_config: MmrCacheConfig, ) -> Result, ChainStorageError> { @@ -516,7 +519,7 @@ pub fn create_lmdb_database>( let _ = std::fs::create_dir_all(&path); let lmdb_store = LMDBBuilder::new() .set_path(path) - .set_environment_size(1_000) + .set_env_config(config) .set_max_number_of_databases(15) .add_database(LMDB_DB_METADATA, flags) .add_database(LMDB_DB_HEADERS, flags) @@ -542,6 +545,8 @@ where D: Digest + Send + Sync return Ok(()); } + LMDBStore::resize_if_required(&self.env, &self.env_config)?; + let mark = Instant::now(); let num_operations = txn.operations.len(); match self.apply_db_transaction(&txn) { @@ -566,7 +571,7 @@ where D: Digest + Send + Sync fn fetch(&self, key: &DbKey) -> Result, ChainStorageError> { Ok(match key { DbKey::Metadata(k) => { - let val: Option = lmdb_get(&self.env, &self.metadata_db, &(k.clone() as u32))?; + let val: Option = lmdb_get(&self.env, &self.metadata_db, &(*k as u32))?; val.map(DbValue::Metadata) }, DbKey::BlockHeader(k) => { @@ -1004,9 +1009,10 @@ mod test { let flags = db::CREATE; let _ = std::fs::remove_dir_all(&path); std::fs::create_dir_all(&path).unwrap(); + let config = LMDBConfig::new_from_mb(1000, 100, 100); let lmdb_store = LMDBBuilder::new() .set_path(path) - .set_environment_size(1_000) + .set_env_config(config) .set_max_number_of_databases(15) .add_database("1", flags) .add_database("2", flags) diff --git a/base_layer/core/src/chain_storage/lmdb_db/lmdb_vec.rs b/base_layer/core/src/chain_storage/lmdb_db/lmdb_vec.rs index 0485e71dd0..70ab11cea8 100644 --- a/base_layer/core/src/chain_storage/lmdb_db/lmdb_vec.rs +++ b/base_layer/core/src/chain_storage/lmdb_db/lmdb_vec.rs @@ -284,7 +284,7 @@ where #[cfg(test)] mod test { use super::*; - use tari_storage::lmdb_store::{db, LMDBBuilder}; + use tari_storage::lmdb_store::{db, LMDBBuilder, LMDBConfig}; use tari_test_utils::paths::create_temporary_data_path; #[test] @@ -293,7 +293,7 @@ mod test { let _ = std::fs::create_dir(&path).unwrap_or_default(); let lmdb_store = LMDBBuilder::new() .set_path(&path) - .set_environment_size(1) + .set_env_config(LMDBConfig::default()) .set_max_number_of_databases(1) .add_database("db", db::CREATE) .build() diff --git a/base_layer/core/tests/chain_storage_tests/chain_backend.rs b/base_layer/core/tests/chain_storage_tests/chain_backend.rs index d005b82da9..556e90e2d5 100644 --- a/base_layer/core/tests/chain_storage_tests/chain_backend.rs +++ b/base_layer/core/tests/chain_storage_tests/chain_backend.rs @@ -48,6 +48,7 @@ use tari_core::{ }; use tari_crypto::tari_utilities::{epoch_time::EpochTime, hex::Hex, Hashable}; use tari_mmr::{MmrCacheConfig, MutableMmr}; +use tari_storage::lmdb_store::LMDBConfig; use tari_test_utils::paths::create_temporary_data_path; fn insert_contains_delete_and_fetch_header(mut db: T) { @@ -93,7 +94,7 @@ fn lmdb_insert_contains_delete_and_fetch_header() { // Perform test { - let db = create_lmdb_database(&temp_path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); insert_contains_delete_and_fetch_header(db); } @@ -138,7 +139,7 @@ fn lmdb_insert_contains_delete_and_fetch_utxo() { // Perform test { - let db = create_lmdb_database(&temp_path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); insert_contains_delete_and_fetch_utxo(db); } @@ -184,7 +185,7 @@ fn lmdb_insert_contains_delete_and_fetch_kernel() { // Perform test { - let db = create_lmdb_database(&temp_path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); insert_contains_delete_and_fetch_kernel(db); } @@ -237,7 +238,7 @@ fn lmdb_insert_contains_delete_and_fetch_orphan() { { let network = Network::LocalNet; let consensus_constants = network.create_consensus_constants(); - let db = create_lmdb_database(&temp_path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); insert_contains_delete_and_fetch_orphan(db, &consensus_constants); } @@ -309,7 +310,7 @@ fn lmdb_spend_utxo_and_unspend_stxo() { // Perform test { - let db = create_lmdb_database(&temp_path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); spend_utxo_and_unspend_stxo(db); } @@ -399,7 +400,7 @@ fn lmdb_insert_fetch_metadata() { // Perform test { - let db = create_lmdb_database(&temp_path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); insert_fetch_metadata(db); } @@ -485,7 +486,7 @@ fn lmdb_fetch_mmr_root_and_proof_for_utxo_and_rp() { // Perform test { - let db = create_lmdb_database(&temp_path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); fetch_mmr_root_and_proof_for_utxo_and_rp(db); } @@ -546,7 +547,7 @@ fn lmdb_fetch_mmr_root_and_proof_for_kernel() { // Perform test { - let db = create_lmdb_database(&temp_path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); fetch_mmr_root_and_proof_for_kernel(db); } @@ -608,7 +609,7 @@ fn lmdb_fetch_future_mmr_root_for_utxo_and_rp() { // Perform test { - let db = create_lmdb_database(&temp_path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); fetch_future_mmr_root_for_utxo_and_rp(db); } @@ -658,7 +659,7 @@ fn lmdb_fetch_future_mmr_root_for_for_kernel() { // Perform test { - let db = create_lmdb_database(&temp_path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); fetch_future_mmr_root_for_for_kernel(db); } @@ -776,7 +777,7 @@ fn lmdb_commit_block_and_create_fetch_checkpoint_and_rewind_mmr() { // Perform test { - let db = create_lmdb_database(&temp_path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); commit_block_and_create_fetch_checkpoint_and_rewind_mmr(db); } @@ -850,7 +851,7 @@ fn lmdb_for_each_orphan() { { let network = Network::LocalNet; let consensus_constants = network.create_consensus_constants(); - let db = create_lmdb_database(&temp_path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); for_each_orphan(db, &consensus_constants); } @@ -908,7 +909,7 @@ fn lmdb_for_each_kernel() { // Perform test { - let db = create_lmdb_database(&temp_path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); for_each_kernel(db); } @@ -966,7 +967,7 @@ fn lmdb_for_each_header() { // Perform test { - let db = create_lmdb_database(&temp_path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); for_each_header(db); } @@ -1025,7 +1026,7 @@ fn lmdb_for_each_utxo() { // Perform test { - let db = create_lmdb_database(&temp_path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); for_each_utxo(db); } @@ -1058,7 +1059,7 @@ fn lmdb_backend_restore() { let path = create_temporary_data_path(); { { - let mut db = create_lmdb_database(&path, MmrCacheConfig::default()).unwrap(); + let mut db = create_lmdb_database(&path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); let mut txn = DbTransaction::new(); txn.insert_orphan(orphan.clone()); txn.insert_utxo(utxo1); @@ -1082,7 +1083,7 @@ fn lmdb_backend_restore() { assert_eq!(db.contains(&DbKey::OrphanBlock(orphan_hash.clone())).unwrap(), true); } // Restore backend storage - let db = create_lmdb_database(&path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); assert_eq!(db.contains(&DbKey::BlockHeader(header.height)).unwrap(), true); assert_eq!(db.contains(&DbKey::BlockHash(header_hash)).unwrap(), true); assert_eq!(db.contains(&DbKey::UnspentOutput(utxo_hash)).unwrap(), true); @@ -1105,7 +1106,7 @@ fn lmdb_mmr_reset_and_commit() { // Perform test { let factories = CryptoFactories::default(); - let mut db = create_lmdb_database(&temp_path, MmrCacheConfig::default()).unwrap(); + let mut db = create_lmdb_database(&temp_path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); let (utxo1, _) = create_utxo(MicroTari(10_000), &factories, None); let (utxo2, _) = create_utxo(MicroTari(15_000), &factories, None); @@ -1320,7 +1321,7 @@ fn lmdb_fetch_checkpoint() { // Perform test { let mmr_cache_config = MmrCacheConfig { rewind_hist_len: 1 }; - let db = create_lmdb_database(&temp_path, mmr_cache_config).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), mmr_cache_config).unwrap(); fetch_checkpoint(db); } @@ -1467,7 +1468,7 @@ fn lmdb_merging_and_fetch_checkpoints_and_stxo_discard() { // Perform test { let mmr_cache_config = MmrCacheConfig { rewind_hist_len: 1 }; - let db = create_lmdb_database(&temp_path, mmr_cache_config).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), mmr_cache_config).unwrap(); merging_and_fetch_checkpoints_and_stxo_discard(db); } @@ -1515,7 +1516,7 @@ fn lmdb_duplicate_utxo() { // Perform test { - let db = create_lmdb_database(&temp_path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); duplicate_utxo(db); } @@ -1559,7 +1560,7 @@ fn lmdb_fetch_last_header() { // Perform test { - let db = create_lmdb_database(&temp_path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); fetch_last_header(db); } @@ -1659,7 +1660,7 @@ fn lmdb_fetch_target_difficulties() { // Perform test { - let db = create_lmdb_database(&temp_path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); fetch_target_difficulties(db); } @@ -1768,7 +1769,7 @@ fn lmdb_fetch_utxo_rp_nodes_and_count() { // Perform test { - let db = create_lmdb_database(&temp_path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); fetch_utxo_rp_mmr_nodes_and_count(db); } @@ -1845,7 +1846,7 @@ fn lmdb_fetch_kernel_nodes_and_count() { // Perform test { - let db = create_lmdb_database(&temp_path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); fetch_kernel_mmr_nodes_and_count(db); } @@ -1937,7 +1938,7 @@ fn lmdb_insert_mmr_node_for_utxo_and_rp() { // Perform test { - let db = create_lmdb_database(&temp_path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); insert_mmr_node_for_utxo_and_rp(db); } diff --git a/base_layer/core/tests/chain_storage_tests/chain_storage.rs b/base_layer/core/tests/chain_storage_tests/chain_storage.rs index bb01fef215..ac22d0b936 100644 --- a/base_layer/core/tests/chain_storage_tests/chain_storage.rs +++ b/base_layer/core/tests/chain_storage_tests/chain_storage.rs @@ -72,6 +72,7 @@ use tari_core::{ }; use tari_crypto::tari_utilities::{hex::Hex, Hashable}; use tari_mmr::{MmrCacheConfig, MutableMmr}; +use tari_storage::lmdb_store::LMDBConfig; use tari_test_utils::{paths::create_temporary_data_path, unpack_enum}; fn init_log() { @@ -1201,7 +1202,7 @@ fn restore_metadata_and_pruning_horizon_update() { let pruning_horizon1: u64 = 1000; let pruning_horizon2: u64 = 900; { - let db = create_lmdb_database(&path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); config.pruning_horizon = pruning_horizon1; let db = BlockchainDatabase::new(db, &rules, validators.clone(), config).unwrap(); @@ -1216,7 +1217,7 @@ fn restore_metadata_and_pruning_horizon_update() { // Restore blockchain db with larger pruning horizon { config.pruning_horizon = 2000; - let db = create_lmdb_database(&path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); let db = BlockchainDatabase::new(db, &rules, validators.clone(), config).unwrap(); let metadata = db.get_chain_metadata().unwrap(); @@ -1227,7 +1228,7 @@ fn restore_metadata_and_pruning_horizon_update() { // Restore blockchain db with smaller pruning horizon update { config.pruning_horizon = 900; - let db = create_lmdb_database(&path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); let db = BlockchainDatabase::new(db, &rules, validators, config).unwrap(); let metadata = db.get_chain_metadata().unwrap(); @@ -1262,7 +1263,7 @@ fn invalid_block() { MockStatelessBlockValidator::new(consensus_manager.clone(), factories.clone()), MockAccumDifficultyValidator {}, ); - let db = create_lmdb_database(&temp_path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(&temp_path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); let mut store = BlockchainDatabase::new(db, &consensus_manager, validators, BlockchainDatabaseConfig::default()).unwrap(); let mut blocks = vec![block0]; diff --git a/base_layer/core/tests/helpers/sample_blockchains.rs b/base_layer/core/tests/helpers/sample_blockchains.rs index e230e388d0..a4a6502d9d 100644 --- a/base_layer/core/tests/helpers/sample_blockchains.rs +++ b/base_layer/core/tests/helpers/sample_blockchains.rs @@ -43,6 +43,7 @@ use tari_core::{ txn_schema, }; use tari_mmr::MmrCacheConfig; +use tari_storage::lmdb_store::LMDBConfig; /// Create a simple 6 block memory-backed database. /// Genesis block: @@ -205,7 +206,7 @@ pub fn create_new_blockchain_lmdb>( .with_consensus_constants(consensus_constants) .with_block(block0.clone()) .build(); - let db = create_lmdb_database(path, MmrCacheConfig::default()).unwrap(); + let db = create_lmdb_database(path, LMDBConfig::default(), MmrCacheConfig::default()).unwrap(); let db = BlockchainDatabase::new(db, &consensus_manager, validators, config).unwrap(); (db, vec![block0], vec![vec![output]], consensus_manager) } diff --git a/base_layer/p2p/src/initialization.rs b/base_layer/p2p/src/initialization.rs index 9fca844ae0..a91c783005 100644 --- a/base_layer/p2p/src/initialization.rs +++ b/base_layer/p2p/src/initialization.rs @@ -43,7 +43,10 @@ use tari_comms::{ PeerManager, }; use tari_comms_dht::{Dht, DhtBuilder, DhtConfig, DhtInitializationError}; -use tari_storage::{lmdb_store::LMDBBuilder, LMDBWrapper}; +use tari_storage::{ + lmdb_store::{LMDBBuilder, LMDBConfig}, + LMDBWrapper, +}; use thiserror::Error; use tower::ServiceBuilder; @@ -134,7 +137,7 @@ where std::fs::create_dir_all(data_path).unwrap(); let datastore = LMDBBuilder::new() .set_path(data_path) - .set_environment_size(50) + .set_env_config(LMDBConfig::default()) .set_max_number_of_databases(1) .add_database(&peer_database_name, lmdb_zero::db::CREATE) .build() @@ -301,7 +304,7 @@ where { let datastore = LMDBBuilder::new() .set_path(&config.datastore_path) - .set_environment_size(50) + .set_env_config(LMDBConfig::default()) .set_max_number_of_databases(1) .add_database(&config.peer_database_name, lmdb_zero::db::CREATE) .build() diff --git a/common/Cargo.toml b/common/Cargo.toml index 7a02121502..baa5048163 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -23,6 +23,7 @@ multiaddr={package="parity-multiaddr", version = "0.7.2"} prost-build = "0.6.1" sha2 = "0.8.0" path-clean = "0.1.0" +tari_storage = { version = "^0.2", path = "../infrastructure/storage"} [dev-dependencies] tempfile = "3.1.0" diff --git a/common/config/presets/windows.toml b/common/config/presets/windows.toml index 170e298ea4..b99c8aa5bf 100644 --- a/common/config/presets/windows.toml +++ b/common/config/presets/windows.toml @@ -116,6 +116,11 @@ network = "rincewind" # almost all use cases. db_type = "lmdb" +# db config defaults +# db_init_size_mb = 1000 +# db_grow_size_mb = 500 +# db_resize_threshold_mb = 100 + # The maximum number of orphans that can be stored in the Orphan block pool. Default value is "720". #orphan_storage_capacity = 720 # The pruning horizon that indicates how many full blocks without pruning must be kept by the base node. Default value diff --git a/common/config/tari_config_sample.toml b/common/config/tari_config_sample.toml index 5861713083..4df6819aea 100644 --- a/common/config/tari_config_sample.toml +++ b/common/config/tari_config_sample.toml @@ -116,6 +116,11 @@ network = "rincewind" # almost all use cases. db_type = "lmdb" +# db config defaults +# db_init_size_mb = 1000 +# db_grow_size_mb = 500 +# db_resize_threshold_mb = 100 + # The maximum number of orphans that can be stored in the Orphan block pool. Default value is "720". #orphan_storage_capacity = 720 # The pruning horizon that indicates how many full blocks without pruning must be kept by the base node. Default value diff --git a/common/src/configuration/global.rs b/common/src/configuration/global.rs index ada4a10cc3..f7883d2a5f 100644 --- a/common/src/configuration/global.rs +++ b/common/src/configuration/global.rs @@ -23,7 +23,7 @@ //! # Global configuration of tari base layer system use super::ConfigurationError; -use config::{Config, Environment}; +use config::{Config, ConfigError, Environment}; use multiaddr::Multiaddr; use std::{ convert::TryInto, @@ -34,6 +34,15 @@ use std::{ str::FromStr, time::Duration, }; +use tari_storage::lmdb_store::LMDBConfig; + +const DB_INIT_DEFAULT_MB: usize = 1000; +const DB_GROW_DEFAULT_MB: usize = 500; +const DB_RESIZE_DEFAULT_MB: usize = 100; + +const DB_INIT_MIN_MB: i64 = 100; +const DB_GROW_MIN_MB: i64 = 20; +const DB_RESIZE_MIN_MB: i64 = 10; //------------------------------------- Main Configuration Struct --------------------------------------// @@ -45,6 +54,7 @@ pub struct GlobalConfig { pub listener_liveness_allowlist_cidrs: Vec, pub data_dir: PathBuf, pub db_type: DatabaseType, + pub db_config: LMDBConfig, pub orphan_storage_capacity: usize, pub pruning_horizon: u64, pub pruned_mode_cleanup_interval: u64, @@ -98,18 +108,18 @@ impl GlobalConfig { fn convert_node_config(network: Network, cfg: Config) -> Result { let net_str = network.to_string().to_lowercase(); - let key = config_string("base_node", &net_str, "db_type"); - let db_type = cfg - .get_str(&key) - .map(|s| s.to_lowercase()) - .map_err(|e| ConfigurationError::new(&key, &e.to_string()))?; - let key = config_string("base_node", &net_str, "data_dir"); let data_dir: PathBuf = cfg .get_str(&key) .map_err(|e| ConfigurationError::new(&key, &e.to_string()))? .into(); + let key = config_string("base_node", &net_str, "db_type"); + let db_type = cfg + .get_str(&key) + .map(|s| s.to_lowercase()) + .map_err(|e| ConfigurationError::new(&key, &e.to_string()))?; + let db_type = match db_type.as_str() { "memory" => Ok(DatabaseType::Memory), "lmdb" => Ok(DatabaseType::LMDB(data_dir.join("db"))), @@ -119,6 +129,65 @@ fn convert_node_config(network: Network, cfg: Config) -> Result { + return Err(ConfigurationError::new( + &key, + &format!("DB initial size must be at least {} MB.", DB_INIT_MIN_MB), + )) + }, + Ok(mb) => mb as usize, + Err(e) => match e { + ConfigError::NotFound(_) => DB_INIT_DEFAULT_MB, // default + other => return Err(ConfigurationError::new(&key, &other.to_string())), + }, + }; + + let key = config_string("base_node", &net_str, "db_grow_size_mb"); + let grow_size_mb = match cfg.get_int(&key) { + Ok(mb) if mb < DB_GROW_MIN_MB => { + return Err(ConfigurationError::new( + &key, + &format!("DB grow size must be at least {} MB.", DB_GROW_MIN_MB), + )) + }, + Ok(mb) => mb as usize, + Err(e) => match e { + ConfigError::NotFound(_) => DB_GROW_DEFAULT_MB, // default + other => return Err(ConfigurationError::new(&key, &other.to_string())), + }, + }; + + let key = config_string("base_node", &net_str, "db_resize_threshold_mb"); + let resize_threshold_mb = match cfg.get_int(&key) { + Ok(mb) if mb < DB_RESIZE_MIN_MB => { + return Err(ConfigurationError::new( + &key, + &format!("DB resize threshold must be at least {} MB.", DB_RESIZE_MIN_MB), + )) + }, + Ok(mb) if mb as usize >= grow_size_mb => { + return Err(ConfigurationError::new( + &key, + "DB resize threshold must be less than grow size.", + )) + }, + Ok(mb) if mb as usize >= init_size_mb => { + return Err(ConfigurationError::new( + &key, + "DB resize threshold must be less than init size.", + )) + }, + Ok(mb) => mb as usize, + Err(e) => match e { + ConfigError::NotFound(_) => DB_RESIZE_DEFAULT_MB, // default + other => return Err(ConfigurationError::new(&key, &other.to_string())), + }, + }; + + let db_config = LMDBConfig::new_from_mb(init_size_mb, grow_size_mb, resize_threshold_mb); + let key = config_string("base_node", &net_str, "orphan_storage_capacity"); let orphan_storage_capacity = cfg .get_int(&key) @@ -336,6 +405,7 @@ fn convert_node_config(network: Network, cfg: Config) -> Result) -> CommsDatabase { let database_name = random::string(8); let datastore = LMDBBuilder::new() .set_path(create_temporary_data_path().to_str().unwrap()) - .set_environment_size(10) + .set_env_config(LMDBConfig::default()) .set_max_number_of_databases(1) .add_database(&database_name, lmdb_zero::db::CREATE) .build() diff --git a/comms/dht/src/test_utils/makers.rs b/comms/dht/src/test_utils/makers.rs index 7e003ba19f..fc3bc8c423 100644 --- a/comms/dht/src/test_utils/makers.rs +++ b/comms/dht/src/test_utils/makers.rs @@ -41,7 +41,7 @@ use tari_crypto::{ keys::PublicKey, tari_utilities::{message_format::MessageFormat, ByteArray}, }; -use tari_storage::lmdb_store::LMDBBuilder; +use tari_storage::lmdb_store::{LMDBBuilder, LMDBConfig}; use tari_test_utils::{paths::create_temporary_data_path, random}; pub fn make_identity(features: PeerFeatures) -> Arc { @@ -167,7 +167,7 @@ pub fn build_peer_manager() -> Arc { let path = create_temporary_data_path(); let datastore = LMDBBuilder::new() .set_path(path.to_str().unwrap()) - .set_environment_size(50) + .set_env_config(LMDBConfig::default()) .set_max_number_of_databases(1) .add_database(&database_name, lmdb_zero::db::CREATE) .build() diff --git a/comms/dht/tests/dht.rs b/comms/dht/tests/dht.rs index e67dac5432..2420fe88e6 100644 --- a/comms/dht/tests/dht.rs +++ b/comms/dht/tests/dht.rs @@ -46,7 +46,10 @@ use tari_comms_dht::{ Dht, DhtBuilder, }; -use tari_storage::{lmdb_store::LMDBBuilder, LMDBWrapper}; +use tari_storage::{ + lmdb_store::{LMDBBuilder, LMDBConfig}, + LMDBWrapper, +}; use tari_test_utils::{ async_assert_eventually, collect_stream, @@ -86,7 +89,7 @@ fn create_peer_storage() -> CommsDatabase { let database_name = random::string(8); let datastore = LMDBBuilder::new() .set_path(create_temporary_data_path()) - .set_environment_size(50) + .set_env_config(LMDBConfig::default()) .set_max_number_of_databases(1) .add_database(&database_name, lmdb_zero::db::CREATE) .build() diff --git a/comms/examples/stress/node.rs b/comms/examples/stress/node.rs index b88d4f0fcf..15a768a200 100644 --- a/comms/examples/stress/node.rs +++ b/comms/examples/stress/node.rs @@ -39,7 +39,10 @@ use tari_comms::{ NodeIdentity, Substream, }; -use tari_storage::{lmdb_store::LMDBBuilder, LMDBWrapper}; +use tari_storage::{ + lmdb_store::{LMDBBuilder, LMDBConfig}, + LMDBWrapper, +}; pub async fn create( node_identity: Option>, @@ -60,7 +63,7 @@ pub async fn create( { let datastore = LMDBBuilder::new() .set_path(database_path.to_str().unwrap()) - .set_environment_size(50) + .set_env_config(LMDBConfig::default()) .set_max_number_of_databases(1) .add_database("peerdb", lmdb_zero::db::CREATE) .build() diff --git a/comms/examples/tor.rs b/comms/examples/tor.rs index afb71926fe..2b3f603e55 100644 --- a/comms/examples/tor.rs +++ b/comms/examples/tor.rs @@ -19,7 +19,10 @@ use tari_comms::{ CommsNode, }; use tari_crypto::tari_utilities::message_format::MessageFormat; -use tari_storage::{lmdb_store::LMDBBuilder, LMDBWrapper}; +use tari_storage::{ + lmdb_store::{LMDBBuilder, LMDBConfig}, + LMDBWrapper, +}; use tempfile::Builder; use thiserror::Error; use tokio::{runtime, task}; @@ -167,7 +170,7 @@ async fn setup_node_with_tor>( { let datastore = LMDBBuilder::new() .set_path(database_path.to_str().unwrap()) - .set_environment_size(50) + .set_env_config(LMDBConfig::default()) .set_max_number_of_databases(1) .add_database("peerdb", lmdb_zero::db::CREATE) .build() diff --git a/comms/src/peer_manager/mod.rs b/comms/src/peer_manager/mod.rs index 4674f40003..9d29c613b1 100644 --- a/comms/src/peer_manager/mod.rs +++ b/comms/src/peer_manager/mod.rs @@ -39,6 +39,7 @@ //! # use lmdb_zero::db; //! # use std::sync::Arc; //! # use tari_storage::LMDBWrapper; +//! # use tari_storage::lmdb_store::LMDBConfig; //! //! let mut rng = rand::rngs::OsRng; //! let (dest_sk, pk) = CommsPublicKey::random_keypair(&mut rng); @@ -55,7 +56,7 @@ //! let database_name = "pm_peer_database"; //! let datastore = LMDBBuilder::new() //! .set_path("/tmp/") -//! .set_environment_size(10) +//! .set_env_config(LMDBConfig::default()) //! .set_max_number_of_databases(1) //! .add_database(database_name, lmdb_zero::db::CREATE) //! .build() diff --git a/infrastructure/storage/src/key_val_store/lmdb_database.rs b/infrastructure/storage/src/key_val_store/lmdb_database.rs index 7a40110f2d..8fe533e4d7 100644 --- a/infrastructure/storage/src/key_val_store/lmdb_database.rs +++ b/infrastructure/storage/src/key_val_store/lmdb_database.rs @@ -109,7 +109,7 @@ where #[cfg(test)] mod test { use super::*; - use crate::lmdb_store::{LMDBBuilder, LMDBError, LMDBStore}; + use crate::lmdb_store::{LMDBBuilder, LMDBConfig, LMDBError, LMDBStore}; use serde::{Deserialize, Serialize}; use std::path::PathBuf; @@ -125,7 +125,7 @@ mod test { std::fs::create_dir(&path).unwrap_or_default(); LMDBBuilder::new() .set_path(&path) - .set_environment_size(50) + .set_env_config(LMDBConfig::default()) .set_max_number_of_databases(2) .add_database(name, lmdb_zero::db::CREATE) .build() diff --git a/infrastructure/storage/src/lmdb_store/error.rs b/infrastructure/storage/src/lmdb_store/error.rs index 8e64e5b5e8..c7db9cc123 100644 --- a/infrastructure/storage/src/lmdb_store/error.rs +++ b/infrastructure/storage/src/lmdb_store/error.rs @@ -21,7 +21,7 @@ // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. use thiserror::Error; -#[derive(Error, Debug)] +#[derive(Error, Debug, Clone)] pub enum LMDBError { #[error("Cannot create LMDB. The path does not exist")] InvalidPath, @@ -36,4 +36,6 @@ pub enum LMDBError { #[from] source: lmdb_zero::error::Error, }, + #[error("An LMDB path error contained invalid UTF8: {0}")] + PathError(#[from] std::str::Utf8Error), } diff --git a/infrastructure/storage/src/lmdb_store/mod.rs b/infrastructure/storage/src/lmdb_store/mod.rs index bddd296c41..dd4d08f037 100644 --- a/infrastructure/storage/src/lmdb_store/mod.rs +++ b/infrastructure/storage/src/lmdb_store/mod.rs @@ -28,4 +28,4 @@ pub use lmdb_zero::{ db, traits::{AsLmdbBytes, FromLmdbBytes}, }; -pub use store::{LMDBBuilder, LMDBDatabase, LMDBStore}; +pub use store::{LMDBBuilder, LMDBConfig, LMDBDatabase, LMDBStore}; diff --git a/infrastructure/storage/src/lmdb_store/store.rs b/infrastructure/storage/src/lmdb_store/store.rs index 1f3d66cc6c..1b6d4a7154 100644 --- a/infrastructure/storage/src/lmdb_store/store.rs +++ b/infrastructure/storage/src/lmdb_store/store.rs @@ -34,22 +34,72 @@ use std::{ }; const LOG_TARGET: &str = "lmdb"; +const BYTES_PER_MB: usize = 1024 * 1024; /// An atomic pointer to an LMDB database instance type DatabaseRef = Arc>; +#[derive(Debug, Clone)] +pub struct LMDBConfig { + init_size_bytes: usize, + grow_size_bytes: usize, + resize_threshold_bytes: usize, +} + +impl LMDBConfig { + /// Specify LMDB config in bytes. + pub fn new(init_size_bytes: usize, grow_size_bytes: usize, resize_threshold_bytes: usize) -> Self { + Self { + init_size_bytes, + grow_size_bytes, + resize_threshold_bytes, + } + } + + /// Specify LMDB config in megabytes. + pub fn new_from_mb(init_size_mb: usize, grow_size_mb: usize, resize_threshold_mb: usize) -> Self { + Self { + init_size_bytes: init_size_mb * BYTES_PER_MB, + grow_size_bytes: grow_size_mb * BYTES_PER_MB, + resize_threshold_bytes: resize_threshold_mb * BYTES_PER_MB, + } + } + + /// Get the initial size of the LMDB environment in bytes. + pub fn init_size_bytes(&self) -> usize { + self.init_size_bytes + } + + /// Get the grow size of the LMDB environment in bytes. The LMDB environment will be resized by this amount when + /// `resize_threshold_bytes` are left. + pub fn grow_size_bytes(&self) -> usize { + self.grow_size_bytes + } + + /// Get the resize threshold in bytes. The LMDB environment will be resized when this much free space is left. + pub fn resize_threshold_bytes(&self) -> usize { + self.resize_threshold_bytes + } +} + +impl Default for LMDBConfig { + fn default() -> Self { + Self::new_from_mb(64, 16, 4) + } +} + /// A builder for [LMDBStore](struct.lmdbstore.html) /// ## Example /// -/// Create a new LMDB database of 500MB in the `db` directory with two named databases: "db1" and "db2" +/// Create a new LMDB database of 64MB in the `db` directory with two named databases: "db1" and "db2" /// /// ``` -/// # use tari_storage::lmdb_store::LMDBBuilder; +/// # use tari_storage::lmdb_store::{LMDBBuilder, LMDBConfig}; /// # use lmdb_zero::db; /// # use std::env; /// let mut store = LMDBBuilder::new() /// .set_path(env::temp_dir()) -/// .set_environment_size(500) +/// .set_env_config(LMDBConfig::default()) /// .set_max_number_of_databases(10) /// .add_database("db1", db::CREATE) /// .add_database("db2", db::CREATE) @@ -59,9 +109,9 @@ type DatabaseRef = Arc>; #[derive(Default)] pub struct LMDBBuilder { path: PathBuf, - db_size_mb: usize, max_dbs: usize, db_names: HashMap, + env_config: LMDBConfig, } impl LMDBBuilder { @@ -71,14 +121,13 @@ impl LMDBBuilder { /// | Parameter | Default | /// |:----------|---------| /// | path | ./store/| - /// | size | 64 MB | /// | named DBs | none | pub fn new() -> LMDBBuilder { LMDBBuilder { path: "./store/".into(), - db_size_mb: 64, db_names: HashMap::new(), max_dbs: 8, + env_config: LMDBConfig::default(), } } @@ -90,10 +139,10 @@ impl LMDBBuilder { self } - /// Sets the size of the environment, in MB. + /// Sets the parameters of the LMDB environment. /// The actual memory will only be allocated when #build() is called - pub fn set_environment_size(mut self, size: usize) -> LMDBBuilder { - self.db_size_mb = size; + pub fn set_env_config(mut self, config: LMDBConfig) -> LMDBBuilder { + self.env_config = config; self } @@ -126,7 +175,7 @@ impl LMDBBuilder { let env = unsafe { let mut builder = EnvBuilder::new()?; - builder.set_mapsize(self.db_size_mb * 1024 * 1024)?; + builder.set_mapsize(self.env_config.init_size_bytes)?; builder.set_maxdbs(max_dbs)?; // Using open::Flags::NOTLS does not compile!?! NOTLS=0x200000 let flags = open::Flags::from_bits(0x200_000).expect("LMDB open::Flag is correct"); @@ -135,33 +184,13 @@ impl LMDBBuilder { let env = Arc::new(env); // Increase map size if usage gets close to the db size - let mut env_info = env.info()?; - let env_stat = env.stat()?; - let size_used = env_stat.psize as usize * env_info.last_pgno; - let mut space_remaining = env_info.mapsize - size_used; - let usage = (size_used as f64 / env_info.mapsize as f64) * 100.0; - if space_remaining <= ((self.db_size_mb * 1024 * 1024) as f64 * 0.5) as usize { - unsafe { - env.set_mapsize(size_used + self.db_size_mb * 1024 * 1024)?; - } - env_info = env.info()?; - space_remaining = env_info.mapsize - size_used; - debug!( - target: LOG_TARGET, - "({}) LMDB environment usage factor {:.*} %., size used {:?} MB, increased by {:?} MB.", - path, - 2, - usage, - size_used / (1024 * 1024), - self.db_size_mb - ); - }; + LMDBStore::resize_if_required(&env, &self.env_config)?; info!( target: LOG_TARGET, "({}) LMDB environment created with a capacity of {} MB, {} MB remaining.", path, - env_info.mapsize / (1024 * 1024), - space_remaining / (1024 * 1024) + env.info()?.mapsize / BYTES_PER_MB, + (env.info()?.mapsize - env.stat()?.psize as usize * env.info()?.last_pgno) / BYTES_PER_MB, ); let mut databases: HashMap = HashMap::new(); @@ -172,13 +201,19 @@ impl LMDBBuilder { let db = Database::open(env.clone(), Some(name), &DatabaseOptions::new(*flags))?; let db = LMDBDatabase { name: name.to_string(), + env_config: self.env_config.clone(), env: env.clone(), db: Arc::new(db), }; databases.insert(name.to_string(), db); trace!(target: LOG_TARGET, "({}) LMDB database '{}' is ready", path, name); } - Ok(LMDBStore { path, env, databases }) + Ok(LMDBStore { + path, + env_config: self.env_config, + env, + databases, + }) } } @@ -238,7 +273,7 @@ impl LMDBBuilder { /// /// ## Serialisation /// -/// The ideal serialiasation format is the one that does the least "bit-twiddling" between memory and the byte array; +/// The ideal serialisation format is the one that does the least "bit-twiddling" between memory and the byte array; /// as well as being as compact as possible. /// /// Candidates include: Bincode, MsgPack, and Protobuf / Cap'nProto. Without spending ages on a comparison, I just @@ -288,14 +323,15 @@ impl LMDBBuilder { /// So after all this, we'll use bincode for the time being to handle serialisation to- and from- LMDB pub struct LMDBStore { path: String, + env_config: LMDBConfig, pub(crate) env: Arc, pub(crate) databases: HashMap, } -/// Close all databases and close the environment. You cannot be guaranteed that the dbs will be closed after calling -/// this function because there still may be threads accessing / writing to a database that will block this call. -/// However, in that case `shutdown` returns an error. impl LMDBStore { + /// Close all databases and close the environment. You cannot be guaranteed that the dbs will be closed after + /// calling this function because there still may be threads accessing / writing to a database that will block + /// this call. However, in that case `shutdown` returns an error. pub fn flush(&self) -> Result<(), lmdb_zero::error::Error> { trace!(target: LOG_TARGET, "Forcing flush of buffers to disk"); self.env.sync(true)?; @@ -312,7 +348,7 @@ impl LMDBStore { e.to_string() ), Ok(info) => { - let size_mb = info.mapsize / 1024 / 1024; + let size_mb = info.mapsize / BYTES_PER_MB; debug!( target: LOG_TARGET, "LMDB Environment information ({}). Map Size={} MB. Last page no={}. Last tx id={}", @@ -356,14 +392,41 @@ impl LMDBStore { } } + pub fn env_config(&self) -> LMDBConfig { + self.env_config.clone() + } + pub fn env(&self) -> Arc { self.env.clone() } + + /// Resize the LMDB environment if the resize threshold is breached. + pub fn resize_if_required(env: &Environment, config: &LMDBConfig) -> Result<(), LMDBError> { + let env_info = env.info()?; + let size_used_bytes = env.stat()?.psize as usize * env_info.last_pgno; + let size_left_bytes = env_info.mapsize - size_used_bytes; + + if size_left_bytes <= config.resize_threshold_bytes { + unsafe { + env.set_mapsize(size_used_bytes + config.grow_size_bytes)?; + } + debug!( + target: LOG_TARGET, + "({}) LMDB size used {:?} MB, environment space left {:?} MB, increased by {:?} MB.", + env.path()?.to_str()?, + size_used_bytes / BYTES_PER_MB, + size_left_bytes / BYTES_PER_MB, + config.grow_size_bytes / BYTES_PER_MB, + ); + } + Ok(()) + } } #[derive(Clone)] pub struct LMDBDatabase { name: String, + env_config: LMDBConfig, env: Arc, db: DatabaseRef, } @@ -377,6 +440,9 @@ impl LMDBDatabase { V: Serialize, { let env = &(*self.db.env()); + + LMDBStore::resize_if_required(env, &self.env_config)?; + let tx = WriteTransaction::new(env)?; { let mut accessor = tx.access(); @@ -437,12 +503,12 @@ impl LMDBDatabase { /// Returns if the database is empty. pub fn is_empty(&self) -> Result { - self.get_stats().and_then(|s| Ok(s.entries > 0)) + self.get_stats().map(|s| s.entries > 0) } /// Returns the total number of entries in this database. pub fn len(&self) -> Result { - self.get_stats().and_then(|s| Ok(s.entries)) + self.get_stats().map(|s| s.entries) } /// Execute function `f` for each value in the database. @@ -643,7 +709,7 @@ impl<'txn, 'db: 'txn> LMDBWriteTransaction<'txn, 'db> { #[cfg(test)] mod test { - use crate::lmdb_store::LMDBBuilder; + use crate::lmdb_store::{LMDBBuilder, LMDBConfig}; use lmdb_zero::db; use std::env; @@ -651,7 +717,7 @@ mod test { fn test_lmdb_builder() { let store = LMDBBuilder::new() .set_path(env::temp_dir()) - .set_environment_size(500) + .set_env_config(LMDBConfig::default()) .set_max_number_of_databases(10) .add_database("db1", db::CREATE) .add_database("db2", db::CREATE) diff --git a/infrastructure/storage/tests/lmdb.rs b/infrastructure/storage/tests/lmdb.rs index 36d385dbac..85a24fa8f6 100644 --- a/infrastructure/storage/tests/lmdb.rs +++ b/infrastructure/storage/tests/lmdb.rs @@ -31,7 +31,7 @@ use std::{ thread, }; use tari_storage::{ - lmdb_store::{db, LMDBBuilder, LMDBDatabase, LMDBError, LMDBStore}, + lmdb_store::{db, LMDBBuilder, LMDBConfig, LMDBDatabase, LMDBError, LMDBStore}, IterationResult, }; use tari_utilities::ExtendBytes; @@ -92,7 +92,7 @@ fn init(name: &str) -> Result { std::fs::create_dir(&path).unwrap_or_default(); LMDBBuilder::new() .set_path(&path) - .set_environment_size(10) + .set_env_config(LMDBConfig::default()) .set_max_number_of_databases(2) .add_database("users", db::CREATE) .build() @@ -278,7 +278,7 @@ fn exists_and_delete() { } #[test] -fn lmbd_resize_on_create() { +fn lmdb_resize_on_create() { let db_env_name = "resize"; { let path = get_path(&db_env_name); @@ -290,7 +290,11 @@ fn lmbd_resize_on_create() { // Create db with large preset environment size let env = LMDBBuilder::new() .set_path(&path) - .set_environment_size(PRESET_SIZE * 100) + .set_env_config(LMDBConfig::new( + 100 * PRESET_SIZE * 1024 * 1024, + 1024 * 1024, + 512 * 1024, + )) .set_max_number_of_databases(1) .add_database(&db_name, db::CREATE) .build() @@ -313,7 +317,7 @@ fn lmbd_resize_on_create() { // Load existing db environment let env = LMDBBuilder::new() .set_path(&path) - .set_environment_size(PRESET_SIZE) + .set_env_config(LMDBConfig::new(PRESET_SIZE * 1024 * 1024, 1024 * 1024, 512 * 1024)) .set_max_number_of_databases(1) .add_database(&db_name, db::CREATE) .build() @@ -330,3 +334,47 @@ fn lmbd_resize_on_create() { } clean_up(&db_env_name); // In Windows file handles must be released before files can be deleted } + +#[test] +fn test_lmdb_resize_before_full() { + let db_env_name = "resize_dynamic"; + { + let path = get_path(&db_env_name); + std::fs::create_dir(&path).unwrap_or_default(); + let db_name = "test_full"; + { + // Create db with 1MB capacity + let store = LMDBBuilder::new() + .set_path(&path) + .set_env_config(LMDBConfig::new(1024 * 1024, 512 * 1024, 100 * 1024)) + .set_max_number_of_databases(1) + .add_database(&db_name, db::CREATE) + .build() + .unwrap(); + let db = store.get_handle(&db_name).unwrap(); + + // Add enough data to exceed our 1MB db + let value = load_users(); + // one insertion requires approx 92KB so after ~11 insertions + // our 1MB env size should be out of space + // however the db should now be allocating additional space as it fills up + for key in 0..32 { + if let Err(_e) = db.insert(&key, &value) { + // println!("LMDBError {:#?}", _e); + panic!("Failed to resize the LMDB store."); + } + } + let env_info = store.env().info().unwrap(); + let psize = store.env().stat().unwrap().psize as usize; + let page_size_total = psize * env_info.last_pgno; + let percent_left = 1.0 - page_size_total as f64 / env_info.mapsize as f64; + + // check the allocated size is now greater than it was initially + assert!(page_size_total > 1024 * 1024); + assert!(percent_left > 0.0); + + store.flush().unwrap(); + } + } + clean_up(&db_env_name); // In Windows file handles must be released before files can be deleted +}