Merge #2193: Implement LMDB automatic resizing

This commit implements the functionality to automatically resize LMDB
environment mapsize as it fills up. This should ensure that new data can always
be inserted as long as there is disk space available.

There were a few minor issues fixed: spelling, docs, and removing an unused
`use` declaration.

Thanks to @hansieodendaal for his contributions

applications/tari_base_node/src/builder.rs

@@ -21,7 +21,7 @@
 // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 use crate::miner;
-use futures::{channel::mpsc, future};
+use futures::future;
 use log::*;
 use std::{
     fs,
@@ -334,7 +334,7 @@ where B: 'static
 }
 
 /// Sets up and initializes the base node, creating the context and database
-/// ## Paramters
+/// ## Parameters
 /// `config` - The configuration for the base node
 /// `node_identity` - The node identity information of the base node
 /// `wallet_node_identity` - The node identity information of the base node's wallet
@@ -367,7 +367,8 @@ pub async fn configure_and_initialize_node(
             NodeContainer::Memory(ctx)
         },
         DatabaseType::LMDB(p) => {
-            let backend = create_lmdb_database(&p, MmrCacheConfig::default()).map_err(|e| e.to_string())?;
+            let backend = create_lmdb_database(&p, config.db_config.clone(), MmrCacheConfig::default())
+                .map_err(|e| e.to_string())?;
             let ctx = build_node_context(
                 backend,
                 network,
@@ -383,8 +384,9 @@ pub async fn configure_and_initialize_node(
     Ok(result)
 }
 
-/// Constructs the base node context, this includes settin up the consensus manager, mempool, base node, wallet, miner
-/// and state machine ## Paramters
+/// Constructs the base node context, this includes setting up the consensus manager, mempool, base node, wallet, miner
+/// and state machine
+/// ## Parameters
 /// `backend` - Backend interface
 /// `network` - The NetworkType (rincewind, mainnet, local)
 /// `base_node_identity` - The node identity information of the base node

base_layer/core/src/chain_storage/error.rs

@@ -25,6 +25,7 @@ use crate::{
     validation::ValidationError,
 };
 use tari_mmr::{error::MerkleMountainRangeError, MerkleProofError};
+use tari_storage::lmdb_store::LMDBError;
 use thiserror::Error;
 use tokio::task;
 
@@ -85,6 +86,11 @@ pub enum ChainStorageError {
     OutOfRange,
     #[error("Value not found: {0}")]
     LmdbValueNotFound(lmdb_zero::Error),
+    #[error("LMDB error: {source}")]
+    LmdbError {
+        #[from]
+        source: LMDBError,
+    },
 }
 
 impl From<LMDBVecError> for ChainStorageError {

base_layer/core/src/chain_storage/lmdb_db/lmdb.rs

@@ -45,21 +45,19 @@ pub fn serialize<T>(data: &T) -> Result<Vec<u8>, ChainStorageError>
 where T: Serialize {
     let size = bincode::serialized_size(&data).map_err(|e| ChainStorageError::AccessError(e.to_string()))?;
     let mut buf = Vec::with_capacity(size as usize);
-    bincode::serialize_into(&mut buf, data)
-        .or_else(|e| {
-            error!(target: LOG_TARGET, "Could not serialize lmdb: {:?}", e);
-            Err(e)
-        })
-        .map_err(|e| ChainStorageError::AccessError(e.to_string()))?;
+    bincode::serialize_into(&mut buf, data).map_err(|e| {
+        error!(target: LOG_TARGET, "Could not serialize lmdb: {:?}", e);
+        ChainStorageError::AccessError(e.to_string())
+    })?;
     Ok(buf)
 }
 
 pub fn deserialize<T>(buf_bytes: &[u8]) -> Result<T, error::Error>
 where T: DeserializeOwned {
     bincode::deserialize(buf_bytes)
-        .or_else(|e| {
+        .map_err(|e| {
             error!(target: LOG_TARGET, "Could not deserialize lmdb: {:?}", e);
-            Err(e)
+            e
         })
         .map_err(|e| error::Error::ValRejected(e.to_string()))
 }

base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs

@@ -90,7 +90,7 @@ use tari_mmr::{
     MmrCacheConfig,
     MutableMmr,
 };
-use tari_storage::lmdb_store::{db, LMDBBuilder, LMDBStore};
+use tari_storage::lmdb_store::{db, LMDBBuilder, LMDBConfig, LMDBStore};
 
 type DatabaseRef = Arc<Database<'static>>;
 
@@ -101,6 +101,7 @@ pub struct LMDBDatabase<D>
 where D: Digest
 {
     env: Arc<Environment>,
+    env_config: LMDBConfig,
     metadata_db: DatabaseRef,
     mem_metadata: ChainMetadata, // Memory copy of stored metadata
     headers_db: DatabaseRef,
@@ -164,6 +165,7 @@ where D: Digest + Send + Sync
             },
             range_proof_checkpoints,
             env,
+            env_config: store.env_config(),
             is_mem_metadata_dirty: false,
         })
     }
@@ -311,7 +313,7 @@ where D: Digest + Send + Sync
     fn op_insert(&mut self, txn: &WriteTransaction<'_>, kv_pair: &DbKeyValuePair) -> Result<(), ChainStorageError> {
         match kv_pair {
             DbKeyValuePair::Metadata(k, v) => {
-                lmdb_replace(&txn, &self.metadata_db, &(k.clone() as u32), &v)?;
+                lmdb_replace(&txn, &self.metadata_db, &(*k as u32), &v)?;
                 self.is_mem_metadata_dirty = true;
             },
             DbKeyValuePair::BlockHeader(k, v) => {
@@ -509,14 +511,15 @@ where D: Digest + Send + Sync
 
 pub fn create_lmdb_database<P: AsRef<Path>>(
     path: P,
+    config: LMDBConfig,
     mmr_cache_config: MmrCacheConfig,
 ) -> Result<LMDBDatabase<HashDigest>, ChainStorageError>
 {
     let flags = db::CREATE;
     let _ = std::fs::create_dir_all(&path);
     let lmdb_store = LMDBBuilder::new()
         .set_path(path)
-        .set_environment_size(1_000)
+        .set_env_config(config)
         .set_max_number_of_databases(15)
         .add_database(LMDB_DB_METADATA, flags)
         .add_database(LMDB_DB_HEADERS, flags)
@@ -542,6 +545,8 @@ where D: Digest + Send + Sync
             return Ok(());
         }
 
+        LMDBStore::resize_if_required(&self.env, &self.env_config)?;
+
         let mark = Instant::now();
         let num_operations = txn.operations.len();
         match self.apply_db_transaction(&txn) {
@@ -566,7 +571,7 @@ where D: Digest + Send + Sync
     fn fetch(&self, key: &DbKey) -> Result<Option<DbValue>, ChainStorageError> {
         Ok(match key {
             DbKey::Metadata(k) => {
-                let val: Option<MetadataValue> = lmdb_get(&self.env, &self.metadata_db, &(k.clone() as u32))?;
+                let val: Option<MetadataValue> = lmdb_get(&self.env, &self.metadata_db, &(*k as u32))?;
                 val.map(DbValue::Metadata)
             },
             DbKey::BlockHeader(k) => {
@@ -1004,9 +1009,10 @@ mod test {
         let flags = db::CREATE;
         let _ = std::fs::remove_dir_all(&path);
         std::fs::create_dir_all(&path).unwrap();
+        let config = LMDBConfig::new_from_mb(1000, 100, 100);
         let lmdb_store = LMDBBuilder::new()
             .set_path(path)
-            .set_environment_size(1_000)
+            .set_env_config(config)
             .set_max_number_of_databases(15)
             .add_database("1", flags)
             .add_database("2", flags)

base_layer/core/src/chain_storage/lmdb_db/lmdb_vec.rs

@@ -284,7 +284,7 @@ where
 #[cfg(test)]
 mod test {
     use super::*;
-    use tari_storage::lmdb_store::{db, LMDBBuilder};
+    use tari_storage::lmdb_store::{db, LMDBBuilder, LMDBConfig};
     use tari_test_utils::paths::create_temporary_data_path;
 
     #[test]
@@ -293,7 +293,7 @@ mod test {
         let _ = std::fs::create_dir(&path).unwrap_or_default();
         let lmdb_store = LMDBBuilder::new()
             .set_path(&path)
-            .set_environment_size(1)
+            .set_env_config(LMDBConfig::default())
             .set_max_number_of_databases(1)
             .add_database("db", db::CREATE)
             .build()