diff --git a/base_layer/core/src/chain_storage/error.rs b/base_layer/core/src/chain_storage/error.rs index 84f68ed2b8..e6182a570b 100644 --- a/base_layer/core/src/chain_storage/error.rs +++ b/base_layer/core/src/chain_storage/error.rs @@ -112,6 +112,8 @@ pub enum ChainStorageError { KeyExists { table_name: &'static str, key: String }, #[error("Database resize required")] DbResizeRequired, + #[error("DB transaction was too large ({0} operations)")] + DbTransactionTooLarge(usize), } impl ChainStorageError { diff --git a/base_layer/core/src/chain_storage/lmdb_db/lmdb.rs b/base_layer/core/src/chain_storage/lmdb_db/lmdb.rs index 6d52017c2a..8ed0e21c2d 100644 --- a/base_layer/core/src/chain_storage/lmdb_db/lmdb.rs +++ b/base_layer/core/src/chain_storage/lmdb_db/lmdb.rs @@ -115,15 +115,15 @@ where { let val_buf = serialize(val)?; txn.access().put(&db, key, &val_buf, put::Flags::empty()).map_err(|e| { - error!( - target: LOG_TARGET, - "Could not insert value into lmdb transaction: {:?}", e - ); if let lmdb_zero::Error::Code(code) = &e { if *code == lmdb_zero::error::MAP_FULL { return ChainStorageError::DbResizeRequired; } } + error!( + target: LOG_TARGET, + "Could not insert value into lmdb transaction: {:?}", e + ); ChainStorageError::AccessError(e.to_string()) }) } @@ -136,6 +136,11 @@ where { let val_buf = serialize(val)?; txn.access().put(&db, key, &val_buf, put::Flags::empty()).map_err(|e| { + if let lmdb_zero::Error::Code(code) = &e { + if *code == lmdb_zero::error::MAP_FULL { + return ChainStorageError::DbResizeRequired; + } + } error!( target: LOG_TARGET, "Could not replace value in lmdb transaction: {:?}", e diff --git a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs index 52ca2c1405..a8c528878e 100644 --- a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs +++ b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs @@ -1335,7 +1335,7 @@ impl BlockchainBackend for LMDBDatabase { let mark = Instant::now(); // Resize this many times before assuming something is not right - const MAX_RESIZES: usize = 3; + const MAX_RESIZES: usize = 5; for i in 0..MAX_RESIZES { let num_operations = txn.operations().len(); match self.apply_db_transaction(&txn) { @@ -1350,13 +1350,17 @@ impl BlockchainBackend for LMDBDatabase { return Ok(()); }, Err(ChainStorageError::DbResizeRequired) => { - info!(target: LOG_TARGET, "Database resize required (iteration: {})", i); + info!( + target: LOG_TARGET, + "Database resize required (resized {} time(s) in this transaction)", + i + 1 + ); // SAFETY: This depends on the thread safety of the caller. Technically, `write` is unsafe too // however we happen to know that `LmdbDatabase` is wrapped in an exclusive write lock in // BlockchainDatabase, so we know there are no other threads taking out LMDB transactions when this // is called. unsafe { - LMDBStore::resize_if_required(&self.env, &self.env_config)?; + LMDBStore::resize(&self.env, &self.env_config)?; } }, Err(e) => { @@ -1366,7 +1370,7 @@ impl BlockchainBackend for LMDBDatabase { } } - Err(ChainStorageError::DbResizeRequired) + Err(ChainStorageError::DbTransactionTooLarge(txn.operations().len())) } fn fetch(&self, key: &DbKey) -> Result, ChainStorageError> { diff --git a/infrastructure/storage/src/lmdb_store/store.rs b/infrastructure/storage/src/lmdb_store/store.rs index 1022cadb54..d5a2bfeab7 100644 --- a/infrastructure/storage/src/lmdb_store/store.rs +++ b/infrastructure/storage/src/lmdb_store/store.rs @@ -412,7 +412,7 @@ impl LMDBStore { ); if size_left_bytes <= config.resize_threshold_bytes { - env.set_mapsize(env_info.mapsize + config.grow_size_bytes)?; + Self::resize(env, config)?; debug!( target: LOG_TARGET, "({}) LMDB size used {:?} MB, environment space left {:?} MB, increased by {:?} MB", @@ -424,6 +424,31 @@ impl LMDBStore { } Ok(()) } + + /// Grows the LMDB environment by the configured amount + /// + /// # Safety + /// This may only be called if no write transactions are active in the current process. Note that the library does + /// not check for this condition, the caller must ensure it explicitly. + /// + /// http://www.lmdb.tech/doc/group__mdb.html#gaa2506ec8dab3d969b0e609cd82e619e5 + pub unsafe fn resize(env: &Environment, config: &LMDBConfig) -> Result<(), LMDBError> { + let env_info = env.info()?; + let current_mapsize = env_info.mapsize; + env.set_mapsize(current_mapsize + config.grow_size_bytes)?; + let env_info = env.info()?; + let new_mapsize = env_info.mapsize; + debug!( + target: LOG_TARGET, + "({}) LMDB MB, mapsize was grown from {:?} MB to {:?} MB, increased by {:?} MB", + env.path()?.to_str()?, + current_mapsize / BYTES_PER_MB, + new_mapsize / BYTES_PER_MB, + config.grow_size_bytes / BYTES_PER_MB, + ); + + Ok(()) + } } #[derive(Clone)] @@ -442,8 +467,7 @@ impl LMDBDatabase { K: AsLmdbBytes + ?Sized, V: Serialize, { - const MAX_RESIZES: usize = 3; - + const MAX_RESIZES: usize = 5; let value = LMDBWriteTransaction::convert_value(value)?; for _ in 0..MAX_RESIZES { match self.write(key, &value) { @@ -456,7 +480,7 @@ impl LMDBDatabase { // SAFETY: We know that there are no open transactions at this point because ... // TODO: we don't guarantee this here but it works because the caller does this. unsafe { - LMDBStore::resize_if_required(&self.env, &self.env_config)?; + LMDBStore::resize(&self.env, &self.env_config)?; } }, Err(e) => return Err(e.into()),