db/mdbx submodule (#1906)

erigontech · Mar 14, 2024 · 9850135 · 9850135
1 parent 66d6fdd
commit 9850135
Show file tree

Hide file tree

Showing 67 changed files with 179 additions and 108 deletions.
diff --git a/cmd/capi/execute.cpp b/cmd/capi/execute.cpp
@@ -32,7 +32,7 @@
 
 #include <silkworm/capi/silkworm.h>
 #include <silkworm/db/access_layer.hpp>
-#include <silkworm/db/mdbx.hpp>
+#include <silkworm/db/mdbx/mdbx.hpp>
 #include <silkworm/db/snapshots/repository.hpp>
 #include <silkworm/infra/common/directories.hpp>
 #include <silkworm/infra/common/log.hpp>

diff --git a/cmd/common/db_max_readers_option.cpp b/cmd/common/db_max_readers_option.cpp
@@ -16,7 +16,7 @@
 
 #include "db_max_readers_option.hpp"
 
-#include <silkworm/db/mdbx.hpp>
+#include <silkworm/db/mdbx/mdbx.hpp>
 
 namespace silkworm::cmd::common {
 

diff --git a/cmd/dev/backend_kv_server.cpp b/cmd/dev/backend_kv_server.cpp
@@ -33,7 +33,7 @@
 #include <silkworm/core/types/evmc_bytes32.hpp>
 #include <silkworm/db/access_layer.hpp>
 #include <silkworm/db/head_info.hpp>
-#include <silkworm/db/mdbx.hpp>
+#include <silkworm/db/mdbx/mdbx.hpp>
 #include <silkworm/infra/common/directories.hpp>
 #include <silkworm/infra/common/log.hpp>
 #include <silkworm/infra/concurrency/awaitable_wait_for_one.hpp>

diff --git a/cmd/dev/check_log_indices.cpp b/cmd/dev/check_log_indices.cpp
@@ -29,9 +29,9 @@
 #include <silkworm/core/common/bytes_to_string.hpp>
 #include <silkworm/core/types/address.hpp>
 #include <silkworm/core/types/evmc_bytes32.hpp>
-#include <silkworm/db/bitmap.hpp>
 #include <silkworm/db/log_cbor.hpp>
-#include <silkworm/db/mdbx.hpp>
+#include <silkworm/db/mdbx/bitmap.hpp>
+#include <silkworm/db/mdbx/mdbx.hpp>
 #include <silkworm/db/tables.hpp>
 #include <silkworm/db/util.hpp>
 #include <silkworm/infra/common/directories.hpp>

diff --git a/cmd/dev/db_toolbox.cpp b/cmd/dev/db_toolbox.cpp
@@ -42,7 +42,7 @@
 #include <silkworm/core/types/block_body_for_storage.hpp>
 #include <silkworm/core/types/evmc_bytes32.hpp>
 #include <silkworm/db/genesis.hpp>
-#include <silkworm/db/mdbx.hpp>
+#include <silkworm/db/mdbx/mdbx.hpp>
 #include <silkworm/db/prune_mode.hpp>
 #include <silkworm/db/stages.hpp>
 #include <silkworm/infra/common/decoding_exception.hpp>

diff --git a/silkworm/capi/silkworm_test.cpp b/silkworm/capi/silkworm_test.cpp
@@ -22,7 +22,7 @@
 #include <catch2/catch.hpp>
 
 #include <silkworm/core/trie/vector_root.hpp>
-#include <silkworm/db/mdbx.hpp>
+#include <silkworm/db/mdbx/mdbx.hpp>
 #include <silkworm/db/snapshots/index.hpp>
 #include <silkworm/db/snapshots/snapshot.hpp>
 #include <silkworm/db/snapshots/test_util/common.hpp>

diff --git a/silkworm/db/CMakeLists.txt b/silkworm/db/CMakeLists.txt
@@ -17,23 +17,22 @@
 include("${SILKWORM_MAIN_DIR}/cmake/common/targets.cmake")
 
 add_subdirectory(etl)
+add_subdirectory(mdbx)
 add_subdirectory(snapshots)
 add_subdirectory(test_util)
 
 find_package(absl REQUIRED)
 find_package(magic_enum REQUIRED)
 find_package(Microsoft.GSL REQUIRED)
 find_package(nlohmann_json REQUIRED)
-find_package(roaring REQUIRED)
 
 set(LIBS_PUBLIC
     absl::btree
     absl::flat_hash_map
     absl::flat_hash_set
     absl::strings
-    mdbx-static
-    roaring::roaring
     silkworm_core
+    silkworm_db_mdbx
 )
 
 # cmake-format: off

diff --git a/silkworm/db/access_layer.cpp b/silkworm/db/access_layer.cpp
@@ -24,7 +24,7 @@
 #include <silkworm/core/common/endian.hpp>
 #include <silkworm/core/types/block_body_for_storage.hpp>
 #include <silkworm/core/types/evmc_bytes32.hpp>
-#include <silkworm/db/bitmap.hpp>
+#include <silkworm/db/mdbx/bitmap.hpp>
 #include <silkworm/db/receipt_cbor.hpp>
 #include <silkworm/db/snapshots/repository.hpp>
 #include <silkworm/db/tables.hpp>

diff --git a/silkworm/db/access_layer.hpp b/silkworm/db/access_layer.hpp
@@ -32,7 +32,7 @@
 #include <silkworm/core/types/block.hpp>
 #include <silkworm/core/types/hash.hpp>
 #include <silkworm/core/types/receipt.hpp>
-#include <silkworm/db/mdbx.hpp>
+#include <silkworm/db/mdbx/mdbx.hpp>
 #include <silkworm/db/util.hpp>
 
 namespace silkworm::snapshots {

diff --git a/silkworm/db/access_layer_test.cpp b/silkworm/db/access_layer_test.cpp
@@ -25,8 +25,8 @@
 #include <silkworm/core/common/test_util.hpp>
 #include <silkworm/core/execution/execution.hpp>
 #include <silkworm/core/protocol/param.hpp>
-#include <silkworm/db/bitmap.hpp>
 #include <silkworm/db/buffer.hpp>
+#include <silkworm/db/mdbx/bitmap.hpp>
 #include <silkworm/db/prune_mode.hpp>
 #include <silkworm/db/stages.hpp>
 #include <silkworm/db/tables.hpp>

diff --git a/silkworm/db/bitmap_test.cpp b/silkworm/db/bitmap_test.cpp
@@ -14,15 +14,14 @@
    limitations under the License.
 */
 
-#include "bitmap.hpp"
-
 #include <map>
 #include <vector>
 
 #include <absl/container/btree_map.h>
 #include <catch2/catch.hpp>
 
-#include <silkworm/db/etl_mdbx_collector.hpp>
+#include <silkworm/db/mdbx/bitmap.hpp>
+#include <silkworm/db/mdbx/etl_mdbx_collector.hpp>
 #include <silkworm/db/test_util/temp_chain_data.hpp>
 
 namespace silkworm::db::bitmap {

diff --git a/silkworm/db/buffer.hpp b/silkworm/db/buffer.hpp
@@ -30,7 +30,7 @@
 #include <silkworm/core/types/block.hpp>
 #include <silkworm/core/types/receipt.hpp>
 #include <silkworm/db/access_layer.hpp>
-#include <silkworm/db/mdbx.hpp>
+#include <silkworm/db/mdbx/mdbx.hpp>
 #include <silkworm/db/util.hpp>
 
 namespace silkworm::db {

diff --git a/silkworm/db/db_utils.hpp b/silkworm/db/db_utils.hpp
@@ -23,7 +23,7 @@
 #include <silkworm/core/types/block.hpp>
 #include <silkworm/core/types/hash.hpp>
 #include <silkworm/db/access_layer.hpp>
-#include <silkworm/db/mdbx.hpp>
+#include <silkworm/db/mdbx/mdbx.hpp>
 
 namespace silkworm {
 

diff --git a/silkworm/db/etl_collector_test.cpp b/silkworm/db/etl_collector_test.cpp
@@ -22,13 +22,12 @@
 
 #include <silkworm/core/common/endian.hpp>
 #include <silkworm/core/common/random_number.hpp>
+#include <silkworm/db/mdbx/etl_mdbx_collector.hpp>
 #include <silkworm/db/tables.hpp>
 #include <silkworm/db/test_util/temp_chain_data.hpp>
 #include <silkworm/infra/common/log.hpp>
 #include <silkworm/infra/test_util/log.hpp>
 
-#include "etl_mdbx_collector.hpp"
-
 namespace silkworm::db::etl {
 
 namespace fs = std::filesystem;

diff --git a/silkworm/db/etl_in_memory_collector_test.cpp b/silkworm/db/etl_in_memory_collector_test.cpp
@@ -20,12 +20,12 @@
 #include <catch2/catch.hpp>
 
 #include <silkworm/core/common/random_number.hpp>
+#include <silkworm/db/etl/in_memory_collector.hpp>
+#include <silkworm/db/mdbx/mdbx.hpp>
 #include <silkworm/db/test_util/temp_chain_data.hpp>
 #include <silkworm/infra/common/log.hpp>
 #include <silkworm/infra/test_util/log.hpp>
 
-#include "etl/in_memory_collector.hpp"
-#include "mdbx.hpp"
 #include "util.hpp"
 
 namespace silkworm::db::etl_mdbx {

diff --git a/silkworm/db/genesis.hpp b/silkworm/db/genesis.hpp
@@ -18,7 +18,7 @@
 
 #include <silkworm/core/state/in_memory_state.hpp>
 #include <silkworm/db/access_layer.hpp>
-#include <silkworm/db/mdbx.hpp>
+#include <silkworm/db/mdbx/mdbx.hpp>
 
 namespace silkworm::db {
 //! \brief Validates provided genesis json payload

diff --git a/silkworm/db/head_info.hpp b/silkworm/db/head_info.hpp
@@ -17,7 +17,7 @@
 #pragma once
 
 #include <silkworm/core/types/head_info.hpp>
-#include <silkworm/db/mdbx.hpp>
+#include <silkworm/db/mdbx/mdbx.hpp>
 
 namespace silkworm::db {
 

diff --git a/silkworm/db/mdbx/CMakeLists.txt b/silkworm/db/mdbx/CMakeLists.txt
@@ -0,0 +1,31 @@
+#[[
+   Copyright 2024 The Silkworm Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+]]
+
+include("${SILKWORM_MAIN_DIR}/cmake/common/targets.cmake")
+
+find_package(absl REQUIRED)
+find_package(roaring REQUIRED)
+
+silkworm_library(
+  silkworm_db_mdbx
+  PUBLIC absl::btree
+         absl::function_ref
+         roaring::roaring
+         mdbx-static
+         silkworm_core
+         silkworm_infra
+         silkworm_db_etl
+)
diff --git a/silkworm/db/bitmap.cpp → silkworm/db/mdbx/bitmap.cpp b/silkworm/db/bitmap.cpp → silkworm/db/mdbx/bitmap.cpp
@@ -19,12 +19,13 @@
 #include <stdexcept>
 
 #include <silkworm/core/common/bytes_to_string.hpp>
-#include <silkworm/db/etl_mdbx_collector.hpp>
-#include <silkworm/db/util.hpp>
+#include <silkworm/core/common/endian.hpp>
 #include <silkworm/infra/common/binary_search.hpp>
 #include <silkworm/infra/common/ensure.hpp>
 #include <silkworm/infra/concurrency/signal_handler.hpp>
 
+#include "etl_mdbx_collector.hpp"
+
 namespace silkworm::db::bitmap {
 
 template <typename BlockUpperBound>

diff --git a/silkworm/db/bitmap.hpp → silkworm/db/mdbx/bitmap.hpp b/silkworm/db/bitmap.hpp → silkworm/db/mdbx/bitmap.hpp
@@ -37,7 +37,8 @@
 
 #include <silkworm/core/common/base.hpp>
 #include <silkworm/core/common/bytes.hpp>
-#include <silkworm/db/mdbx.hpp>
+
+#include "mdbx.hpp"
 
 namespace silkworm::db::etl {
 class Collector;

diff --git a/silkworm/db/etl_mdbx_collector.hpp → silkworm/db/mdbx/etl_mdbx_collector.hpp b/silkworm/db/etl_mdbx_collector.hpp → silkworm/db/mdbx/etl_mdbx_collector.hpp
@@ -16,9 +16,9 @@
 
 #pragma once
 
-#include "etl/collector.hpp"
+#include <silkworm/db/etl/collector.hpp>
+
 #include "mdbx.hpp"
-#include "util.hpp"
 
 namespace silkworm::db::etl_mdbx {
 

diff --git a/silkworm/db/mdbx.cpp → silkworm/db/mdbx/mdbx.cpp b/silkworm/db/mdbx.cpp → silkworm/db/mdbx/mdbx.cpp
@@ -18,7 +18,6 @@
 
 #include <stdexcept>
 
-#include <silkworm/db/util.hpp>
 #include <silkworm/infra/common/log.hpp>
 
 namespace silkworm::db {

diff --git a/silkworm/db/mdbx.hpp → silkworm/db/mdbx/mdbx.hpp b/silkworm/db/mdbx.hpp → silkworm/db/mdbx/mdbx.hpp
@@ -587,4 +587,12 @@ size_t cursor_erase(RWCursor& cursor, ByteView set_key,
 //! \param [in] prefix : Delete keys starting with this prefix
 size_t cursor_erase_prefix(RWCursor& cursor, ByteView prefix);
 
+inline Slice to_slice(ByteView value) {
+    return {value.data(), value.length()};
+}
+
+inline ByteView from_slice(const Slice slice) {
+    return {static_cast<const uint8_t*>(slice.data()), slice.length()};
+}
+
 }  // namespace silkworm::db
diff --git a/silkworm/db/memory_mutation.cpp → silkworm/db/mdbx/memory_mutation.cpp b/silkworm/db/memory_mutation.cpp → silkworm/db/mdbx/memory_mutation.cpp
@@ -18,11 +18,11 @@
 
 #include <gsl/util>
 
-#include <silkworm/db/memory_mutation_cursor.hpp>
-#include <silkworm/db/tables.hpp>
 #include <silkworm/infra/common/directories.hpp>
 #include <silkworm/infra/common/log.hpp>
 
+#include "memory_mutation_cursor.hpp"
+
 namespace silkworm::db {
 
 MemoryDatabase::MemoryDatabase(const std::filesystem::path& tmp_dir) {
@@ -37,28 +37,23 @@ MemoryDatabase::MemoryDatabase(const std::filesystem::path& tmp_dir) {
         .max_size = 512_Mebi,
     };
     memory_env_ = db::open_env(memory_config);
-
-    // Create predefined tables for chaindata schema
-    RWTxnManaged txn{memory_env_};
-    table::check_or_create_chaindata_tables(txn);
-    txn.commit_and_stop();
 }
 
 MemoryDatabase::MemoryDatabase() : MemoryDatabase(TemporaryDirectory::get_unique_temporary_path()) {}
 
-MemoryDatabase::MemoryDatabase(MemoryDatabase&& other) noexcept : memory_env_(std::move(other.memory_env_)) {}
-
 ::mdbx::txn_managed MemoryDatabase::start_rw_txn() {
     return memory_env_.start_write();
 }
 
-MemoryOverlay::MemoryOverlay(const std::filesystem::path& tmp_dir, silkworm::db::ROTxn* txn)
-    : memory_db_{tmp_dir}, txn_(txn) {}
-
-MemoryOverlay::MemoryOverlay(silkworm::db::ROTxn* txn) : memory_db_{}, txn_(txn) {}
-
-MemoryOverlay::MemoryOverlay(MemoryOverlay&& other) noexcept
-    : memory_db_(std::move(other.memory_db_)), txn_(other.txn_) {}
+MemoryOverlay::MemoryOverlay(
+    const std::filesystem::path& tmp_dir,
+    silkworm::db::ROTxn* txn,
+    std::function<std::optional<MapConfig>(const std::string& map_name)> get_map_config,
+    std::string sequence_map_name)
+    : memory_db_(tmp_dir),
+      txn_(txn),
+      get_map_config_(std::move(get_map_config)),
+      sequence_map_name_(std::move(sequence_map_name)) {}
 
 void MemoryOverlay::update_txn(ROTxn* txn) {
     txn_ = txn;
@@ -68,11 +63,21 @@ ::mdbx::txn_managed MemoryOverlay::start_rw_txn() {
     return memory_db_.start_rw_txn();
 }
 
+std::optional<MapConfig> MemoryOverlay::map_config(const std::string& map_name) {
+    return get_map_config_(map_name);
+}
+
+MapConfig MemoryOverlay::sequence_map_config() {
+    return *map_config(sequence_map_name_);
+}
+
 MemoryMutation::MemoryMutation(MemoryOverlay& overlay)
-    : RWTxnManaged{overlay.start_rw_txn()}, overlay_(overlay) {
+    : RWTxnManaged(overlay.start_rw_txn()),
+      overlay_(overlay) {
     // Initialize sequences
-    db::PooledCursor cursor{*overlay_.external_txn(), db::table::kSequence};
-    db::PooledCursor memory_cursor{managed_txn_, db::table::kSequence};
+    const auto sequence_map_config = overlay.sequence_map_config();
+    db::PooledCursor cursor{*overlay_.external_txn(), sequence_map_config};
+    db::PooledCursor memory_cursor{managed_txn_, sequence_map_config};
     for (auto result = cursor.to_first(false); result; result = cursor.to_next(false)) {
         memory_cursor.put(result.key, &result.value, MDBX_put_flags_t::MDBX_UPSERT);
     }
@@ -149,7 +154,7 @@ void MemoryMutation::flush(db::RWTxn& rw_txn) {
 
     // Obliterate entries that need to be deleted
     for (const auto& [table, keys] : this->deleted_entries_) {
-        const auto& table_config = db::table::get_map_config(table);
+        const auto table_config = overlay_.map_config(table);
         if (!table_config) {
             SILK_WARN << "Unknown table " << table << " in memory mutation, ignored";
             continue;
@@ -163,7 +168,7 @@ void MemoryMutation::flush(db::RWTxn& rw_txn) {
     // Iterate over each touched bucket and apply changes accordingly
     const auto tables = db::list_maps(managed_txn_);
     for (const auto& table : tables) {
-        const auto& table_config = db::table::get_map_config(table);
+        const auto table_config = overlay_.map_config(table);
         if (!table_config) {
             SILK_WARN << "Unknown table " << table << " in memory mutation, ignored";
             continue;