feat: Native Merkle Trees (#7037)

This PR is the first of several where we are moving to use native
implementations of Merkle Trees for performance reasons. It includes:

1. The core tree implementations for both indexed trees and append only
trees.
2. The introduction of LMDB as a dependency that we retrieve from GIT
and build ourselves.
3. The creation of a set of RAII wrapper objects around the LMDB
concepts.
4. The creation of a committed/uncommitted store on top of LMDB used by
the trees for state management

---------

Co-authored-by: IlyasRidhuan <[email protected]>
Co-authored-by: Alex Gherghisan <[email protected]>

.github/workflows/ci.yml

@@ -277,7 +277,7 @@ jobs:
         # limit our parallelism to half our cores
         run: earthly-ci --no-output +preset-gcc
 
-  # barretenberg (prover) native and AVM (public VM) tests
+  # barretenberg (prover) native, AVM (public VM) and Merkle tree (world state) tests
   # only ran on x86 for resource reasons (memory intensive)
   bb-native-tests:
     needs: [build-images, changes]

barretenberg/cpp/CMakeLists.txt

@@ -152,6 +152,7 @@ include(cmake/gtest.cmake)
 include(cmake/benchmark.cmake)
 include(cmake/module.cmake)
 include(cmake/msgpack.cmake)
+include(cmake/lmdb.cmake)
 
 # We do not need to bloat barretenberg.wasm with gzip functionality in a browser context as the browser can do this
 if (NOT WASM)

barretenberg/cpp/Earthfile

@@ -31,7 +31,7 @@ preset-release:
 
 preset-release-assert:
     FROM +source
-    RUN cmake --preset clang16-assert -Bbuild && cmake --build build --target bb && rm -rf build/{deps,lib,src}
+    RUN cmake --preset clang16-assert -Bbuild && cmake --build build --target bb crypto_merkle_tree_tests && rm -rf build/{deps,lib,src}
     SAVE ARTIFACT build/bin
 
 preset-debug:

barretenberg/cpp/cmake/lmdb.cmake

@@ -0,0 +1,27 @@
+include(ExternalProject)
+
+set(LMDB_PREFIX "${CMAKE_BINARY_DIR}/_deps/lmdb")
+set(LMDB_INCLUDE "${LMDB_PREFIX}/src/lmdb_repo/libraries/liblmdb")
+set(LMDB_LIB "${LMDB_INCLUDE}/liblmdb.a")
+set(LMDB_OBJECT "${LMDB_INCLUDE}/*.o")
+
+ExternalProject_Add(
+    lmdb_repo
+    PREFIX ${LMDB_PREFIX}
+    GIT_REPOSITORY "https://github.com/LMDB/lmdb.git"
+    GIT_TAG ddd0a773e2f44d38e4e31ec9ed81af81f4e4ccbb
+    BUILD_IN_SOURCE YES
+    CONFIGURE_COMMAND "" # No configure step
+    BUILD_COMMAND make -C libraries/liblmdb -e XCFLAGS=-fPIC liblmdb.a
+    INSTALL_COMMAND ""
+    UPDATE_COMMAND "" # No update step
+    BUILD_BYPRODUCTS ${LMDB_LIB} ${LMDB_INCLUDE}
+)
+
+add_library(lmdb STATIC IMPORTED GLOBAL)
+add_dependencies(lmdb lmdb_repo)
+set_target_properties(lmdb PROPERTIES IMPORTED_LOCATION ${LMDB_LIB})
+
+add_library(lmdb_objects OBJECT IMPORTED GLOBAL)
+add_dependencies(lmdb_objects lmdb_repo)
+set_target_properties(lmdb_objects PROPERTIES IMPORTED_LOCATION ${LMDB_OBJECT})

barretenberg/cpp/scripts/merkle_tree_tests.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -e
+
+# run commands relative to parent directory
+cd $(dirname $0)/..
+
+DEFAULT_TESTS=PersistedIndexedTreeTest.*:PersistedAppendOnlyTreeTest.*:LMDBStoreTest.*
+TEST=${1:-$DEFAULT_TESTS}
+PRESET=${PRESET:-clang16}
+
+cmake --build --preset $PRESET --target crypto_merkle_tree_tests
+./build/bin/crypto_merkle_tree_tests --gtest_filter=$TEST

barretenberg/cpp/src/CMakeLists.txt

@@ -21,6 +21,7 @@ if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
         message(STATUS "Compiling with memory checks.")
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address")
     endif()
+
     if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 18)
         # We target clang18 and need this, eventually warning should be fixed or this will be unconditional.
         add_compile_options(-Wno-vla-cxx-extension)
@@ -41,7 +42,7 @@ if(WASM)
     add_link_options(-Wl,--export-memory,--import-memory,--stack-first,-z,stack-size=1048576,--max-memory=4294967296)
 endif()
 
-include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${MSGPACK_INCLUDE} ${TRACY_INCLUDE})
+include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${MSGPACK_INCLUDE} ${TRACY_INCLUDE} ${LMDB_INCLUDE})
 
 # I feel this should be limited to ecc, however it's currently used in headers that go across libraries,
 # and there currently isn't an easy way to inherit the DDISABLE_ASM parameter.
@@ -87,7 +88,6 @@ add_subdirectory(barretenberg/ultra_honk)
 add_subdirectory(barretenberg/vm)
 add_subdirectory(barretenberg/wasi)
 
-
 if(SMT)
     add_subdirectory(barretenberg/smt_verification)
 endif()
@@ -140,7 +140,6 @@ set(BARRETENBERG_TARGET_OBJECTS
     $<TARGET_OBJECTS:stdlib_honk_verifier_objects>
     $<TARGET_OBJECTS:stdlib_goblin_verifier_objects>
     $<TARGET_OBJECTS:stdlib_keccak_objects>
-    $<TARGET_OBJECTS:crypto_merkle_tree_objects>
     $<TARGET_OBJECTS:stdlib_pedersen_commitment_objects>
     $<TARGET_OBJECTS:stdlib_pedersen_hash_objects>
     $<TARGET_OBJECTS:stdlib_plonk_recursion_objects>
@@ -160,6 +159,11 @@ if(NOT DISABLE_AZTEC_VM)
     list(APPEND BARRETENBERG_TARGET_OBJECTS $<TARGET_OBJECTS:vm_objects>)
 endif()
 
+if(NOT WASM)
+    # enable merkle trees
+    list(APPEND BARRETENBERG_TARGET_OBJECTS $<TARGET_OBJECTS:crypto_merkle_tree_objects>)
+endif()
+
 add_library(
     barretenberg
     STATIC

barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/CMakeLists.txt

@@ -1 +1 @@
-barretenberg_module(append_only_tree_bench crypto_poseidon2 crypto_merkle_tree)
+barretenberg_module(append_only_tree_bench crypto_poseidon2 crypto_pedersen_hash crypto_merkle_tree)

barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp

@@ -1,34 +1,63 @@
 #include "barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp"
-#include "barretenberg/crypto/merkle_tree/array_store.hpp"
+#include "barretenberg/common/thread_pool.hpp"
+#include "barretenberg/crypto/merkle_tree/fixtures.hpp"
 #include "barretenberg/crypto/merkle_tree/hash.hpp"
+#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp"
+#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp"
+#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp"
+#include "barretenberg/crypto/merkle_tree/node_store/array_store.hpp"
+#include "barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp"
+#include "barretenberg/crypto/merkle_tree/response.hpp"
+#include "barretenberg/ecc/curves/bn254/fr.hpp"
 #include "barretenberg/numeric/random/engine.hpp"
 #include <benchmark/benchmark.h>
+#include <cstdint>
+#include <filesystem>
 
 using namespace benchmark;
 using namespace bb::crypto::merkle_tree;
 
-using Pedersen = AppendOnlyTree<ArrayStore, PedersenHashPolicy>;
-using Poseidon2 = AppendOnlyTree<ArrayStore, Poseidon2HashPolicy>;
+namespace {
+using StoreType = CachedTreeStore<LMDBStore, fr>;
+
+using Pedersen = AppendOnlyTree<StoreType, PedersenHashPolicy>;
+using Poseidon2 = AppendOnlyTree<StoreType, Poseidon2HashPolicy>;
 
 const size_t TREE_DEPTH = 32;
 const size_t MAX_BATCH_SIZE = 128;
 
-namespace {
-auto& random_engine = bb::numeric::get_randomness();
-} // namespace
-
 template <typename TreeType> void perform_batch_insert(TreeType& tree, const std::vector<fr>& values)
 {
-    tree.add_values(values);
+    Signal signal(1);
+    auto completion = [&](const TypedResponse<AddDataResponse>&) -> void { signal.signal_level(0); };
+
+    tree.add_values(values, completion);
+    signal.wait_for_level(0);
+}
+
+template <typename TreeType> void commit_tree(TreeType& tree)
+{
+    Signal signal(1);
+    auto completion = [&]() -> void { signal.signal_level(0); };
+    tree.commit(completion);
+    signal.wait_for_level(0);
 }
 
 template <typename TreeType> void append_only_tree_bench(State& state) noexcept
 {
     const size_t batch_size = size_t(state.range(0));
     const size_t depth = TREE_DEPTH;
 
-    ArrayStore store(depth, 1024 * 1024);
-    TreeType tree = TreeType(store, depth);
+    std::string directory = random_temp_directory();
+    std::string name = random_string();
+    std::filesystem::create_directories(directory);
+    uint32_t num_threads = 16;
+    LMDBEnvironment environment = LMDBEnvironment(directory, 1024 * 1024, 2, num_threads);
+
+    LMDBStore db(environment, name, false, false, integer_key_cmp);
+    StoreType store(name, depth, db);
+    ThreadPool workers(num_threads);
+    TreeType tree = TreeType(store, workers);
 
     for (auto _ : state) {
         state.PauseTiming();
@@ -39,6 +68,8 @@ template <typename TreeType> void append_only_tree_bench(State& state) noexcept
         state.ResumeTiming();
         perform_batch_insert(tree, values);
     }
+
+    std::filesystem::remove_all(directory);
 }
 BENCHMARK(append_only_tree_bench<Pedersen>)
     ->Unit(benchmark::kMillisecond)
@@ -51,4 +82,6 @@ BENCHMARK(append_only_tree_bench<Poseidon2>)
     ->Range(2, MAX_BATCH_SIZE)
     ->Iterations(1000);
 
+} // namespace
+
 BENCHMARK_MAIN();

barretenberg/cpp/src/barretenberg/benchmark/client_ivc_bench/CMakeLists.txt

@@ -1 +1 @@
-barretenberg_module(client_ivc_bench client_ivc stdlib_honk_verifier stdlib_sha256 crypto_merkle_tree stdlib_primitives)
+barretenberg_module(client_ivc_bench client_ivc stdlib_honk_verifier stdlib_sha256 stdlib_primitives)

barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/CMakeLists.txt

@@ -1 +1 @@
-barretenberg_module(indexed_tree_bench crypto_poseidon2 crypto_merkle_tree)
+barretenberg_module(indexed_tree_bench crypto_poseidon2 crypto_pedersen_hash crypto_merkle_tree)

barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp

@@ -1,45 +1,59 @@
 #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp"
-#include "barretenberg/crypto/merkle_tree/array_store.hpp"
+#include "barretenberg/crypto/merkle_tree/fixtures.hpp"
 #include "barretenberg/crypto/merkle_tree/hash.hpp"
-#include "barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp"
+#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp"
+#include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp"
+#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp"
+#include "barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp"
+#include "barretenberg/crypto/merkle_tree/response.hpp"
 #include "barretenberg/numeric/random/engine.hpp"
 #include <benchmark/benchmark.h>
+#include <filesystem>
 
 using namespace benchmark;
 using namespace bb::crypto::merkle_tree;
 
-using Poseidon2 = IndexedTree<ArrayStore, LeavesCache, Poseidon2HashPolicy>;
-using Pedersen = IndexedTree<ArrayStore, LeavesCache, PedersenHashPolicy>;
+using StoreType = CachedTreeStore<LMDBStore, NullifierLeafValue>;
 
-const size_t TREE_DEPTH = 32;
-const size_t MAX_BATCH_SIZE = 128;
+using Poseidon2 = IndexedTree<StoreType, Poseidon2HashPolicy>;
+using Pedersen = IndexedTree<StoreType, PedersenHashPolicy>;
 
-namespace {
-auto& random_engine = bb::numeric::get_randomness();
-} // namespace
+const size_t TREE_DEPTH = 40;
+const size_t MAX_BATCH_SIZE = 128;
 
-template <typename TreeType>
-void perform_batch_insert(TreeType& tree, const std::vector<fr>& values, bool single_threaded)
+template <typename TreeType> void add_values(TreeType& tree, const std::vector<NullifierLeafValue>& values)
 {
-    tree.add_or_update_values(values, single_threaded);
+    Signal signal(1);
+    auto completion = [&](const auto&) -> void { signal.signal_level(0); };
+
+    tree.add_or_update_values(values, completion);
+    signal.wait_for_level(0);
 }
 
 template <typename TreeType> void multi_thread_indexed_tree_bench(State& state) noexcept
 {
     const size_t batch_size = size_t(state.range(0));
     const size_t depth = TREE_DEPTH;
 
-    ArrayStore store(depth, 1024 * 1024);
-    TreeType tree = TreeType(store, depth, batch_size);
+    std::string directory = random_temp_directory();
+    std::string name = random_string();
+    std::filesystem::create_directories(directory);
+    uint32_t num_threads = 16;
+    LMDBEnvironment environment = LMDBEnvironment(directory, 1024 * 1024, 2, num_threads);
+
+    LMDBStore db(environment, name, false, false, integer_key_cmp);
+    StoreType store(name, depth, db);
+    ThreadPool workers(num_threads);
+    TreeType tree = TreeType(store, workers, batch_size);
 
     for (auto _ : state) {
         state.PauseTiming();
-        std::vector<fr> values(batch_size);
+        std::vector<NullifierLeafValue> values(batch_size);
         for (size_t i = 0; i < batch_size; ++i) {
             values[i] = fr(random_engine.get_random_uint256());
         }
         state.ResumeTiming();
-        perform_batch_insert(tree, values, false);
+        add_values(tree, values);
     }
 }
 
@@ -48,17 +62,25 @@ template <typename TreeType> void single_thread_indexed_tree_bench(State& state)
     const size_t batch_size = size_t(state.range(0));
     const size_t depth = TREE_DEPTH;
 
-    ArrayStore store(depth, 1024 * 1024);
-    TreeType tree = TreeType(store, depth, batch_size);
+    std::string directory = random_temp_directory();
+    std::string name = random_string();
+    std::filesystem::create_directories(directory);
+    uint32_t num_threads = 1;
+    LMDBEnvironment environment = LMDBEnvironment(directory, 1024 * 1024, 2, num_threads);
+
+    LMDBStore db(environment, name, false, false, integer_key_cmp);
+    StoreType store(name, depth, db);
+    ThreadPool workers(num_threads);
+    TreeType tree = TreeType(store, workers, batch_size);
 
     for (auto _ : state) {
         state.PauseTiming();
-        std::vector<fr> values(batch_size);
+        std::vector<NullifierLeafValue> values(batch_size);
         for (size_t i = 0; i < batch_size; ++i) {
             values[i] = fr(random_engine.get_random_uint256());
         }
         state.ResumeTiming();
-        perform_batch_insert(tree, values, true);
+        add_values(tree, values);
     }
 }
 BENCHMARK(single_thread_indexed_tree_bench<Pedersen>)
@@ -84,4 +106,4 @@ BENCHMARK(multi_thread_indexed_tree_bench<Poseidon2>)
     ->Range(2, MAX_BATCH_SIZE)
     ->Iterations(1000);
 
-BENCHMARK_MAIN();
+BENCHMARK_MAIN();

barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/CMakeLists.txt

@@ -1 +1 @@
-barretenberg_module(merkle_tree_bench crypto_poseidon2 crypto_merkle_tree)
+barretenberg_module(merkle_tree_bench crypto_poseidon2 crypto_pedersen_hash crypto_merkle_tree)

barretenberg/cpp/src/barretenberg/benchmark/simulator_bench/CMakeLists.txt

@@ -1 +1 @@
- barretenberg_module(simulator_bench stdlib_honk_verifier stdlib_sha256 crypto_merkle_tree) 
+ barretenberg_module(simulator_bench stdlib_honk_verifier stdlib_sha256 stdlib_pedersen_hash)

barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/CMakeLists.txt

@@ -3,6 +3,7 @@ barretenberg_module(
   ultra_honk
   stdlib_sha256
   stdlib_keccak
-  crypto_merkle_tree
-  plonk  
+  stdlib_pedersen_hash
+  stdlib_poseidon2
+  plonk
 )