feat(bb): wasmtime and remote benchmarking (AztecProtocol#4204)

- provides a script that runs threaded goblin_bench in WASM with 16 cores - point to https://github.com/AztecProtocol/google-benchmark with changes for our custom WASI environment. This included removing CMake's thread library heuristics (which were unneeded, anyway) and adding checks for a wasm environment when measuring time. See AztecProtocol/barretenberg#837 for followups. - remove conditional BENCHMARKS in CMakeLists.txt. There's no reason to conditionally provide benchmark targets now that WASM can process the google benchmark cmake - add some WASMTIME_ENV_HACK guards around fake SRS and a fixed core count. - helpers for running benchmarks on remote nodes --------- Co-authored-by: ludamad <[email protected]>
Swoir · Jan 30, 2024 · fd27808 · fd27808
1 parent 14e8c5c
commit fd27808
Show file tree

Hide file tree

Showing 56 changed files with 356 additions and 379 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -153,5 +153,8 @@
   "C_Cpp.default.enableConfigurationSquiggles": false,
   "C_Cpp.formatting": "disabled",
   "C_Cpp.vcpkg.enabled": false,
-  "C_Cpp.default.includePath": ["barretenberg/cpp/src"],
+  "C_Cpp.default.includePath": [
+    "barretenberg/cpp/src"
+  ],
+  "cmake.sourceDirectory": "/mnt/user-data/adam/aztec-packages/barretenberg/cpp",
 }
diff --git a/barretenberg/cpp/CMakeLists.txt b/barretenberg/cpp/CMakeLists.txt
@@ -16,14 +16,11 @@ configure_file(
     @ONLY
 )
 
-
 option(DISABLE_ASM "Disable custom assembly" OFF)
 option(DISABLE_ADX "Disable ADX assembly variant" OFF)
 option(MULTITHREADING "Enable multi-threading" ON)
 option(OMP_MULTITHREADING "Enable OMP multi-threading" OFF)
-option(TESTING "Build tests" ON)
-option(BENCHMARKS "Build benchmarks" ON)
-option(FUZZING "Build fuzzing harnesses" OFF)
+option(FUZZING "Build ONLY fuzzing harnesses" OFF)
 option(DISABLE_TBB "Intel Thread Building Blocks" ON)
 option(COVERAGE "Enable collecting coverage from tests" OFF)
 option(ENABLE_ASAN "Address sanitizer for debugging tricky memory corruption" OFF)
@@ -52,30 +49,17 @@ if(FUZZING)
         add_definitions(-DDISABLE_CUSTOM_MUTATORS=1)
     endif()
 
-    set(SANITIZER_OPTIONS "")
-
-    if(ADDRESS_SANITIZER)
-        set(SANITIZER_OPTIONS ${SANITIZER_OPTIONS} -fsanitize=address)
-    endif()
-
-    if(UNDEFINED_BEHAVIOUR_SANITIZER)
-        set(SANITIZER_OPTIONS ${SANITIZER_OPTIONS} -fsanitize=undefined -fno-sanitize=alignment)
-    endif()
-
-    add_compile_options(-fsanitize=fuzzer-no-link ${SANITIZER_OPTIONS})
+    add_compile_options(-fsanitize=fuzzer)
+    add_link_options(-fsanitize=fuzzer)
 
-    set(WASM OFF)
-    set(BENCHMARKS OFF)
     set(MULTITHREADING OFF)
-    set(TESTING OFF)
 endif()
 
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "wasm32")
     message(STATUS "Compiling for WebAssembly.")
     set(WASM ON)
     set(DISABLE_ASM ON)
     set(OMP_MULTITHREADING OFF)
-    set(BENCHMARKS OFF)
     set(DISABLE_TBB 1)
     add_compile_definitions(_WASI_EMULATED_PROCESS_CLOCKS=1)
 endif()

diff --git a/barretenberg/cpp/CMakePresets.json b/barretenberg/cpp/CMakePresets.json
@@ -140,6 +140,19 @@
         "LDFLAGS": "-fsanitize=thread"
       }
     },
+    {
+      "name": "ubsan",
+      "displayName": "Debugging build with undefined behaviour sanitizer on Clang-16",
+      "description": "Build with undefined behaviour sanitizer on clang16 with debugging information",
+      "inherits": "clang16-dbg",
+      "binaryDir": "build-ubsan",
+      "generator": "Unix Makefiles",
+      "environment": {
+        "CFLAGS": "-fsanitize=undefined",
+        "CXXFLAGS": "-fsanitize=undefined",
+        "LDFLAGS": "-fsanitize=undefined"
+      }
+    },
     {
       "name": "msan",
       "displayName": "Debugging build with memory sanitizer on Clang-16",
@@ -151,10 +164,6 @@
         "CFLAGS": "-fsanitize=memory",
         "CXXFLAGS": "-fsanitize=memory",
         "LDFLAGS": "-fsanitize=memory"
-      },
-      "cacheVariables": {
-        "BENCHMARK": "OFF",
-        "TESTING": "OFF"
       }
     },
     {
@@ -228,6 +237,16 @@
         "MULTITHREADING": "ON"
       }
     },
+    {
+      "name": "wasm-bench",
+      "displayName": "WASM benchmarking.",
+      "description": "WASM benchmarking.",
+      "inherits": "wasm-threads",
+      "binaryDir": "build-wasm-bench",
+      "environment": {
+        "CXXFLAGS": "-DWASMTIME_ENV_HACK"
+      }
+    },
     {
       "name": "xray",
       "displayName": "Build with multi-threaded XRay Profiling",
@@ -336,6 +355,11 @@
       "inherits": "default",
       "configurePreset": "tsan"
     },
+    {
+      "name": "ubsan",
+      "inherits": "default",
+      "configurePreset": "ubsan"
+    },
     {
       "name": "coverage",
       "inherits": "default",
@@ -368,6 +392,12 @@
       "jobs": 0,
       "targets": ["barretenberg.wasm"]
     },
+    {
+      "name": "wasm-bench",
+      "configurePreset": "wasm-bench",
+      "inheritConfigureEnvironment": true,
+      "jobs": 0
+    },
     {
       "name": "xray",
       "configurePreset": "xray",

diff --git a/barretenberg/cpp/cmake/benchmark.cmake b/barretenberg/cpp/cmake/benchmark.cmake
@@ -1,25 +1,19 @@
-if(NOT TESTING)
-    set(BENCHMARKS OFF)
-endif()
-
-if(BENCHMARKS)
-    include(FetchContent)
+include(FetchContent)
 
-    FetchContent_Declare(
-        benchmark
-        GIT_REPOSITORY https://github.com/google/benchmark
-        GIT_TAG v1.7.1
-        FIND_PACKAGE_ARGS
-    )
+FetchContent_Declare(
+    benchmark
+    GIT_REPOSITORY https://github.com/AztecProtocol/google-benchmark
+    GIT_TAG 7638387d2727853d970fc9420dcf95cf3e9bd112
+    FIND_PACKAGE_ARGS
+)
 
-    set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Benchmark tests off")
-    set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "Benchmark installation off")
+set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Benchmark tests off")
+set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "Benchmark installation off")
 
-    FetchContent_MakeAvailable(benchmark)
-    if(NOT benchmark_FOUND)
-        # FetchContent_MakeAvailable calls FetchContent_Populate if `find_package` is unsuccessful
-        # so these variables will be available if we reach this case
-        set_property(DIRECTORY ${benchmark_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL)
-        set_property(DIRECTORY ${benchmark_BINARY_DIR} PROPERTY EXCLUDE_FROM_ALL)
-    endif()
+FetchContent_MakeAvailable(benchmark)
+if(NOT benchmark_FOUND)
+    # FetchContent_MakeAvailable calls FetchContent_Populate if `find_package` is unsuccessful
+    # so these variables will be available if we reach this case
+    set_property(DIRECTORY ${benchmark_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL)
+    set_property(DIRECTORY ${benchmark_BINARY_DIR} PROPERTY EXCLUDE_FROM_ALL)
 endif()
diff --git a/barretenberg/cpp/cmake/gtest.cmake b/barretenberg/cpp/cmake/gtest.cmake
@@ -1,51 +1,49 @@
-if(TESTING)
-    include(GoogleTest)
-    include(FetchContent)
-
-    FetchContent_Declare(
-        GTest
-        GIT_REPOSITORY https://github.com/google/googletest.git
-        # Version 1.12.1 is not compatible with WASI-SDK 12
-        GIT_TAG release-1.10.0
-        FIND_PACKAGE_ARGS
+include(GoogleTest)
+include(FetchContent)
+
+FetchContent_Declare(
+    GTest
+    GIT_REPOSITORY https://github.com/google/googletest.git
+    # Version 1.12.1 is not compatible with WASI-SDK 12
+    GIT_TAG release-1.10.0
+    FIND_PACKAGE_ARGS
+)
+
+set(BUILD_GMOCK OFF CACHE BOOL "Build with gMock disabled")
+set(INSTALL_GTEST OFF CACHE BOOL "gTest installation disabled")
+
+FetchContent_MakeAvailable(GTest)
+
+if (NOT GTest_FOUND)
+    # FetchContent_MakeAvailable calls FetchContent_Populate if `find_package` is unsuccessful
+    # so these variables will be available if we reach this case
+    set_property(DIRECTORY ${gtest_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL)
+    set_property(DIRECTORY ${gtest_BINARY_DIR} PROPERTY EXCLUDE_FROM_ALL)
+
+    # Disable all warning when compiling gtest
+    target_compile_options(
+        gtest
+        PRIVATE
+        -w
     )
 
-    set(BUILD_GMOCK OFF CACHE BOOL "Build with gMock disabled")
-    set(INSTALL_GTEST OFF CACHE BOOL "gTest installation disabled")
-
-    FetchContent_MakeAvailable(GTest)
-
-    if (NOT GTest_FOUND)
-        # FetchContent_MakeAvailable calls FetchContent_Populate if `find_package` is unsuccessful
-        # so these variables will be available if we reach this case
-        set_property(DIRECTORY ${gtest_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL)
-        set_property(DIRECTORY ${gtest_BINARY_DIR} PROPERTY EXCLUDE_FROM_ALL)
-
-        # Disable all warning when compiling gtest
-        target_compile_options(
+    if(WASM)
+        target_compile_definitions(
             gtest
             PRIVATE
-            -w
-        )
-
-        if(WASM)
-            target_compile_definitions(
-                gtest
-                PRIVATE
-                -DGTEST_HAS_EXCEPTIONS=0
-                -DGTEST_HAS_STREAM_REDIRECTION=0
-            )
-        endif()
-
-        mark_as_advanced(
-            BUILD_GMOCK BUILD_GTEST BUILD_SHARED_LIBS
-            gmock_build_tests gtest_build_samples gtest_build_tests
-            gtest_disable_pthreads gtest_force_shared_crt gtest_hide_internal_symbols
+            -DGTEST_HAS_EXCEPTIONS=0
+            -DGTEST_HAS_STREAM_REDIRECTION=0
         )
-
-        add_library(GTest::gtest ALIAS gtest)
-        add_library(GTest::gtest_main ALIAS gtest_main)
     endif()
 
-    enable_testing()
+    mark_as_advanced(
+        BUILD_GMOCK BUILD_GTEST BUILD_SHARED_LIBS
+        gmock_build_tests gtest_build_samples gtest_build_tests
+        gtest_disable_pthreads gtest_force_shared_crt gtest_hide_internal_symbols
+    )
+
+    add_library(GTest::gtest ALIAS gtest)
+    add_library(GTest::gtest_main ALIAS gtest_main)
 endif()
+
+enable_testing()
diff --git a/barretenberg/cpp/cmake/module.cmake b/barretenberg/cpp/cmake/module.cmake
@@ -65,7 +65,7 @@ function(barretenberg_module MODULE_NAME)
     endif()
 
     file(GLOB_RECURSE TEST_SOURCE_FILES *.test.cpp)
-    if(TESTING AND TEST_SOURCE_FILES)
+    if(TEST_SOURCE_FILES AND NOT FUZZING)
         add_library(
             ${MODULE_NAME}_test_objects
             OBJECT
@@ -184,7 +184,6 @@ function(barretenberg_module MODULE_NAME)
                 ${MODULE_NAME}_${FUZZER_NAME_STEM}_fuzzer
                 PRIVATE
                 "-fsanitize=fuzzer"
-                ${SANITIZER_OPTIONS}
             )
 
             target_link_libraries(
@@ -196,44 +195,47 @@ function(barretenberg_module MODULE_NAME)
     endif()
 
     file(GLOB_RECURSE BENCH_SOURCE_FILES *.bench.cpp)
-    if(BENCHMARKS AND BENCH_SOURCE_FILES)
-        add_library(
-            ${MODULE_NAME}_bench_objects
-            OBJECT
-            ${BENCH_SOURCE_FILES}
-        )
-        list(APPEND lib_targets ${MODULE_NAME}_bench_objects)
+    if(BENCH_SOURCE_FILES AND NOT FUZZING)
+        foreach(BENCHMARK_SOURCE ${BENCH_SOURCE_FILES})
+            get_filename_component(BENCHMARK_NAME ${BENCHMARK_SOURCE} NAME_WE) # extract name without extension
+            add_library(
+                ${BENCHMARK_NAME}_bench_objects
+                OBJECT
+                ${BENCHMARK_SOURCE}
+            )
+            list(APPEND lib_targets ${BENCHMARK_NAME}_bench_objects)
 
-        target_link_libraries(
-            ${MODULE_NAME}_bench_objects
-            PRIVATE
-            benchmark::benchmark
-            ${TBB_IMPORTED_TARGETS}
-        )
+            target_link_libraries(
+                ${BENCHMARK_NAME}_bench_objects
+                PRIVATE
+                benchmark::benchmark
+                ${TBB_IMPORTED_TARGETS}
+            )
 
-        add_executable(
-            ${MODULE_NAME}_bench
-            $<TARGET_OBJECTS:${MODULE_NAME}_bench_objects>
-        )
-        list(APPEND exe_targets ${MODULE_NAME}_bench)
+            add_executable(
+                ${BENCHMARK_NAME}_bench
+                $<TARGET_OBJECTS:${BENCHMARK_NAME}_bench_objects>
+            )
+            list(APPEND exe_targets ${MODULE_NAME}_bench)
 
-        target_link_libraries(
-            ${MODULE_NAME}_bench
-            PRIVATE
-            ${MODULE_LINK_NAME}
-            ${ARGN}
-            benchmark::benchmark
-            ${TBB_IMPORTED_TARGETS}
-        )
+            target_link_libraries(
+                ${BENCHMARK_NAME}_bench
+                PRIVATE
+                ${MODULE_LINK_NAME}
+                ${ARGN}
+                benchmark::benchmark
+                ${TBB_IMPORTED_TARGETS}
+            )
 
-        # enable msgpack downloading via dependency (solves race condition)
-        add_dependencies(${MODULE_NAME}_bench_objects msgpack-c)
-        add_dependencies(${MODULE_NAME}_bench msgpack-c)
-        add_custom_target(
-            run_${MODULE_NAME}_bench
-            COMMAND ${MODULE_NAME}_bench
-            WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
-        )
+            # enable msgpack downloading via dependency (solves race condition)
+            add_dependencies(${BENCHMARK_NAME}_bench_objects msgpack-c)
+            add_dependencies(${BENCHMARK_NAME}_bench msgpack-c)
+            add_custom_target(
+                run_${BENCHMARK_NAME}_bench
+                COMMAND ${BENCHMARK_NAME}_bench
+                WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+            )
+        endforeach()
     endif()
 
     set(${MODULE_NAME}_lib_targets ${lib_targets} PARENT_SCOPE)

diff --git a/barretenberg/cpp/docs/Fuzzing.md b/barretenberg/cpp/docs/Fuzzing.md
@@ -15,15 +15,11 @@ cmake --build --preset fuzzing
 
 Fuzzing build turns off building tests and benchmarks, since they are incompatible with libfuzzer interface.
 
-To turn on address sanitizer add `-DADDRESS_SANITIZER=ON`. Note that address sanitizer can be used to explore crashes.
+To turn on address sanitizer instead run `cmake --preset asan -DFUZZING=ON`. Note that address sanitizer can be used to explore crashes.
 Sometimes you might have to specify the address of llvm-symbolizer. You have to do it with `export ASAN_SYMBOLIZER_PATH=<PATH_TO_SYMBOLIZER>`.
-For undefined behavior sanitizer `-DUNDEFINED_BEHAVIOUR_SANITIZER=ON`.
+For undefined behavior sanitizer use `cmake --preset ubsan -DFUZZING=ON`.
 Note that the fuzzer can be orders of magnitude slower with ASan (2-3x slower) or UBSan on, so it is best to run a non-sanitized build first, minimize the testcase and then run it for a bit of time with sanitizers.
 
-Building with clang 13 or later is recommended, since libfuzzer contains and by default utilizes the entropic power schedule, which is considered more efficient
-than the standard one present in previous versions.
-You can downloadload the latest clang+llvm release here: https://github.com/llvm/llvm-project/releases
-
 To set up cmake with another version of clang and fuzzing on:
 
 ```bash