Skip to content

Commit

Permalink
feat: bb uses goblin (#3636)
Browse files Browse the repository at this point in the history
"vertical slice" of goblin integrating with ACIR and BB.

Beginning of goblin ultra honk interface for bb. Can only run a basic
test (assert_statement) for now and is not intended for use outside of
the CI check that it's still working, for now.
- adds bb and bb.js command `prove_and_verify_goblin`. Adds bb.js
bindings for goblin prove/verify
- modifies ACIR `dsl` folder to be able to take a goblin builder
- adds CI calls to bb.js and bb that use `prove_and_verify_goblin`
- allowing ability to load grumpkin SRS through memory, needed for bb
and (especially) bb.js. This allows an alternate source of points to be
used other than the default file-based grumpkin loader, which mostly
only works in dev (though revisit: could this work for native bb?)

---------

Co-authored-by: ledwards2225 <[email protected]>
Co-authored-by: ledwards2225 <[email protected]>
Co-authored-by: ludamad <[email protected]>
Co-authored-by: ludamad <[email protected]>
  • Loading branch information
5 people authored Dec 18, 2023
1 parent c0a24fb commit d093266
Show file tree
Hide file tree
Showing 93 changed files with 1,381 additions and 454 deletions.
2 changes: 2 additions & 0 deletions barretenberg/acir_tests/Dockerfile.bb
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,7 @@ COPY . .
# Run every acir test through native bb build prove_then_verify flow.
# This ensures we test independent pk construction through real/garbage witness data paths.
RUN FLOW=prove_then_verify ./run_acir_tests.sh
# TODO(https://github.com/AztecProtocol/barretenberg/issues/811) make this able to run the default test
RUN FLOW=prove_and_verify_goblin ./run_acir_tests.sh assert_statement
# Run 1_mul through native bb build, all_cmds flow, to test all cli args.
RUN VERBOSE=1 FLOW=all_cmds ./run_acir_tests.sh 1_mul
2 changes: 2 additions & 0 deletions barretenberg/acir_tests/Dockerfile.bb.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ COPY . .
ENV VERBOSE=1
# Run double_verify_proof through bb.js on node to check 512k support.
RUN BIN=../ts/dest/node/main.js FLOW=prove_then_verify ./run_acir_tests.sh double_verify_proof
# TODO(https://github.com/AztecProtocol/barretenberg/issues/811) make this able to run double_verify_proof
RUN BIN=../ts/dest/node/main.js FLOW=prove_and_verify_goblin ./run_acir_tests.sh assert_statement
# Run 1_mul through bb.js build, all_cmds flow, to test all cli args.
RUN BIN=../ts/dest/node/main.js FLOW=all_cmds ./run_acir_tests.sh 1_mul
# Run double_verify_proof through bb.js on chrome testing multi-threaded browser support.
Expand Down
6 changes: 6 additions & 0 deletions barretenberg/acir_tests/flows/prove_and_verify_goblin.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu

VFLAG=${VERBOSE:+-v}

$BIN prove_and_verify_goblin $VFLAG -c $CRS_PATH -b ./target/acir.gz
1 change: 1 addition & 0 deletions barretenberg/cpp/.dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

# Important srs_db files.
!srs_db/download_ignition.sh
!srs_db/download_grumpkin.sh
!srs_db/ignition/checksums

# Source code.
Expand Down
1 change: 1 addition & 0 deletions barretenberg/cpp/dockerfiles/Dockerfile.wasm-linux-clang
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,6 @@ RUN ./scripts/strip-wasm.sh
FROM scratch
WORKDIR /usr/src/barretenberg/cpp
COPY . .
COPY --from=builder /usr/src/barretenberg/cpp/srs_db /usr/src/barretenberg/cpp/srs_db
COPY --from=builder /usr/src/barretenberg/cpp/build-wasm/bin/barretenberg.wasm /usr/src/barretenberg/cpp/build-wasm/bin/barretenberg.wasm
COPY --from=builder /usr/src/barretenberg/cpp/build-wasm-threads/bin/barretenberg.wasm /usr/src/barretenberg/cpp/build-wasm-threads/bin/barretenberg.wasm
19 changes: 10 additions & 9 deletions barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-assert
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,20 @@ FROM alpine:3.17 AS builder
RUN apk update \
&& apk upgrade \
&& apk add --no-cache \
build-base \
clang15 \
cmake \
ninja \
git \
curl \
perl \
clang-extra-tools \
bash
build-base \
clang15 \
cmake \
ninja \
git \
curl \
perl \
clang-extra-tools \
bash
WORKDIR /usr/src/barretenberg/cpp
COPY . .
# Build everything to ensure everything builds. All tests will be run from the result of this build.
RUN ./format.sh check && cmake --preset default -DCMAKE_BUILD_TYPE=RelWithAssert -DCI=ON && cmake --build --preset default
RUN srs_db/download_grumpkin.sh

FROM alpine:3.17
RUN apk update && apk add curl libstdc++
Expand Down
4 changes: 2 additions & 2 deletions barretenberg/cpp/scripts/bb-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ TESTS_STR="${TESTS[@]}"
docker run --rm -t $IMAGE_URI /bin/sh -c "\
set -xe; \
cd /usr/src/barretenberg/cpp; \
(cd srs_db && ./download_ignition.sh 1); \
srs_db/download_ignition.sh 1; \
srs_db/download_grumpkin.sh; \
cd build; \
./bin/grumpkin_srs_gen 1048576; \
for BIN in $TESTS_STR; do ./bin/\$BIN; done"
5 changes: 2 additions & 3 deletions barretenberg/cpp/scripts/run_tests
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,7 @@ fi

docker run --rm -t $IMAGE_URI /bin/sh -c "\
set -xe; \
cd /usr/src/barretenberg/cpp/srs_db; \
./download_ignition.sh $NUM_TRANSCRIPTS; \
/usr/src/barretenberg/cpp/srs_db/download_ignition.sh $NUM_TRANSCRIPTS; \
/usr/src/barretenberg/cpp/srs_db/download_grumpkin.sh; \
cd /usr/src/barretenberg/cpp/build; \
./bin/grumpkin_srs_gen 1048576; \
for BIN in $TESTS; do ./bin/\$BIN $@; done"
55 changes: 20 additions & 35 deletions barretenberg/cpp/src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -96,9 +96,9 @@ include(GNUInstallDirs)
# libbarretenberg + libwasi = a wasi "reactor" that implements it's own env (e.g. logstr), e.g. barretenberg.wasm.
# libbarretenberg + env = a wasi "command" that expects a full wasi runtime (e.g. wasmtime), e.g. test binaries.
message(STATUS "Compiling all-in-one barretenberg archive")
add_library(
barretenberg
STATIC

set(BARRETENBERG_TARGET_OBJECTS
$<TARGET_OBJECTS:commitment_schemes_objects>
$<TARGET_OBJECTS:common_objects>
$<TARGET_OBJECTS:crypto_aes128_objects>
$<TARGET_OBJECTS:crypto_blake2s_objects>
Expand All @@ -111,12 +111,17 @@ add_library(
$<TARGET_OBJECTS:crypto_sha256_objects>
$<TARGET_OBJECTS:dsl_objects>
$<TARGET_OBJECTS:ecc_objects>
$<TARGET_OBJECTS:eccvm_objects>
$<TARGET_OBJECTS:examples_objects>
$<TARGET_OBJECTS:flavor_objects>
$<TARGET_OBJECTS:goblin_objects>
$<TARGET_OBJECTS:honk_objects>
$<TARGET_OBJECTS:numeric_objects>
$<TARGET_OBJECTS:plonk_objects>
$<TARGET_OBJECTS:polynomials_objects>
$<TARGET_OBJECTS:proof_system_objects>
$<TARGET_OBJECTS:protogalaxy_objects>
$<TARGET_OBJECTS:relations_objects>
$<TARGET_OBJECTS:srs_objects>
$<TARGET_OBJECTS:stdlib_aes128_objects>
$<TARGET_OBJECTS:stdlib_blake2s_objects>
Expand All @@ -126,9 +131,18 @@ add_library(
$<TARGET_OBJECTS:stdlib_pedersen_commitment_objects>
$<TARGET_OBJECTS:stdlib_pedersen_hash_objects>
$<TARGET_OBJECTS:stdlib_primitives_objects>
$<TARGET_OBJECTS:stdlib_recursion_objects>
$<TARGET_OBJECTS:stdlib_schnorr_objects>
$<TARGET_OBJECTS:stdlib_sha256_objects>
$<TARGET_OBJECTS:sumcheck_objects>
$<TARGET_OBJECTS:transcript_objects>
$<TARGET_OBJECTS:translator_vm_objects>
$<TARGET_OBJECTS:ultra_honk_objects>)

add_library(
barretenberg
STATIC
${BARRETENBERG_TARGET_OBJECTS}
)

if(WASM)
Expand All @@ -144,36 +158,7 @@ if(WASM)
# to implement the functions in env.
add_executable(
barretenberg.wasm
$<TARGET_OBJECTS:common_objects>
$<TARGET_OBJECTS:crypto_aes128_objects>
$<TARGET_OBJECTS:crypto_blake2s_objects>
$<TARGET_OBJECTS:crypto_blake3s_objects>
$<TARGET_OBJECTS:crypto_ecdsa_objects>
$<TARGET_OBJECTS:crypto_keccak_objects>
$<TARGET_OBJECTS:crypto_pedersen_commitment_objects>
$<TARGET_OBJECTS:crypto_pedersen_hash_objects>
$<TARGET_OBJECTS:crypto_schnorr_objects>
$<TARGET_OBJECTS:crypto_sha256_objects>
$<TARGET_OBJECTS:dsl_objects>
$<TARGET_OBJECTS:ecc_objects>
$<TARGET_OBJECTS:examples_objects>
$<TARGET_OBJECTS:honk_objects>
$<TARGET_OBJECTS:numeric_objects>
$<TARGET_OBJECTS:plonk_objects>
$<TARGET_OBJECTS:polynomials_objects>
$<TARGET_OBJECTS:proof_system_objects>
$<TARGET_OBJECTS:srs_objects>
$<TARGET_OBJECTS:stdlib_aes128_objects>
$<TARGET_OBJECTS:stdlib_blake2s_objects>
$<TARGET_OBJECTS:stdlib_blake3s_objects>
$<TARGET_OBJECTS:stdlib_keccak_objects>
$<TARGET_OBJECTS:stdlib_merkle_tree_objects>
$<TARGET_OBJECTS:stdlib_pedersen_commitment_objects>
$<TARGET_OBJECTS:stdlib_pedersen_hash_objects>
$<TARGET_OBJECTS:stdlib_primitives_objects>
$<TARGET_OBJECTS:stdlib_schnorr_objects>
$<TARGET_OBJECTS:stdlib_sha256_objects>
$<TARGET_OBJECTS:transcript_objects>
${BARRETENBERG_TARGET_OBJECTS}
$<TARGET_OBJECTS:wasi_objects>
)

Expand All @@ -194,12 +179,12 @@ if(WASM)
target_link_options(
barretenberg.wasm
PRIVATE
-nostartfiles -Wl,--no-entry,--export-dynamic,--allow-undefined
-nostartfiles -Wl,--no-entry,--export-dynamic
)

target_link_options(
acvm_backend.wasm
PRIVATE
-nostartfiles -Wl,--no-entry,--export-dynamic,--allow-undefined
-nostartfiles -Wl,--no-entry,--export-dynamic
)
endif()
2 changes: 2 additions & 0 deletions barretenberg/cpp/src/barretenberg/bb/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ if (NOT(FUZZING))
add_executable(
bb
main.cpp
get_bn254_crs.cpp
get_grumpkin_crs.cpp
)

target_link_libraries(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,33 +1,27 @@
#pragma once
#include "exec_pipe.hpp"
#include "file_io.hpp"
#include "log.hpp"
#include <barretenberg/ecc/curves/bn254/g1.hpp>
#include <barretenberg/srs/io.hpp>
#include <filesystem>
#include <fstream>
#include <ios>
#include "get_bn254_crs.hpp"

// Gets the transcript URL from the BARRETENBERG_TRANSCRIPT_URL environment variable, if set.
// Otherwise returns the default URL.
inline std::string getTranscriptURL()
namespace {
std::string get_bn254_transcript_url()
{
const char* ENV_VAR_NAME = "BARRETENBERG_TRANSCRIPT_URL";
const std::string DEFAULT_URL = "https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/monomial/transcript00.dat";

const char* env_url = std::getenv(ENV_VAR_NAME);

auto environment_variable_exists = (env_url && *env_url);
auto environment_variable_exists = ((env_url != nullptr) && *env_url);

return environment_variable_exists ? std::string(env_url) : DEFAULT_URL;
}
} // namespace

inline std::vector<uint8_t> download_g1_data(size_t num_points)
std::vector<uint8_t> download_bn254_g1_data(size_t num_points)
{
size_t g1_start = 28;
size_t g1_end = g1_start + num_points * 64 - 1;

std::string url = getTranscriptURL();
std::string url = get_bn254_transcript_url();

std::string command =
"curl -s -H \"Range: bytes=" + std::to_string(g1_start) + "-" + std::to_string(g1_end) + "\" '" + url + "'";
Expand All @@ -38,23 +32,23 @@ inline std::vector<uint8_t> download_g1_data(size_t num_points)
throw std::runtime_error("Failed to download g1 data.");
}

return exec_pipe(command);
return data;
}

inline std::vector<uint8_t> download_g2_data()
std::vector<uint8_t> download_bn254_g2_data()
{
size_t g2_start = 28 + 5040001 * 64;
size_t g2_end = g2_start + 128 - 1;

std::string url = getTranscriptURL();
std::string url = get_bn254_transcript_url();

std::string command =
"curl -s -H \"Range: bytes=" + std::to_string(g2_start) + "-" + std::to_string(g2_end) + "\" '" + url + "'";

return exec_pipe(command);
}

inline std::vector<barretenberg::g1::affine_element> get_g1_data(const std::filesystem::path& path, size_t num_points)
std::vector<barretenberg::g1::affine_element> get_bn254_g1_data(const std::filesystem::path& path, size_t num_points)
{
std::filesystem::create_directories(path);
std::ifstream size_file(path / "size");
Expand All @@ -74,7 +68,7 @@ inline std::vector<barretenberg::g1::affine_element> get_g1_data(const std::file
}

vinfo("downloading crs...");
auto data = download_g1_data(num_points);
auto data = download_bn254_g1_data(num_points);
write_file(path / "g1.dat", data);

std::ofstream new_size_file(path / "size");
Expand All @@ -90,7 +84,7 @@ inline std::vector<barretenberg::g1::affine_element> get_g1_data(const std::file
return points;
}

inline barretenberg::g2::affine_element get_g2_data(const std::filesystem::path& path)
barretenberg::g2::affine_element get_bn254_g2_data(const std::filesystem::path& path)
{
std::filesystem::create_directories(path);

Expand All @@ -100,7 +94,7 @@ inline barretenberg::g2::affine_element get_g2_data(const std::filesystem::path&
barretenberg::srs::IO<curve::BN254>::read_affine_elements_from_buffer(&g2_point, (char*)data.data(), 128);
return g2_point;
} catch (std::exception&) {
auto data = download_g2_data();
auto data = download_bn254_g2_data();
write_file(path / "g2.dat", data);
barretenberg::g2::affine_element g2_point;
barretenberg::srs::IO<curve::BN254>::read_affine_elements_from_buffer(&g2_point, (char*)data.data(), 128);
Expand Down
12 changes: 12 additions & 0 deletions barretenberg/cpp/src/barretenberg/bb/get_bn254_crs.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#pragma once
#include "exec_pipe.hpp"
#include "file_io.hpp"
#include "log.hpp"
#include <barretenberg/ecc/curves/bn254/g1.hpp>
#include <barretenberg/srs/io.hpp>
#include <filesystem>
#include <fstream>
#include <ios>

std::vector<barretenberg::g1::affine_element> get_bn254_g1_data(const std::filesystem::path& path, size_t num_points);
barretenberg::g2::affine_element get_bn254_g2_data(const std::filesystem::path& path);
72 changes: 72 additions & 0 deletions barretenberg/cpp/src/barretenberg/bb/get_grumpkin_crs.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
#include "get_grumpkin_crs.hpp"

// Gets the transcript URL from the BARRETENBERG_GRUMPKIN_TRANSCRIPT_URL environment variable, if set.
// Otherwise returns the default URL.
namespace {
std::string get_grumpkin_transcript_url()
{
const char* ENV_VAR_NAME = "BARRETENBERG_GRUMPKIN_TRANSCRIPT_URL";
const std::string DEFAULT_URL = "https://aztec-ignition.s3.amazonaws.com/TEST%20GRUMPKIN/monomial/transcript00.dat";

const char* env_url = std::getenv(ENV_VAR_NAME);

auto environment_variable_exists = ((env_url != nullptr) && *env_url);

return environment_variable_exists ? env_url : DEFAULT_URL;
}
} // namespace

std::vector<uint8_t> download_grumpkin_g1_data(size_t num_points)
{
size_t g1_start = 28;
size_t g1_end = g1_start + num_points * 64 - 1;

std::string url = get_grumpkin_transcript_url();

std::string command =
"curl -s -H \"Range: bytes=" + std::to_string(g1_start) + "-" + std::to_string(g1_end) + "\" '" + url + "'";

auto data = exec_pipe(command);
// Header + num_points * sizeof point.
if (data.size() < g1_end - g1_start) {
throw std::runtime_error("Failed to download grumpkin g1 data.");
}

return data;
}

std::vector<curve::Grumpkin::AffineElement> get_grumpkin_g1_data(const std::filesystem::path& path, size_t num_points)
{
std::filesystem::create_directories(path);
std::ifstream size_file(path / "grumpkin_size");
size_t size = 0;
if (size_file) {
size_file >> size;
size_file.close();
}
if (size >= num_points) {
vinfo("using cached crs at: ", path);
auto data = read_file(path / "grumpkin_g1.dat", 28 + num_points * 64);
auto points = std::vector<curve::Grumpkin::AffineElement>(num_points);
auto size_of_points_in_bytes = num_points * 64;
barretenberg::srs::IO<curve::Grumpkin>::read_affine_elements_from_buffer(
points.data(), (char*)data.data(), size_of_points_in_bytes);
return points;
}

vinfo("downloading grumpkin crs...");
auto data = download_grumpkin_g1_data(num_points);
write_file(path / "grumpkin_g1.dat", data);

std::ofstream new_size_file(path / "grumpkin_size");
if (!new_size_file) {
throw std::runtime_error("Failed to open size file for writing");
}
new_size_file << num_points;
new_size_file.close();

auto points = std::vector<curve::Grumpkin::AffineElement>(num_points);
barretenberg::srs::IO<curve::Grumpkin>::read_affine_elements_from_buffer(
points.data(), (char*)data.data(), data.size());
return points;
}
11 changes: 11 additions & 0 deletions barretenberg/cpp/src/barretenberg/bb/get_grumpkin_crs.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
#pragma once
#include "exec_pipe.hpp"
#include "file_io.hpp"
#include "log.hpp"
#include <barretenberg/ecc/curves/bn254/g1.hpp>
#include <barretenberg/srs/io.hpp>
#include <filesystem>
#include <fstream>
#include <ios>

std::vector<curve::Grumpkin::AffineElement> get_grumpkin_g1_data(const std::filesystem::path& path, size_t num_points);
Loading

0 comments on commit d093266

Please sign in to comment.