feat: Goblin acir composer (AztecProtocol#4112)

A bit of background: At the end of 2023, we had a small subset of the
acir_tests passing for GUH+Goblin. Early work in 2024 got all of the
tests passing for GUH. (At this stage we were basically running all
acir_tests for UltraPlonk and GoblinUltraHonk). I turned the Goblin
component off because tests were failing in an unpredictable way. (I now
think this is a deterministic issue with the Translator that was showing
up more prominently due to running the Translator hundreds of times via
the acir_tests suite). This work maintains testing of all the noir
programs in the acir_test suite for GUH, but adds back in a full Goblin
workflow for a single arbitrary test. This is done through two
workflows: a "goblin accumulate" workflow (for constructing/verifying
GUH proofs) and a "full goblin" flow (for additionally constructing and
verifying ECCVM/Translator proofs). (Note: this latter flow will
eventually also involve merge logic but that's still not incorporated in
this work).

To make things cleaner, I've introduced a `GoblinAcirComposer` to handle
all Goblin/Honk logic, analogous to the `AcirComposer` for UltraPlonk.

Note: there is still no mechanism for creating goblin ops directly from
acir opcodes. (Eventually this will just happen via a blackbox call to
some recursive verification / folding operation for Honk/Goblin). The
OpQueue being processed by the ECCVM/Translator is just a random set of
ops "appended" to the circuit in question.

barretenberg/acir_tests/Dockerfile.bb

@@ -7,10 +7,13 @@ COPY --from=0 /usr/src/barretenberg/cpp/build /usr/src/barretenberg/cpp/build
 COPY --from=noir-acir-tests /usr/src/noir/test_programs /usr/src/noir/test_programs
 WORKDIR /usr/src/barretenberg/acir_tests
 COPY . .
-# Run every acir test through native bb build prove_then_verify flow.
+# Run every acir test through native bb build prove_then_verify flow for UltraPlonk.
 # This ensures we test independent pk construction through real/garbage witness data paths.
 RUN FLOW=prove_then_verify ./run_acir_tests.sh
-# TODO(https://github.com/AztecProtocol/barretenberg/issues/811) make this able to run the default test
-RUN FLOW=prove_and_verify_goblin ./run_acir_tests.sh
+# This flow is essentially the GoblinUltraHonk equivalent to the UltraPlonk "prove and verify". (This functionality is 
+# accessed via the goblin "accumulate" mechanism).
+RUN FLOW=accumulate_and_verify_goblin ./run_acir_tests.sh
+# This is a "full" Goblin flow. It constructs and verifies four proofs: GoblinUltraHonk, ECCVM, Translator, and merge
+RUN FLOW=prove_and_verify_goblin ./run_acir_tests.sh 6_array
 # Run 1_mul through native bb build, all_cmds flow, to test all cli args.
 RUN VERBOSE=1 FLOW=all_cmds ./run_acir_tests.sh 1_mul

barretenberg/acir_tests/Dockerfile.bb.js

@@ -14,7 +14,10 @@ COPY . .
 ENV VERBOSE=1
 # Run double_verify_proof through bb.js on node to check 512k support.
 RUN BIN=../ts/dest/node/main.js FLOW=prove_then_verify ./run_acir_tests.sh double_verify_proof
-RUN BIN=../ts/dest/node/main.js FLOW=prove_and_verify_goblin ./run_acir_tests.sh double_verify_proof
+# Run a single arbitrary test not involving recursion through bb.js for GoblinUltraHonk
+RUN BIN=../ts/dest/node/main.js FLOW=accumulate_and_verify_goblin ./run_acir_tests.sh 6_array
+# Run a single arbitrary test not involving recursion through bb.js for full Goblin
+RUN BIN=../ts/dest/node/main.js FLOW=prove_and_verify_goblin ./run_acir_tests.sh 6_array
 # Run 1_mul through bb.js build, all_cmds flow, to test all cli args.
 RUN BIN=../ts/dest/node/main.js FLOW=all_cmds ./run_acir_tests.sh 1_mul
 # Run double_verify_proof through bb.js on chrome testing multi-threaded browser support.

barretenberg/acir_tests/flows/accumulate_and_verify_goblin.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -eu
+
+VFLAG=${VERBOSE:+-v}
+
+$BIN accumulate_and_verify_goblin $VFLAG -c $CRS_PATH -b ./target/acir.gz

barretenberg/acir_tests/run_acir_tests.sh

@@ -29,6 +29,7 @@ fi
 
 export BIN CRS_PATH VERBOSE BRANCH
 
+# copy the gzipped acir test data from noir/test_programs to barretenberg/acir_tests
 ./clone_test_vectors.sh
 
 cd acir_tests

barretenberg/cpp/src/barretenberg/bb/main.cpp

@@ -10,6 +10,7 @@
 #include <barretenberg/common/timer.hpp>
 #include <barretenberg/dsl/acir_format/acir_to_constraint_buf.hpp>
 #include <barretenberg/dsl/acir_proofs/acir_composer.hpp>
+#include <barretenberg/dsl/acir_proofs/goblin_acir_composer.hpp>
 #include <barretenberg/srs/global_crs.hpp>
 #include <iostream>
 #include <stdexcept>
@@ -43,6 +44,12 @@ void init_bn254_crs(size_t dyadic_circuit_size)
     srs::init_crs_factory(bn254_g1_data, bn254_g2_data);
 }
 
+/**
+ * @brief Initialize the global crs_factory for grumpkin based on a known dyadic circuit size
+ * @details Grumpkin crs is required only for the ECCVM
+ *
+ * @param dyadic_circuit_size power-of-2 circuit size
+ */
 void init_grumpkin_crs(size_t eccvm_dyadic_circuit_size)
 {
     auto grumpkin_g1_data = get_grumpkin_g1_data(CRS_PATH, eccvm_dyadic_circuit_size);
@@ -115,6 +122,43 @@ bool proveAndVerify(const std::string& bytecodePath, const std::string& witnessP
     return verified;
 }
 
+/**
+ * @brief Constructs and verifies a Honk proof for an ACIR circuit via the Goblin accumulate mechanism
+ *
+ * Communication:
+ * - proc_exit: A boolean value is returned indicating whether the proof is valid.
+ *   an exit code of 0 will be returned for success and 1 for failure.
+ *
+ * @param bytecodePath Path to the file containing the serialized acir constraint system
+ * @param witnessPath Path to the file containing the serialized witness
+ * @return verified
+ */
+bool accumulateAndVerifyGoblin(const std::string& bytecodePath, const std::string& witnessPath)
+{
+    // Populate the acir constraint system and witness from gzipped data
+    auto constraint_system = get_constraint_system(bytecodePath);
+    auto witness = get_witness(witnessPath);
+
+    // Instantiate a Goblin acir composer and construct a bberg circuit from the acir representation
+    acir_proofs::GoblinAcirComposer acir_composer;
+    acir_composer.create_circuit(constraint_system, witness);
+
+    // TODO(https://github.com/AztecProtocol/barretenberg/issues/811): Don't hardcode dyadic circuit size. Currently set
+    // to max circuit size present in acir tests suite.
+    size_t hardcoded_bn254_dyadic_size_hack = 1 << 18;
+    init_bn254_crs(hardcoded_bn254_dyadic_size_hack);
+    size_t hardcoded_grumpkin_dyadic_size_hack = 1 << 10; // For eccvm only
+    init_grumpkin_crs(hardcoded_grumpkin_dyadic_size_hack);
+
+    // Call accumulate to generate a GoblinUltraHonk proof
+    auto proof = acir_composer.accumulate();
+
+    // Verify the GoblinUltraHonk proof
+    auto verified = acir_composer.verify_accumulator(proof);
+
+    return verified;
+}
+
 /**
  * @brief Proves and Verifies an ACIR circuit
  *
@@ -132,11 +176,13 @@ bool proveAndVerifyGoblin(const std::string& bytecodePath,
                           const std::string& witnessPath,
                           [[maybe_unused]] bool recursive)
 {
+    // Populate the acir constraint system and witness from gzipped data
     auto constraint_system = get_constraint_system(bytecodePath);
     auto witness = get_witness(witnessPath);
 
-    acir_proofs::AcirComposer acir_composer;
-    acir_composer.create_goblin_circuit(constraint_system, witness);
+    // Instantiate a Goblin acir composer and construct a bberg circuit from the acir representation
+    acir_proofs::GoblinAcirComposer acir_composer;
+    acir_composer.create_circuit(constraint_system, witness);
 
     // TODO(https://github.com/AztecProtocol/barretenberg/issues/811): Don't hardcode dyadic circuit size. Currently set
     // to max circuit size present in acir tests suite.
@@ -145,9 +191,11 @@ bool proveAndVerifyGoblin(const std::string& bytecodePath,
     size_t hardcoded_grumpkin_dyadic_size_hack = 1 << 10; // For eccvm only
     init_grumpkin_crs(hardcoded_grumpkin_dyadic_size_hack);
 
-    auto proof = acir_composer.create_goblin_proof();
+    // Generate a GoblinUltraHonk proof and a full Goblin proof
+    auto proof = acir_composer.accumulate_and_prove();
 
-    auto verified = acir_composer.verify_goblin_proof(proof);
+    // Verify the GoblinUltraHonk proof and the full Goblin proof
+    auto verified = acir_composer.verify(proof);
 
     return verified;
 }
@@ -458,6 +506,9 @@ int main(int argc, char* argv[])
         if (command == "prove_and_verify") {
             return proveAndVerify(bytecode_path, witness_path, recursive) ? 0 : 1;
         }
+        if (command == "accumulate_and_verify_goblin") {
+            return accumulateAndVerifyGoblin(bytecode_path, witness_path) ? 0 : 1;
+        }
         if (command == "prove_and_verify_goblin") {
             return proveAndVerifyGoblin(bytecode_path, witness_path, recursive) ? 0 : 1;
         }

barretenberg/cpp/src/barretenberg/dsl/acir_proofs/acir_composer.cpp

@@ -3,7 +3,6 @@
 #include "barretenberg/common/throw_or_abort.hpp"
 #include "barretenberg/dsl/acir_format/acir_format.hpp"
 #include "barretenberg/dsl/types.hpp"
-#include "barretenberg/goblin/mock_circuits.hpp"
 #include "barretenberg/plonk/proof_system/proving_key/serialize.hpp"
 #include "barretenberg/plonk/proof_system/verification_key/sol_gen.hpp"
 #include "barretenberg/plonk/proof_system/verification_key/verification_key.hpp"
@@ -61,27 +60,6 @@ std::vector<uint8_t> AcirComposer::create_proof(bool is_recursive)
     return proof;
 }
 
-void AcirComposer::create_goblin_circuit(acir_format::acir_format& constraint_system,
-                                         acir_format::WitnessVector& witness)
-{
-    // Construct a builder using the witness and public input data from acir
-    goblin_builder_ = acir_format::GoblinBuilder{
-        goblin.op_queue, witness, constraint_system.public_inputs, constraint_system.varnum
-    };
-
-    // Populate constraints in the builder via the data in constraint_system
-    acir_format::build_constraints(goblin_builder_, constraint_system, true);
-
-    // TODO(https://github.com/AztecProtocol/barretenberg/issues/817): Add some arbitrary op gates to ensure the
-    // associated polynomials are non-zero and to give ECCVM and Translator some ECC ops to process.
-    GoblinMockCircuits::construct_goblin_ecc_op_circuit(goblin_builder_);
-}
-
-std::vector<uint8_t> AcirComposer::create_goblin_proof()
-{
-    return goblin.construct_proof(goblin_builder_);
-}
-
 std::shared_ptr<bb::plonk::verification_key> AcirComposer::init_verification_key()
 {
     if (!proving_key_) {
@@ -132,11 +110,6 @@ bool AcirComposer::verify_proof(std::vector<uint8_t> const& proof, bool is_recur
     }
 }
 
-bool AcirComposer::verify_goblin_proof(std::vector<uint8_t> const& proof)
-{
-    return goblin.verify_proof({ proof });
-}
-
 std::string AcirComposer::get_solidity_verifier()
 {
     std::ostringstream stream;

barretenberg/cpp/src/barretenberg/dsl/acir_proofs/acir_composer.hpp

@@ -1,6 +1,5 @@
 #pragma once
 #include <barretenberg/dsl/acir_format/acir_format.hpp>
-#include <barretenberg/goblin/goblin.hpp>
 
 namespace acir_proofs {
 
@@ -38,15 +37,8 @@ class AcirComposer {
 
     std::vector<bb::fr> serialize_verification_key_into_fields();
 
-    // Goblin specific methods
-    void create_goblin_circuit(acir_format::acir_format& constraint_system, acir_format::WitnessVector& witness);
-    std::vector<uint8_t> create_goblin_proof();
-    bool verify_goblin_proof(std::vector<uint8_t> const& proof);
-
   private:
     acir_format::Builder builder_;
-    acir_format::GoblinBuilder goblin_builder_;
-    Goblin goblin;
     size_t size_hint_;
     std::shared_ptr<bb::plonk::proving_key> proving_key_;
     std::shared_ptr<bb::plonk::verification_key> verification_key_;

barretenberg/cpp/src/barretenberg/dsl/acir_proofs/c_bind.cpp

@@ -6,6 +6,7 @@
 #include "barretenberg/common/serialize.hpp"
 #include "barretenberg/common/slab_allocator.hpp"
 #include "barretenberg/dsl/acir_format/acir_format.hpp"
+#include "barretenberg/dsl/acir_proofs/goblin_acir_composer.hpp"
 #include "barretenberg/plonk/proof_system/proving_key/serialize.hpp"
 #include "barretenberg/plonk/proof_system/verification_key/verification_key.hpp"
 #include "barretenberg/srs/global_crs.hpp"
@@ -26,6 +27,11 @@ WASM_EXPORT void acir_new_acir_composer(uint32_t const* size_hint, out_ptr out)
     *out = new acir_proofs::AcirComposer(ntohl(*size_hint));
 }
 
+WASM_EXPORT void acir_new_goblin_acir_composer(out_ptr out)
+{
+    *out = new acir_proofs::GoblinAcirComposer();
+}
+
 WASM_EXPORT void acir_delete_acir_composer(in_ptr acir_composer_ptr)
 {
     delete reinterpret_cast<acir_proofs::AcirComposer*>(*acir_composer_ptr);
@@ -57,17 +63,31 @@ WASM_EXPORT void acir_create_proof(in_ptr acir_composer_ptr,
     *out = to_heap_buffer(proof_data);
 }
 
-WASM_EXPORT void acir_create_goblin_proof(in_ptr acir_composer_ptr,
-                                          uint8_t const* acir_vec,
-                                          uint8_t const* witness_vec,
-                                          uint8_t** out)
+WASM_EXPORT void acir_goblin_accumulate(in_ptr acir_composer_ptr,
+                                        uint8_t const* acir_vec,
+                                        uint8_t const* witness_vec,
+                                        uint8_t** out)
 {
-    auto acir_composer = reinterpret_cast<acir_proofs::AcirComposer*>(*acir_composer_ptr);
+    auto acir_composer = reinterpret_cast<acir_proofs::GoblinAcirComposer*>(*acir_composer_ptr);
     auto constraint_system = acir_format::circuit_buf_to_acir_format(from_buffer<std::vector<uint8_t>>(acir_vec));
     auto witness = acir_format::witness_buf_to_witness_data(from_buffer<std::vector<uint8_t>>(witness_vec));
 
-    acir_composer->create_goblin_circuit(constraint_system, witness);
-    auto proof_data = acir_composer->create_goblin_proof();
+    acir_composer->create_circuit(constraint_system, witness);
+    auto proof_data = acir_composer->accumulate();
+    *out = to_heap_buffer(proof_data);
+}
+
+WASM_EXPORT void acir_goblin_prove(in_ptr acir_composer_ptr,
+                                   uint8_t const* acir_vec,
+                                   uint8_t const* witness_vec,
+                                   uint8_t** out)
+{
+    auto acir_composer = reinterpret_cast<acir_proofs::GoblinAcirComposer*>(*acir_composer_ptr);
+    auto constraint_system = acir_format::circuit_buf_to_acir_format(from_buffer<std::vector<uint8_t>>(acir_vec));
+    auto witness = acir_format::witness_buf_to_witness_data(from_buffer<std::vector<uint8_t>>(witness_vec));
+
+    acir_composer->create_circuit(constraint_system, witness);
+    auto proof_data = acir_composer->accumulate_and_prove();
     *out = to_heap_buffer(proof_data);
 }
 
@@ -102,11 +122,18 @@ WASM_EXPORT void acir_get_proving_key(in_ptr acir_composer_ptr, uint8_t const* a
     *out = to_heap_buffer(to_buffer(*pk));
 }
 
-WASM_EXPORT void acir_verify_goblin_proof(in_ptr acir_composer_ptr, uint8_t const* proof_buf, bool* result)
+WASM_EXPORT void acir_goblin_verify_accumulator(in_ptr acir_composer_ptr, uint8_t const* proof_buf, bool* result)
 {
-    auto acir_composer = reinterpret_cast<acir_proofs::AcirComposer*>(*acir_composer_ptr);
+    auto acir_composer = reinterpret_cast<acir_proofs::GoblinAcirComposer*>(*acir_composer_ptr);
+    auto proof = from_buffer<std::vector<uint8_t>>(proof_buf);
+    *result = acir_composer->verify_accumulator(proof);
+}
+
+WASM_EXPORT void acir_goblin_verify(in_ptr acir_composer_ptr, uint8_t const* proof_buf, bool* result)
+{
+    auto acir_composer = reinterpret_cast<acir_proofs::GoblinAcirComposer*>(*acir_composer_ptr);
     auto proof = from_buffer<std::vector<uint8_t>>(proof_buf);
-    *result = acir_composer->verify_goblin_proof(proof);
+    *result = acir_composer->verify(proof);
 }
 
 WASM_EXPORT void acir_verify_proof(in_ptr acir_composer_ptr,

barretenberg/cpp/src/barretenberg/dsl/acir_proofs/c_bind.hpp

@@ -13,6 +13,8 @@ WASM_EXPORT void acir_get_circuit_sizes(uint8_t const* constraint_system_buf,
 
 WASM_EXPORT void acir_new_acir_composer(uint32_t const* size_hint, out_ptr out);
 
+WASM_EXPORT void acir_new_goblin_acir_composer(out_ptr out);
+
 WASM_EXPORT void acir_delete_acir_composer(in_ptr acir_composer_ptr);
 
 WASM_EXPORT void acir_create_circuit(in_ptr acir_composer_ptr,
@@ -32,10 +34,25 @@ WASM_EXPORT void acir_create_proof(in_ptr acir_composer_ptr,
                                    bool const* is_recursive,
                                    uint8_t** out);
 
-WASM_EXPORT void acir_create_goblin_proof(in_ptr acir_composer_ptr,
-                                          uint8_t const* constraint_system_buf,
-                                          uint8_t const* witness_buf,
-                                          uint8_t** out);
+/**
+ * @brief Perform the goblin accumulate operation
+ * @details Constructs a GUH proof and possibly handles transcript merge logic
+ *
+ */
+WASM_EXPORT void acir_goblin_accumulate(in_ptr acir_composer_ptr,
+                                        uint8_t const* constraint_system_buf,
+                                        uint8_t const* witness_buf,
+                                        uint8_t** out);
+
+/**
+ * @brief Construct a full goblin proof
+ * @details Makes a call to accumulate to a final circuit before constructing a Goblin proof
+ *
+ */
+WASM_EXPORT void acir_goblin_prove(in_ptr acir_composer_ptr,
+                                   uint8_t const* constraint_system_buf,
+                                   uint8_t const* witness_buf,
+                                   uint8_t** out);
 
 WASM_EXPORT void acir_load_verification_key(in_ptr acir_composer_ptr, uint8_t const* vk_buf);
 
@@ -50,7 +67,17 @@ WASM_EXPORT void acir_verify_proof(in_ptr acir_composer_ptr,
                                    bool const* is_recursive,
                                    bool* result);
 
-WASM_EXPORT void acir_verify_goblin_proof(in_ptr acir_composer_ptr, uint8_t const* proof_buf, bool* result);
+/**
+ * @brief Verifies a GUH proof produced during goblin accumulation
+ *
+ */
+WASM_EXPORT void acir_goblin_verify_accumulator(in_ptr acir_composer_ptr, uint8_t const* proof_buf, bool* result);
+
+/**
+ * @brief Verifies a full goblin proof (and the GUH proof produced by accumulation)
+ *
+ */
+WASM_EXPORT void acir_goblin_verify(in_ptr acir_composer_ptr, uint8_t const* proof_buf, bool* result);
 
 WASM_EXPORT void acir_get_solidity_verifier(in_ptr acir_composer_ptr, out_str_buf out);