feat: more robust recursion input generator (#8634)

Introduces the method `write_recursion_inputs_honk` in main.cpp that,
given a program, produces the inputs (in the form of a Prover.toml) to a
second noir program that recursively verifies a proof of the first. This
is used to update/simplify the logic in
`regenerate_verify_honk_proof_inputs.sh` as a proof of concept. (Also
replaces `update_verify_honk_proof_inputs.py` with similar logic in
`ProofSurgeon` that deals more dynamically with the public inputs).

The reason for doing this is that similar logic will be needed to create
tests for `noir::verify_proof()` for Oink/PG which will have slightly
different forms. A flow of this kind will also be used to generate
inputs from typescript in the integration and e2e tests.

.github/workflows/ci.yml

@@ -90,6 +90,7 @@ jobs:
               - barretenberg/cpp/src/barretenberg/vm/**
               - barretenberg/cpp/src/barretenberg/**/generated/*
               - barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.{hpp,cpp}
+              - barretenberg/cpp/src/barretenberg/bb/main.cpp
             non-docs:
               - '!(docs/**)'
             non-misc-ci:

barretenberg/acir_tests/regenerate_verify_honk_proof_inputs.sh

@@ -6,9 +6,7 @@ set -eu
 BIN=${BIN:-../cpp/build/bin/bb}
 CRS_PATH=~/.bb-crs
 BRANCH=master
-VERBOSE=${VERBOSE:-}
-RECURSIVE=true
-PROOF_NAME="proof_a"
+VERBOSE=${VERBOSE:+-v}
 
 if [ -f $BIN ]; then
     BIN=$(realpath $BIN)
@@ -18,31 +16,22 @@ fi
 
 export BRANCH
 
-./reset_acir_tests.sh --rebuild-nargo --programs assert_statement_recursive
+# the program for which a proof will be recursively verified
+PROGRAM=assert_statement_recursive
+# the program containing the recursive verifier
+RECURSIVE_PROGRAM=verify_honk_proof
 
-cd acir_tests/assert_statement_recursive
+./reset_acir_tests.sh --rebuild-nargo --programs "$PROGRAM"
+cd "acir_tests/$PROGRAM"
 
-PROOF_DIR=$PWD/proofs
-PROOF_PATH=$PROOF_DIR/$PROOF_NAME
-VFLAG=${VERBOSE:+-v}
-RFLAG=${RECURSIVE:+-r}
-
-echo "Write VK to file for assert_statement..."
-$BIN write_vk_ultra_honk $VFLAG -c $CRS_PATH -o ./target/honk_vk
-
-echo "Write VK as fields for recursion..."
-$BIN vk_as_fields_ultra_honk $VFLAG -c $CRS_PATH -k ./target/honk_vk -o ./target/honk_vk_fields.json
-
-echo "Generate proof to file..."
-[ -d "$PROOF_DIR" ] || mkdir $PWD/proofs
-[ -e "$PROOF_PATH" ] || touch $PROOF_PATH
-$BIN prove_ultra_honk $VFLAG -c $CRS_PATH -b ./target/program.json -o "./proofs/honk_$PROOF_NAME"
+TOML_DIR=../../../../noir/noir-repo/test_programs/execution_success/"$RECURSIVE_PROGRAM"
+if [ ! -d "$TOML_DIR" ]; then
+    echo "Error: Directory $TOML_DIR does not exist."
+    exit 1
+fi
 
-echo "Write proof as fields for recursion..."
-$BIN proof_as_fields_honk $VFLAG -c $CRS_PATH -p "./proofs/honk_$PROOF_NAME" -o "./proofs/honk_${PROOF_NAME}_fields.json"
+echo "Generating recursion inputs and writing to directory $TOML_DIR"
+$BIN write_recursion_inputs_honk $VERBOSE -c $CRS_PATH -b ./target/program.json -o "$TOML_DIR"
 
-# cd back to barretenberg/acir_tests
 cd ../..
-python3 update_verify_honk_proof_inputs.py
-
-./reset_acir_tests.sh --programs verify_honk_proof
+./reset_acir_tests.sh --programs "$RECURSIVE_PROGRAM"

barretenberg/cpp/src/barretenberg/bb/main.cpp

@@ -2,6 +2,7 @@
 #include "barretenberg/client_ivc/client_ivc.hpp"
 #include "barretenberg/common/map.hpp"
 #include "barretenberg/common/serialize.hpp"
+#include "barretenberg/constants.hpp"
 #include "barretenberg/dsl/acir_format/acir_format.hpp"
 #include "barretenberg/dsl/acir_format/proof_surgeon.hpp"
 #include "barretenberg/dsl/acir_proofs/honk_contract.hpp"
@@ -133,6 +134,7 @@ std::string vk_to_json(std::vector<bb::fr> const& data)
     return format("[", join(map(rotated, [](auto fr) { return format("\"", fr, "\""); })), "]");
 }
 
+// WORKTODO: delete?
 std::string honk_vk_to_json(std::vector<bb::fr>& data)
 {
     return format("[", join(map(data, [](auto fr) { return format("\"", fr, "\""); })), "]");
@@ -340,26 +342,26 @@ void client_ivc_prove_output_all_msgpack(const std::string& bytecodePath,
     using TranslatorVK = TranslatorFlavor::VerificationKey;
     using DeciderVerificationKey = ClientIVC::DeciderVerificationKey;
 
+    using namespace acir_format;
+
     init_bn254_crs(1 << 24);
     init_grumpkin_crs(1 << 15);
 
-    auto gzippedBincodes = unpack_from_file<std::vector<std::string>>(bytecodePath);
-    auto witnessMaps = unpack_from_file<std::vector<std::string>>(witnessPath);
+    auto gzipped_bincodes = unpack_from_file<std::vector<std::string>>(bytecodePath);
+    auto witness_data = unpack_from_file<std::vector<std::string>>(witnessPath);
     std::vector<Program> folding_stack;
-    for (size_t i = 0; i < gzippedBincodes.size(); i++) {
+    for (auto [bincode, wit] : zip_view(gzipped_bincodes, witness_data)) {
         // TODO(#7371) there is a lot of copying going on in bincode, we should make sure this writes as a buffer in
         // the future
-        std::vector<uint8_t> buffer =
-            decompressedBuffer(reinterpret_cast<uint8_t*>(&gzippedBincodes[i][0]), gzippedBincodes[i].size()); // NOLINT
-
-        std::vector<acir_format::AcirFormat> constraint_systems =
-            acir_format::program_buf_to_acir_format(buffer,
-                                                    /*honk_recursion=*/false);
-        std::vector<uint8_t> witnessBuffer =
-            decompressedBuffer(reinterpret_cast<uint8_t*>(&witnessMaps[i][0]), witnessMaps[i].size()); // NOLINT
-        acir_format::WitnessVectorStack witness_stack = acir_format::witness_buf_to_witness_stack(witnessBuffer);
-        acir_format::AcirProgramStack program_stack{ constraint_systems, witness_stack };
-        folding_stack.push_back(program_stack.back());
+        std::vector<uint8_t> constraint_buf =
+            decompressedBuffer(reinterpret_cast<uint8_t*>(bincode.data()), bincode.size()); // NOLINT
+        std::vector<uint8_t> witness_buf =
+            decompressedBuffer(reinterpret_cast<uint8_t*>(wit.data()), wit.size()); // NOLINT
+
+        AcirFormat constraints = circuit_buf_to_acir_format(constraint_buf, /*honk_recursion=*/false);
+        WitnessVector witness = witness_buf_to_witness_data(witness_buf);
+
+        folding_stack.push_back(Program{ constraints, witness });
     }
     // TODO(#7371) dedupe this with the rest of the similar code
     // TODO(https://github.com/AztecProtocol/barretenberg/issues/1101): remove use of auto_verify_mode
@@ -370,8 +372,7 @@ void client_ivc_prove_output_all_msgpack(const std::string& bytecodePath,
     // Accumulate the entire program stack into the IVC
     for (Program& program : folding_stack) {
         // Construct a bberg circuit from the acir representation then accumulate it into the IVC
-        auto circuit =
-            acir_format::create_circuit<Builder>(program.constraints, 0, program.witness, false, ivc.goblin.op_queue);
+        auto circuit = create_circuit<Builder>(program.constraints, 0, program.witness, false, ivc.goblin.op_queue);
         ivc.accumulate(circuit);
     }
 
@@ -1075,8 +1076,7 @@ UltraProver_<Flavor> compute_valid_prover(const std::string& bytecodePath, const
     size_t srs_size = builder.get_circuit_subgroup_size(builder.get_total_circuit_size() + num_extra_gates);
     init_bn254_crs(srs_size);
 
-    Prover prover{ builder };
-    return prover;
+    return Prover{ builder };
 }
 
 /**
@@ -1155,13 +1155,11 @@ template <IsUltraFlavor Flavor> bool verify_honk(const std::string& proof_path,
 template <IsUltraFlavor Flavor> void write_vk_honk(const std::string& bytecodePath, const std::string& outputPath)
 {
     using Prover = UltraProver_<Flavor>;
-    using DeciderProvingKey = DeciderProvingKey_<Flavor>;
     using VerificationKey = Flavor::VerificationKey;
 
+    // Construct a verification key from a partial form of the proving key which only has precomputed entities
     Prover prover = compute_valid_prover<Flavor>(bytecodePath, "");
-    DeciderProvingKey& prover_inst = *prover.proving_key;
-    VerificationKey vk(
-        prover_inst.proving_key); // uses a partial form of the proving key which only has precomputed entities
+    VerificationKey vk(prover.proving_key->proving_key);
 
     auto serialized_vk = to_buffer(vk);
     if (outputPath == "-") {
@@ -1173,6 +1171,53 @@ template <IsUltraFlavor Flavor> void write_vk_honk(const std::string& bytecodePa
     }
 }
 
+/**
+ * @brief Write a toml file containing recursive verifier inputs for a given program + witness
+ *
+ * @tparam Flavor
+ * @param bytecodePath Path to the file containing the serialized circuit
+ * @param witnessPath Path to the file containing the serialized witness
+ * @param outputPath Path to write toml file
+ */
+template <IsUltraFlavor Flavor>
+void write_recursion_inputs_honk(const std::string& bytecodePath,
+                                 const std::string& witnessPath,
+                                 const std::string& outputPath)
+{
+    using Builder = Flavor::CircuitBuilder;
+    using Prover = UltraProver_<Flavor>;
+    using VerificationKey = Flavor::VerificationKey;
+    using FF = Flavor::FF;
+
+    bool honk_recursion = true;
+    auto constraints = get_constraint_system(bytecodePath, /*honk_recursion=*/true);
+    auto witness = get_witness(witnessPath);
+    auto builder = acir_format::create_circuit<Builder>(constraints, 0, witness, honk_recursion);
+
+    auto num_extra_gates = builder.get_num_gates_added_to_ensure_nonzero_polynomials();
+    size_t srs_size = builder.get_circuit_subgroup_size(builder.get_total_circuit_size() + num_extra_gates);
+    init_bn254_crs(srs_size);
+
+    // Construct Honk proof and verification key
+    Prover prover{ builder };
+    std::vector<FF> proof = prover.construct_proof();
+    VerificationKey verification_key(prover.proving_key->proving_key);
+
+    // Construct a string with the content of the toml file (vk hash, proof, public inputs, vk)
+    std::string toml_content = acir_format::ProofSurgeon::construct_recursion_inputs_toml_data(proof, verification_key);
+
+    // Write all components to the TOML file
+    std::string toml_path = outputPath + "/Prover.toml";
+    write_file(toml_path, { toml_content.begin(), toml_content.end() });
+
+    // Write to additional dir for noir-sync purposes
+    std::string part_to_remove = "/noir-repo/test_programs/execution_success";
+    size_t pos = toml_path.find(part_to_remove);
+    std::string toml_path_2 = toml_path; // define path here
+    toml_path_2.erase(pos, part_to_remove.length());
+    write_file(toml_path_2, { toml_content.begin(), toml_content.end() });
+}
+
 /**
  * @brief Outputs proof as vector of field elements in readable format.
  *
@@ -1470,6 +1515,9 @@ int main(int argc, char* argv[])
         } else if (command == "vk_as_fields") {
             std::string output_path = get_option(args, "-o", vk_path + "_fields.json");
             vk_as_fields(vk_path, output_path);
+        } else if (command == "write_recursion_inputs_honk") {
+            std::string output_path = get_option(args, "-o", "./target");
+            write_recursion_inputs_honk<UltraFlavor>(bytecode_path, witness_path, output_path);
 #ifndef DISABLE_AZTEC_VM
         } else if (command == "avm_prove") {
             std::filesystem::path avm_bytecode_path = get_option(args, "--avm-bytecode", "./target/avm_bytecode.bin");

barretenberg/cpp/src/barretenberg/dsl/acir_format/proof_surgeon.hpp

@@ -1,8 +1,10 @@
 #pragma once
+#include "barretenberg/common/map.hpp"
 #include "barretenberg/common/serialize.hpp"
 #include "barretenberg/ecc/curves/bn254/fr.hpp"
 #include "barretenberg/plonk_honk_shared/types/aggregation_object_type.hpp"
 #include "barretenberg/serialize/msgpack.hpp"
+#include <barretenberg/common/container.hpp>
 #include <cstdint>
 
 namespace acir_format {
@@ -11,7 +13,47 @@ namespace acir_format {
 static constexpr size_t HONK_RECURSION_PUBLIC_INPUT_OFFSET = 3;
 
 class ProofSurgeon {
+    using FF = bb::fr;
+
+    // construct a string of the form "[<fr_0 hex>, <fr_1 hex>, ...]"
+    static std::string to_json(const std::vector<bb::fr>& data)
+    {
+        return format("[", bb::join(map(data, [](auto fr) { return format("\"", fr, "\""); }), ", "), "]");
+    }
+
   public:
+    /**
+     * @brief Write a toml file containing the inputs to a noir verify_proof call
+     *
+     * @param proof A complete bberg style proof (i.e. contains the public inputs)
+     * @param verification_key
+     * @param toml_path
+     */
+    static std::string construct_recursion_inputs_toml_data(std::vector<FF>& proof, const auto& verification_key)
+    {
+        // Convert verification key to fields
+        std::vector<FF> vkey_fields = verification_key.to_field_elements();
+
+        // Get public inputs by cutting them out of the proof
+        const size_t num_public_inputs_to_extract = verification_key.num_public_inputs - bb::AGGREGATION_OBJECT_SIZE;
+        std::vector<FF> public_inputs =
+            acir_format::ProofSurgeon::cut_public_inputs_from_proof(proof, num_public_inputs_to_extract);
+
+        // Construct json-style output for each component
+        // FF key_hash{ 0 }; // not used for Honk
+        std::string proof_json = to_json(proof);
+        std::string pub_inputs_json = to_json(public_inputs);
+        std::string vk_json = to_json(vkey_fields);
+
+        // Format with labels for noir recursion input
+        std::string toml_content = "key_hash = " + format("\"", FF(0), "\"") + "\n";
+        toml_content += "proof = " + proof_json + "\n";
+        toml_content += "public_inputs = " + pub_inputs_json + "\n";
+        toml_content += "verification_key = " + vk_json + "\n";
+
+        return toml_content;
+    }
+
     /**
      * @brief Reconstruct a bberg style proof from a acir style proof + public inputs
      * @details Insert the public inputs in the middle the proof fields after 'inner_public_input_offset' because this

barretenberg/cpp/src/barretenberg/flavor/flavor.hpp

@@ -172,7 +172,7 @@ class VerificationKey_ : public PrecomputedCommitments {
      *
      * @return std::vector<FF>
      */
-    std::vector<FF> to_field_elements()
+    std::vector<FF> to_field_elements() const
     {
         using namespace bb::field_conversion;
 
@@ -189,7 +189,7 @@ class VerificationKey_ : public PrecomputedCommitments {
         serialize_to_field_buffer(this->contains_recursive_proof, elements);
         serialize_to_field_buffer(this->recursive_proof_public_input_indices, elements);
 
-        for (Commitment& commitment : this->get_all()) {
+        for (const Commitment& commitment : this->get_all()) {
             serialize_to_field_buffer(commitment, elements);
         }
 

barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/mega_flavor.hpp

@@ -562,7 +562,7 @@ class MegaFlavor {
         /**
          * @brief Serialize verification key to field elements
          */
-        std::vector<FF> to_field_elements()
+        std::vector<FF> to_field_elements() const
         {
             using namespace bb::field_conversion;
 
@@ -585,7 +585,7 @@ class MegaFlavor {
             serialize_to_field_buffer(this->databus_propagation_data.kernel_return_data_public_input_idx, elements);
             serialize_to_field_buffer(this->databus_propagation_data.is_kernel, elements);
 
-            for (Commitment& commitment : this->get_all()) {
+            for (const Commitment& commitment : this->get_all()) {
                 serialize_to_field_buffer(commitment, elements);
             }