feat: vk_as_fields, proof_as_fields flows for honk (#6406)

Closes AztecProtocol/barretenberg#982.

Adds the vk_as_fields and proof_as_fields flows for Honk that are used
to generate the Vkey and proof as inputs for a recursive circuit (like
double_verify_proof). I revamped the script, gen_inner_proof_inputs.sh,
that call these flows.

I add new serialization/deserialization for ultra/goblin_ultra flavor
verification keys which are used in the vk_as_fields flow.

Notes: I removed the key_hash output that was part of Plonk's
vk_as_fields output since we don't need it.

.github/spot-runner-action/dist/index.js

@@ -754,8 +754,10 @@ function requestAndWaitForSpot(config) {
                 // wait 10 seconds
                 yield new Promise((r) => setTimeout(r, 5000 * Math.pow(2, backoff)));
                 backoff += 1;
-                core.info("Polling to see if we somehow have an instance up");
-                instanceId = yield ((_a = ec2Client.getInstancesForTags("running")[0]) === null || _a === void 0 ? void 0 : _a.instanceId);
+                if (config.githubActionRunnerConcurrency > 0) {
+                    core.info("Polling to see if we somehow have an instance up");
+                    instanceId = yield ((_a = ec2Client.getInstancesForTags("running")[0]) === null || _a === void 0 ? void 0 : _a.instanceId);
+                }
             }
             if (instanceId) {
                 core.info("Successfully requested/found instance with ID " + instanceId);

.github/spot-runner-action/src/main.ts

@@ -88,8 +88,10 @@ async function requestAndWaitForSpot(config: ActionConfig): Promise<string> {
       // wait 10 seconds
       await new Promise((r) => setTimeout(r, 5000 * 2 ** backoff));
       backoff += 1;
-      core.info("Polling to see if we somehow have an instance up");
-      instanceId = await ec2Client.getInstancesForTags("running")[0]?.instanceId;
+      if (config.githubActionRunnerConcurrency > 0) {
+        core.info("Polling to see if we somehow have an instance up");
+        instanceId = await ec2Client.getInstancesForTags("running")[0]?.instanceId;
+      }
     }
     if (instanceId) {
       core.info("Successfully requested/found instance with ID " + instanceId);

barretenberg/acir_tests/gen_inner_proof_inputs.sh

@@ -28,18 +28,18 @@ VFLAG=${VERBOSE:+-v}
 RFLAG=${RECURSIVE:+-r}
 
 echo "Write VK to file for assert_statement..."
-$BIN write_vk $VFLAG -c $CRS_PATH -o
+$BIN write_vk $VFLAG -c $CRS_PATH -o ./target/vk
 
 echo "Write VK as fields for recursion..."
-$BIN vk_as_fields $VFLAG -c $CRS_PATH
+$BIN vk_as_fields $VFLAG -c $CRS_PATH -k ./target/vk -o ./target/vk_fields.json
 
 echo "Generate proof to file..."
 [ -d "$PROOF_DIR" ] || mkdir $PWD/proofs
 [ -e "$PROOF_PATH" ] || touch $PROOF_PATH
-$BIN prove $VFLAG -c $CRS_PATH -b ./target/program.json -o "./proofs/$PROOF_NAME" $RFLAG
+$BIN prove $VFLAG -c $CRS_PATH -b ./target/program.json -o "./proofs/$PROOF_NAME"
 
 echo "Write proof as fields for recursion..."
-$BIN proof_as_fields $VFLAG -c $CRS_PATH -p "./proofs/$PROOF_NAME"
+$BIN proof_as_fields $VFLAG -c $CRS_PATH -p "./proofs/$PROOF_NAME" -k ./target/vk -o "./proofs/${PROOF_NAME}_fields.json"
 
 cat ./proofs/${PROOF_NAME}_fields.json
 echo

barretenberg/acir_tests/gen_inner_proof_inputs_ultra_honk.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# Env var overrides:
+#   BIN: to specify a different binary to test with (e.g. bb.js or bb.js-dev).
+set -eu
+
+BIN=${BIN:-../cpp/build/bin/bb}
+CRS_PATH=~/.bb-crs
+BRANCH=master
+VERBOSE=${VERBOSE:-}
+RECURSIVE=true
+PROOF_NAME="proof_a"
+
+if [ -f $BIN ]; then
+    BIN=$(realpath $BIN)
+else
+    BIN=$(realpath $(which $BIN))
+fi
+
+export BRANCH
+
+./clone_test_vectors.sh
+
+cd acir_tests/assert_statement_recursive
+
+PROOF_DIR=$PWD/proofs
+PROOF_PATH=$PROOF_DIR/$PROOF_NAME
+VFLAG=${VERBOSE:+-v}
+RFLAG=${RECURSIVE:+-r}
+
+echo "Write VK to file for assert_statement..."
+$BIN write_vk_ultra_honk $VFLAG -c $CRS_PATH -o ./target/vk
+
+echo "Write VK as fields for recursion..."
+$BIN vk_as_fields_ultra_honk $VFLAG -c $CRS_PATH -k ./target/vk -o ./target/vk_fields.json
+
+echo "Generate proof to file..."
+[ -d "$PROOF_DIR" ] || mkdir $PWD/proofs
+[ -e "$PROOF_PATH" ] || touch $PROOF_PATH
+$BIN prove_ultra_honk $VFLAG -c $CRS_PATH -b ./target/program.json -o "./proofs/$PROOF_NAME"
+
+echo "Write proof as fields for recursion..."
+$BIN proof_as_fields_honk $VFLAG -c $CRS_PATH -p "./proofs/$PROOF_NAME" -o "./proofs/${PROOF_NAME}_fields.json"
+
+cat ./proofs/${PROOF_NAME}_fields.json
+echo

barretenberg/cpp/src/barretenberg/bb/main.cpp

@@ -678,6 +678,62 @@ template <IsUltraFlavor Flavor> void write_vk_honk(const std::string& bytecodePa
         vinfo("vk written to: ", outputPath);
     }
 }
+
+/**
+ * @brief Outputs proof as vector of field elements in readable format.
+ *
+ * Communication:
+ * - stdout: The proof as a list of field elements is written to stdout as a string
+ * - Filesystem: The proof as a list of field elements is written to the path specified by outputPath
+ *
+ *
+ * @param proof_path Path to the file containing the serialized proof
+ * @param output_path Path to write the proof to
+ */
+void proof_as_fields_honk(const std::string& proof_path, const std::string& output_path)
+{
+    auto proof = from_buffer<std::vector<bb::fr>>(read_file(proof_path));
+    auto json = proof_to_json(proof);
+
+    if (output_path == "-") {
+        writeStringToStdout(json);
+        vinfo("proof as fields written to stdout");
+    } else {
+        write_file(output_path, { json.begin(), json.end() });
+        vinfo("proof as fields written to: ", output_path);
+    }
+}
+
+/**
+ * @brief Converts a verification key from a byte array into a list of field elements
+ *
+ * Why is this needed?
+ * This follows the same rationale as `proofAsFields`.
+ *
+ * Communication:
+ * - stdout: The verification key as a list of field elements is written to stdout as a string
+ * - Filesystem: The verification key as a list of field elements is written to the path specified by outputPath
+ *
+ * @param vk_path Path to the file containing the serialized verification key
+ * @param output_path Path to write the verification key to
+ */
+template <IsUltraFlavor Flavor> void vk_as_fields_honk(const std::string& vk_path, const std::string& output_path)
+{
+    using VerificationKey = Flavor::VerificationKey;
+
+    auto verification_key = std::make_shared<VerificationKey>(from_buffer<VerificationKey>(read_file(vk_path)));
+    std::vector<bb::fr> data = verification_key->to_field_elements();
+
+    auto json = vk_to_json(data);
+    if (output_path == "-") {
+        writeStringToStdout(json);
+        vinfo("vk as fields written to stdout");
+    } else {
+        write_file(output_path, { json.begin(), json.end() });
+        vinfo("vk as fields written to: ", output_path);
+    }
+}
+
 /**
  * @brief Creates a proof for an ACIR circuit, outputs the proof and verification key in binary and 'field' format
  *
@@ -831,6 +887,15 @@ int main(int argc, char* argv[])
         } else if (command == "write_vk_goblin_ultra_honk") {
             std::string output_path = get_option(args, "-o", "./target/vk");
             write_vk_honk<GoblinUltraFlavor>(bytecode_path, output_path);
+        } else if (command == "proof_as_fields_honk") {
+            std::string output_path = get_option(args, "-o", proof_path + "_fields.json");
+            proof_as_fields_honk(proof_path, output_path);
+        } else if (command == "vk_as_fields_ultra_honk") {
+            std::string output_path = get_option(args, "-o", vk_path + "_fields.json");
+            vk_as_fields_honk<UltraFlavor>(vk_path, output_path);
+        } else if (command == "vk_as_fields_goblin_ultra_honk") {
+            std::string output_path = get_option(args, "-o", vk_path + "_fields.json");
+            vk_as_fields_honk<GoblinUltraFlavor>(vk_path, output_path);
         } else {
             std::cerr << "Unknown command: " << command << "\n";
             return 1;

barretenberg/cpp/src/barretenberg/dsl/acir_proofs/c_bind.cpp

@@ -230,4 +230,19 @@ WASM_EXPORT void acir_write_vk_ultra_honk(uint8_t const* acir_vec, uint8_t** out
     ProverInstance prover_inst(builder);
     VerificationKey vk(prover_inst.proving_key);
     *out = to_heap_buffer(to_buffer(vk));
+}
+
+WASM_EXPORT void acir_proof_as_fields_ultra_honk(uint8_t const* proof_buf, fr::vec_out_buf out)
+{
+    auto proof = from_buffer<std::vector<bb::fr>>(from_buffer<std::vector<uint8_t>>(proof_buf));
+    *out = to_heap_buffer(proof);
+}
+
+WASM_EXPORT void acir_vk_as_fields_ultra_honk(uint8_t const* vk_buf, fr::vec_out_buf out_vkey)
+{
+    using VerificationKey = UltraFlavor::VerificationKey;
+
+    auto verification_key = std::make_shared<VerificationKey>(from_buffer<VerificationKey>(vk_buf));
+    std::vector<bb::fr> vkey_as_fields = verification_key->to_field_elements();
+    *out_vkey = to_heap_buffer(vkey_as_fields);
 }

barretenberg/cpp/src/barretenberg/ecc/fields/field_conversion.hpp

@@ -18,7 +18,7 @@ namespace bb::field_conversion {
  */
 template <typename T> constexpr size_t calc_num_bn254_frs()
 {
-    if constexpr (IsAnyOf<T, uint32_t, bool>) {
+    if constexpr (IsAnyOf<T, uint32_t, uint64_t, bool>) {
         return 1;
     } else if constexpr (IsAnyOf<T, bb::fr, grumpkin::fr>) {
         return T::Params::NUM_BN254_SCALARS;
@@ -48,7 +48,7 @@ template <typename T> T convert_from_bn254_frs(std::span<const bb::fr> fr_vec)
     if constexpr (IsAnyOf<T, bool>) {
         ASSERT(fr_vec.size() == 1);
         return bool(fr_vec[0]);
-    } else if constexpr (IsAnyOf<T, uint32_t, bb::fr>) {
+    } else if constexpr (IsAnyOf<T, uint32_t, uint64_t, bb::fr>) {
         ASSERT(fr_vec.size() == 1);
         return static_cast<T>(fr_vec[0]);
     } else if constexpr (IsAnyOf<T, grumpkin::fr>) {
@@ -89,7 +89,7 @@ std::vector<bb::fr> convert_grumpkin_fr_to_bn254_frs(const grumpkin::fr& val);
  */
 template <typename T> std::vector<bb::fr> convert_to_bn254_frs(const T& val)
 {
-    if constexpr (IsAnyOf<T, bool, uint32_t, bb::fr>) {
+    if constexpr (IsAnyOf<T, bool, uint32_t, uint64_t, bb::fr>) {
         std::vector<bb::fr> fr_vec{ val };
         return fr_vec;
     } else if constexpr (IsAnyOf<T, grumpkin::fr>) {

barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/goblin_ultra_flavor.hpp

@@ -575,6 +575,31 @@ class GoblinUltraFlavor {
                        lagrange_last,
                        lagrange_ecc_op,
                        databus_id);
+
+        /**
+         * @brief Serialize verification key to field elements
+         *
+         * @return std::vector<FF>
+         */
+        std::vector<FF> to_field_elements()
+        {
+            std::vector<FF> elements;
+            std::vector<FF> circuit_size_elements = bb::field_conversion::convert_to_bn254_frs(this->circuit_size);
+            elements.insert(elements.end(), circuit_size_elements.begin(), circuit_size_elements.end());
+            // do the same for the rest of the fields
+            std::vector<FF> num_public_inputs_elements =
+                bb::field_conversion::convert_to_bn254_frs(this->num_public_inputs);
+            elements.insert(elements.end(), num_public_inputs_elements.begin(), num_public_inputs_elements.end());
+            std::vector<FF> pub_inputs_offset_elements =
+                bb::field_conversion::convert_to_bn254_frs(this->pub_inputs_offset);
+            elements.insert(elements.end(), pub_inputs_offset_elements.begin(), pub_inputs_offset_elements.end());
+
+            for (Commitment& comm : this->get_all()) {
+                std::vector<FF> comm_elements = bb::field_conversion::convert_to_bn254_frs(comm);
+                elements.insert(elements.end(), comm_elements.begin(), comm_elements.end());
+            }
+            return elements;
+        }
     };
     /**
      * @brief A container for storing the partially evaluated multivariates produced by sumcheck.

barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/goblin_ultra_recursive_flavor.hpp

@@ -102,6 +102,7 @@ template <typename BuilderType> class GoblinUltraRecursiveFlavor_ {
       public:
         VerificationKey(const size_t circuit_size, const size_t num_public_inputs)
         {
+            // TODO(https://github.com/AztecProtocol/barretenberg/issues/983): Think about if these should be witnesses
             this->circuit_size = circuit_size;
             this->log_circuit_size = numeric::get_msb(circuit_size);
             this->num_public_inputs = num_public_inputs;
@@ -151,6 +152,40 @@ template <typename BuilderType> class GoblinUltraRecursiveFlavor_ {
             this->lagrange_ecc_op = Commitment::from_witness(builder, native_key->lagrange_ecc_op);
             this->databus_id = Commitment::from_witness(builder, native_key->databus_id);
         };
+
+        /**
+         * @brief Deserialize a verification key from a vector of field elements
+         *
+         * @param builder
+         * @param elements
+         */
+        VerificationKey(CircuitBuilder& builder, std::span<FF> elements)
+        {
+            // deserialize circuit size
+            size_t num_frs_read = 0;
+            size_t num_frs_FF = bb::stdlib::field_conversion::calc_num_bn254_frs<CircuitBuilder, FF>();
+            size_t num_frs_Comm = bb::stdlib::field_conversion::calc_num_bn254_frs<CircuitBuilder, Commitment>();
+
+            this->circuit_size = uint64_t(stdlib::field_conversion::convert_from_bn254_frs<CircuitBuilder, FF>(
+                                              builder, elements.subspan(num_frs_read, num_frs_read + num_frs_FF))
+                                              .get_value());
+            num_frs_read += num_frs_FF;
+            this->num_public_inputs = uint64_t(stdlib::field_conversion::convert_from_bn254_frs<CircuitBuilder, FF>(
+                                                   builder, elements.subspan(num_frs_read, num_frs_read + num_frs_FF))
+                                                   .get_value());
+            num_frs_read += num_frs_FF;
+
+            this->pub_inputs_offset = uint64_t(stdlib::field_conversion::convert_from_bn254_frs<CircuitBuilder, FF>(
+                                                   builder, elements.subspan(num_frs_read, num_frs_read + num_frs_FF))
+                                                   .get_value());
+            num_frs_read += num_frs_FF;
+
+            for (Commitment& comm : this->get_all()) {
+                comm = bb::stdlib::field_conversion::convert_from_bn254_frs<CircuitBuilder, Commitment>(
+                    builder, elements.subspan(num_frs_read, num_frs_read + num_frs_Comm));
+                num_frs_read += num_frs_Comm;
+            }
+        }
     };
 
     /**

barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/ultra_flavor.hpp

@@ -546,6 +546,31 @@ class UltraFlavor {
                        table_4,
                        lagrange_first,
                        lagrange_last);
+
+        /**
+         * @brief Serialize verification key to field elements
+         *
+         * @return std::vector<FF>
+         */
+        std::vector<FF> to_field_elements()
+        {
+            std::vector<FF> elements;
+            std::vector<FF> circuit_size_elements = bb::field_conversion::convert_to_bn254_frs(this->circuit_size);
+            elements.insert(elements.end(), circuit_size_elements.begin(), circuit_size_elements.end());
+            // do the same for the rest of the fields
+            std::vector<FF> num_public_inputs_elements =
+                bb::field_conversion::convert_to_bn254_frs(this->num_public_inputs);
+            elements.insert(elements.end(), num_public_inputs_elements.begin(), num_public_inputs_elements.end());
+            std::vector<FF> pub_inputs_offset_elements =
+                bb::field_conversion::convert_to_bn254_frs(this->pub_inputs_offset);
+            elements.insert(elements.end(), pub_inputs_offset_elements.begin(), pub_inputs_offset_elements.end());
+
+            for (Commitment& comm : this->get_all()) {
+                std::vector<FF> comm_elements = bb::field_conversion::convert_to_bn254_frs(comm);
+                elements.insert(elements.end(), comm_elements.begin(), comm_elements.end());
+            }
+            return elements;
+        }
     };
 
     /**

barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/ultra_recursive_flavor.hpp

@@ -274,6 +274,7 @@ template <typename BuilderType> class UltraRecursiveFlavor_ {
       public:
         VerificationKey(const size_t circuit_size, const size_t num_public_inputs)
         {
+            // TODO(https://github.com/AztecProtocol/barretenberg/issues/983): Think about if these should be witnesses
             this->circuit_size = circuit_size;
             this->log_circuit_size = numeric::get_msb(circuit_size);
             this->num_public_inputs = num_public_inputs;
@@ -317,6 +318,40 @@ template <typename BuilderType> class UltraRecursiveFlavor_ {
             this->lagrange_first = Commitment::from_witness(builder, native_key->lagrange_first);
             this->lagrange_last = Commitment::from_witness(builder, native_key->lagrange_last);
         };
+
+        /**
+         * @brief Deserialize a verification key from a vector of field elements
+         *
+         * @param builder
+         * @param elements
+         */
+        VerificationKey(CircuitBuilder& builder, std::span<FF> elements)
+        {
+            // deserialize circuit size
+            size_t num_frs_read = 0;
+            size_t num_frs_FF = bb::stdlib::field_conversion::calc_num_bn254_frs<CircuitBuilder, FF>();
+            size_t num_frs_Comm = bb::stdlib::field_conversion::calc_num_bn254_frs<CircuitBuilder, Commitment>();
+
+            this->circuit_size = uint64_t(stdlib::field_conversion::convert_from_bn254_frs<CircuitBuilder, FF>(
+                                              builder, elements.subspan(num_frs_read, num_frs_read + num_frs_FF))
+                                              .get_value());
+            num_frs_read += num_frs_FF;
+            this->num_public_inputs = uint64_t(stdlib::field_conversion::convert_from_bn254_frs<CircuitBuilder, FF>(
+                                                   builder, elements.subspan(num_frs_read, num_frs_read + num_frs_FF))
+                                                   .get_value());
+            num_frs_read += num_frs_FF;
+
+            this->pub_inputs_offset = uint64_t(stdlib::field_conversion::convert_from_bn254_frs<CircuitBuilder, FF>(
+                                                   builder, elements.subspan(num_frs_read, num_frs_read + num_frs_FF))
+                                                   .get_value());
+            num_frs_read += num_frs_FF;
+
+            for (Commitment& comm : this->get_all()) {
+                comm = bb::stdlib::field_conversion::convert_from_bn254_frs<CircuitBuilder, Commitment>(
+                    builder, elements.subspan(num_frs_read, num_frs_read + num_frs_Comm));
+                num_frs_read += num_frs_Comm;
+            }
+        }
     };
 
     /**