test: Add tests for noir<>ivc integration testing (#7931)

Adds a new suite of "mocked" protocol circuits that follow the same IVC
scheme as the real ones. This allow us to do integration testing between
the noir tools to implement ClientIVC (databus, verify_proof, etc) and
barretenberg.
Added a test in yarn-project using this mocked protocol circuits to
create&verify ClientIVC proofs.

noir-projects/Dockerfile

@@ -17,10 +17,13 @@ ENV PATH="/usr/src/avm-transpiler/target/release:${PATH}"
 WORKDIR /usr/src/noir-projects
 COPY . .
 # Build
+RUN yarn
 WORKDIR /usr/src/noir-projects/noir-contracts
 RUN ./bootstrap.sh
 WORKDIR /usr/src/noir-projects/noir-protocol-circuits
 RUN ./bootstrap.sh
+WORKDIR /usr/src/noir-projects/mock-protocol-circuits
+RUN ./bootstrap.sh
 WORKDIR /usr/src/noir-projects/aztec-nr
 RUN nargo compile --silence-warnings
 

noir-projects/Dockerfile.test

@@ -22,6 +22,7 @@ WORKDIR /usr/src/noir-projects
 COPY . .
 
 # Build & test
+RUN yarn
 RUN cd ./noir-protocol-circuits && ./bootstrap.sh && nargo test --silence-warnings
 
 RUN cd /usr/src/yarn-project/txe && yarn start & \ 

noir-projects/Earthfile

@@ -11,7 +11,8 @@ source:
   WORKDIR /usr/src/noir-projects
 
   # Copy source.
-  COPY --dir aztec-nr noir-contracts noir-protocol-circuits .
+  COPY package.json yarn.lock mega_honk_circuits.json .
+  COPY --dir aztec-nr noir-contracts noir-protocol-circuits mock-protocol-circuits scripts .
 
   # for debugging rebuilds
   RUN echo CONTENT HASH $(find . -type f -exec sha256sum {} ';' | sort | sha256sum | awk '{print $1}') | tee .content-hash
@@ -22,6 +23,7 @@ build-contracts:
   # Install transpiler
   COPY ../avm-transpiler/+build/avm-transpiler /usr/bin/avm-transpiler
 
+  RUN yarn
   RUN cd noir-contracts && NARGO=nargo TRANSPILER=avm-transpiler ./bootstrap.sh
   SAVE ARTIFACT noir-contracts
 
@@ -31,20 +33,34 @@ build-protocol-circuits:
   FROM +source
   RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p ~/.aws && \
       bash -c 'echo -e "[default]\naws_access_key_id=$AWS_ACCESS_KEY_ID\naws_secret_access_key=$AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials'
+  RUN yarn
   RUN cd noir-protocol-circuits && BB_HASH=$bb_source_hash NARGO=nargo ./bootstrap.sh
   SAVE ARTIFACT noir-protocol-circuits
 
+build-mock-protocol-circuits:
+  LOCALLY
+  LET bb_source_hash = $(cd .. && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin)
+  FROM +source
+  RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p ~/.aws && \
+      bash -c 'echo -e "[default]\naws_access_key_id=$AWS_ACCESS_KEY_ID\naws_secret_access_key=$AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials'
+  RUN yarn
+  RUN cd mock-protocol-circuits && BB_HASH=$bb_source_hash NARGO=nargo ./bootstrap.sh
+  SAVE ARTIFACT mock-protocol-circuits
+
 build:
   FROM +source
   BUILD +build-contracts
   BUILD +build-protocol-circuits
+  BUILD +build-mock-protocol-circuits
 
   COPY +build-contracts/noir-contracts ./noir-contracts
   COPY +build-protocol-circuits/noir-protocol-circuits ./noir-protocol-circuits
+  COPY +build-mock-protocol-circuits/mock-protocol-circuits ./mock-protocol-circuits
 
   SAVE ARTIFACT aztec-nr
   SAVE ARTIFACT noir-contracts
   SAVE ARTIFACT noir-protocol-circuits
+  SAVE ARTIFACT mock-protocol-circuits
 
 test:
   FROM ../yarn-project/+txe
@@ -76,6 +92,9 @@ format:
   WORKDIR /usr/src/noir-projects/noir-protocol-circuits
   RUN nargo fmt --check
 
+  WORKDIR /usr/src/noir-projects/mock-protocol-circuits
+  RUN nargo fmt --check
+
   WORKDIR /usr/src/noir-projects/noir-contracts
   RUN nargo fmt --check
 

noir-projects/bootstrap.sh

@@ -37,13 +37,17 @@ esac
 # If builds fail with an amount of free memory greater than this value then it should be increased.
 MIN_PARALLEL_BUILD_MEMORY=32854492
 
+yarn
+
 if [[ AVAILABLE_MEMORY -lt MIN_PARALLEL_BUILD_MEMORY ]]; then
   echo "System does not have enough memory for parallel builds, falling back to sequential"
   ./noir-contracts/bootstrap.sh
   ./noir-protocol-circuits/bootstrap.sh
+  ./mock-protocol-circuits/bootstrap.sh
 else
   ((./noir-contracts/bootstrap.sh) > >(awk -v g="$g" -v r="$r" '{print g "contracts: " r $0}')) &
   ((./noir-protocol-circuits/bootstrap.sh) > >(awk -v b="$b" -v r="$r" '{print  b "protocol-circuits: " r $0}')) &
+  ((./mock-protocol-circuits/bootstrap.sh) > >(awk -v b="$b" -v r="$r" '{print  b "mock-protocol-circuits: " r $0}')) &
 
   for job in $(jobs -p); do
     wait $job || exit 1

noir-projects/bootstrap_cache.sh

@@ -10,3 +10,5 @@ extract_repo_if_working_copy_clean noir-projects \
   /usr/src/noir-projects/noir-protocol-circuits/target ./noir-protocol-circuits
 
 remove_old_images noir-projects
+
+./mock-protocol-circuits/bootstrap.sh

noir-projects/gates_report.sh

@@ -5,6 +5,8 @@ set -eu
 # The script needs some slight updating as `nargo info` expects a complete JSON object, while this script expects a single object field
 # representing a list of circuit reports for a program.
 # The ACIR tests in barretenberg also expect every target bytecode to have the name `acir.gz` while this script expects the same name of the package
+MEGA_HONK_CIRCUIT_PATTERNS=$(jq -r '.[]' mega_honk_circuits.json)
+
 cd noir-protocol-circuits
 PROTOCOL_CIRCUITS_DIR=$PWD
 
@@ -15,7 +17,6 @@ echo "{\"programs\": [" > gates_report.json
 # Bound for checking where to place last parentheses
 NUM_ARTIFACTS=$(ls -1q "$PROTOCOL_CIRCUITS_DIR/target"/*.json | wc -l)
 
-MEGA_HONK_CIRCUIT_PATTERNS=$(jq -r '.[]' mega_honk_circuits.json)
 
 ITER="1"
 for pathname in "$PROTOCOL_CIRCUITS_DIR/target"/*.json; do

noir-projects/mock-protocol-circuits/.gitignore

@@ -0,0 +1,3 @@
+Prover.toml
+Verifier.toml
+target

noir-projects/mock-protocol-circuits/Nargo.toml

@@ -0,0 +1,10 @@
+[workspace]
+members = [
+  "crates/mock-types",
+  "crates/app-creator",
+  "crates/app-reader",
+  "crates/mock-private-kernel-init",
+  "crates/mock-private-kernel-inner",
+  "crates/mock-private-kernel-reset",
+  "crates/mock-private-kernel-tail",
+]

noir-projects/mock-protocol-circuits/Readme.md

@@ -0,0 +1,3 @@
+# Mocked protocol circuits
+
+These simplified circuits act as a way of testing the IVC integration between noir and barretenberg. They follow the same IVC scheme as the real circuits, but are much simpler and easier to reason about.

noir-projects/mock-protocol-circuits/bootstrap.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -eu
+
+cd "$(dirname "$0")"
+
+CMD=${1:-}
+
+if [ -n "$CMD" ]; then
+  if [ "$CMD" = "clean" ]; then
+    git clean -fdx
+    exit 0
+  else
+    echo "Unknown command: $CMD"
+    exit 1
+  fi
+fi
+
+NARGO=${NARGO:-../../noir/noir-repo/target/release/nargo}
+$NARGO compile --silence-warnings
+
+BB_HASH=${BB_HASH:-$(cd ../../ && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin)}
+echo Using BB hash $BB_HASH
+mkdir -p "./target/keys"
+
+for pathname in "./target"/*.json; do    
+    BB_HASH=$BB_HASH node ../scripts/generate_vk_json.js "$pathname" "./target/keys" &
+done
+
+for job in $(jobs -p); do
+  wait $job || exit 1
+done

noir-projects/mock-protocol-circuits/crates/app-creator/Nargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "app_creator"
+type = "bin"
+authors = [""]
+compiler_version = ">=0.32.0"
+
+[dependencies]
+mock_types = { path = "../mock-types" }

noir-projects/mock-protocol-circuits/crates/app-creator/src/main.nr

@@ -0,0 +1,10 @@
+use dep::mock_types::{AppPublicInputs, MAX_COMMITMENTS_PER_CALL};
+
+// Mock app for testing that creates the commitments the user commands.
+// Note: A zero is a null commitment.
+fn main(commitments_to_create: [Field; MAX_COMMITMENTS_PER_CALL]) -> pub AppPublicInputs {
+    let mut result = AppPublicInputs::default();
+    result.commitments = commitments_to_create;
+    result
+}
+

noir-projects/mock-protocol-circuits/crates/app-reader/Nargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "app_reader"
+type = "bin"
+authors = [""]
+compiler_version = ">=0.32.0"
+
+[dependencies]
+mock_types = { path = "../mock-types" }

noir-projects/mock-protocol-circuits/crates/app-reader/src/main.nr

@@ -0,0 +1,10 @@
+use dep::mock_types::{AppPublicInputs, MAX_COMMITMENT_READ_REQUESTS_PER_CALL};
+
+// Mock app for testing that reads the commitments (generates read requests) the user commands.
+// Note: A zero read is a null read.
+fn main(commitments_to_read: [Field; MAX_COMMITMENT_READ_REQUESTS_PER_CALL]) -> pub AppPublicInputs {
+    let mut result = AppPublicInputs::default();
+    result.read_requests = commitments_to_read;
+    result
+}
+

noir-projects/mock-protocol-circuits/crates/mock-private-kernel-init/Nargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "mock_private_kernel_init"
+type = "bin"
+authors = [""]
+compiler_version = ">=0.32.0"
+
+[dependencies]
+mock_types = { path = "../mock-types" }

noir-projects/mock-protocol-circuits/crates/mock-private-kernel-init/src/main.nr

@@ -0,0 +1,8 @@
+use dep::mock_types::{TxRequest, PrivateKernelPublicInputs, PrivateKernelPublicInputsBuilder, AppPublicInputs};
+
+fn main(tx: TxRequest, app_inputs: AppPublicInputs) -> pub PrivateKernelPublicInputs {
+    let mut private_kernel_inputs = PrivateKernelPublicInputsBuilder::from_tx(tx);
+    private_kernel_inputs.ingest_app_inputs(app_inputs);
+    private_kernel_inputs.finish()
+}
+

noir-projects/mock-protocol-circuits/crates/mock-private-kernel-inner/Nargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "mock_private_kernel_inner"
+type = "bin"
+authors = [""]
+compiler_version = ">=0.32.0"
+
+[dependencies]
+mock_types = { path = "../mock-types" }

noir-projects/mock-protocol-circuits/crates/mock-private-kernel-inner/src/main.nr

@@ -0,0 +1,11 @@
+use dep::mock_types::{PrivateKernelPublicInputs, PrivateKernelPublicInputsBuilder, AppPublicInputs};
+
+fn main(
+    prev_kernel_public_inputs: PrivateKernelPublicInputs,
+    app_inputs: AppPublicInputs
+) -> pub PrivateKernelPublicInputs {
+    let mut private_kernel_inputs = PrivateKernelPublicInputsBuilder::from_previous_kernel(prev_kernel_public_inputs);
+    private_kernel_inputs.ingest_app_inputs(app_inputs);
+    private_kernel_inputs.finish()
+}
+

noir-projects/mock-protocol-circuits/crates/mock-private-kernel-reset/Nargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "mock_private_kernel_reset"
+type = "bin"
+authors = [""]
+compiler_version = ">=0.32.0"
+
+[dependencies]
+mock_types = { path = "../mock-types" }

noir-projects/mock-protocol-circuits/crates/mock-private-kernel-reset/src/main.nr

@@ -0,0 +1,22 @@
+use dep::mock_types::{
+    PrivateKernelPublicInputs, PrivateKernelPublicInputsBuilder, MAX_COMMITMENT_READ_REQUESTS_PER_TX,
+    MAX_COMMITMENTS_PER_TX
+};
+
+// Mock reset kernel that reset read requests.
+// It needs hints to locate the commitment that matches the read requests.
+fn main(
+    mut prev_kernel_public_inputs: PrivateKernelPublicInputs,
+    commitment_read_hints: [u32; MAX_COMMITMENT_READ_REQUESTS_PER_TX]
+) -> pub PrivateKernelPublicInputs {
+    for i in 0..MAX_COMMITMENT_READ_REQUESTS_PER_TX {
+        if commitment_read_hints[i] != MAX_COMMITMENTS_PER_TX {
+            assert_eq(
+                prev_kernel_public_inputs.commitments[commitment_read_hints[i]], prev_kernel_public_inputs.read_requests[i]
+            );
+            prev_kernel_public_inputs.read_requests[i] = 0;
+        }
+    }
+    prev_kernel_public_inputs
+}
+

noir-projects/mock-protocol-circuits/crates/mock-private-kernel-tail/Nargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "mock_private_kernel_tail"
+type = "bin"
+authors = [""]
+compiler_version = ">=0.32.0"
+
+[dependencies]
+mock_types = { path = "../mock-types" }

noir-projects/mock-protocol-circuits/crates/mock-private-kernel-tail/src/main.nr

@@ -0,0 +1,15 @@
+use dep::mock_types::{
+    PrivateKernelPublicInputs, PrivateKernelPublicInputsBuilder, KernelPublicInputs,
+    MAX_COMMITMENT_READ_REQUESTS_PER_TX
+};
+
+// The tail kernel finishes the client IVC chain exposing the final public inputs with no remaining calls or unfulfilled read requests.
+fn main(prev_kernel_public_inputs: PrivateKernelPublicInputs) -> pub KernelPublicInputs {
+    assert_eq(prev_kernel_public_inputs.remaining_calls, 0);
+    for i in 0..MAX_COMMITMENT_READ_REQUESTS_PER_TX {
+        assert_eq(prev_kernel_public_inputs.read_requests[i], 0);
+    }
+
+    KernelPublicInputs { commitments: prev_kernel_public_inputs.commitments }
+}
+

noir-projects/mock-protocol-circuits/crates/mock-types/Nargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "mock_types"
+type = "lib"
+authors = [""]
+compiler_version = ">=0.32.0"
+
+[dependencies]