feat: add meta_hwm to private circuit ordering

Co-authored-by: Mitchell Tracy <[email protected]>

yarn-project/circuits.js/src/structs/kernel/private_kernel.ts

@@ -203,6 +203,10 @@ export class PrivateKernelInputsOrdering {
      * The previous kernel data
      */
     public previousKernel: PreviousKernelData,
+    /**
+     * The high water mark for the tx metadata phase.
+     */
+    public metaHwm: Fr,
     /**
      * The sorted new commitments.
      */
@@ -240,6 +244,7 @@ export class PrivateKernelInputsOrdering {
   toBuffer() {
     return serializeToBuffer(
       this.previousKernel,
+      this.metaHwm,
       this.sortedNewCommitments,
       this.sortedNewCommitmentsIndexes,
       this.readCommitmentHints,
@@ -259,6 +264,7 @@ export class PrivateKernelInputsOrdering {
     const reader = BufferReader.asReader(buffer);
     return new PrivateKernelInputsOrdering(
       reader.readObject(PreviousKernelData),
+      reader.readObject(Fr),
       reader.readArray(MAX_NEW_COMMITMENTS_PER_TX, SideEffect),
       reader.readNumbers(MAX_NEW_COMMITMENTS_PER_TX),
       reader.readArray(MAX_READ_REQUESTS_PER_TX, Fr),

yarn-project/noir-protocol-circuits/src/crates/private-kernel-lib/src/private_kernel_ordering.nr

@@ -10,7 +10,7 @@ use dep::types::{
         nullifier_key_validation_request::NullifierKeyValidationRequestContext,
         previous_kernel_data::PreviousKernelData,
         kernel_circuit_public_inputs::{
-            KernelCircuitPublicInputsBuilder, 
+            KernelCircuitPublicInputsBuilder,
             KernelCircuitPublicInputsFinal,
         },
         side_effect::{SideEffect, SideEffectLinkedToNoteHash, Ordered},
@@ -34,6 +34,7 @@ use dep::types::{
 
 struct PrivateKernelInputsOrdering {
     previous_kernel: PreviousKernelData,
+    meta_hwm: u32,
     sorted_new_commitments: [SideEffect; MAX_NEW_COMMITMENTS_PER_TX],
     sorted_new_commitments_indexes: [u32; MAX_NEW_COMMITMENTS_PER_TX],
     read_commitment_hints: [Field; MAX_READ_REQUESTS_PER_TX],
@@ -45,7 +46,7 @@ struct PrivateKernelInputsOrdering {
 
 impl PrivateKernelInputsOrdering {
     fn validate_inputs(self) {
-        assert_eq(array_length(self.previous_kernel.public_inputs.end.private_call_stack), 0, 
+        assert_eq(array_length(self.previous_kernel.public_inputs.end.private_call_stack), 0,
             "Private call stack must be empty when executing the ordering circuit");
     }
 
@@ -200,7 +201,7 @@ impl PrivateKernelInputsOrdering {
         self.validate_inputs();
 
         common::validate_previous_kernel_values(self.previous_kernel.public_inputs.end);
-        
+
         // Do this before any functions can modify the inputs.
         common::initialize_end_values(self.previous_kernel, &mut public_inputs);
 
@@ -210,11 +211,11 @@ impl PrivateKernelInputsOrdering {
 
         self.match_reads_to_commitments(&mut public_inputs);
 
-        self.match_nullifiers_to_commitments_and_squash(&mut public_inputs);        
+        self.match_nullifiers_to_commitments_and_squash(&mut public_inputs);
 
-        PrivateKernelInputsOrdering::apply_commitment_nonces(&mut public_inputs); 
+        PrivateKernelInputsOrdering::apply_commitment_nonces(&mut public_inputs);
 
-        public_inputs.to_final()
+        public_inputs.to_final(self.meta_hwm)
     }
 }
 
@@ -241,6 +242,7 @@ mod tests {
 
     struct PrivateKernelOrderingInputsBuilder {
         previous_kernel: PreviousKernelDataBuilder,
+        meta_hwm: u32,
         read_commitment_hints: [Field; MAX_READ_REQUESTS_PER_TX],
         nullifier_commitment_hints: [Field; MAX_NEW_NULLIFIERS_PER_TX],
     }
@@ -249,6 +251,7 @@ mod tests {
         pub fn new() -> Self {
             PrivateKernelOrderingInputsBuilder {
                 previous_kernel: PreviousKernelDataBuilder::new(),
+                meta_hwm: 0,
                 read_commitment_hints: [0; MAX_READ_REQUESTS_PER_TX],
                 nullifier_commitment_hints: [0; MAX_NEW_NULLIFIERS_PER_TX],
             }
@@ -266,7 +269,7 @@ mod tests {
             self.compute_unique_siloed_commitments(self.previous_kernel.end.new_commitments.storage)
         }
 
-        // A helper function that uses the first nullifer in the previous kernel to compute the unique siloed 
+        // A helper function that uses the first nullifer in the previous kernel to compute the unique siloed
         // commitments for the given commitments.
         pub fn compute_unique_siloed_commitments<N>(self, commitments: [SideEffect; N]) -> [SideEffect; N] {
             let first_nullifier = self.previous_kernel.end.new_nullifiers.get_unchecked(0);
@@ -330,6 +333,7 @@ mod tests {
             }
             let kernel = PrivateKernelInputsOrdering {
                 previous_kernel: self.previous_kernel.finish(),
+                meta_hwm: self.meta_hwm,
                 sorted_new_commitments,
                 sorted_new_commitments_indexes,
                 read_commitment_hints: sorted_read_commitment_hints,
@@ -532,4 +536,63 @@ mod tests {
         builder.previous_kernel.end.new_nullifiers = BoundedVec::new(SideEffectLinkedToNoteHash::empty());
         builder.failed();
     }
+
+    #[test]
+    unconstrained fn partitions_data() {
+        let mut builder = PrivateKernelOrderingInputsBuilder::new();
+
+        let mut sorted_new_commitments = [SideEffect::empty(); 10];
+        let mut sorted_new_nullifiers = [SideEffectLinkedToNoteHash::empty(); 10];
+
+        let mut counter = 1;
+        for i in 0..10 {
+            sorted_new_commitments[i] = SideEffect { value: (i + 1) as Field, counter: counter };
+            counter += 1;
+            sorted_new_nullifiers[i] = SideEffectLinkedToNoteHash { value: (i + 11) as Field, counter: counter, note_hash: 0 };
+            counter += 1;
+        }
+
+        for i in 0..10 {
+            builder.previous_kernel.end.new_commitments.push(sorted_new_commitments[9 - i]);
+            builder.previous_kernel.end.new_nullifiers.push(sorted_new_nullifiers[9 - i]);
+        }
+
+        builder.meta_hwm = 10;
+
+        let public_inputs = builder.execute();
+
+        let sorted_unique_commitments = compute_unique_siloed_commitments(
+            // NOTE(fees) changed the tx nullifier here for this test
+            // TODO(fees) review this after #4334
+            public_inputs.end_meta.new_nullifiers[0].value,
+            sorted_new_commitments
+        );
+
+        for i in 0..5 {
+            let meta_commitment = public_inputs.end_meta.new_commitments[i];
+            let commitment = sorted_unique_commitments[i];
+
+            assert(meta_commitment.eq(commitment));
+        }
+
+        // TODO(fees) do something about the magic indices in this test 🙈
+        // start offset by 1 because 0th nullifier is the tx hash
+        for i in 1..5 {
+            let meta_nullifier = public_inputs.end_meta.new_nullifiers[i];
+            let nullifier = sorted_new_nullifiers[i - 1];
+            assert(meta_nullifier.eq(nullifier));
+        }
+
+        for i in 0..5 {
+            let app_commitment = public_inputs.end.new_commitments[i];
+            let commitment = sorted_unique_commitments[5 + i];
+            assert(app_commitment.eq(commitment));
+        }
+        for i in 0..5 {
+            let app_nullifier = public_inputs.end.new_nullifiers[i];
+            let nullifier = sorted_new_nullifiers[4 + i];
+            // dep::std::println(f"i: {i}: {app_nullifier} {nullifier}");
+            assert(app_nullifier.eq(nullifier));
+        }
+    }
 }

yarn-project/noir-protocol-circuits/src/crates/types/src/abis/combined_accumulated_data.nr

@@ -8,7 +8,8 @@ use crate::{
         public_data_update_request::PublicDataUpdateRequest,
         side_effect::{SideEffect, SideEffectLinkedToNoteHash},
     },
-    mocked::AggregationObject
+    mocked::AggregationObject,
+    utils::arrays::partition_array,
 };
 use crate::constants::{
     MAX_READ_REQUESTS_PER_TX,
@@ -138,20 +139,51 @@ impl CombinedAccumulatedDataBuilder {
         }
     } 
 
-    pub fn to_final(self) -> FinalAccumulatedData {
+    pub fn to_final(self, meta_hwm: u32) -> (FinalAccumulatedData, FinalAccumulatedData) {
         assert_eq(self.read_requests.len, 0, "Final accumulated data: read requests not empty");
         assert_eq(self.nullifier_key_validation_requests.len, 0, "Final accumulated data: nullifier key validation requests not empty");
         assert_eq(self.public_data_update_requests.len, 0, "Final accumulated data: public data update requests not empty");
         assert_eq(self.public_data_reads.len, 0, "Final accumulated data: public data reads not empty");
 
-        FinalAccumulatedData {
+        let (meta_commitments, app_commitments) = partition_array(self.new_commitments.storage, |x: SideEffect| -> bool x.counter() < meta_hwm);
+        let (meta_nullifiers, app_nullifiers) = partition_array(self.new_nullifiers.storage, |x: SideEffectLinkedToNoteHash| x.counter() < meta_hwm);
+
+        // TODO(fees) is this OK? should call stack items be side effects?
+        let (meta_private_call_stack, app_private_call_stack) = partition_array(self.private_call_stack.storage, |x: CallRequest| x.end_side_effect_counter < meta_hwm);
+        let (meta_public_call_stack, app_public_call_stack) = partition_array(self.public_call_stack.storage, |x: CallRequest| x.end_side_effect_counter < meta_hwm);
+
+        let fee = FinalAccumulatedData {
             aggregation_object: self.aggregation_object,
 
-            new_commitments: self.new_commitments.storage,
-            new_nullifiers: self.new_nullifiers.storage,
+            new_commitments: meta_commitments,
+            new_nullifiers: meta_nullifiers,
 
-            private_call_stack: self.private_call_stack.storage,
-            public_call_stack: self.public_call_stack.storage,
+            private_call_stack: meta_private_call_stack,
+            public_call_stack: meta_public_call_stack,
+
+            // TODO(fees) I think we should track L2L1 messages as side effects too
+            new_l2_to_l1_msgs: [0; MAX_NEW_L2_TO_L1_MSGS_PER_TX],
+
+            // TODO(fees) what about logs? I think they should be tracked as side effects
+            encrypted_logs_hash: [0, 0],
+            unencrypted_logs_hash: [0, 0],
+
+            encrypted_log_preimages_length: 0,
+            unencrypted_log_preimages_length: 0,
+
+            new_contracts: [NewContractData::empty(); MAX_NEW_CONTRACTS_PER_TX],
+
+            optionally_revealed_data: self.optionally_revealed_data,
+        };
+
+        let app = FinalAccumulatedData {
+            aggregation_object: self.aggregation_object,
+
+            new_commitments: app_commitments,
+            new_nullifiers: app_nullifiers,
+
+            private_call_stack: app_private_call_stack,
+            public_call_stack: app_public_call_stack,
             new_l2_to_l1_msgs: self.new_l2_to_l1_msgs.storage,
 
             encrypted_logs_hash: self.encrypted_logs_hash,
@@ -162,6 +194,8 @@ impl CombinedAccumulatedDataBuilder {
 
             new_contracts: self.new_contracts.storage,
             optionally_revealed_data: self.optionally_revealed_data,
-        }
-    } 
+        };
+
+        (fee, app)
+    }
 }

yarn-project/noir-protocol-circuits/src/crates/types/src/abis/kernel_circuit_public_inputs.nr

@@ -14,6 +14,8 @@ struct KernelCircuitPublicInputs {
 }
 
 struct KernelCircuitPublicInputsFinal {
+    end_meta: FinalAccumulatedData,
+    // TODO(fees) change this to end_app_logic
     end: FinalAccumulatedData,
     constants: CombinedConstantData,
     is_private: bool,
@@ -34,9 +36,11 @@ impl KernelCircuitPublicInputsBuilder {
         }
     }
 
-    pub fn to_final(self) -> KernelCircuitPublicInputsFinal {
+    pub fn to_final(self, meta_hwm: u32) -> KernelCircuitPublicInputsFinal {
+        let (meta, app) = self.end.to_final(meta_hwm);
         KernelCircuitPublicInputsFinal {
-            end: self.end.to_final(),
+            end_meta: meta,
+            end: app,
             constants: self.constants,
             is_private: self.is_private,
         }

yarn-project/noir-protocol-circuits/src/crates/types/src/utils/arrays.nr

@@ -57,6 +57,31 @@ pub fn array_eq<T, N, S>(array: [T; N], expected: [T; S]) -> bool where T: Empty
     eq
 }
 
+/**
+ * Splits an array in two parts, where the first part contains all elements for which the predicate is true,
+ * and the second part contains all elements for which the predicate is false.
+ */
+pub fn partition_array<T, N, Env>(array: [T; N], predicate: fn[Env](T) -> bool) -> ([T; N], [T; N]) where T: Empty {
+    let mut partition_a = [T::empty(); N];
+    let mut a_idx = 0;
+
+    let mut partition_b = [T::empty(); N];
+    let mut b_idx = 0;
+
+    for i in 0..N {
+        let elem = array[i];
+        if predicate(elem) {
+            partition_a[a_idx] = elem;
+            a_idx += 1;
+        } else {
+            partition_b[b_idx] = elem;
+            b_idx += 1;
+        }
+    }
+
+    (partition_a, partition_b)
+}
+
 #[test]
 fn smoke_validate_array() {
     let valid_array = [];
@@ -102,3 +127,12 @@ fn test_array_length() {
     assert_eq(array_length([123, 0, 456]), 1);
     assert_eq(array_length([0, 123, 0, 456]), 0);
 }
+
+#[test]
+fn test_partition_array() {
+    let array: [u32; 9] = [1, 2, 3, 4, 5, 6, 7, 8, 9];
+    let (partition_a, partition_b) = partition_array(array, |x| x % 2 == 0);
+
+    assert_eq(partition_a, [2, 4, 6, 8, 0, 0, 0, 0, 0]);
+    assert_eq(partition_b, [1, 3, 5, 7, 9, 0, 0, 0, 0]);
+}

yarn-project/noir-protocol-circuits/src/type_conversion.ts

@@ -1110,6 +1110,7 @@ export function mapPrivateKernelInputsOrderingToNoir(
 ): PrivateKernelInputsOrderingNoir {
   return {
     previous_kernel: mapPreviousKernelDataToNoir(inputs.previousKernel),
+    meta_hwm: mapFieldToNoir(inputs.metaHwm),
     sorted_new_commitments: mapTuple(inputs.sortedNewCommitments, mapSideEffectToNoir),
     sorted_new_commitments_indexes: mapTuple(inputs.sortedNewCommitmentsIndexes, mapNumberToNoir),
     read_commitment_hints: mapTuple(inputs.readCommitmentHints, mapFieldToNoir),

yarn-project/pxe/src/kernel_prover/kernel_prover.ts

@@ -195,6 +195,7 @@ export class KernelProver {
 
     const privateInputs = new PrivateKernelInputsOrdering(
       previousKernelData,
+      Fr.ZERO,
       sortedCommitments,
       sortedCommitmentsIndexes,
       readCommitmentHints,