feat: Improve the proving orchestration lifecycle

cspell.json

@@ -81,6 +81,8 @@
     "fargate",
     "Fieldeable",
     "filestat",
+    "finalise",
+    "finalised",
     "flatmap",
     "foundryup",
     "frontend",
@@ -101,6 +103,7 @@
     "herskind",
     "ierc",
     "indexeddb",
+    "initialise",
     "interruptible",
     "isequal",
     "jsons",

yarn-project/circuit-types/src/interfaces/block-prover.ts

@@ -10,8 +10,6 @@ export enum PROVING_STATUS {
 
 export type ProvingSuccess = {
   status: PROVING_STATUS.SUCCESS;
-  block: L2Block;
-  proof: Proof;
 };
 
 export type ProvingFailure = {
@@ -25,17 +23,49 @@ export type ProvingTicket = {
   provingPromise: Promise<ProvingResult>;
 };
 
+export type BlockResult = {
+  block: L2Block;
+  proof: Proof;
+};
+
 /**
  * The interface to the block prover.
  * Provides the ability to generate proofs and build rollups.
  */
 export interface BlockProver {
+  /**
+   * Cancels any block that is currently being built and prepares for a new one to be built
+   * @param numTxs - The complete size of the block, must be a power of 2
+   * @param globalVariables - The global variables for this block
+   * @param l1ToL2Messages - The set of L1 to L2 messages to be included in this block
+   * @param emptyTx - An instance of an empty transaction to be used in this block
+   */
   startNewBlock(
     numTxs: number,
     globalVariables: GlobalVariables,
     l1ToL2Messages: Fr[],
     emptyTx: ProcessedTx,
   ): Promise<ProvingTicket>;
 
+  /**
+   * Add a processed transaction to the current block
+   * @param tx - The transaction to be added
+   */
   addNewTx(tx: ProcessedTx): Promise<void>;
+
+  /**
+   * Cancels the block currently being proven. Proofs already bring built may continue but further proofs should not be started.
+   */
+  cancelBlock(): void;
+
+  /**
+   * Performs the final archive tree insertion for this block and returns the L2Block and Proof instances
+   */
+  finaliseBlock(): Promise<BlockResult>;
+
+  /**
+   * Mark the block as having all the transactions it is going to contain.
+   * Will pad the block to it's complete size with empty transactions and prove all the way to the root rollup.
+   */
+  setBlockCompleted(): Promise<void>;
 }

yarn-project/circuits.js/src/structs/global_variables.ts

@@ -106,6 +106,10 @@ export class GlobalVariables {
     };
   }
 
+  clone(): GlobalVariables {
+    return GlobalVariables.fromBuffer(this.toBuffer());
+  }
+
   isEmpty(): boolean {
     return (
       this.chainId.isZero() &&

yarn-project/circuits.js/src/structs/header.ts

@@ -40,6 +40,10 @@ export class Header {
     return fields;
   }
 
+  clone(): Header {
+    return Header.fromBuffer(this.toBuffer());
+  }
+
   static fromBuffer(buffer: Buffer | BufferReader): Header {
     const reader = BufferReader.asReader(buffer);
 

yarn-project/end-to-end/src/benchmarks/bench_tx_size_fees.test.ts

@@ -56,11 +56,10 @@ describe('benchmarks/tx_size_fees', () => {
   beforeAll(async () => {
     await Promise.all([
       gas.methods.mint_public(aliceWallet.getAddress(), 1000n).send().wait(),
-      token.methods.privately_mint_private_note(1000n).send().wait(),
-      token.methods.mint_public(aliceWallet.getAddress(), 1000n).send().wait(),
-
       gas.methods.mint_public(fpc.address, 1000n).send().wait(),
     ]);
+    await token.methods.privately_mint_private_note(1000n).send().wait();
+    await token.methods.mint_public(aliceWallet.getAddress(), 1000n).send().wait();
   });
 
   it.each<() => Promise<FeePaymentMethod | undefined>>([

yarn-project/end-to-end/src/integration_l1_publisher.test.ts

@@ -12,8 +12,8 @@ import {
 } from '@aztec/aztec.js';
 // eslint-disable-next-line no-restricted-imports
 import {
+  PROVING_STATUS,
   type ProcessedTx,
-  type ProvingSuccess,
   makeEmptyProcessedTx as makeEmptyProcessedTxFromHistoricalTreeRoots,
   makeProcessedTx,
 } from '@aztec/circuit-types';
@@ -142,6 +142,7 @@ describe('L1Publisher integration', () => {
       l2QueueSize: 10,
     };
     const worldStateSynchronizer = new ServerWorldStateSynchronizer(tmpStore, builderDb, blockSource, worldStateConfig);
+    await worldStateSynchronizer.start();
     builder = await TxProver.new({}, worldStateSynchronizer, new WASMSimulator());
     l2Proof = Buffer.alloc(0);
 
@@ -390,8 +391,12 @@ describe('L1Publisher integration', () => {
       );
       const ticket = await buildBlock(globalVariables, txs, currentL1ToL2Messages, makeEmptyProcessedTx());
       const result = await ticket.provingPromise;
-      const block = (result as ProvingSuccess).block;
+      expect(result.status).toBe(PROVING_STATUS.SUCCESS);
+      const blockResult = await builder.finaliseBlock();
+      const block = blockResult.block;
       prevHeader = block.header;
+      blockSource.getL1ToL2Messages.mockResolvedValueOnce(currentL1ToL2Messages);
+      blockSource.getBlocks.mockResolvedValueOnce([block]);
 
       const newL2ToL1MsgsArray = block.body.txEffects.flatMap(txEffect => txEffect.l2ToL1Msgs);
 
@@ -480,8 +485,12 @@ describe('L1Publisher integration', () => {
       );
       const blockTicket = await buildBlock(globalVariables, txs, l1ToL2Messages, makeEmptyProcessedTx());
       const result = await blockTicket.provingPromise;
-      const block = (result as ProvingSuccess).block;
+      expect(result.status).toBe(PROVING_STATUS.SUCCESS);
+      const blockResult = await builder.finaliseBlock();
+      const block = blockResult.block;
       prevHeader = block.header;
+      blockSource.getL1ToL2Messages.mockResolvedValueOnce(l1ToL2Messages);
+      blockSource.getBlocks.mockResolvedValueOnce([block]);
 
       writeJson(`empty_block_${i}`, block, [], AztecAddress.ZERO, deployerAccount.address);
 

yarn-project/prover-client/src/dummy-prover.ts

@@ -1,4 +1,5 @@
 import {
+  type BlockResult,
   L2Block,
   PROVING_STATUS,
   type ProcessedTx,
@@ -30,8 +31,6 @@ export class DummyProver implements ProverClient {
   ): Promise<ProvingTicket> {
     const result: ProvingSuccess = {
       status: PROVING_STATUS.SUCCESS,
-      proof: makeEmptyProof(),
-      block: L2Block.empty(),
     };
     const ticket: ProvingTicket = {
       provingPromise: Promise.resolve(result),
@@ -42,4 +41,17 @@ export class DummyProver implements ProverClient {
   addNewTx(_tx: ProcessedTx): Promise<void> {
     return Promise.resolve();
   }
+
+  cancelBlock(): void {}
+
+  finaliseBlock(): Promise<BlockResult> {
+    return Promise.resolve({
+      block: L2Block.empty(),
+      proof: makeEmptyProof(),
+    });
+  }
+
+  setBlockCompleted(): Promise<void> {
+    return Promise.resolve();
+  }
 }

yarn-project/prover-client/src/orchestrator/block-building-helpers.ts

@@ -9,8 +9,7 @@ import {
   Fr,
   type GlobalVariables,
   KernelData,
-  L1_TO_L2_MSG_SUBTREE_HEIGHT,
-  L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH,
+  type L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH,
   MAX_NEW_NULLIFIERS_PER_TX,
   MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX,
   MembershipWitness,
@@ -200,29 +199,29 @@ export async function executeRootRollupCircuit(
   right: [BaseOrMergeRollupPublicInputs, Proof],
   l1ToL2Roots: RootParityInput,
   newL1ToL2Messages: Tuple<Fr, typeof NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP>,
+  messageTreeSnapshot: AppendOnlyTreeSnapshot,
+  messageTreeRootSiblingPath: Tuple<Fr, typeof L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH>,
   simulator: RollupSimulator,
   prover: RollupProver,
   db: MerkleTreeOperations,
   logger?: DebugLogger,
 ): Promise<[RootRollupPublicInputs, Proof]> {
   logger?.debug(`Running root rollup circuit`);
-  const rootInput = await getRootRollupInput(...left, ...right, l1ToL2Roots, newL1ToL2Messages, db);
-
-  // Update the local trees to include the new l1 to l2 messages
-  await db.appendLeaves(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, newL1ToL2Messages);
+  const rootInput = await getRootRollupInput(
+    ...left,
+    ...right,
+    l1ToL2Roots,
+    newL1ToL2Messages,
+    messageTreeSnapshot,
+    messageTreeRootSiblingPath,
+    db,
+  );
 
   // Simulate and get proof for the root circuit
   const rootOutput = await simulator.rootRollupCircuit(rootInput);
 
   const rootProof = await prover.getRootRollupProof(rootInput, rootOutput);
 
-  //TODO(@PhilWindle) Move this to orchestrator to ensure that we are still on the same block
-  // Update the archive with the latest block header
-  logger?.debug(`Updating and validating root trees`);
-  await db.updateArchive(rootOutput.header);
-
-  await validateRootOutput(rootOutput, db);
-
   return [rootOutput, rootProof];
 }
 
@@ -259,6 +258,8 @@ export async function getRootRollupInput(
   rollupProofRight: Proof,
   l1ToL2Roots: RootParityInput,
   newL1ToL2Messages: Tuple<Fr, typeof NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP>,
+  messageTreeSnapshot: AppendOnlyTreeSnapshot,
+  messageTreeRootSiblingPath: Tuple<Fr, typeof L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH>,
   db: MerkleTreeOperations,
 ) {
   const vks = getVerificationKeys();
@@ -274,21 +275,6 @@ export async function getRootRollupInput(
     return path.toFields();
   };
 
-  const newL1ToL2MessageTreeRootSiblingPathArray = await getSubtreeSiblingPath(
-    MerkleTreeId.L1_TO_L2_MESSAGE_TREE,
-    L1_TO_L2_MSG_SUBTREE_HEIGHT,
-    db,
-  );
-
-  const newL1ToL2MessageTreeRootSiblingPath = makeTuple(
-    L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH,
-    i => (i < newL1ToL2MessageTreeRootSiblingPathArray.length ? newL1ToL2MessageTreeRootSiblingPathArray[i] : Fr.ZERO),
-    0,
-  );
-
-  // Get tree snapshots
-  const startL1ToL2MessageTreeSnapshot = await getTreeSnapshot(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, db);
-
   // Get blocks tree
   const startArchiveSnapshot = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db);
   const newArchiveSiblingPathArray = await getRootTreeSiblingPath(MerkleTreeId.ARCHIVE);
@@ -303,8 +289,8 @@ export async function getRootRollupInput(
     previousRollupData,
     l1ToL2Roots,
     newL1ToL2Messages,
-    newL1ToL2MessageTreeRootSiblingPath,
-    startL1ToL2MessageTreeSnapshot,
+    newL1ToL2MessageTreeRootSiblingPath: messageTreeRootSiblingPath,
+    startL1ToL2MessageTreeSnapshot: messageTreeSnapshot,
     startArchiveSnapshot,
     newArchiveSiblingPath,
   });