From a9d418c07268a38e0c5432983438ea00b97d233b Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Mon, 2 Dec 2024 09:20:46 -0300 Subject: [PATCH] feat: Process blocks in parallel during epoch proving (#10263) Instead of processing blocks in sequencer during epoch proving and trigger proving jobs as each block is processed, we fetch the state prior to each block by forking off a world state following the pending (not proven) chain and process each block (ie execute public functions) in parallel. This means tx execution is less of a bottleneck for proving. Main change is that the epoch orchestrator now requires not a db, but something that can return forks at given block numbers of the db. It also means that the orchestrator accepts out-of-order operations for block building, so multiple blocks can be started, and their txs added in any order (though following the order within each block). Builds on #10174 Fixes #10265 Pending: - Ensuring the state after processing each block matches what we had used from world-state --- .../src/interfaces/epoch-prover.ts | 24 +-- .../src/interfaces/prover-client.ts | 3 +- .../src/interfaces/world_state.ts | 26 ++- .../circuit-types/src/test/factories.ts | 9 +- .../composed/integration_l1_publisher.test.ts | 2 +- .../src/e2e_prover/e2e_prover_test.ts | 1 + yarn-project/end-to-end/src/fixtures/utils.ts | 1 + yarn-project/foundation/package.json | 3 +- .../foundation/src/async-pool/index.ts | 50 ++++++ .../foundation/src/collection/array.test.ts | 20 ++- .../foundation/src/collection/array.ts | 24 +++ yarn-project/foundation/src/config/env_var.ts | 1 + yarn-project/prover-client/package.json | 3 +- .../src/block_builder/index.ts | 1 - .../src/block_builder/light.test.ts | 12 +- .../src/block_builder/light.ts | 29 +++- yarn-project/prover-client/src/index.ts | 3 +- .../prover-client/src/mocks/fixtures.ts | 13 +- .../prover-client/src/mocks/test_context.ts | 104 +++++++++--- .../src/orchestrator/epoch-proving-state.ts | 13 +- .../src/orchestrator/orchestrator.ts | 158 ++++++++++-------- .../orchestrator/orchestrator_errors.test.ts | 88 +++++----- .../orchestrator_failures.test.ts | 38 ++--- .../orchestrator_lifecycle.test.ts | 5 +- .../orchestrator_mixed_blocks.test.ts | 26 +-- ...rchestrator_multi_public_functions.test.ts | 4 +- .../orchestrator_multiple_blocks.test.ts | 58 ++++--- .../orchestrator_public_functions.test.ts | 4 +- .../orchestrator_single_blocks.test.ts | 17 +- .../orchestrator_workflow.test.ts | 15 +- .../{tx-prover => prover-client}/factory.ts | 7 +- .../prover-client/src/prover-client/index.ts | 2 + .../prover-client.ts} | 26 +-- .../src/test/bb_prover_base_rollup.test.ts | 6 +- .../src/test/bb_prover_full_rollup.test.ts | 19 ++- .../src/test/bb_prover_parity.test.ts | 2 +- yarn-project/prover-node/src/config.ts | 21 ++- yarn-project/prover-node/src/factory.ts | 9 +- .../prover-node/src/job/epoch-proving-job.ts | 63 +++---- .../prover-node/src/prover-node.test.ts | 7 +- yarn-project/prover-node/src/prover-node.ts | 26 +-- .../src/block_builder/orchestrator.ts | 43 ----- .../src/client/sequencer-client.ts | 2 +- .../src/sequencer/sequencer.test.ts | 2 +- .../src/sequencer/sequencer.ts | 2 +- .../src/world-state-db/merkle_tree_db.ts | 16 +- 46 files changed, 526 insertions(+), 482 deletions(-) create mode 100644 yarn-project/foundation/src/async-pool/index.ts rename yarn-project/{sequencer-client => prover-client}/src/block_builder/index.ts (85%) rename yarn-project/{sequencer-client => prover-client}/src/block_builder/light.test.ts (99%) rename yarn-project/{sequencer-client => prover-client}/src/block_builder/light.ts (77%) rename yarn-project/prover-client/src/{tx-prover => prover-client}/factory.ts (58%) create mode 100644 yarn-project/prover-client/src/prover-client/index.ts rename yarn-project/prover-client/src/{tx-prover/tx-prover.ts => prover-client/prover-client.ts} (86%) delete mode 100644 yarn-project/sequencer-client/src/block_builder/orchestrator.ts diff --git a/yarn-project/circuit-types/src/interfaces/epoch-prover.ts b/yarn-project/circuit-types/src/interfaces/epoch-prover.ts index 36f5e911b2d..b4bef3658a7 100644 --- a/yarn-project/circuit-types/src/interfaces/epoch-prover.ts +++ b/yarn-project/circuit-types/src/interfaces/epoch-prover.ts @@ -1,25 +1,10 @@ -import { type Fr, type Proof, type RootRollupPublicInputs } from '@aztec/circuits.js'; +import { type Fr, type Header, type Proof, type RootRollupPublicInputs } from '@aztec/circuits.js'; import { type L2Block } from '../l2_block.js'; import { type BlockBuilder } from './block-builder.js'; -/** - * Coordinates the proving of an entire epoch. - * - * Expected usage: - * ``` - * startNewEpoch - * foreach block { - * addNewBlock - * foreach tx { - * addTx - * } - * setBlockCompleted - * } - * finaliseEpoch - * ``` - */ -export interface EpochProver extends BlockBuilder { +/** Coordinates the proving of an entire epoch. */ +export interface EpochProver extends Omit { /** * Starts a new epoch. Must be the first method to be called. * @param epochNumber - The epoch number. @@ -27,6 +12,9 @@ export interface EpochProver extends BlockBuilder { **/ startNewEpoch(epochNumber: number, totalNumBlocks: number): void; + /** Pads the block with empty txs if it hasn't reached the declared number of txs. */ + setBlockCompleted(blockNumber: number, expectedBlockHeader?: Header): Promise; + /** Pads the epoch with empty block roots if needed and blocks until proven. Throws if proving has failed. */ finaliseEpoch(): Promise<{ publicInputs: RootRollupPublicInputs; proof: Proof }>; diff --git a/yarn-project/circuit-types/src/interfaces/prover-client.ts b/yarn-project/circuit-types/src/interfaces/prover-client.ts index bf1ef3d6485..29f8cc4fb53 100644 --- a/yarn-project/circuit-types/src/interfaces/prover-client.ts +++ b/yarn-project/circuit-types/src/interfaces/prover-client.ts @@ -6,7 +6,6 @@ import { z } from 'zod'; import { type TxHash } from '../tx/tx_hash.js'; import { type EpochProver } from './epoch-prover.js'; -import { type MerkleTreeReadOperations } from './merkle_tree_operations.js'; import { type ProvingJobConsumer } from './prover-broker.js'; import { type ProvingJobStatus } from './proving-job.js'; @@ -105,7 +104,7 @@ export interface ProverCache { * Provides the ability to generate proofs and build rollups. */ export interface EpochProverManager { - createEpochProver(db: MerkleTreeReadOperations, cache?: ProverCache): EpochProver; + createEpochProver(cache?: ProverCache): EpochProver; start(): Promise; diff --git a/yarn-project/circuit-types/src/interfaces/world_state.ts b/yarn-project/circuit-types/src/interfaces/world_state.ts index e2d4234da17..4fd93acf259 100644 --- a/yarn-project/circuit-types/src/interfaces/world_state.ts +++ b/yarn-project/circuit-types/src/interfaces/world_state.ts @@ -25,10 +25,17 @@ export interface WorldStateSynchronizerStatus { syncedToL2Block: L2BlockId; } -/** - * Defines the interface for a world state synchronizer. - */ -export interface WorldStateSynchronizer { +/** Provides writeable forks of the world state at a given block number. */ +export interface ForkMerkleTreeOperations { + /** Forks the world state at the given block number, defaulting to the latest one. */ + fork(block?: number): Promise; + + /** Gets a handle that allows reading the state as it was at the given block number. */ + getSnapshot(blockNumber: number): MerkleTreeReadOperations; +} + +/** Defines the interface for a world state synchronizer. */ +export interface WorldStateSynchronizer extends ForkMerkleTreeOperations { /** * Starts the synchronizer. * @returns A promise that resolves once the initial sync is completed. @@ -53,19 +60,8 @@ export interface WorldStateSynchronizer { */ syncImmediate(minBlockNumber?: number): Promise; - /** - * Forks the current in-memory state based off the current committed state, and returns an instance that cannot modify the underlying data store. - */ - fork(block?: number): Promise; - /** * Returns an instance of MerkleTreeAdminOperations that will not include uncommitted data. */ getCommitted(): MerkleTreeReadOperations; - - /** - * Returns a readonly instance of MerkleTreeAdminOperations where the state is as it was at the given block number - * @param block - The block number to look at - */ - getSnapshot(block: number): MerkleTreeReadOperations; } diff --git a/yarn-project/circuit-types/src/test/factories.ts b/yarn-project/circuit-types/src/test/factories.ts index 1a90a045171..72e2c318edf 100644 --- a/yarn-project/circuit-types/src/test/factories.ts +++ b/yarn-project/circuit-types/src/test/factories.ts @@ -53,14 +53,7 @@ export function makeBloatedProcessedTx({ privateOnly?: boolean; } = {}) { seed *= 0x1000; // Avoid clashing with the previous mock values if seed only increases by 1. - - if (!header) { - if (db) { - header = db.getInitialHeader(); - } else { - header = makeHeader(seed); - } - } + header ??= db?.getInitialHeader() ?? makeHeader(seed); const txConstantData = TxConstantData.empty(); txConstantData.historicalHeader = header; diff --git a/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts b/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts index f64f23b9ab5..c2b77cffe2f 100644 --- a/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts +++ b/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts @@ -25,6 +25,7 @@ import { OutboxAbi, RollupAbi } from '@aztec/l1-artifacts'; import { SHA256Trunc, StandardTree } from '@aztec/merkle-tree'; import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types'; import { protocolContractTreeRoot } from '@aztec/protocol-contracts'; +import { LightweightBlockBuilder } from '@aztec/prover-client/block-builder'; import { L1Publisher } from '@aztec/sequencer-client'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { @@ -52,7 +53,6 @@ import { } from 'viem'; import { type PrivateKeyAccount, privateKeyToAccount } from 'viem/accounts'; -import { LightweightBlockBuilder } from '../../../sequencer-client/src/block_builder/light.js'; import { sendL1ToL2Message } from '../fixtures/l1_to_l2_messaging.js'; import { setupL1Contracts } from '../fixtures/utils.js'; diff --git a/yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts b/yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts index 1d200dc9c4c..dd19f122d0a 100644 --- a/yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts +++ b/yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts @@ -269,6 +269,7 @@ export class FullProverTest { proverAgentCount: 2, publisherPrivateKey: `0x${proverNodePrivateKey!.toString('hex')}`, proverNodeMaxPendingJobs: 100, + proverNodeMaxParallelBlocksPerEpoch: 32, proverNodePollingIntervalMs: 100, quoteProviderBasisPointFee: 100, quoteProviderBondAmount: 1000n, diff --git a/yarn-project/end-to-end/src/fixtures/utils.ts b/yarn-project/end-to-end/src/fixtures/utils.ts index 2ff469f815f..db1825bb89c 100644 --- a/yarn-project/end-to-end/src/fixtures/utils.ts +++ b/yarn-project/end-to-end/src/fixtures/utils.ts @@ -682,6 +682,7 @@ export async function createAndSyncProverNode( proverAgentCount: 2, publisherPrivateKey: proverNodePrivateKey, proverNodeMaxPendingJobs: 10, + proverNodeMaxParallelBlocksPerEpoch: 32, proverNodePollingIntervalMs: 200, quoteProviderBasisPointFee: 100, quoteProviderBondAmount: 1000n, diff --git a/yarn-project/foundation/package.json b/yarn-project/foundation/package.json index 46cc1d6d96b..149a879c05c 100644 --- a/yarn-project/foundation/package.json +++ b/yarn-project/foundation/package.json @@ -11,6 +11,7 @@ "./prettier": "./.prettierrc.json", "./abi": "./dest/abi/index.js", "./async-map": "./dest/async-map/index.js", + "./async-pool": "./dest/async-pool/index.js", "./aztec-address": "./dest/aztec-address/index.js", "./collection": "./dest/collection/index.js", "./config": "./dest/config/index.js", @@ -163,4 +164,4 @@ "engines": { "node": ">=18" } -} +} \ No newline at end of file diff --git a/yarn-project/foundation/src/async-pool/index.ts b/yarn-project/foundation/src/async-pool/index.ts new file mode 100644 index 00000000000..67e070933bc --- /dev/null +++ b/yarn-project/foundation/src/async-pool/index.ts @@ -0,0 +1,50 @@ +/* + * Adapted from https://github.com/rxaviers/async-pool/blob/1.x/lib/es6.js + * + * Copyright (c) 2017 Rafael Xavier de Souza http://rafael.xavier.blog.br + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/** Executes the given async function over the iterable, up to a determined number of promises in parallel. */ +export function asyncPool(poolLimit: number, iterable: T[], iteratorFn: (item: T, iterable: T[]) => Promise) { + let i = 0; + const ret: Promise[] = []; + const executing: Set> = new Set(); + const enqueue = (): Promise => { + if (i === iterable.length) { + return Promise.resolve(); + } + const item = iterable[i++]; + const p = Promise.resolve().then(() => iteratorFn(item, iterable)); + ret.push(p); + executing.add(p); + const clean = () => executing.delete(p); + p.then(clean).catch(clean); + let r: Promise = Promise.resolve(); + if (executing.size >= poolLimit) { + r = Promise.race(executing); + } + return r.then(() => enqueue()); + }; + return enqueue().then(() => Promise.all(ret)); +} diff --git a/yarn-project/foundation/src/collection/array.test.ts b/yarn-project/foundation/src/collection/array.test.ts index 97bee2fd7f1..e3be69ff586 100644 --- a/yarn-project/foundation/src/collection/array.test.ts +++ b/yarn-project/foundation/src/collection/array.test.ts @@ -1,4 +1,4 @@ -import { compactArray, removeArrayPaddingEnd, times, unique } from './array.js'; +import { compactArray, maxBy, removeArrayPaddingEnd, times, unique } from './array.js'; describe('times', () => { it('should return an array with the result from all executions', () => { @@ -61,3 +61,21 @@ describe('unique', () => { expect(unique([1n, 2n, 1n])).toEqual([1n, 2n]); }); }); + +describe('maxBy', () => { + it('returns the max value', () => { + expect(maxBy([1, 2, 3], x => x)).toEqual(3); + }); + + it('returns the first max value', () => { + expect(maxBy([{ a: 1 }, { a: 3, b: 1 }, { a: 3, b: 2 }], ({ a }) => a)).toEqual({ a: 3, b: 1 }); + }); + + it('returns undefined for an empty array', () => { + expect(maxBy([], x => x)).toBeUndefined(); + }); + + it('applies the mapping function', () => { + expect(maxBy([1, 2, 3], x => -x)).toEqual(1); + }); +}); diff --git a/yarn-project/foundation/src/collection/array.ts b/yarn-project/foundation/src/collection/array.ts index ea97385aaba..9f37779727e 100644 --- a/yarn-project/foundation/src/collection/array.ts +++ b/yarn-project/foundation/src/collection/array.ts @@ -75,6 +75,20 @@ export function times(n: number, fn: (i: number) => T): T[] { return [...Array(n).keys()].map(i => fn(i)); } +/** + * Executes the given async function n times and returns the results in an array. Awaits each execution before starting the next one. + * @param n - How many times to repeat. + * @param fn - Mapper from index to value. + * @returns The array with the result from all executions. + */ +export async function timesAsync(n: number, fn: (i: number) => Promise): Promise { + const results: T[] = []; + for (let i = 0; i < n; i++) { + results.push(await fn(i)); + } + return results; +} + /** * Returns the serialized size of all non-empty items in an array. * @param arr - Array @@ -121,3 +135,13 @@ export function areArraysEqual(a: T[], b: T[], eq: (a: T, b: T) => boolean = } return true; } + +/** + * Returns the element of the array that has the maximum value of the given function. + * In case of a tie, returns the first element with the maximum value. + * @param arr - The array. + * @param fn - The function to get the value to compare. + */ +export function maxBy(arr: T[], fn: (x: T) => number): T | undefined { + return arr.reduce((max, x) => (fn(x) > fn(max) ? x : max), arr[0]); +} diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts index ce7c17fb3ef..56d5fcd9045 100644 --- a/yarn-project/foundation/src/config/env_var.ts +++ b/yarn-project/foundation/src/config/env_var.ts @@ -115,6 +115,7 @@ export type EnvVar = | 'PROVER_JOB_TIMEOUT_MS' | 'PROVER_NODE_POLLING_INTERVAL_MS' | 'PROVER_NODE_MAX_PENDING_JOBS' + | 'PROVER_NODE_MAX_PARALLEL_BLOCKS_PER_EPOCH' | 'PROVER_PUBLISH_RETRY_INTERVAL_MS' | 'PROVER_PUBLISHER_PRIVATE_KEY' | 'PROVER_REAL_PROOFS' diff --git a/yarn-project/prover-client/package.json b/yarn-project/prover-client/package.json index 4ee446b70af..65012f1a178 100644 --- a/yarn-project/prover-client/package.json +++ b/yarn-project/prover-client/package.json @@ -4,6 +4,7 @@ "type": "module", "exports": { ".": "./dest/index.js", + "./block-builder": "./dest/block_builder/index.js", "./broker": "./dest/proving_broker/index.js", "./prover-agent": "./dest/prover-agent/index.js", "./orchestrator": "./dest/orchestrator/index.js", @@ -103,4 +104,4 @@ "engines": { "node": ">=18" } -} +} \ No newline at end of file diff --git a/yarn-project/sequencer-client/src/block_builder/index.ts b/yarn-project/prover-client/src/block_builder/index.ts similarity index 85% rename from yarn-project/sequencer-client/src/block_builder/index.ts rename to yarn-project/prover-client/src/block_builder/index.ts index c6c151edcc1..b91a260888b 100644 --- a/yarn-project/sequencer-client/src/block_builder/index.ts +++ b/yarn-project/prover-client/src/block_builder/index.ts @@ -1,6 +1,5 @@ import { type BlockBuilder, type MerkleTreeReadOperations } from '@aztec/circuit-types'; -export * from './orchestrator.js'; export * from './light.js'; export interface BlockBuilderFactory { create(db: MerkleTreeReadOperations): BlockBuilder; diff --git a/yarn-project/sequencer-client/src/block_builder/light.test.ts b/yarn-project/prover-client/src/block_builder/light.test.ts similarity index 99% rename from yarn-project/sequencer-client/src/block_builder/light.test.ts rename to yarn-project/prover-client/src/block_builder/light.test.ts index 76f4714823a..de35c68e72f 100644 --- a/yarn-project/sequencer-client/src/block_builder/light.test.ts +++ b/yarn-project/prover-client/src/block_builder/light.test.ts @@ -47,18 +47,18 @@ import { getVKTreeRoot, } from '@aztec/noir-protocol-circuits-types'; import { protocolContractTreeRoot } from '@aztec/protocol-contracts'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; +import { type MerkleTreeAdminDatabase, NativeWorldStateService } from '@aztec/world-state'; + +import { jest } from '@jest/globals'; + import { buildBaseRollupHints, buildHeaderFromCircuitOutputs, getRootTreeSiblingPath, getSubtreeSiblingPath, getTreeSnapshot, -} from '@aztec/prover-client/helpers'; -import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; -import { type MerkleTreeAdminDatabase, NativeWorldStateService } from '@aztec/world-state'; - -import { jest } from '@jest/globals'; - +} from '../orchestrator/block-building-helpers.js'; import { LightweightBlockBuilder } from './light.js'; jest.setTimeout(50_000); diff --git a/yarn-project/sequencer-client/src/block_builder/light.ts b/yarn-project/prover-client/src/block_builder/light.ts similarity index 77% rename from yarn-project/sequencer-client/src/block_builder/light.ts rename to yarn-project/prover-client/src/block_builder/light.ts index 4087b1623bb..3bc5d4a299d 100644 --- a/yarn-project/sequencer-client/src/block_builder/light.ts +++ b/yarn-project/prover-client/src/block_builder/light.ts @@ -1,4 +1,3 @@ -import { createDebugLogger } from '@aztec/aztec.js'; import { type BlockBuilder, L2Block, @@ -9,14 +8,20 @@ import { } from '@aztec/circuit-types'; import { Fr, type GlobalVariables, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/circuits.js'; import { padArrayEnd } from '@aztec/foundation/collection'; +import { createDebugLogger } from '@aztec/foundation/log'; import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types'; import { protocolContractTreeRoot } from '@aztec/protocol-contracts'; -import { buildBaseRollupHints, buildHeaderAndBodyFromTxs, getTreeSnapshot } from '@aztec/prover-client/helpers'; import { type TelemetryClient } from '@aztec/telemetry-client'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { inspect } from 'util'; +import { + buildBaseRollupHints, + buildHeaderAndBodyFromTxs, + getTreeSnapshot, +} from '../orchestrator/block-building-helpers.js'; + /** * Builds a block and its header from a set of processed tx without running any circuits. */ @@ -90,3 +95,23 @@ export class LightweightBlockBuilderFactory { return new LightweightBlockBuilder(db, this.telemetry ?? new NoopTelemetryClient()); } } + +/** + * Creates a block builder under the hood with the given txs and messages and creates a block. + * Automatically adds padding txs to get to a minimum of 2 txs in the block. + * @param db - A db fork to use for block building. + */ +export async function buildBlock( + txs: ProcessedTx[], + globalVariables: GlobalVariables, + l1ToL2Messages: Fr[], + db: MerkleTreeWriteOperations, + telemetry: TelemetryClient = new NoopTelemetryClient(), +) { + const builder = new LightweightBlockBuilder(db, telemetry); + await builder.startNewBlock(Math.max(txs.length, 2), globalVariables, l1ToL2Messages); + for (const tx of txs) { + await builder.addNewTx(tx); + } + return await builder.setBlockCompleted(); +} diff --git a/yarn-project/prover-client/src/index.ts b/yarn-project/prover-client/src/index.ts index 56f3430e2c6..822b565f54a 100644 --- a/yarn-project/prover-client/src/index.ts +++ b/yarn-project/prover-client/src/index.ts @@ -1,6 +1,5 @@ export { EpochProverManager } from '@aztec/circuit-types'; -export * from './tx-prover/tx-prover.js'; +export * from './prover-client/index.js'; export * from './config.js'; -export * from './tx-prover/factory.js'; export * from './proving_broker/prover_cache/memory.js'; diff --git a/yarn-project/prover-client/src/mocks/fixtures.ts b/yarn-project/prover-client/src/mocks/fixtures.ts index 34b7cee5935..f1fc2c856f3 100644 --- a/yarn-project/prover-client/src/mocks/fixtures.ts +++ b/yarn-project/prover-client/src/mocks/fixtures.ts @@ -1,10 +1,4 @@ -import { - MerkleTreeId, - type MerkleTreeReadOperations, - type MerkleTreeWriteOperations, - type ProcessedTx, -} from '@aztec/circuit-types'; -import { makeBloatedProcessedTx } from '@aztec/circuit-types/test'; +import { MerkleTreeId, type MerkleTreeWriteOperations, type ProcessedTx } from '@aztec/circuit-types'; import { AztecAddress, EthAddress, @@ -19,8 +13,6 @@ import { padArrayEnd } from '@aztec/foundation/collection'; import { randomBytes } from '@aztec/foundation/crypto'; import { type DebugLogger } from '@aztec/foundation/log'; import { fileURLToPath } from '@aztec/foundation/url'; -import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types'; -import { protocolContractTreeRoot } from '@aztec/protocol-contracts'; import { NativeACVMSimulator, type SimulationProvider, WASMSimulator } from '@aztec/simulator'; import * as fs from 'fs/promises'; @@ -94,9 +86,6 @@ export async function getSimulationProvider( return new WASMSimulator(); } -export const makeBloatedProcessedTxWithVKRoot = (builderDb: MerkleTreeReadOperations, seed = 0x1) => - makeBloatedProcessedTx({ db: builderDb, vkTreeRoot: getVKTreeRoot(), protocolContractTreeRoot, seed }); - // Updates the expectedDb trees based on the new note hashes, contracts, and nullifiers from these txs export const updateExpectedTreesFromTxs = async (db: MerkleTreeWriteOperations, txs: ProcessedTx[]) => { await db.appendLeaves( diff --git a/yarn-project/prover-client/src/mocks/test_context.ts b/yarn-project/prover-client/src/mocks/test_context.ts index 764a092e813..e2df1346c11 100644 --- a/yarn-project/prover-client/src/mocks/test_context.ts +++ b/yarn-project/prover-client/src/mocks/test_context.ts @@ -1,6 +1,6 @@ import { type BBProverConfig } from '@aztec/bb-prover'; import { - type MerkleTreeWriteOperations, + type L2Block, type ProcessedTx, type ProcessedTxHandler, type PublicExecutionRequest, @@ -8,10 +8,13 @@ import { type Tx, type TxValidator, } from '@aztec/circuit-types'; -import { type Gas, type GlobalVariables, Header } from '@aztec/circuits.js'; +import { makeBloatedProcessedTx } from '@aztec/circuit-types/test'; +import { type AppendOnlyTreeSnapshot, type Gas, type GlobalVariables, Header } from '@aztec/circuits.js'; +import { times } from '@aztec/foundation/collection'; import { Fr } from '@aztec/foundation/fields'; import { type DebugLogger } from '@aztec/foundation/log'; -import { openTmpStore } from '@aztec/kv-store/utils'; +import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types'; +import { protocolContractTreeRoot } from '@aztec/protocol-contracts'; import { PublicProcessor, PublicTxSimulator, @@ -20,32 +23,34 @@ import { type WorldStateDB, } from '@aztec/simulator'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; -import { MerkleTrees } from '@aztec/world-state'; +import { type MerkleTreeAdminDatabase } from '@aztec/world-state'; import { NativeWorldStateService } from '@aztec/world-state/native'; import { jest } from '@jest/globals'; import * as fs from 'fs/promises'; -import { type MockProxy, mock } from 'jest-mock-extended'; +import { mock } from 'jest-mock-extended'; import { TestCircuitProver } from '../../../bb-prover/src/test/test_circuit_prover.js'; import { AvmFinalizedCallResult } from '../../../simulator/src/avm/avm_contract_call_result.js'; import { type AvmPersistableStateManager } from '../../../simulator/src/avm/journal/journal.js'; +import { buildBlock } from '../block_builder/light.js'; import { ProvingOrchestrator } from '../orchestrator/index.js'; import { MemoryProvingQueue } from '../prover-agent/memory-proving-queue.js'; import { ProverAgent } from '../prover-agent/prover-agent.js'; import { getEnvironmentConfig, getSimulationProvider, makeGlobals } from './fixtures.js'; export class TestContext { + private headers: Map = new Map(); + constructor( public publicTxSimulator: PublicTxSimulator, - public worldStateDB: MockProxy, + public worldState: MerkleTreeAdminDatabase, public publicProcessor: PublicProcessor, public simulationProvider: SimulationProvider, public globalVariables: GlobalVariables, - public actualDb: MerkleTreeWriteOperations, public prover: ServerCircuitProver, public proverAgent: ProverAgent, - public orchestrator: ProvingOrchestrator, + public orchestrator: TestProvingOrchestrator, public blockNumber: number, public directoriesToCleanup: string[], public logger: DebugLogger, @@ -57,11 +62,10 @@ export class TestContext { static async new( logger: DebugLogger, - worldState: 'native' | 'legacy' = 'native', proverCount = 4, createProver: (bbConfig: BBProverConfig) => Promise = _ => Promise.resolve(new TestCircuitProver(new NoopTelemetryClient(), new WASMSimulator())), - blockNumber = 3, + blockNumber = 1, ) { const directoriesToCleanup: string[] = []; const globalVariables = makeGlobals(blockNumber); @@ -70,18 +74,9 @@ export class TestContext { const telemetry = new NoopTelemetryClient(); // Separated dbs for public processor and prover - see public_processor for context - let publicDb: MerkleTreeWriteOperations; - let proverDb: MerkleTreeWriteOperations; + const ws = await NativeWorldStateService.tmp(); + const publicDb = await ws.fork(); - if (worldState === 'native') { - const ws = await NativeWorldStateService.tmp(); - publicDb = await ws.fork(); - proverDb = await ws.fork(); - } else { - const ws = await MerkleTrees.new(openTmpStore(), telemetry); - publicDb = await ws.getLatest(); - proverDb = await ws.getLatest(); - } worldStateDB.getMerkleInterface.mockReturnValue(publicDb); const publicTxSimulator = new PublicTxSimulator(publicDb, worldStateDB, telemetry, globalVariables); @@ -118,7 +113,7 @@ export class TestContext { } const queue = new MemoryProvingQueue(telemetry); - const orchestrator = new ProvingOrchestrator(proverDb, queue, telemetry, Fr.ZERO); + const orchestrator = new TestProvingOrchestrator(ws, queue, telemetry, Fr.ZERO); const agent = new ProverAgent(localProver, proverCount); queue.start(); @@ -126,11 +121,10 @@ export class TestContext { return new this( publicTxSimulator, - worldStateDB, + ws, processor, simulationProvider, globalVariables, - proverDb, localProver, agent, orchestrator, @@ -140,6 +134,16 @@ export class TestContext { ); } + public getFork() { + return this.worldState.fork(); + } + + public getHeader(blockNumber: 0): Header; + public getHeader(blockNumber: number): Header | undefined; + public getHeader(blockNumber = 0) { + return blockNumber === 0 ? this.worldState.getCommitted().getInitialHeader() : this.headers.get(blockNumber); + } + async cleanup() { await this.proverAgent.stop(); for (const dir of this.directoriesToCleanup.filter(x => x !== '')) { @@ -147,6 +151,42 @@ export class TestContext { } } + public makeProcessedTx(opts?: Parameters[0]): ProcessedTx; + public makeProcessedTx(seed?: number): ProcessedTx; + public makeProcessedTx(seedOrOpts?: Parameters[0] | number): ProcessedTx { + const opts = typeof seedOrOpts === 'number' ? { seed: seedOrOpts } : seedOrOpts; + const blockNum = (opts?.globalVariables ?? this.globalVariables).blockNumber.toNumber(); + const header = this.getHeader(blockNum - 1); + return makeBloatedProcessedTx({ + header, + vkTreeRoot: getVKTreeRoot(), + protocolContractTreeRoot, + globalVariables: this.globalVariables, + ...opts, + }); + } + + /** Creates a block with the given number of txs and adds it to world-state */ + public async makePendingBlock( + numTxs: number, + numMsgs: number = 0, + blockNumOrGlobals: GlobalVariables | number = this.globalVariables, + makeProcessedTxOpts: (index: number) => Partial[0]> = () => ({}), + ) { + const globalVariables = typeof blockNumOrGlobals === 'number' ? makeGlobals(blockNumOrGlobals) : blockNumOrGlobals; + const blockNum = globalVariables.blockNumber.toNumber(); + const db = await this.worldState.fork(); + const msgs = times(numMsgs, i => new Fr(blockNum * 100 + i)); + const txs = times(numTxs, i => + this.makeProcessedTx({ seed: i + blockNum * 1000, globalVariables, ...makeProcessedTxOpts(i) }), + ); + + const block = await buildBlock(txs, globalVariables, msgs, db); + this.headers.set(blockNum, block.header); + await this.worldState.handleL2BlockAndMessages(block, msgs); + return { block, txs, msgs }; + } + public async processPublicFunctions( txs: Tx[], maxTransactions: number, @@ -217,3 +257,19 @@ export class TestContext { return await this.publicProcessor.process(txs, maxTransactions, txHandler, txValidator); } } + +class TestProvingOrchestrator extends ProvingOrchestrator { + public isVerifyBuiltBlockAgainstSyncedStateEnabled = false; + + // Disable this check by default, since it requires seeding world state with the block being built + // This is only enabled in some tests with multiple blocks that populate the pending chain via makePendingBlock + protected override verifyBuiltBlockAgainstSyncedState( + l2Block: L2Block, + newArchive: AppendOnlyTreeSnapshot, + ): Promise { + if (this.isVerifyBuiltBlockAgainstSyncedStateEnabled) { + return super.verifyBuiltBlockAgainstSyncedState(l2Block, newArchive); + } + return Promise.resolve(); + } +} diff --git a/yarn-project/prover-client/src/orchestrator/epoch-proving-state.ts b/yarn-project/prover-client/src/orchestrator/epoch-proving-state.ts index a13a8d600dc..522d5ba9d70 100644 --- a/yarn-project/prover-client/src/orchestrator/epoch-proving-state.ts +++ b/yarn-project/prover-client/src/orchestrator/epoch-proving-state.ts @@ -59,11 +59,6 @@ export class EpochProvingState { private rejectionCallback: (reason: string) => void, ) {} - /** Returns the current block proving state */ - public get currentBlock(): BlockProvingState | undefined { - return this.blocks.at(-1); - } - // Returns the number of levels of merge rollups public get numMergeLevels() { const totalLeaves = Math.max(2, this.totalNumBlocks); @@ -110,7 +105,7 @@ export class EpochProvingState { archiveTreeSnapshot: AppendOnlyTreeSnapshot, archiveTreeRootSiblingPath: Tuple, previousBlockHash: Fr, - ) { + ): BlockProvingState { const block = new BlockProvingState( this.blocks.length, numTxs, @@ -128,7 +123,7 @@ export class EpochProvingState { if (this.blocks.length === this.totalNumBlocks) { this.provingStateLifecycle = PROVING_STATE_LIFECYCLE.PROVING_STATE_FULL; } - return this.blocks.length - 1; + return block; } // Returns true if this proving state is still valid, false otherwise @@ -180,8 +175,8 @@ export class EpochProvingState { } // Returns a specific transaction proving state - public getBlockProvingState(index: number) { - return this.blocks[index]; + public getBlockProvingStateByBlockNumber(blockNumber: number) { + return this.blocks.find(block => block.blockNumber === blockNumber); } // Returns a set of merge rollup inputs diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.ts index fdf607298e2..156440b3fd2 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.ts @@ -7,6 +7,7 @@ import { } from '@aztec/circuit-types'; import { type EpochProver, + type ForkMerkleTreeOperations, type MerkleTreeWriteOperations, type ProofAndVerificationKey, } from '@aztec/circuit-types/interfaces'; @@ -14,6 +15,7 @@ import { type CircuitName } from '@aztec/circuit-types/stats'; import { AVM_PROOF_LENGTH_IN_FIELDS, AVM_VERIFICATION_KEY_LENGTH_IN_FIELDS, + type AppendOnlyTreeSnapshot, type BaseOrMergeRollupPublicInputs, BaseParityInputs, type BaseRollupHints, @@ -38,7 +40,7 @@ import { makeEmptyRecursiveProof, } from '@aztec/circuits.js'; import { makeTuple } from '@aztec/foundation/array'; -import { padArrayEnd } from '@aztec/foundation/collection'; +import { maxBy, padArrayEnd } from '@aztec/foundation/collection'; import { AbortError } from '@aztec/foundation/error'; import { createDebugLogger } from '@aztec/foundation/log'; import { promiseWithResolvers } from '@aztec/foundation/promise'; @@ -98,9 +100,10 @@ export class ProvingOrchestrator implements EpochProver { private provingPromise: Promise | undefined = undefined; private metrics: ProvingOrchestratorMetrics; + private dbs: Map = new Map(); constructor( - private db: MerkleTreeWriteOperations, + private dbProvider: ForkMerkleTreeOperations, private prover: ServerCircuitProver, telemetryClient: TelemetryClient, private readonly proverId: Fr = Fr.ZERO, @@ -159,24 +162,14 @@ export class ProvingOrchestrator implements EpochProver { throw new Error(`Invalid number of txs for block (got ${numTxs})`); } - if (this.provingState.currentBlock && !this.provingState.currentBlock.block) { - throw new Error(`Must end previous block before starting a new one`); - } - - // TODO(palla/prover): Store block number in the db itself to make this check more reliable, - // and turn this warning into an exception that we throw. - const { blockNumber } = globalVariables; - const dbBlockNumber = (await this.db.getTreeInfo(MerkleTreeId.ARCHIVE)).size - 1n; - if (dbBlockNumber !== blockNumber.toBigInt() - 1n) { - logger.warn( - `Database is at wrong block number (starting block ${blockNumber.toBigInt()} with db at ${dbBlockNumber})`, - ); - } - logger.info( - `Starting block ${globalVariables.blockNumber} for slot ${globalVariables.slotNumber} with ${numTxs} transactions`, + `Starting block ${globalVariables.blockNumber.toNumber()} for slot ${globalVariables.slotNumber.toNumber()} with ${numTxs} transactions`, ); + // Fork world state at the end of the immediately previous block + const db = await this.dbProvider.fork(globalVariables.blockNumber.toNumber() - 1); + this.dbs.set(globalVariables.blockNumber.toNumber(), db); + // we start the block by enqueueing all of the base parity circuits let baseParityInputs: BaseParityInputs[] = []; let l1ToL2MessagesPadded: Tuple; @@ -189,12 +182,12 @@ export class ProvingOrchestrator implements EpochProver { BaseParityInputs.fromSlice(l1ToL2MessagesPadded, i, getVKTreeRoot()), ); - const messageTreeSnapshot = await getTreeSnapshot(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, this.db); + const messageTreeSnapshot = await getTreeSnapshot(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, db); const newL1ToL2MessageTreeRootSiblingPathArray = await getSubtreeSiblingPath( MerkleTreeId.L1_TO_L2_MESSAGE_TREE, L1_TO_L2_MSG_SUBTREE_HEIGHT, - this.db, + db, ); const newL1ToL2MessageTreeRootSiblingPath = makeTuple( @@ -205,18 +198,18 @@ export class ProvingOrchestrator implements EpochProver { ); // Update the local trees to include the new l1 to l2 messages - await this.db.appendLeaves(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, l1ToL2MessagesPadded); - const messageTreeSnapshotAfterInsertion = await getTreeSnapshot(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, this.db); + await db.appendLeaves(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, l1ToL2MessagesPadded); + const messageTreeSnapshotAfterInsertion = await getTreeSnapshot(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, db); // Get archive snapshot before this block lands - const startArchiveSnapshot = await getTreeSnapshot(MerkleTreeId.ARCHIVE, this.db); - const newArchiveSiblingPath = await getRootTreeSiblingPath(MerkleTreeId.ARCHIVE, this.db); - const previousBlockHash = await this.db.getLeafValue( + const startArchiveSnapshot = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db); + const newArchiveSiblingPath = await getRootTreeSiblingPath(MerkleTreeId.ARCHIVE, db); + const previousBlockHash = await db.getLeafValue( MerkleTreeId.ARCHIVE, BigInt(startArchiveSnapshot.nextAvailableLeafIndex - 1), ); - this.provingState!.startNewBlock( + const blockProvingState = this.provingState!.startNewBlock( numTxs, globalVariables, l1ToL2MessagesPadded, @@ -230,7 +223,7 @@ export class ProvingOrchestrator implements EpochProver { // Enqueue base parity circuits for the block for (let i = 0; i < baseParityInputs.length; i++) { - this.enqueueBaseParityCircuit(this.provingState!.currentBlock!, baseParityInputs[i], i); + this.enqueueBaseParityCircuit(blockProvingState, baseParityInputs[i], i); } } @@ -242,33 +235,40 @@ export class ProvingOrchestrator implements EpochProver { [Attributes.TX_HASH]: tx.hash.toString(), })) public async addNewTx(tx: ProcessedTx): Promise { - const provingState = this?.provingState?.currentBlock; - if (!provingState) { - throw new Error(`Invalid proving state, call startNewBlock before adding transactions`); - } + const blockNumber = tx.constants.globalVariables.blockNumber.toNumber(); + try { + const provingState = this.provingState?.getBlockProvingStateByBlockNumber(blockNumber); + if (!provingState) { + throw new Error(`Block proving state for ${blockNumber} not found`); + } - if (!provingState.isAcceptingTransactions()) { - throw new Error(`Rollup not accepting further transactions`); - } + if (!provingState.isAcceptingTransactions()) { + throw new Error(`Rollup not accepting further transactions`); + } - if (!provingState.verifyState()) { - throw new Error(`Invalid proving state when adding a tx`); - } + if (!provingState.verifyState()) { + throw new Error(`Invalid proving state when adding a tx`); + } - validateTx(tx); + validateTx(tx); - logger.info(`Received transaction: ${tx.hash}`); + logger.info(`Received transaction: ${tx.hash}`); - if (tx.isEmpty) { - logger.warn(`Ignoring empty transaction ${tx.hash} - it will not be added to this block`); - return; - } + if (tx.isEmpty) { + logger.warn(`Ignoring empty transaction ${tx.hash} - it will not be added to this block`); + return; + } - const [hints, treeSnapshots] = await this.prepareTransaction(tx, provingState); - this.enqueueFirstProofs(hints, treeSnapshots, tx, provingState); + const [hints, treeSnapshots] = await this.prepareTransaction(tx, provingState); + this.enqueueFirstProofs(hints, treeSnapshots, tx, provingState); - if (provingState.transactionsReceived === provingState.totalNumTxs) { - logger.verbose(`All transactions received for block ${provingState.globalVariables.blockNumber}.`); + if (provingState.transactionsReceived === provingState.totalNumTxs) { + logger.verbose(`All transactions received for block ${provingState.globalVariables.blockNumber}.`); + } + } catch (err: any) { + throw new Error(`Error adding transaction ${tx.hash.toString()} to block ${blockNumber}: ${err.message}`, { + cause: err, + }); } } @@ -276,21 +276,13 @@ export class ProvingOrchestrator implements EpochProver { * Marks the block as full and pads it if required, no more transactions will be accepted. * Computes the block header and updates the archive tree. */ - @trackSpan('ProvingOrchestrator.setBlockCompleted', function () { - const block = this.provingState?.currentBlock; - if (!block) { - return {}; - } - return { - [Attributes.BLOCK_NUMBER]: block.globalVariables.blockNumber.toNumber(), - [Attributes.BLOCK_SIZE]: block.totalNumTxs, - [Attributes.BLOCK_TXS_COUNT]: block.transactionsReceived, - }; - }) - public async setBlockCompleted(expectedHeader?: Header): Promise { - const provingState = this.provingState?.currentBlock; + @trackSpan('ProvingOrchestrator.setBlockCompleted', (blockNumber: number) => ({ + [Attributes.BLOCK_NUMBER]: blockNumber, + })) + public async setBlockCompleted(blockNumber: number, expectedHeader?: Header): Promise { + const provingState = this.provingState?.getBlockProvingStateByBlockNumber(blockNumber); if (!provingState) { - throw new Error(`Invalid proving state, call startNewBlock before adding transactions or completing the block`); + throw new Error(`Block proving state for ${blockNumber} not found`); } if (!provingState.verifyState()) { @@ -313,7 +305,7 @@ export class ProvingOrchestrator implements EpochProver { // base rollup inputs // Then enqueue the proving of all the transactions const unprovenPaddingTx = makeEmptyProcessedTx( - this.db.getInitialHeader(), + this.dbs.get(blockNumber)!.getInitialHeader(), provingState.globalVariables.chainId, provingState.globalVariables.version, getVKTreeRoot(), @@ -362,7 +354,7 @@ export class ProvingOrchestrator implements EpochProver { }) private padEpoch(): Promise { const provingState = this.provingState!; - const lastBlock = provingState.currentBlock?.block; + const lastBlock = maxBy(provingState.blocks, b => b.blockNumber)?.block; if (!lastBlock) { return Promise.reject(new Error(`Epoch needs at least one completed block in order to be padded`)); } @@ -416,13 +408,16 @@ export class ProvingOrchestrator implements EpochProver { // Collect all new nullifiers, commitments, and contracts from all txs in this block to build body const txs = provingState!.allTxs.map(a => a.processedTx); + // Get db for this block + const db = this.dbs.get(provingState.blockNumber)!; + // Given we've applied every change from this block, now assemble the block header // and update the archive tree, so we're ready to start processing the next block const { header, body } = await buildHeaderAndBodyFromTxs( txs, provingState.globalVariables, provingState.newL1ToL2Messages, - this.db, + db, ); if (expectedHeader && !header.equals(expectedHeader)) { @@ -431,10 +426,10 @@ export class ProvingOrchestrator implements EpochProver { } logger.verbose(`Updating archive tree with block ${provingState.blockNumber} header ${header.hash().toString()}`); - await this.db.updateArchive(header); + await db.updateArchive(header); // Assemble the L2 block - const newArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, this.db); + const newArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db); const l2Block = new L2Block(newArchive, header, body); if (!l2Block.body.getTxsEffectsHash().equals(header.contentCommitment.txsEffectsHash)) { @@ -445,10 +440,24 @@ export class ProvingOrchestrator implements EpochProver { ); } + await this.verifyBuiltBlockAgainstSyncedState(l2Block, newArchive); + logger.verbose(`Orchestrator finalised block ${l2Block.number}`); provingState.block = l2Block; } + // Flagged as protected to disable in certain unit tests + protected async verifyBuiltBlockAgainstSyncedState(l2Block: L2Block, newArchive: AppendOnlyTreeSnapshot) { + const syncedArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, this.dbProvider.getSnapshot(l2Block.number)); + if (!syncedArchive.equals(newArchive)) { + throw new Error( + `Archive tree mismatch for block ${l2Block.number}: world state synced to ${inspect( + syncedArchive, + )} but built ${inspect(newArchive)}`, + ); + } + } + // Enqueues the proving of the required padding transactions // If the fully proven padding transaction is not available, this will first be proven private enqueuePaddingTxs( @@ -685,9 +694,11 @@ export class ProvingOrchestrator implements EpochProver { return; } + const db = this.dbs.get(provingState.blockNumber)!; + // We build the base rollup inputs using a mock proof and verification key. // These will be overwritten later once we have proven the tube circuit and any public kernels - const [ms, hints] = await elapsed(buildBaseRollupHints(tx, provingState.globalVariables, this.db)); + const [ms, hints] = await elapsed(buildBaseRollupHints(tx, provingState.globalVariables, db)); if (!tx.isEmpty) { this.metrics.recordBaseRollupInputs(ms); @@ -695,7 +706,7 @@ export class ProvingOrchestrator implements EpochProver { const promises = [MerkleTreeId.NOTE_HASH_TREE, MerkleTreeId.NULLIFIER_TREE, MerkleTreeId.PUBLIC_DATA_TREE].map( async (id: MerkleTreeId) => { - return { key: id, value: await getTreeSnapshot(id, this.db) }; + return { key: id, value: await getTreeSnapshot(id, db) }; }, ); const treeSnapshots: TreeSnapshots = new Map((await Promise.all(promises)).map(obj => [obj.key, obj.value])); @@ -1048,6 +1059,19 @@ export class ProvingOrchestrator implements EpochProver { logger.debug('Block root rollup already started'); return; } + const blockNumber = provingState.blockNumber; + + // TODO(palla/prover): This closes the fork only on the happy path. If this epoch orchestrator + // is aborted and never reaches this point, it will leak the fork. We need to add a global cleanup, + // but have to make sure it only runs once all operations are completed, otherwise some function here + // will attempt to access the fork after it was closed. + logger.debug(`Cleaning up world state fork for ${blockNumber}`); + void this.dbs + .get(blockNumber) + ?.close() + .then(() => this.dbs.delete(blockNumber)) + .catch(err => logger.error(`Error closing db for block ${blockNumber}`, err)); + this.enqueueBlockRootRollup(provingState); } diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_errors.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_errors.test.ts index 56bb5996868..f1a9374e949 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_errors.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_errors.test.ts @@ -1,24 +1,19 @@ -import { makeEmptyProcessedTx } from '@aztec/circuit-types'; import { Fr } from '@aztec/circuits.js'; +import { times } from '@aztec/foundation/collection'; import { createDebugLogger } from '@aztec/foundation/log'; -import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types'; -import { protocolContractTreeRoot } from '@aztec/protocol-contracts'; -import { makeBloatedProcessedTxWithVKRoot } from '../mocks/fixtures.js'; import { TestContext } from '../mocks/test_context.js'; +import { type ProvingOrchestrator } from './orchestrator.js'; const logger = createDebugLogger('aztec:orchestrator-errors'); describe('prover/orchestrator/errors', () => { let context: TestContext; - - const makeEmptyProcessedTestTx = () => { - const header = context.actualDb.getInitialHeader(); - return makeEmptyProcessedTx(header, Fr.ZERO, Fr.ZERO, getVKTreeRoot(), protocolContractTreeRoot); - }; + let orchestrator: ProvingOrchestrator; beforeEach(async () => { context = await TestContext.new(logger); + orchestrator = context.orchestrator; }); afterEach(async () => { @@ -29,73 +24,68 @@ describe('prover/orchestrator/errors', () => { describe('errors', () => { it('throws if adding too many transactions', async () => { - const txs = [ - makeBloatedProcessedTxWithVKRoot(context.actualDb, 1), - makeBloatedProcessedTxWithVKRoot(context.actualDb, 2), - makeBloatedProcessedTxWithVKRoot(context.actualDb, 3), - makeBloatedProcessedTxWithVKRoot(context.actualDb, 4), - ]; + const txs = times(4, i => context.makeProcessedTx(i + 1)); - context.orchestrator.startNewEpoch(1, 1); - await context.orchestrator.startNewBlock(txs.length, context.globalVariables, []); + orchestrator.startNewEpoch(1, 1); + await orchestrator.startNewBlock(txs.length, context.globalVariables, []); for (const tx of txs) { - await context.orchestrator.addNewTx(tx); + await orchestrator.addNewTx(tx); } - await expect(async () => await context.orchestrator.addNewTx(makeEmptyProcessedTestTx())).rejects.toThrow( - 'Rollup not accepting further transactions', + await expect(async () => await orchestrator.addNewTx(context.makeProcessedTx())).rejects.toThrow( + /Rollup not accepting further transactions/, ); - const block = await context.orchestrator.setBlockCompleted(); + const block = await orchestrator.setBlockCompleted(context.blockNumber); expect(block.number).toEqual(context.blockNumber); - await context.orchestrator.finaliseEpoch(); + await orchestrator.finaliseEpoch(); }); it('throws if adding too many blocks', async () => { - context.orchestrator.startNewEpoch(1, 1); - await context.orchestrator.startNewBlock(2, context.globalVariables, []); - await context.orchestrator.setBlockCompleted(); + orchestrator.startNewEpoch(1, 1); + await orchestrator.startNewBlock(2, context.globalVariables, []); + await orchestrator.setBlockCompleted(context.blockNumber); - await expect( - async () => await context.orchestrator.startNewBlock(2, context.globalVariables, []), - ).rejects.toThrow('Epoch not accepting further blocks'); + await expect(async () => await orchestrator.startNewBlock(2, context.globalVariables, [])).rejects.toThrow( + 'Epoch not accepting further blocks', + ); }); it('throws if adding a transaction before starting epoch', async () => { - await expect(async () => await context.orchestrator.addNewTx(makeEmptyProcessedTestTx())).rejects.toThrow( - `Invalid proving state, call startNewBlock before adding transactions`, + await expect(async () => await orchestrator.addNewTx(context.makeProcessedTx())).rejects.toThrow( + /Block proving state for 1 not found/, ); }); it('throws if adding a transaction before starting block', async () => { - context.orchestrator.startNewEpoch(1, 1); - await expect(async () => await context.orchestrator.addNewTx(makeEmptyProcessedTestTx())).rejects.toThrow( - `Invalid proving state, call startNewBlock before adding transactions`, + orchestrator.startNewEpoch(1, 1); + await expect(async () => await orchestrator.addNewTx(context.makeProcessedTx())).rejects.toThrow( + /Block proving state for 1 not found/, ); }); it('throws if completing a block before start', async () => { - context.orchestrator.startNewEpoch(1, 1); - await expect(async () => await context.orchestrator.setBlockCompleted()).rejects.toThrow( - 'Invalid proving state, call startNewBlock before adding transactions or completing the block', + orchestrator.startNewEpoch(1, 1); + await expect(async () => await orchestrator.setBlockCompleted(context.blockNumber)).rejects.toThrow( + /Block proving state for 1 not found/, ); }); it('throws if setting an incomplete block as completed', async () => { - context.orchestrator.startNewEpoch(1, 1); - await context.orchestrator.startNewBlock(3, context.globalVariables, []); - await expect(async () => await context.orchestrator.setBlockCompleted()).rejects.toThrow( + orchestrator.startNewEpoch(1, 1); + await orchestrator.startNewBlock(3, context.globalVariables, []); + await expect(async () => await orchestrator.setBlockCompleted(context.blockNumber)).rejects.toThrow( `Block not ready for completion: expecting ${3} more transactions.`, ); }); it('throws if adding to a cancelled block', async () => { - context.orchestrator.startNewEpoch(1, 1); - await context.orchestrator.startNewBlock(2, context.globalVariables, []); - context.orchestrator.cancel(); + orchestrator.startNewEpoch(1, 1); + await orchestrator.startNewBlock(2, context.globalVariables, []); + orchestrator.cancel(); - await expect(async () => await context.orchestrator.addNewTx(makeEmptyProcessedTestTx())).rejects.toThrow( + await expect(async () => await orchestrator.addNewTx(context.makeProcessedTx())).rejects.toThrow( 'Invalid proving state when adding a tx', ); }); @@ -103,25 +93,25 @@ describe('prover/orchestrator/errors', () => { it.each([[-4], [0], [1], [8.1]] as const)( 'fails to start a block with %i transactions', async (blockSize: number) => { - context.orchestrator.startNewEpoch(1, 1); + orchestrator.startNewEpoch(1, 1); await expect( - async () => await context.orchestrator.startNewBlock(blockSize, context.globalVariables, []), + async () => await orchestrator.startNewBlock(blockSize, context.globalVariables, []), ).rejects.toThrow(`Invalid number of txs for block (got ${blockSize})`); }, ); it.each([[-4], [0], [8.1]] as const)('fails to start an epoch with %i blocks', (epochSize: number) => { - context.orchestrator.startNewEpoch(1, 1); - expect(() => context.orchestrator.startNewEpoch(1, epochSize)).toThrow( + orchestrator.startNewEpoch(1, 1); + expect(() => orchestrator.startNewEpoch(1, epochSize)).toThrow( `Invalid number of blocks for epoch (got ${epochSize})`, ); }); it('rejects if too many l1 to l2 messages are provided', async () => { const l1ToL2Messages = new Array(100).fill(new Fr(0n)); - context.orchestrator.startNewEpoch(1, 1); + orchestrator.startNewEpoch(1, 1); await expect( - async () => await context.orchestrator.startNewBlock(2, context.globalVariables, l1ToL2Messages), + async () => await orchestrator.startNewBlock(2, context.globalVariables, l1ToL2Messages), ).rejects.toThrow('Too many L1 to L2 messages'); }); }); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_failures.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_failures.test.ts index 40dd1b10901..709f044575f 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_failures.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_failures.test.ts @@ -1,17 +1,12 @@ +import { TestCircuitProver } from '@aztec/bb-prover'; import { type ServerCircuitProver } from '@aztec/circuit-types'; -import { makeBloatedProcessedTx } from '@aztec/circuit-types/test'; -import { Fr } from '@aztec/circuits.js'; -import { times } from '@aztec/foundation/collection'; +import { timesAsync } from '@aztec/foundation/collection'; import { createDebugLogger } from '@aztec/foundation/log'; -import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types'; -import { protocolContractTreeRoot } from '@aztec/protocol-contracts'; import { WASMSimulator } from '@aztec/simulator'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { jest } from '@jest/globals'; -import { TestCircuitProver } from '../../../bb-prover/src/test/test_circuit_prover.js'; -import { makeGlobals } from '../mocks/fixtures.js'; import { TestContext } from '../mocks/test_context.js'; import { ProvingOrchestrator } from './orchestrator.js'; @@ -34,29 +29,20 @@ describe('prover/orchestrator/failures', () => { beforeEach(() => { mockProver = new TestCircuitProver(new NoopTelemetryClient(), new WASMSimulator()); - orchestrator = new ProvingOrchestrator(context.actualDb, mockProver, new NoopTelemetryClient()); + orchestrator = new ProvingOrchestrator(context.worldState, mockProver, new NoopTelemetryClient()); }); const run = async (message: string) => { + // We need at least 3 blocks, 3 txs, and 1 message to ensure all circuits are used + // We generate them and add them as part of the pending chain + const blocks = await timesAsync(3, i => context.makePendingBlock(3, 1, i + 1, j => ({ privateOnly: j === 1 }))); + orchestrator.startNewEpoch(1, 3); - // We need at least 3 blocks and 3 txs to ensure all circuits are used - for (let i = 0; i < 3; i++) { - const globalVariables = makeGlobals(i + 1); - const txs = times(3, j => - makeBloatedProcessedTx({ - db: context.actualDb, - globalVariables, - vkTreeRoot: getVKTreeRoot(), - protocolContractTreeRoot, - seed: i * 10 + j + 1, - privateOnly: j === 1, - }), - ); - const msgs = [new Fr(i + 100)]; + for (const { block, txs, msgs } of blocks) { // these operations could fail if the target circuit fails before adding all blocks or txs try { - await orchestrator.startNewBlock(txs.length, globalVariables, msgs); + await orchestrator.startNewBlock(txs.length, block.header.globalVariables, msgs); let allTxsAdded = true; for (const tx of txs) { try { @@ -68,9 +54,11 @@ describe('prover/orchestrator/failures', () => { } if (!allTxsAdded) { - await expect(orchestrator.setBlockCompleted()).rejects.toThrow(`Block proving failed: ${message}`); + await expect(orchestrator.setBlockCompleted(block.number)).rejects.toThrow( + `Block proving failed: ${message}`, + ); } else { - await orchestrator.setBlockCompleted(); + await orchestrator.setBlockCompleted(block.number); } } catch (err) { break; diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_lifecycle.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_lifecycle.test.ts index ba76c3d0c23..5325d22cd01 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_lifecycle.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_lifecycle.test.ts @@ -1,6 +1,5 @@ import { type ServerCircuitProver } from '@aztec/circuit-types'; import { NUM_BASE_PARITY_PER_ROOT_PARITY } from '@aztec/circuits.js'; -import { makeGlobalVariables } from '@aztec/circuits.js/testing'; import { createDebugLogger } from '@aztec/foundation/log'; import { type PromiseWithResolvers, promiseWithResolvers } from '@aztec/foundation/promise'; import { sleep } from '@aztec/foundation/sleep'; @@ -28,7 +27,7 @@ describe('prover/orchestrator/lifecycle', () => { describe('lifecycle', () => { it('cancels proving requests', async () => { const prover: ServerCircuitProver = new TestCircuitProver(new NoopTelemetryClient()); - const orchestrator = new ProvingOrchestrator(context.actualDb, prover, new NoopTelemetryClient()); + const orchestrator = new ProvingOrchestrator(context.worldState, prover, new NoopTelemetryClient()); const spy = jest.spyOn(prover, 'getBaseParityProof'); const deferredPromises: PromiseWithResolvers[] = []; @@ -39,7 +38,7 @@ describe('prover/orchestrator/lifecycle', () => { }); orchestrator.startNewEpoch(1, 1); - await orchestrator.startNewBlock(2, makeGlobalVariables(1), []); + await orchestrator.startNewBlock(2, context.globalVariables, []); await sleep(1); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_mixed_blocks.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_mixed_blocks.test.ts index e4ebaee303e..986b1f7f0c3 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_mixed_blocks.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_mixed_blocks.test.ts @@ -1,38 +1,27 @@ -import { MerkleTreeId } from '@aztec/circuit-types'; import { NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/circuits.js'; import { fr } from '@aztec/circuits.js/testing'; import { range } from '@aztec/foundation/array'; import { times } from '@aztec/foundation/collection'; import { createDebugLogger } from '@aztec/foundation/log'; -import { type MerkleTreeAdminDatabase } from '@aztec/world-state'; -import { NativeWorldStateService } from '@aztec/world-state/native'; -import { makeBloatedProcessedTxWithVKRoot, updateExpectedTreesFromTxs } from '../mocks/fixtures.js'; import { TestContext } from '../mocks/test_context.js'; const logger = createDebugLogger('aztec:orchestrator-mixed-blocks'); describe('prover/orchestrator/mixed-blocks', () => { let context: TestContext; - let expectsDb: MerkleTreeAdminDatabase; beforeEach(async () => { context = await TestContext.new(logger); - expectsDb = await NativeWorldStateService.tmp(); }); afterEach(async () => { await context.cleanup(); - await expectsDb.close(); }); describe('blocks', () => { it('builds an unbalanced L2 block', async () => { - const txs = [ - makeBloatedProcessedTxWithVKRoot(context.actualDb, 1), - makeBloatedProcessedTxWithVKRoot(context.actualDb, 2), - makeBloatedProcessedTxWithVKRoot(context.actualDb, 3), - ]; + const txs = times(3, i => context.makeProcessedTx(i + 1)); const l1ToL2Messages = range(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 1 + 0x400).map(fr); @@ -42,13 +31,13 @@ describe('prover/orchestrator/mixed-blocks', () => { await context.orchestrator.addNewTx(tx); } - const block = await context.orchestrator.setBlockCompleted(); + const block = await context.orchestrator.setBlockCompleted(context.blockNumber); await context.orchestrator.finaliseEpoch(); expect(block.number).toEqual(context.blockNumber); }); it.each([2, 4, 5, 8] as const)('builds an L2 block with %i bloated txs', async (totalCount: number) => { - const txs = times(totalCount, (i: number) => makeBloatedProcessedTxWithVKRoot(context.actualDb, i + 1)); + const txs = times(totalCount, i => context.makeProcessedTx(i + 1)); const l1ToL2Messages = range(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 1 + 0x400).map(fr); @@ -59,16 +48,9 @@ describe('prover/orchestrator/mixed-blocks', () => { await context.orchestrator.addNewTx(tx); } - const block = await context.orchestrator.setBlockCompleted(); + const block = await context.orchestrator.setBlockCompleted(context.blockNumber); await context.orchestrator.finaliseEpoch(); expect(block.number).toEqual(context.blockNumber); - - const fork = await expectsDb.fork(); - await updateExpectedTreesFromTxs(fork, txs); - const noteHashTreeAfter = await context.actualDb.getTreeInfo(MerkleTreeId.NOTE_HASH_TREE); - - const expectedNoteHashTreeAfter = await fork.getTreeInfo(MerkleTreeId.NOTE_HASH_TREE).then(t => t.root); - expect(noteHashTreeAfter.root).toEqual(expectedNoteHashTreeAfter); }); }); }); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_multi_public_functions.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_multi_public_functions.test.ts index e805a15dd3b..a84f751ec63 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_multi_public_functions.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_multi_public_functions.test.ts @@ -35,7 +35,7 @@ describe('prover/orchestrator/public-functions', () => { }), ); for (const tx of txs) { - tx.data.constants.historicalHeader = context.actualDb.getInitialHeader(); + tx.data.constants.historicalHeader = context.getHeader(0); tx.data.constants.vkTreeRoot = getVKTreeRoot(); tx.data.constants.protocolContractTreeRoot = protocolContractTreeRoot; } @@ -56,7 +56,7 @@ describe('prover/orchestrator/public-functions', () => { await context.orchestrator.addNewTx(tx); } - const block = await context.orchestrator.setBlockCompleted(); + const block = await context.orchestrator.setBlockCompleted(context.blockNumber); await context.orchestrator.finaliseEpoch(); expect(block.number).toEqual(context.blockNumber); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_multiple_blocks.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_multiple_blocks.test.ts index 5919fa383bf..baa3ad189ab 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_multiple_blocks.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_multiple_blocks.test.ts @@ -1,9 +1,6 @@ -import { makeBloatedProcessedTx } from '@aztec/circuit-types/test'; +import { timesAsync } from '@aztec/foundation/collection'; import { createDebugLogger } from '@aztec/foundation/log'; -import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types'; -import { protocolContractTreeRoot } from '@aztec/protocol-contracts'; -import { makeGlobals } from '../mocks/fixtures.js'; import { TestContext } from '../mocks/test_context.js'; const logger = createDebugLogger('aztec:orchestrator-multi-blocks'); @@ -13,6 +10,7 @@ describe('prover/orchestrator/multi-block', () => { beforeEach(async () => { context = await TestContext.new(logger); + context.orchestrator.isVerifyBuiltBlockAgainstSyncedStateEnabled = true; }); afterEach(async () => { @@ -21,34 +19,44 @@ describe('prover/orchestrator/multi-block', () => { describe('multiple blocks', () => { it.each([1, 4, 5])('builds an epoch with %s blocks in sequence', async (numBlocks: number) => { - context.orchestrator.startNewEpoch(1, numBlocks); - let header = context.actualDb.getInitialHeader(); - - for (let i = 0; i < numBlocks; i++) { - logger.info(`Creating block ${i + 1000}`); - const tx = makeBloatedProcessedTx({ - header, - vkTreeRoot: getVKTreeRoot(), - protocolContractTreeRoot, - seed: i + 1, - }); + logger.info(`Seeding world state with ${numBlocks} blocks`); + const txCount = 1; + const blocks = await timesAsync(numBlocks, i => context.makePendingBlock(txCount, 0, i + 1)); - const blockNum = i + 1000; - const globals = makeGlobals(blockNum); + logger.info(`Starting new epoch with ${numBlocks}`); + context.orchestrator.startNewEpoch(1, numBlocks); + for (const { block, txs } of blocks) { + await context.orchestrator.startNewBlock(Math.max(txCount, 2), block.header.globalVariables, []); + for (const tx of txs) { + await context.orchestrator.addNewTx(tx); + } + await context.orchestrator.setBlockCompleted(block.number); + } - // This will need to be a 2 tx block - await context.orchestrator.startNewBlock(2, globals, []); + logger.info('Finalising epoch'); + const epoch = await context.orchestrator.finaliseEpoch(); + expect(epoch.publicInputs.endBlockNumber.toNumber()).toEqual(numBlocks); + expect(epoch.proof).toBeDefined(); + }); - await context.orchestrator.addNewTx(tx); + it.each([1, 4, 5])('builds an epoch with %s blocks in parallel', async (numBlocks: number) => { + logger.info(`Seeding world state with ${numBlocks} blocks`); + const txCount = 1; + const blocks = await timesAsync(numBlocks, i => context.makePendingBlock(txCount, 0, i + 1)); - // we need to complete the block as we have not added a full set of txs - const block = await context.orchestrator.setBlockCompleted(); - header = block!.header; - } + logger.info(`Starting new epoch with ${numBlocks}`); + context.orchestrator.startNewEpoch(1, numBlocks); + await Promise.all( + blocks.map(async ({ block, txs }) => { + await context.orchestrator.startNewBlock(Math.max(txCount, 2), block.header.globalVariables, []); + await Promise.all(txs.map(tx => context.orchestrator.addNewTx(tx))); + await context.orchestrator.setBlockCompleted(block.number); + }), + ); logger.info('Finalising epoch'); const epoch = await context.orchestrator.finaliseEpoch(); - expect(epoch.publicInputs.endBlockNumber.toNumber()).toEqual(1000 + numBlocks - 1); + expect(epoch.publicInputs.endBlockNumber.toNumber()).toEqual(numBlocks); expect(epoch.proof).toBeDefined(); }); }); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_public_functions.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_public_functions.test.ts index 91c34a355f2..7e0221fc716 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_public_functions.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_public_functions.test.ts @@ -35,7 +35,7 @@ describe('prover/orchestrator/public-functions', () => { numberOfNonRevertiblePublicCallRequests, numberOfRevertiblePublicCallRequests, }); - tx.data.constants.historicalHeader = context.actualDb.getInitialHeader(); + tx.data.constants.historicalHeader = context.getHeader(0); tx.data.constants.vkTreeRoot = getVKTreeRoot(); tx.data.constants.protocolContractTreeRoot = protocolContractTreeRoot; @@ -49,7 +49,7 @@ describe('prover/orchestrator/public-functions', () => { await context.orchestrator.addNewTx(processedTx); } - const block = await context.orchestrator.setBlockCompleted(); + const block = await context.orchestrator.setBlockCompleted(context.blockNumber); await context.orchestrator.finaliseEpoch(); expect(block.number).toEqual(context.blockNumber); }, diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_single_blocks.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_single_blocks.test.ts index 5c82382d054..e790fa7d378 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_single_blocks.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_single_blocks.test.ts @@ -1,10 +1,10 @@ import { NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/circuits.js'; import { fr } from '@aztec/circuits.js/testing'; import { range } from '@aztec/foundation/array'; +import { times } from '@aztec/foundation/collection'; import { createDebugLogger } from '@aztec/foundation/log'; import { sleep } from '@aztec/foundation/sleep'; -import { makeBloatedProcessedTxWithVKRoot } from '../mocks/fixtures.js'; import { TestContext } from '../mocks/test_context.js'; const logger = createDebugLogger('aztec:orchestrator-single-blocks'); @@ -25,13 +25,13 @@ describe('prover/orchestrator/blocks', () => { context.orchestrator.startNewEpoch(1, 1); await context.orchestrator.startNewBlock(2, context.globalVariables, []); - const block = await context.orchestrator.setBlockCompleted(); + const block = await context.orchestrator.setBlockCompleted(context.blockNumber); await context.orchestrator.finaliseEpoch(); expect(block.number).toEqual(context.blockNumber); }); it('builds a block with 1 transaction', async () => { - const txs = [makeBloatedProcessedTxWithVKRoot(context.actualDb, 1)]; + const txs = [context.makeProcessedTx(1)]; // This will need to be a 2 tx block context.orchestrator.startNewEpoch(1, 1); @@ -41,18 +41,13 @@ describe('prover/orchestrator/blocks', () => { await context.orchestrator.addNewTx(tx); } - const block = await context.orchestrator.setBlockCompleted(); + const block = await context.orchestrator.setBlockCompleted(context.blockNumber); await context.orchestrator.finaliseEpoch(); expect(block.number).toEqual(context.blockNumber); }); it('builds a block concurrently with transaction simulation', async () => { - const txs = [ - makeBloatedProcessedTxWithVKRoot(context.actualDb, 1), - makeBloatedProcessedTxWithVKRoot(context.actualDb, 2), - makeBloatedProcessedTxWithVKRoot(context.actualDb, 3), - makeBloatedProcessedTxWithVKRoot(context.actualDb, 4), - ]; + const txs = times(4, i => context.makeProcessedTx(i + 1)); const l1ToL2Messages = range(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, 1 + 0x400).map(fr); @@ -64,7 +59,7 @@ describe('prover/orchestrator/blocks', () => { await sleep(1000); } - const block = await context.orchestrator.setBlockCompleted(); + const block = await context.orchestrator.setBlockCompleted(context.blockNumber); await context.orchestrator.finaliseEpoch(); expect(block.number).toEqual(context.blockNumber); }); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts index 7675933f239..7525c9e16ed 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts @@ -17,11 +17,9 @@ import { createDebugLogger } from '@aztec/foundation/log'; import { promiseWithResolvers } from '@aztec/foundation/promise'; import { sleep } from '@aztec/foundation/sleep'; import { ProtocolCircuitVks } from '@aztec/noir-protocol-circuits-types'; -import { type MerkleTreeReadOperations } from '@aztec/world-state'; import { type MockProxy, mock } from 'jest-mock-extended'; -import { makeBloatedProcessedTxWithVKRoot } from '../mocks/fixtures.js'; import { TestContext } from '../mocks/test_context.js'; import { type ProvingOrchestrator } from './orchestrator.js'; @@ -30,7 +28,6 @@ const logger = createDebugLogger('aztec:orchestrator-workflow'); describe('prover/orchestrator', () => { describe('workflow', () => { let orchestrator: ProvingOrchestrator; - let actualDb: MerkleTreeReadOperations; let globalVariables: GlobalVariables; let context: TestContext; @@ -39,8 +36,8 @@ describe('prover/orchestrator', () => { beforeEach(async () => { mockProver = mock(); - context = await TestContext.new(logger, 'native', 4, () => Promise.resolve(mockProver)); - ({ actualDb, orchestrator, globalVariables } = context); + context = await TestContext.new(logger, 4, () => Promise.resolve(mockProver)); + ({ orchestrator, globalVariables } = context); }); it('calls root parity circuit only when ready', async () => { @@ -103,20 +100,20 @@ describe('prover/orchestrator', () => { describe('with simulated prover', () => { beforeEach(async () => { context = await TestContext.new(logger); - ({ actualDb, orchestrator, globalVariables } = context); + ({ orchestrator, globalVariables } = context); }); it('waits for block to be completed before enqueueing block root proof', async () => { orchestrator.startNewEpoch(1, 1); await orchestrator.startNewBlock(2, globalVariables, []); - await orchestrator.addNewTx(makeBloatedProcessedTxWithVKRoot(actualDb, 1)); - await orchestrator.addNewTx(makeBloatedProcessedTxWithVKRoot(actualDb, 2)); + await orchestrator.addNewTx(context.makeProcessedTx(1)); + await orchestrator.addNewTx(context.makeProcessedTx(2)); // wait for the block root proof to try to be enqueued await sleep(1000); // now finish the block - await orchestrator.setBlockCompleted(); + await orchestrator.setBlockCompleted(context.blockNumber); const result = await orchestrator.finaliseEpoch(); expect(result.proof).toBeDefined(); diff --git a/yarn-project/prover-client/src/tx-prover/factory.ts b/yarn-project/prover-client/src/prover-client/factory.ts similarity index 58% rename from yarn-project/prover-client/src/tx-prover/factory.ts rename to yarn-project/prover-client/src/prover-client/factory.ts index 07a65a8c57c..45e10ed630b 100644 --- a/yarn-project/prover-client/src/tx-prover/factory.ts +++ b/yarn-project/prover-client/src/prover-client/factory.ts @@ -1,14 +1,15 @@ -import { type ProvingJobBroker } from '@aztec/circuit-types'; +import { type ForkMerkleTreeOperations, type ProvingJobBroker } from '@aztec/circuit-types'; import { type TelemetryClient } from '@aztec/telemetry-client'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { type ProverClientConfig } from '../config.js'; -import { TxProver } from './tx-prover.js'; +import { ProverClient } from './prover-client.js'; export function createProverClient( config: ProverClientConfig, + worldState: ForkMerkleTreeOperations, broker: ProvingJobBroker, telemetry: TelemetryClient = new NoopTelemetryClient(), ) { - return TxProver.new(config, broker, telemetry); + return ProverClient.new(config, worldState, broker, telemetry); } diff --git a/yarn-project/prover-client/src/prover-client/index.ts b/yarn-project/prover-client/src/prover-client/index.ts new file mode 100644 index 00000000000..dc8c2be6612 --- /dev/null +++ b/yarn-project/prover-client/src/prover-client/index.ts @@ -0,0 +1,2 @@ +export * from './factory.js'; +export * from './prover-client.js'; diff --git a/yarn-project/prover-client/src/tx-prover/tx-prover.ts b/yarn-project/prover-client/src/prover-client/prover-client.ts similarity index 86% rename from yarn-project/prover-client/src/tx-prover/tx-prover.ts rename to yarn-project/prover-client/src/prover-client/prover-client.ts index 9bd34df56ca..3cc5b9aa32b 100644 --- a/yarn-project/prover-client/src/tx-prover/tx-prover.ts +++ b/yarn-project/prover-client/src/prover-client/prover-client.ts @@ -3,7 +3,7 @@ import { type ActualProverConfig, type EpochProver, type EpochProverManager, - type MerkleTreeWriteOperations, + type ForkMerkleTreeOperations, type ProverCache, type ProvingJobBroker, type ProvingJobConsumer, @@ -25,11 +25,8 @@ import { InlineProofStore } from '../proving_broker/proof_store.js'; import { InMemoryProverCache } from '../proving_broker/prover_cache/memory.js'; import { ProvingAgent } from '../proving_broker/proving_agent.js'; -/** - * A prover factory. - * TODO(palla/prover-node): Rename this class - */ -export class TxProver implements EpochProverManager { +/** Manages proving of epochs by orchestrating the proving of individual blocks relying on a pool of prover agents. */ +export class ProverClient implements EpochProverManager { private running = false; private agents: ProvingAgent[] = []; @@ -37,6 +34,7 @@ export class TxProver implements EpochProverManager { private constructor( private config: ProverClientConfig, + private worldState: ForkMerkleTreeOperations, private telemetry: TelemetryClient, private orchestratorClient: ProvingJobProducer, private agentClient?: ProvingJobConsumer, @@ -47,9 +45,9 @@ export class TxProver implements EpochProverManager { this.cacheDir = this.config.cacheDir ? join(this.config.cacheDir, `tx_prover_${this.config.proverId}`) : undefined; } - public createEpochProver(db: MerkleTreeWriteOperations, cache: ProverCache = new InMemoryProverCache()): EpochProver { + public createEpochProver(cache: ProverCache = new InMemoryProverCache()): EpochProver { return new ProvingOrchestrator( - db, + this.worldState, new CachingBrokerFacade(this.orchestratorClient, cache), this.telemetry, this.config.proverId, @@ -104,12 +102,16 @@ export class TxProver implements EpochProverManager { /** * Creates a new prover client and starts it * @param config - The prover configuration. - * @param vks - The verification keys for the prover - * @param worldStateSynchronizer - An instance of the world state + * @param worldState - An instance of the world state * @returns An instance of the prover, constructed and started. */ - public static async new(config: ProverClientConfig, broker: ProvingJobBroker, telemetry: TelemetryClient) { - const prover = new TxProver(config, telemetry, broker, broker); + public static async new( + config: ProverClientConfig, + worldState: ForkMerkleTreeOperations, + broker: ProvingJobBroker, + telemetry: TelemetryClient, + ) { + const prover = new ProverClient(config, worldState, telemetry, broker, broker); await prover.start(); return prover; } diff --git a/yarn-project/prover-client/src/test/bb_prover_base_rollup.test.ts b/yarn-project/prover-client/src/test/bb_prover_base_rollup.test.ts index e8b644a8a26..154ac6c71dd 100644 --- a/yarn-project/prover-client/src/test/bb_prover_base_rollup.test.ts +++ b/yarn-project/prover-client/src/test/bb_prover_base_rollup.test.ts @@ -27,7 +27,7 @@ describe('prover/bb_prover/base-rollup', () => { prover = await BBNativeRollupProver.new(bbConfig, new NoopTelemetryClient()); return prover; }; - context = await TestContext.new(logger, 'native', 1, buildProver); + context = await TestContext.new(logger, 1, buildProver); }); afterAll(async () => { @@ -35,7 +35,7 @@ describe('prover/bb_prover/base-rollup', () => { }); it('proves the base rollup', async () => { - const header = context.actualDb.getInitialHeader(); + const header = context.getHeader(0); const chainId = context.globalVariables.chainId; const version = context.globalVariables.version; const vkTreeRoot = getVKTreeRoot(); @@ -59,7 +59,7 @@ describe('prover/bb_prover/base-rollup', () => { const tubeData = new PrivateTubeData(tubeProof.inputs, tubeProof.proof, vkData); - const baseRollupHints = await buildBaseRollupHints(tx, context.globalVariables, context.actualDb); + const baseRollupHints = await buildBaseRollupHints(tx, context.globalVariables, await context.getFork()); const baseRollupInputs = new PrivateBaseRollupInputs(tubeData, baseRollupHints as PrivateBaseRollupHints); logger.verbose('Proving base rollups'); diff --git a/yarn-project/prover-client/src/test/bb_prover_full_rollup.test.ts b/yarn-project/prover-client/src/test/bb_prover_full_rollup.test.ts index e573e8572f7..89827c6e02e 100644 --- a/yarn-project/prover-client/src/test/bb_prover_full_rollup.test.ts +++ b/yarn-project/prover-client/src/test/bb_prover_full_rollup.test.ts @@ -8,6 +8,7 @@ import { getTestData, isGenerateTestDataEnabled, writeTestData } from '@aztec/fo import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; +import { buildBlock } from '../block_builder/light.js'; import { makeGlobals } from '../mocks/fixtures.js'; import { TestContext } from '../mocks/test_context.js'; @@ -16,16 +17,16 @@ describe('prover/bb_prover/full-rollup', () => { let prover: BBNativeRollupProver; let log: DebugLogger; - beforeAll(async () => { + beforeEach(async () => { const buildProver = async (bbConfig: BBProverConfig) => { prover = await BBNativeRollupProver.new(bbConfig, new NoopTelemetryClient()); return prover; }; log = createDebugLogger('aztec:bb-prover-full-rollup'); - context = await TestContext.new(log, 'native', 1, buildProver); + context = await TestContext.new(log, 1, buildProver); }); - afterAll(async () => { + afterEach(async () => { await context.cleanup(); }); @@ -38,7 +39,7 @@ describe('prover/bb_prover/full-rollup', () => { async (blockCount, totalBlocks, nonEmptyTxs, totalTxs) => { log.info(`Proving epoch with ${blockCount}/${totalBlocks} blocks with ${nonEmptyTxs}/${totalTxs} non-empty txs`); - const initialHeader = context.actualDb.getInitialHeader(); + const initialHeader = context.getHeader(0); context.orchestrator.startNewEpoch(1, totalBlocks); for (let blockNum = 1; blockNum <= blockCount; blockNum++) { @@ -60,7 +61,11 @@ describe('prover/bb_prover/full-rollup', () => { expect(failed.length).toBe(0); log.info(`Setting block as completed`); - await context.orchestrator.setBlockCompleted(); + await context.orchestrator.setBlockCompleted(blockNum); + + log.info(`Updating world state with new block`); + const block = await buildBlock(processed, globals, l1ToL2Messages, await context.worldState.fork()); + await context.worldState.handleL2BlockAndMessages(block, l1ToL2Messages); } log.info(`Awaiting proofs`); @@ -89,7 +94,7 @@ describe('prover/bb_prover/full-rollup', () => { }), ); for (const tx of txs) { - tx.data.constants.historicalHeader = context.actualDb.getInitialHeader(); + tx.data.constants.historicalHeader = context.getHeader(0); } const l1ToL2Messages = makeTuple( @@ -106,7 +111,7 @@ describe('prover/bb_prover/full-rollup', () => { expect(processed.length).toBe(numTransactions); expect(failed.length).toBe(0); - await context.orchestrator.setBlockCompleted(); + await context.orchestrator.setBlockCompleted(context.blockNumber); const result = await context.orchestrator.finaliseEpoch(); await expect(prover.verifyProof('RootRollupArtifact', result.proof)).resolves.not.toThrow(); diff --git a/yarn-project/prover-client/src/test/bb_prover_parity.test.ts b/yarn-project/prover-client/src/test/bb_prover_parity.test.ts index a845a1de4cc..1763fd1b400 100644 --- a/yarn-project/prover-client/src/test/bb_prover_parity.test.ts +++ b/yarn-project/prover-client/src/test/bb_prover_parity.test.ts @@ -36,7 +36,7 @@ describe('prover/bb_prover/parity', () => { bbProver = await BBNativeRollupProver.new(bbConfig, new NoopTelemetryClient()); return bbProver; }; - context = await TestContext.new(logger, 'native', 1, buildProver); + context = await TestContext.new(logger, 1, buildProver); }); afterAll(async () => { diff --git a/yarn-project/prover-node/src/config.ts b/yarn-project/prover-node/src/config.ts index 34a59b0a338..b609ffd7d5c 100644 --- a/yarn-project/prover-node/src/config.ts +++ b/yarn-project/prover-node/src/config.ts @@ -46,10 +46,14 @@ export type ProverNodeConfig = ArchiverConfig & DataStoreConfig & ProverCoordinationConfig & ProverBondManagerConfig & - QuoteProviderConfig & { - proverNodeMaxPendingJobs: number; - proverNodePollingIntervalMs: number; - }; + QuoteProviderConfig & + SpecificProverNodeConfig; + +type SpecificProverNodeConfig = { + proverNodeMaxPendingJobs: number; + proverNodePollingIntervalMs: number; + proverNodeMaxParallelBlocksPerEpoch: number; +}; export type QuoteProviderConfig = { quoteProviderBasisPointFee: number; @@ -57,9 +61,7 @@ export type QuoteProviderConfig = { quoteProviderUrl?: string; }; -const specificProverNodeConfigMappings: ConfigMappingsType< - Pick -> = { +const specificProverNodeConfigMappings: ConfigMappingsType = { proverNodeMaxPendingJobs: { env: 'PROVER_NODE_MAX_PENDING_JOBS', description: 'The maximum number of pending jobs for the prover node', @@ -70,6 +72,11 @@ const specificProverNodeConfigMappings: ConfigMappingsType< description: 'The interval in milliseconds to poll for new jobs', ...numberConfigHelper(1000), }, + proverNodeMaxParallelBlocksPerEpoch: { + env: 'PROVER_NODE_MAX_PARALLEL_BLOCKS_PER_EPOCH', + description: 'The Maximum number of blocks to process in parallel while proving an epoch', + ...numberConfigHelper(32), + }, }; const quoteProviderConfigMappings: ConfigMappingsType = { diff --git a/yarn-project/prover-node/src/factory.ts b/yarn-project/prover-node/src/factory.ts index 2f54b4b7f7d..7190d81ee66 100644 --- a/yarn-project/prover-node/src/factory.ts +++ b/yarn-project/prover-node/src/factory.ts @@ -21,7 +21,7 @@ import { ClaimsMonitor } from './monitors/claims-monitor.js'; import { EpochMonitor } from './monitors/epoch-monitor.js'; import { ProverCacheManager } from './prover-cache/cache_manager.js'; import { createProverCoordination } from './prover-coordination/factory.js'; -import { ProverNode } from './prover-node.js'; +import { ProverNode, type ProverNodeOptions } from './prover-node.js'; import { HttpQuoteProvider } from './quote-provider/http.js'; import { SimpleQuoteProvider } from './quote-provider/simple.js'; import { QuoteSigner } from './quote-signer.js'; @@ -43,12 +43,12 @@ export async function createProverNode( const archiver = deps.archiver ?? (await createArchiver(config, telemetry, { blockUntilSync: true })); log.verbose(`Created archiver and synced to block ${await archiver.getBlockNumber()}`); - const worldStateConfig = { ...config, worldStateProvenBlocksOnly: true }; + const worldStateConfig = { ...config, worldStateProvenBlocksOnly: false }; const worldStateSynchronizer = await createWorldStateSynchronizer(worldStateConfig, archiver, telemetry); await worldStateSynchronizer.start(); const broker = deps.broker ?? (await createAndStartProvingBroker(config)); - const prover = await createProverClient(config, broker, telemetry); + const prover = await createProverClient(config, worldStateSynchronizer, broker, telemetry); // REFACTOR: Move publisher out of sequencer package and into an L1-related package const publisher = deps.publisher ?? new L1Publisher(config, telemetry); @@ -65,9 +65,10 @@ export async function createProverNode( const quoteProvider = createQuoteProvider(config); const quoteSigner = createQuoteSigner(config); - const proverNodeConfig = { + const proverNodeConfig: ProverNodeOptions = { maxPendingJobs: config.proverNodeMaxPendingJobs, pollingIntervalMs: config.proverNodePollingIntervalMs, + maxParallelBlocksPerEpoch: config.proverNodeMaxParallelBlocksPerEpoch, }; const claimsMonitor = new ClaimsMonitor(publisher, proverNodeConfig); diff --git a/yarn-project/prover-node/src/job/epoch-proving-job.ts b/yarn-project/prover-node/src/job/epoch-proving-job.ts index c50e6682be4..56deb373a96 100644 --- a/yarn-project/prover-node/src/job/epoch-proving-job.ts +++ b/yarn-project/prover-node/src/job/epoch-proving-job.ts @@ -2,17 +2,16 @@ import { EmptyTxValidator, type EpochProver, type EpochProvingJobState, + type ForkMerkleTreeOperations, type L1ToL2MessageSource, type L2Block, type L2BlockSource, - MerkleTreeId, - type MerkleTreeWriteOperations, type ProcessedTx, type ProverCoordination, type Tx, type TxHash, } from '@aztec/circuit-types'; -import { KernelCircuitPublicInputs, NULLIFIER_SUBTREE_HEIGHT, PublicDataTreeLeaf } from '@aztec/circuits.js'; +import { asyncPool } from '@aztec/foundation/async-pool'; import { createDebugLogger } from '@aztec/foundation/log'; import { promiseWithResolvers } from '@aztec/foundation/promise'; import { Timer } from '@aztec/foundation/timer'; @@ -36,7 +35,7 @@ export class EpochProvingJob { private runPromise: Promise | undefined; constructor( - private db: MerkleTreeWriteOperations, + private dbProvider: ForkMerkleTreeOperations, private epochNumber: bigint, private blocks: L2Block[], private prover: EpochProver, @@ -46,6 +45,7 @@ export class EpochProvingJob { private l1ToL2MessageSource: L1ToL2MessageSource, private coordination: ProverCoordination, private metrics: ProverNodeMetrics, + private config: { parallelBlockLimit: number } = { parallelBlockLimit: 32 }, private cleanUp: (job: EpochProvingJob) => Promise = () => Promise.resolve(), ) { this.uuid = crypto.randomUUID(); @@ -75,19 +75,13 @@ export class EpochProvingJob { try { this.prover.startNewEpoch(epochNumber, epochSize); - // Get the genesis header if the first block of the epoch is the first block of the chain - let previousHeader = - this.blocks[0].number === 1 - ? this.db.getInitialHeader() - : await this.l2BlockSource.getBlockHeader(this.blocks[0].number - 1); - - for (const block of this.blocks) { - // Gather all data to prove this block + await asyncPool(this.config.parallelBlockLimit, this.blocks, async block => { const globalVariables = block.header.globalVariables; const txHashes = block.body.txEffects.map(tx => tx.txHash); const txCount = block.body.numberOfTxsIncludingPadded; const l1ToL2Messages = await this.getL1ToL2Messages(block); const txs = await this.getTxs(txHashes); + const previousHeader = await this.getBlockHeader(block.number - 1); this.log.verbose(`Starting block processing`, { number: block.number, @@ -105,27 +99,23 @@ export class EpochProvingJob { await this.prover.startNewBlock(txCount, globalVariables, l1ToL2Messages); // Process public fns - const publicProcessor = this.publicProcessorFactory.create(this.db, previousHeader, globalVariables); + const db = await this.dbProvider.fork(block.number - 1); + const publicProcessor = this.publicProcessorFactory.create(db, previousHeader, globalVariables); await this.processTxs(publicProcessor, txs, txCount); + await db.close(); this.log.verbose(`Processed all txs for block`, { blockNumber: block.number, blockHash: block.hash().toString(), uuid: this.uuid, }); - if (txCount > txs.length) { - // If this block has a padding tx, ensure that the public processor's db has its state - await this.addPaddingTxState(); - } - - // Mark block as completed and update archive tree - await this.prover.setBlockCompleted(block.header); - previousHeader = block.header; - } + // Mark block as completed to pad it + await this.prover.setBlockCompleted(block.number, block.header); + }); this.state = 'awaiting-prover'; const { publicInputs, proof } = await this.prover.finaliseEpoch(); - this.log.info(`Finalised proof for epoch`, { epochNumber, uuid: this.uuid }); + this.log.info(`Finalised proof for epoch`, { epochNumber, uuid: this.uuid, duration: timer.ms() }); this.state = 'publishing-proof'; const [fromBlock, toBlock] = [this.blocks[0].number, this.blocks.at(-1)!.number]; @@ -150,6 +140,14 @@ export class EpochProvingJob { } } + /* Returns the header for the given block number, or undefined for block zero. */ + private getBlockHeader(blockNumber: number) { + if (blockNumber === 0) { + return undefined; + } + return this.l2BlockSource.getBlockHeader(blockNumber); + } + private async getTxs(txHashes: TxHash[]): Promise { const txs = await Promise.all( txHashes.map(txHash => this.coordination.getTxByHash(txHash).then(tx => [txHash, tx] as const)), @@ -185,25 +183,6 @@ export class EpochProvingJob { return processedTxs; } - - private async addPaddingTxState() { - const emptyKernelOutput = KernelCircuitPublicInputs.empty(); - await this.db.appendLeaves(MerkleTreeId.NOTE_HASH_TREE, emptyKernelOutput.end.noteHashes); - await this.db.batchInsert( - MerkleTreeId.NULLIFIER_TREE, - emptyKernelOutput.end.nullifiers.map(n => n.toBuffer()), - NULLIFIER_SUBTREE_HEIGHT, - ); - const allPublicDataWrites = emptyKernelOutput.end.publicDataWrites - .filter(write => !write.isEmpty()) - .map(({ leafSlot, value }) => new PublicDataTreeLeaf(leafSlot, value)); - - await this.db.batchInsert( - MerkleTreeId.PUBLIC_DATA_TREE, - allPublicDataWrites.map(x => x.toBuffer()), - 0, - ); - } } export { type EpochProvingJobState }; diff --git a/yarn-project/prover-node/src/prover-node.test.ts b/yarn-project/prover-node/src/prover-node.test.ts index 008b2443cc4..bc8ca80897b 100644 --- a/yarn-project/prover-node/src/prover-node.test.ts +++ b/yarn-project/prover-node/src/prover-node.test.ts @@ -75,7 +75,6 @@ describe('prover-node', () => { let jobs: { job: MockProxy; cleanUp: (job: EpochProvingJob) => Promise; - db: MerkleTreeWriteOperations; epochNumber: bigint; }[]; @@ -121,7 +120,7 @@ describe('prover-node', () => { bondManager = mock(); telemetryClient = new NoopTelemetryClient(); - config = { maxPendingJobs: 3, pollingIntervalMs: 10 }; + config = { maxPendingJobs: 3, pollingIntervalMs: 10, maxParallelBlocksPerEpoch: 32 }; // World state returns a new mock db every time it is asked to fork worldState.fork.mockImplementation(() => Promise.resolve(mock())); @@ -378,15 +377,13 @@ describe('prover-node', () => { protected override doCreateEpochProvingJob( epochNumber: bigint, _blocks: L2Block[], - publicDb: MerkleTreeWriteOperations, - _proverDb: MerkleTreeWriteOperations, _cache: ProverCache, _publicProcessorFactory: PublicProcessorFactory, cleanUp: (job: EpochProvingJob) => Promise, ): EpochProvingJob { const job = mock({ getState: () => 'processing', run: () => Promise.resolve() }); job.getId.mockReturnValue(jobs.length.toString()); - jobs.push({ epochNumber, job, cleanUp, db: publicDb }); + jobs.push({ epochNumber, job, cleanUp }); return job; } diff --git a/yarn-project/prover-node/src/prover-node.ts b/yarn-project/prover-node/src/prover-node.ts index 0c63bc79b40..d4ea397d245 100644 --- a/yarn-project/prover-node/src/prover-node.ts +++ b/yarn-project/prover-node/src/prover-node.ts @@ -6,7 +6,6 @@ import { type L1ToL2MessageSource, type L2Block, type L2BlockSource, - type MerkleTreeWriteOperations, type ProverCache, type ProverCoordination, type ProverNodeApi, @@ -35,6 +34,7 @@ import { type QuoteSigner } from './quote-signer.js'; export type ProverNodeOptions = { pollingIntervalMs: number; maxPendingJobs: number; + maxParallelBlocksPerEpoch: number; }; /** @@ -71,6 +71,7 @@ export class ProverNode implements ClaimsMonitorHandler, EpochMonitorHandler, Pr this.options = { pollingIntervalMs: 1_000, maxPendingJobs: 100, + maxParallelBlocksPerEpoch: 32, ...compact(options), }; @@ -246,10 +247,6 @@ export class ProverNode implements ClaimsMonitorHandler, EpochMonitorHandler, Pr // Fast forward world state to right before the target block and get a fork this.log.verbose(`Creating proving job for epoch ${epochNumber} for block range ${fromBlock} to ${toBlock}`); await this.worldState.syncImmediate(fromBlock - 1); - // NB: separated the dbs as both a block builder and public processor need to track and update tree state - // see public_processor.ts for context - const publicDb = await this.worldState.fork(fromBlock - 1); - const proverDb = await this.worldState.fork(fromBlock - 1); // Create a processor using the forked world state const publicProcessorFactory = new PublicProcessorFactory(this.contractDataSource, this.telemetryClient); @@ -258,22 +255,12 @@ export class ProverNode implements ClaimsMonitorHandler, EpochMonitorHandler, Pr const proverCache = await this.proverCacheManager.openCache(epochNumber, epochHash); const cleanUp = async () => { - await publicDb.close(); - await proverDb.close(); await proverCache.close(); await this.proverCacheManager.removeStaleCaches(epochNumber); this.jobs.delete(job.getId()); }; - const job = this.doCreateEpochProvingJob( - epochNumber, - blocks, - publicDb, - proverDb, - proverCache, - publicProcessorFactory, - cleanUp, - ); + const job = this.doCreateEpochProvingJob(epochNumber, blocks, proverCache, publicProcessorFactory, cleanUp); this.jobs.set(job.getId(), job); return job; } @@ -282,23 +269,22 @@ export class ProverNode implements ClaimsMonitorHandler, EpochMonitorHandler, Pr protected doCreateEpochProvingJob( epochNumber: bigint, blocks: L2Block[], - publicDb: MerkleTreeWriteOperations, - proverDb: MerkleTreeWriteOperations, proverCache: ProverCache, publicProcessorFactory: PublicProcessorFactory, cleanUp: () => Promise, ) { return new EpochProvingJob( - publicDb, + this.worldState, epochNumber, blocks, - this.prover.createEpochProver(proverDb, proverCache), + this.prover.createEpochProver(proverCache), publicProcessorFactory, this.publisher, this.l2BlockSource, this.l1ToL2MessageSource, this.coordination, this.metrics, + { parallelBlockLimit: this.options.maxParallelBlocksPerEpoch }, cleanUp, ); } diff --git a/yarn-project/sequencer-client/src/block_builder/orchestrator.ts b/yarn-project/sequencer-client/src/block_builder/orchestrator.ts deleted file mode 100644 index 862963f10fe..00000000000 --- a/yarn-project/sequencer-client/src/block_builder/orchestrator.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { TestCircuitProver } from '@aztec/bb-prover'; -import { - type BlockBuilder, - type L2Block, - type MerkleTreeWriteOperations, - type ProcessedTx, -} from '@aztec/circuit-types'; -import { type Fr, type GlobalVariables } from '@aztec/circuits.js'; -import { ProvingOrchestrator } from '@aztec/prover-client/orchestrator'; -import { type SimulationProvider } from '@aztec/simulator'; -import { type TelemetryClient } from '@aztec/telemetry-client'; -import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; - -/** - * Implements a block simulator using a test circuit prover under the hood, which just simulates circuits and outputs empty proofs. - * This class is unused at the moment, but could be leveraged by a prover-node to ascertain that it can prove a block before - * committing to proving it by sending a quote. - */ -export class OrchestratorBlockBuilder implements BlockBuilder { - private orchestrator: ProvingOrchestrator; - constructor(db: MerkleTreeWriteOperations, simulationProvider: SimulationProvider, telemetry: TelemetryClient) { - const testProver = new TestCircuitProver(telemetry, simulationProvider); - this.orchestrator = new ProvingOrchestrator(db, testProver, telemetry); - } - - startNewBlock(numTxs: number, globalVariables: GlobalVariables, l1ToL2Messages: Fr[]): Promise { - return this.orchestrator.startNewBlock(numTxs, globalVariables, l1ToL2Messages); - } - setBlockCompleted(): Promise { - return this.orchestrator.setBlockCompleted(); - } - addNewTx(tx: ProcessedTx): Promise { - return this.orchestrator.addNewTx(tx); - } -} - -export class OrchestratorBlockBuilderFactory { - constructor(private simulationProvider: SimulationProvider, private telemetry?: TelemetryClient) {} - - create(db: MerkleTreeWriteOperations): BlockBuilder { - return new OrchestratorBlockBuilder(db, this.simulationProvider, this.telemetry ?? new NoopTelemetryClient()); - } -} diff --git a/yarn-project/sequencer-client/src/client/sequencer-client.ts b/yarn-project/sequencer-client/src/client/sequencer-client.ts index 404b062696a..98ff97db320 100644 --- a/yarn-project/sequencer-client/src/client/sequencer-client.ts +++ b/yarn-project/sequencer-client/src/client/sequencer-client.ts @@ -2,11 +2,11 @@ import { type L1ToL2MessageSource, type L2BlockSource, type WorldStateSynchroniz import { type ContractDataSource } from '@aztec/circuits.js'; import { type EthAddress } from '@aztec/foundation/eth-address'; import { type P2P } from '@aztec/p2p'; +import { LightweightBlockBuilderFactory } from '@aztec/prover-client/block-builder'; import { PublicProcessorFactory } from '@aztec/simulator'; import { type TelemetryClient } from '@aztec/telemetry-client'; import { type ValidatorClient } from '@aztec/validator-client'; -import { LightweightBlockBuilderFactory } from '../block_builder/index.js'; import { type SequencerClientConfig } from '../config.js'; import { GlobalVariableBuilder } from '../global_variable_builder/index.js'; import { L1Publisher } from '../publisher/index.js'; diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts index e1b3f8bb71a..e5a55462b97 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts @@ -36,6 +36,7 @@ import { randomBytes } from '@aztec/foundation/crypto'; import { Signature } from '@aztec/foundation/eth-signature'; import { type Writeable } from '@aztec/foundation/types'; import { type P2P, P2PClientState } from '@aztec/p2p'; +import { type BlockBuilderFactory } from '@aztec/prover-client/block-builder'; import { type PublicProcessor, type PublicProcessorFactory } from '@aztec/simulator'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { type ValidatorClient } from '@aztec/validator-client'; @@ -43,7 +44,6 @@ import { type ValidatorClient } from '@aztec/validator-client'; import { expect } from '@jest/globals'; import { type MockProxy, mock, mockFn } from 'jest-mock-extended'; -import { type BlockBuilderFactory } from '../block_builder/index.js'; import { type GlobalVariableBuilder } from '../global_variable_builder/global_builder.js'; import { type L1Publisher } from '../publisher/l1-publisher.js'; import { TxValidatorFactory } from '../tx_validator/tx_validator_factory.js'; diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index 3380025bb3c..8c9750ff043 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -26,13 +26,13 @@ import { createDebugLogger } from '@aztec/foundation/log'; import { RunningPromise } from '@aztec/foundation/running-promise'; import { Timer, elapsed } from '@aztec/foundation/timer'; import { type P2P } from '@aztec/p2p'; +import { type BlockBuilderFactory } from '@aztec/prover-client/block-builder'; import { type PublicProcessorFactory } from '@aztec/simulator'; import { Attributes, type TelemetryClient, type Tracer, trackSpan } from '@aztec/telemetry-client'; import { type ValidatorClient } from '@aztec/validator-client'; import { inspect } from 'util'; -import { type BlockBuilderFactory } from '../block_builder/index.js'; import { type GlobalVariableBuilder } from '../global_variable_builder/global_builder.js'; import { type L1Publisher } from '../publisher/l1-publisher.js'; import { prettyLogViemErrorMsg } from '../publisher/utils.js'; diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts index a9e9389b687..6410c8bc16d 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts @@ -1,5 +1,5 @@ import { type L2Block, type MerkleTreeId } from '@aztec/circuit-types'; -import { type MerkleTreeReadOperations, type MerkleTreeWriteOperations } from '@aztec/circuit-types/interfaces'; +import { type ForkMerkleTreeOperations, type MerkleTreeReadOperations } from '@aztec/circuit-types/interfaces'; import { type Fr, MAX_NULLIFIERS_PER_TX, MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX } from '@aztec/circuits.js'; import { type IndexedTreeSnapshot, type TreeSnapshot } from '@aztec/merkle-tree'; @@ -32,7 +32,7 @@ export type TreeSnapshots = { [MerkleTreeId.ARCHIVE]: TreeSnapshot; }; -export interface MerkleTreeAdminDatabase { +export interface MerkleTreeAdminDatabase extends ForkMerkleTreeOperations { /** * Handles a single L2 block (i.e. Inserts the new note hashes into the merkle tree). * @param block - The L2 block to handle. @@ -45,18 +45,6 @@ export interface MerkleTreeAdminDatabase { */ getCommitted(): MerkleTreeReadOperations; - /** - * Gets a handle that allows reading the state as it was at the given block number - * @param blockNumber - The block number to get the snapshot for - */ - getSnapshot(blockNumber: number): MerkleTreeReadOperations; - - /** - * Forks the database at its current state. - * @param blockNumber - The block number to fork at. If not provided, the current block number is used. - */ - fork(blockNumber?: number): Promise; - /** * Removes all historical snapshots up to but not including the given block number * @param toBlockNumber The block number of the new oldest historical block