refactor: remove prover-pool (#6727)

This PR replaces `ProverPool` with adding configurable concurrency level
to individual agents. This matches the expection that in production a
prover would have multiple independent agents running on separate
machines.

The `aztec start --prover` command was also changed to start a single
agent (which can run multiple proving jobs in parallel)

yarn-project/Earthfile

@@ -184,6 +184,7 @@ end-to-end:
     ENV BB_BINARY_PATH=/usr/src/barretenberg/cpp/build/bin/bb
     ENV ACVM_WORKING_DIRECTORY=/usr/src/acvm
     ENV ACVM_BINARY_PATH=/usr/src/noir/noir-repo/target/release/acvm
+    ENV PROVER_AGENT_CONCURRENCY=8
     RUN mkdir -p $BB_WORKING_DIRECTORY $ACVM_WORKING_DIRECTORY
     COPY +anvil/anvil /opt/foundry/bin/anvil
     COPY +end-to-end-prod/usr/src /usr/src

yarn-project/aztec-node/src/aztec-node/server.ts

@@ -147,9 +147,7 @@ export class AztecNodeService implements AztecNode {
 
     // start the prover if we have been told to
     const simulationProvider = await getSimulationProvider(config, log);
-    const prover = config.disableProver
-      ? await DummyProver.new()
-      : await TxProver.new(config, simulationProvider, worldStateSynchronizer);
+    const prover = config.disableProver ? await DummyProver.new() : await TxProver.new(config, worldStateSynchronizer);
 
     // now create the sequencer
     const sequencer = config.disableSequencer

yarn-project/aztec/package.json

@@ -31,6 +31,7 @@
     "@aztec/archiver": "workspace:^",
     "@aztec/aztec-node": "workspace:^",
     "@aztec/aztec.js": "workspace:^",
+    "@aztec/bb-prover": "workspace:^",
     "@aztec/builder": "workspace:^",
     "@aztec/circuit-types": "workspace:^",
     "@aztec/circuits.js": "workspace:^",

yarn-project/aztec/src/cli/cmds/start_node.ts

@@ -6,7 +6,7 @@ import {
 import { NULL_KEY } from '@aztec/ethereum';
 import { type ServerList } from '@aztec/foundation/json-rpc/server';
 import { type LogFn } from '@aztec/foundation/log';
-import { createProvingJobSourceServer } from '@aztec/prover-client/prover-pool';
+import { createProvingJobSourceServer } from '@aztec/prover-client/prover-agent';
 import { type PXEServiceConfig, createPXERpcServer, getPXEServiceConfig } from '@aztec/pxe';
 
 import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts';

yarn-project/aztec/src/cli/cmds/start_prover.ts

@@ -1,60 +1,56 @@
-import { type ProvingJobSource } from '@aztec/circuit-types';
-import { type ProverClientConfig, getProverEnvVars } from '@aztec/prover-client';
-import { ProverPool, createProvingJobSourceClient } from '@aztec/prover-client/prover-pool';
-
-import { tmpdir } from 'node:os';
+import { BBNativeRollupProver, TestCircuitProver } from '@aztec/bb-prover';
+import { type ServerCircuitProver } from '@aztec/circuit-types';
+import { getProverEnvVars } from '@aztec/prover-client';
+import { ProverAgent, createProvingJobSourceClient } from '@aztec/prover-client/prover-agent';
 
 import { type ServiceStarter, parseModuleOptions } from '../util.js';
 
-type ProverOptions = ProverClientConfig &
-  Partial<{
-    proverUrl: string;
-    agents: string;
-    acvmBinaryPath?: string;
-    bbBinaryPath?: string;
-    simulate?: string;
-  }>;
-
 export const startProver: ServiceStarter = async (options, signalHandlers, logger) => {
-  const proverOptions: ProverOptions = {
-    proverUrl: process.env.PROVER_URL,
+  const proverOptions = {
     ...getProverEnvVars(),
     ...parseModuleOptions(options.prover),
   };
-  let source: ProvingJobSource;
 
-  if (typeof proverOptions.proverUrl === 'string') {
-    logger(`Connecting to prover at ${proverOptions.proverUrl}`);
-    source = createProvingJobSourceClient(proverOptions.proverUrl, 'provingJobSource');
-  } else {
+  if (!proverOptions.nodeUrl) {
     throw new Error('Starting prover without an orchestrator is not supported');
   }
 
-  const agentCount = proverOptions.agents ? parseInt(proverOptions.agents, 10) : proverOptions.proverAgents;
-  if (agentCount === 0 || !Number.isSafeInteger(agentCount)) {
-    throw new Error('Cannot start prover without agents');
-  }
-
-  let pool: ProverPool;
-  if (proverOptions.simulate) {
-    pool = ProverPool.testPool(undefined, agentCount);
-  } else if (proverOptions.acvmBinaryPath && proverOptions.bbBinaryPath) {
-    pool = ProverPool.nativePool(
-      {
-        acvmBinaryPath: proverOptions.acvmBinaryPath,
-        bbBinaryPath: proverOptions.bbBinaryPath,
-        acvmWorkingDirectory: tmpdir(),
-        bbWorkingDirectory: tmpdir(),
-      },
-      agentCount,
-    );
+  logger(`Connecting to prover at ${proverOptions.nodeUrl}`);
+  const source = createProvingJobSourceClient(proverOptions.nodeUrl, 'provingJobSource');
+
+  const agentConcurrency =
+    // string if it was set as a CLI option, ie start --prover proverAgentConcurrency=10
+    typeof proverOptions.proverAgentConcurrency === 'string'
+      ? parseInt(proverOptions.proverAgentConcurrency, 10)
+      : proverOptions.proverAgentConcurrency;
+
+  const pollInterval =
+    // string if it was set as a CLI option, ie start --prover proverAgentPollInterval=10
+    typeof proverOptions.proverAgentPollInterval === 'string'
+      ? parseInt(proverOptions.proverAgentPollInterval, 10)
+      : proverOptions.proverAgentPollInterval;
+
+  let circuitProver: ServerCircuitProver;
+  if (proverOptions.realProofs) {
+    if (!proverOptions.acvmBinaryPath || !proverOptions.bbBinaryPath) {
+      throw new Error('Cannot start prover without simulation or native prover options');
+    }
+
+    circuitProver = await BBNativeRollupProver.new({
+      acvmBinaryPath: proverOptions.acvmBinaryPath,
+      bbBinaryPath: proverOptions.bbBinaryPath,
+      acvmWorkingDirectory: proverOptions.acvmWorkingDirectory,
+      bbWorkingDirectory: proverOptions.bbWorkingDirectory,
+    });
   } else {
-    throw new Error('Cannot start prover without simulation or native prover options');
+    circuitProver = new TestCircuitProver();
   }
 
-  logger(`Starting ${agentCount} prover agents`);
-  await pool.start(source);
-  signalHandlers.push(() => pool.stop());
+  const agent = new ProverAgent(circuitProver, agentConcurrency, pollInterval);
+  agent.start(source);
+  logger(`Started prover agent with concurrency limit of ${agentConcurrency}`);
+
+  signalHandlers.push(() => agent.stop());
 
   return Promise.resolve([]);
 };

yarn-project/aztec/tsconfig.json

@@ -18,6 +18,9 @@
     {
       "path": "../aztec.js"
     },
+    {
+      "path": "../bb-prover"
+    },
     {
       "path": "../builder"
     },

yarn-project/bb-prover/src/config.ts

@@ -0,0 +1,9 @@
+export interface BBConfig {
+  bbBinaryPath: string;
+  bbWorkingDirectory: string;
+}
+
+export interface ACVMConfig {
+  acvmBinaryPath: string;
+  acvmWorkingDirectory: string;
+}

yarn-project/bb-prover/src/index.ts

@@ -1,3 +1,4 @@
 export * from './prover/index.js';
 export * from './test/index.js';
 export * from './verifier/index.js';
+export * from './config.js';

yarn-project/bb-prover/src/prover/bb_prover.ts

@@ -63,6 +63,7 @@ import {
   verifyProof,
   writeProofAsFields,
 } from '../bb/execute.js';
+import type { ACVMConfig, BBConfig } from '../config.js';
 import { PublicKernelArtifactMapping } from '../mappings/mappings.js';
 import { mapProtocolArtifactNameToCircuitName } from '../stats.js';
 import { extractVkData } from '../verification_key/verification_key_data.js';
@@ -71,14 +72,10 @@ const logger = createDebugLogger('aztec:bb-prover');
 
 const CIRCUITS_WITHOUT_AGGREGATION: Set<ServerProtocolArtifact> = new Set(['BaseParityArtifact']);
 
-export type BBProverConfig = {
-  bbBinaryPath: string;
-  bbWorkingDirectory: string;
-  acvmBinaryPath: string;
-  acvmWorkingDirectory: string;
+export interface BBProverConfig extends BBConfig, ACVMConfig {
   // list of circuits supported by this prover. defaults to all circuits if empty
   circuitFilter?: ServerProtocolArtifact[];
-};
+}
 
 /**
  * Prover implementation that uses barretenberg native proving

yarn-project/bb-prover/src/verifier/bb_verifier.ts

@@ -6,13 +6,9 @@ import { type ProtocolArtifact, ProtocolCircuitArtifacts } from '@aztec/noir-pro
 import * as fs from 'fs/promises';
 
 import { BB_RESULT, generateContractForCircuit, generateKeyForNoirCircuit, verifyProof } from '../bb/execute.js';
+import { type BBConfig } from '../config.js';
 import { extractVkData } from '../verification_key/verification_key_data.js';
 
-export type BBConfig = {
-  bbBinaryPath: string;
-  bbWorkingDirectory: string;
-};
-
 export class BBCircuitVerifier {
   private constructor(
     private config: BBConfig,

yarn-project/circuit-types/src/interfaces/prover-client.ts

@@ -5,10 +5,16 @@ import { type ProvingJobSource } from './proving-job.js';
  * The prover configuration.
  */
 export type ProverConfig = {
-  /** How many agents to run */
-  proverAgents: number;
+  /** The URL to the Aztec node to take proving jobs from */
+  nodeUrl?: string;
   /** Whether to construct real proofs */
   realProofs: boolean;
+  /** Whether this prover has a local prover agent */
+  proverAgentEnabled: boolean;
+  /** The interval agents poll for jobs at */
+  proverAgentPollInterval: number;
+  /** The maximum number of proving jobs to be run in parallel */
+  proverAgentConcurrency: number;
 };
 
 /**

yarn-project/end-to-end/src/benchmarks/bench_prover.test.ts

@@ -1,11 +1,9 @@
 import { getSchnorrAccount, getSchnorrWallet } from '@aztec/accounts/schnorr';
-import { type AztecNodeService } from '@aztec/aztec-node';
 import { TxStatus } from '@aztec/aztec.js';
 import { type AccountWallet } from '@aztec/aztec.js/wallet';
 import { CompleteAddress, Fq, Fr } from '@aztec/circuits.js';
 import { FPCContract, GasTokenContract, TestContract, TokenContract } from '@aztec/noir-contracts.js';
 import { GasTokenAddress } from '@aztec/protocol-contracts/gas-token';
-import { ProverPool } from '@aztec/prover-client/prover-pool';
 import { type PXEService, createPXEService } from '@aztec/pxe';
 
 import { jest } from '@jest/globals';
@@ -39,7 +37,6 @@ describe('benchmarks/proving', () => {
 
   let acvmCleanup: () => Promise<void>;
   let bbCleanup: () => Promise<void>;
-  let proverPool: ProverPool;
 
   // setup the environment quickly using fake proofs
   beforeAll(async () => {
@@ -48,7 +45,7 @@ describe('benchmarks/proving', () => {
       {
         // do setup with fake proofs
         realProofs: false,
-        proverAgents: 4,
+        proverAgentConcurrency: 4,
         proverAgentPollInterval: 10,
         minTxsPerBlock: 1,
       },
@@ -110,25 +107,14 @@ describe('benchmarks/proving', () => {
     acvmCleanup = acvmConfig.cleanup;
     bbCleanup = bbConfig.cleanup;
 
-    proverPool = ProverPool.nativePool(
-      {
-        ...acvmConfig,
-        ...bbConfig,
-      },
-      2,
-      10,
-    );
-
     ctx.logger.info('Stopping fake provers');
     await ctx.aztecNode.setConfig({
-      // stop the fake provers
-      proverAgents: 0,
+      proverAgentConcurrency: 1,
       realProofs: true,
       minTxsPerBlock: 2,
     });
 
     ctx.logger.info('Starting real provers');
-    await proverPool.start((ctx.aztecNode as AztecNodeService).getProver().getProvingJobSource());
 
     ctx.logger.info('Starting PXEs configured with real proofs');
     provingPxes = [];
@@ -161,7 +147,6 @@ describe('benchmarks/proving', () => {
     for (const pxe of provingPxes) {
       await pxe.stop();
     }
-    await proverPool.stop();
     await ctx.teardown();
     await acvmCleanup();
     await bbCleanup();

yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts

@@ -39,7 +39,6 @@ import { AvailabilityOracleAbi, InboxAbi, OutboxAbi, RollupAbi } from '@aztec/l1
 import { SHA256Trunc, StandardTree } from '@aztec/merkle-tree';
 import { TxProver } from '@aztec/prover-client';
 import { type L1Publisher, getL1Publisher } from '@aztec/sequencer-client';
-import { WASMSimulator } from '@aztec/simulator';
 import { MerkleTrees, ServerWorldStateSynchronizer, type WorldStateConfig } from '@aztec/world-state';
 
 import { beforeEach, describe, expect, it } from '@jest/globals';
@@ -145,7 +144,7 @@ describe('L1Publisher integration', () => {
     };
     const worldStateSynchronizer = new ServerWorldStateSynchronizer(tmpStore, builderDb, blockSource, worldStateConfig);
     await worldStateSynchronizer.start();
-    builder = await TxProver.new(config, new WASMSimulator(), worldStateSynchronizer);
+    builder = await TxProver.new(config, worldStateSynchronizer);
     l2Proof = makeEmptyProof();
 
     publisher = getL1Publisher({

yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts

@@ -1,5 +1,4 @@
 import { SchnorrAccountContractArtifact, getSchnorrAccount } from '@aztec/accounts/schnorr';
-import { type AztecNodeService } from '@aztec/aztec-node';
 import {
   type AccountWalletWithSecretKey,
   type AztecNode,
@@ -18,7 +17,6 @@ import {
 import { BBCircuitVerifier, BBNativeProofCreator } from '@aztec/bb-prover';
 import { RollupAbi } from '@aztec/l1-artifacts';
 import { TokenContract } from '@aztec/noir-contracts.js';
-import { ProverPool } from '@aztec/prover-client/prover-pool';
 import { type PXEService } from '@aztec/pxe';
 
 // @ts-expect-error solc-js doesn't publish its types https://github.com/ethereum/solc-js/issues/689
@@ -67,7 +65,6 @@ export class FullProverTest {
   tokenSim!: TokenSimulator;
   aztecNode!: AztecNode;
   pxe!: PXEService;
-  private proverPool!: ProverPool;
   private provenComponents: ProvenSetup[] = [];
   private bbConfigCleanup?: () => Promise<void>;
   private acvmConfigCleanup?: () => Promise<void>;
@@ -153,25 +150,13 @@ export class FullProverTest {
 
     this.circuitProofVerifier = await BBCircuitVerifier.new(bbConfig);
 
-    this.proverPool = ProverPool.nativePool(
-      {
-        ...acvmConfig,
-        ...bbConfig,
-      },
-      4,
-      10,
-    );
-
     this.logger.debug(`Configuring the node for real proofs...`);
     await this.aztecNode.setConfig({
-      // stop the fake provers
-      proverAgents: 0,
+      proverAgentConcurrency: 1,
       realProofs: true,
       minTxsPerBlock: 2, // min 2 txs per block
     });
 
-    await this.proverPool.start((this.aztecNode as AztecNodeService).getProver().getProvingJobSource());
-
     this.proofCreator = new BBNativeProofCreator(bbConfig.bbBinaryPath, bbConfig.bbWorkingDirectory);
 
     this.logger.debug(`Main setup completed, initializing full prover PXE and Node...`);

yarn-project/prover-client/package.json

@@ -4,7 +4,7 @@
   "type": "module",
   "exports": {
     ".": "./dest/index.js",
-    "./prover-pool": "./dest/prover-pool/index.js"
+    "./prover-agent": "./dest/prover-agent/index.js"
   },
   "typedocOptions": {
     "entryPoints": [