feat: add a prover-node to the proving e2e tests

.github/workflows/devnet-deploys.yml

@@ -75,7 +75,7 @@ env:
   TF_VAR_BOT_PRIVATE_TRANSFERS_PER_TX: 0 # no private transfers
   TF_VAR_BOT_PUBLIC_TRANSFERS_PER_TX: 1
   TF_VAR_BOT_TX_MINED_WAIT_SECONDS: 2400
-  TF_VAR_BOT_NO_WAIT_FOR_TRANSFERS: true
+  TF_VAR_BOT_FOLLOW_CHAIN: "PROVEN"
   TF_VAR_BOT_TX_INTERVAL_SECONDS: 180
   TF_VAR_BOT_COUNT: 1
 

barretenberg/cpp/src/barretenberg/bb/main.cpp

@@ -1447,6 +1447,9 @@ int main(int argc, char* argv[])
         } else if (command == "prove_keccak_ultra_honk") {
             std::string output_path = get_option(args, "-o", "./proofs/proof");
             prove_honk<UltraKeccakFlavor>(bytecode_path, witness_path, output_path);
+        } else if (command == "prove_keccak_ultra_honk_output_all") {
+            std::string output_path = get_option(args, "-o", "./proofs/proof");
+            prove_honk_output_all<UltraKeccakFlavor>(bytecode_path, witness_path, output_path);
         } else if (command == "verify_ultra_honk") {
             return verify_honk<UltraFlavor>(proof_path, vk_path) ? 0 : 1;
         } else if (command == "verify_keccak_ultra_honk") {

yarn-project/aztec/terraform/bot/main.tf

@@ -167,7 +167,7 @@ resource "aws_ecs_task_definition" "aztec-bot" {
         { name = "BOT_PRIVATE_TRANSFERS_PER_TX", value = var.BOT_PRIVATE_TRANSFERS_PER_TX },
         { name = "BOT_PUBLIC_TRANSFERS_PER_TX", value = var.BOT_PUBLIC_TRANSFERS_PER_TX },
         { name = "BOT_TX_MINED_WAIT_SECONDS", value = var.BOT_TX_MINED_WAIT_SECONDS },
-        { name = "BOT_NO_WAIT_FOR_TRANSFERS", value = var.BOT_NO_WAIT_FOR_TRANSFERS },
+        { name = "BOT_FOLLOW_CHAIN", value = var.BOT_FOLLOW_CHAIN },
         { name = "AZTEC_NODE_URL", value = "http://${var.DEPLOY_TAG}-aztec-node-1.local/${var.DEPLOY_TAG}/aztec-node-1/${var.API_KEY}" },
         { name = "PXE_PROVER_ENABLED", value = tostring(var.PROVING_ENABLED) },
         { name = "NETWORK", value = var.DEPLOY_TAG }

yarn-project/aztec/terraform/bot/variables.tf

@@ -43,9 +43,8 @@ variable "BOT_TX_MINED_WAIT_SECONDS" {
   type = string
 }
 
-variable "BOT_NO_WAIT_FOR_TRANSFERS" {
-  type    = string
-  default = true
+variable "BOT_FOLLOW_CHAIN" {
+  type = string
 }
 
 variable "PROVING_ENABLED" {

yarn-project/bot/src/bot.ts

@@ -71,13 +71,20 @@ export class Bot {
 
     const txHash = await tx.getTxHash();
 
-    if (this.config.noWaitForTransfers) {
+    if (this.config.followChain === 'NONE') {
       this.log.info(`Transaction ${txHash} sent, not waiting for it to be mined`);
       return;
     }
 
-    this.log.verbose(`Awaiting tx ${txHash} to be mined (timeout ${this.config.txMinedWaitSeconds}s)`, logCtx);
-    const receipt = await tx.wait({ timeout: this.config.txMinedWaitSeconds });
+    this.log.verbose(
+      `Awaiting tx ${txHash} to be on the ${this.config.followChain} (timeout ${this.config.txMinedWaitSeconds}s)`,
+      logCtx,
+    );
+    const receipt = await tx.wait({
+      timeout: this.config.txMinedWaitSeconds,
+      provenTimeout: this.config.txMinedWaitSeconds,
+      proven: this.config.followChain === 'PROVEN',
+    });
     this.log.info(`Tx ${receipt.txHash} mined in block ${receipt.blockNumber}`, logCtx);
   }
 

yarn-project/bot/src/config.ts

@@ -7,6 +7,9 @@ import {
   numberConfigHelper,
 } from '@aztec/foundation/config';
 
+const botFollowChain = ['NONE', 'PENDING', 'PROVEN'] as const;
+type BotFollowChain = (typeof botFollowChain)[number];
+
 export type BotConfig = {
   /** URL to the PXE for sending txs, or undefined if an in-proc PXE is used. */
   pxeUrl: string | undefined;
@@ -28,8 +31,7 @@ export type BotConfig = {
   noStart: boolean;
   /** How long to wait for a tx to be mined before reporting an error. */
   txMinedWaitSeconds: number;
-  /** Don't wait for transfer transactions. */
-  noWaitForTransfers: boolean;
+  followChain: BotFollowChain;
 };
 
 export const botConfigMappings: ConfigMappingsType<BotConfig> = {
@@ -86,10 +88,16 @@ export const botConfigMappings: ConfigMappingsType<BotConfig> = {
     description: 'How long to wait for a tx to be mined before reporting an error.',
     ...numberConfigHelper(180),
   },
-  noWaitForTransfers: {
-    env: 'BOT_NO_WAIT_FOR_TRANSFERS',
-    description: "Don't wait for transfer transactions.",
-    ...booleanConfigHelper(),
+  followChain: {
+    env: 'BOT_FOLLOW_CHAIN',
+    description: 'Which chain the bot follows',
+    defaultValue: 'none',
+    parseEnv(val) {
+      if (!botFollowChain.includes(val as any)) {
+        throw new Error(`Invalid value for BOT_FOLLOW_CHAIN: ${val}`);
+      }
+      return val as BotFollowChain;
+    },
   },
 };
 

yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts

@@ -1,9 +1,11 @@
 import { SchnorrAccountContractArtifact, getSchnorrAccount } from '@aztec/accounts/schnorr';
+import { type Archiver, createArchiver } from '@aztec/archiver';
 import {
   type AccountWalletWithSecretKey,
   type AztecNode,
   type CompleteAddress,
   type DebugLogger,
+  type DeployL1Contracts,
   ExtendedNote,
   type Fq,
   Fr,
@@ -15,9 +17,12 @@ import {
   deployL1Contract,
 } from '@aztec/aztec.js';
 import { BBCircuitVerifier } from '@aztec/bb-prover';
+import { createStore } from '@aztec/kv-store/utils';
 import { RollupAbi } from '@aztec/l1-artifacts';
 import { TokenContract } from '@aztec/noir-contracts.js';
+import { type ProverNode, type ProverNodeConfig, createProverNode } from '@aztec/prover-node';
 import { type PXEService } from '@aztec/pxe';
+import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
 
 // TODO(#7373): Deploy honk solidity verifier
 // @ts-expect-error solc-js doesn't publish its types https://github.com/ethereum/solc-js/issues/689
@@ -34,7 +39,7 @@ import {
   createSnapshotManager,
   publicDeployAccounts,
 } from '../fixtures/snapshot_manager.js';
-import { setupPXEService } from '../fixtures/utils.js';
+import { getPrivateKeyFromIndex, setupPXEService } from '../fixtures/utils.js';
 import { TokenSimulator } from '../simulators/token_simulator.js';
 
 const { E2E_DATA_PATH: dataPath } = process.env;
@@ -72,6 +77,9 @@ export class FullProverTest {
   circuitProofVerifier?: BBCircuitVerifier;
   provenAssets: TokenContract[] = [];
   private context!: SubsystemsContext;
+  private proverNode!: ProverNode;
+  private simulatedProverNode!: ProverNode;
+  private l1Contracts!: DeployL1Contracts;
 
   constructor(testName: string, private minNumberOfTxsPerBlock: number) {
     this.logger = createDebugLogger(`aztec:full_prover_test:${testName}`);
@@ -109,7 +117,7 @@ export class FullProverTest {
           FullProverTest.TOKEN_DECIMALS,
         )
           .send()
-          .deployed();
+          .deployed({ proven: true });
         this.logger.verbose(`Token deployed to ${asset.address}`);
 
         return { tokenContractAddress: asset.address };
@@ -133,7 +141,12 @@ export class FullProverTest {
 
   async setup() {
     this.context = await this.snapshotManager.setup();
-    ({ pxe: this.pxe, aztecNode: this.aztecNode } = this.context);
+    ({
+      pxe: this.pxe,
+      aztecNode: this.aztecNode,
+      proverNode: this.simulatedProverNode,
+      deployL1ContractsValues: this.l1Contracts,
+    } = this.context);
 
     // Configure a full prover PXE
 
@@ -153,12 +166,11 @@ export class FullProverTest {
 
     this.logger.debug(`Configuring the node for real proofs...`);
     await this.aztecNode.setConfig({
-      proverAgentConcurrency: 2,
       realProofs: true,
       minTxsPerBlock: this.minNumberOfTxsPerBlock,
     });
 
-    this.logger.debug(`Main setup completed, initializing full prover PXE and Node...`);
+    this.logger.debug(`Main setup completed, initializing full prover PXE, Node, and Prover Node...`);
 
     for (let i = 0; i < 2; i++) {
       const result = await setupPXEService(
@@ -204,7 +216,45 @@ export class FullProverTest {
       this.provenAssets.push(asset);
     }
 
-    this.logger.debug(`Full prover PXE started!!`);
+    this.logger.info(`Full prover PXE started`);
+
+    // Shutdown the current, simulated prover node
+    this.logger.verbose('Shutting down simulated prover node');
+    await this.simulatedProverNode.stop();
+
+    // Creating temp store and archiver for fully proven prover node
+
+    this.logger.verbose('Starting archiver for new prover node');
+    const store = await createStore({ dataDirectory: undefined }, this.l1Contracts.l1ContractAddresses.rollupAddress);
+
+    const archiver = await createArchiver(
+      { ...this.context.aztecNodeConfig, dataDirectory: undefined },
+      store,
+      new NoopTelemetryClient(),
+      { blockUntilSync: true },
+    );
+
+    // The simulated prover node (now shutdown) used private key index 2
+    const proverNodePrivateKey = getPrivateKeyFromIndex(2);
+
+    this.logger.verbose('Starting fully proven prover node');
+    const proverConfig: ProverNodeConfig = {
+      ...this.context.aztecNodeConfig,
+      txProviderNodeUrl: undefined,
+      dataDirectory: undefined,
+      proverId: new Fr(81),
+      realProofs: true,
+      proverAgentConcurrency: 2,
+      publisherPrivateKey: `0x${proverNodePrivateKey!.toString('hex')}`,
+    };
+    this.proverNode = await createProverNode(proverConfig, {
+      aztecNodeTxProvider: this.aztecNode,
+      archiver: archiver as Archiver,
+    });
+    this.proverNode.start();
+
+    this.logger.info('Prover node started');
+
     return this;
   }
 
@@ -222,6 +272,9 @@ export class FullProverTest {
       await this.provenComponents[i].teardown();
     }
 
+    // clean up the full prover node
+    await this.proverNode.stop();
+
     await this.bbConfigCleanup?.();
     await this.acvmConfigCleanup?.();
   }
@@ -246,17 +299,19 @@ export class FullProverTest {
         const { fakeProofsAsset: asset, accounts } = this;
         const amount = 10000n;
 
+        const waitOpts = { proven: true };
+
         this.logger.verbose(`Minting ${amount} publicly...`);
-        await asset.methods.mint_public(accounts[0].address, amount).send().wait();
+        await asset.methods.mint_public(accounts[0].address, amount).send().wait(waitOpts);
 
         this.logger.verbose(`Minting ${amount} privately...`);
         const secret = Fr.random();
         const secretHash = computeSecretHash(secret);
-        const receipt = await asset.methods.mint_private(amount, secretHash).send().wait();
+        const receipt = await asset.methods.mint_private(amount, secretHash).send().wait(waitOpts);
 
         await this.addPendingShieldNoteToPXE(0, amount, secretHash, receipt.txHash);
         const txClaim = asset.methods.redeem_shield(accounts[0].address, amount, secret).send();
-        await txClaim.wait({ debug: true });
+        await txClaim.wait({ debug: true, proven: true });
         this.logger.verbose(`Minting complete.`);
 
         return { amount };

yarn-project/end-to-end/src/e2e_prover/full.test.ts

@@ -18,7 +18,7 @@ describe('full_prover', () => {
     await t.applyBaseSnapshots();
     await t.applyMintSnapshot();
     await t.setup();
-    await t.deployVerifier();
+    // await t.deployVerifier();
     ({ provenAssets, accounts, tokenSim, logger } = t);
   });
 
@@ -63,8 +63,8 @@ describe('full_prover', () => {
       const sentPrivateTx = privateInteraction.send({ skipPublicSimulation: true });
       const sentPublicTx = publicInteraction.send({ skipPublicSimulation: true });
       await Promise.all([
-        sentPrivateTx.wait({ timeout: 1200, interval: 10 }),
-        sentPublicTx.wait({ timeout: 1200, interval: 10 }),
+        sentPrivateTx.wait({ timeout: 60, interval: 10, proven: true, provenTimeout: 1200 }),
+        sentPublicTx.wait({ timeout: 60, interval: 10, proven: true, provenTimeout: 1200 }),
       ]);
       tokenSim.transferPrivate(accounts[0].address, accounts[1].address, privateSendAmount);
       tokenSim.transferPublic(accounts[0].address, accounts[1].address, publicSendAmount);

yarn-project/end-to-end/src/e2e_prover/with_padding.test.ts

@@ -10,7 +10,7 @@ describe('full_prover_with_padding_tx', () => {
     await t.applyBaseSnapshots();
     await t.applyMintSnapshot();
     await t.setup();
-    await t.deployVerifier();
+    // await t.deployVerifier();
     ({ provenAssets, accounts, tokenSim, logger } = t);
   });
 

yarn-project/end-to-end/src/e2e_prover_node.test.ts

@@ -5,10 +5,8 @@ import {
   type AztecAddress,
   type DebugLogger,
   EthAddress,
-  type FieldsOf,
   Fr,
   SignerlessWallet,
-  type TxReceipt,
   computeSecretHash,
   createDebugLogger,
   retryUntil,
@@ -35,7 +33,6 @@ describe('e2e_prover_node', () => {
   let recipient: AztecAddress;
   let contract: StatefulTestContract;
   let msgTestContract: TestContract;
-  let txReceipts: FieldsOf<TxReceipt>[];
 
   let logger: DebugLogger;
   let snapshotManager: ISnapshotManager;
@@ -84,42 +81,28 @@ describe('e2e_prover_node', () => {
       },
     );
 
-    await snapshotManager.snapshot(
-      'create-blocks',
-      async ctx => {
-        const msgSender = ctx.deployL1ContractsValues.walletClient.account.address;
-        const txReceipt1 = await msgTestContract.methods
-          .consume_message_from_arbitrary_sender_private(msgContent, msgSecret, EthAddress.fromString(msgSender))
-          .send()
-          .wait();
-        const txReceipt2 = await contract.methods.create_note(recipient, recipient, 10).send().wait();
-        const txReceipt3 = await contract.methods.increment_public_value(recipient, 20).send().wait();
-        return { txReceipts: [txReceipt1, txReceipt2, txReceipt3] };
-      },
-      data => {
-        txReceipts = data.txReceipts;
-        return Promise.resolve();
-      },
-    );
-
     ctx = await snapshotManager.setup();
   });
 
   it('submits three blocks, then prover proves the first two', async () => {
+    // Stop the current prover node
+    await ctx.proverNode.stop();
+
+    const msgSender = ctx.deployL1ContractsValues.walletClient.account.address;
+    const txReceipt1 = await msgTestContract.methods
+      .consume_message_from_arbitrary_sender_private(msgContent, msgSecret, EthAddress.fromString(msgSender))
+      .send()
+      .wait();
+    const txReceipt2 = await contract.methods.create_note(recipient, recipient, 10).send().wait();
+    const txReceipt3 = await contract.methods.increment_public_value(recipient, 20).send().wait();
+
     // Check everything went well during setup and txs were mined in two different blocks
-    const [txReceipt1, txReceipt2, txReceipt3] = txReceipts;
     const firstBlock = txReceipt1.blockNumber!;
     const secondBlock = firstBlock + 1;
     expect(txReceipt2.blockNumber).toEqual(secondBlock);
     expect(txReceipt3.blockNumber).toEqual(firstBlock + 2);
     expect(await contract.methods.get_public_value(recipient).simulate()).toEqual(20n);
     expect(await contract.methods.summed_values(recipient).simulate()).toEqual(10n);
-    expect(await ctx.aztecNode.getProvenBlockNumber()).toEqual(0);
-
-    // Trick archiver into thinking everything has been proven up to this point.
-    // TODO: Add cheat code to flag current block as proven on L1, which will be needed when we assert on L1 that proofs do not have any gaps.
-    await (ctx.aztecNode.getBlockSource() as Archiver).setProvenBlockNumber(firstBlock - 1);
-    expect(await ctx.aztecNode.getProvenBlockNumber()).toEqual(firstBlock - 1);
 
     // Kick off a prover node
     await sleep(1000);
@@ -141,13 +124,20 @@ describe('e2e_prover_node', () => {
     await expect(proverNode.startProof(firstBlock, firstBlock)).rejects.toThrow(/behind the current world state/i);
 
     // Await until proofs get submitted
-    await retryUntil(async () => (await ctx.aztecNode.getProvenBlockNumber()) === secondBlock, 'proven', 10, 1);
+    await retryUntil(async () => (await ctx.aztecNode.getProvenBlockNumber()) === secondBlock, 'proven', 60, 1);
     expect(await ctx.aztecNode.getProvenBlockNumber()).toEqual(secondBlock);
 
     // Check that the prover id made it to the emitted event
     const { publicClient, l1ContractAddresses } = ctx.deployL1ContractsValues;
     const logs = await retrieveL2ProofVerifiedEvents(publicClient, l1ContractAddresses.rollupAddress, 1n);
-    expect(logs[0].l2BlockNumber).toEqual(BigInt(firstBlock));
-    expect(logs[0].proverId.toString()).toEqual(proverId.toString());
+    expect(logs.length).toEqual(secondBlock);
+
+    const expectedBlockNumbers = [firstBlock, secondBlock];
+    const logsSlice = logs.slice(firstBlock - 1);
+    for (let i = 0; i < 2; i++) {
+      const log = logsSlice[i];
+      expect(log.l2BlockNumber).toEqual(BigInt(expectedBlockNumbers[i]));
+      expect(log.proverId.toString()).toEqual(proverId.toString());
+    }
   });
 });