feat: add api for inclusion proof of outgoing message in block #4562

.circleci/config.yml

@@ -900,6 +900,18 @@ jobs:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_ordering.test.ts
           aztec_manifest_key: end-to-end
+
+  e2e-outbox:
+    docker:
+      - image: aztecprotocol/alpine-build-image
+    resource_class: small
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: "Test"
+          command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_outbox.test.ts
+          aztec_manifest_key: end-to-end
 
   uniswap-trade-on-l1-from-l2:
     docker:
@@ -1467,6 +1479,7 @@ workflows:
       - e2e-inclusion-proofs-contract: *e2e_test
       - e2e-pending-note-hashes-contract: *e2e_test
       - e2e-ordering: *e2e_test
+      - e2e-outbox: *e2e_test
       - e2e-counter: *e2e_test
       - e2e-private-voting: *e2e_test
       - uniswap-trade-on-l1-from-l2: *e2e_test
@@ -1530,6 +1543,7 @@ workflows:
             - e2e-inclusion-proofs-contract
             - e2e-pending-note-hashes-contract
             - e2e-ordering
+            - e2e-outbox
             - e2e-counter
             - e2e-private-voting
             - uniswap-trade-on-l1-from-l2

yarn-project/aztec-node/src/aztec-node/server.ts

@@ -32,6 +32,7 @@ import {
   Header,
   INITIAL_L2_BLOCK_NUM,
   L1_TO_L2_MSG_TREE_HEIGHT,
+  L2_TO_L1_MESSAGE_LENGTH,
   NOTE_HASH_TREE_HEIGHT,
   NULLIFIER_TREE_HEIGHT,
   NullifierLeafPreimage,
@@ -44,7 +45,8 @@ import { AztecAddress } from '@aztec/foundation/aztec-address';
 import { createDebugLogger } from '@aztec/foundation/log';
 import { AztecKVStore } from '@aztec/kv-store';
 import { AztecLmdbStore } from '@aztec/kv-store/lmdb';
-import { initStoreForRollup } from '@aztec/kv-store/utils';
+import { initStoreForRollup, openTmpStore } from '@aztec/kv-store/utils';
+import { SHA256, StandardTree } from '@aztec/merkle-tree';
 import { AztecKVTxPool, P2P, createP2PClient } from '@aztec/p2p';
 import {
   GlobalVariableBuilder,
@@ -111,7 +113,7 @@ export class AztecNodeService implements AztecNode {
     const log = createDebugLogger('aztec:node');
     const storeLog = createDebugLogger('aztec:node:lmdb');
     const store = await initStoreForRollup(
-      AztecLmdbStore.open(config.dataDirectory, storeLog),
+      AztecLmdbStore.open(config.dataDirectory, false, storeLog),
       config.l1Contracts.rollupAddress,
       storeLog,
     );
@@ -424,6 +426,47 @@ export class AztecNodeService implements AztecNode {
     return committedDb.getSiblingPath(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, leafIndex);
   }
 
+  /**
+   * Returns the index of a l2ToL1Message in a virtual l2 to l1 data tree as well as its sibling path.
+   * This tree is considered virtual because it is created on-demand by: taking all the l2ToL1 messages
+   * in a single block, and then using them to make a variable depth append-only tree with these messages as leaves.
+   * The tree is discarded immediately after calculating what we need from it.
+   * @param blockNumber - The block number at which to get the data.
+   * @param l2ToL1Message - The l2ToL1Message get the index / sibling path for.
+   * @returns A tuple of the index and the sibling path of the L2ToL1Message.
+   */
+  public async getL2ToL1MessageIndexAndSiblingPath(
+    blockNumber: number | 'latest',
+    l2ToL1Message: Fr,
+  ): Promise<[number, SiblingPath<number>]> {
+    const block = await this.blockSource.getBlock(blockNumber === 'latest' ? await this.getBlockNumber() : blockNumber);
+
+    if (block === undefined) {
+      throw new Error('Block is not defined');
+    }
+
+    const l2ToL1Messages = block.body.txEffects.flatMap(txEffect => txEffect.l2ToL1Msgs);
+
+    if (l2ToL1Messages.length !== L2_TO_L1_MESSAGE_LENGTH * block.body.txEffects.length) {
+      throw new Error('L2 to L1 Messages are not padded');
+    }
+
+    const indexOfL2ToL1Message = l2ToL1Messages.findIndex(l2ToL1MessageInBlock =>
+      l2ToL1MessageInBlock.equals(l2ToL1Message),
+    );
+
+    if (indexOfL2ToL1Message === -1) {
+      throw new Error('The L2ToL1Message you are trying to prove inclusion of does not exist');
+    }
+
+    const treeHeight = Math.ceil(Math.log2(l2ToL1Messages.length));
+
+    const tree = new StandardTree(openTmpStore(true), new SHA256(), 'temp_outhash_sibling_path', treeHeight);
+    await tree.appendLeaves(l2ToL1Messages.map(l2ToL1Msg => l2ToL1Msg.toBuffer()));
+
+    return [indexOfL2ToL1Message, await tree.getSiblingPath(BigInt(indexOfL2ToL1Message), true)];
+  }
+
   /**
    * Returns a sibling path for a leaf in the committed blocks tree.
    * @param blockNumber - The block number at which to get the data.

yarn-project/aztec.js/src/index.ts

@@ -115,6 +115,7 @@ export {
   merkleTreeIds,
   mockTx,
   Comparator,
+  SiblingPath,
 } from '@aztec/circuit-types';
 export { NodeInfo } from '@aztec/types/interfaces';
 

yarn-project/aztec/src/cli/cmds/start_archiver.ts

@@ -24,7 +24,7 @@ export const startArchiver = async (options: any, signalHandlers: (() => Promise
 
   const storeLog = createDebugLogger('aztec:archiver:lmdb');
   const store = await initStoreForRollup(
-    AztecLmdbStore.open(archiverConfig.dataDirectory, storeLog),
+    AztecLmdbStore.open(archiverConfig.dataDirectory, false, storeLog),
     archiverConfig.l1Contracts.rollupAddress,
     storeLog,
   );

yarn-project/circuit-types/src/interfaces/aztec-node.ts

@@ -93,6 +93,20 @@ export interface AztecNode {
     leafIndex: bigint,
   ): Promise<SiblingPath<typeof L1_TO_L2_MSG_TREE_HEIGHT>>;
 
+  /**
+   * Returns the index of a l2ToL1Message in a virtual l2 to l1 data tree as well as its sibling path.
+   * This tree is considered virtual because it is created on-demand by: taking all the l2ToL1 messages
+   * in a single block, and then using them to make a variable depth append-only tree with these messages as leaves.
+   * The tree is discarded immediately after calculating what we need from it.
+   * @param blockNumber - The block number at which to get the data.
+   * @param l2ToL1Message - The l2ToL1Message get the index / sibling path for.
+   * @returns A tuple of the index and the sibling path of the L2ToL1Message.
+   */
+  getL2ToL1MessageIndexAndSiblingPath(
+    blockNumber: number | 'latest',
+    l2ToL1Message: Fr,
+  ): Promise<[number, SiblingPath<number>]>;
+
   /**
    * Returns a sibling path for a leaf in the committed historic blocks tree.
    * @param blockNumber - The block number at which to get the data.

yarn-project/end-to-end/src/e2e_outbox.test.ts

@@ -0,0 +1,62 @@
+import { AccountWalletWithPrivateKey, AztecNode, EthAddress, Fr, SiblingPath } from '@aztec/aztec.js';
+import { SHA256 } from '@aztec/merkle-tree';
+import { TestContract } from '@aztec/noir-contracts.js';
+
+import { beforeEach, describe, expect, it } from '@jest/globals';
+
+import { setup } from './fixtures/utils.js';
+
+describe('E2E Outbox Tests', () => {
+  let teardown: () => void;
+  let aztecNode: AztecNode;
+  const sha256 = new SHA256();
+  let contract: TestContract;
+  let wallets: AccountWalletWithPrivateKey[];
+
+  beforeEach(async () => {
+    ({ teardown, aztecNode, wallets } = await setup(1));
+
+    const receipt = await TestContract.deploy(wallets[0]).send().wait();
+    contract = receipt.contract;
+  }, 100_000);
+
+  afterAll(() => teardown());
+
+  it('Inserts a new out message, and verifies sibling paths of both the new message, and its zeroed sibling', async () => {
+    const { blockNumber } = await contract.methods
+      .create_l2_to_l1_message_arbitrary_recipient_public(42069, EthAddress.ZERO)
+      .send()
+      .wait();
+
+    const block = await aztecNode.getBlock(blockNumber!);
+
+    const l2ToL1Messages = block?.body.txEffects.flatMap(txEffect =>
+      txEffect.l2ToL1Msgs.map(l2ToL1Message => l2ToL1Message.toBuffer()),
+    );
+
+    const [, siblingPath] = await aztecNode.getL2ToL1MessageIndexAndSiblingPath(blockNumber!, Fr.ZERO);
+
+    expect(siblingPath.pathSize).toBe(2);
+
+    const expectedSiblingPath = new SiblingPath(siblingPath.pathSize, [
+      l2ToL1Messages![0],
+      sha256.hash(Fr.ZERO.toBuffer(), Fr.ZERO.toBuffer()),
+    ]);
+
+    expect(siblingPath.toString()).toBe(expectedSiblingPath.toString());
+
+    const [, siblingPathAlt] = await aztecNode.getL2ToL1MessageIndexAndSiblingPath(
+      blockNumber!,
+      Fr.fromBuffer(l2ToL1Messages![0]),
+    );
+
+    expect(siblingPathAlt.pathSize).toBe(2);
+
+    const expectedSiblingPathAlt = new SiblingPath(siblingPath.pathSize, [
+      Fr.ZERO.toBuffer(),
+      sha256.hash(Fr.ZERO.toBuffer(), Fr.ZERO.toBuffer()),
+    ]);
+
+    expect(siblingPathAlt.toString()).toBe(expectedSiblingPathAlt.toString());
+  }, 360_000);
+});

yarn-project/kv-store/src/lmdb/store.ts

@@ -45,12 +45,20 @@ export class AztecLmdbStore implements AztecKVStore {
    * different rollup instances.
    *
    * @param path - A path on the disk to store the database. Optional
+   * @param ephemeral - true if the store should only exist in memory and not automatically be flushed to disk. Optional
    * @param log - A logger to use. Optional
    * @returns The store
    */
-  static open(path?: string, log = createDebugLogger('aztec:kv-store:lmdb')): AztecLmdbStore {
+  static open(
+    path?: string,
+    ephemeral: boolean = false,
+    log = createDebugLogger('aztec:kv-store:lmdb'),
+  ): AztecLmdbStore {
     log.info(`Opening LMDB database at ${path || 'temporary location'}`);
-    const rootDb = open({ path });
+    const rootDb = open({
+      path,
+      noSync: ephemeral,
+    });
     return new AztecLmdbStore(rootDb);
   }
 

yarn-project/kv-store/src/utils.ts

@@ -34,8 +34,9 @@ export async function initStoreForRollup<T extends AztecKVStore>(
 
 /**
  * Opens a temporary store for testing purposes.
+ * @param ephemeral - true if the store should only exist in memory and not automatically be flushed to disk. Optional
  * @returns A new store
  */
-export function openTmpStore(): AztecKVStore {
-  return AztecLmdbStore.open();
+export function openTmpStore(ephemeral: boolean = false): AztecKVStore {
+  return AztecLmdbStore.open(undefined, ephemeral);
 }

yarn-project/merkle-tree/src/index.ts

@@ -3,6 +3,7 @@ export * from './interfaces/indexed_tree.js';
 export * from './interfaces/merkle_tree.js';
 export * from './interfaces/update_only_tree.js';
 export * from './pedersen.js';
+export * from './sha_256.js';
 export * from './sparse_tree/sparse_tree.js';
 export { StandardIndexedTree } from './standard_indexed_tree/standard_indexed_tree.js';
 export { StandardIndexedTreeWithAppend } from './standard_indexed_tree/test/standard_indexed_tree_with_append.js';

yarn-project/merkle-tree/src/sha_256.ts

@@ -0,0 +1,25 @@
+import { sha256 } from '@aztec/foundation/crypto';
+import { Hasher } from '@aztec/types/interfaces';
+
+/**
+ * A helper class encapsulating SHA256 hash functionality.
+ * @deprecated Don't call SHA256 directly in production code. Instead, create suitably-named functions for specific
+ * purposes.
+ */
+export class SHA256 implements Hasher {
+  /*
+   * @deprecated Don't call SHA256 directly in production code. Instead, create suitably-named functions for specific
+   * purposes.
+   */
+  public hash(lhs: Uint8Array, rhs: Uint8Array): Buffer {
+    return sha256(Buffer.concat([Buffer.from(lhs), Buffer.from(rhs)]));
+  }
+
+  /*
+   * @deprecated Don't call SHA256 directly in production code. Instead, create suitably-named functions for specific
+   * purposes.
+   */
+  public hashInputs(inputs: Buffer[]): Buffer {
+    return sha256(Buffer.concat(inputs));
+  }
+}