diff --git a/cspell.json b/cspell.json index b489357e0e5..f6c12ff5d82 100644 --- a/cspell.json +++ b/cspell.json @@ -79,6 +79,7 @@ "erc", "falsey", "fargate", + "Fieldeable", "filestat", "flatmap", "foundryup", diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 03966222617..93317645c7c 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -458,8 +458,15 @@ export class AztecNodeService implements AztecNode { const treeHeight = Math.ceil(Math.log2(l2ToL1Messages.length)); // The root of this tree is the out_hash calculated in Noir => we truncate to match Noir's SHA - const tree = new StandardTree(openTmpStore(true), new SHA256Trunc(), 'temp_outhash_sibling_path', treeHeight); - await tree.appendLeaves(l2ToL1Messages.map(l2ToL1Msg => l2ToL1Msg.toBuffer())); + const tree = new StandardTree( + openTmpStore(true), + new SHA256Trunc(), + 'temp_outhash_sibling_path', + treeHeight, + 0n, + Fr, + ); + await tree.appendLeaves(l2ToL1Messages); return [indexOfL2ToL1Message, await tree.getSiblingPath(indexOfL2ToL1Message, true)]; } diff --git a/yarn-project/end-to-end/src/e2e_blacklist_token_contract.test.ts b/yarn-project/end-to-end/src/e2e_blacklist_token_contract.test.ts index 4fbf7bacd4f..bd5bcb70637 100644 --- a/yarn-project/end-to-end/src/e2e_blacklist_token_contract.test.ts +++ b/yarn-project/end-to-end/src/e2e_blacklist_token_contract.test.ts @@ -38,14 +38,14 @@ describe('e2e_blacklist_token_contract', () => { let tokenSim: TokenSimulator; - let slowUpdateTreeSimulator: SparseTree; + let slowUpdateTreeSimulator: SparseTree; let cheatCodes: CheatCodes; const getMembershipProof = async (index: bigint, includeUncommitted: boolean) => { return { index, - value: Fr.fromBuffer(slowUpdateTreeSimulator.getLeafValue(index, includeUncommitted)!), + value: slowUpdateTreeSimulator.getLeafValue(index, includeUncommitted)!, // eslint-disable-next-line camelcase sibling_path: (await slowUpdateTreeSimulator.getSiblingPath(index, includeUncommitted)).toFields(), }; @@ -101,9 +101,9 @@ describe('e2e_blacklist_token_contract', () => { await wallets[accountIndex].addNote(extendedNote); }; - const updateSlowTree = async (tree: SparseTree, wallet: Wallet, index: AztecAddress, value: bigint) => { + const updateSlowTree = async (tree: SparseTree, wallet: Wallet, index: AztecAddress, value: bigint) => { await wallet.addCapsule(getUpdateCapsule(await getUpdateProof(value, index.toBigInt()))); - await tree.updateLeaf(new Fr(value).toBuffer(), index.toBigInt()); + await tree.updateLeaf(new Fr(value), index.toBigInt()); }; beforeAll(async () => { @@ -113,7 +113,7 @@ describe('e2e_blacklist_token_contract', () => { slowTree = await SlowTreeContract.deploy(wallets[0]).send().deployed(); const depth = 254; - slowUpdateTreeSimulator = await newTree(SparseTree, openTmpStore(), new Pedersen(), 'test', depth); + slowUpdateTreeSimulator = await newTree(SparseTree, openTmpStore(), new Pedersen(), 'test', Fr, depth); // Add account[0] as admin await updateSlowTree(slowUpdateTreeSimulator, wallets[0], accounts[0].address, 4n); diff --git a/yarn-project/end-to-end/src/e2e_dapp_subscription.test.ts b/yarn-project/end-to-end/src/e2e_dapp_subscription.test.ts index 37ed31610a7..000b96e116a 100644 --- a/yarn-project/end-to-end/src/e2e_dapp_subscription.test.ts +++ b/yarn-project/end-to-end/src/e2e_dapp_subscription.test.ts @@ -28,7 +28,7 @@ import { setup, } from './fixtures/utils.js'; -jest.setTimeout(1_000_000); +jest.setTimeout(100_000); const TOKEN_NAME = 'BananaCoin'; const TOKEN_SYMBOL = 'BC'; diff --git a/yarn-project/end-to-end/src/e2e_slow_tree.test.ts b/yarn-project/end-to-end/src/e2e_slow_tree.test.ts index ffc41d35586..4e4d1c80f79 100644 --- a/yarn-project/end-to-end/src/e2e_slow_tree.test.ts +++ b/yarn-project/end-to-end/src/e2e_slow_tree.test.ts @@ -23,11 +23,11 @@ describe('e2e_slow_tree', () => { it('Messing around with noir slow tree', async () => { const depth = 254; - const slowUpdateTreeSimulator = await newTree(SparseTree, openTmpStore(), new Pedersen(), 'test', depth); + const slowUpdateTreeSimulator = await newTree(SparseTree, openTmpStore(), new Pedersen(), 'test', Fr, depth); const getMembershipProof = async (index: bigint, includeUncommitted: boolean) => { return { index, - value: Fr.fromBuffer(slowUpdateTreeSimulator.getLeafValue(index, includeUncommitted)!), + value: slowUpdateTreeSimulator.getLeafValue(index, includeUncommitted)!, // eslint-disable-next-line camelcase sibling_path: (await slowUpdateTreeSimulator.getSiblingPath(index, includeUncommitted)).toFields(), }; @@ -102,7 +102,7 @@ describe('e2e_slow_tree', () => { .update_at_public(await getUpdateProof(1n, key)) .send() .wait(); - await slowUpdateTreeSimulator.updateLeaf(new Fr(1).toBuffer(), key); + await slowUpdateTreeSimulator.updateLeaf(new Fr(1), key); // Update below. _root = { @@ -140,7 +140,7 @@ describe('e2e_slow_tree', () => { const t2 = computeNextChange(BigInt(await cheatCodes.eth.timestamp())); await wallet.addCapsule(getUpdateCapsule(await getUpdateProof(4n, key))); await contract.methods.update_at_private(key, 4n).send().wait(); - await slowUpdateTreeSimulator.updateLeaf(new Fr(4).toBuffer(), key); + await slowUpdateTreeSimulator.updateLeaf(new Fr(4), key); _root = { before: Fr.fromBuffer(slowUpdateTreeSimulator.getRoot(false)).toBigInt(), after: Fr.fromBuffer(slowUpdateTreeSimulator.getRoot(true)).toBigInt(), diff --git a/yarn-project/end-to-end/src/integration_l1_publisher.test.ts b/yarn-project/end-to-end/src/integration_l1_publisher.test.ts index 3361d04a52f..050cf0da6bf 100644 --- a/yarn-project/end-to-end/src/integration_l1_publisher.test.ts +++ b/yarn-project/end-to-end/src/integration_l1_publisher.test.ts @@ -428,8 +428,15 @@ describe('L1Publisher integration', () => { const treeHeight = Math.ceil(Math.log2(newL2ToL1MsgsArray.length)); - const tree = new StandardTree(openTmpStore(true), new SHA256Trunc(), 'temp_outhash_sibling_path', treeHeight); - await tree.appendLeaves(newL2ToL1MsgsArray.map(l2ToL1Msg => l2ToL1Msg.toBuffer())); + const tree = new StandardTree( + openTmpStore(true), + new SHA256Trunc(), + 'temp_outhash_sibling_path', + treeHeight, + 0n, + Fr, + ); + await tree.appendLeaves(newL2ToL1MsgsArray); const expectedRoot = tree.getRoot(true); const [actualRoot] = await outbox.read.roots([block.header.globalVariables.blockNumber.toBigInt()]); diff --git a/yarn-project/foundation/src/serialize/buffer_reader.ts b/yarn-project/foundation/src/serialize/buffer_reader.ts index 0e637d9e69d..d3bb2e12bea 100644 --- a/yarn-project/foundation/src/serialize/buffer_reader.ts +++ b/yarn-project/foundation/src/serialize/buffer_reader.ts @@ -312,3 +312,14 @@ export class BufferReader { return this.buffer.length; } } + +/** + * A deserializer + */ +export interface FromBuffer { + /** + * Deserializes an object from a buffer + * @param buffer - The buffer to deserialize. + */ + fromBuffer(buffer: Buffer): T; +} diff --git a/yarn-project/merkle-tree/src/interfaces/append_only_tree.ts b/yarn-project/merkle-tree/src/interfaces/append_only_tree.ts index 77dd7ae9e5d..ba6d0ad983b 100644 --- a/yarn-project/merkle-tree/src/interfaces/append_only_tree.ts +++ b/yarn-project/merkle-tree/src/interfaces/append_only_tree.ts @@ -1,13 +1,17 @@ -import { TreeSnapshotBuilder } from '../snapshots/snapshot_builder.js'; +import { Bufferable } from '@aztec/foundation/serialize'; + +import { TreeSnapshot, TreeSnapshotBuilder } from '../snapshots/snapshot_builder.js'; import { MerkleTree } from './merkle_tree.js'; /** * A Merkle tree that supports only appending leaves and not updating existing leaves. */ -export interface AppendOnlyTree extends MerkleTree, TreeSnapshotBuilder { +export interface AppendOnlyTree + extends MerkleTree, + TreeSnapshotBuilder> { /** * Appends a set of leaf values to the tree. * @param leaves - The set of leaves to be appended. */ - appendLeaves(leaves: Buffer[]): Promise; + appendLeaves(leaves: T[]): Promise; } diff --git a/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts b/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts index 13c14f059dc..44bd849479b 100644 --- a/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts +++ b/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts @@ -1,7 +1,9 @@ import { SiblingPath } from '@aztec/circuit-types'; import { IndexedTreeLeaf, IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; +import { IndexedTreeSnapshot, TreeSnapshot, TreeSnapshotBuilder } from '../snapshots/snapshot_builder.js'; import { AppendOnlyTree } from './append_only_tree.js'; +import { MerkleTree } from './merkle_tree.js'; /** * Factory for creating leaf preimages. @@ -73,7 +75,10 @@ export interface BatchInsertionResult, + TreeSnapshotBuilder, + Omit, keyof TreeSnapshotBuilder>> { /** * Finds the index of the largest leaf whose value is less than or equal to the provided value. * @param newValue - The new value to be inserted into the tree. diff --git a/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts b/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts index 912158a70cf..dd5c30a78f0 100644 --- a/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts +++ b/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts @@ -1,4 +1,5 @@ import { SiblingPath } from '@aztec/circuit-types'; +import { Bufferable } from '@aztec/foundation/serialize'; /** * Defines the interface for a source of sibling paths. @@ -15,7 +16,7 @@ export interface SiblingPathSource { /** * Defines the interface for a Merkle tree. */ -export interface MerkleTree extends SiblingPathSource { +export interface MerkleTree extends SiblingPathSource { /** * Returns the current root of the tree. * @param includeUncommitted - Set to true to include uncommitted updates in the calculated root. @@ -48,7 +49,7 @@ export interface MerkleTree extends SiblingPathSource { * @param index - The index of the leaf value to be returned. * @param includeUncommitted - Set to true to include uncommitted updates in the data set. */ - getLeafValue(index: bigint, includeUncommitted: boolean): Buffer | undefined; + getLeafValue(index: bigint, includeUncommitted: boolean): T | undefined; /** * Returns the index of a leaf given its value, or undefined if no leaf with that value is found. @@ -56,7 +57,7 @@ export interface MerkleTree extends SiblingPathSource { * @param includeUncommitted - Indicates whether to include uncommitted data. * @returns The index of the first leaf found with a given value (undefined if not found). */ - findLeafIndex(leaf: Buffer, includeUncommitted: boolean): bigint | undefined; + findLeafIndex(leaf: T, includeUncommitted: boolean): bigint | undefined; /** * Returns the first index containing a leaf value after `startIndex`. @@ -65,5 +66,5 @@ export interface MerkleTree extends SiblingPathSource { * @param includeUncommitted - Indicates whether to include uncommitted data. * @returns The index of the first leaf found with a given value (undefined if not found). */ - findLeafIndexAfter(leaf: Buffer, startIndex: bigint, includeUncommitted: boolean): bigint | undefined; + findLeafIndexAfter(leaf: T, startIndex: bigint, includeUncommitted: boolean): bigint | undefined; } diff --git a/yarn-project/merkle-tree/src/interfaces/update_only_tree.ts b/yarn-project/merkle-tree/src/interfaces/update_only_tree.ts index 06ce3a24096..f1e40d37dbd 100644 --- a/yarn-project/merkle-tree/src/interfaces/update_only_tree.ts +++ b/yarn-project/merkle-tree/src/interfaces/update_only_tree.ts @@ -1,14 +1,18 @@ -import { TreeSnapshotBuilder } from '../snapshots/snapshot_builder.js'; +import { Bufferable } from '@aztec/foundation/serialize'; + +import { TreeSnapshot, TreeSnapshotBuilder } from '../snapshots/snapshot_builder.js'; import { MerkleTree } from './merkle_tree.js'; /** * A Merkle tree that supports updates at arbitrary indices but not appending. */ -export interface UpdateOnlyTree extends MerkleTree, TreeSnapshotBuilder { +export interface UpdateOnlyTree + extends MerkleTree, + TreeSnapshotBuilder> { /** * Updates a leaf at a given index in the tree. * @param leaf - The leaf value to be updated. * @param index - The leaf to be updated. */ - updateLeaf(leaf: Buffer, index: bigint): Promise; + updateLeaf(leaf: T, index: bigint): Promise; } diff --git a/yarn-project/merkle-tree/src/load_tree.ts b/yarn-project/merkle-tree/src/load_tree.ts index a4bf1e8853a..043d242f43a 100644 --- a/yarn-project/merkle-tree/src/load_tree.ts +++ b/yarn-project/merkle-tree/src/load_tree.ts @@ -1,3 +1,4 @@ +import { Bufferable, FromBuffer } from '@aztec/foundation/serialize'; import { AztecKVStore } from '@aztec/kv-store'; import { Hasher } from '@aztec/types/interfaces'; @@ -11,13 +12,22 @@ import { TreeBase, getTreeMeta } from './tree_base.js'; * @param name - Name of the tree. * @returns The newly created tree. */ -export function loadTree( - c: new (store: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint, root: Buffer) => T, +export function loadTree, D extends FromBuffer>( + c: new ( + store: AztecKVStore, + hasher: Hasher, + name: string, + depth: number, + size: bigint, + deserializer: D, + root: Buffer, + ) => T, store: AztecKVStore, hasher: Hasher, name: string, + deserializer: D, ): Promise { const { root, depth, size } = getTreeMeta(store, name); - const tree = new c(store, hasher, name, depth, size, root); + const tree = new c(store, hasher, name, depth, size, deserializer, root); return Promise.resolve(tree); } diff --git a/yarn-project/merkle-tree/src/new_tree.ts b/yarn-project/merkle-tree/src/new_tree.ts index 5c354e21851..a4d73af3693 100644 --- a/yarn-project/merkle-tree/src/new_tree.ts +++ b/yarn-project/merkle-tree/src/new_tree.ts @@ -1,3 +1,4 @@ +import { Bufferable, FromBuffer } from '@aztec/foundation/serialize'; import { AztecKVStore } from '@aztec/kv-store'; import { Hasher } from '@aztec/types/interfaces'; @@ -13,15 +14,16 @@ import { TreeBase } from './tree_base.js'; * @param prefilledSize - A number of leaves that are prefilled with values. * @returns The newly created tree. */ -export async function newTree( - c: new (store: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint) => T, +export async function newTree, D extends FromBuffer>( + c: new (store: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint, deserializer: D) => T, store: AztecKVStore, hasher: Hasher, name: string, + deserializer: D, depth: number, prefilledSize = 1, ): Promise { - const tree = new c(store, hasher, name, depth, 0n); + const tree = new c(store, hasher, name, depth, 0n, deserializer); await tree.init(prefilledSize); return tree; } diff --git a/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.test.ts b/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.test.ts index b6b6dd3e46d..84a205a7741 100644 --- a/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.test.ts +++ b/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.test.ts @@ -1,4 +1,5 @@ import { randomBytes } from '@aztec/foundation/crypto'; +import { FromBuffer } from '@aztec/foundation/serialize'; import { AztecKVStore } from '@aztec/kv-store'; import { openTmpStore } from '@aztec/kv-store/utils'; @@ -8,14 +9,15 @@ import { describeSnapshotBuilderTestSuite } from './snapshot_builder_test_suite. describe('AppendOnlySnapshot', () => { let tree: StandardTree; - let snapshotBuilder: AppendOnlySnapshotBuilder; + let snapshotBuilder: AppendOnlySnapshotBuilder; let db: AztecKVStore; beforeEach(async () => { db = openTmpStore(); const hasher = new Pedersen(); - tree = await newTree(StandardTree, db, hasher, 'test', 4); - snapshotBuilder = new AppendOnlySnapshotBuilder(db, tree, hasher); + const deserializer: FromBuffer = { fromBuffer: b => b }; + tree = await newTree(StandardTree, db, hasher, 'test', deserializer, 4); + snapshotBuilder = new AppendOnlySnapshotBuilder(db, tree, hasher, deserializer); }); describeSnapshotBuilderTestSuite( diff --git a/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.ts b/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.ts index a3870f9ae23..2549bf1544b 100644 --- a/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.ts +++ b/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.ts @@ -1,4 +1,5 @@ import { SiblingPath } from '@aztec/circuit-types'; +import { Bufferable, FromBuffer, serializeToBuffer } from '@aztec/foundation/serialize'; import { AztecKVStore, AztecMap } from '@aztec/kv-store'; import { Hasher } from '@aztec/types/interfaces'; @@ -37,19 +38,24 @@ type SnapshotMetadata = { * Best case: O(H) database reads + O(1) hashes * Worst case: O(H) database reads + O(H) hashes */ -export class AppendOnlySnapshotBuilder implements TreeSnapshotBuilder { +export class AppendOnlySnapshotBuilder implements TreeSnapshotBuilder> { #nodeValue: AztecMap, Buffer>; #nodeLastModifiedByBlock: AztecMap, number>; #snapshotMetadata: AztecMap; - constructor(private db: AztecKVStore, private tree: TreeBase & AppendOnlyTree, private hasher: Hasher) { + constructor( + private db: AztecKVStore, + private tree: TreeBase & AppendOnlyTree, + private hasher: Hasher, + private deserializer: FromBuffer, + ) { const treeName = tree.getName(); this.#nodeValue = db.openMap(`append_only_snapshot:${treeName}:node`); this.#nodeLastModifiedByBlock = db.openMap(`append_ony_snapshot:${treeName}:block`); this.#snapshotMetadata = db.openMap(`append_only_snapshot:${treeName}:snapshot_metadata`); } - getSnapshot(block: number): Promise { + getSnapshot(block: number): Promise> { const meta = this.#getSnapshotMeta(block); if (typeof meta === 'undefined') { @@ -65,11 +71,12 @@ export class AppendOnlySnapshotBuilder implements TreeSnapshotBuilder { meta.root, this.tree, this.hasher, + this.deserializer, ), ); } - snapshot(block: number): Promise { + snapshot(block: number): Promise> { return this.db.transaction(() => { const meta = this.#getSnapshotMeta(block); if (typeof meta !== 'undefined') { @@ -82,6 +89,7 @@ export class AppendOnlySnapshotBuilder implements TreeSnapshotBuilder { meta.root, this.tree, this.hasher, + this.deserializer, ); } @@ -136,6 +144,7 @@ export class AppendOnlySnapshotBuilder implements TreeSnapshotBuilder { root, this.tree, this.hasher, + this.deserializer, ); }); } @@ -148,15 +157,16 @@ export class AppendOnlySnapshotBuilder implements TreeSnapshotBuilder { /** * a */ -class AppendOnlySnapshot implements TreeSnapshot { +class AppendOnlySnapshot implements TreeSnapshot { constructor( private nodes: AztecMap, private nodeHistory: AztecMap, private block: number, private leafCount: bigint, private historicalRoot: Buffer, - private tree: TreeBase & AppendOnlyTree, + private tree: TreeBase & AppendOnlyTree, private hasher: Hasher, + private deserializer: FromBuffer, ) {} public getSiblingPath(index: bigint): SiblingPath { @@ -191,7 +201,7 @@ class AppendOnlySnapshot implements TreeSnapshot { return this.historicalRoot; } - getLeafValue(index: bigint): Buffer | undefined { + getLeafValue(index: bigint): T | undefined { const leafLevel = this.getDepth(); const blockNumber = this.#getBlockNumberThatModifiedNode(leafLevel, index); @@ -202,7 +212,8 @@ class AppendOnlySnapshot implements TreeSnapshot { // leaf was set some time in the past if (blockNumber <= this.block) { - return this.nodes.get(historicalNodeKey(leafLevel, index)); + const val = this.nodes.get(historicalNodeKey(leafLevel, index)); + return val ? this.deserializer.fromBuffer(val) : undefined; } // leaf has been set but in a block in the future @@ -249,15 +260,16 @@ class AppendOnlySnapshot implements TreeSnapshot { return this.nodeHistory.get(nodeModifiedAtBlockKey(level, index)); } - findLeafIndex(value: Buffer): bigint | undefined { + findLeafIndex(value: T): bigint | undefined { return this.findLeafIndexAfter(value, 0n); } - findLeafIndexAfter(value: Buffer, startIndex: bigint): bigint | undefined { + findLeafIndexAfter(value: T, startIndex: bigint): bigint | undefined { + const valueBuffer = serializeToBuffer(value); const numLeaves = this.getNumLeaves(); for (let i = startIndex; i < numLeaves; i++) { const currentValue = this.getLeafValue(i); - if (currentValue && currentValue.equals(value)) { + if (currentValue && serializeToBuffer(currentValue).equals(valueBuffer)) { return i; } } diff --git a/yarn-project/merkle-tree/src/snapshots/base_full_snapshot.ts b/yarn-project/merkle-tree/src/snapshots/base_full_snapshot.ts index 223554a215d..89a3d5af722 100644 --- a/yarn-project/merkle-tree/src/snapshots/base_full_snapshot.ts +++ b/yarn-project/merkle-tree/src/snapshots/base_full_snapshot.ts @@ -1,4 +1,5 @@ import { SiblingPath } from '@aztec/circuit-types'; +import { Bufferable, FromBuffer, serializeToBuffer } from '@aztec/foundation/serialize'; import { AztecKVStore, AztecMap } from '@aztec/kv-store'; import { TreeBase } from '../tree_base.js'; @@ -30,7 +31,7 @@ type SnapshotMetadata = { * Worst case space complexity: O(N * M) * Sibling path access: O(H) database reads */ -export abstract class BaseFullTreeSnapshotBuilder +export abstract class BaseFullTreeSnapshotBuilder, S extends TreeSnapshot> implements TreeSnapshotBuilder { protected nodes: AztecMap; @@ -119,12 +120,13 @@ export abstract class BaseFullTreeSnapshotBuilder implements TreeSnapshot { constructor( protected db: AztecMap, protected historicRoot: Buffer, protected numLeaves: bigint, - protected tree: TreeBase, + protected tree: TreeBase, + protected deserializer: FromBuffer, ) {} getSiblingPath(index: bigint): SiblingPath { @@ -141,13 +143,13 @@ export class BaseFullTreeSnapshot implements TreeSnapshot { return new SiblingPath(this.tree.getDepth() as N, siblings); } - getLeafValue(index: bigint): Buffer | undefined { + getLeafValue(index: bigint): T | undefined { let leafNode: Buffer | undefined = undefined; for (const [node, _sibling] of this.pathFromRootToLeaf(index)) { leafNode = node; } - return leafNode; + return leafNode ? this.deserializer.fromBuffer(leafNode) : undefined; } getDepth(): number { @@ -202,15 +204,16 @@ export class BaseFullTreeSnapshot implements TreeSnapshot { return path; } - findLeafIndex(value: Buffer): bigint | undefined { + findLeafIndex(value: T): bigint | undefined { return this.findLeafIndexAfter(value, 0n); } - public findLeafIndexAfter(value: Buffer, startIndex: bigint): bigint | undefined { + public findLeafIndexAfter(value: T, startIndex: bigint): bigint | undefined { const numLeaves = this.getNumLeaves(); + const buffer = serializeToBuffer(value); for (let i = startIndex; i < numLeaves; i++) { const currentValue = this.getLeafValue(i); - if (currentValue && currentValue.equals(value)) { + if (currentValue && serializeToBuffer(currentValue).equals(buffer)) { return i; } } diff --git a/yarn-project/merkle-tree/src/snapshots/full_snapshot.test.ts b/yarn-project/merkle-tree/src/snapshots/full_snapshot.test.ts index 2c0ea129f8a..bc01c0b9ea1 100644 --- a/yarn-project/merkle-tree/src/snapshots/full_snapshot.test.ts +++ b/yarn-project/merkle-tree/src/snapshots/full_snapshot.test.ts @@ -1,4 +1,5 @@ import { randomBytes } from '@aztec/foundation/crypto'; +import { FromBuffer } from '@aztec/foundation/serialize'; import { AztecKVStore } from '@aztec/kv-store'; import { openTmpStore } from '@aztec/kv-store/utils'; @@ -8,13 +9,14 @@ import { describeSnapshotBuilderTestSuite } from './snapshot_builder_test_suite. describe('FullSnapshotBuilder', () => { let tree: StandardTree; - let snapshotBuilder: FullTreeSnapshotBuilder; + let snapshotBuilder: FullTreeSnapshotBuilder; let db: AztecKVStore; beforeEach(async () => { db = openTmpStore(); - tree = await newTree(StandardTree, db, new Pedersen(), 'test', 4); - snapshotBuilder = new FullTreeSnapshotBuilder(db, tree); + const deserializer: FromBuffer = { fromBuffer: b => b }; + tree = await newTree(StandardTree, db, new Pedersen(), 'test', deserializer, 4); + snapshotBuilder = new FullTreeSnapshotBuilder(db, tree, deserializer); }); describeSnapshotBuilderTestSuite( diff --git a/yarn-project/merkle-tree/src/snapshots/full_snapshot.ts b/yarn-project/merkle-tree/src/snapshots/full_snapshot.ts index 73cce3b05e7..9417dadf67b 100644 --- a/yarn-project/merkle-tree/src/snapshots/full_snapshot.ts +++ b/yarn-project/merkle-tree/src/snapshots/full_snapshot.ts @@ -1,3 +1,6 @@ +import { Bufferable, FromBuffer } from '@aztec/foundation/serialize'; +import { AztecKVStore } from '@aztec/kv-store'; + import { TreeBase } from '../tree_base.js'; import { BaseFullTreeSnapshot, BaseFullTreeSnapshotBuilder } from './base_full_snapshot.js'; import { TreeSnapshot, TreeSnapshotBuilder } from './snapshot_builder.js'; @@ -16,11 +19,15 @@ import { TreeSnapshot, TreeSnapshotBuilder } from './snapshot_builder.js'; * Worst case space complexity: O(N * M) * Sibling path access: O(H) database reads */ -export class FullTreeSnapshotBuilder - extends BaseFullTreeSnapshotBuilder - implements TreeSnapshotBuilder +export class FullTreeSnapshotBuilder + extends BaseFullTreeSnapshotBuilder, TreeSnapshot> + implements TreeSnapshotBuilder> { - protected openSnapshot(root: Buffer, numLeaves: bigint): TreeSnapshot { - return new BaseFullTreeSnapshot(this.nodes, root, numLeaves, this.tree); + constructor(db: AztecKVStore, tree: TreeBase, private deserializer: FromBuffer) { + super(db, tree); + } + + protected openSnapshot(root: Buffer, numLeaves: bigint): TreeSnapshot { + return new BaseFullTreeSnapshot(this.nodes, root, numLeaves, this.tree, this.deserializer); } } diff --git a/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.test.ts b/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.test.ts index 81f5ccb5b20..beedabaec47 100644 --- a/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.test.ts +++ b/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.test.ts @@ -9,7 +9,15 @@ import { IndexedTreeSnapshotBuilder } from './indexed_tree_snapshot.js'; import { describeSnapshotBuilderTestSuite } from './snapshot_builder_test_suite.js'; class NullifierTree extends StandardIndexedTreeWithAppend { - constructor(db: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint = 0n, root?: Buffer) { + constructor( + db: AztecKVStore, + hasher: Hasher, + name: string, + depth: number, + size: bigint = 0n, + _noop: any, + root?: Buffer, + ) { super(db, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf, root); } } @@ -21,7 +29,7 @@ describe('IndexedTreeSnapshotBuilder', () => { beforeEach(async () => { db = openTmpStore(); - tree = await newTree(NullifierTree, db, new Pedersen(), 'test', 4); + tree = await newTree(NullifierTree, db, new Pedersen(), 'test', { fromBuffer: (b: Buffer) => b }, 4); snapshotBuilder = new IndexedTreeSnapshotBuilder(db, tree, NullifierLeafPreimage); }); diff --git a/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.ts b/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.ts index 8a787fd2067..c608daa1118 100644 --- a/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.ts +++ b/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.ts @@ -10,11 +10,11 @@ const snapshotLeafValue = (node: Buffer, index: bigint) => 'snapshot:leaf:' + no /** a */ export class IndexedTreeSnapshotBuilder - extends BaseFullTreeSnapshotBuilder + extends BaseFullTreeSnapshotBuilder, IndexedTreeSnapshot> implements TreeSnapshotBuilder { leaves: AztecMap; - constructor(store: AztecKVStore, tree: IndexedTree & TreeBase, private leafPreimageBuilder: PreimageFactory) { + constructor(store: AztecKVStore, tree: IndexedTree & TreeBase, private leafPreimageBuilder: PreimageFactory) { super(store, tree); this.leaves = store.openMap('indexed_tree_snapshot:' + tree.getName()); } @@ -32,16 +32,16 @@ export class IndexedTreeSnapshotBuilder } /** A snapshot of an indexed tree at a particular point in time */ -class IndexedTreeSnapshotImpl extends BaseFullTreeSnapshot implements IndexedTreeSnapshot { +class IndexedTreeSnapshotImpl extends BaseFullTreeSnapshot implements IndexedTreeSnapshot { constructor( db: AztecMap, private leaves: AztecMap, historicRoot: Buffer, numLeaves: bigint, - tree: IndexedTree & TreeBase, + tree: IndexedTree & TreeBase, private leafPreimageBuilder: PreimageFactory, ) { - super(db, historicRoot, numLeaves, tree); + super(db, historicRoot, numLeaves, tree, { fromBuffer: buf => buf }); } getLeafValue(index: bigint): Buffer | undefined { diff --git a/yarn-project/merkle-tree/src/snapshots/snapshot_builder.ts b/yarn-project/merkle-tree/src/snapshots/snapshot_builder.ts index cecac01a206..bfdc3065846 100644 --- a/yarn-project/merkle-tree/src/snapshots/snapshot_builder.ts +++ b/yarn-project/merkle-tree/src/snapshots/snapshot_builder.ts @@ -1,10 +1,11 @@ import { SiblingPath } from '@aztec/circuit-types'; +import { Bufferable } from '@aztec/foundation/serialize'; import { IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; /** * An interface for a tree that can record snapshots of its contents. */ -export interface TreeSnapshotBuilder { +export interface TreeSnapshotBuilder> { /** * Creates a snapshot of the tree at the given version. * @param block - The version to snapshot the tree at. @@ -21,7 +22,7 @@ export interface TreeSnapshotBuilder { /** * A tree snapshot */ -export interface TreeSnapshot { +export interface TreeSnapshot { /** * Returns the current root of the tree. */ @@ -41,7 +42,7 @@ export interface TreeSnapshot { * Returns the value of a leaf at the specified index. * @param index - The index of the leaf value to be returned. */ - getLeafValue(index: bigint): Buffer | undefined; + getLeafValue(index: bigint): T | undefined; /** * Returns the sibling path for a requested leaf index. @@ -55,7 +56,7 @@ export interface TreeSnapshot { * @param value - The leaf value to look for. * @returns The index of the first leaf found with a given value (undefined if not found). */ - findLeafIndex(value: Buffer): bigint | undefined; + findLeafIndex(value: T): bigint | undefined; /** * Returns the first index containing a leaf value after `startIndex`. @@ -63,11 +64,11 @@ export interface TreeSnapshot { * @param startIndex - The index to start searching from (used when skipping nullified messages) * @returns The index of the first leaf found with a given value (undefined if not found). */ - findLeafIndexAfter(leaf: Buffer, startIndex: bigint): bigint | undefined; + findLeafIndexAfter(leaf: T, startIndex: bigint): bigint | undefined; } /** A snapshot of an indexed tree */ -export interface IndexedTreeSnapshot extends TreeSnapshot { +export interface IndexedTreeSnapshot extends TreeSnapshot { /** * Gets the historical data for a leaf * @param index - The index of the leaf to get the data for diff --git a/yarn-project/merkle-tree/src/snapshots/snapshot_builder_test_suite.ts b/yarn-project/merkle-tree/src/snapshots/snapshot_builder_test_suite.ts index a10842121b9..4dcf97bd5b4 100644 --- a/yarn-project/merkle-tree/src/snapshots/snapshot_builder_test_suite.ts +++ b/yarn-project/merkle-tree/src/snapshots/snapshot_builder_test_suite.ts @@ -1,14 +1,18 @@ import { randomBigInt } from '@aztec/foundation/crypto'; +import { Bufferable } from '@aztec/foundation/serialize'; + +import { jest } from '@jest/globals'; import { TreeBase } from '../tree_base.js'; -import { TreeSnapshotBuilder } from './snapshot_builder.js'; +import { TreeSnapshot, TreeSnapshotBuilder } from './snapshot_builder.js'; + +jest.setTimeout(50_000); /** Creates a test suit for snapshots */ -export function describeSnapshotBuilderTestSuite( - getTree: () => T, - getSnapshotBuilder: () => S, - modifyTree: (tree: T) => Promise, -) { +export function describeSnapshotBuilderTestSuite< + T extends TreeBase, + S extends TreeSnapshotBuilder>, +>(getTree: () => T, getSnapshotBuilder: () => S, modifyTree: (tree: T) => Promise) { describe('SnapshotBuilder', () => { let tree: T; let snapshotBuilder: S; @@ -34,7 +38,9 @@ export function describeSnapshotBuilderTestSuite { diff --git a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts b/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts index 69ad6c2f3cb..4d11aa49359 100644 --- a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts +++ b/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts @@ -15,12 +15,28 @@ import { SparseTree } from './sparse_tree.js'; const log = createDebugLogger('aztec:sparse_tree_test'); -const createDb = async (db: AztecKVStore, hasher: Hasher, name: string, depth: number): Promise => { - return await newTree(SparseTree, db, hasher, name, depth); +const createDb = async ( + db: AztecKVStore, + hasher: Hasher, + name: string, + depth: number, +): Promise> => { + return await newTree( + SparseTree, + db, + hasher, + name, + { + fromBuffer: (buffer: Buffer): Buffer => buffer, + }, + depth, + ); }; -const createFromName = async (db: AztecKVStore, hasher: Hasher, name: string): Promise => { - return await loadTree(SparseTree, db, hasher, name); +const createFromName = async (db: AztecKVStore, hasher: Hasher, name: string): Promise> => { + return await loadTree(SparseTree, db, hasher, name, { + fromBuffer: (buffer: Buffer): Buffer => buffer, + }); }; const TEST_TREE_DEPTH = 3; diff --git a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.ts b/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.ts index 6630e7bb1bb..646f01a2412 100644 --- a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.ts +++ b/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.ts @@ -1,3 +1,5 @@ +import { Bufferable, serializeToBuffer } from '@aztec/foundation/serialize'; + import { UpdateOnlyTree } from '../interfaces/update_only_tree.js'; import { FullTreeSnapshotBuilder } from '../snapshots/full_snapshot.js'; import { TreeSnapshot } from '../snapshots/snapshot_builder.js'; @@ -6,20 +8,21 @@ import { INITIAL_LEAF, TreeBase } from '../tree_base.js'; /** * A Merkle tree implementation that uses a LevelDB database to store the tree. */ -export class SparseTree extends TreeBase implements UpdateOnlyTree { - #snapshotBuilder = new FullTreeSnapshotBuilder(this.store, this); +export class SparseTree extends TreeBase implements UpdateOnlyTree { + #snapshotBuilder = new FullTreeSnapshotBuilder(this.store, this, this.deserializer); /** * Updates a leaf in the tree. * @param leaf - New contents of the leaf. * @param index - Index of the leaf to be updated. */ - public updateLeaf(leaf: Buffer, index: bigint): Promise { + public updateLeaf(value: T, index: bigint): Promise { if (index > this.maxIndex) { throw Error(`Index out of bounds. Index ${index}, max index: ${this.maxIndex}.`); } + const leaf = serializeToBuffer(value); const insertingZeroElement = leaf.equals(INITIAL_LEAF); - const originallyZeroElement = this.getLeafValue(index, true)?.equals(INITIAL_LEAF); + const originallyZeroElement = this.getLeafBuffer(index, true)?.equals(INITIAL_LEAF); if (insertingZeroElement && originallyZeroElement) { return Promise.resolve(); } @@ -35,19 +38,19 @@ export class SparseTree extends TreeBase implements UpdateOnlyTree { return Promise.resolve(); } - public snapshot(block: number): Promise { + public snapshot(block: number): Promise> { return this.#snapshotBuilder.snapshot(block); } - public getSnapshot(block: number): Promise { + public getSnapshot(block: number): Promise> { return this.#snapshotBuilder.getSnapshot(block); } - public findLeafIndex(_value: Buffer, _includeUncommitted: boolean): bigint | undefined { + public findLeafIndex(_value: T, _includeUncommitted: boolean): bigint | undefined { throw new Error('Finding leaf index is not supported for sparse trees'); } - public findLeafIndexAfter(_value: Buffer, _startIndex: bigint, _includeUncommitted: boolean): bigint | undefined { + public findLeafIndexAfter(_value: T, _startIndex: bigint, _includeUncommitted: boolean): bigint | undefined { throw new Error('Finding leaf index is not supported for sparse trees'); } } diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts index 5239e04d6a4..42e9a3cbfbe 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts @@ -1,6 +1,7 @@ import { SiblingPath } from '@aztec/circuit-types'; import { TreeInsertionStats } from '@aztec/circuit-types/stats'; import { toBufferBE } from '@aztec/foundation/bigint-buffer'; +import { FromBuffer } from '@aztec/foundation/serialize'; import { Timer } from '@aztec/foundation/timer'; import { IndexedTreeLeaf, IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; import { AztecKVStore, AztecMap } from '@aztec/kv-store'; @@ -51,10 +52,14 @@ function getEmptyLowLeafWitness( }; } +export const noopDeserializer: FromBuffer = { + fromBuffer: (buf: Buffer) => buf, +}; + /** * Standard implementation of an indexed tree. */ -export class StandardIndexedTree extends TreeBase implements IndexedTree { +export class StandardIndexedTree extends TreeBase implements IndexedTree { #snapshotBuilder = new IndexedTreeSnapshotBuilder(this.store, this, this.leafPreimageFactory); protected cachedLeafPreimages: { [key: string]: IndexedTreeLeafPreimage } = {}; @@ -71,7 +76,7 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { protected leafFactory: LeafFactory, root?: Buffer, ) { - super(store, hasher, name, depth, size, root); + super(store, hasher, name, depth, size, noopDeserializer, root); this.leaves = store.openMap(`tree_${name}_leaves`); this.leafIndex = store.openMap(`tree_${name}_leaf_index`); } diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts index 7454aad786f..a605e19025e 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts @@ -7,6 +7,7 @@ import { PublicDataTreeLeafPreimage, } from '@aztec/circuits.js'; import { toBufferBE } from '@aztec/foundation/bigint-buffer'; +import { FromBuffer } from '@aztec/foundation/serialize'; import { AztecKVStore } from '@aztec/kv-store'; import { openTmpStore } from '@aztec/kv-store/utils'; import { Hasher } from '@aztec/types/interfaces'; @@ -16,23 +17,43 @@ import { treeTestSuite } from '../../test/test_suite.js'; import { StandardIndexedTreeWithAppend } from './standard_indexed_tree_with_append.js'; class NullifierTree extends StandardIndexedTreeWithAppend { - constructor(store: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint = 0n, root?: Buffer) { + constructor( + store: AztecKVStore, + hasher: Hasher, + name: string, + depth: number, + size: bigint = 0n, + _noop: any, + root?: Buffer, + ) { super(store, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf, root); } } class PublicDataTree extends StandardIndexedTreeWithAppend { - constructor(store: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint = 0n, root?: Buffer) { + constructor( + store: AztecKVStore, + hasher: Hasher, + name: string, + depth: number, + size: bigint = 0n, + _noop: any, + root?: Buffer, + ) { super(store, hasher, name, depth, size, PublicDataTreeLeafPreimage, PublicDataTreeLeaf, root); } } +const noopDeserializer: FromBuffer = { + fromBuffer: (buffer: Buffer) => buffer, +}; + const createDb = async (store: AztecKVStore, hasher: Hasher, name: string, depth: number, prefilledSize = 1) => { - return await newTree(NullifierTree, store, hasher, name, depth, prefilledSize); + return await newTree(NullifierTree, store, hasher, name, noopDeserializer, depth, prefilledSize); }; const createFromName = async (store: AztecKVStore, hasher: Hasher, name: string) => { - return await loadTree(NullifierTree, store, hasher, name); + return await loadTree(NullifierTree, store, hasher, name, noopDeserializer); }; const createNullifierTreeLeafHashInputs = (value: number, nextIndex: number, nextValue: number) => { @@ -521,7 +542,7 @@ describe('StandardIndexedTreeSpecific', () => { it('should be able to upsert leaves', async () => { // Create a depth-3 indexed merkle tree const db = openTmpStore(); - const tree = await newTree(PublicDataTree, db, pedersen, 'test', 3, 1); + const tree = await newTree(PublicDataTree, db, pedersen, 'test', {}, 3, 1); /** * Initial state: diff --git a/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts b/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts index 465f5a59a64..baf96a21df9 100644 --- a/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts +++ b/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts @@ -1,4 +1,5 @@ import { randomBytes } from '@aztec/foundation/crypto'; +import { FromBuffer } from '@aztec/foundation/serialize'; import { AztecKVStore } from '@aztec/kv-store'; import { openTmpStore } from '@aztec/kv-store/utils'; import { Hasher } from '@aztec/types/interfaces'; @@ -11,12 +12,16 @@ import { PedersenWithCounter } from '../test/utils/pedersen_with_counter.js'; import { INITIAL_LEAF } from '../tree_base.js'; import { StandardTree } from './standard_tree.js'; +const noopDeserializer: FromBuffer = { + fromBuffer: (buffer: Buffer) => buffer, +}; + const createDb = async (store: AztecKVStore, hasher: Hasher, name: string, depth: number) => { - return await newTree(StandardTree, store, hasher, name, depth); + return await newTree(StandardTree, store, hasher, name, noopDeserializer, depth); }; const createFromName = async (store: AztecKVStore, hasher: Hasher, name: string) => { - return await loadTree(StandardTree, store, hasher, name); + return await loadTree(StandardTree, store, hasher, name, noopDeserializer); }; treeTestSuite('StandardTree', createDb, createFromName); diff --git a/yarn-project/merkle-tree/src/standard_tree/standard_tree.ts b/yarn-project/merkle-tree/src/standard_tree/standard_tree.ts index 4c28b5e74f6..45c8cc6a6ce 100644 --- a/yarn-project/merkle-tree/src/standard_tree/standard_tree.ts +++ b/yarn-project/merkle-tree/src/standard_tree/standard_tree.ts @@ -1,4 +1,5 @@ import { TreeInsertionStats } from '@aztec/circuit-types/stats'; +import { Bufferable, serializeToBuffer } from '@aztec/foundation/serialize'; import { Timer } from '@aztec/foundation/timer'; import { AppendOnlyTree } from '../interfaces/append_only_tree.js'; @@ -9,15 +10,15 @@ import { TreeBase } from '../tree_base.js'; /** * A Merkle tree implementation that uses a LevelDB database to store the tree. */ -export class StandardTree extends TreeBase implements AppendOnlyTree { - #snapshotBuilder = new AppendOnlySnapshotBuilder(this.store, this, this.hasher); +export class StandardTree extends TreeBase implements AppendOnlyTree { + #snapshotBuilder = new AppendOnlySnapshotBuilder(this.store, this, this.hasher, this.deserializer); /** * Appends the given leaves to the tree. * @param leaves - The leaves to append. * @returns Empty promise. */ - public appendLeaves(leaves: Buffer[]): Promise { + public appendLeaves(leaves: T[]): Promise { this.hasher.reset(); const timer = new Timer(); super.appendLeaves(leaves); @@ -34,22 +35,23 @@ export class StandardTree extends TreeBase implements AppendOnlyTree { return Promise.resolve(); } - public snapshot(blockNumber: number): Promise { + public snapshot(blockNumber: number): Promise> { return this.#snapshotBuilder.snapshot(blockNumber); } - public getSnapshot(blockNumber: number): Promise { + public getSnapshot(blockNumber: number): Promise> { return this.#snapshotBuilder.getSnapshot(blockNumber); } - public findLeafIndex(value: Buffer, includeUncommitted: boolean): bigint | undefined { + public findLeafIndex(value: T, includeUncommitted: boolean): bigint | undefined { return this.findLeafIndexAfter(value, 0n, includeUncommitted); } - public findLeafIndexAfter(value: Buffer, startIndex: bigint, includeUncommitted: boolean): bigint | undefined { + public findLeafIndexAfter(value: T, startIndex: bigint, includeUncommitted: boolean): bigint | undefined { + const buffer = serializeToBuffer(value); for (let i = startIndex; i < this.getNumLeaves(includeUncommitted); i++) { const currentValue = this.getLeafValue(i, includeUncommitted); - if (currentValue && currentValue.equals(value)) { + if (currentValue && serializeToBuffer(currentValue).equals(buffer)) { return i; } } diff --git a/yarn-project/merkle-tree/src/tree_base.ts b/yarn-project/merkle-tree/src/tree_base.ts index 7d7c2fc5035..db2ed88bfb2 100644 --- a/yarn-project/merkle-tree/src/tree_base.ts +++ b/yarn-project/merkle-tree/src/tree_base.ts @@ -1,6 +1,7 @@ import { SiblingPath } from '@aztec/circuit-types'; import { toBigIntLE, toBufferLE } from '@aztec/foundation/bigint-buffer'; import { DebugLogger, createDebugLogger } from '@aztec/foundation/log'; +import { Bufferable, FromBuffer, serializeToBuffer } from '@aztec/foundation/serialize'; import { AztecKVStore, AztecMap, AztecSingleton } from '@aztec/kv-store'; import { Hasher } from '@aztec/types/interfaces'; @@ -44,7 +45,7 @@ export const INITIAL_LEAF = Buffer.from('000000000000000000000000000000000000000 /** * A Merkle tree implementation that uses a LevelDB database to store the tree. */ -export abstract class TreeBase implements MerkleTree { +export abstract class TreeBase implements MerkleTree { protected readonly maxIndex: bigint; protected cachedSize?: bigint; private root!: Buffer; @@ -62,6 +63,7 @@ export abstract class TreeBase implements MerkleTree { private name: string, private depth: number, protected size: bigint = 0n, + protected deserializer: FromBuffer, root?: Buffer, ) { if (!(depth >= 1 && depth <= MAX_DEPTH)) { @@ -173,7 +175,22 @@ export abstract class TreeBase implements MerkleTree { * @param includeUncommitted - Indicates whether to include uncommitted changes. * @returns Leaf value at the given index or undefined. */ - public getLeafValue(index: bigint, includeUncommitted: boolean): Buffer | undefined { + public getLeafValue(index: bigint, includeUncommitted: boolean): T | undefined { + const buf = this.getLatestValueAtIndex(this.depth, index, includeUncommitted); + if (buf) { + return this.deserializer.fromBuffer(buf); + } else { + return undefined; + } + } + + /** + * Gets the value at the given index. + * @param index - The index of the leaf. + * @param includeUncommitted - Indicates whether to include uncommitted changes. + * @returns Leaf value at the given index or undefined. + */ + public getLeafBuffer(index: bigint, includeUncommitted: boolean): Buffer | undefined { return this.getLatestValueAtIndex(this.depth, index, includeUncommitted); } @@ -292,7 +309,7 @@ export abstract class TreeBase implements MerkleTree { * `getLatestValueAtIndex` will return a value from cache (because at least one of the 2 children was * touched in previous iteration). */ - protected appendLeaves(leaves: Buffer[]): void { + protected appendLeaves(leaves: T[]): void { const numLeaves = this.getNumLeaves(true); if (numLeaves + BigInt(leaves.length) - 1n > this.maxIndex) { throw Error(`Can't append beyond max index. Max index: ${this.maxIndex}`); @@ -303,7 +320,7 @@ export abstract class TreeBase implements MerkleTree { let level = this.depth; for (let i = 0; i < leaves.length; i++) { const cacheKey = indexToKeyHash(this.name, level, firstIndex + BigInt(i)); - this.cache[cacheKey] = leaves[i]; + this.cache[cacheKey] = serializeToBuffer(leaves[i]); } let lastIndex = firstIndex + BigInt(leaves.length); @@ -330,7 +347,7 @@ export abstract class TreeBase implements MerkleTree { * @param includeUncommitted - Indicates whether to include uncommitted data. * @returns The index of the first leaf found with a given value (undefined if not found). */ - abstract findLeafIndex(value: Buffer, includeUncommitted: boolean): bigint | undefined; + abstract findLeafIndex(value: T, includeUncommitted: boolean): bigint | undefined; /** * Returns the first index containing a leaf value after `startIndex`. @@ -339,5 +356,5 @@ export abstract class TreeBase implements MerkleTree { * @param includeUncommitted - Indicates whether to include uncommitted data. * @returns The index of the first leaf found with a given value (undefined if not found). */ - abstract findLeafIndexAfter(leaf: Buffer, startIndex: bigint, includeUncommitted: boolean): bigint | undefined; + abstract findLeafIndexAfter(leaf: T, startIndex: bigint, includeUncommitted: boolean): bigint | undefined; } diff --git a/yarn-project/noir-protocol-circuits-types/src/noir_test_gen.test.ts b/yarn-project/noir-protocol-circuits-types/src/noir_test_gen.test.ts index 1586c8c3bbe..8082a6c7801 100644 --- a/yarn-project/noir-protocol-circuits-types/src/noir_test_gen.test.ts +++ b/yarn-project/noir-protocol-circuits-types/src/noir_test_gen.test.ts @@ -98,7 +98,7 @@ describe('Data generation for noir tests', () => { it('Computes a note hash tree', async () => { const indexes = new Array(128).fill(null).map((_, i) => BigInt(i)); - const leaves = indexes.map(i => new Fr(i + 1n).toBuffer()); + const leaves = indexes.map(i => new Fr(i + 1n)); const db = openTmpStore(); @@ -107,6 +107,8 @@ describe('Data generation for noir tests', () => { new Pedersen(), `${MerkleTreeId[MerkleTreeId.NOTE_HASH_TREE]}`, NOTE_HASH_TREE_HEIGHT, + 0n, + Fr, ); await noteHashTree.appendLeaves(leaves); diff --git a/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts b/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts index 3376ec35d62..5156edd6301 100644 --- a/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts +++ b/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts @@ -94,7 +94,7 @@ export async function buildBaseRollupInput( // Update the note hash trees with the new items being inserted to get the new roots // that will be used by the next iteration of the base rollup circuit, skipping the empty ones - const newNoteHashes = tx.data.combinedData.newNoteHashes.map(x => x.value.toBuffer()); + const newNoteHashes = tx.data.combinedData.newNoteHashes.map(x => x.value); await db.appendLeaves(MerkleTreeId.NOTE_HASH_TREE, newNoteHashes); // The read witnesses for a given TX should be generated before the writes of the same TX are applied. @@ -214,10 +214,7 @@ export async function executeRootRollupCircuit( const rootInput = await getRootRollupInput(...left, ...right, l1ToL2Roots, newL1ToL2Messages, db); // Update the local trees to include the new l1 to l2 messages - await db.appendLeaves( - MerkleTreeId.L1_TO_L2_MESSAGE_TREE, - newL1ToL2Messages.map(m => m.toBuffer()), - ); + await db.appendLeaves(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, newL1ToL2Messages); // Simulate and get proof for the root circuit const rootOutput = await simulator.rootRollupCircuit(rootInput); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts index 8d11730cb78..c4af9a479dc 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.test.ts @@ -136,7 +136,7 @@ describe('prover/tx-prover', () => { .sort(sideEffectCmp), SideEffect.empty(), MAX_NEW_NOTE_HASHES_PER_TX, - ).map(l => l.value.toBuffer()), + ).map(l => l.value), ), ); await expectsDb.batchInsert( diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts index 10d5878e81f..d00a57a7fef 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts @@ -162,7 +162,7 @@ describe('sequencer', () => { // We make a nullifier from tx1 a part of the nullifier tree, so it gets rejected as double spend const doubleSpendNullifier = doubleSpendTx.data.end.newNullifiers[0].value.toBuffer(); - merkleTreeOps.findLeafIndex.mockImplementation((treeId: MerkleTreeId, value: Buffer) => { + merkleTreeOps.findLeafIndex.mockImplementation((treeId: MerkleTreeId, value: any) => { return Promise.resolve( treeId === MerkleTreeId.NULLIFIER_TREE && value.equals(doubleSpendNullifier) ? 1n : undefined, ); diff --git a/yarn-project/sequencer-client/src/simulator/public_executor.ts b/yarn-project/sequencer-client/src/simulator/public_executor.ts index 1f1d582431c..0e83ad98ab0 100644 --- a/yarn-project/sequencer-client/src/simulator/public_executor.ts +++ b/yarn-project/sequencer-client/src/simulator/public_executor.ts @@ -235,11 +235,7 @@ export class WorldStateDB implements CommitmentsDB { // We iterate over messages until we find one whose nullifier is not in the nullifier tree --> we need to check // for nullifiers because messages can have duplicates. do { - messageIndex = (await this.db.findLeafIndexAfter( - MerkleTreeId.L1_TO_L2_MESSAGE_TREE, - messageHash.toBuffer(), - startIndex, - ))!; + messageIndex = (await this.db.findLeafIndexAfter(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, messageHash, startIndex))!; if (messageIndex === undefined) { throw new Error(`No non-nullified L1 to L2 message found for message hash ${messageHash.toString()}`); } @@ -259,7 +255,7 @@ export class WorldStateDB implements CommitmentsDB { } public async getCommitmentIndex(commitment: Fr): Promise { - return await this.db.findLeafIndex(MerkleTreeId.NOTE_HASH_TREE, commitment.toBuffer()); + return await this.db.findLeafIndex(MerkleTreeId.NOTE_HASH_TREE, commitment); } public async getNullifierIndex(nullifier: Fr): Promise { diff --git a/yarn-project/simulator/src/client/private_execution.test.ts b/yarn-project/simulator/src/client/private_execution.test.ts index 6495690acb1..003ecf44bc2 100644 --- a/yarn-project/simulator/src/client/private_execution.test.ts +++ b/yarn-project/simulator/src/client/private_execution.test.ts @@ -83,7 +83,7 @@ describe('Private Execution test suite', () => { l1ToL2Messages: L1_TO_L2_MSG_TREE_HEIGHT, }; - let trees: { [name: keyof typeof treeHeights]: AppendOnlyTree } = {}; + let trees: { [name: keyof typeof treeHeights]: AppendOnlyTree } = {}; const txContextFields: FieldsOf = { isFeePaymentTx: false, isRebatePaymentTx: false, @@ -127,11 +127,11 @@ describe('Private Execution test suite', () => { if (!trees[name]) { const db = openTmpStore(); const pedersen = new Pedersen(); - trees[name] = await newTree(StandardTree, db, pedersen, name, treeHeights[name]); + trees[name] = await newTree(StandardTree, db, pedersen, name, Fr, treeHeights[name]); } const tree = trees[name]; - await tree.appendLeaves(leaves.map(l => l.toBuffer())); + await tree.appendLeaves(leaves); // Create a new snapshot. const newSnap = new AppendOnlyTreeSnapshot(Fr.fromBuffer(tree.getRoot(true)), Number(tree.getNumLeaves(true))); diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts index d1c716adc30..4da1279012b 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts @@ -114,8 +114,10 @@ describe('server_world_state_synchronizer', () => { new SHA256Trunc(), 'empty_subtree_in_hash', L1_TO_L2_MSG_SUBTREE_HEIGHT, + 0n, + Fr, ); - await tree.appendLeaves(l1ToL2Messages.map(msg => msg.toBuffer())); + await tree.appendLeaves(l1ToL2Messages); inHash = tree.getRoot(true); }); diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts index 583ceff28a8..001fd91abc2 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts @@ -245,8 +245,10 @@ export class ServerWorldStateSynchronizer implements WorldStateSynchronizer { new SHA256Trunc(), 'temp_in_hash_check', L1_TO_L2_MSG_SUBTREE_HEIGHT, + 0n, + Fr, ); - await tree.appendLeaves(l1ToL2Messages.map(msg => msg.toBuffer())); + await tree.appendLeaves(l1ToL2Messages); if (!tree.getRoot(true).equals(inHash)) { throw new Error('Obtained L1 to L2 messages failed to be hashed to the block inHash'); diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts index a6f8382b318..7df7a084f32 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts @@ -1,4 +1,5 @@ -import { MAX_NEW_NULLIFIERS_PER_TX, MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX } from '@aztec/circuits.js'; +import { MerkleTreeId } from '@aztec/circuit-types'; +import { Fr, MAX_NEW_NULLIFIERS_PER_TX, MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX } from '@aztec/circuits.js'; import { IndexedTreeSnapshot, TreeSnapshot } from '@aztec/merkle-tree'; import { MerkleTreeOperations } from './merkle_tree_operations.js'; @@ -40,6 +41,14 @@ type MerkleTreeSetters = | 'handleL2BlockAndMessages' | 'batchInsert'; +export type TreeSnapshots = { + [MerkleTreeId.NULLIFIER_TREE]: IndexedTreeSnapshot; + [MerkleTreeId.NOTE_HASH_TREE]: TreeSnapshot; + [MerkleTreeId.PUBLIC_DATA_TREE]: IndexedTreeSnapshot; + [MerkleTreeId.L1_TO_L2_MESSAGE_TREE]: TreeSnapshot; + [MerkleTreeId.ARCHIVE]: TreeSnapshot; +}; + /** * Defines the interface for operations on a set of Merkle Trees configuring whether to return committed or uncommitted data. */ @@ -52,5 +61,5 @@ export type MerkleTreeDb = { * Returns a snapshot of the current state of the trees. * @param block - The block number to take the snapshot at. */ - getSnapshot(block: number): Promise>; + getSnapshot(block: number): Promise; }; diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_operations.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_operations.ts index c418fa2a1e2..3cacb652fe0 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_operations.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_operations.ts @@ -2,7 +2,7 @@ import { L2Block, MerkleTreeId, SiblingPath } from '@aztec/circuit-types'; import { Fr, Header, NullifierLeafPreimage, StateReference } from '@aztec/circuits.js'; import { createDebugLogger } from '@aztec/foundation/log'; import { IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; -import { BatchInsertionResult } from '@aztec/merkle-tree'; +import { AppendOnlyTree, BatchInsertionResult, IndexedTree } from '@aztec/merkle-tree'; /** * Type alias for the nullifier tree ID. @@ -32,6 +32,24 @@ export interface TreeInfo { depth: number; } +export type MerkleTreeMap = { + [MerkleTreeId.NULLIFIER_TREE]: IndexedTree; + [MerkleTreeId.NOTE_HASH_TREE]: AppendOnlyTree; + [MerkleTreeId.PUBLIC_DATA_TREE]: IndexedTree; + [MerkleTreeId.L1_TO_L2_MESSAGE_TREE]: AppendOnlyTree; + [MerkleTreeId.ARCHIVE]: AppendOnlyTree; +}; + +type LeafTypes = { + [MerkleTreeId.NULLIFIER_TREE]: Buffer; + [MerkleTreeId.NOTE_HASH_TREE]: Fr; + [MerkleTreeId.PUBLIC_DATA_TREE]: Buffer; + [MerkleTreeId.L1_TO_L2_MESSAGE_TREE]: Fr; + [MerkleTreeId.ARCHIVE]: Fr; +}; + +export type MerkleTreeLeafType = LeafTypes[ID]; + /** * Defines the interface for operations on a set of Merkle Trees. */ @@ -41,7 +59,7 @@ export interface MerkleTreeOperations { * @param treeId - The tree to be updated. * @param leaves - The set of leaves to be appended. */ - appendLeaves(treeId: MerkleTreeId, leaves: Buffer[]): Promise; + appendLeaves(treeId: ID, leaves: MerkleTreeLeafType[]): Promise; /** * Returns information about the given tree. @@ -71,8 +89,8 @@ export interface MerkleTreeOperations { * @param treeId - The tree for which the previous value index is required. * @param value - The value to be queried. */ - getPreviousValueIndex( - treeId: IndexedTreeId, + getPreviousValueIndex( + treeId: ID, value: bigint, ): Promise< | { @@ -93,7 +111,7 @@ export interface MerkleTreeOperations { * @param treeId - The tree for which leaf data should be returned. * @param index - The index of the leaf required. */ - getLeafPreimage(treeId: IndexedTreeId, index: bigint): Promise; + getLeafPreimage(treeId: ID, index: bigint): Promise; /** * Update the leaf data at the given index. @@ -101,14 +119,14 @@ export interface MerkleTreeOperations { * @param leaf - The updated leaf value. * @param index - The index of the leaf to be updated. */ - updateLeaf(treeId: IndexedTreeId, leaf: NullifierLeafPreimage | Buffer, index: bigint): Promise; + updateLeaf(treeId: ID, leaf: NullifierLeafPreimage | Buffer, index: bigint): Promise; /** * Returns the index containing a leaf value. * @param treeId - The tree for which the index should be returned. * @param value - The value to search for in the tree. */ - findLeafIndex(treeId: MerkleTreeId, value: Buffer): Promise; + findLeafIndex(treeId: ID, value: MerkleTreeLeafType): Promise; /** * Returns the first index containing a leaf value after `startIndex`. @@ -116,14 +134,21 @@ export interface MerkleTreeOperations { * @param value - The value to search for in the tree. * @param startIndex - The index to start searching from (used when skipping nullified messages) */ - findLeafIndexAfter(treeId: MerkleTreeId, value: Buffer, startIndex: bigint): Promise; + findLeafIndexAfter( + treeId: ID, + value: MerkleTreeLeafType, + startIndex: bigint, + ): Promise; /** * Gets the value for a leaf in the tree. * @param treeId - The tree for which the index should be returned. * @param index - The index of the leaf. */ - getLeafValue(treeId: MerkleTreeId, index: bigint): Promise; + getLeafValue( + treeId: ID, + index: bigint, + ): Promise | undefined>; /** * Inserts the block hash into the archive. @@ -139,8 +164,8 @@ export interface MerkleTreeOperations { * @param subtreeHeight - Height of the subtree. * @returns The witness data for the leaves to be updated when inserting the new ones. */ - batchInsert( - treeId: MerkleTreeId, + batchInsert( + treeId: ID, leaves: Buffer[], subtreeHeight: number, ): Promise>; diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts index b6282ee2abd..aad3c019a6b 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts @@ -4,7 +4,13 @@ import { IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; import { BatchInsertionResult } from '@aztec/merkle-tree'; import { MerkleTreeDb } from './merkle_tree_db.js'; -import { HandleL2BlockAndMessagesResult, MerkleTreeOperations, TreeInfo } from './merkle_tree_operations.js'; +import { + HandleL2BlockAndMessagesResult, + IndexedTreeId, + MerkleTreeLeafType, + MerkleTreeOperations, + TreeInfo, +} from './merkle_tree_operations.js'; /** * Wraps a MerkleTreeDbOperations to call all functions with a preset includeUncommitted flag. @@ -44,7 +50,7 @@ export class MerkleTreeOperationsFacade implements MerkleTreeOperations { * @param leaves - The set of leaves to be appended. * @returns The tree info of the specified tree. */ - appendLeaves(treeId: MerkleTreeId, leaves: Buffer[]): Promise { + appendLeaves(treeId: ID, leaves: MerkleTreeLeafType[]): Promise { return this.trees.appendLeaves(treeId, leaves); } @@ -66,8 +72,8 @@ export class MerkleTreeOperationsFacade implements MerkleTreeOperations { * @param includeUncommitted - If true, the uncommitted changes are included in the search. * @returns The found leaf index and a flag indicating if the corresponding leaf's value is equal to `newValue`. */ - getPreviousValueIndex( - treeId: MerkleTreeId.NULLIFIER_TREE, + getPreviousValueIndex( + treeId: ID, value: bigint, ): Promise< | { @@ -92,7 +98,7 @@ export class MerkleTreeOperationsFacade implements MerkleTreeOperations { * @param index - The index to insert into. * @returns Empty promise. */ - updateLeaf(treeId: MerkleTreeId.NULLIFIER_TREE, leaf: NullifierLeafPreimage, index: bigint): Promise { + updateLeaf(treeId: ID, leaf: NullifierLeafPreimage, index: bigint): Promise { return this.trees.updateLeaf(treeId, leaf, index); } @@ -102,8 +108,8 @@ export class MerkleTreeOperationsFacade implements MerkleTreeOperations { * @param index - The index of the leaf to get. * @returns Leaf preimage. */ - async getLeafPreimage( - treeId: MerkleTreeId.NULLIFIER_TREE, + async getLeafPreimage( + treeId: ID, index: bigint, ): Promise { const preimage = await this.trees.getLeafPreimage(treeId, index, this.includeUncommitted); @@ -116,7 +122,7 @@ export class MerkleTreeOperationsFacade implements MerkleTreeOperations { * @param value - The leaf value to look for. * @returns The index of the first leaf found with a given value (undefined if not found). */ - findLeafIndex(treeId: MerkleTreeId, value: Buffer): Promise { + findLeafIndex(treeId: ID, value: MerkleTreeLeafType): Promise { return this.trees.findLeafIndex(treeId, value, this.includeUncommitted); } @@ -126,7 +132,11 @@ export class MerkleTreeOperationsFacade implements MerkleTreeOperations { * @param value - The value to search for in the tree. * @param startIndex - The index to start searching from (used when skipping nullified messages) */ - findLeafIndexAfter(treeId: MerkleTreeId, value: Buffer, startIndex: bigint): Promise { + findLeafIndexAfter( + treeId: ID, + value: MerkleTreeLeafType, + startIndex: bigint, + ): Promise { return this.trees.findLeafIndexAfter(treeId, value, startIndex, this.includeUncommitted); } @@ -137,8 +147,13 @@ export class MerkleTreeOperationsFacade implements MerkleTreeOperations { * @param includeUncommitted - Indicates whether to include uncommitted changes. * @returns Leaf value at the given index (undefined if not found). */ - getLeafValue(treeId: MerkleTreeId, index: bigint): Promise { - return this.trees.getLeafValue(treeId, index, this.includeUncommitted); + getLeafValue( + treeId: ID, + index: bigint, + ): Promise | undefined> { + return this.trees.getLeafValue(treeId, index, this.includeUncommitted) as Promise< + MerkleTreeLeafType | undefined + >; } /** @@ -184,7 +199,7 @@ export class MerkleTreeOperationsFacade implements MerkleTreeOperations { * @returns The data for the leaves to be updated when inserting the new ones. */ public batchInsert( - treeId: MerkleTreeId, + treeId: IndexedTreeId, leaves: Buffer[], subtreeHeight: number, ): Promise> { diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts index 59e958522c6..188213d66c7 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts @@ -1,10 +1,16 @@ import { MerkleTreeId, SiblingPath } from '@aztec/circuit-types'; import { AppendOnlyTreeSnapshot, Fr, Header, PartialStateReference, StateReference } from '@aztec/circuits.js'; import { IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; -import { BatchInsertionResult, IndexedTreeSnapshot, TreeSnapshot } from '@aztec/merkle-tree'; +import { BatchInsertionResult, IndexedTreeSnapshot } from '@aztec/merkle-tree'; -import { MerkleTreeDb } from './merkle_tree_db.js'; -import { HandleL2BlockAndMessagesResult, MerkleTreeOperations, TreeInfo } from './merkle_tree_operations.js'; +import { MerkleTreeDb, TreeSnapshots } from './merkle_tree_db.js'; +import { + HandleL2BlockAndMessagesResult, + IndexedTreeId, + MerkleTreeLeafType, + MerkleTreeOperations, + TreeInfo, +} from './merkle_tree_operations.js'; /** * Merkle tree operations on readonly tree snapshots. @@ -12,47 +18,56 @@ import { HandleL2BlockAndMessagesResult, MerkleTreeOperations, TreeInfo } from ' export class MerkleTreeSnapshotOperationsFacade implements MerkleTreeOperations { #treesDb: MerkleTreeDb; #blockNumber: number; - #treeSnapshots: ReadonlyArray = []; + #treeSnapshots: TreeSnapshots = {} as any; constructor(trees: MerkleTreeDb, blockNumber: number) { this.#treesDb = trees; this.#blockNumber = blockNumber; } - async #getTreeSnapshot(merkleTreeId: number): Promise { - if (this.#treeSnapshots[merkleTreeId]) { - return this.#treeSnapshots[merkleTreeId]; + async #getTreeSnapshot(treeId: MerkleTreeId): Promise { + if (this.#treeSnapshots[treeId]) { + return this.#treeSnapshots[treeId]; } this.#treeSnapshots = await this.#treesDb.getSnapshot(this.#blockNumber); - return this.#treeSnapshots[merkleTreeId]!; + return this.#treeSnapshots[treeId]!; } - async findLeafIndex(treeId: MerkleTreeId, value: Buffer): Promise { + async findLeafIndex(treeId: ID, value: MerkleTreeLeafType): Promise { const tree = await this.#getTreeSnapshot(treeId); - return tree.findLeafIndex(value); + // TODO #5448 fix "as any" + return tree.findLeafIndex(value as any); } - async findLeafIndexAfter(treeId: MerkleTreeId, value: Buffer, startIndex: bigint): Promise { + async findLeafIndexAfter( + treeId: MerkleTreeId, + value: MerkleTreeLeafType, + startIndex: bigint, + ): Promise { const tree = await this.#getTreeSnapshot(treeId); - return tree.findLeafIndexAfter(value, startIndex); + // TODO #5448 fix "as any" + return tree.findLeafIndexAfter(value as any, startIndex); } - async getLeafPreimage( - treeId: MerkleTreeId.NULLIFIER_TREE, + async getLeafPreimage( + treeId: ID, index: bigint, ): Promise { const snapshot = (await this.#getTreeSnapshot(treeId)) as IndexedTreeSnapshot; return snapshot.getLatestLeafPreimageCopy(BigInt(index)); } - async getLeafValue(treeId: MerkleTreeId, index: bigint): Promise { + async getLeafValue( + treeId: ID, + index: bigint, + ): Promise | undefined> { const snapshot = await this.#getTreeSnapshot(treeId); - return snapshot.getLeafValue(BigInt(index)); + return snapshot.getLeafValue(BigInt(index)) as MerkleTreeLeafType | undefined; } async getPreviousValueIndex( - treeId: MerkleTreeId.NULLIFIER_TREE, + treeId: IndexedTreeId, value: bigint, ): Promise< | { diff --git a/yarn-project/world-state/src/world-state-db/merkle_trees.ts b/yarn-project/world-state/src/world-state-db/merkle_trees.ts index 541874b2d89..352d94e8fb7 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_trees.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_trees.ts @@ -40,10 +40,17 @@ import { } from '@aztec/merkle-tree'; import { Hasher } from '@aztec/types/interfaces'; -import { INITIAL_NULLIFIER_TREE_SIZE, INITIAL_PUBLIC_DATA_TREE_SIZE, MerkleTreeDb } from './merkle_tree_db.js'; +import { + INITIAL_NULLIFIER_TREE_SIZE, + INITIAL_PUBLIC_DATA_TREE_SIZE, + MerkleTreeDb, + TreeSnapshots, +} from './merkle_tree_db.js'; import { HandleL2BlockAndMessagesResult, IndexedTreeId, + MerkleTreeLeafType, + MerkleTreeMap, MerkleTreeOperations, TreeInfo, } from './merkle_tree_operations.js'; @@ -53,7 +60,15 @@ import { MerkleTreeOperationsFacade } from './merkle_tree_operations_facade.js'; * The nullifier tree is an indexed tree. */ class NullifierTree extends StandardIndexedTree { - constructor(store: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint = 0n, root?: Buffer) { + constructor( + store: AztecKVStore, + hasher: Hasher, + name: string, + depth: number, + size: bigint = 0n, + _noop: any, + root?: Buffer, + ) { super(store, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf, root); } } @@ -62,7 +77,15 @@ class NullifierTree extends StandardIndexedTree { * The public data tree is an indexed tree. */ class PublicDataTree extends StandardIndexedTree { - constructor(store: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint = 0n, root?: Buffer) { + constructor( + store: AztecKVStore, + hasher: Hasher, + name: string, + depth: number, + size: bigint = 0n, + _noop: any, + root?: Buffer, + ) { super(store, hasher, name, depth, size, PublicDataTreeLeafPreimage, PublicDataTreeLeaf, root); } } @@ -71,7 +94,8 @@ class PublicDataTree extends StandardIndexedTree { * A convenience class for managing multiple merkle trees. */ export class MerkleTrees implements MerkleTreeDb { - private trees: (AppendOnlyTree | UpdateOnlyTree)[] = []; + // gets initialized in #init + private trees: MerkleTreeMap = null as any; private jobQueue = new SerialQueue(); private constructor(private store: AztecKVStore, private log: DebugLogger) {} @@ -101,14 +125,16 @@ export class MerkleTrees implements MerkleTreeDb { this.store, hasher, `${MerkleTreeId[MerkleTreeId.NULLIFIER_TREE]}`, + {}, NULLIFIER_TREE_HEIGHT, INITIAL_NULLIFIER_TREE_SIZE, ); - const noteHashTree: AppendOnlyTree = await initializeTree( + const noteHashTree: AppendOnlyTree = await initializeTree( StandardTree, this.store, hasher, `${MerkleTreeId[MerkleTreeId.NOTE_HASH_TREE]}`, + Fr, NOTE_HASH_TREE_HEIGHT, ); const publicDataTree = await initializeTree( @@ -116,21 +142,24 @@ export class MerkleTrees implements MerkleTreeDb { this.store, hasher, `${MerkleTreeId[MerkleTreeId.PUBLIC_DATA_TREE]}`, + {}, PUBLIC_DATA_TREE_HEIGHT, INITIAL_PUBLIC_DATA_TREE_SIZE, ); - const l1Tol2MessageTree: AppendOnlyTree = await initializeTree( + const l1Tol2MessageTree: AppendOnlyTree = await initializeTree( StandardTree, this.store, hasher, `${MerkleTreeId[MerkleTreeId.L1_TO_L2_MESSAGE_TREE]}`, + Fr, L1_TO_L2_MSG_TREE_HEIGHT, ); - const archive: AppendOnlyTree = await initializeTree( + const archive: AppendOnlyTree = await initializeTree( StandardTree, this.store, hasher, `${MerkleTreeId[MerkleTreeId.ARCHIVE]}`, + Fr, ARCHIVE_HEIGHT, ); this.trees = [nullifierTree, noteHashTree, publicDataTree, l1Tol2MessageTree, archive]; @@ -230,7 +259,7 @@ export class MerkleTrees implements MerkleTreeDb { treeId: MerkleTreeId, index: bigint, includeUncommitted: boolean, - ): Promise { + ): Promise | undefined> { return await this.synchronize(() => Promise.resolve(this.trees[treeId].getLeafValue(index, includeUncommitted))); } @@ -255,7 +284,7 @@ export class MerkleTrees implements MerkleTreeDb { * @param leaves - The leaves to append. * @returns Empty promise. */ - public async appendLeaves(treeId: MerkleTreeId, leaves: Buffer[]): Promise { + public async appendLeaves(treeId: ID, leaves: MerkleTreeLeafType[]): Promise { return await this.synchronize(() => this.#appendLeaves(treeId, leaves)); } @@ -328,14 +357,15 @@ export class MerkleTrees implements MerkleTreeDb { * @param includeUncommitted - Indicates whether to include uncommitted data. * @returns The index of the first leaf found with a given value (undefined if not found). */ - public async findLeafIndex( - treeId: MerkleTreeId, - value: Buffer, + public async findLeafIndex( + treeId: ID, + value: MerkleTreeLeafType, includeUncommitted: boolean, ): Promise { return await this.synchronize(() => { const tree = this.trees[treeId]; - return Promise.resolve(tree.findLeafIndex(value, includeUncommitted)); + // TODO #5448 fix "as any" + return Promise.resolve(tree.findLeafIndex(value as any, includeUncommitted)); }); } @@ -346,15 +376,16 @@ export class MerkleTrees implements MerkleTreeDb { * @param startIndex - The index to start searching from (used when skipping nullified messages) * @param includeUncommitted - Indicates whether to include uncommitted data. */ - public async findLeafIndexAfter( - treeId: MerkleTreeId, - value: Buffer, + public async findLeafIndexAfter( + treeId: ID, + value: MerkleTreeLeafType, startIndex: bigint, includeUncommitted: boolean, ): Promise { return await this.synchronize(() => { const tree = this.trees[treeId]; - return Promise.resolve(tree.findLeafIndexAfter(value, startIndex, includeUncommitted)); + // TODO #5448 fix "as any" + return Promise.resolve(tree.findLeafIndexAfter(value as any, startIndex, includeUncommitted)); }); } @@ -391,7 +422,7 @@ export class MerkleTrees implements MerkleTreeDb { SubtreeHeight extends number, SubtreeSiblingPathHeight extends number, >( - treeId: MerkleTreeId, + treeId: IndexedTreeId, leaves: Buffer[], subtreeHeight: SubtreeHeight, ): Promise> { @@ -421,7 +452,7 @@ export class MerkleTrees implements MerkleTreeDb { } const blockHash = header.hash(); - await this.#appendLeaves(MerkleTreeId.ARCHIVE, [blockHash.toBuffer()]); + await this.#appendLeaves(MerkleTreeId.ARCHIVE, [blockHash]); } /** @@ -455,20 +486,21 @@ export class MerkleTrees implements MerkleTreeDb { * @param leaves - Leaves to append. * @returns Empty promise. */ - async #appendLeaves(treeId: MerkleTreeId, leaves: Buffer[]): Promise { + async #appendLeaves(treeId: ID, leaves: MerkleTreeLeafType[]): Promise { const tree = this.trees[treeId]; if (!('appendLeaves' in tree)) { throw new Error('Tree does not support `appendLeaves` method'); } - return await tree.appendLeaves(leaves); + // TODO #5448 fix "as any" + return await tree.appendLeaves(leaves as any[]); } - async #updateLeaf(treeId: IndexedTreeId, leaf: Buffer, index: bigint): Promise { + async #updateLeaf(treeId: IndexedTreeId, leaf: MerkleTreeLeafType, index: bigint): Promise { const tree = this.trees[treeId]; if (!('updateLeaf' in tree)) { throw new Error('Tree does not support `updateLeaf` method'); } - return await tree.updateLeaf(leaf, index); + return await (tree as UpdateOnlyTree).updateLeaf(leaf, index); } /** @@ -476,7 +508,7 @@ export class MerkleTrees implements MerkleTreeDb { * @returns Empty promise. */ async #commit(): Promise { - for (const tree of this.trees) { + for (const tree of Object.values(this.trees)) { await tree.commit(); } } @@ -486,17 +518,31 @@ export class MerkleTrees implements MerkleTreeDb { * @returns Empty promise. */ async #rollback(): Promise { - for (const tree of this.trees) { + for (const tree of Object.values(this.trees)) { await tree.rollback(); } } - public getSnapshot(blockNumber: number) { - return Promise.all(this.trees.map(tree => tree.getSnapshot(blockNumber))); + public async getSnapshot(blockNumber: number): Promise { + const snapshots = await Promise.all([ + this.trees[MerkleTreeId.NULLIFIER_TREE].getSnapshot(blockNumber), + this.trees[MerkleTreeId.NOTE_HASH_TREE].getSnapshot(blockNumber), + this.trees[MerkleTreeId.PUBLIC_DATA_TREE].getSnapshot(blockNumber), + this.trees[MerkleTreeId.L1_TO_L2_MESSAGE_TREE].getSnapshot(blockNumber), + this.trees[MerkleTreeId.ARCHIVE].getSnapshot(blockNumber), + ]); + + return { + [MerkleTreeId.NULLIFIER_TREE]: snapshots[0], + [MerkleTreeId.NOTE_HASH_TREE]: snapshots[1], + [MerkleTreeId.PUBLIC_DATA_TREE]: snapshots[2], + [MerkleTreeId.L1_TO_L2_MESSAGE_TREE]: snapshots[3], + [MerkleTreeId.ARCHIVE]: snapshots[4], + }; } async #snapshot(blockNumber: number): Promise { - for (const tree of this.trees) { + for (const tree of Object.values(this.trees)) { await tree.snapshot(blockNumber); } } @@ -535,10 +581,7 @@ export class MerkleTrees implements MerkleTreeDb { [MerkleTreeId.NOTE_HASH_TREE, l2Block.body.txEffects.flatMap(txEffect => txEffect.noteHashes)], [MerkleTreeId.L1_TO_L2_MESSAGE_TREE, l1ToL2MessagesPadded], ] as const) { - await this.#appendLeaves( - tree, - leaves.map(fr => fr.toBuffer()), - ); + await this.#appendLeaves(tree, leaves); } // Sync the indexed trees