feat: persistent archiver store

cspell.json

@@ -226,7 +226,8 @@
     "auditability",
     "hardfork",
     "composablity",
-    "counterparty"
+    "counterparty",
+    "lmdb"
   ],
   "ignorePaths": [
     "node_modules/",
@@ -253,7 +254,5 @@
     "lib",
     "*.cmake"
   ],
-  "flagWords": [
-    "anonymous"
-  ]
+  "flagWords": ["anonymous"]
 }

yarn-project/archiver/package.json

@@ -41,6 +41,7 @@
     "@aztec/types": "workspace:^",
     "@types/lodash.omit": "^4.5.7",
     "debug": "^4.3.4",
+    "lmdb": "^2.9.1",
     "lodash.omit": "^4.5.0",
     "tsc-watch": "^6.0.0",
     "tslib": "^2.5.0",

yarn-project/archiver/src/archiver/archiver.test.ts

@@ -11,7 +11,8 @@ import times from 'lodash.times';
 import { Chain, HttpTransport, Log, PublicClient, Transaction, encodeFunctionData, toHex } from 'viem';
 
 import { Archiver } from './archiver.js';
-import { ArchiverDataStore, MemoryArchiverStore } from './archiver_store.js';
+import { ArchiverDataStore } from './archiver_store.js';
+import { MemoryArchiverStore } from './memory_archiver_store/memory_archiver_store.js';
 
 describe('Archiver', () => {
   const rollupAddress = EthAddress.ZERO.toString();

yarn-project/archiver/src/archiver/archiver.ts

@@ -25,17 +25,20 @@ import {
   TxHash,
 } from '@aztec/types';
 
+import { RootDatabase } from 'lmdb';
 import omit from 'lodash.omit';
 import { Chain, HttpTransport, PublicClient, createPublicClient, getContract, http } from 'viem';
 
-import { ArchiverDataStore, MemoryArchiverStore } from './archiver_store.js';
+import { ArchiverDataStore } from './archiver_store.js';
 import { ArchiverConfig } from './config.js';
 import {
   retrieveBlocks,
   retrieveNewCancelledL1ToL2Messages,
   retrieveNewContractData,
   retrieveNewPendingL1ToL2Messages,
 } from './data_retrieval.js';
+import { LMDBArchiverStore } from './lmdb_archiver_store.js';
+import { MemoryArchiverStore } from './memory_archiver_store/memory_archiver_store.js';
 
 /**
  * Pulls L2 blocks in a non-blocking manner and provides interface for their retrieval.
@@ -93,10 +96,15 @@ export class Archiver implements L2BlockSource, L2LogsSource, ContractDataSource
   /**
    * Creates a new instance of the Archiver and blocks until it syncs from chain.
    * @param config - The archiver's desired configuration.
+   * @param rootDB - The database for the archiver.
    * @param blockUntilSynced - If true, blocks until the archiver has fully synced.
    * @returns - An instance of the archiver.
    */
-  public static async createAndSync(config: ArchiverConfig, blockUntilSynced = true): Promise<Archiver> {
+  public static async createAndSync(
+    config: ArchiverConfig,
+    rootDB?: RootDatabase,
+    blockUntilSynced = true,
+  ): Promise<Archiver> {
     const chain = createEthereumChain(config.rpcUrl, config.apiKey);
     const publicClient = createPublicClient({
       chain: chain.chainInfo,
@@ -113,7 +121,9 @@ export class Archiver implements L2BlockSource, L2LogsSource, ContractDataSource
     });
     const searchStartBlock = Number((await registryContract.read.getCurrentSnapshot()).blockNumber);
 
-    const archiverStore = new MemoryArchiverStore(config.maxLogs ?? 1000);
+    const archiverStore = rootDB
+      ? new LMDBArchiverStore(rootDB, config.maxLogs ?? 1000)
+      : new MemoryArchiverStore(config.maxLogs ?? 1000);
     const archiver = new Archiver(
       publicClient,
       config.l1Contracts.rollupAddress,
@@ -252,26 +262,22 @@ export class Archiver implements L2BlockSource, L2LogsSource, ContractDataSource
 
     this.log(`Retrieved ${retrievedBlocks.retrievedData.length} block(s) from chain`);
 
-    // store encrypted logs from L2 Blocks that we have retrieved
-    const encryptedLogs = retrievedBlocks.retrievedData.map(block => {
-      return block.newEncryptedLogs!;
-    });
-    await this.store.addLogs(encryptedLogs, LogType.ENCRYPTED);
-
-    // store unencrypted logs from L2 Blocks that we have retrieved
-    const unencryptedLogs = retrievedBlocks.retrievedData.map(block => {
-      return block.newUnencryptedLogs!;
-    });
-    await this.store.addLogs(unencryptedLogs, LogType.UNENCRYPTED);
+    await Promise.all(
+      retrievedBlocks.retrievedData.map(block =>
+        this.store.addLogs(block.newEncryptedLogs, block.newUnencryptedLogs, block.number),
+      ),
+    );
 
     // store contracts for which we have retrieved L2 blocks
     const lastKnownL2BlockNum = retrievedBlocks.retrievedData[retrievedBlocks.retrievedData.length - 1].number;
-    retrievedContracts.retrievedData.forEach(async ([contracts, l2BlockNum], index) => {
-      this.log(`Retrieved extended contract data for l2 block number: ${index}`);
-      if (l2BlockNum <= lastKnownL2BlockNum) {
-        await this.store.addExtendedContractData(contracts, l2BlockNum);
-      }
-    });
+    await Promise.all(
+      retrievedContracts.retrievedData.map(async ([contracts, l2BlockNum]) => {
+        this.log(`Retrieved extended contract data for l2 block number: ${l2BlockNum}`);
+        if (l2BlockNum <= lastKnownL2BlockNum) {
+          await this.store.addExtendedContractData(contracts, l2BlockNum);
+        }
+      }),
+    );
 
     // from retrieved L2Blocks, confirm L1 to L2 messages that have been published
     // from each l2block fetch all messageKeys in a flattened array: