feat: Track world state db versions and wipe the state upon version c…

…hange (#9946)

Implements a very basic mechanism of tracking changes to the world state
DB structure and deleting the world state when a change is detected.

yarn-project/world-state/src/native/native_world_state.test.ts

@@ -23,7 +23,8 @@ import { join } from 'path';
 import { assertSameState, compareChains, mockBlock } from '../test/utils.js';
 import { INITIAL_NULLIFIER_TREE_SIZE, INITIAL_PUBLIC_DATA_TREE_SIZE } from '../world-state-db/merkle_tree_db.js';
 import { type WorldStateStatusSummary } from './message.js';
-import { NativeWorldStateService } from './native_world_state.js';
+import { NativeWorldStateService, WORLD_STATE_VERSION_FILE } from './native_world_state.js';
+import { WorldStateVersion } from './world_state_version.js';
 
 describe('NativeWorldState', () => {
   let dataDir: string;
@@ -58,6 +59,8 @@ describe('NativeWorldState', () => {
       await expect(
         ws.getCommitted().findLeafIndex(MerkleTreeId.NOTE_HASH_TREE, block.body.txEffects[0].noteHashes[0]),
       ).resolves.toBeDefined();
+      const status = await ws.getStatusSummary();
+      expect(status.unfinalisedBlockNumber).toBe(1n);
       await ws.close();
     });
 
@@ -77,6 +80,41 @@ describe('NativeWorldState', () => {
       await expect(
         ws.getCommitted().findLeafIndex(MerkleTreeId.NOTE_HASH_TREE, block.body.txEffects[0].noteHashes[0]),
       ).resolves.toBeUndefined();
+      const status = await ws.getStatusSummary();
+      expect(status.unfinalisedBlockNumber).toBe(0n);
+      await ws.close();
+    });
+
+    it('clears the database if the world state version is different', async () => {
+      // open ws against the data again
+      let ws = await NativeWorldStateService.new(rollupAddress, dataDir, defaultDBMapSize);
+      // db should be empty
+      let emptyStatus = await ws.getStatusSummary();
+      expect(emptyStatus.unfinalisedBlockNumber).toBe(0n);
+
+      // populate it and then close it
+      const fork = await ws.fork();
+      ({ block, messages } = await mockBlock(1, 2, fork));
+      await fork.close();
+
+      const status = await ws.handleL2BlockAndMessages(block, messages);
+      expect(status.summary.unfinalisedBlockNumber).toBe(1n);
+      await ws.close();
+      // we open up the version file that was created and modify the version to be older
+      const fullPath = join(dataDir, 'world_state', WORLD_STATE_VERSION_FILE);
+      const storedWorldStateVersion = await WorldStateVersion.readVersion(fullPath);
+      expect(storedWorldStateVersion).toBeDefined();
+      const modifiedVersion = new WorldStateVersion(
+        storedWorldStateVersion!.version - 1,
+        storedWorldStateVersion!.rollupAddress,
+      );
+      await modifiedVersion.writeVersionFile(fullPath);
+
+      // Open the world state again and it should be empty
+      ws = await NativeWorldStateService.new(rollupAddress, dataDir, defaultDBMapSize);
+      // db should be empty
+      emptyStatus = await ws.getStatusSummary();
+      expect(emptyStatus.unfinalisedBlockNumber).toBe(0n);
       await ws.close();
     });
 

yarn-project/world-state/src/native/native_world_state.ts

@@ -24,7 +24,7 @@ import { padArrayEnd } from '@aztec/foundation/collection';
 import { createDebugLogger } from '@aztec/foundation/log';
 
 import assert from 'assert/strict';
-import { mkdir, mkdtemp, readFile, rm, writeFile } from 'fs/promises';
+import { mkdir, mkdtemp, rm } from 'fs/promises';
 import { tmpdir } from 'os';
 import { join } from 'path';
 
@@ -40,8 +40,16 @@ import {
   worldStateRevision,
 } from './message.js';
 import { NativeWorldState } from './native_world_state_instance.js';
+import { WorldStateVersion } from './world_state_version.js';
 
-const ROLLUP_ADDRESS_FILE = 'rollup_address';
+export const WORLD_STATE_VERSION_FILE = 'version';
+
+// A crude way of maintaining DB versioning
+// We don't currently have any method of performing data migrations
+// should the world state db structure change
+// For now we will track versions using this hardcoded value and delete
+// the state if a change is detected
+export const WORLD_STATE_DB_VERSION = 1; // The initial version
 
 export class NativeWorldStateService implements MerkleTreeDatabase {
   protected initialHeader: Header | undefined;
@@ -60,17 +68,24 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
     cleanup = () => Promise.resolve(),
   ): Promise<NativeWorldStateService> {
     const worldStateDirectory = join(dataDir, 'world_state');
-    const rollupAddressFile = join(worldStateDirectory, ROLLUP_ADDRESS_FILE);
-    const currentRollupStr = await readFile(rollupAddressFile, 'utf8').catch(() => undefined);
-    const currentRollupAddress = currentRollupStr ? EthAddress.fromString(currentRollupStr.trim()) : undefined;
+    const versionFile = join(worldStateDirectory, WORLD_STATE_VERSION_FILE);
+    const storedWorldStateVersion = await WorldStateVersion.readVersion(versionFile);
 
-    if (currentRollupAddress && !rollupAddress.equals(currentRollupAddress)) {
-      log.warn('Rollup address changed, deleting database');
+    if (!storedWorldStateVersion) {
+      log.warn('No world state version found, deleting world state directory');
+      await rm(worldStateDirectory, { recursive: true, force: true });
+    } else if (!rollupAddress.equals(storedWorldStateVersion.rollupAddress)) {
+      log.warn('Rollup address changed, deleting world state directory');
+      await rm(worldStateDirectory, { recursive: true, force: true });
+    } else if (storedWorldStateVersion.version != WORLD_STATE_DB_VERSION) {
+      log.warn('World state version change detected, deleting world state directory');
       await rm(worldStateDirectory, { recursive: true, force: true });
     }
 
+    const newWorldStateVersion = new WorldStateVersion(WORLD_STATE_DB_VERSION, rollupAddress);
+
     await mkdir(worldStateDirectory, { recursive: true });
-    await writeFile(rollupAddressFile, rollupAddress.toString(), 'utf8');
+    await newWorldStateVersion.writeVersionFile(versionFile);
 
     const instance = new NativeWorldState(worldStateDirectory, dbMapSizeKb);
     const worldState = new this(instance, log, cleanup);

yarn-project/world-state/src/native/world_state_version.ts

@@ -0,0 +1,41 @@
+import { EthAddress } from '@aztec/circuits.js';
+
+import { readFile, writeFile } from 'fs/promises';
+
+export class WorldStateVersion {
+  constructor(readonly version: number, readonly rollupAddress: EthAddress) {}
+
+  static async readVersion(filename: string) {
+    const versionData = await readFile(filename, 'utf-8').catch(() => undefined);
+    if (versionData === undefined) {
+      return undefined;
+    }
+    const versionJSON = JSON.parse(versionData);
+    if (versionJSON.version === undefined || versionJSON.rollupAddress === undefined) {
+      return undefined;
+    }
+    return WorldStateVersion.fromJSON(versionJSON);
+  }
+
+  public async writeVersionFile(filename: string) {
+    const data = JSON.stringify(this.toJSON());
+    await writeFile(filename, data, 'utf-8');
+  }
+
+  toJSON() {
+    return {
+      version: this.version,
+      rollupAddress: this.rollupAddress.toChecksumString(),
+    };
+  }
+
+  static fromJSON(obj: any): WorldStateVersion {
+    const version = obj.version;
+    const rollupAddress = EthAddress.fromString(obj.rollupAddress);
+    return new WorldStateVersion(version, rollupAddress);
+  }
+
+  static empty() {
+    return new WorldStateVersion(0, EthAddress.ZERO);
+  }
+}