From 209d4849a19cf4c8c6429df81c55f0363c636b3c Mon Sep 17 00:00:00 2001 From: PhilWindle <60546371+PhilWindle@users.noreply.github.com> Date: Tue, 19 Nov 2024 16:16:22 +0000 Subject: [PATCH] feat: Track world state db versions and wipe the state upon version change (#9946) Implements a very basic mechanism of tracking changes to the world state DB structure and deleting the world state when a change is detected. --- .../src/native/native_world_state.test.ts | 40 +++++++++++++++++- .../src/native/native_world_state.ts | 31 ++++++++++---- .../src/native/world_state_version.ts | 41 +++++++++++++++++++ 3 files changed, 103 insertions(+), 9 deletions(-) create mode 100644 yarn-project/world-state/src/native/world_state_version.ts diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts index 069c081eb3c..dbfd92a0b24 100644 --- a/yarn-project/world-state/src/native/native_world_state.test.ts +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -23,7 +23,8 @@ import { join } from 'path'; import { assertSameState, compareChains, mockBlock } from '../test/utils.js'; import { INITIAL_NULLIFIER_TREE_SIZE, INITIAL_PUBLIC_DATA_TREE_SIZE } from '../world-state-db/merkle_tree_db.js'; import { type WorldStateStatusSummary } from './message.js'; -import { NativeWorldStateService } from './native_world_state.js'; +import { NativeWorldStateService, WORLD_STATE_VERSION_FILE } from './native_world_state.js'; +import { WorldStateVersion } from './world_state_version.js'; describe('NativeWorldState', () => { let dataDir: string; @@ -58,6 +59,8 @@ describe('NativeWorldState', () => { await expect( ws.getCommitted().findLeafIndex(MerkleTreeId.NOTE_HASH_TREE, block.body.txEffects[0].noteHashes[0]), ).resolves.toBeDefined(); + const status = await ws.getStatusSummary(); + expect(status.unfinalisedBlockNumber).toBe(1n); await ws.close(); }); @@ -77,6 +80,41 @@ describe('NativeWorldState', () => { await expect( ws.getCommitted().findLeafIndex(MerkleTreeId.NOTE_HASH_TREE, block.body.txEffects[0].noteHashes[0]), ).resolves.toBeUndefined(); + const status = await ws.getStatusSummary(); + expect(status.unfinalisedBlockNumber).toBe(0n); + await ws.close(); + }); + + it('clears the database if the world state version is different', async () => { + // open ws against the data again + let ws = await NativeWorldStateService.new(rollupAddress, dataDir, defaultDBMapSize); + // db should be empty + let emptyStatus = await ws.getStatusSummary(); + expect(emptyStatus.unfinalisedBlockNumber).toBe(0n); + + // populate it and then close it + const fork = await ws.fork(); + ({ block, messages } = await mockBlock(1, 2, fork)); + await fork.close(); + + const status = await ws.handleL2BlockAndMessages(block, messages); + expect(status.summary.unfinalisedBlockNumber).toBe(1n); + await ws.close(); + // we open up the version file that was created and modify the version to be older + const fullPath = join(dataDir, 'world_state', WORLD_STATE_VERSION_FILE); + const storedWorldStateVersion = await WorldStateVersion.readVersion(fullPath); + expect(storedWorldStateVersion).toBeDefined(); + const modifiedVersion = new WorldStateVersion( + storedWorldStateVersion!.version - 1, + storedWorldStateVersion!.rollupAddress, + ); + await modifiedVersion.writeVersionFile(fullPath); + + // Open the world state again and it should be empty + ws = await NativeWorldStateService.new(rollupAddress, dataDir, defaultDBMapSize); + // db should be empty + emptyStatus = await ws.getStatusSummary(); + expect(emptyStatus.unfinalisedBlockNumber).toBe(0n); await ws.close(); }); diff --git a/yarn-project/world-state/src/native/native_world_state.ts b/yarn-project/world-state/src/native/native_world_state.ts index 8dc112fecfc..9af6299f157 100644 --- a/yarn-project/world-state/src/native/native_world_state.ts +++ b/yarn-project/world-state/src/native/native_world_state.ts @@ -24,7 +24,7 @@ import { padArrayEnd } from '@aztec/foundation/collection'; import { createDebugLogger } from '@aztec/foundation/log'; import assert from 'assert/strict'; -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'fs/promises'; +import { mkdir, mkdtemp, rm } from 'fs/promises'; import { tmpdir } from 'os'; import { join } from 'path'; @@ -40,8 +40,16 @@ import { worldStateRevision, } from './message.js'; import { NativeWorldState } from './native_world_state_instance.js'; +import { WorldStateVersion } from './world_state_version.js'; -const ROLLUP_ADDRESS_FILE = 'rollup_address'; +export const WORLD_STATE_VERSION_FILE = 'version'; + +// A crude way of maintaining DB versioning +// We don't currently have any method of performing data migrations +// should the world state db structure change +// For now we will track versions using this hardcoded value and delete +// the state if a change is detected +export const WORLD_STATE_DB_VERSION = 1; // The initial version export class NativeWorldStateService implements MerkleTreeDatabase { protected initialHeader: Header | undefined; @@ -60,17 +68,24 @@ export class NativeWorldStateService implements MerkleTreeDatabase { cleanup = () => Promise.resolve(), ): Promise { const worldStateDirectory = join(dataDir, 'world_state'); - const rollupAddressFile = join(worldStateDirectory, ROLLUP_ADDRESS_FILE); - const currentRollupStr = await readFile(rollupAddressFile, 'utf8').catch(() => undefined); - const currentRollupAddress = currentRollupStr ? EthAddress.fromString(currentRollupStr.trim()) : undefined; + const versionFile = join(worldStateDirectory, WORLD_STATE_VERSION_FILE); + const storedWorldStateVersion = await WorldStateVersion.readVersion(versionFile); - if (currentRollupAddress && !rollupAddress.equals(currentRollupAddress)) { - log.warn('Rollup address changed, deleting database'); + if (!storedWorldStateVersion) { + log.warn('No world state version found, deleting world state directory'); + await rm(worldStateDirectory, { recursive: true, force: true }); + } else if (!rollupAddress.equals(storedWorldStateVersion.rollupAddress)) { + log.warn('Rollup address changed, deleting world state directory'); + await rm(worldStateDirectory, { recursive: true, force: true }); + } else if (storedWorldStateVersion.version != WORLD_STATE_DB_VERSION) { + log.warn('World state version change detected, deleting world state directory'); await rm(worldStateDirectory, { recursive: true, force: true }); } + const newWorldStateVersion = new WorldStateVersion(WORLD_STATE_DB_VERSION, rollupAddress); + await mkdir(worldStateDirectory, { recursive: true }); - await writeFile(rollupAddressFile, rollupAddress.toString(), 'utf8'); + await newWorldStateVersion.writeVersionFile(versionFile); const instance = new NativeWorldState(worldStateDirectory, dbMapSizeKb); const worldState = new this(instance, log, cleanup); diff --git a/yarn-project/world-state/src/native/world_state_version.ts b/yarn-project/world-state/src/native/world_state_version.ts new file mode 100644 index 00000000000..20707d354ab --- /dev/null +++ b/yarn-project/world-state/src/native/world_state_version.ts @@ -0,0 +1,41 @@ +import { EthAddress } from '@aztec/circuits.js'; + +import { readFile, writeFile } from 'fs/promises'; + +export class WorldStateVersion { + constructor(readonly version: number, readonly rollupAddress: EthAddress) {} + + static async readVersion(filename: string) { + const versionData = await readFile(filename, 'utf-8').catch(() => undefined); + if (versionData === undefined) { + return undefined; + } + const versionJSON = JSON.parse(versionData); + if (versionJSON.version === undefined || versionJSON.rollupAddress === undefined) { + return undefined; + } + return WorldStateVersion.fromJSON(versionJSON); + } + + public async writeVersionFile(filename: string) { + const data = JSON.stringify(this.toJSON()); + await writeFile(filename, data, 'utf-8'); + } + + toJSON() { + return { + version: this.version, + rollupAddress: this.rollupAddress.toChecksumString(), + }; + } + + static fromJSON(obj: any): WorldStateVersion { + const version = obj.version; + const rollupAddress = EthAddress.fromString(obj.rollupAddress); + return new WorldStateVersion(version, rollupAddress); + } + + static empty() { + return new WorldStateVersion(0, EthAddress.ZERO); + } +}