Merge pull request #6899 from LiskHQ/6752-add-stateroot-block-gen

Add stateRoot when generating a block - Closes #6752

elements/lisk-chain/src/chain.ts

@@ -22,7 +22,6 @@ import {
 	GENESIS_BLOCK_VERSION,
 } from './constants';
 import { DataAccess } from './data_access';
-import { StateStore } from './state_store';
 import { validateGenesisBlock } from './validate';
 import {
 	blockSchema,
@@ -32,6 +31,7 @@ import {
 } from './schema';
 import { Block } from './block';
 import { BlockHeader } from './block_header';
+import { CurrentState } from './state_store/smt_store';
 
 interface ChainConstructor {
 	// Constants
@@ -183,21 +183,21 @@ export class Chain {
 
 	public async saveBlock(
 		block: Block,
-		stateStore: StateStore,
+		state: CurrentState,
 		finalizedHeight: number,
 		{ removeFromTempTable } = {
 			removeFromTempTable: false,
 		},
 	): Promise<void> {
-		await this.dataAccess.saveBlock(block, stateStore, finalizedHeight, removeFromTempTable);
+		await this.dataAccess.saveBlock(block, state, finalizedHeight, removeFromTempTable);
 		this.dataAccess.addBlockHeader(block.header);
 		this._finalizedHeight = finalizedHeight;
 		this._lastBlock = block;
 	}
 
 	public async removeBlock(
 		block: Block,
-		stateStore: StateStore,
+		state: CurrentState,
 		{ saveTempBlock } = { saveTempBlock: false },
 	): Promise<void> {
 		if (block.header.version === GENESIS_BLOCK_VERSION) {
@@ -210,7 +210,7 @@ export class Chain {
 			throw new Error('PreviousBlock is null.');
 		}
 
-		await this.dataAccess.deleteBlock(block, stateStore, saveTempBlock);
+		await this.dataAccess.deleteBlock(block, state, saveTempBlock);
 		await this.dataAccess.removeBlockHeader(block.header.id);
 		this._lastBlock = secondLastBlock;
 	}

elements/lisk-chain/src/data_access/data_access.ts

@@ -11,6 +11,7 @@
  *
  * Removal or modification of this copyright notice is prohibited.
  */
+
 import { KVStore, NotFoundError } from '@liskhq/lisk-db';
 import { Transaction } from '../transaction';
 import { RawBlock } from '../types';
@@ -19,8 +20,8 @@ import { Block } from '../block';
 
 import { BlockCache } from './cache';
 import { Storage as StorageAccess } from './storage';
-import { StateStore } from '../state_store';
 import { BlockAssets } from '../block_assets';
+import { CurrentState } from '../state_store';
 
 interface DAConstructor {
 	readonly db: KVStore;
@@ -282,12 +283,13 @@ export class DataAccess {
 	*/
 	public async saveBlock(
 		block: Block,
-		stateStore: StateStore,
+		state: CurrentState,
 		finalizedHeight: number,
 		removeFromTemp = false,
 	): Promise<void> {
 		const { id: blockID, height } = block.header;
 		const encodedHeader = block.header.getBytes();
+
 		const encodedPayload = [];
 		for (const tx of block.payload) {
 			const txID = tx.id;
@@ -301,26 +303,23 @@ export class DataAccess {
 			encodedHeader,
 			encodedPayload,
 			block.assets.getBytes(),
-			stateStore,
+			state,
 			removeFromTemp,
 		);
 	}
 
-	public async deleteBlock(
-		block: Block,
-		stateStore: StateStore,
-		saveToTemp = false,
-	): Promise<void> {
+	public async deleteBlock(block: Block, state: CurrentState, saveToTemp = false): Promise<void> {
 		const { id: blockID, height } = block.header;
 		const txIDs = block.payload.map(tx => tx.id);
+
 		const encodedBlock = block.getBytes();
 		await this._storage.deleteBlock(
 			blockID,
 			height,
 			txIDs,
 			block.assets.getBytes(),
 			encodedBlock,
-			stateStore,
+			state,
 			saveToTemp,
 		);
 	}

elements/lisk-chain/src/data_access/storage.ts

@@ -15,7 +15,6 @@ import { KVStore, formatInt, getFirstPrefix, getLastPrefix, NotFoundError } from
 import { codec } from '@liskhq/lisk-codec';
 import { hash } from '@liskhq/lisk-cryptography';
 import { RawBlock, StateDiff } from '../types';
-import { StateStore } from '../state_store';
 
 import {
 	DB_KEY_BLOCKS_ID,
@@ -29,6 +28,7 @@ import {
 } from '../db_keys';
 import { concatDBKeys } from '../utils';
 import { stateDiffSchema } from '../schema';
+import { CurrentState } from '../state_store';
 
 const bytesArraySchema = {
 	$id: 'lisk-chain/bytesarray',
@@ -54,10 +54,7 @@ const decodeByteArray = (val: Buffer): Buffer[] => {
 	return decoded.list;
 };
 
-const encodeByteArray = (val: Buffer[]): Buffer => {
-	const encoded = codec.encode(bytesArraySchema, { list: val });
-	return codec.encode(bytesArraySchema, encoded);
-};
+const encodeByteArray = (val: Buffer[]): Buffer => codec.encode(bytesArraySchema, { list: val });
 
 export class Storage {
 	private readonly _db: KVStore;
@@ -341,11 +338,11 @@ export class Storage {
 		header: Buffer,
 		payload: { id: Buffer; value: Buffer }[],
 		assets: Buffer[],
-		stateStore: StateStore,
+		state: CurrentState,
 		removeFromTemp = false,
 	): Promise<void> {
 		const heightBuf = formatInt(height);
-		const batch = this._db.batch();
+		const { batch, diff } = state;
 		batch.put(concatDBKeys(DB_KEY_BLOCKS_ID, id), header);
 		batch.put(concatDBKeys(DB_KEY_BLOCKS_HEIGHT, heightBuf), id);
 		if (payload.length > 0) {
@@ -363,7 +360,7 @@ export class Storage {
 		if (removeFromTemp) {
 			batch.del(concatDBKeys(DB_KEY_TEMPBLOCKS_HEIGHT, heightBuf));
 		}
-		const diff = stateStore.finalize(batch);
+
 		const encodedDiff = codec.encode(stateDiffSchema, diff);
 		batch.put(concatDBKeys(DB_KEY_DIFF_STATE, formatInt(height)), encodedDiff);
 		const finalizedHeightBytes = Buffer.alloc(4);
@@ -380,10 +377,10 @@ export class Storage {
 		txIDs: Buffer[],
 		assets: Buffer[],
 		fullBlock: Buffer,
-		stateStore: StateStore,
+		state: CurrentState,
 		saveToTemp = false,
 	): Promise<StateDiff> {
-		const batch = this._db.batch();
+		const { batch, smt, smtStore } = state;
 		const heightBuf = formatInt(height);
 		batch.del(concatDBKeys(DB_KEY_BLOCKS_ID, id));
 		batch.del(concatDBKeys(DB_KEY_BLOCKS_HEIGHT, heightBuf));
@@ -410,20 +407,23 @@ export class Storage {
 			updated: updatedStates,
 			deleted: deletedStates,
 		} = codec.decode<StateDiff>(stateDiffSchema, stateDiff);
+
 		// Delete all the newly created states
 		for (const key of createdStates) {
 			batch.del(key);
+			await smt.remove(key);
 		}
 		// Revert all deleted values
 		for (const { key, value: previousValue } of deletedStates) {
 			batch.put(key, previousValue);
+			await smt.update(key, previousValue);
 		}
 		for (const { key, value: previousValue } of updatedStates) {
 			batch.put(key, previousValue);
+			await smt.update(key, previousValue);
 		}
-		// ignore diff created while deleting
-		stateStore.finalize(batch);
 
+		smtStore.finalize(batch);
 		// Delete stored diff at particular height
 		batch.del(diffKey);
 

elements/lisk-chain/src/db_keys.ts

@@ -27,3 +27,6 @@ export const DB_KEY_FINALIZED_HEIGHT = Buffer.from([27]);
 
 // 51-75 for diff state
 export const DB_KEY_DIFF_STATE = Buffer.from([51]);
+
+// 76 for smt of state
+export const DB_KEY_STATE_SMT = Buffer.from([76]);

elements/lisk-chain/src/index.ts

@@ -31,7 +31,7 @@ export type { RawBlock } from './types';
 export { Slots } from './slots';
 export { concatDBKeys } from './utils';
 
-export { StateStore, NotFoundError } from './state_store';
+export { StateStore, NotFoundError, CurrentState, SMTStore } from './state_store';
 export { Block } from './block';
 export { BlockAsset, BlockAssets } from './block_assets';
 export { BlockHeader, BlockHeaderAttrs, BlockHeaderJSON } from './block_header';

elements/lisk-chain/src/state_store/cache_db.ts

@@ -12,6 +12,7 @@
  * Removal or modification of this copyright notice is prohibited.
  */
 
+import { SparseMerkleTree } from '@liskhq/lisk-tree';
 import { StateDiff } from '../types';
 import { copyBuffer } from './utils';
 import { DatabaseWriter } from './types';
@@ -157,16 +158,18 @@ export class CacheDB {
 		return newDB;
 	}
 
-	public finalize(batch: DatabaseWriter): StateDiff {
+	public async finalize(batch: DatabaseWriter, smt: SparseMerkleTree): Promise<StateDiff> {
 		const diff: StateDiff = {
 			created: [],
 			deleted: [],
 			updated: [],
 		};
+
 		for (const [key, value] of Object.entries(this._data)) {
 			const keyBytes = Buffer.from(key, 'binary');
 			if (value.init === undefined) {
 				diff.created.push(keyBytes);
+				await smt.update(keyBytes, value.value);
 				batch.put(keyBytes, value.value);
 				continue;
 			}
@@ -175,6 +178,7 @@ export class CacheDB {
 					key: keyBytes,
 					value: value.init,
 				});
+				await smt.remove(keyBytes);
 				batch.del(keyBytes);
 				continue;
 			}
@@ -183,9 +187,11 @@ export class CacheDB {
 					key: keyBytes,
 					value: value.init,
 				});
+				await smt.update(keyBytes, value.value);
 				batch.put(keyBytes, value.value);
 			}
 		}
+
 		return diff;
 	}
 }

elements/lisk-chain/src/state_store/index.ts

@@ -13,4 +13,5 @@
  */
 
 export { NotFoundError } from './errors';
+export { CurrentState, SMTStore } from './smt_store';
 export { StateStore, IterateOptions } from './state_store';

elements/lisk-chain/src/state_store/smt_store.ts

@@ -0,0 +1,80 @@
+/*
+ * Copyright © 2021 Lisk Foundation
+ *
+ * See the LICENSE file at the top-level directory of this distribution
+ * for licensing information.
+ *
+ * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation,
+ * no part of this software, including this file, may be copied, modified,
+ * propagated, or distributed except according to the terms contained in the
+ * LICENSE file.
+ *
+ * Removal or modification of this copyright notice is prohibited.
+ */
+
+import { BatchChain, NotFoundError as DBNotFoundError } from '@liskhq/lisk-db';
+import { SparseMerkleTree } from '@liskhq/lisk-tree';
+import { dataStructures } from '@liskhq/lisk-utils';
+import { StateStore } from './state_store';
+import { DB_KEY_STATE_SMT } from '../db_keys';
+import { StateDiff } from '../types';
+import { NotFoundError } from './errors';
+import { DatabaseReader, DatabaseWriter } from './types';
+
+export interface CurrentState {
+	batch: BatchChain;
+	diff: StateDiff;
+	stateStore: StateStore;
+	smt: SparseMerkleTree;
+	smtStore: SMTStore;
+}
+
+export class SMTStore {
+	private readonly _db: DatabaseReader;
+	private readonly _data: dataStructures.BufferMap<Buffer>;
+
+	public constructor(db: DatabaseReader, data?: dataStructures.BufferMap<Buffer>) {
+		this._db = db;
+		this._data = data ?? new dataStructures.BufferMap();
+	}
+
+	public async get(key: Buffer): Promise<Buffer> {
+		const prefixedKey = this._getKey(key);
+		const cachedValue = this._data.get(prefixedKey);
+		if (cachedValue) {
+			return cachedValue;
+		}
+		try {
+			const storedValue = await this._db.get(prefixedKey);
+			this._data.set(prefixedKey, storedValue);
+			return storedValue;
+		} catch (error) {
+			if (error instanceof DBNotFoundError) {
+				throw new NotFoundError(key);
+			}
+			throw error;
+		}
+	}
+
+	// eslint-disable-next-line @typescript-eslint/require-await
+	public async set(key: Buffer, value: Buffer): Promise<void> {
+		const prefixedKey = this._getKey(key);
+		this._data.set(prefixedKey, value);
+	}
+
+	// eslint-disable-next-line @typescript-eslint/require-await
+	public async del(key: Buffer): Promise<void> {
+		const prefixedKey = this._getKey(key);
+		this._data.delete(prefixedKey);
+	}
+
+	public finalize(batch: DatabaseWriter): void {
+		for (const [key, value] of this._data.entries()) {
+			batch.put(key, value);
+		}
+	}
+
+	private _getKey(key: Buffer): Buffer {
+		return Buffer.concat([DB_KEY_STATE_SMT, key]);
+	}
+}

elements/lisk-chain/src/state_store/state_store.ts

@@ -14,6 +14,7 @@
 
 import { codec, Schema } from '@liskhq/lisk-codec';
 import { NotFoundError as DBNotFoundError } from '@liskhq/lisk-db';
+import { SparseMerkleTree } from '@liskhq/lisk-tree';
 import { DB_KEY_STATE_STORE } from '../db_keys';
 import { StateDiff } from '../types';
 import { CacheDB } from './cache_db';
@@ -60,6 +61,7 @@ export class StateStore {
 			Buffer.concat([DB_KEY_STATE_STORE, moduleIDBuffer, storePrefixBuffer]),
 			this._cache,
 		);
+
 		return subStore;
 	}
 
@@ -229,8 +231,8 @@ export class StateStore {
 		this._snapshot = undefined;
 	}
 
-	public finalize(batch: DatabaseWriter): StateDiff {
-		return this._cache.finalize(batch);
+	public async finalize(batch: DatabaseWriter, smt: SparseMerkleTree): Promise<StateDiff> {
+		return this._cache.finalize(batch, smt);
 	}
 
 	private async _ensureCache(prefixedKey: Buffer): Promise<boolean> {