Blockchain: More Modern and Flexible Consensus Layout / Tree Shaking …

…Optimization (#3504)

* Switch to a more flexible Blockchain consensusDict options structure allowing to pass in different consensus objects

* Remove misplaced (non consensus checkand useless (already checked in block header) difficulty equals 0 check from CasperConsensus

* Shift to a new consensus semantics adhering to the fact that *consensus* and *consensus validation* is basically the same, allowing for more flexible instantiation and optional consensus object usage

* Remove redundant validateConsensus flag, fix block validation, clique and custom consensus tests

* Readd validateConsensus flag (default: true) to allow for more fine-grained settings to use the mechanism (e.g. Clique) but skip the respective validation

* Find a middle ground between convenience (allow mainnet default blockchain without need for ethash passing in) and consensus availability validation (now in the validate() call)

* Add clique example

* Client fixes

* Fix VM test

* Minor

* Remove Ethash dependency from Blockchain, adjust EthashConsensus ethash object integration

* Re-add CasperConsensus exports

* Rebuild package-lock.json

* EtashConsensus fix

* Fixes

* Fix client CLI tests

* Cleaner consensus check on validate call

* More consistent check for consensus object, re-add test for custom consensus transition

* Re-add difficulty check for PoS, re-activate removed test

packages/blockchain/examples/clique.ts

@@ -0,0 +1,12 @@
+import { createBlockchain, CliqueConsensus, ConsensusDict } from '@ethereumjs/blockchain'
+import { Chain, Common, ConsensusAlgorithm, Hardfork } from '@ethereumjs/common'
+
+const common = new Common({ chain: Chain.Goerli, hardfork: Hardfork.London })
+
+const consensusDict: ConsensusDict = {}
+consensusDict[ConsensusAlgorithm.Clique] = new CliqueConsensus()
+const blockchain = await createBlockchain({
+  consensusDict,
+  common,
+})
+console.log(`Created blockchain with ${blockchain.consensus.algorithm} consensus algorithm`)

packages/blockchain/package.json

@@ -49,7 +49,6 @@
   "dependencies": {
     "@ethereumjs/block": "^5.2.0",
     "@ethereumjs/common": "^4.3.0",
-    "@ethereumjs/ethash": "^3.0.3",
     "@ethereumjs/rlp": "^5.0.2",
     "@ethereumjs/trie": "^6.2.0",
     "@ethereumjs/tx": "^5.3.0",
@@ -58,7 +57,9 @@
     "ethereum-cryptography": "^2.2.1",
     "lru-cache": "10.1.0"
   },
-  "devDependencies": {},
+  "devDependencies": {
+    "@ethereumjs/ethash": "^3.0.3"
+  },
   "engines": {
     "node": ">=18"
   }

packages/blockchain/src/blockchain.ts

@@ -14,7 +14,7 @@ import {
   equalsBytes,
 } from '@ethereumjs/util'
 
-import { CasperConsensus, CliqueConsensus, EthashConsensus } from './consensus/index.js'
+import { CasperConsensus } from './consensus/casper.js'
 import {
   DBOp,
   DBSaveLookups,
@@ -30,17 +30,24 @@ import type {
   BlockchainInterface,
   BlockchainOptions,
   Consensus,
+  ConsensusDict,
   OnBlock,
 } from './types.js'
 import type { HeaderData } from '@ethereumjs/block'
 import type { CliqueConfig } from '@ethereumjs/common'
 import type { BigIntLike, DB, DBObject, GenesisState } from '@ethereumjs/util'
 
 /**
- * This class stores and interacts with blocks.
+ * Blockchain implementation to create and maintain a valid canonical chain
+ * of block headers or blocks with support for reorgs and the ability to provide
+ * custom DB backends.
+ *
+ * By default consensus validation is not provided since with the swith to
+ * Proof-of-Stake consensus is validated by the Ethereum consensus layer.
+ * If consensus validation is desired for Etash or Clique blockchains the
+ * optional `consensusDict` option can be used to pass in validation objects.
  */
 export class Blockchain implements BlockchainInterface {
-  consensus: Consensus
   db: DB<Uint8Array | string, Uint8Array | string | DBObject>
   dbManager: DBManager
   events: AsyncEventEmitter<BlockchainEvents>
@@ -70,8 +77,9 @@ export class Blockchain implements BlockchainInterface {
 
   public readonly common: Common
   private _hardforkByHeadBlockNumber: boolean
-  private readonly _validateConsensus: boolean
   private readonly _validateBlocks: boolean
+  private readonly _validateConsensus: boolean
+  private _consensusDict: ConsensusDict
 
   /**
    * This is used to track which canonical blocks are deleted. After a method calls
@@ -103,8 +111,8 @@ export class Blockchain implements BlockchainInterface {
     }
 
     this._hardforkByHeadBlockNumber = opts.hardforkByHeadBlockNumber ?? false
-    this._validateConsensus = opts.validateConsensus ?? true
     this._validateBlocks = opts.validateBlocks ?? true
+    this._validateConsensus = opts.validateConsensus ?? false
     this._customGenesisState = opts.genesisState
 
     this.db = opts.db !== undefined ? opts.db : new MapDB()
@@ -113,38 +121,13 @@ export class Blockchain implements BlockchainInterface {
 
     this.events = new AsyncEventEmitter()
 
-    if (opts.consensus) {
-      this.consensus = opts.consensus
-    } else {
-      switch (this.common.consensusAlgorithm()) {
-        case ConsensusAlgorithm.Casper:
-          this.consensus = new CasperConsensus()
-          break
-        case ConsensusAlgorithm.Clique:
-          this.consensus = new CliqueConsensus()
-          break
-        case ConsensusAlgorithm.Ethash:
-          this.consensus = new EthashConsensus()
-          break
-        default:
-          throw new Error(`consensus algorithm ${this.common.consensusAlgorithm()} not supported`)
-      }
-    }
+    this._consensusDict = {}
+    this._consensusDict[ConsensusAlgorithm.Casper] = new CasperConsensus()
 
-    if (this._validateConsensus) {
-      if (this.common.consensusType() === ConsensusType.ProofOfWork) {
-        if (this.common.consensusAlgorithm() !== ConsensusAlgorithm.Ethash) {
-          throw new Error('consensus validation only supported for pow ethash algorithm')
-        }
-      }
-      if (this.common.consensusType() === ConsensusType.ProofOfAuthority) {
-        if (this.common.consensusAlgorithm() !== ConsensusAlgorithm.Clique) {
-          throw new Error(
-            'consensus (signature) validation only supported for poa clique algorithm'
-          )
-        }
-      }
+    if (opts.consensusDict !== undefined) {
+      this._consensusDict = { ...this._consensusDict, ...opts.consensusDict }
     }
+    this._consensusCheck()
 
     this._heads = {}
 
@@ -155,6 +138,22 @@ export class Blockchain implements BlockchainInterface {
     }
   }
 
+  private _consensusCheck() {
+    if (this._validateConsensus && this.consensus === undefined) {
+      throw new Error(
+        `Consensus object for ${this.common.consensusAlgorithm()} must be passed (see consensusDict option) if consensus validation is activated`
+      )
+    }
+  }
+
+  /**
+   * Returns an eventual consensus object matching the current consensus algorithm from Common
+   * or undefined if non available
+   */
+  get consensus(): Consensus | undefined {
+    return this._consensusDict[this.common.consensusAlgorithm()]
+  }
+
   /**
    * Returns a deep copy of this {@link Blockchain} instance.
    *
@@ -390,7 +389,7 @@ export class Blockchain implements BlockchainInterface {
         }
 
         if (this._validateConsensus) {
-          await this.consensus.validateConsensus(block)
+          await this.consensus!.validateConsensus(block)
         }
 
         // set total difficulty in the current context scope
@@ -453,7 +452,7 @@ export class Blockchain implements BlockchainInterface {
         const ops = dbOps.concat(this._saveHeadOps())
         await this.dbManager.batch(ops)
 
-        await this.consensus.newBlock(block, commonAncestor, ancestorHeaders)
+        await this.consensus?.newBlock(block, commonAncestor, ancestorHeaders)
       } catch (e) {
         // restore head to the previouly sane state
         this._heads = oldHeads
@@ -499,7 +498,7 @@ export class Blockchain implements BlockchainInterface {
       throw new Error(`invalid timestamp ${header.errorStr()}`)
     }
 
-    if (!(header.common.consensusType() === 'pos')) await this.consensus.validateDifficulty(header)
+    if (!(header.common.consensusType() === 'pos')) await this.consensus?.validateDifficulty(header)
 
     if (this.common.consensusAlgorithm() === ConsensusAlgorithm.Clique) {
       const period = (this.common.consensusConfig() as CliqueConfig).period
@@ -1221,31 +1220,9 @@ export class Blockchain implements BlockchainInterface {
       timestamp,
     })
 
-    // If custom consensus algorithm is used, skip merge hardfork consensus checks
-    if (!Object.values(ConsensusAlgorithm).includes(this.consensus.algorithm as ConsensusAlgorithm))
-      return
-
-    switch (this.common.consensusAlgorithm()) {
-      case ConsensusAlgorithm.Casper:
-        if (!(this.consensus instanceof CasperConsensus)) {
-          this.consensus = new CasperConsensus()
-        }
-        break
-      case ConsensusAlgorithm.Clique:
-        if (!(this.consensus instanceof CliqueConsensus)) {
-          this.consensus = new CliqueConsensus()
-        }
-        break
-      case ConsensusAlgorithm.Ethash:
-        if (!(this.consensus instanceof EthashConsensus)) {
-          this.consensus = new EthashConsensus()
-        }
-        break
-      default:
-        throw new Error(`consensus algorithm ${this.common.consensusAlgorithm()} not supported`)
-    }
-    await this.consensus.setup({ blockchain: this })
-    await this.consensus.genesisInit(this.genesisBlock)
+    this._consensusCheck()
+    await this.consensus?.setup({ blockchain: this })
+    await this.consensus?.genesisInit(this.genesisBlock)
   }
 
   /**

packages/blockchain/src/consensus/casper.ts

@@ -21,6 +21,9 @@ export class CasperConsensus implements Consensus {
   public async validateConsensus(): Promise<void> {}
 
   public async validateDifficulty(header: BlockHeader): Promise<void> {
+    // TODO: This is not really part of consensus validation and it should be analyzed
+    // if it is possible to replace by a more generic hardfork check between block and
+    // blockchain along adding new blocks or headers
     if (header.difficulty !== BIGINT_0) {
       const msg = 'invalid difficulty.  PoS blocks must have difficulty 0'
       throw new Error(`${msg} ${header.errorStr()}`)

packages/blockchain/src/consensus/ethash.ts

@@ -1,26 +1,28 @@
 import { ConsensusAlgorithm } from '@ethereumjs/common'
-import { Ethash } from '@ethereumjs/ethash'
 
 import type { Blockchain } from '../index.js'
 import type { Consensus, ConsensusOptions } from '../types.js'
 import type { Block, BlockHeader } from '@ethereumjs/block'
 
+type MinimalEthashInterface = {
+  cacheDB?: any
+  verifyPOW(block: Block): Promise<boolean>
+}
+
 /**
  * This class encapsulates Ethash-related consensus functionality when used with the Blockchain class.
  */
 export class EthashConsensus implements Consensus {
   blockchain: Blockchain | undefined
   algorithm: ConsensusAlgorithm
-  _ethash: Ethash | undefined
+  _ethash: MinimalEthashInterface
 
-  constructor() {
+  constructor(ethash: MinimalEthashInterface) {
     this.algorithm = ConsensusAlgorithm.Ethash
+    this._ethash = ethash
   }
 
   async validateConsensus(block: Block): Promise<void> {
-    if (!this._ethash) {
-      throw new Error('blockchain not provided')
-    }
     const valid = await this._ethash.verifyPOW(block)
     if (!valid) {
       throw new Error('invalid POW')
@@ -44,7 +46,7 @@ export class EthashConsensus implements Consensus {
   public async genesisInit(): Promise<void> {}
   public async setup({ blockchain }: ConsensusOptions): Promise<void> {
     this.blockchain = blockchain
-    this._ethash = new Ethash(this.blockchain!.db as any)
+    this._ethash.cacheDB = this.blockchain!.db as any
   }
   public async newBlock(): Promise<void> {}
 }

packages/blockchain/src/constructors.ts

@@ -17,7 +17,7 @@ import type { Chain } from '@ethereumjs/common'
 export async function createBlockchain(opts: BlockchainOptions = {}) {
   const blockchain = new Blockchain(opts)
 
-  await blockchain.consensus.setup({ blockchain })
+  await blockchain.consensus?.setup({ blockchain })
 
   let stateRoot = opts.genesisBlock?.header.stateRoot ?? opts.genesisStateRoot
   if (stateRoot === undefined) {
@@ -56,7 +56,7 @@ export async function createBlockchain(opts: BlockchainOptions = {}) {
     DBSetBlockOrHeader(genesisBlock).map((op) => dbOps.push(op))
     DBSaveLookups(genesisHash, BIGINT_0).map((op) => dbOps.push(op))
     await blockchain.dbManager.batch(dbOps)
-    await blockchain.consensus.genesisInit(genesisBlock)
+    await blockchain.consensus?.genesisInit(genesisBlock)
   }
 
   // At this point, we can safely set the genesis:

packages/blockchain/src/types.ts

@@ -10,7 +10,7 @@ export type BlockchainEvents = {
 }
 
 export interface BlockchainInterface {
-  consensus: Consensus
+  consensus: Consensus | undefined
   /**
    * Adds a block to the blockchain.
    *
@@ -132,6 +132,10 @@ export interface GenesisOptions {
   genesisStateRoot?: Uint8Array
 }
 
+export type ConsensusDict = {
+  [consensusAlgorithm: ConsensusAlgorithm | string]: Consensus
+}
+
 /**
  * This are the options that the Blockchain constructor can receive.
  */
@@ -161,17 +165,6 @@ export interface BlockchainOptions extends GenesisOptions {
    */
   db?: DB<Uint8Array | string | number, Uint8Array | string | DBObject>
 
-  /**
-   * This flags indicates if a block should be validated along the consensus algorithm
-   * or protocol used by the chain, e.g. by verifying the PoW on the block.
-   *
-   * Supported consensus types and algorithms (taken from the `Common` instance):
-   * - 'pow' with 'ethash' algorithm (validates the proof-of-work)
-   * - 'poa' with 'clique' algorithm (verifies the block signatures)
-   * Default: `true`.
-   */
-  validateConsensus?: boolean
-
   /**
    * This flag indicates if protocol-given consistency checks on
    * block headers and included uncles and transactions should be performed,
@@ -181,9 +174,40 @@ export interface BlockchainOptions extends GenesisOptions {
   validateBlocks?: boolean
 
   /**
-   * Optional custom consensus that implements the {@link Consensus} class
+   * Validate the consensus with the respective consensus implementation passed
+   * to `consensusDict` (see respective option) `CapserConsensus` (which effectively
+   * does nothing) is available by default.
+   *
+   * For the build-in validation classes the following validations take place.
+   * - 'pow' with 'ethash' algorithm (validates the proof-of-work)
+   * - 'poa' with 'clique' algorithm (verifies the block signatures)
+   * Default: `false`.
+   */
+  validateConsensus?: boolean
+
+  /**
+   * Optional dictionary with consensus objects (adhering to the {@link Consensus} interface)
+   * if consensus validation is wished for certain consensus algorithms.
+   *
+   * Since consensus validation moved to the Ethereum consensus layer with Proof-of-Stake
+   * consensus is not validated by default. For `ConsensusAlgorithm.Ethash` and
+   * `ConsensusAlgorith.Clique` consensus validation can be activated by passing in the
+   * respective consensus validation objects `EthashConsensus` or `CliqueConsensus`.
+   *
+   * ```ts
+   * import { CliqueConsensus, createBlockchain } from '@ethereumjs/blockchain'
+   * import type { ConsensusDict } from '@ethereumjs/blockchain'
+   *
+   * const consensusDict: ConsensusDict = {}
+   * consensusDict[ConsensusAlgorithm.Clique] = new CliqueConsensus()
+   * const blockchain = await createBlockchain({ common, consensusDict })
+   * ```
+   *
+   * Additionally it is possible to provide a fully custom consensus implementation.
+   * Note that this needs a custom `Common` object passed to the blockchain where
+   * the `ConsensusAlgorithm` string matches the string used here.
    */
-  consensus?: Consensus
+  consensusDict?: ConsensusDict
 }
 
 /**