Save blocks to db

packages/chopsticks/src/blockchain/block-builder.ts

@@ -135,6 +135,8 @@ const initNewBlock = async (head: Block, header: Header, inherents: HexString[],
     }
   }
 
+  await head.chain.saveBlockToDB(newBlock)
+
   return {
     block: newBlock,
     layers: layers,
@@ -287,6 +289,8 @@ export const buildBlock = async (
     storageDiff,
   })
 
+  await head.chain.saveBlockToDB(finalBlock)
+
   logger.info(
     {
       number: newBlock.number,

packages/chopsticks/src/blockchain/index.ts

@@ -8,6 +8,7 @@ import type { TransactionValidity } from '@polkadot/types/interfaces/txqueue'
 
 import { Api } from '../api'
 import { Block } from './block'
+import { Block as BlockEntity } from '../db/entities'
 import { BuildBlockMode, BuildBlockParams, DownwardMessage, HorizontalMessage, TxPool } from './txpool'
 import { HeadState } from './head-state'
 import { InherentProvider } from './inherent'
@@ -108,6 +109,45 @@ export class Blockchain {
     return this.#txpool
   }
 
+  async saveBlockToDB(block: Block) {
+    if (this.db) {
+      const { hash, number, header, extrinsics } = block
+      // delete old ones with the same block number if any, keep the latest one
+      await this.db.getRepository(BlockEntity).delete({ number })
+      await this.db.getRepository(BlockEntity).upsert(
+        {
+          hash,
+          number,
+          header,
+          extrinsics: await extrinsics,
+          parentHash: (await block.parentBlock)?.hash,
+          storageDiff: await block.storageDiff(),
+        },
+        ['hash'],
+      )
+    }
+  }
+
+  /**
+   * Try to load block from db and register it
+   * If pass in number, get block by number, else get block by hash
+   */
+  async loadBlockFromDB(key: number | HexString): Promise<Block | undefined> {
+    if (this.db) {
+      const blockData = await this.db
+        .getRepository(BlockEntity)
+        .findOne({ where: { [typeof key === 'number' ? 'number' : 'hash']: key } })
+      if (blockData) {
+        const { hash, number, header, extrinsics, parentHash, storageDiff } = blockData
+        const parentBlock = parentHash ? this.#blocksByHash[parentHash] : undefined
+        const block = new Block(this, number, hash, parentBlock, { header, extrinsics, storageDiff })
+        this.#registerBlock(block)
+        return block
+      }
+    }
+    return undefined
+  }
+
   async getBlockAt(number?: number): Promise<Block | undefined> {
     if (number === undefined) {
       return this.head
@@ -116,6 +156,10 @@ export class Blockchain {
       return undefined
     }
     if (!this.#blocksByNumber.has(number)) {
+      const blockFromDB = await this.loadBlockFromDB(number)
+      if (blockFromDB) {
+        return blockFromDB
+      }
       const hash = await this.api.getBlockHash(number)
       const block = new Block(this, number, hash)
       this.#registerBlock(block)
@@ -135,9 +179,12 @@ export class Blockchain {
       } else {
         const loadingBlock = (async () => {
           try {
-            const header = await this.api.getHeader(hash)
-            const block = new Block(this, Number(header.number), hash)
-            this.#registerBlock(block)
+            const blockFromDB = await this.loadBlockFromDB(hash)
+            if (!blockFromDB) {
+              const header = await this.api.getHeader(hash)
+              const block = new Block(this, Number(header.number), hash)
+              this.#registerBlock(block)
+            }
           } catch (e) {
             logger.debug(`getBlock(${hash}) failed: ${e}`)
           }
@@ -162,6 +209,10 @@ export class Blockchain {
       this.#blocksByNumber.delete(block.number)
     }
     delete this.#blocksByHash[block.hash]
+    // delete from db
+    if (this.db) {
+      this.db.getRepository(BlockEntity).delete({ hash: block.hash })
+    }
   }
 
   async setHead(block: Block): Promise<void> {

packages/chopsticks/src/db/entities.ts

@@ -1,4 +1,6 @@
 import { Column, Entity, PrimaryColumn } from 'typeorm'
+import type { Header } from '@polkadot/types/interfaces'
+import type { HexString } from '@polkadot/util/types'
 
 @Entity()
 export class KeyValuePair {
@@ -11,3 +13,24 @@ export class KeyValuePair {
   @Column({ nullable: true })
   value!: string
 }
+
+@Entity()
+export class Block {
+  @PrimaryColumn({ type: 'varchar' })
+  hash!: HexString
+
+  @Column()
+  number!: number
+
+  @Column({ type: 'simple-json', nullable: true })
+  header!: Header
+
+  @Column({ type: 'varchar', nullable: true })
+  parentHash!: HexString
+
+  @Column('simple-array', { nullable: true })
+  extrinsics!: HexString[]
+
+  @Column({ type: 'simple-json', nullable: true })
+  storageDiff!: Record<HexString, HexString | null>
+}

packages/e2e/src/blocks-save.test.ts

@@ -0,0 +1,27 @@
+import { afterAll, describe, expect, it } from 'vitest'
+
+import networks from './networks'
+
+describe('block-save', async () => {
+  const acala = await networks.acala({ db: 'db.sqlite' })
+  const { chain, dev } = acala
+
+  afterAll(async () => {
+    await acala.teardown()
+  })
+
+  it('saved blocks data', async () => {
+    await dev.newBlock({ count: 2 })
+
+    const numberOfBlocks = await chain.db?.getRepository('Block').count()
+    expect(numberOfBlocks).toEqual(3)
+
+    const block = await chain.getBlockAt(chain.head.number)
+    const blockData = await chain.db?.getRepository('Block').findOne({ where: { number: chain.head.number } })
+    expect(blockData?.hash).toEqual(block?.hash)
+    expect(JSON.stringify(blockData?.header)).toEqual(JSON.stringify(block?.header))
+    expect(blockData?.parentHash).toEqual((await block?.parentBlock)?.hash)
+    expect(JSON.stringify(blockData?.extrinsics)).toEqual(JSON.stringify(await block?.extrinsics))
+    expect(JSON.stringify(blockData?.storageDiff)).toEqual(JSON.stringify(await block?.storageDiff()))
+  })
+})