able to clear storage map

configs/kusama.yml

@@ -1,6 +1,6 @@
 endpoint: wss://kusama-rpc.dwellir.com
 mock-signature-host: true
-block: 15463000
+block: 15616800
 db: ./db.sqlite
 
 import-storage:
@@ -22,3 +22,39 @@ import-storage:
     Members: [5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY]
   TechnicalMembership:
     Members: [5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY]
+  FellowshipCollective:
+    $removePrefix:
+      - IdToIndex
+      - IndexToId
+      - MemberCount
+      - Members
+    IdToIndex:
+      - [[0, 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY], 0]
+      - [[1, 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY], 0]
+      - [[2, 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY], 0]
+      - [[3, 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY], 0]
+      - [[4, 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY], 0]
+      - [[5, 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY], 0]
+      - [[6, 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY], 0]
+      - [[7, 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY], 0]
+    IndexToId:
+      - [[0, 0], 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY]
+      - [[1, 0], 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY]
+      - [[2, 0], 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY]
+      - [[3, 0], 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY]
+      - [[4, 0], 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY]
+      - [[5, 0], 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY]
+      - [[6, 0], 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY]
+      - [[7, 0], 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY]
+    MemberCount:
+      - [[0], 1]
+      - [[1], 1]
+      - [[2], 1]
+      - [[3], 1]
+      - [[4], 1]
+      - [[5], 1]
+      - [[6], 1]
+      - [[7], 1]
+    Members:
+      - [[5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY], rank: 7]
+    Voting: []

e2e/import-storage/index.test.ts

@@ -1,7 +1,7 @@
 import { describe, expect, it } from 'vitest'
 import path from 'path'
 
-import { chain, setupApi } from '../helper'
+import { api, chain, setupApi } from '../helper'
 import { importStorage, overrideWasm } from '../../src/utils/import-storage'
 
 setupApi({
@@ -19,7 +19,7 @@ describe('import-storage', () => {
 
     await importStorage(chain, path.join(__dirname, './storage.ok.yml'))
 
-    expect(await block?.get(sudoKey)).toBeUndefined
+    expect(await block?.get(sudoKey)).toBeUndefined()
   })
 
   it('handle errors', async () => {
@@ -43,4 +43,34 @@ describe('import-storage', () => {
     // can produce blocks
     await expect(chain.newBlock()).resolves.toContain({ number: blockNumber + 1 })
   })
+
+  it('able to reset storage map', async () => {
+    await importStorage(chain, {
+      Tokens: {
+        $removePrefix: ['Accounts'],
+        Accounts: [
+          [
+            ['5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY', { Token: 'ACA' }],
+            {
+              free: 1000000000000000,
+            },
+          ],
+        ],
+      },
+    })
+
+    const entries = await api.query.tokens.accounts.entries()
+    expect(entries).toMatchInlineSnapshot(`
+      [
+        [
+          "0x99971b5749ac43e0235e41b0d37869188ee7418a6531173d60d1f6a82d8f4d51de1e86a9a8c739864cf3cc5ec2bea59fd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d01a12dfa1fa4ab9a0000",
+          {
+            "free": 1000000000000000,
+            "frozen": 0,
+            "reserved": 0,
+          },
+        ],
+      ]
+    `)
+  })
 })

src/blockchain/storage-layer.ts

@@ -7,7 +7,8 @@ import { defaultLogger } from '../logger'
 const logger = defaultLogger.child({ name: 'layer' })
 
 export const enum StorageValueKind {
-  Deleted,
+  Deleted = 'Deleted',
+  DeletedPrefix = 'DeletedPrefix',
 }
 
 export type StorageValue = string | StorageValueKind | undefined
@@ -58,6 +59,7 @@ export class RemoteStorageLayer implements StorageLayerProvider {
 export class StorageLayer implements StorageLayerProvider {
   readonly #store: Record<string, StorageValue | Promise<StorageValue>> = {}
   readonly #keys: string[] = []
+  readonly #deletedPrefix: string[] = []
   #parent?: StorageLayerProvider
 
   constructor(parent?: StorageLayerProvider) {
@@ -86,6 +88,10 @@ export class StorageLayer implements StorageLayerProvider {
       return this.#store[key]
     }
 
+    if (this.#deletedPrefix.some((prefix) => key.startsWith(prefix))) {
+      return StorageValueKind.Deleted
+    }
+
     if (this.#parent) {
       const val = this.#parent.get(key, false)
       if (cache) {
@@ -103,6 +109,15 @@ export class StorageLayer implements StorageLayerProvider {
         this.#store[key] = value
         this.#removeKey(key)
         break
+      case StorageValueKind.DeletedPrefix:
+        this.#deletedPrefix.push(key)
+        for (const k of this.#keys) {
+          if (k.startsWith(key)) {
+            this.#store[k] = StorageValueKind.Deleted
+            this.#removeKey(k)
+          }
+        }
+        break
       case undefined:
         delete this.#store[key]
         this.#removeKey(key)
@@ -126,6 +141,10 @@ export class StorageLayer implements StorageLayerProvider {
   async foldInto(into: StorageLayer): Promise<StorageLayerProvider | undefined> {
     const newParent = await this.#parent?.foldInto(into)
 
+    for (const deletedPrefix of this.#deletedPrefix) {
+      into.set(deletedPrefix, StorageValueKind.DeletedPrefix)
+    }
+
     for (const key of this.#keys) {
       const value = await this.#store[key]
       into.set(key, value)
@@ -141,12 +160,18 @@ export class StorageLayer implements StorageLayerProvider {
   }
 
   async getKeysPaged(prefix: string, pageSize: number, startKey: string): Promise<string[]> {
-    await this.fold()
-    // TODO: maintain a list of fetched ranges to avoid fetching the same range multiple times
-    const remote = (await this.#parent?.getKeysPaged(prefix, pageSize, startKey)) ?? []
-    for (const key of remote) {
-      this.#addKey(key)
+    if (!this.#deletedPrefix.some((prefix) => startKey.startsWith(prefix))) {
+      await this.fold()
+      // TODO: maintain a list of fetched ranges to avoid fetching the same range multiple times
+      const remote = (await this.#parent?.getKeysPaged(prefix, pageSize, startKey)) ?? []
+      for (const key of remote) {
+        if (this.#deletedPrefix.some((prefix) => key.startsWith(prefix))) {
+          continue
+        }
+        this.#addKey(key)
+      }
     }
+
     let idx = _.sortedIndex(this.#keys, startKey)
     if (this.#keys[idx] === startKey) {
       ++idx

src/utils/set-storage.ts

@@ -1,8 +1,10 @@
 import { DecoratedMeta } from '@polkadot/types/metadata/decorate/types'
 import { StorageKey } from '@polkadot/types'
 import { stringCamelCase } from '@polkadot/util/string'
+import { u8aToHex } from '@polkadot/util'
 
 import { Blockchain } from '../blockchain'
+import { StorageValueKind } from '../blockchain/storage-layer'
 
 type RawStorageValues = [string, string | null][]
 type StorageConfig = Record<string, Record<string, any>>
@@ -19,6 +21,17 @@ function objectToStorageItems(meta: DecoratedMeta, storage: StorageConfig): RawS
     for (const storageName in section) {
       const storage = section[storageName]
 
+      if (storageName === '$removePrefix') {
+        for (const mapName of storage) {
+          const storageEntry = pallet[stringCamelCase(mapName)]
+          if (!storageEntry) throw Error(`Cannot find storage ${mapName} in pallet ${sectionName}`)
+
+          const prefix = storageEntry.keyPrefix()
+          storageItems.push([u8aToHex(prefix), StorageValueKind.DeletedPrefix])
+        }
+        continue
+      }
+
       const storageEntry = pallet[stringCamelCase(storageName)]
       if (!storageEntry) throw Error(`Cannot find storage ${storageName} in pallet ${sectionName}`)