diff --git a/.gitignore b/.gitignore index 664de805..eb6afd87 100644 --- a/.gitignore +++ b/.gitignore @@ -122,6 +122,7 @@ dist *.sqlite *.sqlite-journal *.wasm +!packages/e2e/blobs/*.wasm *.db *.sqlite-shm *.sqlite-wal diff --git a/configs/polkadot.yml b/configs/polkadot.yml index 19b90551..0b329c0e 100644 --- a/configs/polkadot.yml +++ b/configs/polkadot.yml @@ -1,5 +1,5 @@ endpoint: - - wss://rpc.polkadot.io + - wss://rpc.ibp.network/polkadot - wss://polkadot-rpc.dwellir.com mock-signature-host: true block: ${env.POLKADOT_BLOCK_NUMBER} diff --git a/packages/core/src/api.ts b/packages/core/src/api.ts index ab6c402b..73713560 100644 --- a/packages/core/src/api.ts +++ b/packages/core/src/api.ts @@ -2,6 +2,7 @@ import { ExtDef } from '@polkadot/types/extrinsic/signedExtensions/types' import { HexString } from '@polkadot/util/types' import { ProviderInterface, ProviderInterfaceCallback } from '@polkadot/rpc-provider/types' import { prefixedChildKey, splitChildKey, stripChildPrefix } from './utils/index.js' +import _ from 'lodash' export type ChainProperties = { ss58Format?: number @@ -150,6 +151,26 @@ export class Api { } } + async getStorageBatch(prefix: HexString, keys: HexString[], hash?: HexString) { + const [child] = splitChildKey(prefix) + if (child) { + // child storage key, use childstate_getStorageEntries + // strip child prefix from keys + const params: any[] = [child, keys.map((key) => stripChildPrefix(key))] + if (hash) params.push(hash) + return this.#provider + .send('childstate_getStorageEntries', params, !!hash) + .then((values) => _.zip(keys, values) as [HexString, HexString | null][]) + } else { + // main storage key, use state_getStorageAt + const params: any[] = [keys] + if (hash) params.push(hash) + return this.#provider + .send('state_queryStorageAt', params, !!hash) + .then((result) => (result[0]?.['changes'] as [HexString, HexString | null][]) || []) + } + } + async subscribeRemoteNewHeads(cb: ProviderInterfaceCallback) { if (!this.#provider.hasSubscriptions) { throw new Error('subscribeRemoteNewHeads only works with subscriptions') diff --git a/packages/core/src/blockchain/block.ts b/packages/core/src/blockchain/block.ts index d45915b0..606677dd 100644 --- a/packages/core/src/blockchain/block.ts +++ b/packages/core/src/blockchain/block.ts @@ -176,14 +176,11 @@ export class Block { * Get paged storage keys. */ async getKeysPaged(options: { prefix?: string; startKey?: string; pageSize: number }): Promise { - const layer = new StorageLayer(this.storage) - await layer.fold() - const prefix = options.prefix ?? '0x' const startKey = options.startKey ?? '0x' const pageSize = options.pageSize - return layer.getKeysPaged(prefix, pageSize, startKey) + return this.storage.getKeysPaged(prefix, pageSize, startKey) } /** diff --git a/packages/core/src/blockchain/get-keys-paged.test.ts b/packages/core/src/blockchain/get-keys-paged.test.ts new file mode 100644 index 00000000..0e29913e --- /dev/null +++ b/packages/core/src/blockchain/get-keys-paged.test.ts @@ -0,0 +1,257 @@ +import { Api } from '../api.js' +import { RemoteStorageLayer, StorageLayer, StorageValueKind } from './storage-layer.js' +import { describe, expect, it, vi } from 'vitest' +import _ from 'lodash' + +describe('getKeysPaged', () => { + const hash = '0x1111111111111111111111111111111111111111111111111111111111111111' + + const allKeys = [ + '0x0000000000000000000000000000000000000000000000000000000000000000_00', + '0x0000000000000000000000000000000000000000000000000000000000000000_03', + '0x0000000000000000000000000000000000000000000000000000000000000000_04', + '0x1111111111111111111111111111111111111111111111111111111111111111_01', + '0x1111111111111111111111111111111111111111111111111111111111111111_02', + '0x1111111111111111111111111111111111111111111111111111111111111111_03', + '0x1111111111111111111111111111111111111111111111111111111111111111_06', + '0x1111111111111111111111111111111111111111111111111111111111111111_07', + '0x2222222222222222222222222222222222222222222222222222222222222222_21', + '0x2222222222222222222222222222222222222222222222222222222222222222_23', + '0x2222222222222222222222222222222222222222222222222222222222222222_26', + ] + + Api.prototype['getKeysPaged'] = vi.fn(async (prefix, pageSize, startKey) => { + const withPrefix = allKeys.filter((k) => k.startsWith(prefix) && k > startKey) + const result = withPrefix.slice(0, pageSize) + return result + }) + const mockApi = new Api(undefined!) + + const remoteLayer = new RemoteStorageLayer(mockApi, hash, undefined) + const storageLayer = new StorageLayer(remoteLayer) + + it('mocked api works', async () => { + expect( + await mockApi.getKeysPaged( + '0x1111111111111111111111111111111111111111111111111111111111111111', + 1, + '0x1111111111111111111111111111111111111111111111111111111111111111', + hash, + ), + ).toEqual(['0x1111111111111111111111111111111111111111111111111111111111111111_01']) + + expect( + await mockApi.getKeysPaged( + '0x2222222222222222222222222222222222222222222222222222222222222222', + 4, + '0x2222222222222222222222222222222222222222222222222222222222222222', + hash, + ), + ).toEqual([ + '0x2222222222222222222222222222222222222222222222222222222222222222_21', + '0x2222222222222222222222222222222222222222222222222222222222222222_23', + '0x2222222222222222222222222222222222222222222222222222222222222222_26', + ]) + + expect( + await mockApi.getKeysPaged( + '0x1111111111111111111111111111111111111111111111111111111111111111', + 10, + '0x1111111111111111111111111111111111111111111111111111111111111111_02', + hash, + ), + ).toEqual([ + '0x1111111111111111111111111111111111111111111111111111111111111111_03', + '0x1111111111111111111111111111111111111111111111111111111111111111_06', + '0x1111111111111111111111111111111111111111111111111111111111111111_07', + ]) + + expect( + await mockApi.getKeysPaged( + '0x1111111111111111111111111111111111111111111111111111111111111111', + 2, + '0x1111111111111111111111111111111111111111111111111111111111111111_04', + hash, + ), + ).toEqual([ + '0x1111111111111111111111111111111111111111111111111111111111111111_06', + '0x1111111111111111111111111111111111111111111111111111111111111111_07', + ]) + + expect( + await mockApi.getKeysPaged( + '0x1111111111111111111111111111111111111111111111111111111111111111', + 2, + '0x1111111111111111111111111111111111111111111111111111111111111111_07', + hash, + ), + ).toEqual([]) + }) + + it('remote layer works', async () => { + expect( + await remoteLayer.getKeysPaged( + '0x1111111111111111111111111111111111111111111111111111111111111111', + 3, + '0x1111111111111111111111111111111111111111111111111111111111111111', + ), + ).toEqual([ + '0x1111111111111111111111111111111111111111111111111111111111111111_01', + '0x1111111111111111111111111111111111111111111111111111111111111111_02', + '0x1111111111111111111111111111111111111111111111111111111111111111_03', + ]) + + expect( + await remoteLayer.getKeysPaged( + '0x1111111111111111111111111111111111111111111111111111111111111111', + 10, + '0x1111111111111111111111111111111111111111111111111111111111111111_03', + ), + ).toEqual([ + '0x1111111111111111111111111111111111111111111111111111111111111111_06', + '0x1111111111111111111111111111111111111111111111111111111111111111_07', + ]) + }) + + it('storage layer works', async () => { + expect( + await mockApi.getKeysPaged( + '0x1111111111111111111111111111111111111111111111111111111111111111', + 1, + '0x1111111111111111111111111111111111111111111111111111111111111111', + hash, + ), + ).toEqual(['0x1111111111111111111111111111111111111111111111111111111111111111_01']) + + expect( + await mockApi.getKeysPaged( + '0x1111111111111111111111111111111111111111111111111111111111111111', + 10, + '0x1111111111111111111111111111111111111111111111111111111111111111_02', + hash, + ), + ).toEqual([ + '0x1111111111111111111111111111111111111111111111111111111111111111_03', + '0x1111111111111111111111111111111111111111111111111111111111111111_06', + '0x1111111111111111111111111111111111111111111111111111111111111111_07', + ]) + }) + + it('updated values', async () => { + const layer2 = new StorageLayer(storageLayer) + layer2.setAll([ + ['0x1111111111111111111111111111111111111111111111111111111111111111_00', '0x00'], + ['0x1111111111111111111111111111111111111111111111111111111111111111_04', '0x04'], + ]) + expect( + await layer2.getKeysPaged( + '0x1111111111111111111111111111111111111111111111111111111111111111', + 10, + '0x1111111111111111111111111111111111111111111111111111111111111111_03', + ), + ).toEqual([ + '0x1111111111111111111111111111111111111111111111111111111111111111_04', + '0x1111111111111111111111111111111111111111111111111111111111111111_06', + '0x1111111111111111111111111111111111111111111111111111111111111111_07', + ]) + + expect( + await layer2.getKeysPaged( + '0x1111111111111111111111111111111111111111111111111111111111111111', + 10, + '0x1111111111111111111111111111111111111111111111111111111111111111', + ), + ).toEqual([ + '0x1111111111111111111111111111111111111111111111111111111111111111_00', + '0x1111111111111111111111111111111111111111111111111111111111111111_01', + '0x1111111111111111111111111111111111111111111111111111111111111111_02', + '0x1111111111111111111111111111111111111111111111111111111111111111_03', + '0x1111111111111111111111111111111111111111111111111111111111111111_04', + '0x1111111111111111111111111111111111111111111111111111111111111111_06', + '0x1111111111111111111111111111111111111111111111111111111111111111_07', + ]) + + expect( + await layer2.getKeysPaged( + '0x1111111111111111111111111111111111111111111111111111111111111111', + 10, + '0x1111111111111111111111111111111111111111111111111111111111111111_04', + ), + ).toEqual([ + '0x1111111111111111111111111111111111111111111111111111111111111111_06', + '0x1111111111111111111111111111111111111111111111111111111111111111_07', + ]) + + expect( + await layer2.getKeysPaged( + '0x1111111111111111111111111111111111111111111111111111111111111111', + 10, + '0x1111111111111111111111111111111111111111111111111111111111111111_07', + ), + ).toEqual([]) + + const layer3 = new StorageLayer(layer2) + layer3.setAll([ + ['0x1111111111111111111111111111111111111111111111111111111111111111_03', '0x03'], + ['0x1111111111111111111111111111111111111111111111111111111111111111_04', null], + ['0x1111111111111111111111111111111111111111111111111111111111111111_06', '0x06'], + ]) + + expect( + await layer3.getKeysPaged( + '0x1111111111111111111111111111111111111111111111111111111111111111', + 10, + '0x1111111111111111111111111111111111111111111111111111111111111111_02', + ), + ).toEqual([ + '0x1111111111111111111111111111111111111111111111111111111111111111_03', + '0x1111111111111111111111111111111111111111111111111111111111111111_06', + '0x1111111111111111111111111111111111111111111111111111111111111111_07', + ]) + + const layer4 = new StorageLayer(layer3) + layer4.setAll([ + ['0x1111111111111111111111111111111111111111111111111111111111111111_03', null], + ['0x1111111111111111111111111111111111111111111111111111111111111111_04', '0x04'], + ['0x1111111111111111111111111111111111111111111111111111111111111111_06', null], + ['0x1111111111111111111111111111111111111111111111111111111111111111_08', '0x08'], + ]) + + expect( + await layer4.getKeysPaged( + '0x1111111111111111111111111111111111111111111111111111111111111111', + 10, + '0x1111111111111111111111111111111111111111111111111111111111111111_02', + ), + ).toEqual([ + '0x1111111111111111111111111111111111111111111111111111111111111111_04', + '0x1111111111111111111111111111111111111111111111111111111111111111_07', + '0x1111111111111111111111111111111111111111111111111111111111111111_08', + ]) + + const layer5 = new StorageLayer(layer4) + layer5.setAll([ + ['0x1111111111111111111111111111111111111111111111111111111111111111', StorageValueKind.DeletedPrefix], + ['0x1111111111111111111111111111111111111111111111111111111111111111_09', '0x09'], + ]) + expect( + await layer5.getKeysPaged( + '0x1111111111111111111111111111111111111111111111111111111111111111', + 10, + '0x1111111111111111111111111111111111111111111111111111111111111111', + ), + ).toEqual(['0x1111111111111111111111111111111111111111111111111111111111111111_09']) + + const layer6 = new StorageLayer(layer5) + layer6.setAll([ + ['0x1111111111111111111111111111111111111111111111111111111111111111', StorageValueKind.DeletedPrefix], + ]) + expect( + await layer6.getKeysPaged( + '0x1111111111111111111111111111111111111111111111111111111111111111', + 10, + '0x1111111111111111111111111111111111111111111111111111111111111111', + ), + ).toEqual([]) + }) +}) diff --git a/packages/core/src/blockchain/storage-layer.ts b/packages/core/src/blockchain/storage-layer.ts index 3d1848ed..0c32d521 100644 --- a/packages/core/src/blockchain/storage-layer.ts +++ b/packages/core/src/blockchain/storage-layer.ts @@ -2,9 +2,10 @@ import { HexString } from '@polkadot/util/types' import _ from 'lodash' import { Api } from '../api.js' +import { CHILD_PREFIX_LENGTH, PREFIX_LENGTH, isPrefixedChildKey } from '../utils/index.js' import { Database } from '../database.js' import { defaultLogger } from '../logger.js' -import KeyCache, { PREFIX_LENGTH } from '../utils/key-cache.js' +import KeyCache from '../utils/key-cache.js' const logger = defaultLogger.child({ name: 'layer' }) @@ -40,7 +41,8 @@ export class RemoteStorageLayer implements StorageLayerProvider { readonly #api: Api readonly #at: string readonly #db: Database | undefined - readonly #keyCache = new KeyCache() + readonly #keyCache = new KeyCache(PREFIX_LENGTH) + readonly #defaultChildKeyCache = new KeyCache(CHILD_PREFIX_LENGTH) constructor(api: Api, at: string, db: Database | undefined) { this.#api = api @@ -69,15 +71,21 @@ export class RemoteStorageLayer implements StorageLayerProvider { async getKeysPaged(prefix: string, pageSize: number, startKey: string): Promise { if (pageSize > BATCH_SIZE) throw new Error(`pageSize must be less or equal to ${BATCH_SIZE}`) logger.trace({ at: this.#at, prefix, pageSize, startKey }, 'RemoteStorageLayer getKeysPaged') + + const isChild = isPrefixedChildKey(prefix as HexString) + const minPrefixLen = isChild ? CHILD_PREFIX_LENGTH : PREFIX_LENGTH + // can't handle keyCache without prefix - if (prefix.length < PREFIX_LENGTH || startKey.length < PREFIX_LENGTH) { + if (prefix.length < minPrefixLen || startKey.length < minPrefixLen) { return this.#api.getKeysPaged(prefix, pageSize, startKey, this.#at) } let batchComplete = false const keysPaged: string[] = [] while (keysPaged.length < pageSize) { - const nextKey = await this.#keyCache.next(startKey as any) + const nextKey = isChild + ? await this.#defaultChildKeyCache.next(startKey as HexString) + : await this.#keyCache.next(startKey as HexString) if (nextKey) { keysPaged.push(nextKey) startKey = nextKey @@ -94,13 +102,26 @@ export class RemoteStorageLayer implements StorageLayerProvider { // feed the key cache if (batch.length > 0) { - this.#keyCache.feed([startKey, ...(batch as any)]) + if (isChild) { + this.#defaultChildKeyCache.feed([startKey, ...batch] as HexString[]) + } else { + this.#keyCache.feed([startKey, ...batch] as HexString[]) + } } if (batch.length === 0) { // no more keys were found break } + + if (this.#db) { + // batch fetch storage values and save to db, they may be used later + this.#api.getStorageBatch(prefix as HexString, batch as HexString[], this.#at as HexString).then((storage) => { + for (const [key, value] of storage) { + this.#db!.saveStorage(this.#at as HexString, key as HexString, value) + } + }) + } } return keysPaged } @@ -209,32 +230,114 @@ export class StorageLayer implements StorageLayerProvider { } async getKeysPaged(prefix: string, pageSize: number, startKey: string): Promise { - if (!this.#deletedPrefix.some((dp) => startKey.startsWith(dp))) { - const remote = (await this.#parent?.getKeysPaged(prefix, pageSize, startKey)) ?? [] - for (const key of remote) { - if (this.#store.get(key) === StorageValueKind.Deleted) { - continue + let parentFetchComplete = false + const parentFetchKeys = async (batchSize: number, startKey: string) => { + if (!this.#deletedPrefix.some((dp) => startKey.startsWith(dp))) { + const newKeys: string[] = [] + while (newKeys.length < batchSize) { + const remote = (await this.#parent?.getKeysPaged(prefix, batchSize, startKey)) ?? [] + if (remote.length) { + startKey = remote[remote.length - 1] + } + for (const key of remote) { + if (this.#store.get(key) === StorageValueKind.Deleted) { + continue + } + if (this.#deletedPrefix.some((dp) => key.startsWith(dp))) { + continue + } + newKeys.push(key) + } + if (remote.length < batchSize) { + parentFetchComplete = true + break + } + } + return newKeys + } else { + parentFetchComplete = true + return [] + } + } + + const res: string[] = [] + + const foundNextKey = (key: string) => { + // make sure keys are unique + if (!res.includes(key)) { + res.push(key) + } + } + + const iterLocalKeys = (prefix: string, startKey: string, includeFirst: boolean, endKey?: string) => { + let idx = this.#keys.findIndex((x) => x.startsWith(startKey)) + if (this.#keys[idx] !== startKey) { + idx = this.#keys.findIndex((x) => x.startsWith(prefix) && x > startKey) + const key = this.#keys[idx] + if (key) { + if (endKey && key >= endKey) { + return startKey + } + foundNextKey(key) + ++idx } - if (this.#deletedPrefix.some((dp) => key.startsWith(dp))) { - continue + } + if (idx !== -1) { + if (includeFirst) { + const key = this.#keys[idx] + if (key) { + foundNextKey(key) + } } - this.#addKey(key) + while (res.length < pageSize) { + ++idx + const key: string = this.#keys[idx] + if (!key || !key.startsWith(prefix)) { + break + } + if (endKey && key >= endKey) { + break + } + foundNextKey(key) + } + return _.last(res) ?? startKey } + return startKey } - let idx = _.sortedIndex(this.#keys, startKey) - if (this.#keys[idx] === startKey) { - ++idx + if (prefix !== startKey && this.#keys.find((x) => x === startKey)) { + startKey = iterLocalKeys(prefix, startKey, false) } - const res: string[] = [] - while (res.length < pageSize) { - const key: string = this.#keys[idx] - if (!key || !key.startsWith(prefix)) { - break + + // then iterate the parent keys + let keys = await parentFetchKeys(pageSize - res.length, startKey) + if (keys.length) { + let idx = 0 + while (res.length < pageSize) { + const key = keys[idx] + if (!key || !key.startsWith(prefix)) { + if (parentFetchComplete) { + break + } else { + keys = await parentFetchKeys(pageSize - res.length, _.last(keys)!) + continue + } + } + + const keyPosition = _.sortedIndex(this.#keys, key) + const localParentKey = this.#keys[keyPosition - 1] + if (localParentKey < key) { + startKey = iterLocalKeys(prefix, startKey, false, key) + } + + foundNextKey(key) + ++idx } - res.push(key) - ++idx } + if (res.length < pageSize) { + iterLocalKeys(prefix, startKey, prefix === startKey) + } + return res } diff --git a/packages/core/src/utils/index.ts b/packages/core/src/utils/index.ts index c70d1936..441846d9 100644 --- a/packages/core/src/utils/index.ts +++ b/packages/core/src/utils/index.ts @@ -84,7 +84,10 @@ export function defer() { const DEFAULT_CHILD_STORAGE = '0x3a6368696c645f73746f726167653a64656661756c743a' // length of the child storage key -const CHILD_LENGTH = DEFAULT_CHILD_STORAGE.length + 64 +export const CHILD_PREFIX_LENGTH = DEFAULT_CHILD_STORAGE.length + 64 + +// 0x + 32 module + 32 method +export const PREFIX_LENGTH = 66 // returns a key that is prefixed with the child storage key export const prefixedChildKey = (prefix: HexString, key: HexString) => prefix + hexStripPrefix(key) @@ -95,9 +98,9 @@ export const isPrefixedChildKey = (key: HexString) => key.startsWith(DEFAULT_CHI // returns a key that is split into the child storage key and the rest export const splitChildKey = (key: HexString) => { if (!key.startsWith(DEFAULT_CHILD_STORAGE)) return [] - if (key.length < CHILD_LENGTH) return [] - const child = key.slice(0, CHILD_LENGTH) - const rest = key.slice(CHILD_LENGTH) + if (key.length < CHILD_PREFIX_LENGTH) return [] + const child = key.slice(0, CHILD_PREFIX_LENGTH) + const rest = key.slice(CHILD_PREFIX_LENGTH) return [child, hexAddPrefix(rest)] as [HexString, HexString] } diff --git a/packages/core/src/utils/key-cache.test.ts b/packages/core/src/utils/key-cache.test.ts index 8a9eff7d..6784bd32 100644 --- a/packages/core/src/utils/key-cache.test.ts +++ b/packages/core/src/utils/key-cache.test.ts @@ -21,7 +21,7 @@ const KEY_16 = '0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371d describe('key cache', () => { it('should be able to fee keys', async () => { - const keyCache = new KeyCache() + const keyCache = new KeyCache(66) keyCache.feed([KEY_0, KEY_1, KEY_2, KEY_3, KEY_4]) expect(await keyCache.next(KEY_1)).toBe(KEY_2) expect(await keyCache.next(KEY_3)).toBe(KEY_4) @@ -33,7 +33,7 @@ describe('key cache', () => { }) it("should be able to feed keys that don't intersect", async () => { - const keyCache = new KeyCache() + const keyCache = new KeyCache(66) keyCache.feed([KEY_3, KEY_4, KEY_5, KEY_6]) keyCache.feed([KEY_7, KEY_8, KEY_9, KEY_10]) expect(keyCache.ranges.length).toBe(2) diff --git a/packages/core/src/utils/key-cache.ts b/packages/core/src/utils/key-cache.ts index 7f465491..c22e0d91 100644 --- a/packages/core/src/utils/key-cache.ts +++ b/packages/core/src/utils/key-cache.ts @@ -1,24 +1,23 @@ import { HexString } from '@polkadot/util/types' import _ from 'lodash' -// 0x + 32 module + 32 method -export const PREFIX_LENGTH = 66 - export default class KeyCache { + constructor(readonly prefixLength: number) {} + readonly ranges: Array<{ prefix: string; keys: string[] }> = [] feed(keys: HexString[]) { - const _keys = keys.filter((key) => key.length >= PREFIX_LENGTH) + const _keys = keys.filter((key) => key.length >= this.prefixLength) if (_keys.length === 0) return - const startKey = _keys[0].slice(PREFIX_LENGTH) - const endKey = _keys[_keys.length - 1].slice(PREFIX_LENGTH) - const grouped = _.groupBy(_keys, (key) => key.slice(0, PREFIX_LENGTH)) + const startKey = _keys[0].slice(this.prefixLength) + const endKey = _keys[_keys.length - 1].slice(this.prefixLength) + const grouped = _.groupBy(_keys, (key) => key.slice(0, this.prefixLength)) for (const [prefix, keys] of Object.entries(grouped)) { const ranges = this.ranges.filter((range) => range.prefix === prefix) if (ranges.length === 0) { // no existing range with prefix - this.ranges.push({ prefix, keys: keys.map((i) => i.slice(PREFIX_LENGTH)) }) + this.ranges.push({ prefix, keys: keys.map((i) => i.slice(this.prefixLength)) }) continue } @@ -27,14 +26,14 @@ export default class KeyCache { const startPosition = _.sortedIndex(range.keys, startKey) if (startPosition >= 0 && range.keys[startPosition] === startKey) { // found existing range with prefix - range.keys.splice(startPosition, keys.length, ...keys.map((i) => i.slice(PREFIX_LENGTH))) + range.keys.splice(startPosition, keys.length, ...keys.map((i) => i.slice(this.prefixLength))) merged = true break } const endPosition = _.sortedIndex(range.keys, endKey) if (endPosition >= 0 && range.keys[endPosition] === endKey) { // found existing range with prefix - range.keys.splice(0, endPosition + 1, ...keys.map((i) => i.slice(PREFIX_LENGTH))) + range.keys.splice(0, endPosition + 1, ...keys.map((i) => i.slice(this.prefixLength))) merged = true break } @@ -42,7 +41,7 @@ export default class KeyCache { // insert new prefix with range if (!merged) { - this.ranges.push({ prefix, keys: keys.map((i) => i.slice(PREFIX_LENGTH)) }) + this.ranges.push({ prefix, keys: keys.map((i) => i.slice(this.prefixLength)) }) } } @@ -50,10 +49,15 @@ export default class KeyCache { } async next(startKey: HexString): Promise { - if (startKey.length < PREFIX_LENGTH) return - const prefix = startKey.slice(0, PREFIX_LENGTH) - const key = startKey.slice(PREFIX_LENGTH) + if (startKey.length < this.prefixLength) return + const prefix = startKey.slice(0, this.prefixLength) + const key = startKey.slice(this.prefixLength) for (const range of this.ranges.filter((range) => range.prefix === prefix)) { + if (key.length === 0) { + // if key is empty then find the range with first key empty + if (range.keys[0] !== '') continue + return [prefix, range.keys[1]].join('') as HexString + } const index = _.sortedIndex(range.keys, key) if (range.keys[index] !== key) continue const nextKey = range.keys[index + 1] diff --git a/packages/core/src/wasm-executor/index.ts b/packages/core/src/wasm-executor/index.ts index 7b574de7..8d22b3dc 100644 --- a/packages/core/src/wasm-executor/index.ts +++ b/packages/core/src/wasm-executor/index.ts @@ -5,9 +5,8 @@ import { randomAsHex } from '@polkadot/util-crypto' import _ from 'lodash' import { Block } from '../blockchain/block.js' -import { PREFIX_LENGTH } from '../utils/key-cache.js' +import { PREFIX_LENGTH, stripChildPrefix } from '../utils/index.js' import { defaultLogger, truncate } from '../logger.js' -import { stripChildPrefix } from '../utils/index.js' import type { JsCallback } from '@acala-network/chopsticks-executor' export { JsCallback } diff --git a/packages/e2e/blobs/shibuya-118.wasm b/packages/e2e/blobs/shibuya-118.wasm new file mode 100644 index 00000000..1d4da9bb Binary files /dev/null and b/packages/e2e/blobs/shibuya-118.wasm differ diff --git a/packages/e2e/src/build-block.test.ts b/packages/e2e/src/build-block.test.ts index 39e0eb8a..26ae6db3 100644 --- a/packages/e2e/src/build-block.test.ts +++ b/packages/e2e/src/build-block.test.ts @@ -12,16 +12,23 @@ const KUSAMA_STORAGE = { }, } -describe.runIf(process.env.CI).each([ +describe.runIf(process.env.CI || process.env.RUN_ALL).each([ { chain: 'Polkadot', endpoint: 'https://rpc.polkadot.io' }, { chain: 'Statemint', endpoint: 'wss://statemint-rpc.dwellir.com' }, // { chain: 'Polkadot Collectives', endpoint: 'wss://sys.ibp.network/collectives-polkadot' }, - { chain: 'Acala', endpoint: 'wss://acala-rpc-1.aca-api.network' }, + { chain: 'Acala', endpoint: 'wss://acala-rpc.aca-api.network' }, { chain: 'Kusama', endpoint: 'wss://kusama-rpc.polkadot.io', storage: KUSAMA_STORAGE }, { chain: 'Statemine', endpoint: 'wss://statemine-rpc-tn.dwellir.com' }, - { chain: 'Karura', endpoint: 'wss://karura-rpc-3.aca-api.network' }, - + { + chain: 'Karura', + endpoint: [ + 'wss://karura-rpc-0.aca-api.network', + 'wss://karura-rpc-1.aca-api.network', + 'wss://karura-rpc-2.aca-api.network', + 'wss://karura-rpc-3.aca-api.network', + ], + }, { chain: 'Westend', endpoint: 'wss://westend-rpc.polkadot.io' }, { chain: 'Westmint', endpoint: 'wss://westmint-rpc.polkadot.io' }, // { chain: 'Westend Collectives', endpoint: 'wss://sys.ibp.network/collectives-westend' }, diff --git a/packages/e2e/src/crowdloan.redeem.test.ts b/packages/e2e/src/crowdloan.redeem.test.ts index a96da441..8771e83e 100644 --- a/packages/e2e/src/crowdloan.redeem.test.ts +++ b/packages/e2e/src/crowdloan.redeem.test.ts @@ -19,29 +19,64 @@ describe('Polkadot Crowdloan Refund', async () => { }) }, 200_000) - it.runIf(process.env.CI)( + it.runIf(process.env.CI || process.env.RUN_ALL)( "should refund Acala's contributors", async () => { // trigger refund await expect(api.tx.crowdloan.refund(3336).signAndSend(alice)).resolves.toBeTruthy() await dev.newBlock() - // some address get refund - expect((await api.query.system.events()).toHuman()).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - event: expect.objectContaining({ - method: 'Transfer', - section: 'balances', - data: expect.objectContaining({ - from: '13UVJyLnbVp77Z2t6qZV4fNpRjDHppL6c87bHcZKG48tKJad', - to: '111DbHPUxncZcffEfy1BrtFZNDUzK7hHchLpmJYFEFG4hy1', - amount: '1,000,000,000,000', + { + // 1000 accounts get refunded and crowdloan is partially refunded + const events = await api.query.system.events() + expect(events.filter((event) => event.event.method === 'Transfer').length === 1000).toBeTruthy() + expect(events.toHuman()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + event: expect.objectContaining({ + method: 'Transfer', + section: 'balances', + data: expect.objectContaining({ + from: '13UVJyLnbVp77Z2t6qZV4fNpRjDHppL6c87bHcZKG48tKJad', + to: '111DbHPUxncZcffEfy1BrtFZNDUzK7hHchLpmJYFEFG4hy1', + amount: '1,000,000,000,000', + }), }), }), - }), - ]), - ) + expect.objectContaining({ + event: expect.objectContaining({ + method: 'PartiallyRefunded', + section: 'crowdloan', + data: expect.objectContaining({ + paraId: '3,336', + }), + }), + }), + ]), + ) + } + + await expect(api.tx.crowdloan.refund(3336).signAndSend(alice)).resolves.toBeTruthy() + await dev.newBlock() + + { + // 1000 accounts get refunded and crowdloan is partially refunded + const events = await api.query.system.events() + expect(events.filter((event) => event.event.method === 'Transfer').length === 1000).toBeTruthy() + expect(events.toHuman()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + event: expect.objectContaining({ + method: 'PartiallyRefunded', + section: 'crowdloan', + data: expect.objectContaining({ + paraId: '3,336', + }), + }), + }), + ]), + ) + } }, { timeout: 400_000 }, ) @@ -54,8 +89,8 @@ describe('Polkadot Crowdloan Refund', async () => { section: 'balances', data: expect.objectContaining({ from: '13UVJyLnbVp77Z2t6qZV4fNpRjDHppL6c87bHcZKG48tKJad', - to: '1E8EcginNpZRZezwa1A5eQT6crLQQj5R4T3pLKFbyJX3VU8', - amount: '500,000,000,000', + to: '1TkyFWT8PkGiFAD4pcnq8nB2RModtod1Hk4yLoVYbMtzagW', + amount: '50,000,000,000', }), }), }), @@ -63,14 +98,14 @@ describe('Polkadot Crowdloan Refund', async () => { // trigger refund await expect( - api.tx.crowdloan.withdraw('1E8EcginNpZRZezwa1A5eQT6crLQQj5R4T3pLKFbyJX3VU8', 3336).signAndSend(alice), + api.tx.crowdloan.withdraw('1TkyFWT8PkGiFAD4pcnq8nB2RModtod1Hk4yLoVYbMtzagW', 3336).signAndSend(alice), ).resolves.toBeTruthy() await dev.newBlock() expect((await api.query.system.events()).toHuman()).toEqual(expectedEvent) // doing the same thing again should fail because the funds are already withdrawn await expect( - api.tx.crowdloan.withdraw('1E8EcginNpZRZezwa1A5eQT6crLQQj5R4T3pLKFbyJX3VU8', 3336).signAndSend(alice), + api.tx.crowdloan.withdraw('1TkyFWT8PkGiFAD4pcnq8nB2RModtod1Hk4yLoVYbMtzagW', 3336).signAndSend(alice), ).resolves.toBeTruthy() await dev.newBlock() expect((await api.query.system.events()).toHuman()).not.toEqual(expectedEvent) diff --git a/packages/e2e/src/hrmp.test.ts b/packages/e2e/src/hrmp.test.ts index 654bd2e1..ea07c472 100644 --- a/packages/e2e/src/hrmp.test.ts +++ b/packages/e2e/src/hrmp.test.ts @@ -24,21 +24,31 @@ const acalaHRMP: Record = { describe('HRMP', () => { it('Statemine handles horizonal messages', async () => { - const statemine = await setupContext({ endpoint: 'wss://statemine-rpc-tn.dwellir.com' }) + const statemine = await setupContext({ + endpoint: 'wss://statemine-rpc-tn.dwellir.com', + db: !process.env.RUN_TESTS_WITHOUT_DB ? 'e2e-tests-db.sqlite' : undefined, + }) await statemine.chain.newBlock({ horizontalMessages: statemineHRMP }) await checkSystemEvents(statemine, 'xcmpQueue', 'Success').toMatchSnapshot() await statemine.teardown() }) it('Acala handles horizonal messages', async () => { - const acala = await setupContext({ endpoint: 'wss://acala-rpc.aca-api.network' }) + const acala = await setupContext({ + endpoint: 'wss://acala-rpc.aca-api.network', + db: !process.env.RUN_TESTS_WITHOUT_DB ? 'e2e-tests-db.sqlite' : undefined, + }) await acala.chain.newBlock({ horizontalMessages: acalaHRMP }) await checkSystemEvents(acala, 'xcmpQueue', 'Success').toMatchSnapshot() await acala.teardown() }) it('Statemine handles horizonal messages block#5,800,000', async () => { - const statemine = await setupContext({ endpoint: 'wss://statemine-rpc-tn.dwellir.com', blockNumber: 5_800_000 }) + const statemine = await setupContext({ + endpoint: 'wss://statemine-rpc-tn.dwellir.com', + blockNumber: 5_800_000, + db: !process.env.RUN_TESTS_WITHOUT_DB ? 'e2e-tests-db.sqlite' : undefined, + }) await statemine.chain.newBlock({ horizontalMessages: statemineHRMP }) await checkSystemEvents(statemine, 'xcmpQueue', 'Success').toMatchSnapshot() await statemine.teardown() diff --git a/packages/e2e/src/migration.test.ts b/packages/e2e/src/migration.test.ts new file mode 100644 index 00000000..9f7a3359 --- /dev/null +++ b/packages/e2e/src/migration.test.ts @@ -0,0 +1,67 @@ +import { afterAll, describe, expect, it } from 'vitest' +import { setupContext, testingPairs } from '@acala-network/chopsticks-testing' + +describe('Migration', async () => { + const { alice } = testingPairs() + + const { api, dev, teardown } = await setupContext({ + wasmOverride: new URL('../blobs/shibuya-118.wasm', import.meta.url).pathname, + blockNumber: 5335600, + endpoint: 'wss://shibuya-rpc.dwellir.com', + db: !process.env.RUN_TESTS_WITHOUT_DB ? 'e2e-tests-db.sqlite' : undefined, + timeout: 400_000, + }) + + afterAll(async () => await teardown()) + + it.runIf(process.env.CI || process.env.RUN_ALL)( + 'migrate all entries', + async () => { + { + const version = await api.query.system.lastRuntimeUpgrade() + expect(version.toHuman()).toMatchInlineSnapshot(` + { + "specName": "shibuya", + "specVersion": "115", + } + `) + } + await dev.setStorage({ + System: { + Account: [[[alice.address], { providers: 1, data: { free: '100000000000000000000000' } }]], + }, + }) + + await dev.newBlock() + { + const version = await api.query.system.lastRuntimeUpgrade() + expect(version.toHuman()).toMatchInlineSnapshot(` + { + "specName": "shibuya", + "specVersion": "118", + } + `) + } + + for (const items of [301 /*, 295*/]) { + // number of entries migrated, matches with onchain data + // first call will migrate 301 entries, second call will migrate 295 entries + await api.tx.dappStakingMigration.migrate(null).signAndSend(alice) + await dev.newBlock() + const events = await api.query.system.events() + expect(events.toHuman()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + event: expect.objectContaining({ + section: 'dappStakingMigration', + method: 'EntriesMigrated', + data: [`${items}`], + }), + }), + ]), + ) + } + }, + { timeout: 400_000 }, + ) +}) diff --git a/packages/e2e/src/networks.ts b/packages/e2e/src/networks.ts index 236a8c5e..3d73a47e 100644 --- a/packages/e2e/src/networks.ts +++ b/packages/e2e/src/networks.ts @@ -5,7 +5,7 @@ import { SetupOption, setupContext } from '@acala-network/chopsticks-testing' dotenvConfig const endpoints = { - polkadot: ['wss://rpc.polkadot.io', 'wss://polkadot-rpc.dwellir.com'], + polkadot: ['wss://rpc.ibp.network/polkadot', 'wss://polkadot-rpc.dwellir.com'], acala: ['wss://acala-rpc.aca-api.network', 'wss://acala-rpc.dwellir.com'], } @@ -25,7 +25,7 @@ export default { wasmOverride: process.env.POLKADOT_WASM, blockNumber: toNumber(process.env.POLKADOT_BLOCK_NUMBER) || 14500000, endpoint: process.env.POLKADOT_ENDPOINT ?? endpoints.polkadot, - db: process.env.DB_PATH, + db: !process.env.RUN_TESTS_WITHOUT_DB ? 'e2e-tests-db.sqlite' : undefined, ...options, }), acala: (options?: Partial) => @@ -33,7 +33,7 @@ export default { wasmOverride: process.env.ACALA_WASM, blockNumber: toNumber(process.env.ACALA_BLOCK_NUMBER) || 3000000, endpoint: process.env.ACALA_ENDPOINT ?? endpoints.acala, - db: process.env.DB_PATH, + db: !process.env.RUN_TESTS_WITHOUT_DB ? 'e2e-tests-db.sqlite' : undefined, ...options, }), } diff --git a/packages/e2e/src/storage-migrate.test.ts b/packages/e2e/src/storage-migrate.test.ts index 52785047..31bc86e1 100644 --- a/packages/e2e/src/storage-migrate.test.ts +++ b/packages/e2e/src/storage-migrate.test.ts @@ -26,7 +26,7 @@ setupApi({ }, }) -describe.runIf(process.env.CI)('storage-migrate', async () => { +describe.runIf(process.env.CI || process.env.RUN_ALL)('storage-migrate', async () => { it( 'no empty keys', async () => {