Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

improve keysPaged #641

Merged
merged 5 commits into from
Jan 23, 2024
Merged

improve keysPaged #641

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
044968b
improve keysPaged
ermalkaleci Jan 23, 2024
0e1fcbb
use db
ermalkaleci Jan 23, 2024
b1dc02a
update test
ermalkaleci Jan 23, 2024
ce1d556
lint
ermalkaleci Jan 23, 2024
1e009bb
cleanup
ermalkaleci Jan 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ dist
*.sqlite
*.sqlite-journal
*.wasm
!packages/e2e/blobs/*.wasm
*.db
*.sqlite-shm
*.sqlite-wal
Expand Down
2 changes: 1 addition & 1 deletion configs/polkadot.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
endpoint:
- wss://rpc.polkadot.io
- wss://rpc.ibp.network/polkadot
- wss://polkadot-rpc.dwellir.com
mock-signature-host: true
block: ${env.POLKADOT_BLOCK_NUMBER}
Expand Down
21 changes: 21 additions & 0 deletions packages/core/src/api.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { ExtDef } from '@polkadot/types/extrinsic/signedExtensions/types'
import { HexString } from '@polkadot/util/types'
import { ProviderInterface, ProviderInterfaceCallback } from '@polkadot/rpc-provider/types'
import { prefixedChildKey, splitChildKey, stripChildPrefix } from './utils/index.js'
import _ from 'lodash'

export type ChainProperties = {
ss58Format?: number
Expand Down Expand Up @@ -150,6 +151,26 @@ export class Api {
}
}

async getStorageBatch(prefix: HexString, keys: HexString[], hash?: HexString) {
const [child] = splitChildKey(prefix)
if (child) {
// child storage key, use childstate_getStorageEntries
// strip child prefix from keys
const params: any[] = [child, keys.map((key) => stripChildPrefix(key))]
if (hash) params.push(hash)
return this.#provider
.send<HexString[]>('childstate_getStorageEntries', params, !!hash)
.then((values) => _.zip(keys, values) as [HexString, HexString | null][])
} else {
// main storage key, use state_getStorageAt
const params: any[] = [keys]
if (hash) params.push(hash)
return this.#provider
.send<HexString[]>('state_queryStorageAt', params, !!hash)
.then((result) => (result[0]?.['changes'] as [HexString, HexString | null][]) || [])
}
}

async subscribeRemoteNewHeads(cb: ProviderInterfaceCallback) {
if (!this.#provider.hasSubscriptions) {
throw new Error('subscribeRemoteNewHeads only works with subscriptions')
Expand Down
5 changes: 1 addition & 4 deletions packages/core/src/blockchain/block.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,14 +176,11 @@ export class Block {
* Get paged storage keys.
*/
async getKeysPaged(options: { prefix?: string; startKey?: string; pageSize: number }): Promise<string[]> {
const layer = new StorageLayer(this.storage)
await layer.fold()

const prefix = options.prefix ?? '0x'
const startKey = options.startKey ?? '0x'
const pageSize = options.pageSize

return layer.getKeysPaged(prefix, pageSize, startKey)
return this.storage.getKeysPaged(prefix, pageSize, startKey)
}

/**
Expand Down
257 changes: 257 additions & 0 deletions packages/core/src/blockchain/get-keys-paged.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,257 @@
import { Api } from '../api.js'
import { RemoteStorageLayer, StorageLayer, StorageValueKind } from './storage-layer.js'
import { describe, expect, it, vi } from 'vitest'
import _ from 'lodash'

describe('getKeysPaged', () => {
const hash = '0x1111111111111111111111111111111111111111111111111111111111111111'

const allKeys = [
'0x0000000000000000000000000000000000000000000000000000000000000000_00',
'0x0000000000000000000000000000000000000000000000000000000000000000_03',
'0x0000000000000000000000000000000000000000000000000000000000000000_04',
'0x1111111111111111111111111111111111111111111111111111111111111111_01',
'0x1111111111111111111111111111111111111111111111111111111111111111_02',
'0x1111111111111111111111111111111111111111111111111111111111111111_03',
'0x1111111111111111111111111111111111111111111111111111111111111111_06',
'0x1111111111111111111111111111111111111111111111111111111111111111_07',
'0x2222222222222222222222222222222222222222222222222222222222222222_21',
'0x2222222222222222222222222222222222222222222222222222222222222222_23',
'0x2222222222222222222222222222222222222222222222222222222222222222_26',
]

Api.prototype['getKeysPaged'] = vi.fn(async (prefix, pageSize, startKey) => {
const withPrefix = allKeys.filter((k) => k.startsWith(prefix) && k > startKey)
const result = withPrefix.slice(0, pageSize)
return result
})
const mockApi = new Api(undefined!)

const remoteLayer = new RemoteStorageLayer(mockApi, hash, undefined)
const storageLayer = new StorageLayer(remoteLayer)

it('mocked api works', async () => {
expect(
await mockApi.getKeysPaged(
'0x1111111111111111111111111111111111111111111111111111111111111111',
1,
'0x1111111111111111111111111111111111111111111111111111111111111111',
hash,
),
).toEqual(['0x1111111111111111111111111111111111111111111111111111111111111111_01'])

expect(
await mockApi.getKeysPaged(
'0x2222222222222222222222222222222222222222222222222222222222222222',
4,
'0x2222222222222222222222222222222222222222222222222222222222222222',
hash,
),
).toEqual([
'0x2222222222222222222222222222222222222222222222222222222222222222_21',
'0x2222222222222222222222222222222222222222222222222222222222222222_23',
'0x2222222222222222222222222222222222222222222222222222222222222222_26',
])

expect(
await mockApi.getKeysPaged(
'0x1111111111111111111111111111111111111111111111111111111111111111',
10,
'0x1111111111111111111111111111111111111111111111111111111111111111_02',
hash,
),
).toEqual([
'0x1111111111111111111111111111111111111111111111111111111111111111_03',
'0x1111111111111111111111111111111111111111111111111111111111111111_06',
'0x1111111111111111111111111111111111111111111111111111111111111111_07',
])

expect(
await mockApi.getKeysPaged(
'0x1111111111111111111111111111111111111111111111111111111111111111',
2,
'0x1111111111111111111111111111111111111111111111111111111111111111_04',
hash,
),
).toEqual([
'0x1111111111111111111111111111111111111111111111111111111111111111_06',
'0x1111111111111111111111111111111111111111111111111111111111111111_07',
])

expect(
await mockApi.getKeysPaged(
'0x1111111111111111111111111111111111111111111111111111111111111111',
2,
'0x1111111111111111111111111111111111111111111111111111111111111111_07',
hash,
),
).toEqual([])
})

it('remote layer works', async () => {
expect(
await remoteLayer.getKeysPaged(
'0x1111111111111111111111111111111111111111111111111111111111111111',
3,
'0x1111111111111111111111111111111111111111111111111111111111111111',
),
).toEqual([
'0x1111111111111111111111111111111111111111111111111111111111111111_01',
'0x1111111111111111111111111111111111111111111111111111111111111111_02',
'0x1111111111111111111111111111111111111111111111111111111111111111_03',
])

expect(
await remoteLayer.getKeysPaged(
'0x1111111111111111111111111111111111111111111111111111111111111111',
10,
'0x1111111111111111111111111111111111111111111111111111111111111111_03',
),
).toEqual([
'0x1111111111111111111111111111111111111111111111111111111111111111_06',
'0x1111111111111111111111111111111111111111111111111111111111111111_07',
])
})

it('storage layer works', async () => {
expect(
await mockApi.getKeysPaged(
'0x1111111111111111111111111111111111111111111111111111111111111111',
1,
'0x1111111111111111111111111111111111111111111111111111111111111111',
hash,
),
).toEqual(['0x1111111111111111111111111111111111111111111111111111111111111111_01'])

expect(
await mockApi.getKeysPaged(
'0x1111111111111111111111111111111111111111111111111111111111111111',
10,
'0x1111111111111111111111111111111111111111111111111111111111111111_02',
hash,
),
).toEqual([
'0x1111111111111111111111111111111111111111111111111111111111111111_03',
'0x1111111111111111111111111111111111111111111111111111111111111111_06',
'0x1111111111111111111111111111111111111111111111111111111111111111_07',
])
})

it('updated values', async () => {
const layer2 = new StorageLayer(storageLayer)
layer2.setAll([
['0x1111111111111111111111111111111111111111111111111111111111111111_00', '0x00'],
['0x1111111111111111111111111111111111111111111111111111111111111111_04', '0x04'],
])
expect(
await layer2.getKeysPaged(
'0x1111111111111111111111111111111111111111111111111111111111111111',
10,
'0x1111111111111111111111111111111111111111111111111111111111111111_03',
),
).toEqual([
'0x1111111111111111111111111111111111111111111111111111111111111111_04',
'0x1111111111111111111111111111111111111111111111111111111111111111_06',
'0x1111111111111111111111111111111111111111111111111111111111111111_07',
])

expect(
await layer2.getKeysPaged(
'0x1111111111111111111111111111111111111111111111111111111111111111',
10,
'0x1111111111111111111111111111111111111111111111111111111111111111',
),
).toEqual([
'0x1111111111111111111111111111111111111111111111111111111111111111_00',
'0x1111111111111111111111111111111111111111111111111111111111111111_01',
'0x1111111111111111111111111111111111111111111111111111111111111111_02',
'0x1111111111111111111111111111111111111111111111111111111111111111_03',
'0x1111111111111111111111111111111111111111111111111111111111111111_04',
'0x1111111111111111111111111111111111111111111111111111111111111111_06',
'0x1111111111111111111111111111111111111111111111111111111111111111_07',
])

expect(
await layer2.getKeysPaged(
'0x1111111111111111111111111111111111111111111111111111111111111111',
10,
'0x1111111111111111111111111111111111111111111111111111111111111111_04',
),
).toEqual([
'0x1111111111111111111111111111111111111111111111111111111111111111_06',
'0x1111111111111111111111111111111111111111111111111111111111111111_07',
])

expect(
await layer2.getKeysPaged(
'0x1111111111111111111111111111111111111111111111111111111111111111',
10,
'0x1111111111111111111111111111111111111111111111111111111111111111_07',
),
).toEqual([])

const layer3 = new StorageLayer(layer2)
layer3.setAll([
['0x1111111111111111111111111111111111111111111111111111111111111111_03', '0x03'],
['0x1111111111111111111111111111111111111111111111111111111111111111_04', null],
['0x1111111111111111111111111111111111111111111111111111111111111111_06', '0x06'],
])

expect(
await layer3.getKeysPaged(
'0x1111111111111111111111111111111111111111111111111111111111111111',
10,
'0x1111111111111111111111111111111111111111111111111111111111111111_02',
),
).toEqual([
'0x1111111111111111111111111111111111111111111111111111111111111111_03',
'0x1111111111111111111111111111111111111111111111111111111111111111_06',
'0x1111111111111111111111111111111111111111111111111111111111111111_07',
])

const layer4 = new StorageLayer(layer3)
layer4.setAll([
['0x1111111111111111111111111111111111111111111111111111111111111111_03', null],
['0x1111111111111111111111111111111111111111111111111111111111111111_04', '0x04'],
['0x1111111111111111111111111111111111111111111111111111111111111111_06', null],
['0x1111111111111111111111111111111111111111111111111111111111111111_08', '0x08'],
])

expect(
await layer4.getKeysPaged(
'0x1111111111111111111111111111111111111111111111111111111111111111',
10,
'0x1111111111111111111111111111111111111111111111111111111111111111_02',
),
).toEqual([
'0x1111111111111111111111111111111111111111111111111111111111111111_04',
'0x1111111111111111111111111111111111111111111111111111111111111111_07',
'0x1111111111111111111111111111111111111111111111111111111111111111_08',
])

const layer5 = new StorageLayer(layer4)
layer5.setAll([
['0x1111111111111111111111111111111111111111111111111111111111111111', StorageValueKind.DeletedPrefix],
['0x1111111111111111111111111111111111111111111111111111111111111111_09', '0x09'],
])
expect(
await layer5.getKeysPaged(
'0x1111111111111111111111111111111111111111111111111111111111111111',
10,
'0x1111111111111111111111111111111111111111111111111111111111111111',
),
).toEqual(['0x1111111111111111111111111111111111111111111111111111111111111111_09'])

const layer6 = new StorageLayer(layer5)
layer6.setAll([
['0x1111111111111111111111111111111111111111111111111111111111111111', StorageValueKind.DeletedPrefix],
])
expect(
await layer6.getKeysPaged(
'0x1111111111111111111111111111111111111111111111111111111111111111',
10,
'0x1111111111111111111111111111111111111111111111111111111111111111',
),
).toEqual([])
})
})
Loading