From 873f8d40de671681ea609c127caeeff8e098ab78 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Tue, 9 Aug 2022 12:07:49 +1000 Subject: [PATCH] wip: preliminary DB fixes. --- benches/gitgc.ts | 20 ++++----- src/acl/ACL.ts | 4 +- src/discovery/Discovery.ts | 21 ++++----- src/gestalts/GestaltGraph.ts | 2 +- src/nodes/NodeGraph.ts | 52 ++++++++++------------- src/notifications/NotificationsManager.ts | 13 +++--- src/sigchain/Sigchain.ts | 29 +++++++------ src/utils/utils.ts | 7 ++- src/vaults/VaultManager.ts | 1 - tests/nodes/utils.test.ts | 4 +- 10 files changed, 78 insertions(+), 75 deletions(-) diff --git a/benches/gitgc.ts b/benches/gitgc.ts index 983065cf1f..5026436fbb 100644 --- a/benches/gitgc.ts +++ b/benches/gitgc.ts @@ -2,7 +2,7 @@ import path from 'path'; import b from 'benny'; import { suiteCommon } from './utils'; -async function main () { +async function main() { let map = new Map(); let obj = {}; let arr: any = []; @@ -18,10 +18,10 @@ async function main () { for (let i = 0; i < 1000; i++) { map.delete(i); } - for (const i of map) { + for (const _i of map) { // NOOP } - } + }; }), b.add('obj', async () => { obj = {}; @@ -32,26 +32,26 @@ async function main () { for (let i = 0; i < 1000; i++) { delete obj[i]; } - for (const i in obj) { + for (const _i in obj) { // NOOP } }; }), b.add('arr', async () => { - // you first have to count the number of objects + // You first have to count the number of objects arr = []; return async () => { - // you have to iterate for each object + // You have to iterate for each object // then for each value in length for (let i = 0; i < 1000; i++) { if (i === arr.length) { - // double the vector + // Double the vector arr.length = arr.length * 2 || 2; } arr[i] = { id: i, mark: false }; - // arr.push({ id: i, mark: false}); + // Arr.push({ id: i, mark: false}); } - // this has to iterate the length of the array + // This has to iterate the length of the array // but stop as soon as it reaches the end // it gets complicate, but for 5x improvement // it could be interesting @@ -74,7 +74,7 @@ async function main () { for (let i = 0; i < 1000; i++) { set.delete(i); } - for (const i of set) { + for (const _i of set) { // NOOP } }; diff --git a/src/acl/ACL.ts b/src/acl/ACL.ts index 62d5bfa706..db9a61d7b7 100644 --- a/src/acl/ACL.ts +++ b/src/acl/ACL.ts @@ -133,7 +133,7 @@ class ACL { return this.withTransactionF(async (tran) => this.getNodePerms(tran)); } const permIds: Record> = {}; - for await (const [keyPath, value] of tran.iterator(undefined, [ + for await (const [keyPath, value] of tran.iterator([ ...this.aclNodesDbPath, ])) { const key = keyPath[0] as Buffer; @@ -175,8 +175,8 @@ class ACL { } const vaultPerms: Record> = {}; for await (const [keyPath, nodeIds] of tran.iterator>( - { valueAsBuffer: false }, [...this.aclVaultsDbPath], + { valueAsBuffer: false }, )) { const key = keyPath[0] as Buffer; const vaultId = IdInternal.fromBuffer(key); diff --git a/src/discovery/Discovery.ts b/src/discovery/Discovery.ts index 5d33b5cbfd..3c5e57a492 100644 --- a/src/discovery/Discovery.ts +++ b/src/discovery/Discovery.ts @@ -130,10 +130,11 @@ class Discovery { } // Getting latest ID and creating ID generator let latestId: DiscoveryQueueId | undefined; - const keyIterator = this.db.iterator( - { limit: 1, reverse: true, values: false }, - this.discoveryQueueDbPath, - ); + const keyIterator = this.db.iterator(this.discoveryQueueDbPath, { + limit: 1, + reverse: true, + values: false, + }); for await (const [keyPath] of keyIterator) { const key = keyPath[0] as Buffer; latestId = IdInternal.fromBuffer(key); @@ -204,8 +205,8 @@ class Discovery { // Processing queue this.logger.debug('DiscoveryQueue is processing'); for await (const [keyPath, vertex] of this.db.iterator( - { valueAsBuffer: false }, this.discoveryQueueDbPath, + { valueAsBuffer: false }, )) { const key = keyPath[0] as Buffer; const vertexId = IdInternal.fromBuffer(key); @@ -425,10 +426,10 @@ class Discovery { protected async queueIsEmpty(): Promise { return await this.lock.withF(async () => { let nextDiscoveryQueueId: DiscoveryQueueId | undefined; - const keyIterator = this.db.iterator( - { limit: 1, values: false }, - this.discoveryQueueDbPath, - ); + const keyIterator = this.db.iterator(this.discoveryQueueDbPath, { + limit: 1, + values: false, + }); for await (const [keyPath] of keyIterator) { const key = keyPath[0] as Buffer; nextDiscoveryQueueId = IdInternal.fromBuffer(key); @@ -449,8 +450,8 @@ class Discovery { [this.db.transaction(), this.lock.lock()], async ([tran]) => { const valueIterator = tran.iterator( - { valueAsBuffer: false }, this.discoveryQueueDbPath, + { valueAsBuffer: false }, ); for await (const [, value] of valueIterator) { if (value === gestaltKey) { diff --git a/src/gestalts/GestaltGraph.ts b/src/gestalts/GestaltGraph.ts index 0bb6c7cd11..f61b234955 100644 --- a/src/gestalts/GestaltGraph.ts +++ b/src/gestalts/GestaltGraph.ts @@ -104,8 +104,8 @@ class GestaltGraph { } const unvisited: Map = new Map(); for await (const [k, gKs] of tran.iterator( - { valueAsBuffer: false }, [...this.gestaltGraphMatrixDbPath], + { valueAsBuffer: false }, )) { const gK = k.toString() as GestaltKey; unvisited.set(gK, gKs); diff --git a/src/nodes/NodeGraph.ts b/src/nodes/NodeGraph.ts index a05610d33a..e37140138c 100644 --- a/src/nodes/NodeGraph.ts +++ b/src/nodes/NodeGraph.ts @@ -192,11 +192,11 @@ class NodeGraph { } for await (const [keyPath, nodeData] of tran.iterator( + this.nodeGraphBucketsDbPath, { reverse: order !== 'asc', valueAsBuffer: false, }, - this.nodeGraphBucketsDbPath, )) { const { nodeId } = nodesUtils.parseBucketsDbKey(keyPath); yield [nodeId, nodeData]; @@ -273,10 +273,10 @@ class NodeGraph { const bucketKey = nodesUtils.bucketKey(bucketIndex); // Remove the oldest entry in the bucket const oldestNodeIds: Array = []; - for await (const [keyPath] of tran.iterator({ limit }, [ - ...this.nodeGraphLastUpdatedDbPath, - bucketKey, - ])) { + for await (const [keyPath] of tran.iterator( + [...this.nodeGraphLastUpdatedDbPath, bucketKey], + { limit }, + )) { const { nodeId } = nodesUtils.parseLastUpdatedBucketDbKey(keyPath); oldestNodeIds.push(nodeId); } @@ -338,11 +338,11 @@ class NodeGraph { const bucket: NodeBucket = []; if (sort === 'nodeId' || sort === 'distance') { for await (const [key, nodeData] of tran.iterator( + [...this.nodeGraphBucketsDbPath, bucketKey], { reverse: order !== 'asc', valueAsBuffer: false, }, - [...this.nodeGraphBucketsDbPath, bucketKey], )) { const nodeId = nodesUtils.parseBucketDbKey(key[0] as Buffer); bucket.push([nodeId, nodeData]); @@ -356,15 +356,15 @@ class NodeGraph { } } else if (sort === 'lastUpdated') { const bucketDbIterator = tran.iterator( - { valueAsBuffer: false }, [...this.nodeGraphBucketsDbPath, bucketKey], + { valueAsBuffer: false }, ); try { for await (const [, nodeIdBuffer] of tran.iterator( + [...this.nodeGraphLastUpdatedDbPath, bucketKey], { reverse: order !== 'asc', }, - [...this.nodeGraphLastUpdatedDbPath, bucketKey], )) { const nodeId = IdInternal.fromBuffer(nodeIdBuffer); bucketDbIterator.seek(nodeIdBuffer); @@ -375,7 +375,7 @@ class NodeGraph { bucket.push([nodeId, nodeData]); } } finally { - await bucketDbIterator.end(); + await bucketDbIterator.destroy(); // FIXME: should this be `.destroy` now? } } return bucket; @@ -410,11 +410,11 @@ class NodeGraph { let bucket: NodeBucket = []; if (sort === 'nodeId' || sort === 'distance') { for await (const [key, nodeData] of tran.iterator( + this.nodeGraphBucketsDbPath, { reverse: order !== 'asc', valueAsBuffer: false, }, - this.nodeGraphBucketsDbPath, )) { const { bucketIndex: bucketIndex_, nodeId } = nodesUtils.parseBucketsDbKey(key); @@ -452,15 +452,15 @@ class NodeGraph { } } else if (sort === 'lastUpdated') { const bucketsDbIterator = tran.iterator( - { valueAsBuffer: false }, this.nodeGraphBucketsDbPath, + { valueAsBuffer: false }, ); try { for await (const [key] of tran.iterator( + this.nodeGraphLastUpdatedDbPath, { reverse: order !== 'asc', }, - this.nodeGraphLastUpdatedDbPath, )) { const { bucketIndex: bucketIndex_, nodeId } = nodesUtils.parseLastUpdatedBucketsDbKey(key); @@ -488,7 +488,7 @@ class NodeGraph { yield [bucketIndex, bucket]; } } finally { - await bucketsDbIterator.end(); + await bucketsDbIterator.destroy(); // FIXME: destroy? } } } @@ -524,8 +524,8 @@ class NodeGraph { // Iterating over all entries across all buckets for await (const [key, nodeData] of tran.iterator( - { valueAsBuffer: false }, this.nodeGraphBucketsDbPath, + { valueAsBuffer: false }, )) { // The key is a combined bucket key and node ID const { bucketIndex: bucketIndexOld, nodeId } = @@ -551,12 +551,9 @@ class NodeGraph { } else { let oldestIndexKey: KeyPath | undefined = undefined; let oldestNodeId: NodeId | undefined = undefined; - for await (const [key] of tran.iterator( - { - limit: 1, - }, - indexPathNew, - )) { + for await (const [key] of tran.iterator(indexPathNew, { + limit: 1, + })) { oldestIndexKey = key; ({ nodeId: oldestNodeId } = nodesUtils.parseLastUpdatedBucketDbKey(key)); @@ -716,12 +713,12 @@ class NodeGraph { const remainingLimit = limit - nodeIds.length; // Iterate over lower buckets for await (const [key, nodeData] of tran.iterator( + this.nodeGraphBucketsDbPath, { lt: [bucketIdKey, ''], limit: remainingLimit, valueAsBuffer: false, }, - this.nodeGraphBucketsDbPath, )) { const info = nodesUtils.parseBucketsDbKey(key); nodeIds.push([info.nodeId, nodeData]); @@ -732,20 +729,17 @@ class NodeGraph { const bucketId = Buffer.from(nodesUtils.bucketKey(startingBucket + 1)); const remainingLimit = limit - nodeIds.length; // Iterate over ids further away - tran.iterator( - { - gt: [bucketId, ''], - limit: remainingLimit, - }, - this.nodeGraphBucketsDbPath, - ); + tran.iterator(this.nodeGraphBucketsDbPath, { + gt: [bucketId, ''], + limit: remainingLimit, + }); for await (const [key, nodeData] of tran.iterator( + this.nodeGraphBucketsDbPath, { gt: [bucketId, ''], limit: remainingLimit, valueAsBuffer: false, }, - this.nodeGraphBucketsDbPath, )) { const info = nodesUtils.parseBucketsDbKey(key); nodeIds.push([info.nodeId, nodeData]); diff --git a/src/notifications/NotificationsManager.ts b/src/notifications/NotificationsManager.ts index ac91a0cf14..9b58945990 100644 --- a/src/notifications/NotificationsManager.ts +++ b/src/notifications/NotificationsManager.ts @@ -139,10 +139,11 @@ class NotificationsManager { // Getting latest ID and creating ID generator let latestId: NotificationId | undefined; - const keyIterator = tran.iterator( - { limit: 1, reverse: true, values: false }, - this.notificationsMessagesDbPath, - ); + const keyIterator = tran.iterator(this.notificationsMessagesDbPath, { + limit: 1, + reverse: true, + values: false, + }); for await (const [keyPath] of keyIterator) { const key = keyPath[0] as Buffer; latestId = IdInternal.fromBuffer(key); @@ -368,8 +369,8 @@ class NotificationsManager { ): Promise> { const notificationIds: Array = []; const messageIterator = tran.iterator( - { valueAsBuffer: false }, this.notificationsMessagesDbPath, + { valueAsBuffer: false }, ); for await (const [keyPath, notification] of messageIterator) { const key = keyPath[0] as Buffer; @@ -391,8 +392,8 @@ class NotificationsManager { ): Promise> { const notifications: Array = []; for await (const [, notification] of tran.iterator( - { valueAsBuffer: false }, this.notificationsMessagesDbPath, + { valueAsBuffer: false }, )) { if (type === 'all') { notifications.push(notification); diff --git a/src/sigchain/Sigchain.ts b/src/sigchain/Sigchain.ts index 06631cdd59..d69e30c579 100644 --- a/src/sigchain/Sigchain.ts +++ b/src/sigchain/Sigchain.ts @@ -286,9 +286,10 @@ class Sigchain { return this.withTransactionF(async (tran) => this.getChainData(tran)); } const chainData: ChainDataEncoded = {}; - const readIterator = tran.iterator({ valueAsBuffer: false }, [ - ...this.sigchainClaimsDbPath, - ]); + const readIterator = tran.iterator( + this.sigchainClaimsDbPath, + { valueAsBuffer: false }, + ); for await (const [keyPath, claimEncoded] of readIterator) { const key = keyPath[0] as Buffer; const claimId = IdInternal.fromBuffer(key); @@ -316,9 +317,10 @@ class Sigchain { ); } const relevantClaims: Array = []; - const readIterator = tran.iterator({ valueAsBuffer: false }, [ - ...this.sigchainClaimsDbPath, - ]); + const readIterator = tran.iterator( + this.sigchainClaimsDbPath, + { valueAsBuffer: false }, + ); for await (const [, claim] of readIterator) { const decodedClaim = claimsUtils.decodeClaim(claim); if (decodedClaim.payload.data.type === claimType) { @@ -400,9 +402,9 @@ class Sigchain { return this.withTransactionF(async (tran) => this.getSeqMap(tran)); } const map: Record = {}; - const claimStream = tran.iterator({ values: false }, [ - ...this.sigchainClaimsDbPath, - ]); + const claimStream = tran.iterator(this.sigchainClaimsDbPath, { + values: false, + }); let seq = 1; for await (const [keyPath] of claimStream) { const key = keyPath[0] as Buffer; @@ -416,10 +418,11 @@ class Sigchain { tran: DBTransaction, ): Promise { let latestId: ClaimId | undefined; - const keyStream = tran.iterator( - { limit: 1, reverse: true, values: false }, - [...this.sigchainClaimsDbPath], - ); + const keyStream = tran.iterator(this.sigchainClaimsDbPath, { + limit: 1, + reverse: true, + values: false, + }); for await (const [keyPath] of keyStream) { latestId = IdInternal.fromBuffer(keyPath[0] as Buffer); } diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 84f3194485..a55cc48922 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -1,4 +1,9 @@ -import type { FileSystem, Timer, PromiseDeconstructed, Callback } from '../types'; +import type { + FileSystem, + Timer, + PromiseDeconstructed, + Callback, +} from '../types'; import os from 'os'; import process from 'process'; import path from 'path'; diff --git a/src/vaults/VaultManager.ts b/src/vaults/VaultManager.ts index e6fa716f69..6bd32803d1 100644 --- a/src/vaults/VaultManager.ts +++ b/src/vaults/VaultManager.ts @@ -425,7 +425,6 @@ class VaultManager { const vaults: VaultList = new Map(); // Stream of vaultName VaultId key value pairs for await (const [vaultNameBuffer, vaultIdBuffer] of tran.iterator( - undefined, this.vaultsNamesDbPath, )) { const vaultName = vaultNameBuffer.toString() as VaultName; diff --git a/tests/nodes/utils.test.ts b/tests/nodes/utils.test.ts index 64d7c7afe9..c2c1dfee8f 100644 --- a/tests/nodes/utils.test.ts +++ b/tests/nodes/utils.test.ts @@ -122,7 +122,7 @@ describe('nodes/utils', () => { data.sort((a, b) => Buffer.compare(a.key, b.key)); let i = 0; - for await (const [key] of db.iterator({}, bucketsDbPath)) { + for await (const [key] of db.iterator(bucketsDbPath)) { const { bucketIndex, bucketKey, nodeId } = nodesUtils.parseBucketsDbKey( key as Array, ); @@ -162,7 +162,7 @@ describe('nodes/utils', () => { // the bucket key and last updated and node ID data.sort((a, b) => Buffer.compare(a.key, b.key)); let i = 0; - for await (const [key] of db.iterator({}, lastUpdatedDbPath)) { + for await (const [key] of db.iterator(lastUpdatedDbPath)) { const { bucketIndex, bucketKey, lastUpdated, nodeId } = nodesUtils.parseLastUpdatedBucketsDbKey(key as Array); expect(bucketIndex).toBe(data[i].bucketIndex);