From d93641651c551794d29b05d9cea73c0b6deb9782 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Fri, 22 Sep 2023 13:04:31 -0400 Subject: [PATCH 1/9] upgrade kysley for tuple support --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2ccc091..ddc4db6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "license": "Apache-2.0", "dependencies": { "@tbd54566975/dwn-sdk-js": "0.1.0", - "kysely": "0.25.0", + "kysely": "0.26.3", "multiformats": "12.0.1", "readable-stream": "4.4.2" }, @@ -3332,9 +3332,9 @@ "dev": true }, "node_modules/kysely": { - "version": "0.25.0", - "resolved": "https://registry.npmjs.org/kysely/-/kysely-0.25.0.tgz", - "integrity": "sha512-srn0efIMu5IoEBk0tBmtGnoUss4uwvxtbFQWG/U2MosfqIace1l43IFP1PmEpHRDp+Z79xIcKEqmHH3dAvQdQA==", + "version": "0.26.3", + "resolved": "https://registry.npmjs.org/kysely/-/kysely-0.26.3.tgz", + "integrity": "sha512-yWSgGi9bY13b/W06DD2OCDDHQmq1kwTGYlQ4wpZkMOJqMGCstVCFIvxCCVG4KfY1/3G0MhDAcZsip/Lw8/vJWw==", "engines": { "node": ">=14.0.0" } diff --git a/package.json b/package.json index 9781655..08489c4 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "react-native": "./dist/esm/src/main.js", "dependencies": { "@tbd54566975/dwn-sdk-js": "0.1.0", - "kysely": "0.25.0", + "kysely": "0.26.3", "multiformats": "12.0.1", "readable-stream": "4.4.2" }, From ffb5c1e0c0fc8c5a505a964aabcc632bc4e982e7 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Thu, 21 Sep 2023 10:43:54 -0400 Subject: [PATCH 2/9] upgrade sdk and queries, first pass -- tests fail --- package-lock.json | 113 +++++++++------------------------------ package.json | 2 +- src/message-store-sql.ts | 65 ++++++++++++++++++---- src/utils/filter.ts | 9 +++- 4 files changed, 88 insertions(+), 101 deletions(-) diff --git a/package-lock.json b/package-lock.json index ddc4db6..59dfb8b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "0.1.0", "license": "Apache-2.0", "dependencies": { - "@tbd54566975/dwn-sdk-js": "0.1.0", + "@tbd54566975/dwn-sdk-js": "0.2.2", "kysely": "0.26.3", "multiformats": "12.0.1", "readable-stream": "4.4.2" @@ -883,9 +883,9 @@ "dev": true }, "node_modules/@tbd54566975/dwn-sdk-js": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/@tbd54566975/dwn-sdk-js/-/dwn-sdk-js-0.1.0.tgz", - "integrity": "sha512-oH5s2P5855mIkPkbHeYuRWNgyQKU7nO6ccmtKYr0qMeYgIYUjBNpvIkgqllfx7ObsfLPU7myFYmlxqHAiFZGRA==", + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@tbd54566975/dwn-sdk-js/-/dwn-sdk-js-0.2.2.tgz", + "integrity": "sha512-n0bbi91GrsPCZTG5Z/iBwlzhKEVET6b9GT4T1cdGRT0CHGNQdW+jHrKhhNzBIVDF15z92VWOA9Ogyug2KJ7J4Q==", "dependencies": { "@ipld/dag-cbor": "9.0.3", "@js-temporal/polyfill": "0.4.4", @@ -894,7 +894,7 @@ "abstract-level": "1.0.3", "ajv": "8.12.0", "blockstore-core": "4.2.0", - "cross-fetch": "3.1.6", + "cross-fetch": "4.0.0", "eciesjs": "0.4.0", "flat": "5.0.2", "interface-blockstore": "5.2.3", @@ -908,8 +908,7 @@ "multiformats": "11.0.2", "randombytes": "2.1.0", "readable-stream": "4.4.0", - "secp256k1": "5.0.0", - "ulid": "2.3.0", + "ulidx": "2.1.0", "uuid": "8.3.2", "varint": "6.0.0" }, @@ -1633,11 +1632,6 @@ "npm": ">=7.0.0" } }, - "node_modules/bn.js": { - "version": "4.12.0", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", - "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" - }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -1660,11 +1654,6 @@ "node": ">=8" } }, - "node_modules/brorand": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz", - "integrity": "sha512-cKV8tMCEpQs4hK/ik71d6LrPOnpkpGBR0wzxqr68g2m/LB2GxVYQroAjMJZRVM1Y4BCjCKc3vAamxSzOY2RP+w==" - }, "node_modules/browser-level": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/browser-level/-/browser-level-1.0.1.tgz", @@ -1978,11 +1967,11 @@ "dev": true }, "node_modules/cross-fetch": { - "version": "3.1.6", - "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.6.tgz", - "integrity": "sha512-riRvo06crlE8HiqOwIpQhxwdOk4fOeR7FVM/wXoxchFEqMNUjvbs3bfo4OTgMEMHzppd4DxFBDbyySj8Cv781g==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.0.0.tgz", + "integrity": "sha512-e4a5N8lVvuLgAWgnCrLr2PP0YyDOTHa9H/Rj54dirp61qXnNq46m82bRhNqIA5VccJtWBvPTFRV3TtvHUKPB1g==", "dependencies": { - "node-fetch": "^2.6.11" + "node-fetch": "^2.6.12" } }, "node_modules/cross-spawn": { @@ -2139,20 +2128,6 @@ "@noble/curves": "^1.1.0" } }, - "node_modules/elliptic": { - "version": "6.5.4", - "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz", - "integrity": "sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ==", - "dependencies": { - "bn.js": "^4.11.9", - "brorand": "^1.1.0", - "hash.js": "^1.0.0", - "hmac-drbg": "^1.0.1", - "inherits": "^2.0.4", - "minimalistic-assert": "^1.0.1", - "minimalistic-crypto-utils": "^1.0.1" - } - }, "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", @@ -2784,15 +2759,6 @@ "node": ">=8" } }, - "node_modules/hash.js": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz", - "integrity": "sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==", - "dependencies": { - "inherits": "^2.0.3", - "minimalistic-assert": "^1.0.1" - } - }, "node_modules/he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", @@ -2802,16 +2768,6 @@ "he": "bin/he" } }, - "node_modules/hmac-drbg": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", - "integrity": "sha512-Tti3gMqLdZfhOQY1Mzf/AanLiqh1WTiJgEj26ZuYQ9fbkLomzGchCws4FyrSd4VkpBfiNhaE1On+lOz894jvXg==", - "dependencies": { - "hash.js": "^1.0.3", - "minimalistic-assert": "^1.0.0", - "minimalistic-crypto-utils": "^1.0.1" - } - }, "node_modules/html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", @@ -3339,6 +3295,11 @@ "node": ">=14.0.0" } }, + "node_modules/layerr": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/layerr/-/layerr-2.0.1.tgz", + "integrity": "sha512-z0730CwG/JO24evdORnyDkwG1Q7b7mF2Tp1qRQ0YvrMMARbt1DFG694SOv439Gm7hYKolyZyaB49YIrYIfZBdg==" + }, "node_modules/level": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/level/-/level-8.0.0.tgz", @@ -3516,16 +3477,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/minimalistic-assert": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", - "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==" - }, - "node_modules/minimalistic-crypto-utils": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz", - "integrity": "sha512-JIYlbt6g8i5jKfJ3xz7rF0LXmv2TkDxBLUkiBeZ7bAx4GnnNMr8xFpGnOxn6GhTEHx3SjRrZEoU+j04prX1ktg==" - }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -3776,11 +3727,6 @@ "node": ">=10" } }, - "node_modules/node-addon-api": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz", - "integrity": "sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA==" - }, "node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", @@ -4540,20 +4486,6 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "dev": true }, - "node_modules/secp256k1": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/secp256k1/-/secp256k1-5.0.0.tgz", - "integrity": "sha512-TKWX8xvoGHrxVdqbYeZM9w+izTF4b9z3NhSaDkdn81btvuh+ivbIMGT/zQvDtTFWhRlThpoz6LEYTr7n8A5GcA==", - "hasInstallScript": true, - "dependencies": { - "elliptic": "^6.5.4", - "node-addon-api": "^5.0.0", - "node-gyp-build": "^4.2.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, "node_modules/semver": { "version": "7.5.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.3.tgz", @@ -5074,12 +5006,15 @@ "multiformats": "^12.0.1" } }, - "node_modules/ulid": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/ulid/-/ulid-2.3.0.tgz", - "integrity": "sha512-keqHubrlpvT6G2wH0OEfSW4mquYRcbe/J8NMmveoQOjUqmo+hXtO+ORCpWhdbZ7k72UtY61BL7haGxW6enBnjw==", - "bin": { - "ulid": "bin/cli.js" + "node_modules/ulidx": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/ulidx/-/ulidx-2.1.0.tgz", + "integrity": "sha512-DlMi97oP9HASI3kLCjBlOhAG1SoisUrEqC2PJ7itiFbq9q5Zo0JejupXeu2Gke99W62epNzA4MFNToNiq8A5LA==", + "dependencies": { + "layerr": "^2.0.1" + }, + "engines": { + "node": ">=16" } }, "node_modules/uri-js": { diff --git a/package.json b/package.json index 08489c4..f95b7c5 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ }, "react-native": "./dist/esm/src/main.js", "dependencies": { - "@tbd54566975/dwn-sdk-js": "0.1.0", + "@tbd54566975/dwn-sdk-js": "0.2.2", "kysely": "0.26.3", "multiformats": "12.0.1", "readable-stream": "4.4.2" diff --git a/src/message-store-sql.ts b/src/message-store-sql.ts index e5a51b2..dab9bee 100644 --- a/src/message-store-sql.ts +++ b/src/message-store-sql.ts @@ -1,4 +1,10 @@ -import { executeUnlessAborted, Filter, GenericMessage, MessageStore, MessageStoreOptions } from '@tbd54566975/dwn-sdk-js'; +import { + executeUnlessAborted, + Filter, + GenericMessage, + MessageStore, + MessageStoreOptions, +} from '@tbd54566975/dwn-sdk-js'; import { Kysely } from 'kysely'; import { Database } from './database.js'; import * as block from 'multiformats/block'; @@ -138,11 +144,27 @@ export class MessageStoreSql implements MessageStore { return this.parseEncodedMessage(result.encodedMessageBytes, options); } + private getOrderBy( + messageSort?: MessageSort + ):{ property: 'dateCreated' | 'datePublished' | 'messageTimestamp', direction: SortOrder } { + if(messageSort?.dateCreated !== undefined) { + return { property: 'dateCreated', direction: messageSort.dateCreated }; + } else if(messageSort?.datePublished !== undefined) { + return { property: 'datePublished', direction: messageSort.datePublished }; + } else if (messageSort?.messageTimestamp !== undefined) { + return { property: 'messageTimestamp', direction: messageSort.messageTimestamp }; + } else { + return { property: 'messageTimestamp', direction: SortOrder.Ascending }; + } + } + async query( tenant: string, - filter: Filter, + filters: Filter[], + messageSort?: MessageSort, + pagination?: Pagination, options?: MessageStoreOptions - ): Promise { + ): Promise<{ messages: GenericMessage[], paginationMessageCid?: string }> { if (!this.#db) { throw new Error( 'Connection to database not open. Call `open` before using `query`.' @@ -151,23 +173,32 @@ export class MessageStoreSql implements MessageStore { options?.signal?.throwIfAborted(); + const {property, direction} = this.getOrderBy(messageSort); let query = this.#db .selectFrom('messageStore') .selectAll() .where('tenant', '=', tenant); - query = filterSelectQuery(filter, query); + query = filterSelectQuery(filters, query); + query = query + .orderBy(property, direction === SortOrder.Ascending ? 'asc' : 'desc') + .orderBy('messageCid', 'desc'); + + if (pagination?.limit !== undefined) { + query = query.limit(pagination.limit); + } const results = await executeUnlessAborted( query.execute(), options?.signal ); - const messages = results.map(async (result) => { - return this.parseEncodedMessage(result.encodedMessageBytes, options); - }); + const messages:GenericMessage[] = []; + for (const result of results) { + messages.push(await this.parseEncodedMessage(result.encodedMessageBytes, options)); + } - return await Promise.all(messages); + return { messages: (await Promise.all(messages)) }; } async delete( @@ -221,4 +252,20 @@ export class MessageStoreSql implements MessageStore { return message; } -} \ No newline at end of file +} + +export type Pagination = { + messageCid?: string + limit?: number +}; + +export enum SortOrder { + Descending = -1, + Ascending = 1 +} + +export type MessageSort = { + dateCreated?: SortOrder; + datePublished?: SortOrder; + messageTimestamp?: SortOrder; +}; \ No newline at end of file diff --git a/src/utils/filter.ts b/src/utils/filter.ts index 5488ce7..173c795 100644 --- a/src/utils/filter.ts +++ b/src/utils/filter.ts @@ -3,13 +3,17 @@ import { DynamicModule, SelectQueryBuilder } from 'kysely'; import { sanitizedString } from './sanitize.js'; export function filterSelectQuery( - filter: Filter, + filters: Filter[], query: SelectQueryBuilder ): SelectQueryBuilder { + if (filters.length === 0) { + return query; + } + const filter = filters[0]; + // for (let filter of filters) { for (let property in filter) { const value = filter[property]; const column = new DynamicModule().ref(property); - if (Array.isArray(value)) { // OneOfFilter query = query.where(column, 'in', value); } else if (typeof value === 'object') { // RangeFilter @@ -29,6 +33,7 @@ export function filterSelectQuery', + eb.selectFrom('messageStore').select('messageTimestamp').where('messageCid', '=', pagination.messageCid!).limit(1), + ) + ); + } + query = filterSelectQuery(filters, query); query = query .orderBy(property, direction === SortOrder.Ascending ? 'asc' : 'desc') .orderBy('messageCid', 'desc'); - if (pagination?.limit !== undefined) { - query = query.limit(pagination.limit); + if (pagination?.limit !== undefined && pagination?.limit > 0) { + // we grab one additional record to decide if we return a cursor or not. + query = query.limit(pagination.limit + 1); } const results = await executeUnlessAborted( query.execute(), options?.signal ); - const messages:GenericMessage[] = []; for (const result of results) { messages.push(await this.parseEncodedMessage(result.encodedMessageBytes, options)); } + let paginationMessageCid: string|undefined; + if (pagination?.limit !== undefined && messages.length > pagination.limit) { + messages.splice(messages.length - 1); + const lastMessage = messages.at(-1)!; + paginationMessageCid = await Message.getCid(lastMessage); + } - return { messages: (await Promise.all(messages)) }; + return { messages, paginationMessageCid }; } async delete( From 42e5afb73e31e2059027de9ef783af209e54c8ea Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Fri, 22 Sep 2023 15:31:55 -0400 Subject: [PATCH 4/9] store encodedData when available, sort and paginate according to messageCid cusror --- package-lock.json | 47 ++++++++++--- package.json | 1 + src/database.ts | 1 + src/message-store-sql.ts | 146 +++++++++++++++++++++++++-------------- src/utils/filter.ts | 59 ++++++++++++---- 5 files changed, 180 insertions(+), 74 deletions(-) diff --git a/package-lock.json b/package-lock.json index 59dfb8b..e3d5311 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.1.0", "license": "Apache-2.0", "dependencies": { + "@ipld/dag-cbor": "^9.0.5", "@tbd54566975/dwn-sdk-js": "0.2.2", "kysely": "0.26.3", "multiformats": "12.0.1", @@ -525,11 +526,11 @@ "dev": true }, "node_modules/@ipld/dag-cbor": { - "version": "9.0.3", - "resolved": "https://registry.npmjs.org/@ipld/dag-cbor/-/dag-cbor-9.0.3.tgz", - "integrity": "sha512-A2UFccS0+sARK9xwXiVZIaWbLbPxLGP3UZOjBeOMWfDY04SXi8h1+t4rHBzOlKYF/yWNm3RbFLyclWO7hZcy4g==", + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/@ipld/dag-cbor/-/dag-cbor-9.0.5.tgz", + "integrity": "sha512-TyqgtxEojc98rvxg4NGM+73JzQeM4+tK2VQes/in2mdyhO+1wbGuBijh1tvi9BErQ/dEblxs9v4vEQSX8mFCIw==", "dependencies": { - "cborg": "^2.0.1", + "cborg": "^4.0.0", "multiformats": "^12.0.1" }, "engines": { @@ -916,6 +917,36 @@ "node": ">= 18" } }, + "node_modules/@tbd54566975/dwn-sdk-js/node_modules/@ipld/dag-cbor": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/@ipld/dag-cbor/-/dag-cbor-9.0.3.tgz", + "integrity": "sha512-A2UFccS0+sARK9xwXiVZIaWbLbPxLGP3UZOjBeOMWfDY04SXi8h1+t4rHBzOlKYF/yWNm3RbFLyclWO7hZcy4g==", + "dependencies": { + "cborg": "^2.0.1", + "multiformats": "^12.0.1" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@tbd54566975/dwn-sdk-js/node_modules/@ipld/dag-cbor/node_modules/multiformats": { + "version": "12.1.1", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.1.tgz", + "integrity": "sha512-GBSToTmri2vJYs8wqcZQ8kB21dCaeTOzHTIAlr8J06C1eL6UbzqURXFZ5Fl0EYm9GAFz1IlYY8SxGOs9G9NJRg==", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@tbd54566975/dwn-sdk-js/node_modules/cborg": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/cborg/-/cborg-2.0.5.tgz", + "integrity": "sha512-xVW1rSIw1ZXbkwl2XhJ7o/jAv0vnVoQv/QlfQxV8a7V5PlA4UU/AcIiXqmpyybwNWy/GPQU1m/aBVNIWr7/T0w==", + "bin": { + "cborg": "cli.js" + } + }, "node_modules/@tbd54566975/dwn-sdk-js/node_modules/multiformats": { "version": "11.0.2", "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", @@ -1802,11 +1833,11 @@ } }, "node_modules/cborg": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/cborg/-/cborg-2.0.5.tgz", - "integrity": "sha512-xVW1rSIw1ZXbkwl2XhJ7o/jAv0vnVoQv/QlfQxV8a7V5PlA4UU/AcIiXqmpyybwNWy/GPQU1m/aBVNIWr7/T0w==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/cborg/-/cborg-4.0.3.tgz", + "integrity": "sha512-poLvpK30KT5KI8gzDx3J/IuVCbsLqMT2fEbOrOuX0H7Hyj8yg5LezeWhRh9aLa5Z6MfPC5sriW3HVJF328M8LQ==", "bin": { - "cborg": "cli.js" + "cborg": "lib/bin.js" } }, "node_modules/chai": { diff --git a/package.json b/package.json index f95b7c5..1387012 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ }, "react-native": "./dist/esm/src/main.js", "dependencies": { + "@ipld/dag-cbor": "^9.0.5", "@tbd54566975/dwn-sdk-js": "0.2.2", "kysely": "0.26.3", "multiformats": "12.0.1", diff --git a/src/database.ts b/src/database.ts index 7967e32..5159162 100644 --- a/src/database.ts +++ b/src/database.ts @@ -11,6 +11,7 @@ export interface MessageStoreTable { tenant: string; messageCid: string; encodedMessageBytes: Uint8Array; + encodedData: string | null; // "indexes" start interface: string | null; method: string | null; diff --git a/src/message-store-sql.ts b/src/message-store-sql.ts index eea3870..751aa14 100644 --- a/src/message-store-sql.ts +++ b/src/message-store-sql.ts @@ -1,10 +1,15 @@ import { + DwnInterfaceName, + DwnMethodName, executeUnlessAborted, Filter, GenericMessage, Message, MessageStore, MessageStoreOptions, + MessageSort, + Pagination, + SortOrder } from '@tbd54566975/dwn-sdk-js'; import { Kysely } from 'kysely'; import { Database } from './database.js'; @@ -35,6 +40,7 @@ export class MessageStoreSql implements MessageStore { .ifNotExists() .addColumn('tenant', 'text', (col) => col.notNull()) .addColumn('messageCid', 'varchar(60)', (col) => col.notNull()) + .addColumn('encodedData', 'text') // we optionally store encoded data if it is below a threshold // "indexes" start .addColumn('interface', 'text') .addColumn('method', 'text') @@ -92,8 +98,23 @@ export class MessageStoreSql implements MessageStore { options?.signal?.throwIfAborted(); + // gets the encoded data and removes it from the message + const getEncodedData = (message: GenericMessage): { message: GenericMessage, encodedData: string|null} => { + let encodedData: string|null = null; + if (message.descriptor.interface === DwnInterfaceName.Records && message.descriptor.method === DwnMethodName.Write) { + const data = (message as any).encodedData as string|undefined; + if(data) { + delete (message as any).encodedData; + encodedData = data; + } + } + return { message, encodedData }; + }; + + const { message: messageToProcess, encodedData} = getEncodedData(message); + const encodedMessageBlock = await executeUnlessAborted( - block.encode({ value: message, codec: cbor, hasher: sha256}), + block.encode({ value: messageToProcess, codec: cbor, hasher: sha256}), options?.signal ); @@ -101,6 +122,7 @@ export class MessageStoreSql implements MessageStore { const encodedMessageBytes = Buffer.from(encodedMessageBlock.bytes); sanitizeRecords(indexes); + await executeUnlessAborted( this.#db .insertInto('messageStore') @@ -108,7 +130,8 @@ export class MessageStoreSql implements MessageStore { tenant, messageCid, encodedMessageBytes, - ...indexes, + encodedData, + ...indexes }) .executeTakeFirstOrThrow(), options?.signal @@ -142,21 +165,7 @@ export class MessageStoreSql implements MessageStore { return undefined; } - return this.parseEncodedMessage(result.encodedMessageBytes, options); - } - - private getOrderBy( - messageSort?: MessageSort - ):{ property: 'dateCreated' | 'datePublished' | 'messageTimestamp', direction: SortOrder } { - if(messageSort?.dateCreated !== undefined) { - return { property: 'dateCreated', direction: messageSort.dateCreated }; - } else if(messageSort?.datePublished !== undefined) { - return { property: 'datePublished', direction: messageSort.datePublished }; - } else if (messageSort?.messageTimestamp !== undefined) { - return { property: 'messageTimestamp', direction: messageSort.messageTimestamp }; - } else { - return { property: 'messageTimestamp', direction: SortOrder.Ascending }; - } + return this.parseEncodedMessage(result.encodedMessageBytes, result.encodedData, options); } async query( @@ -174,24 +183,40 @@ export class MessageStoreSql implements MessageStore { options?.signal?.throwIfAborted(); - const {property, direction} = this.getOrderBy(messageSort); let query = this.#db .selectFrom('messageStore') .selectAll() .where('tenant', '=', tenant); - if(pagination?.messageCid !== undefined) { - query = query.where(eb => - eb.cmpr('messageTimestamp', '>', - eb.selectFrom('messageStore').select('messageTimestamp').where('messageCid', '=', pagination.messageCid!).limit(1), - ) - ); + // if query is sorted by date published, only show records which are published + if(messageSort?.datePublished !== undefined) { + query = query.where('published', '=', 'true'); } query = filterSelectQuery(filters, query); + const { property: sortProperty, direction: sortDirection } = this.getOrderBy(messageSort); + + if(pagination?.messageCid !== undefined) { + + const messageCid = pagination.messageCid; + query = query.where(({ eb, selectFrom, refTuple }) => { + const direction = sortDirection === SortOrder.Ascending ? '>' : '<'; + + // fetches the sort property tuple from the database based on the messageCid. + const cursor = selectFrom('messageStore') + .select([sortProperty, 'messageCid']) + .where('tenant', '=', tenant) + .where('messageCid', '=', messageCid) + .limit(1).$asTuple(sortProperty, 'messageCid'); + + return eb(refTuple(sortProperty, 'messageCid'), direction, cursor); + }); + } + + // sorting by the provided sort property, the tiebreak is always in ascending order regardless of sort query = query - .orderBy(property, direction === SortOrder.Ascending ? 'asc' : 'desc') - .orderBy('messageCid', 'desc'); + .orderBy(sortProperty, sortDirection === SortOrder.Ascending ? 'asc' : 'desc') + .orderBy('messageCid', 'asc'); if (pagination?.limit !== undefined && pagination?.limit > 0) { // we grab one additional record to decide if we return a cursor or not. @@ -202,18 +227,11 @@ export class MessageStoreSql implements MessageStore { query.execute(), options?.signal ); - const messages:GenericMessage[] = []; - for (const result of results) { - messages.push(await this.parseEncodedMessage(result.encodedMessageBytes, options)); - } - let paginationMessageCid: string|undefined; - if (pagination?.limit !== undefined && messages.length > pagination.limit) { - messages.splice(messages.length - 1); - const lastMessage = messages.at(-1)!; - paginationMessageCid = await Message.getCid(lastMessage); - } - return { messages, paginationMessageCid }; + const messages: Promise[] = results.map((r:any) => this.parseEncodedMessage(r.encodedMessageBytes, r.encodedData, options)); + + // returns the pruned the messages and a potential paginationMessageCid + return this.getPaginationResults(messages, pagination?.limit); } async delete( @@ -253,6 +271,7 @@ export class MessageStoreSql implements MessageStore { private async parseEncodedMessage( encodedMessageBytes: Uint8Array, + encodedData: string | null | undefined, options?: MessageStoreOptions ): Promise { options?.signal?.throwIfAborted(); @@ -264,23 +283,46 @@ export class MessageStoreSql implements MessageStore { }); const message = decodedBlock.value as GenericMessage; + if (message !== undefined && encodedData !== undefined && encodedData !== null) { + (message as any).encodedData = encodedData; + } return message; } -} - -export type Pagination = { - messageCid?: string - limit?: number -}; + /** + * Gets the pagination Message Cid if there is messages to paginate. Accepts more messages than the limit. + * + * @param messages a list of messages, potentially larger than the provided limit. + * @param limit the maximum number of messages to be returned + * + * @returns the pruned message results and an optional paginationMessageCid + */ + private async getPaginationResults( + messages: Promise[], limit?: number + ): Promise<{ messages: GenericMessage[], paginationMessageCid?: string }>{ + if (limit !== undefined && messages.length > limit) { + messages = messages.slice(0, limit); + const lastMessage = messages.at(-1); + return { + messages : await Promise.all(messages), + paginationMessageCid : lastMessage ? await Message.getCid(await lastMessage) : undefined + }; + } -export enum SortOrder { - Descending = -1, - Ascending = 1 -} + return { messages: await Promise.all(messages) }; + } -export type MessageSort = { - dateCreated?: SortOrder; - datePublished?: SortOrder; - messageTimestamp?: SortOrder; -}; \ No newline at end of file + private getOrderBy( + messageSort?: MessageSort + ):{ property: 'dateCreated' | 'datePublished' | 'messageTimestamp', direction: SortOrder } { + if(messageSort?.dateCreated !== undefined) { + return { property: 'dateCreated', direction: messageSort.dateCreated }; + } else if(messageSort?.datePublished !== undefined) { + return { property: 'datePublished', direction: messageSort.datePublished }; + } else if (messageSort?.messageTimestamp !== undefined) { + return { property: 'messageTimestamp', direction: messageSort.messageTimestamp }; + } else { + return { property: 'messageTimestamp', direction: SortOrder.Ascending }; + } + } +} \ No newline at end of file diff --git a/src/utils/filter.ts b/src/utils/filter.ts index 173c795..2c2ffc7 100644 --- a/src/utils/filter.ts +++ b/src/utils/filter.ts @@ -1,39 +1,70 @@ import { Filter } from '@tbd54566975/dwn-sdk-js'; -import { DynamicModule, SelectQueryBuilder } from 'kysely'; +import { DynamicModule, ExpressionBuilder, OperandExpression, SelectQueryBuilder, SqlBool } from 'kysely'; import { sanitizedString } from './sanitize.js'; +/** + * Takes multiple Filters and returns a single query. + * Each filter is evaluated as an OR operation. + * + * @param filters Array of filters to be evaluated as OR operations + * @param query the QueryBuilder. + * @returns The modified QueryBuilder. + */ export function filterSelectQuery( filters: Filter[], query: SelectQueryBuilder ): SelectQueryBuilder { - if (filters.length === 0) { - return query; - } - const filter = filters[0]; - // for (let filter of filters) { + return query.where((eb) => { + // we are building multiple OR queries out of each individual filter. + const or: OperandExpression[] = []; + for (let filter of filters) { + // processFilter will take a single filter and create AND operations from each property. + // if an array of values are passed for a property, it will treat it as an OR(IN) within the individual filter property. + // it returns a single OperandExpression to be evaluated. + or.push(processFilter(eb, filter)); + } + // Evaluate the array of expressions as an OR operation. + return eb.or(or); + }); +} + +/** + * Returns an array of OperandExpressions for a single filter. + * Each property within the filter is evaluated as an AND operand. + * This way each Filer has to be a complete match, but the collection of filters can be evaluated as chosen. + * + * @param eb The ExpressionBuilder from the query. + * @param filter The filter to be evaluated. + * @returns An array of OperandExpressions to be evaluated by the caller. + */ +function processFilter( + eb: ExpressionBuilder, + filter: Filter +):OperandExpression { + const andOperands: OperandExpression[] = []; for (let property in filter) { const value = filter[property]; const column = new DynamicModule().ref(property); if (Array.isArray(value)) { // OneOfFilter - query = query.where(column, 'in', value); + andOperands.push(eb(column, 'in', value)); } else if (typeof value === 'object') { // RangeFilter if (value.gt) { - query = query.where(column, '>', sanitizedString(value.gt)); + andOperands.push(eb(column, '>', sanitizedString(value.gt))); } if (value.gte) { - query = query.where(column, '>=', sanitizedString(value.gte)); + andOperands.push(eb(column, '>=', sanitizedString(value.gte))); } if (value.lt) { - query = query.where(column, '<', sanitizedString(value.lt)); + andOperands.push(eb(column, '<', sanitizedString(value.lt))); } if (value.lte) { - query = query.where(column, '<=', sanitizedString(value.lte)); + andOperands.push(eb(column, '<=', sanitizedString(value.lte))); } } else { // EqualFilter - query = query.where(column, '=', sanitizedString(value)); + andOperands.push(eb(column, '=', sanitizedString(value))); } } - // } - return query; + // evaluate the the collected operands as an AND operation. + return eb.and(andOperands); } \ No newline at end of file From 0b3bbb4cfe6347e5d22b363b90d4855cb0cfbf83 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Tue, 26 Sep 2023 09:40:23 -0400 Subject: [PATCH 5/9] added more comments --- src/message-store-sql.ts | 18 +++++++++++++----- src/utils/filter.ts | 11 +++++------ 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/message-store-sql.ts b/src/message-store-sql.ts index 751aa14..70faec7 100644 --- a/src/message-store-sql.ts +++ b/src/message-store-sql.ts @@ -193,22 +193,25 @@ export class MessageStoreSql implements MessageStore { query = query.where('published', '=', 'true'); } + // add filters to query query = filterSelectQuery(filters, query); + + // extract sort property and direction from the supplied messageSort const { property: sortProperty, direction: sortDirection } = this.getOrderBy(messageSort); if(pagination?.messageCid !== undefined) { - const messageCid = pagination.messageCid; query = query.where(({ eb, selectFrom, refTuple }) => { const direction = sortDirection === SortOrder.Ascending ? '>' : '<'; - // fetches the sort property tuple from the database based on the messageCid. + // fetches the cursor as a sort property tuple from the database based on the messageCid. const cursor = selectFrom('messageStore') .select([sortProperty, 'messageCid']) .where('tenant', '=', tenant) .where('messageCid', '=', messageCid) .limit(1).$asTuple(sortProperty, 'messageCid'); + // https://kysely-org.github.io/kysely-apidoc/interfaces/ExpressionBuilder.html#refTuple return eb(refTuple(sortProperty, 'messageCid'), direction, cursor); }); } @@ -219,7 +222,7 @@ export class MessageStoreSql implements MessageStore { .orderBy('messageCid', 'asc'); if (pagination?.limit !== undefined && pagination?.limit > 0) { - // we grab one additional record to decide if we return a cursor or not. + // we query for one additional record to decide if we return a pagination cursor or not. query = query.limit(pagination.limit + 1); } @@ -228,9 +231,10 @@ export class MessageStoreSql implements MessageStore { options?.signal ); + // extracts the full encoded message from the stored blob for each result item. const messages: Promise[] = results.map((r:any) => this.parseEncodedMessage(r.encodedMessageBytes, r.encodedData, options)); - // returns the pruned the messages and a potential paginationMessageCid + // returns the pruned the messages, since we have and additional record from above, and a potential paginationMessageCid return this.getPaginationResults(messages, pagination?.limit); } @@ -283,6 +287,9 @@ export class MessageStoreSql implements MessageStore { }); const message = decodedBlock.value as GenericMessage; + // If encodedData is stored within the MessageStore we include it in the response. + // We store encodedData when the data is below a certain threshold. + // https://github.com/TBD54566975/dwn-sdk-js/pull/456 if (message !== undefined && encodedData !== undefined && encodedData !== null) { (message as any).encodedData = encodedData; } @@ -290,7 +297,8 @@ export class MessageStoreSql implements MessageStore { } /** - * Gets the pagination Message Cid if there is messages to paginate. Accepts more messages than the limit. + * Gets the pagination Message Cid if there are additional messages to paginate. + * Accepts more messages than the limit, as we query for additional records to check if we should paginate. * * @param messages a list of messages, potentially larger than the provided limit. * @param limit the maximum number of messages to be returned diff --git a/src/utils/filter.ts b/src/utils/filter.ts index 2c2ffc7..56e4485 100644 --- a/src/utils/filter.ts +++ b/src/utils/filter.ts @@ -7,8 +7,8 @@ import { sanitizedString } from './sanitize.js'; * Each filter is evaluated as an OR operation. * * @param filters Array of filters to be evaluated as OR operations - * @param query the QueryBuilder. - * @returns The modified QueryBuilder. + * @param query the incoming QueryBuilder. + * @returns The modified QueryBuilder respecting the provided filters. */ export function filterSelectQuery( filters: Filter[], @@ -18,9 +18,7 @@ export function filterSelectQuery[] = []; for (let filter of filters) { - // processFilter will take a single filter and create AND operations from each property. - // if an array of values are passed for a property, it will treat it as an OR(IN) within the individual filter property. - // it returns a single OperandExpression to be evaluated. + // processFilter will take a single filter adding it to the query to be evaluated as an OR operation with the other filters. or.push(processFilter(eb, filter)); } // Evaluate the array of expressions as an OR operation. @@ -30,7 +28,8 @@ export function filterSelectQuery Date: Tue, 26 Sep 2023 11:49:55 -0400 Subject: [PATCH 6/9] added comment about removing encodedData from message before encoding for storage --- src/message-store-sql.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/message-store-sql.ts b/src/message-store-sql.ts index 70faec7..38aa6e5 100644 --- a/src/message-store-sql.ts +++ b/src/message-store-sql.ts @@ -99,6 +99,8 @@ export class MessageStoreSql implements MessageStore { options?.signal?.throwIfAborted(); // gets the encoded data and removes it from the message + // we remove it from the message as it would cause the `encodedMessageBytes` to be greater than the + // maximum bytes allowed by SQL const getEncodedData = (message: GenericMessage): { message: GenericMessage, encodedData: string|null} => { let encodedData: string|null = null; if (message.descriptor.interface === DwnInterfaceName.Records && message.descriptor.method === DwnMethodName.Write) { From dbcf1959e04c580087b71399bac2b7639b0c1895 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Tue, 26 Sep 2023 12:45:00 -0400 Subject: [PATCH 7/9] bump version --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index e3d5311..3c6aa45 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "license": "Apache-2.0", "dependencies": { "@ipld/dag-cbor": "^9.0.5", - "@tbd54566975/dwn-sdk-js": "0.2.2", + "@tbd54566975/dwn-sdk-js": "0.2.3", "kysely": "0.26.3", "multiformats": "12.0.1", "readable-stream": "4.4.2" @@ -884,9 +884,9 @@ "dev": true }, "node_modules/@tbd54566975/dwn-sdk-js": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/@tbd54566975/dwn-sdk-js/-/dwn-sdk-js-0.2.2.tgz", - "integrity": "sha512-n0bbi91GrsPCZTG5Z/iBwlzhKEVET6b9GT4T1cdGRT0CHGNQdW+jHrKhhNzBIVDF15z92VWOA9Ogyug2KJ7J4Q==", + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@tbd54566975/dwn-sdk-js/-/dwn-sdk-js-0.2.3.tgz", + "integrity": "sha512-T3Yy6kY6zftdVgsX2C0D2bIAmWQQVCFrLB95+BN/zoAAA29LogOr807Kx15QHrKmILWidVfYt/ZwsPHl4k5bDQ==", "dependencies": { "@ipld/dag-cbor": "9.0.3", "@js-temporal/polyfill": "0.4.4", diff --git a/package.json b/package.json index 1387012..4464739 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "react-native": "./dist/esm/src/main.js", "dependencies": { "@ipld/dag-cbor": "^9.0.5", - "@tbd54566975/dwn-sdk-js": "0.2.2", + "@tbd54566975/dwn-sdk-js": "0.2.3", "kysely": "0.26.3", "multiformats": "12.0.1", "readable-stream": "4.4.2" From 210a221092bd6bdf2fa3657949c63a8cb9db3b87 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Tue, 26 Sep 2023 14:48:30 -0400 Subject: [PATCH 8/9] new release version --- package.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 4464739..3ed9f0b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tbd54566975/dwn-sql-store", - "version": "0.1.0", + "version": "0.2.0", "description": "SQL backed implementations of DWN MessageStore, DataStore, and EventLog", "type": "module", "license": "Apache-2.0", @@ -78,6 +78,10 @@ { "name": "Adam Leos", "url": "https://github.com/adam4leos" + }, + { + "name": "Liran Cohen", + "url": "https://github.com/lirancohen" } ] } From 63c2b132fb9371cec6f135ba18c98c84c21d4b48 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Tue, 26 Sep 2023 15:18:41 -0400 Subject: [PATCH 9/9] npm i after version bump --- package-lock.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3c6aa45..64f8e12 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@tbd54566975/dwn-sql-store", - "version": "0.1.0", + "version": "0.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@tbd54566975/dwn-sql-store", - "version": "0.1.0", + "version": "0.2.0", "license": "Apache-2.0", "dependencies": { "@ipld/dag-cbor": "^9.0.5",