From dce370dd8dbab6aa31d046a4340cdbf173fff4eb Mon Sep 17 00:00:00 2001 From: LiranCohen Date: Wed, 29 Nov 2023 14:16:43 -0500 Subject: [PATCH] Support RecordsQuery pagination in Web5 (#268) * Add pagination messageCid to RecordsQueryResponse and tests supporting pagination/sort * cursor instead of paginationsMessagecid --------- Signed-off-by: Frank Hinek Co-authored-by: Frank Hinek --- README.md | 14 ++++ packages/agent/src/sync-manager.ts | 15 ++-- packages/api/README.md | 14 ++++ packages/api/src/dwn-api.ts | 9 +- packages/api/tests/dwn-api.spec.ts | 130 +++++++++++++++++++++++++++++ 5 files changed, 173 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index a324672ff..8da44abbc 100644 --- a/README.md +++ b/README.md @@ -224,6 +224,20 @@ The query `request` contains the following properties: - **`recipient`** - _`string`_ (_optional_): the DID in the `recipient` field of the record. - **`schema`** - _`URI string`_ (_optional_): the URI of the schema bucket in which to query. - **`dataFormat`** - _`Media Type string`_ (_optional_): the IANA string corresponding with the format of the data to filter for. See IANA's Media Type list here: https://www.iana.org/assignments/media-types/media-types.xhtml + - **`dateSort`** - _`DateSort`_ (_optional_): the `DateSort` value of the date field and direction to sort records by. Defaults to `CreatedAscending`. + - **`pagination`** - _`object`_ (_optional_): the properties used to paginate results. + - **`limit`** - _`number`_ (_optional_): the number of records that should be returned with this query. `undefined` returns all records. + - **`cursor`** - _`messageCid string`_ (_optional_): the `messageCid` of the records toc continue paginating from. This value is returned as a `cursor` in the response object of a `query` if there are more results beyond the `limit`. + +#### **Response** + +The query `response` contains the following properties: + +- **`status`** - _`object`_: the status of the `request`: + - **`code`** - _`number`_: the `Response Status` code, following the response code patterns for `HTTP Response Status Codes`. + - **`detail`** _`string`_: a detailed message describing the response. +- **`records`** - _`Records array`_ (_optional_): the array of `Records` returned if the request was successful. +- **`cursor`** - _`messageCid string`_ (_optional_): the `messageCid` of the last message returned in the results if there are exist additional records beyond the specified `limit` in the `query`. ### **`web5.dwn.records.create(request)`** diff --git a/packages/agent/src/sync-manager.ts b/packages/agent/src/sync-manager.ts index 4d1e8d8b6..edf5cb5bc 100644 --- a/packages/agent/src/sync-manager.ts +++ b/packages/agent/src/sync-manager.ts @@ -1,17 +1,20 @@ -import { DataStream } from '@tbd54566975/dwn-sdk-js'; -import { Convert } from '@web5/common'; -import { utils as didUtils } from '@web5/dids'; -import { Level } from 'level'; -import { webReadableToIsomorphicNodeReadable } from './utils.js'; +import type { AbstractBatchOperation, AbstractLevel } from 'abstract-level'; import type { EventsGetReply, GenericMessage, MessagesGetReply, RecordsWriteMessage, } from '@tbd54566975/dwn-sdk-js'; -import type { AbstractBatchOperation, AbstractLevel } from 'abstract-level'; + +import { Level } from 'level'; +import { Convert } from '@web5/common'; +import { utils as didUtils } from '@web5/dids'; +import { DataStream } from '@tbd54566975/dwn-sdk-js'; + import type { Web5ManagedAgent } from './types/agent.js'; +import { webReadableToIsomorphicNodeReadable } from './utils.js'; + export interface SyncManager { agent: Web5ManagedAgent; registerIdentity(options: { did: string }): Promise; diff --git a/packages/api/README.md b/packages/api/README.md index f1921e21f..45b122ce9 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -217,6 +217,20 @@ The query `request` contains the following properties: - **`recipient`** - _`string`_ (_optional_): the DID in the `recipient` field of the record. - **`schema`** - _`URI string`_ (_optional_): the URI of the schema bucket in which to query. - **`dataFormat`** - _`Media Type string`_ (_optional_): the IANA string corresponding with the format of the data to filter for. See IANA's Media Type list here: https://www.iana.org/assignments/media-types/media-types.xhtml + - **`dateSort`** - _`DateSort`_ (_optional_): the `DateSort` value of the date field and direction to sort records by. Defaults to `CreatedAscending`. + - **`pagination`** - _`object`_ (_optional_): the properties used to paginate results. + - **`limit`** - _`number`_ (_optional_): the number of records that should be returned with this query. `undefined` returns all records. + - **`cursor`** - _`messageCid string`_ (_optional_): the `messageCid` of the records toc continue paginating from. This value is returned as a `cursor` in the response object of a `query` if there are more results beyond the `limit`. + +#### **Response** + +The query `response` contains the following properties: + +- **`status`** - _`object`_: the status of the `request`: + - **`code`** - _`number`_: the `Response Status` code, following the response code patterns for `HTTP Response Status Codes`. + - **`detail`** _`string`_: a detailed message describing the response. +- **`records`** - _`Records array`_ (_optional_): the array of `Records` returned if the request was successful. +- **`cursor`** - _`messageCid string`_ (_optional_): the `messageCid` of the last message returned in the results if there are exist additional records beyond the specified `limit` in the `query`. ### **`web5.dwn.records.create(request)`** diff --git a/packages/api/src/dwn-api.ts b/packages/api/src/dwn-api.ts index 359b8050e..c28ec50ec 100644 --- a/packages/api/src/dwn-api.ts +++ b/packages/api/src/dwn-api.ts @@ -16,8 +16,8 @@ import { isEmptyObject } from '@web5/common'; import { DwnInterfaceName, DwnMethodName, RecordsWrite } from '@tbd54566975/dwn-sdk-js'; import { Record } from './record.js'; -import { Protocol } from './protocol.js'; import { dataToBlob } from './utils.js'; +import { Protocol } from './protocol.js'; /** * Status code and detailed message for a response. @@ -131,6 +131,9 @@ export type RecordsQueryRequest = { */ export type RecordsQueryResponse = ResponseStatus & { records?: Record[] + + /** If there are additional results, the messageCid of the last record will be returned as a pagination cursor. */ + cursor?: string; }; /** @@ -360,7 +363,7 @@ export class DwnApi { agentResponse = await this.agent.processDwnRequest(agentRequest); } - const { reply: { entries, status } } = agentResponse; + const { reply: { entries, status, cursor } } = agentResponse; const records = entries.map((entry: RecordsQueryReplyEntry) => { const recordOptions = { @@ -381,7 +384,7 @@ export class DwnApi { return record; }); - return { records, status }; + return { records, status, cursor }; }, /** diff --git a/packages/api/tests/dwn-api.spec.ts b/packages/api/tests/dwn-api.spec.ts index 1320f5165..54a514cc6 100644 --- a/packages/api/tests/dwn-api.spec.ts +++ b/packages/api/tests/dwn-api.spec.ts @@ -1,7 +1,9 @@ import type { PortableDid } from '@web5/dids'; +import sinon from 'sinon'; import { expect } from 'chai'; import { TestManagedAgent } from '@web5/agent'; +import { DateSort } from '@tbd54566975/dwn-sdk-js'; import { DwnApi } from '../src/dwn-api.js'; import { testDwnUrl } from './test-config.js'; @@ -528,6 +530,134 @@ describe('DwnApi', () => { expect(result.records!.length).to.equal(1); expect(result.records![0].id).to.equal(writeResult.record!.id); }); + + it('returns cursor when there are additional results', async () => { + for(let i = 0; i < 3; i++ ) { + const writeResult = await dwnAlice.records.write({ + data : `Hello, world ${i + 1}!`, + message : { + schema : 'foo/bar', + dataFormat : 'text/plain' + } + }); + + expect(writeResult.status.code).to.equal(202); + expect(writeResult.status.detail).to.equal('Accepted'); + expect(writeResult.record).to.exist; + } + + const results = await dwnAlice.records.query({ + message: { + filter: { + schema: 'foo/bar' + }, + pagination: { limit: 2 } // set a limit of 2 + } + }); + + expect(results.status.code).to.equal(200); + expect(results.records).to.exist; + expect(results.records!.length).to.equal(2); + expect(results.cursor).to.exist; + + const additionalResults = await dwnAlice.records.query({ + message: { + filter: { + schema: 'foo/bar' + }, + pagination: { limit: 2, cursor: results.cursor} + } + }); + expect(additionalResults.status.code).to.equal(200); + expect(additionalResults.records).to.exist; + expect(additionalResults.records!.length).to.equal(1); + expect(additionalResults.cursor).to.not.exist; + }); + + it('sorts results based on provided query sort parameter', async () => { + const clock = sinon.useFakeTimers(); + + const items = []; + const publishedItems = []; + for(let i = 0; i < 6; i++ ) { + const writeResult = await dwnAlice.records.write({ + data : `Hello, world ${i + 1}!`, + message : { + published : i % 2 == 0 ? true : false, + schema : 'foo/bar', + dataFormat : 'text/plain' + } + }); + + expect(writeResult.status.code).to.equal(202); + expect(writeResult.status.detail).to.equal('Accepted'); + expect(writeResult.record).to.exist; + + items.push(writeResult.record.id); // add id to list in the order it was inserted + if (writeResult.record.published === true) { + publishedItems.push(writeResult.record.id); // add published records separately + } + + clock.tick(1000 * 1); // travel forward one second + } + clock.restore(); + + // query in ascending order by the dateCreated field + const createdAscResults = await dwnAlice.records.query({ + message: { + filter: { + schema: 'foo/bar' + }, + dateSort: DateSort.CreatedAscending // same as default + } + }); + expect(createdAscResults.status.code).to.equal(200); + expect(createdAscResults.records).to.exist; + expect(createdAscResults.records!.length).to.equal(6); + expect(createdAscResults.records.map(r => r.id)).to.eql(items); + + // query in descending order by the dateCreated field + const createdDescResults = await dwnAlice.records.query({ + message: { + filter: { + schema: 'foo/bar' + }, + dateSort: DateSort.CreatedDescending + } + }); + expect(createdDescResults.status.code).to.equal(200); + expect(createdDescResults.records).to.exist; + expect(createdDescResults.records!.length).to.equal(6); + expect(createdDescResults.records.map(r => r.id)).to.eql([...items].reverse()); + + // query in ascending order by the datePublished field, this will only return published records + const publishedAscResults = await dwnAlice.records.query({ + message: { + filter: { + schema: 'foo/bar' + }, + dateSort: DateSort.PublishedAscending + } + }); + expect(publishedAscResults.status.code).to.equal(200); + expect(publishedAscResults.records).to.exist; + expect(publishedAscResults.records!.length).to.equal(3); + expect(publishedAscResults.records.map(r => r.id)).to.eql(publishedItems); + + // query in desscending order by the datePublished field, this will only return published records + const publishedDescResults = await dwnAlice.records.query({ + message: { + filter: { + schema: 'foo/bar' + }, + dateSort: DateSort.PublishedDescending + } + }); + expect(publishedDescResults.status.code).to.equal(200); + expect(publishedDescResults.records).to.exist; + expect(publishedDescResults.records!.length).to.equal(3); + expect(publishedDescResults.records.map(r => r.id)).to.eql([...publishedItems].reverse()); + }); }); describe('from: did', () => {