Support RecordsQuery pagination in Web5 (#268)

* Add pagination messageCid to RecordsQueryResponse and tests supporting pagination/sort * cursor instead of paginationsMessagecid --------- Signed-off-by: Frank Hinek <[email protected]> Co-authored-by: Frank Hinek <[email protected]>
decentralized-identity · Nov 29, 2023 · dce370d · dce370d
1 parent 9c7c866
commit dce370d
Show file tree

Hide file tree

Showing 5 changed files with 173 additions and 9 deletions.
diff --git a/README.md b/README.md
@@ -224,6 +224,20 @@ The query `request` contains the following properties:
     - **`recipient`** - _`string`_ (_optional_): the DID in the `recipient` field of the record.
     - **`schema`** - _`URI string`_ (_optional_): the URI of the schema bucket in which to query.
     - **`dataFormat`** - _`Media Type string`_ (_optional_): the IANA string corresponding with the format of the data to filter for. See IANA's Media Type list here: https://www.iana.org/assignments/media-types/media-types.xhtml
+  - **`dateSort`** - _`DateSort`_ (_optional_): the `DateSort` value of the date field and direction to sort records by. Defaults to `CreatedAscending`.
+  - **`pagination`** - _`object`_ (_optional_): the properties used to paginate results.
+    - **`limit`** - _`number`_ (_optional_): the number of records that should be returned with this query. `undefined` returns all records.
+    - **`cursor`** - _`messageCid string`_ (_optional_): the `messageCid` of the records toc continue paginating from. This value is returned as a `cursor` in the response object of a `query` if there are more results beyond the `limit`.
+
+#### **Response**
+
+The query `response` contains the following properties:
+
+- **`status`** - _`object`_: the status of the `request`:
+  - **`code`** - _`number`_: the `Response Status` code, following the response code patterns for `HTTP Response Status Codes`.
+  - **`detail`** _`string`_: a detailed message describing the response.
+- **`records`** - _`Records array`_ (_optional_): the array of `Records` returned if the request was successful.
+- **`cursor`** - _`messageCid string`_ (_optional_): the `messageCid` of the last message returned in the results if there are exist additional records beyond the specified `limit` in the `query`.
 
 ### **`web5.dwn.records.create(request)`**
 

diff --git a/packages/agent/src/sync-manager.ts b/packages/agent/src/sync-manager.ts
@@ -1,17 +1,20 @@
-import { DataStream } from '@tbd54566975/dwn-sdk-js';
-import { Convert } from '@web5/common';
-import { utils as didUtils } from '@web5/dids';
-import { Level } from 'level';
-import { webReadableToIsomorphicNodeReadable } from './utils.js';
+import type { AbstractBatchOperation, AbstractLevel } from 'abstract-level';
 import type {
   EventsGetReply,
   GenericMessage,
   MessagesGetReply,
   RecordsWriteMessage,
 } from '@tbd54566975/dwn-sdk-js';
-import type { AbstractBatchOperation, AbstractLevel } from 'abstract-level';
+
+import { Level } from 'level';
+import { Convert } from '@web5/common';
+import { utils as didUtils } from '@web5/dids';
+import { DataStream } from '@tbd54566975/dwn-sdk-js';
+
 import type { Web5ManagedAgent } from './types/agent.js';
 
+import { webReadableToIsomorphicNodeReadable } from './utils.js';
+
 export interface SyncManager {
   agent: Web5ManagedAgent;
   registerIdentity(options: { did: string }): Promise<void>;

diff --git a/packages/api/README.md b/packages/api/README.md
@@ -217,6 +217,20 @@ The query `request` contains the following properties:
     - **`recipient`** - _`string`_ (_optional_): the DID in the `recipient` field of the record.
     - **`schema`** - _`URI string`_ (_optional_): the URI of the schema bucket in which to query.
     - **`dataFormat`** - _`Media Type string`_ (_optional_): the IANA string corresponding with the format of the data to filter for. See IANA's Media Type list here: https://www.iana.org/assignments/media-types/media-types.xhtml
+  - **`dateSort`** - _`DateSort`_ (_optional_): the `DateSort` value of the date field and direction to sort records by. Defaults to `CreatedAscending`.
+  - **`pagination`** - _`object`_ (_optional_): the properties used to paginate results.
+    - **`limit`** - _`number`_ (_optional_): the number of records that should be returned with this query. `undefined` returns all records.
+    - **`cursor`** - _`messageCid string`_ (_optional_): the `messageCid` of the records toc continue paginating from. This value is returned as a `cursor` in the response object of a `query` if there are more results beyond the `limit`.
+
+#### **Response**
+
+The query `response` contains the following properties:
+
+- **`status`** - _`object`_: the status of the `request`:
+  - **`code`** - _`number`_: the `Response Status` code, following the response code patterns for `HTTP Response Status Codes`.
+  - **`detail`** _`string`_: a detailed message describing the response.
+- **`records`** - _`Records array`_ (_optional_): the array of `Records` returned if the request was successful.
+- **`cursor`** - _`messageCid string`_ (_optional_): the `messageCid` of the last message returned in the results if there are exist additional records beyond the specified `limit` in the `query`.
 
 ### **`web5.dwn.records.create(request)`**
 

diff --git a/packages/api/src/dwn-api.ts b/packages/api/src/dwn-api.ts
@@ -16,8 +16,8 @@ import { isEmptyObject } from '@web5/common';
 import { DwnInterfaceName, DwnMethodName, RecordsWrite } from '@tbd54566975/dwn-sdk-js';
 
 import { Record } from './record.js';
-import { Protocol } from './protocol.js';
 import { dataToBlob } from './utils.js';
+import { Protocol } from './protocol.js';
 
 /**
  * Status code and detailed message for a response.
@@ -131,6 +131,9 @@ export type RecordsQueryRequest = {
  */
 export type RecordsQueryResponse = ResponseStatus & {
   records?: Record[]
+
+  /** If there are additional results, the messageCid of the last record will be returned as a pagination cursor. */
+  cursor?: string;
 };
 
 /**
@@ -360,7 +363,7 @@ export class DwnApi {
           agentResponse = await this.agent.processDwnRequest(agentRequest);
         }
 
-        const { reply: { entries, status } } = agentResponse;
+        const { reply: { entries, status, cursor } } = agentResponse;
 
         const records = entries.map((entry: RecordsQueryReplyEntry) => {
           const recordOptions = {
@@ -381,7 +384,7 @@ export class DwnApi {
           return record;
         });
 
-        return { records, status };
+        return { records, status, cursor };
       },
 
       /**

diff --git a/packages/api/tests/dwn-api.spec.ts b/packages/api/tests/dwn-api.spec.ts
@@ -1,7 +1,9 @@
 import type { PortableDid } from '@web5/dids';
 
+import sinon from 'sinon';
 import { expect } from 'chai';
 import { TestManagedAgent } from '@web5/agent';
+import { DateSort } from '@tbd54566975/dwn-sdk-js';
 
 import { DwnApi } from '../src/dwn-api.js';
 import { testDwnUrl } from './test-config.js';
@@ -528,6 +530,134 @@ describe('DwnApi', () => {
         expect(result.records!.length).to.equal(1);
         expect(result.records![0].id).to.equal(writeResult.record!.id);
       });
+
+      it('returns cursor when there are additional results', async () => {
+        for(let i = 0; i < 3; i++ ) {
+          const writeResult = await dwnAlice.records.write({
+            data    : `Hello, world ${i + 1}!`,
+            message : {
+              schema     : 'foo/bar',
+              dataFormat : 'text/plain'
+            }
+          });
+
+          expect(writeResult.status.code).to.equal(202);
+          expect(writeResult.status.detail).to.equal('Accepted');
+          expect(writeResult.record).to.exist;
+        }
+
+        const results = await dwnAlice.records.query({
+          message: {
+            filter: {
+              schema: 'foo/bar'
+            },
+            pagination: { limit: 2 } // set a limit of 2
+          }
+        });
+
+        expect(results.status.code).to.equal(200);
+        expect(results.records).to.exist;
+        expect(results.records!.length).to.equal(2);
+        expect(results.cursor).to.exist;
+
+        const additionalResults = await dwnAlice.records.query({
+          message: {
+            filter: {
+              schema: 'foo/bar'
+            },
+            pagination: { limit: 2, cursor: results.cursor}
+          }
+        });
+        expect(additionalResults.status.code).to.equal(200);
+        expect(additionalResults.records).to.exist;
+        expect(additionalResults.records!.length).to.equal(1);
+        expect(additionalResults.cursor).to.not.exist;
+      });
+
+      it('sorts results based on provided query sort parameter', async () => {
+        const clock = sinon.useFakeTimers();
+
+        const items = [];
+        const publishedItems = [];
+        for(let i = 0; i < 6; i++ ) {
+          const writeResult = await dwnAlice.records.write({
+            data    : `Hello, world ${i + 1}!`,
+            message : {
+              published  : i % 2 == 0 ? true : false,
+              schema     : 'foo/bar',
+              dataFormat : 'text/plain'
+            }
+          });
+
+          expect(writeResult.status.code).to.equal(202);
+          expect(writeResult.status.detail).to.equal('Accepted');
+          expect(writeResult.record).to.exist;
+
+          items.push(writeResult.record.id); // add id to list in the order it was inserted
+          if (writeResult.record.published === true) {
+            publishedItems.push(writeResult.record.id); // add published records separately
+          }
+
+          clock.tick(1000 * 1); // travel forward one second
+        }
+        clock.restore();
+
+        // query in ascending order by the dateCreated field
+        const createdAscResults = await dwnAlice.records.query({
+          message: {
+            filter: {
+              schema: 'foo/bar'
+            },
+            dateSort: DateSort.CreatedAscending // same as default
+          }
+        });
+        expect(createdAscResults.status.code).to.equal(200);
+        expect(createdAscResults.records).to.exist;
+        expect(createdAscResults.records!.length).to.equal(6);
+        expect(createdAscResults.records.map(r => r.id)).to.eql(items);
+
+        // query in descending order by the dateCreated field
+        const createdDescResults = await dwnAlice.records.query({
+          message: {
+            filter: {
+              schema: 'foo/bar'
+            },
+            dateSort: DateSort.CreatedDescending
+          }
+        });
+        expect(createdDescResults.status.code).to.equal(200);
+        expect(createdDescResults.records).to.exist;
+        expect(createdDescResults.records!.length).to.equal(6);
+        expect(createdDescResults.records.map(r => r.id)).to.eql([...items].reverse());
+
+        // query in ascending order by the datePublished field, this will only return published records
+        const publishedAscResults = await dwnAlice.records.query({
+          message: {
+            filter: {
+              schema: 'foo/bar'
+            },
+            dateSort: DateSort.PublishedAscending
+          }
+        });
+        expect(publishedAscResults.status.code).to.equal(200);
+        expect(publishedAscResults.records).to.exist;
+        expect(publishedAscResults.records!.length).to.equal(3);
+        expect(publishedAscResults.records.map(r => r.id)).to.eql(publishedItems);
+
+        // query in desscending order by the datePublished field, this will only return published records
+        const publishedDescResults = await dwnAlice.records.query({
+          message: {
+            filter: {
+              schema: 'foo/bar'
+            },
+            dateSort: DateSort.PublishedDescending
+          }
+        });
+        expect(publishedDescResults.status.code).to.equal(200);
+        expect(publishedDescResults.records).to.exist;
+        expect(publishedDescResults.records!.length).to.equal(3);
+        expect(publishedDescResults.records.map(r => r.id)).to.eql([...publishedItems].reverse());
+      });
     });
 
     describe('from: did', () => {