From 3f632b89ce8e03743010b933b78a738a15815d00 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Wed, 14 Aug 2024 15:59:55 -0400 Subject: [PATCH] update dwn-sdk and account for 204 in sync --- package.json | 2 +- packages/agent/package.json | 2 +- packages/agent/src/sync-engine-level.ts | 13 +- .../agent/tests/sync-engine-level.spec.ts | 162 +++++++++++++++++- packages/api/package.json | 2 +- packages/dev-env/docker-compose.yaml | 2 +- pnpm-lock.yaml | 67 +++++--- 7 files changed, 219 insertions(+), 31 deletions(-) diff --git a/package.json b/package.json index 6d99435f2..bcdf387ea 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "@changesets/cli": "^2.27.5", "@npmcli/package-json": "5.0.0", "@typescript-eslint/eslint-plugin": "7.9.0", - "@web5/dwn-server": "0.4.5", + "@web5/dwn-server": "0.4.6", "audit-ci": "^7.0.1", "eslint-plugin-mocha": "10.4.3", "globals": "^13.24.0", diff --git a/packages/agent/package.json b/packages/agent/package.json index 9fb8ca688..685267f32 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -71,7 +71,7 @@ "dependencies": { "@noble/ciphers": "0.5.3", "@scure/bip39": "1.2.2", - "@tbd54566975/dwn-sdk-js": "0.4.4", + "@tbd54566975/dwn-sdk-js": "0.4.5", "@web5/common": "1.0.0", "@web5/crypto": "workspace:*", "@web5/dids": "1.1.0", diff --git a/packages/agent/src/sync-engine-level.ts b/packages/agent/src/sync-engine-level.ts index 446d2ac9a..1ebe45bdb 100644 --- a/packages/agent/src/sync-engine-level.ts +++ b/packages/agent/src/sync-engine-level.ts @@ -243,9 +243,6 @@ export class SyncEngineLevel implements SyncEngine { }); // Update the watermark and add the messageCid to the Sync Message Store if either: - // - 202: message was successfully written to the remote DWN - // - 409: message was already present on the remote DWN - // - RecordsDelete and the status code is 404: the initial write message was not found or the message was already deleted if (SyncEngineLevel.syncMessageReplyIsSuccessful(reply)) { await this.addMessage(did, messageCid); deleteOperations.push({ type: 'del', key: key }); @@ -315,8 +312,18 @@ export class SyncEngineLevel implements SyncEngine { } } + /** + * 202: message was successfully written to the remote DWN + * 204: an initial write message was written without any data, cannot yet be read until a subsequent message is written with data + * 409: message was already present on the remote DWN + * RecordsDelete and the status code is 404: the initial write message was not found or the message was already deleted + */ private static syncMessageReplyIsSuccessful(reply: UnionMessageReply): boolean { return reply.status.code === 202 || + // a 204 status code is returned when the message was accepted without any data. + // This is the case for an initial RecordsWrite messages for records that have been updated. + // For context: https://github.com/TBD54566975/dwn-sdk-js/issues/695 + reply.status.code === 204 || reply.status.code === 409 || ( // If the message is a RecordsDelete and the status code is 404, the initial write message was not found or the message was already deleted diff --git a/packages/agent/tests/sync-engine-level.spec.ts b/packages/agent/tests/sync-engine-level.spec.ts index 380590b68..a1f6b967a 100644 --- a/packages/agent/tests/sync-engine-level.spec.ts +++ b/packages/agent/tests/sync-engine-level.spec.ts @@ -1,7 +1,7 @@ import sinon from 'sinon'; import { expect } from 'chai'; import { utils as cryptoUtils } from '@web5/crypto'; -import { DwnConstant, DwnInterfaceName, DwnMethodName, Jws, ProtocolDefinition, Time } from '@tbd54566975/dwn-sdk-js'; +import { DwnConstant, DwnInterfaceName, DwnMethodName, Jws, Message, ProtocolDefinition, Time } from '@tbd54566975/dwn-sdk-js'; import type { BearerIdentity } from '../src/bearer-identity.js'; @@ -934,6 +934,84 @@ describe('SyncEngineLevel', () => { }); describe('pull()', () => { + it('synchronizes records that have been updated', async () => { + // Write a test record to Alice's remote DWN. + let writeResponse1 = await testHarness.agent.dwn.sendRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsWrite, + messageParams : { + dataFormat : 'text/plain', + schema : randomSchema + }, + dataStream: new Blob(['Hello, world!']) + }); + + // Get the record ID of the test record. + const testRecordId = writeResponse1.message!.recordId; + + // const update the record + let updateResponse = await testHarness.agent.dwn.sendRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsWrite, + messageParams : { + recordId : testRecordId, + dataFormat : 'text/plain', + schema : randomSchema, + dateCreated : writeResponse1.message!.descriptor.dateCreated + }, + dataStream: new Blob(['Hello, world updated!']) + }); + expect(updateResponse.reply.status.code).to.equal(202); + expect(updateResponse.message!.recordId).to.equal(testRecordId); + + const updateMessageCid = updateResponse.messageCid; + + // Confirm the record does NOT exist on Alice's local DWN. + let queryResponse = await testHarness.agent.dwn.processRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsQuery, + messageParams : { + filter: { + dataFormat : 'text/plain', + schema : randomSchema + } + } + }); + let localDwnQueryReply = queryResponse.reply; + expect(localDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. + expect(localDwnQueryReply.entries).to.have.length(0); // Record doesn't exist on local DWN. + + // Register Alice's DID to be synchronized. + await testHarness.agent.sync.registerIdentity({ + did : alice.did.uri, + options : { + protocols: [] + } + }); + + // Execute Sync to pull all records from Alice's remote DWN to Alice's local DWN. + await syncEngine.pull(); + + // Confirm the record now DOES exist on Alice's local DWN. + queryResponse = await testHarness.agent.dwn.processRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsQuery, + messageParams : { filter: { recordId: testRecordId } } + }); + localDwnQueryReply = queryResponse.reply; + expect(localDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. + expect(localDwnQueryReply.entries).to.have.length(1); // Record does exist on local DWN. + + // remove `initialWrite` from the response to generate an accurate messageCid + const { initialWrite, ...rawMessage } = localDwnQueryReply.entries![0] + const queriedMessageCid = await Message.getCid(rawMessage); + expect(queriedMessageCid).to.equal(updateMessageCid); + }); + it('silently ignores sendDwnRequest for a messageCid that does not exist on a remote DWN', async () => { // scenario: The messageCids returned from the remote eventLog contains a Cid that is not found in the remote DWN // this could happen when a record is updated, only the initial write and the most recent state are kept. @@ -1194,7 +1272,7 @@ describe('SyncEngineLevel', () => { }); it('synchronizes records for 1 identity from remote DWN to local DWN', async () => { - // Write a test record to Alice's remote DWN. + // Write a test record to Alice's remote DWN. let writeResponse = await testHarness.agent.dwn.sendRequest({ author : alice.did.uri, target : alice.did.uri, @@ -1248,8 +1326,6 @@ describe('SyncEngineLevel', () => { expect(localDwnQueryReply.entries).to.have.length(1); // Record does exist on local DWN. - - // Add another record for a subsequent sync. let writeResponse2 = await testHarness.agent.dwn.sendRequest({ author : alice.did.uri, @@ -1430,6 +1506,84 @@ describe('SyncEngineLevel', () => { }); describe('push()', () => { + it('synchronizes records that have been updated', async () => { + // Write a test record to Alice's local DWN. + let writeResponse1 = await testHarness.agent.dwn.processRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsWrite, + messageParams : { + dataFormat : 'text/plain', + schema : randomSchema + }, + dataStream: new Blob(['Hello, world!']) + }); + + // Get the record ID of the test record. + const testRecordId = writeResponse1.message!.recordId; + + // const update the record + let updateResponse = await testHarness.agent.dwn.processRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsWrite, + messageParams : { + recordId : testRecordId, + dataFormat : 'text/plain', + schema : randomSchema, + dateCreated : writeResponse1.message!.descriptor.dateCreated + }, + dataStream: new Blob(['Hello, world updated!']) + }); + expect(updateResponse.reply.status.code).to.equal(202); + expect(updateResponse.message!.recordId).to.equal(testRecordId); + + const updateMessageCid = updateResponse.messageCid; + + // Confirm the record does NOT exist on Alice's remote DWN. + let queryResponse = await testHarness.agent.dwn.sendRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsQuery, + messageParams : { + filter: { + dataFormat : 'text/plain', + schema : randomSchema + } + } + }); + let remoteDwnQueryReply = queryResponse.reply; + expect(remoteDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. + expect(remoteDwnQueryReply.entries).to.have.length(0); // Record doesn't exist on local DWN. + + // Register Alice's DID to be synchronized. + await testHarness.agent.sync.registerIdentity({ + did : alice.did.uri, + options : { + protocols: [] + } + }); + + // Execute Sync to pull all records from Alice's remote DWN to Alice's local DWN. + await syncEngine.push(); + + // Confirm the record now DOES exist on Alice's local DWN. + queryResponse = await testHarness.agent.dwn.sendRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsQuery, + messageParams : { filter: { recordId: testRecordId } } + }); + remoteDwnQueryReply = queryResponse.reply; + expect(remoteDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. + expect(remoteDwnQueryReply.entries).to.have.length(1); // Record does exist on local DWN. + + // remove `initialWrite` from the response to generate an accurate messageCid + const { initialWrite, ...rawMessage } = remoteDwnQueryReply.entries![0] + const queriedMessageCid = await Message.getCid(rawMessage); + expect(queriedMessageCid).to.equal(updateMessageCid); + }); + it('silently ignores a messageCid from the eventLog that does not exist on the local DWN', async () => { // It's important to create a new DID here to avoid conflicts with the previous test on the remote DWN, // since we are not clearing the remote DWN's storage before each test. diff --git a/packages/api/package.json b/packages/api/package.json index be8c293b7..da1048f58 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -85,7 +85,7 @@ }, "devDependencies": { "@playwright/test": "1.45.3", - "@tbd54566975/dwn-sdk-js": "0.4.4", + "@tbd54566975/dwn-sdk-js": "0.4.5", "@types/chai": "4.3.6", "@types/eslint": "8.56.10", "@types/mocha": "10.0.1", diff --git a/packages/dev-env/docker-compose.yaml b/packages/dev-env/docker-compose.yaml index df9cfebf2..541bc2415 100644 --- a/packages/dev-env/docker-compose.yaml +++ b/packages/dev-env/docker-compose.yaml @@ -3,6 +3,6 @@ version: "3.98" services: dwn-server: container_name: dwn-server - image: ghcr.io/tbd54566975/dwn-server:0.4.5 + image: ghcr.io/tbd54566975/dwn-server:0.4.6 ports: - "3000:3000" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 889dc08f8..7998f7ac5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -28,8 +28,8 @@ importers: specifier: 7.9.0 version: 7.9.0(@typescript-eslint/parser@7.14.1(eslint@9.7.0)(typescript@5.5.4))(eslint@9.7.0)(typescript@5.5.4) '@web5/dwn-server': - specifier: 0.4.5 - version: 0.4.5 + specifier: 0.4.6 + version: 0.4.6 audit-ci: specifier: ^7.0.1 version: 7.1.0 @@ -52,8 +52,8 @@ importers: specifier: 1.2.2 version: 1.2.2 '@tbd54566975/dwn-sdk-js': - specifier: 0.4.4 - version: 0.4.4 + specifier: 0.4.5 + version: 0.4.5 '@web5/common': specifier: 1.0.0 version: 1.0.0 @@ -180,8 +180,8 @@ importers: specifier: 1.45.3 version: 1.45.3 '@tbd54566975/dwn-sdk-js': - specifier: 0.4.4 - version: 0.4.4 + specifier: 0.4.5 + version: 0.4.5 '@types/chai': specifier: 4.3.6 version: 4.3.6 @@ -2096,12 +2096,12 @@ packages: '@sphereon/ssi-types@0.26.0': resolution: {integrity: sha512-r4JQIN7rnPunEv0HvCFC1ZCc9qlWcegYvhJbMJqSvyFE6VhmT5NNdH9jNV9QetgMa0yo5r3k+TnHNv3nH58Dmg==} - '@tbd54566975/dwn-sdk-js@0.4.4': - resolution: {integrity: sha512-i4jBrKOn6ypaFQJLNG8Kc9AvR1nmZrq3A+9M4O9xiDbH60+QMPf2HsNuVOTu6Ve9V7IU8NorI/GoSryD0TknpQ==} + '@tbd54566975/dwn-sdk-js@0.4.5': + resolution: {integrity: sha512-UGcq9PX32oQ3sB9LfQms82Ce5secNJXUUe+W163Am2vOAAjJ8AyFG9CaIrXO8HMyEO1yZ7l3bBv57tYM9Zf70A==} engines: {node: '>= 18'} - '@tbd54566975/dwn-sql-store@0.6.4': - resolution: {integrity: sha512-ivQt83MMMYsXC4qSD61lKINuJ5KnGIz4p0H61oh/zqd4ivg7ofrTnnVZBdKopvchREZ1ozxI5hcRRuBLkuLbkw==} + '@tbd54566975/dwn-sql-store@0.6.5': + resolution: {integrity: sha512-ZPdz7Ck7NMNCIOuZv+oxfxrw5lZJI/SukcLVd7PMseWZvT28D/gRcHgt0MfizeE9jLv9+ONI+0k8uc/O2NILyw==} engines: {node: '>=18'} '@tootallnate/quickjs-emscripten@0.23.0': @@ -2483,6 +2483,10 @@ packages: resolution: {integrity: sha512-z1CsgycTqiXEsS6pPlJDDLGAeGsgzfdBeWvyxLXTgh08Q8ACULmEGRXjSsgWHFn6DO6MpWFn55h/hF4wZZRxvA==} engines: {node: '>=18.0.0'} + '@web5/crypto@1.0.3': + resolution: {integrity: sha512-gZJKo0scX+L53E2K/5cgEiFYxejzHP2RSg64ncF6TitOnCNxUyWjofovgufb+u3ZpGC4iuliD7V0o1C+V73Law==} + engines: {node: '>=18.0.0'} + '@web5/dids@1.1.0': resolution: {integrity: sha512-d9pKf/DW+ziUiV5g3McC71utyAhQyT1tYGPbQSYWt2ji6FHGNC6tffHMfLXXK/W+vbwV3eNTn06JqTXRaYhxBA==} engines: {node: '>=18.0.0'} @@ -2491,8 +2495,12 @@ packages: resolution: {integrity: sha512-LKc6Okl2iz78QGJCsd8QKQq3LdtmfQ9cfiRKu1BU4ITWteWsg4JD089hKmslNDd2KKnEf9LE72TqEYWxr/e8JA==} engines: {node: '>=18.0.0'} - '@web5/dwn-server@0.4.5': - resolution: {integrity: sha512-fEoL3IU/RwkOUj5kKuoBaBRY2OJ1h/gEzEgeH0nDluNg1S8T96kOCDxSgPbuVMY+aofj7JmQZQm0oAmq4tH2Eg==} + '@web5/dids@1.1.3': + resolution: {integrity: sha512-M9EfsEYcOtYuEvUQjow4vpxXbD0Sz5H8EuDXMtwuvP4UdYL0ATl+60F8+8HDmwPFeUy6M2wxuoixrLDwSRFwZA==} + engines: {node: '>=18.0.0'} + + '@web5/dwn-server@0.4.6': + resolution: {integrity: sha512-92uNTJDBHGprneQtuD0A4XbcWirWHY+MrQVKd4fyrjwMh7cUMF3uE4l3QnBwn8kquzRrU+3sekcgzNjHvmTfHw==} hasBin: true '@webassemblyjs/ast@1.12.1': @@ -7344,7 +7352,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@tbd54566975/dwn-sdk-js@0.4.4': + '@tbd54566975/dwn-sdk-js@0.4.5': dependencies: '@ipld/dag-cbor': 9.0.3 '@js-temporal/polyfill': 0.4.4 @@ -7352,7 +7360,7 @@ snapshots: '@noble/curves': 1.4.2 '@noble/ed25519': 2.0.0 '@noble/secp256k1': 2.0.0 - '@web5/dids': 1.1.0 + '@web5/dids': 1.1.3 abstract-level: 1.0.3 ajv: 8.12.0 blockstore-core: 4.2.0 @@ -7377,10 +7385,10 @@ snapshots: - encoding - supports-color - '@tbd54566975/dwn-sql-store@0.6.4': + '@tbd54566975/dwn-sql-store@0.6.5': dependencies: '@ipld/dag-cbor': 9.0.5 - '@tbd54566975/dwn-sdk-js': 0.4.4 + '@tbd54566975/dwn-sdk-js': 0.4.5 kysely: 0.26.3 multiformats: 12.0.1 readable-stream: 4.4.2 @@ -8287,6 +8295,13 @@ snapshots: '@noble/hashes': 1.3.3 '@web5/common': 1.0.0 + '@web5/crypto@1.0.3': + dependencies: + '@noble/ciphers': 0.5.3 + '@noble/curves': 1.3.0 + '@noble/hashes': 1.4.0 + '@web5/common': 1.0.1 + '@web5/dids@1.1.0': dependencies: '@decentralized-identity/ion-sdk': 1.0.4 @@ -8311,10 +8326,22 @@ snapshots: level: 8.0.1 ms: 2.1.3 - '@web5/dwn-server@0.4.5': + '@web5/dids@1.1.3': dependencies: - '@tbd54566975/dwn-sdk-js': 0.4.4 - '@tbd54566975/dwn-sql-store': 0.6.4 + '@decentralized-identity/ion-sdk': 1.0.4 + '@dnsquery/dns-packet': 6.1.1 + '@web5/common': 1.0.0 + '@web5/crypto': 1.0.3 + abstract-level: 1.0.4 + bencode: 4.0.0 + buffer: 6.0.3 + level: 8.0.1 + ms: 2.1.3 + + '@web5/dwn-server@0.4.6': + dependencies: + '@tbd54566975/dwn-sdk-js': 0.4.5 + '@tbd54566975/dwn-sql-store': 0.6.5 better-sqlite3: 8.7.0 body-parser: 1.20.2 bytes: 3.1.2 @@ -10205,7 +10232,7 @@ snapshots: ipfs-unixfs-exporter@13.1.5: dependencies: - '@ipld/dag-cbor': 9.0.3 + '@ipld/dag-cbor': 9.0.5 '@ipld/dag-pb': 4.1.2 '@multiformats/murmur3': 2.1.8 err-code: 3.0.1