Skip to content

Commit

Permalink
update dwn-sdk and account for 204 in sync
Browse files Browse the repository at this point in the history
  • Loading branch information
LiranCohen committed Aug 14, 2024
1 parent 85ec6a5 commit 3f632b8
Showing 7 changed files with 219 additions and 31 deletions.
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -31,7 +31,7 @@
"@changesets/cli": "^2.27.5",
"@npmcli/package-json": "5.0.0",
"@typescript-eslint/eslint-plugin": "7.9.0",
"@web5/dwn-server": "0.4.5",
"@web5/dwn-server": "0.4.6",
"audit-ci": "^7.0.1",
"eslint-plugin-mocha": "10.4.3",
"globals": "^13.24.0",
2 changes: 1 addition & 1 deletion packages/agent/package.json
Original file line number Diff line number Diff line change
@@ -71,7 +71,7 @@
"dependencies": {
"@noble/ciphers": "0.5.3",
"@scure/bip39": "1.2.2",
"@tbd54566975/dwn-sdk-js": "0.4.4",
"@tbd54566975/dwn-sdk-js": "0.4.5",
"@web5/common": "1.0.0",
"@web5/crypto": "workspace:*",
"@web5/dids": "1.1.0",
13 changes: 10 additions & 3 deletions packages/agent/src/sync-engine-level.ts
Original file line number Diff line number Diff line change
@@ -243,9 +243,6 @@ export class SyncEngineLevel implements SyncEngine {
});

// Update the watermark and add the messageCid to the Sync Message Store if either:
// - 202: message was successfully written to the remote DWN
// - 409: message was already present on the remote DWN
// - RecordsDelete and the status code is 404: the initial write message was not found or the message was already deleted
if (SyncEngineLevel.syncMessageReplyIsSuccessful(reply)) {
await this.addMessage(did, messageCid);
deleteOperations.push({ type: 'del', key: key });
@@ -315,8 +312,18 @@ export class SyncEngineLevel implements SyncEngine {
}
}

/**
* 202: message was successfully written to the remote DWN
* 204: an initial write message was written without any data, cannot yet be read until a subsequent message is written with data
* 409: message was already present on the remote DWN
* RecordsDelete and the status code is 404: the initial write message was not found or the message was already deleted
*/
private static syncMessageReplyIsSuccessful(reply: UnionMessageReply): boolean {
return reply.status.code === 202 ||
// a 204 status code is returned when the message was accepted without any data.
// This is the case for an initial RecordsWrite messages for records that have been updated.
// For context: https://github.com/TBD54566975/dwn-sdk-js/issues/695
reply.status.code === 204 ||
reply.status.code === 409 ||
(
// If the message is a RecordsDelete and the status code is 404, the initial write message was not found or the message was already deleted
162 changes: 158 additions & 4 deletions packages/agent/tests/sync-engine-level.spec.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import sinon from 'sinon';
import { expect } from 'chai';
import { utils as cryptoUtils } from '@web5/crypto';
import { DwnConstant, DwnInterfaceName, DwnMethodName, Jws, ProtocolDefinition, Time } from '@tbd54566975/dwn-sdk-js';
import { DwnConstant, DwnInterfaceName, DwnMethodName, Jws, Message, ProtocolDefinition, Time } from '@tbd54566975/dwn-sdk-js';

import type { BearerIdentity } from '../src/bearer-identity.js';

@@ -934,6 +934,84 @@ describe('SyncEngineLevel', () => {
});

describe('pull()', () => {
it('synchronizes records that have been updated', async () => {
// Write a test record to Alice's remote DWN.
let writeResponse1 = await testHarness.agent.dwn.sendRequest({
author : alice.did.uri,
target : alice.did.uri,
messageType : DwnInterface.RecordsWrite,
messageParams : {
dataFormat : 'text/plain',
schema : randomSchema
},
dataStream: new Blob(['Hello, world!'])
});

// Get the record ID of the test record.
const testRecordId = writeResponse1.message!.recordId;

// const update the record
let updateResponse = await testHarness.agent.dwn.sendRequest({
author : alice.did.uri,
target : alice.did.uri,
messageType : DwnInterface.RecordsWrite,
messageParams : {
recordId : testRecordId,
dataFormat : 'text/plain',
schema : randomSchema,
dateCreated : writeResponse1.message!.descriptor.dateCreated
},
dataStream: new Blob(['Hello, world updated!'])
});
expect(updateResponse.reply.status.code).to.equal(202);
expect(updateResponse.message!.recordId).to.equal(testRecordId);

const updateMessageCid = updateResponse.messageCid;

// Confirm the record does NOT exist on Alice's local DWN.
let queryResponse = await testHarness.agent.dwn.processRequest({
author : alice.did.uri,
target : alice.did.uri,
messageType : DwnInterface.RecordsQuery,
messageParams : {
filter: {
dataFormat : 'text/plain',
schema : randomSchema
}
}
});
let localDwnQueryReply = queryResponse.reply;
expect(localDwnQueryReply.status.code).to.equal(200); // Query was successfully executed.
expect(localDwnQueryReply.entries).to.have.length(0); // Record doesn't exist on local DWN.

// Register Alice's DID to be synchronized.
await testHarness.agent.sync.registerIdentity({
did : alice.did.uri,
options : {
protocols: []
}
});

// Execute Sync to pull all records from Alice's remote DWN to Alice's local DWN.
await syncEngine.pull();

// Confirm the record now DOES exist on Alice's local DWN.
queryResponse = await testHarness.agent.dwn.processRequest({
author : alice.did.uri,
target : alice.did.uri,
messageType : DwnInterface.RecordsQuery,
messageParams : { filter: { recordId: testRecordId } }
});
localDwnQueryReply = queryResponse.reply;
expect(localDwnQueryReply.status.code).to.equal(200); // Query was successfully executed.
expect(localDwnQueryReply.entries).to.have.length(1); // Record does exist on local DWN.

// remove `initialWrite` from the response to generate an accurate messageCid
const { initialWrite, ...rawMessage } = localDwnQueryReply.entries![0]
const queriedMessageCid = await Message.getCid(rawMessage);
expect(queriedMessageCid).to.equal(updateMessageCid);
});

it('silently ignores sendDwnRequest for a messageCid that does not exist on a remote DWN', async () => {
// scenario: The messageCids returned from the remote eventLog contains a Cid that is not found in the remote DWN
// this could happen when a record is updated, only the initial write and the most recent state are kept.
@@ -1194,7 +1272,7 @@ describe('SyncEngineLevel', () => {
});

it('synchronizes records for 1 identity from remote DWN to local DWN', async () => {
// Write a test record to Alice's remote DWN.
// Write a test record to Alice's remote DWN.
let writeResponse = await testHarness.agent.dwn.sendRequest({
author : alice.did.uri,
target : alice.did.uri,
@@ -1248,8 +1326,6 @@ describe('SyncEngineLevel', () => {
expect(localDwnQueryReply.entries).to.have.length(1); // Record does exist on local DWN.




// Add another record for a subsequent sync.
let writeResponse2 = await testHarness.agent.dwn.sendRequest({
author : alice.did.uri,
@@ -1430,6 +1506,84 @@ describe('SyncEngineLevel', () => {
});

describe('push()', () => {
it('synchronizes records that have been updated', async () => {
// Write a test record to Alice's local DWN.
let writeResponse1 = await testHarness.agent.dwn.processRequest({
author : alice.did.uri,
target : alice.did.uri,
messageType : DwnInterface.RecordsWrite,
messageParams : {
dataFormat : 'text/plain',
schema : randomSchema
},
dataStream: new Blob(['Hello, world!'])
});

// Get the record ID of the test record.
const testRecordId = writeResponse1.message!.recordId;

// const update the record
let updateResponse = await testHarness.agent.dwn.processRequest({
author : alice.did.uri,
target : alice.did.uri,
messageType : DwnInterface.RecordsWrite,
messageParams : {
recordId : testRecordId,
dataFormat : 'text/plain',
schema : randomSchema,
dateCreated : writeResponse1.message!.descriptor.dateCreated
},
dataStream: new Blob(['Hello, world updated!'])
});
expect(updateResponse.reply.status.code).to.equal(202);
expect(updateResponse.message!.recordId).to.equal(testRecordId);

const updateMessageCid = updateResponse.messageCid;

// Confirm the record does NOT exist on Alice's remote DWN.
let queryResponse = await testHarness.agent.dwn.sendRequest({
author : alice.did.uri,
target : alice.did.uri,
messageType : DwnInterface.RecordsQuery,
messageParams : {
filter: {
dataFormat : 'text/plain',
schema : randomSchema
}
}
});
let remoteDwnQueryReply = queryResponse.reply;
expect(remoteDwnQueryReply.status.code).to.equal(200); // Query was successfully executed.
expect(remoteDwnQueryReply.entries).to.have.length(0); // Record doesn't exist on local DWN.

// Register Alice's DID to be synchronized.
await testHarness.agent.sync.registerIdentity({
did : alice.did.uri,
options : {
protocols: []
}
});

// Execute Sync to pull all records from Alice's remote DWN to Alice's local DWN.
await syncEngine.push();

// Confirm the record now DOES exist on Alice's local DWN.
queryResponse = await testHarness.agent.dwn.sendRequest({
author : alice.did.uri,
target : alice.did.uri,
messageType : DwnInterface.RecordsQuery,
messageParams : { filter: { recordId: testRecordId } }
});
remoteDwnQueryReply = queryResponse.reply;
expect(remoteDwnQueryReply.status.code).to.equal(200); // Query was successfully executed.
expect(remoteDwnQueryReply.entries).to.have.length(1); // Record does exist on local DWN.

// remove `initialWrite` from the response to generate an accurate messageCid
const { initialWrite, ...rawMessage } = remoteDwnQueryReply.entries![0]
const queriedMessageCid = await Message.getCid(rawMessage);
expect(queriedMessageCid).to.equal(updateMessageCid);
});

it('silently ignores a messageCid from the eventLog that does not exist on the local DWN', async () => {
// It's important to create a new DID here to avoid conflicts with the previous test on the remote DWN,
// since we are not clearing the remote DWN's storage before each test.
2 changes: 1 addition & 1 deletion packages/api/package.json
Original file line number Diff line number Diff line change
@@ -85,7 +85,7 @@
},
"devDependencies": {
"@playwright/test": "1.45.3",
"@tbd54566975/dwn-sdk-js": "0.4.4",
"@tbd54566975/dwn-sdk-js": "0.4.5",
"@types/chai": "4.3.6",
"@types/eslint": "8.56.10",
"@types/mocha": "10.0.1",
2 changes: 1 addition & 1 deletion packages/dev-env/docker-compose.yaml
Original file line number Diff line number Diff line change
@@ -3,6 +3,6 @@ version: "3.98"
services:
dwn-server:
container_name: dwn-server
image: ghcr.io/tbd54566975/dwn-server:0.4.5
image: ghcr.io/tbd54566975/dwn-server:0.4.6
ports:
- "3000:3000"
Loading

0 comments on commit 3f632b8

Please sign in to comment.