From 39e87a2bfbd68454364e4ab1d04bd43a0719c8e6 Mon Sep 17 00:00:00 2001 From: Nik Graf Date: Mon, 3 Jun 2024 19:49:48 +0200 Subject: [PATCH 1/8] add onPendingChangesUpdated --- .../YjsPendingChangesCallbackExample.tsx | 105 ++++++++++++++++++ documentation/pages/docs/_meta.json | 1 + documentation/pages/docs/api/client.mdx | 10 ++ .../docs/other-examples/pending-changes.mdx | 17 +++ packages/secsync/src/createSyncMachine.ts | 91 ++++++++++----- packages/secsync/src/types.ts | 1 + 6 files changed, 195 insertions(+), 30 deletions(-) create mode 100644 documentation/components/YjsPendingChangesCallbackExample/YjsPendingChangesCallbackExample.tsx create mode 100644 documentation/pages/docs/other-examples/pending-changes.mdx diff --git a/documentation/components/YjsPendingChangesCallbackExample/YjsPendingChangesCallbackExample.tsx b/documentation/components/YjsPendingChangesCallbackExample/YjsPendingChangesCallbackExample.tsx new file mode 100644 index 0000000..ce62e02 --- /dev/null +++ b/documentation/components/YjsPendingChangesCallbackExample/YjsPendingChangesCallbackExample.tsx @@ -0,0 +1,105 @@ +import sodium, { KeyPair } from "libsodium-wrappers"; +import React, { useRef, useState } from "react"; +import { DevTool } from "secsync-react-devtool"; +import { useYjsSync } from "secsync-react-yjs"; +import * as Yjs from "yjs"; +import { useYArray } from "../../hooks/useYArray"; + +const websocketEndpoint = + process.env.NODE_ENV === "development" + ? "ws://localhost:4000" + : "wss://secsync.fly.dev"; + +type Props = { + documentId: string; + showDevTool: boolean; +}; + +export const YjsPendingChangesCallbackExample: React.FC = ({ + documentId, + showDevTool, +}) => { + const documentKey = sodium.from_base64( + "MTcyipWZ6Kiibd5fATw55i9wyEU7KbdDoTE_MRgDR98" + ); + + const [authorKeyPair] = useState(() => { + return sodium.crypto_sign_keypair(); + }); + + const yDocRef = useRef(new Yjs.Doc()); + const yTodos: Yjs.Array = yDocRef.current.getArray("todos"); + const todos = useYArray(yTodos); + const [newTodoText, setNewTodoText] = useState(""); + + const [state, send] = useYjsSync({ + onPendingChangesUpdated: (allChanges) => { + console.log("pending changes", allChanges); + }, + yDoc: yDocRef.current, + documentId, + signatureKeyPair: authorKeyPair, + websocketEndpoint, + websocketSessionKey: "your-secret-session-key", + getNewSnapshotData: async ({ id }) => { + return { + data: Yjs.encodeStateAsUpdateV2(yDocRef.current), + key: documentKey, + publicData: {}, + }; + }, + getSnapshotKey: async () => { + return documentKey; + }, + shouldSendSnapshot: ({ snapshotUpdatesCount }) => { + // create a new snapshot if the active snapshot has more than 100 updates + return snapshotUpdatesCount > 3; + }, + isValidClient: async (signingPublicKey: string) => { + return true; + }, + sodium, + logging: "error", + }); + + return ( + <> +
+
{ + event.preventDefault(); + yTodos.push([newTodoText]); + setNewTodoText(""); + }} + > + setNewTodoText(event.target.value)} + value={newTodoText} + className="new-todo" + /> + +
+ +
    + {todos.map((entry, index) => { + return ( +
  • +
    {entry}
    +
    • + ); + })} +
+
+ +
+ + + ); +}; diff --git a/documentation/pages/docs/_meta.json b/documentation/pages/docs/_meta.json index 0804d13..5a0dca7 100644 --- a/documentation/pages/docs/_meta.json +++ b/documentation/pages/docs/_meta.json @@ -1,6 +1,7 @@ { "getting-started": "Getting started", "integration-examples": "Integration Examples", + "other-examples": "Other Examples", "api": "API", "error-handling": "Error Handling", "security_and_privacy": "Security & Privacy", diff --git a/documentation/pages/docs/api/client.mdx b/documentation/pages/docs/api/client.mdx index af9682b..aebbf9e 100644 --- a/documentation/pages/docs/api/client.mdx +++ b/documentation/pages/docs/api/client.mdx @@ -229,6 +229,16 @@ The `knownSnapshotInfo` can be stored locally in order to provide it to `loadDoc }) => void | Promise ``` +#### onPendingChangesUpdated + +Required: `false` + +A callback that is invoked every time the pending changes of a document are updated. It combines all unsynced changes, changes from a snapshot in flight and changes from updates in flight. + +```ts +(pendingChanges: any[]) => void | Promise +``` + #### loadDocumentParams Required: `false` diff --git a/documentation/pages/docs/other-examples/pending-changes.mdx b/documentation/pages/docs/other-examples/pending-changes.mdx new file mode 100644 index 0000000..2492ebf --- /dev/null +++ b/documentation/pages/docs/other-examples/pending-changes.mdx @@ -0,0 +1,17 @@ +## Pending Changes Example + +This example shows how to use the `onPendingChangesUpdated` callback to get the unsynced changes. This is useful to update a local storage to avoid changes getting lost e.g. in a local-first setting. + +See the `console.log` output in the browser console to see the pending changes. + +import SimpleExampleWrapper from "../../../components/SimpleExampleWrapper"; +import { YjsPendingChangesCallbackExample } from "../../../components/YjsPendingChangesCallbackExample/YjsPendingChangesCallbackExample"; + + + +## Code + +The code for this example is located [here](https://github.com/serenity-kit/secsync/blob/main/documentation/components/YjsPendingChangesCallbackExample/YjsPendingChangesCallbackExample.tsx). diff --git a/packages/secsync/src/createSyncMachine.ts b/packages/secsync/src/createSyncMachine.ts index 5548d51..cc49787 100644 --- a/packages/secsync/src/createSyncMachine.ts +++ b/packages/secsync/src/createSyncMachine.ts @@ -1174,6 +1174,16 @@ export const createSyncMachine = () => { }; }, actions: { + invokePendingChangesUpdatedCallback: ({ context, event }) => { + const allPendingChanges = [ + ...context._pendingChangesQueue, + ...(context._snapshotInFlight?.changes || []), + ...context._updatesInFlight.reduce((accumulator, updateInFlight) => { + return [...accumulator, ...updateInFlight.changes]; + }, [] as any[]), + ]; + context.onPendingChangesUpdated?.(allPendingChanges); + }, resetWebsocketRetries: assign({ _websocketRetries: 0, }), @@ -1293,10 +1303,10 @@ export const createSyncMachine = () => { if (castedEvent.output.handledQueue === "incoming") { return { _incomingQueue: context._incomingQueue.slice(1), - // because changes might have been add while processing the queue ans creating a - // snapshot or update we can't just overwrite the _pendingChangesQueue - // instead we need to track how many to remove from the beginning of the list - // and of some should be restored also add them to the list + // Because changes might have been added while processing the queue and therefor + // creating a snapshot or update we can't just overwrite the _pendingChangesQueue. + // Instead we need to track how many to remove from the beginning of the list + // and of some should be restored also add them to the list. _pendingChangesQueue: castedEvent.output.pendingChangesToPrepend.concat( context._pendingChangesQueue.slice( @@ -1321,10 +1331,10 @@ export const createSyncMachine = () => { } else if (castedEvent.output.handledQueue === "customMessage") { return { _customMessageQueue: context._customMessageQueue.slice(1), - // because changes might have been add while processing the queue ans creating a - // snapshot or update we can't just overwrite the _pendingChangesQueue - // instead we need to track how many to remove from the beginning of the list - // and of some should be restored also add them to the list + // Because changes might have been added while processing the queue and therefor + // creating a snapshot or update we can't just overwrite the _pendingChangesQueue. + // Instead we need to track how many to remove from the beginning of the list + // and of some should be restored also add them to the list. _pendingChangesQueue: castedEvent.output.pendingChangesToPrepend.concat( context._pendingChangesQueue.slice( @@ -1348,10 +1358,10 @@ export const createSyncMachine = () => { }; } else if (castedEvent.output.handledQueue === "pending") { return { - // because changes might have been add while processing the queue ans creating a - // snapshot or update we can't just overwrite the _pendingChangesQueue - // instead we need to track how many to remove from the beginning of the list - // and of some should be restored also add them to the list + // Because changes might have been added while processing the queue and therefor + // creating a snapshot or update we can't just overwrite the _pendingChangesQueue. + // Instead we need to track how many to remove from the beginning of the list + // and of some should be restored also add them to the list. _pendingChangesQueue: castedEvent.output.pendingChangesToPrepend.concat( context._pendingChangesQueue.slice( @@ -1510,7 +1520,10 @@ export const createSyncMachine = () => { target: "connected", }, ADD_CHANGES: { - actions: ["addToPendingUpdatesQueue"], + actions: [ + "addToPendingUpdatesQueue", + "invokePendingChangesUpdatedCallback", + ], }, }, }, @@ -1528,7 +1541,10 @@ export const createSyncMachine = () => { target: "processingQueues", }, ADD_CHANGES: { - actions: ["addToPendingUpdatesQueue"], + actions: [ + "addToPendingUpdatesQueue", + "invokePendingChangesUpdatedCallback", + ], target: "processingQueues", }, }, @@ -1542,7 +1558,10 @@ export const createSyncMachine = () => { actions: ["addToCustomMessageQueue"], }, ADD_CHANGES: { - actions: ["addToPendingUpdatesQueue"], + actions: [ + "addToPendingUpdatesQueue", + "invokePendingChangesUpdatedCallback", + ], }, }, invoke: { @@ -1574,27 +1593,36 @@ export const createSyncMachine = () => { ) ); }, - actions: ["removeOldestItemFromQueueAndUpdateContext"], + actions: [ + "removeOldestItemFromQueueAndUpdateContext", + "invokePendingChangesUpdatedCallback", + ], target: "checkingForMoreQueueItems", }, { - actions: ["removeOldestItemFromQueueAndUpdateContext"], + actions: [ + "removeOldestItemFromQueueAndUpdateContext", + "invokePendingChangesUpdatedCallback", + ], target: "#syncMachine.disconnected", }, ], onError: { - actions: assign(({ context, event }) => { - return { - _documentDecryptionState: - // @ts-expect-error documentDecryptionState is dynamically added to the error event - event.error?.documentDecryptionState || - context._documentDecryptionState, - _snapshotAndUpdateErrors: [ - event.error as Error, - ...context._snapshotAndUpdateErrors, - ], - }; - }), + actions: [ + assign(({ context, event }) => { + return { + _documentDecryptionState: + // @ts-expect-error documentDecryptionState is dynamically added to the error event + event.error?.documentDecryptionState || + context._documentDecryptionState, + _snapshotAndUpdateErrors: [ + event.error as Error, + ...context._snapshotAndUpdateErrors, + ], + }; + }), + "invokePendingChangesUpdatedCallback", + ], target: "#syncMachine.failed", }, }, @@ -1645,7 +1673,10 @@ export const createSyncMachine = () => { }, on: { ADD_CHANGES: { - actions: ["addToPendingUpdatesQueue"], + actions: [ + "addToPendingUpdatesQueue", + "invokePendingChangesUpdatedCallback", + ], }, CONNECT: { target: "connecting", diff --git a/packages/secsync/src/types.ts b/packages/secsync/src/types.ts index 91351a3..fe2e2fc 100644 --- a/packages/secsync/src/types.ts +++ b/packages/secsync/src/types.ts @@ -154,6 +154,7 @@ export type SyncMachineConfig = { type: OnDocumentUpdatedEventType; knownSnapshotInfo: SnapshotInfoWithUpdateClocks; }) => void | Promise; + onPendingChangesUpdated?: (allChanges: any[]) => void; onCustomMessage?: (message: any) => Promise | void; loadDocumentParams?: LoadDocumentParams; additionalAuthenticationDataValidations?: AdditionalAuthenticationDataValidations; From b041ba3c840e63bfd25712888521ec349a5d71a7 Mon Sep 17 00:00:00 2001 From: Nik Graf Date: Mon, 3 Jun 2024 22:35:58 +0200 Subject: [PATCH 2/8] fix sync crashing due sending events to remove websocket actor --- packages/secsync/src/createSyncMachine.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/secsync/src/createSyncMachine.ts b/packages/secsync/src/createSyncMachine.ts index cc49787..f9808c6 100644 --- a/packages/secsync/src/createSyncMachine.ts +++ b/packages/secsync/src/createSyncMachine.ts @@ -1416,6 +1416,9 @@ export const createSyncMachine = () => { shouldReconnect: ({ context }) => { return context._websocketShouldReconnect; }, + hasActiveWebsocket: ({ context }) => { + return Boolean(context._websocketActor); + }, }, }).createMachine({ context: ({ input }) => { @@ -1467,9 +1470,11 @@ export const createSyncMachine = () => { initial: "connecting", on: { SEND: { + guard: "hasActiveWebsocket", actions: forwardTo("websocketActor"), }, ADD_EPHEMERAL_MESSAGE: { + guard: "hasActiveWebsocket", actions: sendTo("websocketActor", ({ context, event }) => { return { type: "SEND_EPHEMERAL_MESSAGE", From 386a0f49a88b9343671858c5cf5e72f7027221a7 Mon Sep 17 00:00:00 2001 From: Nik Graf Date: Tue, 4 Jun 2024 08:24:48 +0200 Subject: [PATCH 3/8] update changelog --- packages/secsync/CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/secsync/CHANGELOG.md b/packages/secsync/CHANGELOG.md index 7e2537a..4dde6ee 100644 --- a/packages/secsync/CHANGELOG.md +++ b/packages/secsync/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- added `onPendingChangesUpdated` callback + +### Fixed + +- fix sync crashing due sending events to remove websocket actor + ## [0.4.0] - 2024-06-01 ### Changed From 4742e314481433cc72b3f64a45d647a10b72be8e Mon Sep 17 00:00:00 2001 From: Nik Graf Date: Tue, 4 Jun 2024 10:27:28 +0200 Subject: [PATCH 4/8] add pendingChanges and implement local-first example --- .../YjsLocalFirstExample.tsx} | 57 +++++++++++++++---- .../YjsLocalFirstExample/deserialize.ts | 12 ++++ .../YjsLocalFirstExample/serialize.ts | 8 +++ documentation/pages/docs/api/client.mdx | 7 +++ .../docs/other-examples/pending-changes.mdx | 17 ------ .../docs/other-examples/yjs-local-first.mdx | 17 ++++++ packages/secsync/CHANGELOG.md | 1 + packages/secsync/src/createSyncMachine.ts | 2 +- packages/secsync/src/types.ts | 1 + 9 files changed, 94 insertions(+), 28 deletions(-) rename documentation/components/{YjsPendingChangesCallbackExample/YjsPendingChangesCallbackExample.tsx => YjsLocalFirstExample/YjsLocalFirstExample.tsx} (63%) create mode 100644 documentation/components/YjsLocalFirstExample/deserialize.ts create mode 100644 documentation/components/YjsLocalFirstExample/serialize.ts delete mode 100644 documentation/pages/docs/other-examples/pending-changes.mdx create mode 100644 documentation/pages/docs/other-examples/yjs-local-first.mdx diff --git a/documentation/components/YjsPendingChangesCallbackExample/YjsPendingChangesCallbackExample.tsx b/documentation/components/YjsLocalFirstExample/YjsLocalFirstExample.tsx similarity index 63% rename from documentation/components/YjsPendingChangesCallbackExample/YjsPendingChangesCallbackExample.tsx rename to documentation/components/YjsLocalFirstExample/YjsLocalFirstExample.tsx index ce62e02..0184d8f 100644 --- a/documentation/components/YjsPendingChangesCallbackExample/YjsPendingChangesCallbackExample.tsx +++ b/documentation/components/YjsLocalFirstExample/YjsLocalFirstExample.tsx @@ -1,9 +1,11 @@ import sodium, { KeyPair } from "libsodium-wrappers"; -import React, { useRef, useState } from "react"; +import React, { useEffect, useRef, useState } from "react"; import { DevTool } from "secsync-react-devtool"; import { useYjsSync } from "secsync-react-yjs"; import * as Yjs from "yjs"; import { useYArray } from "../../hooks/useYArray"; +import { deserialize } from "./deserialize"; +import { serialize } from "./serialize"; const websocketEndpoint = process.env.NODE_ENV === "development" @@ -15,26 +17,58 @@ type Props = { showDevTool: boolean; }; -export const YjsPendingChangesCallbackExample: React.FC = ({ +export const YjsLocalFirstExample: React.FC = ({ documentId, showDevTool, }) => { const documentKey = sodium.from_base64( "MTcyipWZ6Kiibd5fATw55i9wyEU7KbdDoTE_MRgDR98" ); - + const [newTodoText, setNewTodoText] = useState(""); const [authorKeyPair] = useState(() => { return sodium.crypto_sign_keypair(); }); - const yDocRef = useRef(new Yjs.Doc()); - const yTodos: Yjs.Array = yDocRef.current.getArray("todos"); - const todos = useYArray(yTodos); - const [newTodoText, setNewTodoText] = useState(""); + // load initial data from localStorage + const [initialData] = useState(() => { + const yDoc = new Yjs.Doc(); + // load full document + const serializedDoc = localStorage.getItem(`doc:state:${documentId}`); + if (serializedDoc) { + Yjs.applyUpdateV2(yDoc, deserialize(serializedDoc)); + } + + // loads the pendingChanges from localStorage + const pendingChanges = localStorage.getItem(`doc:pending:${documentId}`); + + return { + yDoc, + pendingChanges: pendingChanges ? deserialize(pendingChanges) : [], + }; + }); + + // create the yDocRef + const yDocRef = useRef(initialData.yDoc); + + // update the document in localStorage after every change (could be debounced) + useEffect(() => { + const onUpdate = (update: any) => { + const fullYDoc = Yjs.encodeStateAsUpdateV2(yDocRef.current); + localStorage.setItem(`doc:state:${documentId}`, serialize(fullYDoc)); + }; + yDocRef.current.on("updateV2", onUpdate); + + return () => { + yDocRef.current.off("updateV2", onUpdate); + }; + }, []); const [state, send] = useYjsSync({ + // pass in the pending changes + pendingChanges: initialData.pendingChanges, + // callback to store the pending changes in onPendingChangesUpdated: (allChanges) => { - console.log("pending changes", allChanges); + localStorage.setItem(`doc:pending:${documentId}`, serialize(allChanges)); }, yDoc: yDocRef.current, documentId, @@ -53,15 +87,18 @@ export const YjsPendingChangesCallbackExample: React.FC = ({ }, shouldSendSnapshot: ({ snapshotUpdatesCount }) => { // create a new snapshot if the active snapshot has more than 100 updates - return snapshotUpdatesCount > 3; + return snapshotUpdatesCount > 10; }, isValidClient: async (signingPublicKey: string) => { return true; }, sodium, - logging: "error", + logging: "debug", }); + const yTodos: Yjs.Array = yDocRef.current.getArray("todos"); + const todos = useYArray(yTodos); + return ( <>
diff --git a/documentation/components/YjsLocalFirstExample/deserialize.ts b/documentation/components/YjsLocalFirstExample/deserialize.ts new file mode 100644 index 0000000..5a50270 --- /dev/null +++ b/documentation/components/YjsLocalFirstExample/deserialize.ts @@ -0,0 +1,12 @@ +export const deserialize = (data: string) => { + return JSON.parse(data, (key, value) => { + if ( + typeof value === "object" && + value !== null && + value.type === "Uint8Array" + ) { + return new Uint8Array(value.data); + } + return value; + }); +}; diff --git a/documentation/components/YjsLocalFirstExample/serialize.ts b/documentation/components/YjsLocalFirstExample/serialize.ts new file mode 100644 index 0000000..8f9053e --- /dev/null +++ b/documentation/components/YjsLocalFirstExample/serialize.ts @@ -0,0 +1,8 @@ +export const serialize = (data: any) => { + return JSON.stringify(data, (key, value) => { + if (value instanceof Uint8Array) { + return { type: "Uint8Array", data: Array.from(value) }; + } + return value; + }); +}; diff --git a/documentation/pages/docs/api/client.mdx b/documentation/pages/docs/api/client.mdx index aebbf9e..e145d34 100644 --- a/documentation/pages/docs/api/client.mdx +++ b/documentation/pages/docs/api/client.mdx @@ -239,6 +239,13 @@ A callback that is invoked every time the pending changes of a document are upda (pendingChanges: any[]) => void | Promise ``` +#### pendingChanges + +Required: `false` + +When initializing secsync `pendingChanges` can be passed in. These changes will be synced as soon as a connection is established. The pendingChanges can be +received via the `onPendingChangesUpdated` callback. + #### loadDocumentParams Required: `false` diff --git a/documentation/pages/docs/other-examples/pending-changes.mdx b/documentation/pages/docs/other-examples/pending-changes.mdx deleted file mode 100644 index 2492ebf..0000000 --- a/documentation/pages/docs/other-examples/pending-changes.mdx +++ /dev/null @@ -1,17 +0,0 @@ -## Pending Changes Example - -This example shows how to use the `onPendingChangesUpdated` callback to get the unsynced changes. This is useful to update a local storage to avoid changes getting lost e.g. in a local-first setting. - -See the `console.log` output in the browser console to see the pending changes. - -import SimpleExampleWrapper from "../../../components/SimpleExampleWrapper"; -import { YjsPendingChangesCallbackExample } from "../../../components/YjsPendingChangesCallbackExample/YjsPendingChangesCallbackExample"; - - - -## Code - -The code for this example is located [here](https://github.com/serenity-kit/secsync/blob/main/documentation/components/YjsPendingChangesCallbackExample/YjsPendingChangesCallbackExample.tsx). diff --git a/documentation/pages/docs/other-examples/yjs-local-first.mdx b/documentation/pages/docs/other-examples/yjs-local-first.mdx new file mode 100644 index 0000000..3fd412f --- /dev/null +++ b/documentation/pages/docs/other-examples/yjs-local-first.mdx @@ -0,0 +1,17 @@ +## Yjs Local-first Example + +This example shows how to use secsync in a local-first setup. In this example the full document and the `pendingChanges` are stored in localStorage. In case the API is not available the changes won't be lost. Once re-connected to the API, Secsync will send the pending changes. + +This makes use of the `pendingChanges` parameter and `onPendingChangesUpdated` callback parameter. + +import SimpleExampleWrapper from "../../../components/SimpleExampleWrapper"; +import { YjsLocalFirstExample } from "../../../components/YjsLocalFirstExample/YjsLocalFirstExample"; + + + +## Code + +The code for this example is located [here](https://github.com/serenity-kit/secsync/blob/main/documentation/components/YjsLocalFirstExample/YjsLocalFirstExample.tsx). diff --git a/packages/secsync/CHANGELOG.md b/packages/secsync/CHANGELOG.md index 4dde6ee..87dd2d1 100644 --- a/packages/secsync/CHANGELOG.md +++ b/packages/secsync/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - added `onPendingChangesUpdated` callback +- added `pendingChanges` config ### Fixed diff --git a/packages/secsync/src/createSyncMachine.ts b/packages/secsync/src/createSyncMachine.ts index f9808c6..0146e49 100644 --- a/packages/secsync/src/createSyncMachine.ts +++ b/packages/secsync/src/createSyncMachine.ts @@ -1451,7 +1451,7 @@ export const createSyncMachine = () => { _snapshotInFlight: null, // it is needed so the the snapshotInFlight can be applied as the activeSnapshot once the server confirmed that it has been saved _incomingQueue: [], _customMessageQueue: [], - _pendingChangesQueue: [], + _pendingChangesQueue: input.pendingChanges || [], _snapshotInfosWithUpdateClocks: [], _websocketShouldReconnect: false, _websocketRetries: 0, diff --git a/packages/secsync/src/types.ts b/packages/secsync/src/types.ts index fe2e2fc..1b4f793 100644 --- a/packages/secsync/src/types.ts +++ b/packages/secsync/src/types.ts @@ -158,6 +158,7 @@ export type SyncMachineConfig = { onCustomMessage?: (message: any) => Promise | void; loadDocumentParams?: LoadDocumentParams; additionalAuthenticationDataValidations?: AdditionalAuthenticationDataValidations; + pendingChanges?: any[]; /** default: "off" */ logging?: "off" | "error" | "debug"; }; From c1e4ac8988742de3ec79a13b8d634858126bf1f6 Mon Sep 17 00:00:00 2001 From: Nik Graf Date: Tue, 4 Jun 2024 10:59:53 +0200 Subject: [PATCH 5/8] fix specification --- documentation/pages/docs/specification.mdx | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/documentation/pages/docs/specification.mdx b/documentation/pages/docs/specification.mdx index e4d6e8e..8bfbba0 100644 --- a/documentation/pages/docs/specification.mdx +++ b/documentation/pages/docs/specification.mdx @@ -9,11 +9,8 @@ The public data of a message always contains the document ID in order to be able ## Encryption All messages use the AEAD xchacha20poly1305_ietf construction. The nonce is always public and must be included along with the ciphertext. -In order to commit to one (plaintext, AAD) pair matching only one (ciphertext, authentication tag) pair every message to be encrypted is prefixed with 4 NUL bytes. After every decryption the prefix is verified and an error thrown in case it's not valid. - -**TODO**: still needs to verified with a cryptographer if this technique and especially if 4 NUL bytes are sufficient -read more here https://soatok.blog/2023/04/03/asymmetric-cryptographic-commitments/#what-is-commitment and -here https://eprint.iacr.org/2019/016 +In order to commit to one (plaintext, AAD) pair matching only one (ciphertext, authentication tag) pair every message is prefixed with a commitment tag. +It is generated from an `HMAC(nonce + ciphertext + AAD, key)`. Before every decryption the commitment tag is verified and an error thrown in case it's not valid. While the AEAD construction already includes a MAC we additionally sign the data to verify its author. This is redundant and would not be necessary, but has been chosen to be able to rely on established constructions instead of implementing our own Signcryption like https://github.com/jedisct1/libsodium-signcryption. From 4cce07d7d545679760c51fa66d0e805c58d156b5 Mon Sep 17 00:00:00 2001 From: Nik Graf Date: Tue, 4 Jun 2024 12:39:36 +0200 Subject: [PATCH 6/8] update security explaination --- .../pages/docs/security_and_privacy/considerations.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/pages/docs/security_and_privacy/considerations.mdx b/documentation/pages/docs/security_and_privacy/considerations.mdx index c3b38dc..08c31e3 100644 --- a/documentation/pages/docs/security_and_privacy/considerations.mdx +++ b/documentation/pages/docs/security_and_privacy/considerations.mdx @@ -52,7 +52,7 @@ Secsync does not provide any metadata protection guarantees. ## Cryptographic Analysis -Security guarantees have been analyzed using [Verifpal](https://verifpal.com/) and can be found at [https://github.com/serenity-kit/secsync/tree/main/verifpal](https://github.com/serenity-kit/secsync/tree/main/verifpal). The proofs show the confidentiality and freshness between clients in two scenarios: +Security guarantees have been analyzed using [Verifpal](https://verifpal.com/) and can be found at [https://github.com/serenity-kit/secsync/tree/main/verifpal_and_threat_library](https://github.com/serenity-kit/secsync/tree/main/verifpal_and_threat_library). The proofs show the confidentiality and freshness between clients in two scenarios: 1. Client A sends a message to client B and the public key of each message is NOT verified to belong to a known client with access to the document. 1. Client A sends a message to client B and the public key of each message is verified to belong to a known client with access to the document. From a49585d3c5578df82fd129e9c828654b7b276da0 Mon Sep 17 00:00:00 2001 From: Nik Graf Date: Tue, 4 Jun 2024 12:39:51 +0200 Subject: [PATCH 7/8] add authorized authors example --- .../AuthorizedAuthorsExample.tsx | 213 ++++++++++++++++++ .../other-examples/authorized-authors.mdx | 24 ++ 2 files changed, 237 insertions(+) create mode 100644 documentation/components/AuthorizedAuthorsExample/AuthorizedAuthorsExample.tsx create mode 100644 documentation/pages/docs/other-examples/authorized-authors.mdx diff --git a/documentation/components/AuthorizedAuthorsExample/AuthorizedAuthorsExample.tsx b/documentation/components/AuthorizedAuthorsExample/AuthorizedAuthorsExample.tsx new file mode 100644 index 0000000..f8b0447 --- /dev/null +++ b/documentation/components/AuthorizedAuthorsExample/AuthorizedAuthorsExample.tsx @@ -0,0 +1,213 @@ +import sodium from "libsodium-wrappers"; +import React, { useId, useRef, useState } from "react"; +import { DevTool } from "secsync-react-devtool"; +import { useYjsSync } from "secsync-react-yjs"; +import * as Yjs from "yjs"; +import { useYArray } from "../../hooks/useYArray"; + +const websocketEndpoint = + process.env.NODE_ENV === "development" + ? "ws://localhost:4000" + : "wss://secsync.fly.dev"; + +type Props = { + documentId: string; + showDevTool: boolean; +}; + +const authorizedAuthors = [ + { + privateKey: + "90gI4rbA8cApZe72j3oJ0f31ymuleLuZdsaKm64jEUwlcH4y5KshGsNcNaWzQaBJKp3cHdUEnnP3bgXloyOytA", + publicKey: "JXB-MuSrIRrDXDWls0GgSSqd3B3VBJ5z924F5aMjsrQ", + }, + { + privateKey: + "dko_5CR064h36HQmOuYPcBQIS-xdM7wSQJAJjCnIO9SvQ-tKBL_BvFFw-AhkHPSKRPb3F6kw5kfTV3GGJ4awYg", + publicKey: "r0PrSgS_wbxRcPgIZBz0ikT29xepMOZH01dxhieGsGI", + }, + { + privateKey: + "rTzxM6i7bVjH3eG3jqvyI1E6ZFzJhNYsvUwiJxAfTlOC5jWTkIieCdRxpG3mXySar6HS1z5bL0Rdx1azVTr3jw", + publicKey: "guY1k5CIngnUcaRt5l8kmq-h0tc-Wy9EXcdWs1U6948", + }, + { + privateKey: + "MH_NDZ4aUfCT0cpCr-tZ8Hhvh4MYpw56IZkdbwVmUQn0wCzxJ_IYw_IzyZ3kxdOXIQFtg-UxykGK8hjiA-x1hg", + publicKey: "9MAs8SfyGMPyM8md5MXTlyEBbYPlMcpBivIY4gPsdYY", + }, + { + privateKey: + "ZcknsT8b9HdksFl9SxQbs0soLsWfNoGIGP52E4-8FcIQSgu83cXNvcqcYPAFO0wUatx_h19GM34sz-8u5RfejA", + publicKey: "EEoLvN3Fzb3KnGDwBTtMFGrcf4dfRjN-LM_vLuUX3ow", + }, +]; + +const nonAuthorizedAuthors = [ + { + privateKey: + "wjNC0kxBFJzaBN02Gr3a87pJlGUj6LTAM4PNMT2hFTxuGHbOc48VvXd4lhLJELmV4q7ahne0H3nCs271MSx_mA", + publicKey: "bhh2znOPFb13eJYSyRC5leKu2oZ3tB95wrNu9TEsf5g", + }, +]; + +export const AuthorizedAuthorsExample: React.FC = ({ + documentId, + showDevTool, +}) => { + const [authorKeyPair, setAuthorKeyPair] = useState( + null + ); + + const selectId = useId(); + + if (authorKeyPair === null) { + return ( +
+
+ + + +

+ NOTE: Choosing the same author in multiple clients will result in + errors when trying to create update in parallel. +

+
+
+ ); + } + + return ; +}; + +type TodosProps = { + documentId: string; + authorKeyPair: sodium.KeyPair; +}; + +export const Todos: React.FC = ({ documentId, authorKeyPair }) => { + const documentKey = sodium.from_base64( + "MTcyipWZ6Kiibd5fATw55i9wyEU7KbdDoTE_MRgDR98" + ); + + const yDocRef = useRef(new Yjs.Doc()); + const yTodos: Yjs.Array = yDocRef.current.getArray("todos"); + const todos = useYArray(yTodos); + const [newTodoText, setNewTodoText] = useState(""); + + const [state, send] = useYjsSync({ + yDoc: yDocRef.current, + documentId, + signatureKeyPair: authorKeyPair, + websocketEndpoint, + websocketSessionKey: "your-secret-session-key", + getNewSnapshotData: async ({ id }) => { + return { + data: Yjs.encodeStateAsUpdateV2(yDocRef.current), + key: documentKey, + publicData: {}, + }; + }, + getSnapshotKey: async () => { + return documentKey; + }, + shouldSendSnapshot: ({ snapshotUpdatesCount }) => { + // create a new snapshot if the active snapshot has more than 100 updates + return snapshotUpdatesCount > 100; + }, + isValidClient: async (signingPublicKey: string) => { + return authorizedAuthors.some((author) => { + console.log( + author.publicKey, + signingPublicKey, + author.publicKey === signingPublicKey + ); + return author.publicKey === signingPublicKey; + }); + }, + sodium, + logging: "debug", + }); + + return ( + <> +
+
{ + event.preventDefault(); + yTodos.push([newTodoText]); + setNewTodoText(""); + }} + > + setNewTodoText(event.target.value)} + value={newTodoText} + className="new-todo" + /> + +
+ +
    + {todos.map((entry, index) => { + return ( +
  • +
    {entry}
    +
    • + ); + })} +
+
+ +
+ + + ); +}; diff --git a/documentation/pages/docs/other-examples/authorized-authors.mdx b/documentation/pages/docs/other-examples/authorized-authors.mdx new file mode 100644 index 0000000..e1ac6b5 --- /dev/null +++ b/documentation/pages/docs/other-examples/authorized-authors.mdx @@ -0,0 +1,24 @@ +## Authorized Authors Example + +This example demonstrates how to only allow a fixed set of authors. It leverages the `isValidClient` callback which will be invoked vor every update. In case all `snapshots`, `updates` and `ephemeralMessages` pass the `isValidClient` callback and `true` is returned all changes are accepted. In case `isValidClient` returns `false` or an error is thrown the change is rejected. Depending on the case it's simply ignored or the document ends up in a broken stage. More details can be found at [Error Handling](/docs/error-handling). + +Example: + +1. Select the first authorized author and add an item in the list +2. Open another tab and select the first non-authorized author and add an item in the list + +The entry will not be synced to the first tab and in the `DevTool` box below you can see the errors. You can also refresh and see how the document is reconstructed (except the last update was a snapshot). In this case the previous snapshot could be loaded, but that case is not handled in this example. + +NOTE: Choosing the same author in multiple clients will result in errors when trying to create update in parallel. + +import SimpleExampleWrapper from "../../../components/SimpleExampleWrapper"; +import { AuthorizedAuthorsExample } from "../../../components/AuthorizedAuthorsExample/AuthorizedAuthorsExample"; + + + +## Code + +The code for this example is located [here](https://github.com/serenity-kit/secsync/blob/main/documentation/components/AuthorizedAuthorsExample/AuthorizedAuthorsExample.tsx). From 75fc486e431f207245111320a684ea385f5ca891 Mon Sep 17 00:00:00 2001 From: Nik Graf Date: Tue, 4 Jun 2024 14:03:33 +0200 Subject: [PATCH 8/8] add logo and improve homepage style --- documentation/components/Logo.tsx | 4 +++- documentation/pages/_meta.json | 3 +++ documentation/public/favicon.png | Bin 689 -> 835 bytes documentation/public/favicon.svg | 10 +++++++--- documentation/public/secsync-logo.svg | 9 +++++++++ documentation/theme.config.tsx | 2 +- 6 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 documentation/public/secsync-logo.svg diff --git a/documentation/components/Logo.tsx b/documentation/components/Logo.tsx index 73cacde..018cbd2 100644 --- a/documentation/components/Logo.tsx +++ b/documentation/components/Logo.tsx @@ -4,10 +4,12 @@ export type LogoProps = { hoverEffect?: boolean; }; +import LogoSvg from "../public/secsync-logo.svg"; + export const Logo = ({ color = "currentColor", height = 20 }: LogoProps) => { return (
- Secsync +
); }; diff --git a/documentation/pages/_meta.json b/documentation/pages/_meta.json index 5a0f328..f0e5d35 100644 --- a/documentation/pages/_meta.json +++ b/documentation/pages/_meta.json @@ -2,6 +2,9 @@ "index": { "title": "Secsync", "type": "page", + "theme": { + "typesetting": "article" + }, "display": "hidden" }, "docs": { diff --git a/documentation/public/favicon.png b/documentation/public/favicon.png index 7600090371c3242a6a100fd89d5d3ad763a2ca65..b711c962f2246c70811865dae365cb9182341d7c 100644 GIT binary patch delta 827 zcmV-B1H}BX1;Yj*iBL{Q4GJ0x0000DNk~Le0000w0000x2nGNE04nF1jgjpae*gh< zQb$4nuFf3k00004XF*Lt006O%3;baP0008mNkl~`Fxhs>7<_bdOaBqhth7had5lcf~v796p_tlGf|`-p;s#xYfUI35s)QP54>I_){@a?vytodn)~mG zVMbM06S`inr6?>;Ram)UxZm&6YPF;&WZ-biOBGo0C_?Lw6`^6K6%(qgf2r1lhFMgK zV!A*kuNi@u$Y|cznFh^E9L5dg0M%WVHBt9*G)N zR^;pCSu7Td)Vadsq689mruN{xC!u9x>vdpQ9`c@c$_#WpgtC=vzd@W=5ysA0E|+RC z-Yqg}i@IUh?RH`tYAQ92+N*2ur&>kzzN~arE1P%N3?upTsHh(me;A)vpw8OZ#bXi{ zNvX`7Dn^N$Pl{tV(NRUq(km2y)A@B`Y#_AL=|~k2v$`kkcDw3#YJVC58;jgvXgKex zKKflT%9bIU;|-dHJbi2$CUu?;O{0XCX~vU=F4Q(DQU;?UA3~{XwuP4UU=msP#_Xim zh91;;lsZEU*s61!e=)OV!;#^me5PQQgolXl0q&%_p>rCRE5w$O?>>!XusbfsOl6ecLZve?|x-%ie(bNSwWJ?9znhg z3{`3j3=J&|48MRv4KElNN(~qoUL`OvSj}Ky5HFasE6@fg(UKbBnda-upao=eFt9QT zF)#yJj6lf1D8&FW4aj2fVw8rngBUfSYM2-p+A|qgplYIkGzfSAF-Q-DW?sOEFmVAB zT(!aiW&|6gEq)LG2Oz~+;1OBOz`!jG!i)^F=12fdi_8p(D2ed(u}aR*)k{ptPfFFR z$SnZrVz8;O0y1+`OA-|-a&z*EttxDlz$&bOY>=?Nk^)#sNw%$0gl~X?bAC~(f|;Iy zo`I4bmx6+VO;JjkRgjAtR6CGotCUevQedU8UtV6WS8lAAUzDzIXlZGwZ(yWvWTXpJ zp<7&;SCUwvn^&w1F$89gOKNd)QD#9&W`3Rm$lS!F{L&IzB_)tWZ~$>KW+6%?4_<0f}1q7iFdbZ3dZdXJ`Xfi6REI$3`DyIg(=_J_U;cy=up0 zqYn=@J1)t%hwQ+RH1c$D42j@;d%=+xm=u@|5A07@nDOAU - - + + + + + + + diff --git a/documentation/public/secsync-logo.svg b/documentation/public/secsync-logo.svg new file mode 100644 index 0000000..397c0ae --- /dev/null +++ b/documentation/public/secsync-logo.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/documentation/theme.config.tsx b/documentation/theme.config.tsx index bb49978..01c4a4d 100644 --- a/documentation/theme.config.tsx +++ b/documentation/theme.config.tsx @@ -17,7 +17,7 @@ const config: DocsThemeConfig = { ), logo: ( // wrapper needed so it looks vertically centered in header -
+
),