From c4688cf72449d7ee041f4e817165d955b9545e16 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Wed, 23 Aug 2023 14:51:57 +0100 Subject: [PATCH] WIP add coreOwnership custom getWinner and initial mapDoc Add verifyCoreOwnership function Add mapAndValidateCoreOwnership move some code around add mapDoc and getWinner options to indexWriter Add tests & small fix Fix types and use defaultGetWinner from @mapeo/sqlite-indexer --- package-lock.json | 22 ++--- package.json | 4 +- src/core-ownership.js | 96 ++++++++++++++++++++ src/index-writer/index.js | 15 +++- src/mapeo-project.js | 9 ++ tests/core-ownership.js | 184 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 315 insertions(+), 15 deletions(-) create mode 100644 src/core-ownership.js create mode 100644 tests/core-ownership.js diff --git a/package-lock.json b/package-lock.json index 7f6f39633..59abb54c7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,9 +12,9 @@ "@digidem/types": "^2.0.0", "@fastify/type-provider-typebox": "^3.3.0", "@hyperswarm/secret-stream": "^6.1.2", - "@mapeo/crypto": "^1.0.0-alpha.5", + "@mapeo/crypto": "^1.0.0-alpha.7", "@mapeo/schema": "^3.0.0-next.8", - "@mapeo/sqlite-indexer": "^1.0.0-alpha.5", + "@mapeo/sqlite-indexer": "^1.0.0-alpha.6", "@sinclair/typebox": "^0.29.6", "b4a": "^1.6.3", "base32.js": "^0.1.0", @@ -1579,10 +1579,12 @@ "integrity": "sha512-vYY5EIxCPzEXEWL/vTjdHy4g92tv1ApUQCjPJsj9gEoXLNNVwJlwwgRZisuvgFBZ3zeLzQygrbehERSpYdmFZA==" }, "node_modules/@mapeo/crypto": { - "version": "1.0.0-alpha.5", - "resolved": "https://registry.npmjs.org/@mapeo/crypto/-/crypto-1.0.0-alpha.5.tgz", - "integrity": "sha512-c1lTsJs9jSKAtY9MBi4pmRj/wdaRvIt9uFA36o5TGmjyr3LQ5uVMRwau5XXQmaPse2aUhCzrEXevoG4Hsrjp8g==", + "version": "1.0.0-alpha.7", + "resolved": "https://registry.npmjs.org/@mapeo/crypto/-/crypto-1.0.0-alpha.7.tgz", + "integrity": "sha512-O4qkHYnHxjk1G4LxI4uttidjWHQX48/d7kWAq6ukfZZkuT9j8u24RBCBWOoUfVO5qACjjooUwxv+I/qvbBqmGQ==", "dependencies": { + "@types/b4a": "^1.6.0", + "b4a": "^1.6.4", "base-x": "^3.0.9", "base32.js": "^0.1.0", "compact-encoding": "^2.5.1", @@ -1591,7 +1593,8 @@ "crc": "^3.8.0", "derive-key": "^1.0.1", "lodash": "^4.17.21", - "sodium-universal": "^3.0.4" + "sodium-universal": "^3.0.4", + "z32": "^1.0.0" } }, "node_modules/@mapeo/crypto/node_modules/sodium-universal": { @@ -1727,9 +1730,9 @@ } }, "node_modules/@mapeo/sqlite-indexer": { - "version": "1.0.0-alpha.5", - "resolved": "https://registry.npmjs.org/@mapeo/sqlite-indexer/-/sqlite-indexer-1.0.0-alpha.5.tgz", - "integrity": "sha512-80I+5Tr+3pFaxqloi9VKCtL53lH4aelrc4XodBVtObN7oNhIDYnWqVNOxlOlIXQhHY0igjCjt7Y4EIKLbHJJUg==", + "version": "1.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@mapeo/sqlite-indexer/-/sqlite-indexer-1.0.0-alpha.6.tgz", + "integrity": "sha512-iLUePxr2kHgsWfFTuJAKjTSZCRuVsIVNbQVyLEkN0pX/2dWzljCxCRvO+9rc1x+bThUas96ZAzCedqeeqC0zRw==", "dependencies": { "@types/better-sqlite3": "^7.6.4", "better-sqlite3": "^8.4.0" @@ -1857,7 +1860,6 @@ "version": "1.6.0", "resolved": "https://registry.npmjs.org/@types/b4a/-/b4a-1.6.0.tgz", "integrity": "sha512-rYU2r5nSUPyKyufWijxgTjsFp2kLCwydj2TmKU4StJeGPHS/Fs5KHgP89DNF0jddyeAbN5mdjNDqIrjIHca60g==", - "dev": true, "dependencies": { "@types/node": "*" } diff --git a/package.json b/package.json index 4a7ca71e4..ac5cd89ab 100644 --- a/package.json +++ b/package.json @@ -95,9 +95,9 @@ "@digidem/types": "^2.0.0", "@fastify/type-provider-typebox": "^3.3.0", "@hyperswarm/secret-stream": "^6.1.2", - "@mapeo/crypto": "^1.0.0-alpha.5", + "@mapeo/crypto": "^1.0.0-alpha.7", "@mapeo/schema": "^3.0.0-next.8", - "@mapeo/sqlite-indexer": "^1.0.0-alpha.5", + "@mapeo/sqlite-indexer": "^1.0.0-alpha.6", "@sinclair/typebox": "^0.29.6", "b4a": "^1.6.3", "base32.js": "^0.1.0", diff --git a/src/core-ownership.js b/src/core-ownership.js new file mode 100644 index 000000000..e8364509c --- /dev/null +++ b/src/core-ownership.js @@ -0,0 +1,96 @@ +import { verifySignature } from '@mapeo/crypto' +import { NAMESPACES } from './core-manager/index.js' +import { parseVersionId } from '@mapeo/schema' +import { defaultGetWinner } from '@mapeo/sqlite-indexer' +import assert from 'node:assert' +import sodium from 'sodium-universal' + +/** + * @typedef {Extract, { schemaName: 'coreOwnership' }>} CoreOwnershipWithSignatures + */ + +/** + * - Validate that the doc is written to the core identified by doc.authCoreId + * - Verify the signatures + * - Remove the signatures (we don't add them to the indexer) + * - Set doc.links to an empty array - this forces the indexer to treat every + * document as a fork, so getWinner is called for every doc, which resolves to + * the doc with the lowest index (e.g. the first) + * + * @param {CoreOwnershipWithSignatures} doc + * @param {import('@mapeo/schema').VersionIdObject} version + * @returns {import('@mapeo/schema').CoreOwnership} + */ +export function mapAndValidateCoreOwnership(doc, { coreKey }) { + if (doc.authCoreId !== coreKey.toString('hex')) { + throw new Error('Invalid coreOwnership record: mismatched authCoreId') + } + if (!verifyCoreOwnership(doc)) { + throw new Error('Invalid coreOwnership record: signatures are invalid') + } + // eslint-disable-next-line no-unused-vars + const { identitySignature, coreSignatures, ...docWithoutSignatures } = doc + docWithoutSignatures.links = [] + return docWithoutSignatures +} + +/** + * Verify the signatures of a coreOwnership record, which verify that the device + * with the identityKey matching the docIds does own (e.g. can write to) cores + * with the given core IDs + * + * @param {CoreOwnershipWithSignatures} doc + * @returns {boolean} + */ +function verifyCoreOwnership(doc) { + const { coreSignatures, identitySignature } = doc + for (const namespace of NAMESPACES) { + const signature = coreSignatures[namespace] + const coreKey = Buffer.from(doc[`${namespace}CoreId`], 'hex') + assert.equal( + signature.length, + sodium.crypto_sign_BYTES, + 'Invalid core ownership signature' + ) + assert.equal( + coreKey.length, + sodium.crypto_sign_PUBLICKEYBYTES, + 'Invalid core ownership coreId' + ) + const isValidSignature = verifySignature(coreKey, signature, coreKey) + if (!isValidSignature) return false + } + const identityPublicKey = Buffer.from(doc.docId, 'hex') + assert.equal(identitySignature.length, sodium.crypto_sign_BYTES) + assert.equal(identityPublicKey.length, sodium.crypto_sign_PUBLICKEYBYTES) + const isValidIdentitySignature = verifySignature( + identityPublicKey, + identitySignature, + identityPublicKey + ) + if (!isValidIdentitySignature) return false + return true +} + +/** + * For coreOwnership records, we only trust the first record written to the core. + * + * @type {NonNullable[0]['getWinner']>} + */ +export function getWinner(docA, docB) { + if ( + 'schemaName' in docA && + docA.schemaName === 'coreOwnership' && + 'schemaName' in docB && + docB.schemaName === 'coreOwnership' + ) { + // Assumes docA and docB have same coreKey, so we choose the first one + // written to the core + const docAindex = parseVersionId(docA.versionId).index + const docBindex = parseVersionId(docB.versionId).index + if (docAindex < docBindex) return docA + return docB + } else { + return defaultGetWinner(docA, docB) + } +} diff --git a/src/index-writer/index.js b/src/index-writer/index.js index 893fbf1a6..f03cb12a5 100644 --- a/src/index-writer/index.js +++ b/src/index-writer/index.js @@ -9,6 +9,9 @@ import { getBacklinkTableName } from '../schema/utils.js' /** * @typedef {import('@mapeo/schema').MapeoDoc} MapeoDoc */ +/** + * @typedef {ReturnType} MapeoDocInternal + */ /** * @template {MapeoDocTables} [TTables=MapeoDocTables] @@ -16,13 +19,17 @@ import { getBacklinkTableName } from '../schema/utils.js' export class IndexWriter { /** @type {Map} */ #indexers = new Map() + #mapDoc /** * * @param {object} opts * @param {import('better-sqlite3').Database} opts.sqlite * @param {TTables[]} opts.tables + * @param {(doc: MapeoDocInternal, version: import('@mapeo/schema').VersionIdObject) => MapeoDoc} [opts.mapDoc] optionally transform a document prior to indexing. Can also validate, if an error is thrown then the document will not be indexed + * @param {typeof import('@mapeo/sqlite-indexer').defaultGetWinner} [opts.getWinner] custom function to determine the "winner" of two forked documents. Defaults to choosing the document with the most recent `updatedAt` */ - constructor({ tables, sqlite }) { + constructor({ tables, sqlite, mapDoc = (d) => d, getWinner }) { + this.#mapDoc = mapDoc for (const table of tables) { const config = getTableConfig(table) const schemaName = /** @type {(typeof table)['_']['name']} */ ( @@ -31,6 +38,7 @@ export class IndexWriter { const indexer = new SqliteIndexer(sqlite, { docTableName: config.name, backlinkTableName: getBacklinkTableName(config.name), + getWinner, }) this.#indexers.set(schemaName, indexer) } @@ -51,9 +59,10 @@ export class IndexWriter { const queued = {} for (const { block, key, index } of entries) { try { - var doc = decode(block, { coreKey: key, index }) + const version = { coreKey: key, index } + var doc = this.#mapDoc(decode(block, version), version) } catch (e) { - // Unknown entry - silently ignore + // Unknown or invalid entry - silently ignore continue } if (queued[doc.schemaName]) { diff --git a/src/mapeo-project.js b/src/mapeo-project.js index 41a1723f9..1b2ca21e1 100644 --- a/src/mapeo-project.js +++ b/src/mapeo-project.js @@ -14,6 +14,7 @@ import RAM from 'random-access-memory' import Database from 'better-sqlite3' import path from 'path' import { RandomAccessFilePool } from './core-manager/random-access-file-pool.js' +import { getWinner, mapAndValidateCoreOwnership } from './core-ownership.js' import { valueOf } from './utils.js' /** @typedef {Omit} EditableProjectSettings */ @@ -97,6 +98,14 @@ export class MapeoProject { const indexWriter = new IndexWriter({ tables: [observationTable, presetTable, fieldTable], sqlite, + getWinner, + mapDoc: (doc, version) => { + if (doc.schemaName === 'coreOwnership') { + return mapAndValidateCoreOwnership(doc, version) + } else { + return doc + } + }, }) this.#dataStores = { config: new DataStore({ diff --git a/tests/core-ownership.js b/tests/core-ownership.js new file mode 100644 index 000000000..cb44205e1 --- /dev/null +++ b/tests/core-ownership.js @@ -0,0 +1,184 @@ +// @ts-check +import test from 'brittle' +import { KeyManager, sign } from '@mapeo/crypto' +import sodium from 'sodium-universal' +import { + mapAndValidateCoreOwnership, + getWinner, +} from '../src/core-ownership.js' +import { randomBytes } from 'node:crypto' +import { parseVersionId, getVersionId } from '@mapeo/schema' + +test('Valid coreOwnership record', (t) => { + const validDoc = generateValidDoc() + const version = parseVersionId(validDoc.versionId) + + const mappedDoc = mapAndValidateCoreOwnership(validDoc, version) + + t.ok(validDoc.links.length > 0, 'original doc has links') + t.alike(mappedDoc.links, [], 'links are stripped from mapped doc') + t.absent( + 'coreSignatures' in mappedDoc, + 'coreSignatures are stripped from mapped doc' + ) + t.absent( + 'identitySignature' in mappedDoc, + 'identitySignature is stripped from mapped doc' + ) +}) + +test('Invalid coreOwnership signatures', (t) => { + const validDoc = generateValidDoc() + const version = parseVersionId(validDoc.versionId) + + for (const key of Object.keys(validDoc.coreSignatures)) { + const invalidDoc = { + ...validDoc, + coreSignatures: { + ...validDoc.coreSignatures, + [key]: randomBytes(sodium.crypto_sign_BYTES), + }, + } + t.exception(() => mapAndValidateCoreOwnership(invalidDoc, version)) + } + + const invalidDoc = { + ...validDoc, + identitySignature: randomBytes(sodium.crypto_sign_BYTES), + } + t.exception(() => mapAndValidateCoreOwnership(invalidDoc, version)) +}) + +test('Invalid coreOwnership docId and coreIds', (t) => { + const validDoc = generateValidDoc() + const version = parseVersionId(validDoc.versionId) + + for (const key of Object.keys(validDoc.coreSignatures)) { + const invalidDoc = { + ...validDoc, + [`${key}CoreId`]: randomBytes(32).toString('hex'), + } + t.exception(() => mapAndValidateCoreOwnership(invalidDoc, version)) + } + + const invalidDoc = { + ...validDoc, + docId: randomBytes(32).toString('hex'), + } + t.exception(() => mapAndValidateCoreOwnership(invalidDoc, version)) +}) + +test('Invalid coreOwnership docId and coreIds (wrong length)', (t) => { + const validDoc = generateValidDoc() + const version = parseVersionId(validDoc.versionId) + + for (const key of Object.keys(validDoc.coreSignatures)) { + const invalidDoc = { + ...validDoc, + [`${key}CoreId`]: validDoc[`${key}CoreId`].slice(0, -1), + } + t.exception(() => mapAndValidateCoreOwnership(invalidDoc, version)) + } + + const invalidDoc = { + ...validDoc, + docId: validDoc.docId.slice(0, -1), + } + t.exception(() => mapAndValidateCoreOwnership(invalidDoc, version)) +}) + +test('Invalid - different coreKey', (t) => { + const validDoc = generateValidDoc() + const version = { + ...parseVersionId(validDoc.versionId), + coreKey: randomBytes(32), + } + t.exception(() => mapAndValidateCoreOwnership(validDoc, version)) +}) + +test('getWinner (coreOwnership)', (t) => { + const validDoc = generateValidDoc() + const version = parseVersionId(validDoc.versionId) + + const docA = { + ...validDoc, + versionId: getVersionId({ ...version, index: 5 }), + } + const docB = { + ...validDoc, + versionId: getVersionId({ ...version, index: 6 }), + } + + t.is(getWinner(docA, docB), docA, 'Doc with lowest index picked as winner') + t.is(getWinner(docB, docA), docA, 'Doc with lowest index picked as winner') +}) + +test('getWinner (default)', (t) => { + const docA = { + docId: 'A', + schemaName: 'other', + versionId: 'abcd', + links: [], + updatedAt: new Date(1999, 0, 1).toISOString(), + } + const docB = { + docId: 'A', + schemaName: 'other', + versionId: '1234', + links: ['1'], + updatedAt: new Date(1999, 0, 2).toISOString(), + } + t.is(getWinner(docA, docB), docB, 'Doc with last updatedAt picked as winner') + t.is(getWinner(docB, docA), docB, 'Doc with last updatedAt picked as winner') + + docA.updatedAt = docB.updatedAt + + t.is(getWinner(docA, docB), docA, 'Deterministic winner if same updatedAt') + t.is(getWinner(docB, docA), docA, 'Deterministic winner if same updatedAt') +}) + +function generateValidDoc() { + const km = new KeyManager(randomBytes(16)) + const projectKey = randomBytes(32) + + const coreKeypairs = { + auth: km.getHypercoreKeypair('auth', projectKey), + config: km.getHypercoreKeypair('config', projectKey), + data: km.getHypercoreKeypair('data', projectKey), + blobIndex: km.getHypercoreKeypair('blobIndex', projectKey), + blob: km.getHypercoreKeypair('blob', projectKey), + } + + /** @type {ReturnType} */ + const validDoc = { + docId: km.getIdentityKeypair().publicKey.toString('hex'), + versionId: getVersionId({ coreKey: coreKeypairs.auth.publicKey, index: 1 }), + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + links: ['5678/0'], + schemaName: 'coreOwnership', + authCoreId: coreKeypairs.auth.publicKey.toString('hex'), + configCoreId: coreKeypairs.config.publicKey.toString('hex'), + dataCoreId: coreKeypairs.data.publicKey.toString('hex'), + blobIndexCoreId: coreKeypairs.blobIndex.publicKey.toString('hex'), + blobCoreId: coreKeypairs.blob.publicKey.toString('hex'), + coreSignatures: { + auth: sign(coreKeypairs.auth.publicKey, coreKeypairs.auth.secretKey), + config: sign( + coreKeypairs.config.publicKey, + coreKeypairs.config.secretKey + ), + data: sign(coreKeypairs.data.publicKey, coreKeypairs.data.secretKey), + blob: sign(coreKeypairs.blob.publicKey, coreKeypairs.blob.secretKey), + blobIndex: sign( + coreKeypairs.blobIndex.publicKey, + coreKeypairs.blobIndex.secretKey + ), + }, + identitySignature: sign( + km.getIdentityKeypair().publicKey, + km.getIdentityKeypair().secretKey + ), + } + return validDoc +}