From 2a558ea976628cdd5b3a0fdaf9a6e97afe416b8b Mon Sep 17 00:00:00 2001 From: Andrew Chou Date: Thu, 31 Aug 2023 14:18:33 -0400 Subject: [PATCH 1/4] update @mapeo/crypto --- package-lock.json | 13 +++++++------ package.json | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9525d851..914cb81f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@digidem/types": "^2.0.0", "@fastify/type-provider-typebox": "^3.3.0", "@hyperswarm/secret-stream": "^6.1.2", - "@mapeo/crypto": "^1.0.0-alpha.7", + "@mapeo/crypto": "^1.0.0-alpha.8", "@mapeo/schema": "^3.0.0-next.8", "@mapeo/sqlite-indexer": "^1.0.0-alpha.6", "@sinclair/typebox": "^0.29.6", @@ -1580,9 +1580,9 @@ "integrity": "sha512-vYY5EIxCPzEXEWL/vTjdHy4g92tv1ApUQCjPJsj9gEoXLNNVwJlwwgRZisuvgFBZ3zeLzQygrbehERSpYdmFZA==" }, "node_modules/@mapeo/crypto": { - "version": "1.0.0-alpha.7", - "resolved": "https://registry.npmjs.org/@mapeo/crypto/-/crypto-1.0.0-alpha.7.tgz", - "integrity": "sha512-O4qkHYnHxjk1G4LxI4uttidjWHQX48/d7kWAq6ukfZZkuT9j8u24RBCBWOoUfVO5qACjjooUwxv+I/qvbBqmGQ==", + "version": "1.0.0-alpha.8", + "resolved": "https://registry.npmjs.org/@mapeo/crypto/-/crypto-1.0.0-alpha.8.tgz", + "integrity": "sha512-2pIykZTFWINwtdsk2Fl3+3UdpQZrt1/IwMgvglwZlxyqKM7LpcuAda871BFiK1CcEsDlaTT63FBtIszEXnnawg==", "dependencies": { "@types/b4a": "^1.6.0", "b4a": "^1.6.4", @@ -8545,8 +8545,9 @@ } }, "node_modules/z32": { - "version": "1.0.0", - "license": "MIT", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/z32/-/z32-1.0.1.tgz", + "integrity": "sha512-Uytfqf6VEVchHKZDw0NRdCViOARHP84uzvOw0CXCMLOwhgHZUL9XibpEPLLQN10mCVLxOlGCQWbkV7km7yNYcw==", "dependencies": { "b4a": "^1.5.3" } diff --git a/package.json b/package.json index 14abfa21..7f12ea38 100644 --- a/package.json +++ b/package.json @@ -95,7 +95,7 @@ "@digidem/types": "^2.0.0", "@fastify/type-provider-typebox": "^3.3.0", "@hyperswarm/secret-stream": "^6.1.2", - "@mapeo/crypto": "^1.0.0-alpha.7", + "@mapeo/crypto": "^1.0.0-alpha.8", "@mapeo/schema": "^3.0.0-next.8", "@mapeo/sqlite-indexer": "^1.0.0-alpha.6", "@sinclair/typebox": "^0.29.6", From 6c100697f883223642fc0b6dcd393b9080b2c755 Mon Sep 17 00:00:00 2001 From: Andrew Chou Date: Tue, 5 Sep 2023 14:15:49 -0400 Subject: [PATCH 2/4] update usage of encrypt/decrypt local message --- src/mapeo-manager.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index e3a118bf..237cb4c0 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -71,8 +71,9 @@ export class MapeoManager { * @returns {ProjectKeys} */ #decodeProjectKeysCipher(keysCipher, projectId) { + const nonce = Buffer.from(projectId, 'hex') return ProjectKeys.decode( - this.#keyManager.decryptLocalMessage(keysCipher, projectId) + this.#keyManager.decryptLocalMessage(keysCipher, nonce) ) } @@ -98,13 +99,15 @@ export class MapeoManager { */ #saveToProjectKeysTable({ projectId, projectKeys, projectInfo }) { const encoded = ProjectKeys.encode(projectKeys).finish() + const nonce = Buffer.from(projectId, 'hex') + this.#db .insert(projectKeysTable) .values({ projectId, keysCipher: this.#keyManager.encryptLocalMessage( Buffer.from(encoded.buffer, encoded.byteOffset, encoded.byteLength), - projectId + nonce ), projectInfo, }) From 65cf62fd33f78b016d5a226ca0f9090e2c90843a Mon Sep 17 00:00:00 2001 From: Andrew Chou Date: Tue, 5 Sep 2023 14:54:13 -0400 Subject: [PATCH 3/4] update client db schema --- .../{0000_steady_jackpot.sql => 0000_needy_hex.sql} | 1 + drizzle/client/meta/0000_snapshot.json | 9 ++++++++- drizzle/client/meta/_journal.json | 4 ++-- src/schema/client.js | 1 + 4 files changed, 12 insertions(+), 3 deletions(-) rename drizzle/client/{0000_steady_jackpot.sql => 0000_needy_hex.sql} (93%) diff --git a/drizzle/client/0000_steady_jackpot.sql b/drizzle/client/0000_needy_hex.sql similarity index 93% rename from drizzle/client/0000_steady_jackpot.sql rename to drizzle/client/0000_needy_hex.sql index 3c8176df..cb7e7268 100644 --- a/drizzle/client/0000_steady_jackpot.sql +++ b/drizzle/client/0000_needy_hex.sql @@ -4,6 +4,7 @@ CREATE TABLE `project_backlink` ( --> statement-breakpoint CREATE TABLE `projectKeys` ( `projectId` text PRIMARY KEY NOT NULL, + `projectPublicId` text NOT NULL, `keysCipher` blob NOT NULL, `projectInfo` text DEFAULT '{}' NOT NULL ); diff --git a/drizzle/client/meta/0000_snapshot.json b/drizzle/client/meta/0000_snapshot.json index 6697cd68..3df04e6e 100644 --- a/drizzle/client/meta/0000_snapshot.json +++ b/drizzle/client/meta/0000_snapshot.json @@ -1,7 +1,7 @@ { "version": "5", "dialect": "sqlite", - "id": "a4afa0b6-5fd1-4c8d-bc09-7e3f3fc1928f", + "id": "e047380e-7d10-442c-b5e4-0791b8970eeb", "prevId": "00000000-0000-0000-0000-000000000000", "tables": { "project_backlink": { @@ -30,6 +30,13 @@ "notNull": true, "autoincrement": false }, + "projectPublicId": { + "name": "projectPublicId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, "keysCipher": { "name": "keysCipher", "type": "blob", diff --git a/drizzle/client/meta/_journal.json b/drizzle/client/meta/_journal.json index ae010144..037433ce 100644 --- a/drizzle/client/meta/_journal.json +++ b/drizzle/client/meta/_journal.json @@ -5,8 +5,8 @@ { "idx": 0, "version": "5", - "when": 1693337118674, - "tag": "0000_steady_jackpot", + "when": 1693938675535, + "tag": "0000_needy_hex", "breakpoints": true } ] diff --git a/src/schema/client.js b/src/schema/client.js index 13c0938e..67c4b224 100644 --- a/src/schema/client.js +++ b/src/schema/client.js @@ -18,6 +18,7 @@ export const projectTable = sqliteTable('project', toColumns(schemas.project)) export const projectBacklinkTable = backlinkTable(projectTable) export const projectKeysTable = sqliteTable('projectKeys', { projectId: text('projectId').notNull().primaryKey(), + projectPublicId: text('projectPublicId').notNull(), keysCipher: blob('keysCipher', { mode: 'buffer' }).notNull(), projectInfo: projectInfoColumn('projectInfo') .default( From ebce746ba60aa7aca33cf223bd1cf4fab7cc1966 Mon Sep 17 00:00:00 2001 From: Andrew Chou Date: Tue, 5 Sep 2023 15:05:29 -0400 Subject: [PATCH 4/4] update manager to return and handle public ids --- src/mapeo-manager.js | 94 +++++++++++++++++++++++++++++--------------- src/mapeo-project.js | 5 +-- src/types.ts | 5 +++ src/utils.js | 31 +++++++++++++++ 4 files changed, 100 insertions(+), 35 deletions(-) diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index 237cb4c0..75585354 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -10,10 +10,17 @@ import { IndexWriter } from './index-writer/index.js' import { MapeoProject } from './mapeo-project.js' import { projectKeysTable, projectTable } from './schema/client.js' import { ProjectKeys } from './generated/keys.js' -import { deNullify } from './utils.js' +import { + deNullify, + projectIdToNonce, + projectKeyToId, + projectKeyToPublicId, +} from './utils.js' import { RandomAccessFilePool } from './core-manager/random-access-file-pool.js' /** @typedef {import("@mapeo/schema").ProjectValue} ProjectValue */ +/** @typedef {import('./types.js').ProjectId} ProjectId */ +/** @typedef {import('./types.js').ProjectPublicId} ProjectPublicId */ const CLIENT_SQLITE_FILE_NAME = 'client.db' @@ -27,7 +34,7 @@ export class MapeoManager { #keyManager #projectSettingsIndexWriter #db - /** @type {Map} */ + /** @type {Map} */ #activeProjects /** @type {import('./types.js').CoreStorage} */ #coreStorage @@ -67,18 +74,18 @@ export class MapeoManager { /** * @param {Buffer} keysCipher - * @param {string} projectId + * @param {ProjectId} projectId * @returns {ProjectKeys} */ #decodeProjectKeysCipher(keysCipher, projectId) { - const nonce = Buffer.from(projectId, 'hex') + const nonce = projectIdToNonce(projectId) return ProjectKeys.decode( this.#keyManager.decryptLocalMessage(keysCipher, nonce) ) } /** - * @param {string} projectId + * @param {ProjectId} projectId * @returns {Pick[0], 'dbPath' | 'coreStorage'>} */ #projectStorage(projectId) { @@ -93,18 +100,25 @@ export class MapeoManager { /** * @param {Object} opts - * @param {string} opts.projectId + * @param {ProjectId} opts.projectId + * @param {ProjectPublicId} opts.projectPublicId * @param {ProjectKeys} opts.projectKeys * @param {import('./generated/rpc.js').Invite_ProjectInfo} [opts.projectInfo] */ - #saveToProjectKeysTable({ projectId, projectKeys, projectInfo }) { + #saveToProjectKeysTable({ + projectId, + projectPublicId, + projectKeys, + projectInfo, + }) { const encoded = ProjectKeys.encode(projectKeys).finish() - const nonce = Buffer.from(projectId, 'hex') + const nonce = projectIdToNonce(projectId) this.#db .insert(projectKeysTable) .values({ projectId, + projectPublicId, keysCipher: this.#keyManager.encryptLocalMessage( Buffer.from(encoded.buffer, encoded.byteOffset, encoded.byteLength), nonce @@ -117,7 +131,7 @@ export class MapeoManager { /** * Create a new project. * @param {import('type-fest').Simplify>>} [settings] - * @returns {Promise} + * @returns {Promise} */ async createProject(settings = {}) { // 1. Create project keypair @@ -141,10 +155,14 @@ export class MapeoManager { encryptionKeys, } - // TODO: Update to use @mapeo/crypto when ready (https://github.com/digidem/mapeo-core-next/issues/171) - const projectId = projectKeypair.publicKey.toString('hex') + const projectId = projectKeyToId(keys.projectKey) + const projectPublicId = projectKeyToPublicId(keys.projectKey) - this.#saveToProjectKeysTable({ projectId, projectKeys: keys }) + this.#saveToProjectKeysTable({ + projectId, + projectPublicId, + projectKeys: keys, + }) // 4. Create MapeoProject instance const project = new MapeoProject({ @@ -161,36 +179,40 @@ export class MapeoManager { await project.$setProjectSettings(settings) // TODO: Close the project instance instead of keeping it around - // https://github.com/digidem/mapeo-core-next/issues/207 - this.#activeProjects.set(projectId, project) + this.#activeProjects.set(projectPublicId, project) - // 6. Return project id - return projectId + // 6. Return project public id + return projectPublicId } /** - * @param {string} projectId + * @param {ProjectPublicId} projectPublicId * @returns {Promise} */ - async getProject(projectId) { + async getProject(projectPublicId) { // 1. Check for existing active project - const activeProject = this.#activeProjects.get(projectId) + const activeProject = this.#activeProjects.get(projectPublicId) if (activeProject) return activeProject // 2. Create project instance const projectKeysTableResult = this.#db .select({ + projectId: projectKeysTable.projectId, keysCipher: projectKeysTable.keysCipher, }) .from(projectKeysTable) - .where(eq(projectKeysTable.projectId, projectId)) + .where(eq(projectKeysTable.projectPublicId, projectPublicId)) .get() if (!projectKeysTableResult) { - throw new Error(`NotFound: project ID ${projectId} not found`) + throw new Error(`NotFound: project ID ${projectPublicId} not found`) } + const projectId = /** @type {ProjectId} */ ( + projectKeysTableResult.projectId + ) + const projectKeys = this.#decodeProjectKeysCipher( projectKeysTableResult.keysCipher, projectId @@ -205,13 +227,13 @@ export class MapeoManager { }) // 3. Keep track of project instance as we know it's a properly existing project - this.#activeProjects.set(projectId, project) + this.#activeProjects.set(projectPublicId, project) return project } /** - * @returns {Promise & { projectId: string, createdAt?: string, updatedAt?: string }>>} + * @returns {Promise & { projectId: ProjectPublicId, createdAt?: string, updatedAt?: string }>>} */ async listProjects() { // We use the project keys table as the source of truth for projects that exist @@ -220,6 +242,7 @@ export class MapeoManager { const allProjectKeysResult = this.#db .select({ projectId: projectKeysTable.projectId, + projectPublicId: projectKeysTable.projectPublicId, projectInfo: projectKeysTable.projectInfo, }) .from(projectKeysTable) @@ -235,17 +258,21 @@ export class MapeoManager { .from(projectTable) .all() - /** @type {Array & { projectId: string, createdAt?: string, updatedAt?: string }>} */ + /** @type {Array & { projectId: ProjectPublicId, createdAt?: string, updatedAt?: string }>} */ const result = [] - for (const { projectId, projectInfo } of allProjectKeysResult) { + for (const { + projectId, + projectPublicId, + projectInfo, + } of allProjectKeysResult) { const existingProject = allProjectsResult.find( (p) => p.projectId === projectId ) result.push( deNullify({ - projectId, + projectId: /** @type {ProjectPublicId} */ (projectPublicId), createdAt: existingProject?.createdAt, updatedAt: existingProject?.updatedAt, name: existingProject?.name || projectInfo.name, @@ -258,20 +285,22 @@ export class MapeoManager { /** * @param {import('./generated/rpc.js').Invite} invite - * @returns {Promise} + * @returns {Promise} */ async addProject({ projectKey, encryptionKeys, projectInfo }) { - const projectId = projectKey.toString('hex') + const projectPublicId = projectKeyToPublicId(projectKey) // 1. Check for an active project - const activeProject = this.#activeProjects.get(projectId) + const activeProject = this.#activeProjects.get(projectPublicId) if (activeProject) { - throw new Error(`Project with ID ${projectId} already exists`) + throw new Error(`Project with ID ${projectPublicId} already exists`) } // 2. Check if the project exists in the project keys table // If it does, that means the project has already been either created or added before + const projectId = projectKeyToId(projectKey) + const projectExists = this.#db .select() .from(projectKeysTable) @@ -279,7 +308,7 @@ export class MapeoManager { .get() if (projectExists) { - throw new Error(`Project with ID ${projectId} already exists`) + throw new Error(`Project with ID ${projectPublicId} already exists`) } // TODO: Relies on completion of https://github.com/digidem/mapeo-core-next/issues/233 @@ -288,6 +317,7 @@ export class MapeoManager { // 4. Update the project keys table this.#saveToProjectKeysTable({ projectId, + projectPublicId, projectKeys: { projectKey, encryptionKeys, @@ -295,6 +325,6 @@ export class MapeoManager { projectInfo, }) - return projectId + return projectPublicId } } diff --git a/src/mapeo-project.js b/src/mapeo-project.js index 6750419f..9f0bbe35 100644 --- a/src/mapeo-project.js +++ b/src/mapeo-project.js @@ -27,7 +27,7 @@ import { mapAndValidateCoreOwnership, } from './core-ownership.js' import { Capabilities } from './capabilities.js' -import { valueOf } from './utils.js' +import { projectKeyToId, valueOf } from './utils.js' /** @typedef {Omit} EditableProjectSettings */ @@ -69,8 +69,7 @@ export class MapeoProject { projectSecretKey, encryptionKeys, }) { - // TODO: Update to use @mapeo/crypto when ready (https://github.com/digidem/mapeo-core-next/issues/171) - this.#projectId = projectKey.toString('hex') + this.#projectId = projectKeyToId(projectKey) ///////// 1. Setup database const sqlite = new Database(dbPath) diff --git a/src/types.ts b/src/types.ts index 977d8ba3..5d13e42d 100644 --- a/src/types.ts +++ b/src/types.ts @@ -4,6 +4,7 @@ import type { ValueOf, RequireAtLeastOne, SetOptional, + Opaque, } from 'type-fest' import { SUPPORTED_BLOB_VARIANTS } from './blob-store/index.js' import { MapeoCommon, MapeoDoc, MapeoValue, decode } from '@mapeo/schema' @@ -142,6 +143,10 @@ export type TopicKey = Buffer export type TopicId = string /** 52 character base32 encoding of `Topic` Buffer */ export type MdnsTopicId = string +/** hex string representation of project key buffer */ +export type ProjectId = Opaque +/** z32-encoded hash of project key */ +export type ProjectPublicId = Opaque // TODO: Figure out where those extra fields come from and find more elegant way to represent this export type RawDhtConnectionStream = Duplex & { diff --git a/src/utils.js b/src/utils.js index a2761cbc..0938fed1 100644 --- a/src/utils.js +++ b/src/utils.js @@ -1,4 +1,5 @@ import b4a from 'b4a' +import { projectKeyToPublicId as keyToPublicId } from '@mapeo/crypto' /** * @param {String|Buffer} id @@ -90,3 +91,33 @@ export function valueOf(doc) { const { docId, versionId, links, forks, createdAt, updatedAt, ...rest } = doc return rest } + +/** + * Create an internal ID from a project key + * @param {Buffer} projectKey + * @returns {import('./types.js').ProjectId} + */ +export function projectKeyToId(projectKey) { + return /** @type {import('./types.js').ProjectId} */ ( + projectKey.toString('hex') + ) +} + +/** + * Create a public ID from a project key + * @param {Buffer} projectKey + * @returns {import('./types.js').ProjectPublicId} + */ +export function projectKeyToPublicId(projectKey) { + return /** @type {import('./types.js').ProjectPublicId} */ ( + keyToPublicId(projectKey) + ) +} + +/** + * @param {import('./types.js').ProjectId} projectId + * @returns {Buffer} 24-byte nonce (same length as sodium.crypto_aead_xchacha20poly1305_ietf_NPUBBYTES) + */ +export function projectIdToNonce(projectId) { + return Buffer.from(projectId, 'hex').subarray(0, 24) +}