From fa40030a4572d4442aaa295f9ac54a44a35b4e10 Mon Sep 17 00:00:00 2001 From: Spencer T Brody Date: Thu, 5 Nov 2020 17:39:14 -0500 Subject: [PATCH 1/4] fix(document): Enforce schema when loading genesis record --- .../core/src/__tests__/ceramic-api.test.ts | 91 ++----------------- packages/core/src/__tests__/document.test.ts | 73 +++++++++++++++ packages/core/src/document.ts | 7 ++ 3 files changed, 90 insertions(+), 81 deletions(-) diff --git a/packages/core/src/__tests__/ceramic-api.test.ts b/packages/core/src/__tests__/ceramic-api.test.ts index ce59e64021..1857cb3be9 100644 --- a/packages/core/src/__tests__/ceramic-api.test.ts +++ b/packages/core/src/__tests__/ceramic-api.test.ts @@ -377,74 +377,6 @@ describe('Ceramic API', () => { await ceramic.close() }) - it('update schema so existing doc no longer conforms', async () => { - ceramic = await createCeramic() - - const controller = ceramic.context.did.id - - // Create doc with content that has type 'string'. - const tileDocParams: TileParams = { - metadata: { - controllers: [controller] - }, - content: { a: 'x' }, - } - const doc = await ceramic.createDocument(DOCTYPE_TILE, tileDocParams) - await syncDoc(doc) - - // Create schema that enforces that the content value is a string - const schemaDoc = await ceramic.createDocument(DOCTYPE_TILE, { - content: stringMapSchema, - metadata: { controllers: [controller] } - }) - // wait for anchor - await syncDoc(schemaDoc) - expect(schemaDoc.state.anchorStatus).toEqual(AnchorStatus.ANCHORED) - const schemaV0Id = DocID.fromBytes(schemaDoc.id.bytes, schemaDoc.tip.toString()) - - // Assign the schema to the conforming document. - await doc.change({ - metadata: { - controllers: [controller], schema: schemaDoc.id.toString() - } - }) - await syncDoc(doc) - expect(doc.content).toEqual({ a: 'x' }) - - // Update schema so that existing doc no longer conforms - const updatedSchema = cloneDeep(stringMapSchema) - updatedSchema.additionalProperties.type = "number" - await schemaDoc.change({content: updatedSchema}) - await syncDoc(schemaDoc) - - // Test that we can load the existing document without issue - const doc2 = await ceramic.loadDocument(doc.id) - expect(doc2.content).toEqual(doc.content) - - // Test that updating the existing document fails if it doesn't conform to the most recent - // version of the schema, when specifying just the schema document ID without a version - try { - await doc.change({ - content: {a: 'y'}, - metadata: {controllers: [controller], schema: schemaDoc.id.toString() } - }) - throw new Error('Should not be able to update the document with invalid content') - } catch (e) { - expect(e.message).toEqual('Validation Error: data[\'a\'] should be number') - } - - // Test that we can update the existing document according to the original schema by manually - // specifying the old version of the schema - await doc.change({ - content: { a: 'z' }, - metadata: { controllers: [controller], schema: schemaV0Id.toString() } - }) - await syncDoc(doc) - expect(doc.content).toEqual({ a: 'z' }) - - await ceramic.close() - }) - it('Pin schema to a specific version', async () => { ceramic = await createCeramic() @@ -455,10 +387,11 @@ describe('Ceramic API', () => { metadata: { controllers: [controller] }, - content: { a: 'x' }, + content: { stuff: 'a' }, } const doc = await ceramic.createDocument(DOCTYPE_TILE, tileDocParams) await syncDoc(doc) + expect(doc.content).toEqual({ stuff: 'a' }) // Create schema that enforces that the content value is a string const schemaDoc = await ceramic.createDocument(DOCTYPE_TILE, { @@ -473,12 +406,11 @@ describe('Ceramic API', () => { // Assign the schema to the conforming document, specifying current version of the schema explicitly await doc.change({ - metadata: { - controllers: [controller], schema: schemaV0Id.toString(), - } + metadata: { controllers: [controller], schema: schemaV0Id.toString() }, + content: {stuff: 'b'} }) await syncDoc(doc) - expect(doc.content).toEqual({ a: 'x' }) + expect(doc.content).toEqual({ stuff: 'b' }) expect(doc.metadata.schema).toEqual(schemaV0Id.toString()) // Update schema so that existing doc no longer conforms @@ -489,30 +421,27 @@ describe('Ceramic API', () => { expect(doc.metadata.schema.toString()).toEqual(schemaV0Id.toString()) - // Test that we can load the existing document without issue - const doc2 = await ceramic.loadDocument(doc.id) - expect(doc2.content).toEqual(doc.content) - expect(doc2.metadata).toEqual(doc.metadata) - // Test that we can update the existing document according to the original schema when taking // the schema docID from the existing document. await doc.change({ - content: { a: 'y' }, + content: { stuff: 'c' }, metadata: { controllers: [controller], schema: doc.metadata.schema.toString() } }) await syncDoc(doc) + expect(doc.content).toEqual({ stuff: 'c' }) // Test that updating the existing document fails if it doesn't conform to the most recent // version of the schema, when specifying just the schema document ID without a version try { await doc.change({ - content: {a: 'z'}, + content: {stuff: 'd'}, metadata: {controllers: [controller], schema: schemaDoc.id.toString() } }) throw new Error('Should not be able to update the document with invalid content') } catch (e) { - expect(e.message).toEqual('Validation Error: data[\'a\'] should be number') + expect(e.message).toEqual('Validation Error: data[\'stuff\'] should be number') } + expect(doc.content).toEqual({ stuff: 'c' }) await ceramic.close() }) diff --git a/packages/core/src/__tests__/document.test.ts b/packages/core/src/__tests__/document.test.ts index 551ddfe7b1..2b7b5d2159 100644 --- a/packages/core/src/__tests__/document.test.ts +++ b/packages/core/src/__tests__/document.test.ts @@ -102,6 +102,15 @@ const create = async (params: TileParams, ceramic: Ceramic, context: Context, op return await ceramic._createDocFromGenesis(record, opts) } +const stringMapSchema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "StringMap", + "type": "object", + "additionalProperties": { + "type": "string" + } +} + let stateStore: LevelStateStore let pinStore: PinStore let pinning: PinningBackend @@ -130,6 +139,7 @@ describe('Document', () => { let findHandler: any; let anchorService: AnchorService; let ceramic: Ceramic; + let ceramicWithoutSchemaValidation: Ceramic; let context: Context; beforeEach(() => { @@ -177,6 +187,9 @@ describe('Document', () => { ceramic = new Ceramic(dispatcher, pinStore, context) ceramic._doctypeHandlers['tile'] = doctypeHandler + + ceramicWithoutSchemaValidation = new Ceramic(dispatcher, pinStore, context, false) + ceramicWithoutSchemaValidation._doctypeHandlers['tile'] = doctypeHandler }) it('is created correctly', async () => { @@ -326,6 +339,66 @@ describe('Document', () => { await doc1._handleTip(tipInvalidUpdate) expect(doc1.content).toEqual(newContent) }) + + it('Enforces schema at document creation', async () => { + const schemaDoc = await create({ content: stringMapSchema, metadata: { controllers } }, ceramic, context) + await anchorUpdate(schemaDoc) + + try { + const docParams = { + content: {stuff: 1}, + metadata: {controllers, schema: schemaDoc.id.toString()} + } + await create(docParams, ceramic, context) + throw new Error('Should not be able to create a document with an invalid schema') + } catch (e) { + expect(e.message).toEqual('Validation Error: data[\'stuff\'] should be string') + } + }) + + it('Enforces schema at document update', async () => { + const schemaDoc = await create({ content: stringMapSchema, metadata: { controllers } }, ceramic, context) + await anchorUpdate(schemaDoc) + + const docParams = { + content: {stuff: 1}, + metadata: {controllers} + } + const doc = await create(docParams, ceramic, context) + await anchorUpdate(doc) + + try { + const updateRec = await TileDoctype._makeRecord(doc.doctype, user, null, doc.controllers, schemaDoc.id.toString()) + await doc.applyRecord(updateRec) + throw new Error('Should not be able to assign a schema to a document that does not conform') + } catch (e) { + expect(e.message).toEqual('Validation Error: data[\'stuff\'] should be string') + } + }) + + it('Enforces schema when loading genesis record', async () => { + const schemaDoc = await create({ content: stringMapSchema, metadata: { controllers } }, ceramic, context) + await anchorUpdate(schemaDoc) + + const docParams = { + content: {stuff: 1}, + metadata: {controllers, schema: schemaDoc.id.toString()} + } + // Create a document that isn't conforming to the schema + const doc = await create(docParams, ceramicWithoutSchemaValidation, context) + await anchorUpdate(doc) + + expect(doc.content).toEqual({stuff:1}) + expect(doc.metadata.schema).toEqual(schemaDoc.id.toString()) + + try { + await Document.load(doc.id, findHandler, dispatcher, pinStore, context, {skipWait:true}) + throw new Error('Should not be able to assign a schema to a document that does not conform') + } catch (e) { + expect(e.message).toEqual('Validation Error: data[\'stuff\'] should be string') + } + }) + }) describe('Network update logic', () => { diff --git a/packages/core/src/document.ts b/packages/core/src/document.ts index 74ce5ab666..cac15cd003 100644 --- a/packages/core/src/document.ts +++ b/packages/core/src/document.ts @@ -145,6 +145,13 @@ class Document extends EventEmitter { doc._doctype.state = await doc._doctypeHandler.applyRecord(record, doc._genesisCid, context) } + if (validate) { + const schema = await Document.loadSchema(context.api, doc._doctype) + if (schema) { + Utils.validate(doc._doctype.content, schema) + } + } + await doc._register(opts) return doc } From 879367c5015a098bb6a013d57173bac85ebc3ed0 Mon Sep 17 00:00:00 2001 From: Spencer T Brody Date: Fri, 6 Nov 2020 15:17:42 -0500 Subject: [PATCH 2/4] Improve test robustness by manually anchoring --- .../core/src/__tests__/ceramic-api.test.ts | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/packages/core/src/__tests__/ceramic-api.test.ts b/packages/core/src/__tests__/ceramic-api.test.ts index 1857cb3be9..2d8c0b68fb 100644 --- a/packages/core/src/__tests__/ceramic-api.test.ts +++ b/packages/core/src/__tests__/ceramic-api.test.ts @@ -38,12 +38,14 @@ const createIPFS =(overrideConfig: Record = {}): Promise => { - await new Promise(resolve => { +const anchorDoc = async (ceramic: Ceramic, doc: any): Promise => { + const p = new Promise(resolve => { doc.on('change', () => { resolve() }) }) + await ceramic.context.anchorService.anchor() + await p } describe('Ceramic API', () => { @@ -67,6 +69,7 @@ describe('Ceramic API', () => { const createCeramic = async (c: CeramicConfig = {}): Promise => { c.topic = topic + c.anchorOnRequest = false const ceramic = await Ceramic.create(ipfs, c) const config = { @@ -108,7 +111,7 @@ describe('Ceramic API', () => { }) // wait for anchor (new version) - await syncDoc(docOg) + await anchorDoc(ceramic, docOg) expect(docOg.state.log.length).toEqual(2) expect(docOg.content).toEqual({ test: 321 }) @@ -119,7 +122,7 @@ describe('Ceramic API', () => { await docOg.change({ content: { test: 'abcde' } }) // wait for anchor (new version) - await syncDoc(docOg) + await anchorDoc(ceramic, docOg) expect(docOg.state.log.length).toEqual(4) expect(docOg.content).toEqual({ test: 'abcde' }) @@ -339,7 +342,7 @@ describe('Ceramic API', () => { content: { a: 1 }, } const doc = await ceramic.createDocument(DOCTYPE_TILE, tileDocParams) - await syncDoc(doc) + await anchorDoc(ceramic, doc) // Create schema that enforces that the content value is a string, which would reject // the document created above. @@ -348,7 +351,7 @@ describe('Ceramic API', () => { metadata: { controllers: [controller] } }) // wait for anchor - await syncDoc(schemaDoc) + await anchorDoc(ceramic, schemaDoc) expect(schemaDoc.state.anchorStatus).toEqual(AnchorStatus.ANCHORED) // Update the schema to expect a number, so now the original doc should conform to the new @@ -357,7 +360,7 @@ describe('Ceramic API', () => { updatedSchema.additionalProperties.type = "number" await schemaDoc.change({content: updatedSchema}) // wait for anchor - await syncDoc(schemaDoc) + await anchorDoc(ceramic, schemaDoc) expect(schemaDoc.state.anchorStatus).toEqual(AnchorStatus.ANCHORED) // Test that we can assign the updated schema to the document without error. @@ -366,7 +369,7 @@ describe('Ceramic API', () => { controllers: [controller], schema: schemaDoc.id.toString() } }) - await syncDoc(doc) + await anchorDoc(ceramic, doc) expect(doc.content).toEqual({ a: 1 }) // Test that we can reload the document without issue @@ -390,7 +393,7 @@ describe('Ceramic API', () => { content: { stuff: 'a' }, } const doc = await ceramic.createDocument(DOCTYPE_TILE, tileDocParams) - await syncDoc(doc) + await anchorDoc(ceramic, doc) expect(doc.content).toEqual({ stuff: 'a' }) // Create schema that enforces that the content value is a string @@ -400,7 +403,7 @@ describe('Ceramic API', () => { }) // wait for anchor - await syncDoc(schemaDoc) + await anchorDoc(ceramic, schemaDoc) expect(schemaDoc.state.anchorStatus).toEqual(AnchorStatus.ANCHORED) const schemaV0Id = DocID.fromBytes(schemaDoc.id.bytes, schemaDoc.tip.toString()) @@ -409,7 +412,7 @@ describe('Ceramic API', () => { metadata: { controllers: [controller], schema: schemaV0Id.toString() }, content: {stuff: 'b'} }) - await syncDoc(doc) + await anchorDoc(ceramic, doc) expect(doc.content).toEqual({ stuff: 'b' }) expect(doc.metadata.schema).toEqual(schemaV0Id.toString()) @@ -417,7 +420,7 @@ describe('Ceramic API', () => { const updatedSchema = cloneDeep(stringMapSchema) updatedSchema.additionalProperties.type = "number" await schemaDoc.change({content: updatedSchema}) - await syncDoc(schemaDoc) + await anchorDoc(ceramic, schemaDoc) expect(doc.metadata.schema.toString()).toEqual(schemaV0Id.toString()) @@ -427,7 +430,7 @@ describe('Ceramic API', () => { content: { stuff: 'c' }, metadata: { controllers: [controller], schema: doc.metadata.schema.toString() } }) - await syncDoc(doc) + await anchorDoc(ceramic, doc) expect(doc.content).toEqual({ stuff: 'c' }) // Test that updating the existing document fails if it doesn't conform to the most recent From 84313774c725e198fb7208451e3a9f2aff307db7 Mon Sep 17 00:00:00 2001 From: Spencer T Brody Date: Mon, 9 Nov 2020 14:45:00 -0500 Subject: [PATCH 3/4] Increase robustness of ceramic-anchor-test --- .../core/src/__tests__/ceramic-anchor.test.ts | 50 ++++++++++--------- .../core/src/__tests__/ceramic-api.test.ts | 22 +++++--- 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/packages/core/src/__tests__/ceramic-anchor.test.ts b/packages/core/src/__tests__/ceramic-anchor.test.ts index 514c4f1465..34aa0a3e21 100644 --- a/packages/core/src/__tests__/ceramic-anchor.test.ts +++ b/packages/core/src/__tests__/ceramic-anchor.test.ts @@ -1,6 +1,6 @@ import Ceramic from '../ceramic' import IdentityWallet from 'identity-wallet' -import { Doctype } from "@ceramicnetwork/common" +import {AnchorStatus, Doctype} from "@ceramicnetwork/common" import { TileDoctype } from "@ceramicnetwork/doctype-tile" import tmp from 'tmp-promise' import IPFS from 'ipfs' @@ -50,18 +50,27 @@ const createCeramic = async (ipfs: IPFSApi, anchorManual: boolean, topic: string return ceramic } -const anchor = async (ceramic: Ceramic): Promise => { - await ceramic.context.anchorService.anchor() -} - -const syncDoc = async (doctype: Doctype): Promise => { - await new Promise(resolve => { - doctype.on('change', () => { +const registerChangeListener = function (doc: Doctype): Promise { + return new Promise(resolve => { + doc.on('change', () => { resolve() }) }) } +/** + * Registers a listener for change notifications on a document, instructs the anchor service to + * perform an anchor, then waits for the change listener to resolve, indicating that the document + * got anchored. + * @param ceramic + * @param doc + */ +const anchorDoc = async (ceramic: Ceramic, doc: any): Promise => { + const changeHandle = registerChangeListener(doc) + await ceramic.context.anchorService.anchor() + await changeHandle +} + describe('Ceramic anchoring', () => { jest.setTimeout(60000) let ipfs1: IPFSApi; @@ -127,11 +136,11 @@ describe('Ceramic anchoring', () => { await doctype1.change({ content: { a: 2 }, metadata: { controllers: [controller] } }, { applyOnly: false }) await doctype1.change({ content: { a: 3 }, metadata: { controllers: [controller] } }, { applyOnly: false }) - await anchor(ceramic1) - await syncDoc(doctype1) + await anchorDoc(ceramic1, doctype1) expect(doctype1.content).toEqual({ a: 3 }) expect(doctype1.state.log.length).toEqual(3) + expect(doctype1.state.anchorStatus).toEqual(AnchorStatus.ANCHORED) const doctype2 = await ceramic2.loadDocument(doctype1.id) expect(doctype1.content).toEqual(doctype2.content) @@ -181,8 +190,7 @@ describe('Ceramic anchoring', () => { expect(doctype1.state.log.length).toEqual(2) - await anchor(ceramic1) - await syncDoc(doctype1) + await anchorDoc(ceramic1, doctype1) expect(doctype1.content).toEqual({ a: 123, b: 4567 }) expect(doctype1.state.log.length).toEqual(2) @@ -209,8 +217,7 @@ describe('Ceramic anchoring', () => { expect(doctype1.state.log.length).toEqual(2) - await anchor(ceramic1) - await syncDoc(doctype1) + await anchorDoc(ceramic1, doctype1) expect(doctype1.content).toEqual({ a: 123 }) expect(doctype1.state.log.length).toEqual(2) @@ -237,8 +244,7 @@ describe('Ceramic anchoring', () => { await doctype1.change({ content: { x: doctype1.content.x + 1 }, metadata: { controllers: [controller] } }, { applyOnly: true }) await doctype1.change({ content: { x: doctype1.content.x + 1 }, metadata: { controllers: [controller] } }, { applyOnly: false }) - await anchor(ceramic1) - await syncDoc(doctype1) + await anchorDoc(ceramic1, doctype1) expect(doctype1.content).toEqual({ x: 3 }) expect(doctype1.state.log.length).toEqual(3) @@ -264,8 +270,7 @@ describe('Ceramic anchoring', () => { await doctype1.change({ content: { x: doctype1.content.x + 1 }, metadata: { controllers: [controller] } }, { applyOnly: true }) await doctype1.change({ content: { x: doctype1.content.x + 1 }, metadata: { controllers: [controller] } }, { applyOnly: false }) - await anchor(ceramic1) - await syncDoc(doctype1) + await anchorDoc(ceramic1, doctype1) expect(doctype1.content).toEqual({ x: 3 }) expect(doctype1.state.log.length).toEqual(3) @@ -295,8 +300,7 @@ describe('Ceramic anchoring', () => { await doctype1.change({ content: { x: doctype1.content.x + 1 }, metadata: { controllers: [controller] } }, { applyOnly: false }) - await anchor(ceramic1) - await syncDoc(doctype1) + await anchorDoc(ceramic1, doctype1) expect(doctype1.content).toEqual({ x: 3 }) expect(doctype1.state.log.length).toEqual(3) @@ -305,8 +309,7 @@ describe('Ceramic anchoring', () => { await doctype1.change({ content: { x: doctype1.content.x + 1 }, metadata: { controllers: [controller] } }, { applyOnly: true }) await doctype1.change({ content: { x: doctype1.content.x + 1 }, metadata: { controllers: [controller] } }, { applyOnly: false }) - await anchor(ceramic1) - await syncDoc(doctype1) + await anchorDoc(ceramic1, doctype1) expect(doctype1.content).toEqual({ x: 6 }) expect(doctype1.state.log.length).toEqual(5) @@ -333,8 +336,7 @@ describe('Ceramic anchoring', () => { await doctype1.change({ content: { x: 7 }, metadata: { controllers: [controller] } }, { applyOnly: false }) await cloned.change({ content: { x: 5 }, metadata: { controllers: [controller] } }, { applyOnly: false }) - await anchor(ceramic1) - await syncDoc(doctype1) + await anchorDoc(ceramic1, doctype1) expect(doctype1.content).toEqual({ x: 7 }) expect(doctype1.state.log.length).toEqual(3) diff --git a/packages/core/src/__tests__/ceramic-api.test.ts b/packages/core/src/__tests__/ceramic-api.test.ts index 2d8c0b68fb..fe06292b5a 100644 --- a/packages/core/src/__tests__/ceramic-api.test.ts +++ b/packages/core/src/__tests__/ceramic-api.test.ts @@ -33,19 +33,25 @@ const createIPFS =(overrideConfig: Record = {}): Promise => { - const p = new Promise(resolve => { +const registerChangeListener = async (doc: any): Promise => { + return new Promise(resolve => { doc.on('change', () => { resolve() }) }) +} + +/** + * Registers a listener for change notifications on a document, instructs the anchor service to + * perform an anchor, then waits for the change listener to resolve, indicating that the document + * got anchored. + * @param ceramic + * @param doc + */ +const anchorDoc = async (ceramic: Ceramic, doc: any): Promise => { + const changeHandle = await registerChangeListener(doc) await ceramic.context.anchorService.anchor() - await p + await changeHandle } describe('Ceramic API', () => { From 719f1270a3781abe829b34d458c7d777de4efdeb Mon Sep 17 00:00:00 2001 From: Spencer T Brody Date: Mon, 9 Nov 2020 17:02:46 -0500 Subject: [PATCH 4/4] Change registerChangeListener function in ceramic-api-test to be synchronous --- packages/core/src/__tests__/ceramic-api.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/__tests__/ceramic-api.test.ts b/packages/core/src/__tests__/ceramic-api.test.ts index fe06292b5a..8bbc2e4f70 100644 --- a/packages/core/src/__tests__/ceramic-api.test.ts +++ b/packages/core/src/__tests__/ceramic-api.test.ts @@ -33,7 +33,7 @@ const createIPFS =(overrideConfig: Record = {}): Promise => { +const registerChangeListener = function (doc: any): Promise { return new Promise(resolve => { doc.on('change', () => { resolve() @@ -49,7 +49,7 @@ const registerChangeListener = async (doc: any): Promise => { * @param doc */ const anchorDoc = async (ceramic: Ceramic, doc: any): Promise => { - const changeHandle = await registerChangeListener(doc) + const changeHandle = registerChangeListener(doc) await ceramic.context.anchorService.anchor() await changeHandle }