From dd796bd984cc0bcd426f9e721cf0cf5d76cb185c Mon Sep 17 00:00:00 2001 From: Espen Hovlandsdal Date: Sat, 2 Dec 2017 15:13:15 +0100 Subject: [PATCH] [import] Generate document IDs on client side --- packages/@sanity/import/package.json | 1 + .../@sanity/import/src/assignDocumentId.js | 11 ++++++ .../@sanity/import/src/documentHasErrors.js | 4 +++ packages/@sanity/import/src/import.js | 7 +++- .../test/fixtures/invalid-id-format.ndjson | 3 ++ .../fixtures/valid-but-missing-ids.ndjson | 3 ++ packages/@sanity/import/test/import.test.js | 34 ++++++++++++++++--- 7 files changed, 58 insertions(+), 5 deletions(-) create mode 100644 packages/@sanity/import/src/assignDocumentId.js create mode 100644 packages/@sanity/import/test/fixtures/invalid-id-format.ndjson create mode 100644 packages/@sanity/import/test/fixtures/valid-but-missing-ids.ndjson diff --git a/packages/@sanity/import/package.json b/packages/@sanity/import/package.json index 4e9caf421cb6..4950b899a917 100644 --- a/packages/@sanity/import/package.json +++ b/packages/@sanity/import/package.json @@ -25,6 +25,7 @@ ], "dependencies": { "@sanity/mutator": "^0.120.0", + "@sanity/uuid": "^0.120.0", "debug": "^2.6.3", "get-uri": "^2.0.1", "lodash": "^4.17.4", diff --git a/packages/@sanity/import/src/assignDocumentId.js b/packages/@sanity/import/src/assignDocumentId.js new file mode 100644 index 000000000000..570d5402db42 --- /dev/null +++ b/packages/@sanity/import/src/assignDocumentId.js @@ -0,0 +1,11 @@ +const uuid = require('@sanity/uuid') + +function assignDocumentId(doc) { + if (doc._id) { + return doc + } + + return Object.assign({_id: uuid()}, doc) +} + +module.exports = assignDocumentId diff --git a/packages/@sanity/import/src/documentHasErrors.js b/packages/@sanity/import/src/documentHasErrors.js index 0fb2592f0470..52470a17e049 100644 --- a/packages/@sanity/import/src/documentHasErrors.js +++ b/packages/@sanity/import/src/documentHasErrors.js @@ -3,6 +3,10 @@ function documentHasErrors(doc) { return `Document contained an invalid "_id" property - must be a string` } + if (typeof doc._id !== 'undefined' && !/^[a-z0-9_.-]+$/i.test(doc._id)) { + return `Document ID "${doc._id}" is not valid: Please use alphanumeric document IDs. Dashes (-) and underscores (_) are also allowed.` + } + if (typeof doc._type !== 'string') { return `Document did not contain required "_type" property of type string` } diff --git a/packages/@sanity/import/src/import.js b/packages/@sanity/import/src/import.js index f73b31c8532b..20fdf9a8617b 100644 --- a/packages/@sanity/import/src/import.js +++ b/packages/@sanity/import/src/import.js @@ -4,6 +4,7 @@ const validateOptions = require('./validateOptions') const streamToArray = require('./streamToArray') const {getAssetRefs, unsetAssetRefs} = require('./assetRefs') const assignArrayKeys = require('./assignArrayKeys') +const assignDocumentId = require('./assignDocumentId') const uploadAssets = require('./uploadAssets') const documentHasErrors = require('./documentHasErrors') const batchDocuments = require('./batchDocuments') @@ -28,9 +29,13 @@ async function importDocuments(input, opts) { documents.some(documentHasErrors.validate) } + // Assign document IDs for document that do not have one. This is necessary + // for us to strengthen references and import assets properly. + const ided = documents.map(doc => assignDocumentId(doc)) + // User might not have applied `_key` on array elements which are objects; // if this is the case, generate random keys to help realtime engine - const keyed = documents.map(doc => assignArrayKeys(doc)) + const keyed = ided.map(doc => assignArrayKeys(doc)) // Sanity prefers to have a `_type` on every object. Make sure references // has `_type` set to `reference`. diff --git a/packages/@sanity/import/test/fixtures/invalid-id-format.ndjson b/packages/@sanity/import/test/fixtures/invalid-id-format.ndjson new file mode 100644 index 000000000000..1996909334cd --- /dev/null +++ b/packages/@sanity/import/test/fixtures/invalid-id-format.ndjson @@ -0,0 +1,3 @@ +{"_id": "espen", "_type": "employee", "name": "Espen"} +{"_id": "pk#123", "_type": "employee", "name": "Per-Kristian"} +{"_id": "Even", "_type": "employee", "name": "Even"} diff --git a/packages/@sanity/import/test/fixtures/valid-but-missing-ids.ndjson b/packages/@sanity/import/test/fixtures/valid-but-missing-ids.ndjson new file mode 100644 index 000000000000..ce7c4ebd6b33 --- /dev/null +++ b/packages/@sanity/import/test/fixtures/valid-but-missing-ids.ndjson @@ -0,0 +1,3 @@ +{"_type": "employee", "name": "Espen"} +{"_id":"pk", "_type": "employee", "name": "Per-Kristian"} +{"_type": "employee", "name": "Bjørge"} diff --git a/packages/@sanity/import/test/import.test.js b/packages/@sanity/import/test/import.test.js index a7d5d0847216..53fa52b7e177 100644 --- a/packages/@sanity/import/test/import.test.js +++ b/packages/@sanity/import/test/import.test.js @@ -12,6 +12,7 @@ const defaultClient = sanityClient({ token: 'foo' }) +const uuidMatcher = /^[a-z0-9]+-[a-z0-9]+-[a-z0-9]+-[a-z0-9]+-[a-z0-9]+$/ const importOptions = {client: defaultClient} const fixturesDir = path.join(__dirname, 'fixtures') const getFixturePath = fix => path.join(fixturesDir, `${fix}.ndjson`) @@ -51,6 +52,13 @@ test('rejects on invalid `_id` property', async () => { ) }) +test('rejects on invalid `_id` property format', async () => { + await expect(importer(getFixtureStream('invalid-id-format'), importOptions)).rejects.toHaveProperty( + 'message', + 'Failed to parse line #2: Document ID "pk#123" is not valid: Please use alphanumeric document IDs. Dashes (-) and underscores (_) are also allowed.' + ) +}) + test('rejects on missing `_type` property', async () => { await expect(importer(getFixtureStream('missing-type'), importOptions)).rejects.toHaveProperty( 'message', @@ -67,25 +75,43 @@ test('rejects on missing `_type` property (from array)', async () => { test('accepts an array as source', async () => { const docs = getFixtureArray('employees') - const client = getSanityClient(getMockEmployeeHandler()) + const client = getSanityClient(getMockMutationHandler()) const res = await importer(docs, {client}) expect(res).toBe(2) }) test('accepts a stream as source', async () => { - const client = getSanityClient(getMockEmployeeHandler()) + const client = getSanityClient(getMockMutationHandler()) const res = await importer(getFixtureStream('employees'), {client}) expect(res).toBe(2) }) -function getMockEmployeeHandler() { +test('generates uuids for documents without id', async () => { + const match = body => { + expect(body.mutations[0].create._id).toMatch(uuidMatcher) + expect(body.mutations[1].create._id).toBe('pk') + expect(body.mutations[2].create._id).toMatch(uuidMatcher) + } + + const client = getSanityClient(getMockMutationHandler(match)) + const res = await importer(getFixtureStream('valid-but-missing-ids'), {client}) + expect(res).toBe(3) +}) + +function getMockMutationHandler(match = 'employee creation') { return req => { const options = req.context.options const uri = options.uri || options.url if (uri.includes('/data/mutate')) { const body = JSON.parse(options.body) - expect(body).toMatchSnapshot('employee creation') + + if (typeof match === 'function') { + match(body) + } else { + expect(body).toMatchSnapshot(match) + } + const results = body.mutations.map(mut => ({ id: mut.create.id, operation: 'create'