diff --git a/packages/gatsby/src/db/__tests__/__snapshots__/node-tracking-test.js.snap b/packages/gatsby/src/db/__tests__/__snapshots__/node-tracking-test.js.snap new file mode 100644 index 0000000000000..e397a70cb812c --- /dev/null +++ b/packages/gatsby/src/db/__tests__/__snapshots__/node-tracking-test.js.snap @@ -0,0 +1,22 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`track root nodes Node sanitization Remove not supported fields / values 1`] = ` +Object { + "children": Array [], + "id": "id1", + "inlineArray": Array [ + 1, + 2, + 3, + ], + "inlineObject": Object { + "field": "fieldOfFirstNode", + }, + "internal": Object { + "contentDigest": "digest1", + "owner": "test", + "type": "Test", + }, + "parent": null, +} +`; diff --git a/packages/gatsby/src/db/__tests__/node-tracking-test.js b/packages/gatsby/src/db/__tests__/node-tracking-test.js index c627eabc499fc..8143f5f1ca10c 100644 --- a/packages/gatsby/src/db/__tests__/node-tracking-test.js +++ b/packages/gatsby/src/db/__tests__/node-tracking-test.js @@ -3,7 +3,11 @@ const { boundActionCreators: { createNode }, } = require(`../../redux/actions`) const { getNode } = require(`../../db/nodes`) -const { findRootNodeAncestor, trackDbNodes } = require(`../node-tracking`) +const { + findRootNodeAncestor, + trackDbNodes, + trackInlineObjectsInRootNode, +} = require(`../node-tracking`) const { run: runQuery } = require(`../nodes-query`) require(`./fixtures/ensure-loki`)() @@ -136,4 +140,70 @@ describe(`track root nodes`, () => { expect(findRootNodeAncestor(result[0].inlineObject)).toEqual(result[0]) }) }) + + describe(`Node sanitization`, () => { + let testNode + beforeEach(() => { + testNode = { + id: `id1`, + parent: null, + children: [], + unsupported: () => {}, + inlineObject: { + field: `fieldOfFirstNode`, + re: /re/, + }, + inlineArray: [1, 2, 3, Symbol(`test`)], + internal: { + type: `Test`, + contentDigest: `digest1`, + owner: `test`, + }, + } + }) + + it(`Remove not supported fields / values`, () => { + const result = trackInlineObjectsInRootNode(testNode, true) + expect(result).toMatchSnapshot() + expect(result.unsupported).not.toBeDefined() + expect(result.inlineObject.re).not.toBeDefined() + expect(result.inlineArray[3]).not.toBeDefined() + }) + + it(`Doesn't mutate original`, () => { + trackInlineObjectsInRootNode(testNode, true) + expect(testNode.unsupported).toBeDefined() + expect(testNode.inlineObject.re).toBeDefined() + expect(testNode.inlineArray[3]).toBeDefined() + }) + + it(`Create copy of node if it has to remove anything`, () => { + const result = trackInlineObjectsInRootNode(testNode, true) + expect(result).not.toBe(testNode) + }) + + it(`Doesn't create clones if it doesn't have to`, () => { + const testNodeWithoutUnserializableData = { + id: `id1`, + parent: null, + children: [], + inlineObject: { + field: `fieldOfFirstNode`, + }, + inlineArray: [1, 2, 3], + internal: { + type: `Test`, + contentDigest: `digest1`, + owner: `test`, + }, + } + + const result = trackInlineObjectsInRootNode( + testNodeWithoutUnserializableData, + true + ) + // should be same instance + expect(result).toBe(testNodeWithoutUnserializableData) + }) + }) }) diff --git a/packages/gatsby/src/db/node-tracking.js b/packages/gatsby/src/db/node-tracking.js index b344a6266136c..79b583f608bce 100644 --- a/packages/gatsby/src/db/node-tracking.js +++ b/packages/gatsby/src/db/node-tracking.js @@ -9,18 +9,87 @@ const rootNodeMap = new WeakMap() const getRootNodeId = node => rootNodeMap.get(node) +/** + * @param {Object} data + * @returns {Object} data without undefined values + */ +const omitUndefined = data => { + const isPlainObject = _.isPlainObject(data) + if (isPlainObject) { + return _.pickBy(data, p => p !== undefined) + } + + return data.filter(p => p !== undefined) +} + +/** + * @param {*} data + * @return {boolean} + */ +const isTypeSupported = data => { + if (data === null) { + return true + } + + const type = typeof data + const isSupported = + type === `number` || + type === `string` || + type === `boolean` || + data instanceof Date + + return isSupported +} + /** * Add link between passed data and Node. This function shouldn't be used * directly. Use higher level `trackInlineObjectsInRootNode` * @see trackInlineObjectsInRootNode * @param {(Object|Array)} data Inline object or array * @param {string} nodeId Id of node that contains data passed in first parameter + * @param {boolean} sanitize Wether to strip objects of unuspported and not serializable fields + * @param {string} [ignore] Fieldname that doesn't need to be tracked and sanitized + * */ -const addRootNodeToInlineObject = (data, nodeId) => { - if (_.isPlainObject(data) || _.isArray(data)) { - _.each(data, o => addRootNodeToInlineObject(o, nodeId)) - rootNodeMap.set(data, nodeId) +const addRootNodeToInlineObject = (data, nodeId, sanitize, isNode = false) => { + const isPlainObject = _.isPlainObject(data) + + if (isPlainObject || _.isArray(data)) { + let returnData = data + if (sanitize) { + returnData = isPlainObject ? {} : [] + } + let anyFieldChanged = false + _.each(data, (o, key) => { + if (isNode && key === `internal`) { + returnData[key] = o + return + } + returnData[key] = addRootNodeToInlineObject(o, nodeId, sanitize) + + if (returnData[key] !== o) { + anyFieldChanged = true + } + }) + + if (anyFieldChanged) { + data = omitUndefined(returnData) + } + + // don't need to track node itself + if (!isNode) { + rootNodeMap.set(data, nodeId) + } + + // arrays and plain objects are supported - no need to to sanitize + return data } + + if (sanitize && !isTypeSupported(data)) { + return undefined + } + // either supported or not sanitizing + return data } /** @@ -28,17 +97,8 @@ const addRootNodeToInlineObject = (data, nodeId) => { * and that Node object. * @param {Node} node Root Node */ -const trackInlineObjectsInRootNode = node => { - _.each(node, (v, k) => { - // Ignore the node internal object. - if (k === `internal`) { - return - } - addRootNodeToInlineObject(v, node.id) - }) - - return node -} +const trackInlineObjectsInRootNode = (node, sanitize = false) => + addRootNodeToInlineObject(node, node.id, sanitize, true) exports.trackInlineObjectsInRootNode = trackInlineObjectsInRootNode /** diff --git a/packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap b/packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap new file mode 100644 index 0000000000000..14658aae15f86 --- /dev/null +++ b/packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap @@ -0,0 +1,25 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`redux db should write cache to disk 1`] = ` +Object { + "componentDataDependencies": Object { + "connections": Object {}, + "nodes": Object {}, + }, + "components": Map { + "/Users/username/dev/site/src/templates/my-sweet-new-page.js" => Object { + "componentPath": "/Users/username/dev/site/src/templates/my-sweet-new-page.js", + "isInBootstrap": true, + "pages": Array [ + "/my-sweet-new-page/", + ], + "query": "", + }, + }, + "jsonDataPaths": Object {}, + "staticQueryComponents": Map {}, + "status": Object { + "plugins": Object {}, + }, +} +`; diff --git a/packages/gatsby/src/redux/__tests__/index.js b/packages/gatsby/src/redux/__tests__/index.js index 9581cf4780001..7c14ff137b613 100644 --- a/packages/gatsby/src/redux/__tests__/index.js +++ b/packages/gatsby/src/redux/__tests__/index.js @@ -1,19 +1,32 @@ -const path = require(`path`) -const fs = require(`fs-extra`) -const { saveState, store } = require(`../index`) +const _ = require(`lodash`) + +const writeToCache = jest.spyOn(require(`../persist`), `writeToCache`) +const { saveState, store, readState } = require(`../index`) + const { actions: { createPage }, } = require(`../actions`) -jest.mock(`fs-extra`) +const mockWrittenContent = new Map() +jest.mock(`fs-extra`, () => { + return { + writeFileSync: jest.fn((file, content) => + mockWrittenContent.set(file, content) + ), + readFileSync: jest.fn(file => mockWrittenContent.get(file)), + } +}) describe(`redux db`, () => { + const initialComponentsState = _.cloneDeep(store.getState().components) + beforeEach(() => { store.dispatch( createPage( { path: `/my-sweet-new-page/`, - component: path.resolve(`./src/templates/my-sweet-new-page.js`), + // seems like jest serializer doesn't play nice with Maps on Windows + component: `/Users/username/dev/site/src/templates/my-sweet-new-page.js`, // The context is passed as props to the component as well // as into the component's GraphQL query. context: { @@ -24,15 +37,34 @@ describe(`redux db`, () => { ) ) - fs.writeFile.mockClear() + writeToCache.mockClear() + mockWrittenContent.clear() + }) + + it(`expect components state to be empty initially`, () => { + expect(initialComponentsState).toEqual(new Map()) }) + it(`should write cache to disk`, async () => { await saveState() - expect(fs.writeFile).toBeCalledWith( - expect.stringContaining(`.cache/redux-state.json`), - expect.stringContaining(`my-sweet-new-page.js`) - ) + expect(writeToCache).toBeCalled() + + // reset state in memory + store.dispatch({ + type: `DELETE_CACHE`, + }) + // make sure store in memory is empty + expect(store.getState().components).toEqual(initialComponentsState) + + // read data that was previously cached + const data = readState() + + // make sure data was read and is not the same as our clean redux state + expect(data.components).not.toEqual(initialComponentsState) + + // yuck - loki and redux will have different shape of redux state (nodes and nodesByType) + expect(_.omit(data, [`nodes`, `nodesByType`])).toMatchSnapshot() }) it(`does not write to the cache when DANGEROUSLY_DISABLE_OOM is set`, async () => { @@ -40,7 +72,7 @@ describe(`redux db`, () => { await saveState() - expect(fs.writeFile).not.toBeCalled() + expect(writeToCache).not.toBeCalled() delete process.env.DANGEROUSLY_DISABLE_OOM }) diff --git a/packages/gatsby/src/redux/actions.js b/packages/gatsby/src/redux/actions.js index 5db90337089fb..cd235cb7a0986 100644 --- a/packages/gatsby/src/redux/actions.js +++ b/packages/gatsby/src/redux/actions.js @@ -591,7 +591,7 @@ actions.createNode = ( ) } - trackInlineObjectsInRootNode(node) + node = trackInlineObjectsInRootNode(node, true) const oldNode = getNode(node.id) diff --git a/packages/gatsby/src/redux/index.js b/packages/gatsby/src/redux/index.js index eba0ef95aa2c9..9f598a0171ef3 100644 --- a/packages/gatsby/src/redux/index.js +++ b/packages/gatsby/src/redux/index.js @@ -1,68 +1,42 @@ const Redux = require(`redux`) const _ = require(`lodash`) -const fs = require(`fs-extra`) + const mitt = require(`mitt`) -const stringify = require(`json-stringify-safe`) // Create event emitter for actions const emitter = mitt() // Reducers const reducers = require(`./reducers`) +const { writeToCache, readFromCache } = require(`./persist`) -const objectToMap = obj => { - let map = new Map() - Object.keys(obj).forEach(key => { - map.set(key, obj[key]) - }) - return map -} - -const mapToObject = map => { - const obj = {} - for (let [key, value] of map) { - obj[key] = value +// Read old node data from cache. +const readState = () => { + try { + const state = readFromCache() + if (state.nodes) { + // re-create nodesByType + state.nodesByType = new Map() + state.nodes.forEach(node => { + const { type } = node.internal + if (!state.nodesByType.has(type)) { + state.nodesByType.set(type, new Map()) + } + state.nodesByType.get(type).set(node.id, node) + }) + } + return state + } catch (e) { + // ignore errors. } - return obj + return {} } -// Read from cache the old node data. -let initialState = {} -try { - const file = fs.readFileSync(`${process.cwd()}/.cache/redux-state.json`) - // Apparently the file mocking in node-tracking-test.js - // can override the file reading replacing the mocked string with - // an already parsed object. - if (Buffer.isBuffer(file) || typeof file === `string`) { - initialState = JSON.parse(file) - } - if (initialState.staticQueryComponents) { - initialState.staticQueryComponents = objectToMap( - initialState.staticQueryComponents - ) - } - if (initialState.components) { - initialState.components = objectToMap(initialState.components) - } - if (initialState.nodes) { - initialState.nodes = objectToMap(initialState.nodes) - - initialState.nodesByType = new Map() - initialState.nodes.forEach(node => { - const { type } = node.internal - if (!initialState.nodesByType.has(type)) { - initialState.nodesByType.set(type, new Map()) - } - initialState.nodesByType.get(type).set(node.id, node) - }) - } -} catch (e) { - // ignore errors. -} +exports.readState = readState const store = Redux.createStore( Redux.combineReducers({ ...reducers }), - initialState, + readState(), Redux.applyMiddleware(function multi({ dispatch }) { return next => action => Array.isArray(action) @@ -87,13 +61,7 @@ function saveState() { `staticQueryComponents`, ]) - pickedState.staticQueryComponents = mapToObject( - pickedState.staticQueryComponents - ) - pickedState.components = mapToObject(pickedState.components) - pickedState.nodes = pickedState.nodes ? mapToObject(pickedState.nodes) : [] - const stringified = stringify(pickedState, null, 2) - return fs.writeFile(`${process.cwd()}/.cache/redux-state.json`, stringified) + return writeToCache(pickedState) } exports.saveState = saveState diff --git a/packages/gatsby/src/redux/persist.js b/packages/gatsby/src/redux/persist.js new file mode 100644 index 0000000000000..2436056955c05 --- /dev/null +++ b/packages/gatsby/src/redux/persist.js @@ -0,0 +1,47 @@ +const v8 = require(`v8`) +const fs = require(`fs-extra`) +const stringify = require(`json-stringify-safe`) + +const objectToMap = obj => { + const map = new Map() + Object.keys(obj).forEach(key => { + map.set(key, obj[key]) + }) + return map +} + +const mapToObject = map => { + const obj = {} + for (let [key, value] of map) { + obj[key] = value + } + return obj +} + +const jsonStringify = contents => { + contents.staticQueryComponents = mapToObject(contents.staticQueryComponents) + contents.components = mapToObject(contents.components) + contents.nodes = contents.nodes ? mapToObject(contents.nodes) : {} + return stringify(contents, null, 2) +} + +const jsonParse = buffer => { + const parsed = JSON.parse(buffer.toString(`utf8`)) + parsed.staticQueryComponents = objectToMap(parsed.staticQueryComponents) + parsed.components = objectToMap(parsed.components) + parsed.nodes = objectToMap(parsed.nodes || {}) + return parsed +} + +const useV8 = Boolean(v8.serialize) +const [serialize, deserialize, file] = useV8 + ? [v8.serialize, v8.deserialize, `${process.cwd()}/.cache/redux.state`] + : [jsonStringify, jsonParse, `${process.cwd()}/.cache/redux-state.json`] + +const readFromCache = () => deserialize(fs.readFileSync(file)) + +const writeToCache = contents => { + fs.writeFileSync(file, serialize(contents)) +} + +module.exports = { readFromCache, writeToCache }