From 7d31fe75f8d73bb52e7f10384033429bafbc8186 Mon Sep 17 00:00:00 2001 From: Anthony Marcar Date: Tue, 30 Oct 2018 06:20:36 +1100 Subject: [PATCH] feat(gatsby): Add nodes db module (#9416) In preparation for #9338, I've added a new nodes db module that can be implemented by an appropriate db nodes backend. Which is only redux as of now. The loki PR will eventually be implemented so that it can be featured flagged on/off using this functionality. I also added `getNodesByType` since it makes sense even without loki. --- .../gatsby-source-contentful/package.json | 2 +- .../src/gatsby-node.js | 6 +- .../gatsby-transformer-remark/package.json | 2 +- .../src/__tests__/extend-node.js | 93 +++++++++++-------- .../src/extend-node-type.js | 24 +++-- .../package.json | 2 +- .../src/gatsby-node.js | 6 +- .../gatsby-transformer-sharp/package.json | 2 +- .../src/gatsby-node.js | 6 +- packages/gatsby/src/bootstrap/index.js | 4 + .../__snapshots__/load-plugins.js.snap | 28 +++--- .../gatsby/src/bootstrap/load-plugins/load.js | 16 +++- packages/gatsby/src/commands/repl.js | 12 ++- packages/gatsby/src/db/loki/nodes.js | 12 +++ packages/gatsby/src/db/nodes.js | 16 ++++ .../internal-data-bridge/gatsby-node.js | 2 +- packages/gatsby/src/redux/__tests__/nodes.js | 3 +- packages/gatsby/src/redux/actions.js | 5 +- packages/gatsby/src/redux/index.js | 87 ----------------- packages/gatsby/src/redux/nodes.js | 91 ++++++++++++++++++ packages/gatsby/src/redux/plugin-runner.js | 5 +- .../schema/__tests__/node-tracking-test.js | 2 +- .../gatsby/src/schema/__tests__/run-sift.js | 2 +- .../src/schema/build-node-connections.js | 7 +- .../gatsby/src/schema/build-node-types.js | 12 ++- .../src/schema/infer-graphql-input-fields.js | 6 +- .../gatsby/src/schema/infer-graphql-type.js | 13 ++- packages/gatsby/src/schema/node-tracking.js | 2 +- packages/gatsby/src/schema/run-sift.js | 2 +- packages/gatsby/src/schema/types/type-file.js | 8 +- packages/gatsby/src/utils/api-runner-node.js | 7 +- packages/gatsby/src/utils/source-nodes.js | 7 +- 32 files changed, 284 insertions(+), 208 deletions(-) create mode 100644 packages/gatsby/src/db/loki/nodes.js create mode 100644 packages/gatsby/src/db/nodes.js create mode 100644 packages/gatsby/src/redux/nodes.js diff --git a/packages/gatsby-source-contentful/package.json b/packages/gatsby-source-contentful/package.json index 817abc3b7fa7b..56e09ead6fde7 100644 --- a/packages/gatsby-source-contentful/package.json +++ b/packages/gatsby-source-contentful/package.json @@ -34,7 +34,7 @@ ], "license": "MIT", "peerDependencies": { - "gatsby": ">2.0.0-alpha" + "gatsby": "^2.0.33" }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-contentful", "scripts": { diff --git a/packages/gatsby-source-contentful/src/gatsby-node.js b/packages/gatsby-source-contentful/src/gatsby-node.js index 9f922498dc0c4..ac425b77f332c 100644 --- a/packages/gatsby-source-contentful/src/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/gatsby-node.js @@ -216,7 +216,7 @@ exports.sourceNodes = async ( // Check if there are any ContentfulAsset nodes and if gatsby-image is installed. If so, // add fragments for ContentfulAsset and gatsby-image. The fragment will cause an error // if there's not ContentfulAsset nodes and without gatsby-image, the fragment is useless. -exports.onPreExtractQueries = async ({ store, getNodes }) => { +exports.onPreExtractQueries = async ({ store, getNodesByType }) => { const program = store.getState().program const CACHE_DIR = path.resolve( @@ -224,9 +224,7 @@ exports.onPreExtractQueries = async ({ store, getNodes }) => { ) await fs.ensureDir(CACHE_DIR) - const nodes = getNodes() - - if (!nodes.some(n => n.internal.type === `ContentfulAsset`)) { + if (getNodesByType(`ContentfulAsset`).length == 0) { return } diff --git a/packages/gatsby-transformer-remark/package.json b/packages/gatsby-transformer-remark/package.json index b5fd86d2671d1..796e94ffb0066 100644 --- a/packages/gatsby-transformer-remark/package.json +++ b/packages/gatsby-transformer-remark/package.json @@ -42,7 +42,7 @@ ], "license": "MIT", "peerDependencies": { - "gatsby": ">2.0.0-alpha" + "gatsby": "^2.0.33" }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-transformer-remark", "scripts": { diff --git a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js index 51fcb01df9803..53b3eeca6b783 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js +++ b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js @@ -11,7 +11,12 @@ const { const extendNodeType = require(`../extend-node-type`) // given a set of nodes and a query, return the result of the query -async function queryResult(nodes, fragment, { types = [] } = {}, additionalParameters) { +async function queryResult( + nodes, + fragment, + { types = [] } = {}, + additionalParameters +) { const inferredFields = inferObjectStructureFromNodes({ nodes, types: [...types], @@ -23,7 +28,7 @@ async function queryResult(nodes, fragment, { types = [] } = {}, additionalParam get: () => null, set: () => null, }, - getNodes: () => [], + getNodesByType: type => [], ...additionalParameters, }, { @@ -70,7 +75,13 @@ async function queryResult(nodes, fragment, { types = [] } = {}, additionalParam return result } -const bootstrapTest = (label, content, query, test, additionalParameters = {}) => { +const bootstrapTest = ( + label, + content, + query, + test, + additionalParameters = {} +) => { const node = { id: `whatever`, children: [], @@ -82,7 +93,7 @@ const bootstrapTest = (label, content, query, test, additionalParameters = {}) = // Make some fake functions its expecting. const loadNodeContent = node => Promise.resolve(node.content) - it(label, async (done) => { + it(label, async done => { node.content = content const createNode = markdownNode => { queryResult( @@ -96,8 +107,7 @@ const bootstrapTest = (label, content, query, test, additionalParameters = {}) = try { test(result.data.listNode[0]) done() - } - catch(err) { + } catch (err) { done.fail(err) } }) @@ -106,19 +116,19 @@ const bootstrapTest = (label, content, query, test, additionalParameters = {}) = const actions = { createNode, createParentChildLink } const createNodeId = jest.fn() createNodeId.mockReturnValue(`uuid-from-gatsby`) - await onCreateNode({ - node, - loadNodeContent, - actions, - createNodeId, - }, - { ...additionalParameters } + await onCreateNode( + { + node, + loadNodeContent, + actions, + createNodeId, + }, + { ...additionalParameters } ) - }) + }) } describe(`Excerpt is generated correctly from schema`, () => { - bootstrapTest( `correctly loads an excerpt`, `--- @@ -131,7 +141,7 @@ Where oh where is my little pony?`, title } `, - (node) => { + node => { expect(node).toMatchSnapshot() expect(node.excerpt).toMatch(`Where oh where is my little pony?`) } @@ -148,7 +158,7 @@ date: "2017-09-18T23:19:51.246Z" title } `, - (node) => { + node => { expect(node).toMatchSnapshot() expect(node.excerpt).toMatch(``) } @@ -171,7 +181,7 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid title } `, - (node) => { + node => { expect(node).toMatchSnapshot() expect(node.excerpt).toMatch(`Where oh where is my little pony?`) }, @@ -194,7 +204,7 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid title } `, - (node) => { + node => { expect(node).toMatchSnapshot() expect(node.excerpt.length).toBe(139) } @@ -208,7 +218,7 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid title } `, - (node) => { + node => { expect(node).toMatchSnapshot() expect(node.excerpt.length).toBe(46) } @@ -222,7 +232,7 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid title } `, - (node) => { + node => { expect(node).toMatchSnapshot() expect(node.excerpt.length).toBe(50) } @@ -248,16 +258,15 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid frontmatter { title }`, - (node) => { + node => { expect(node).toMatchSnapshot() - expect(node.wordCount).toEqual( - { + expect(node.wordCount).toEqual({ paragraphs: 2, sentences: 19, words: 150, - } - ) - }) + }) + } + ) const content = `--- title: "my little pony" @@ -276,16 +285,15 @@ date: "2017-09-18T23:19:51.246Z" frontmatter { title }`, - (node) => { + node => { expect(node).toMatchSnapshot() - expect(node.wordCount).toEqual( - { + expect(node.wordCount).toEqual({ paragraphs: null, sentences: null, words: null, - } - ) - }) + }) + } + ) bootstrapTest( `correctly uses a default value for timeToRead`, @@ -294,10 +302,11 @@ date: "2017-09-18T23:19:51.246Z" frontmatter { title }`, - (node) => { + node => { expect(node).toMatchSnapshot() expect(node.timeToRead).toBe(1) - }) + } + ) }) describe(`Table of contents is generated correctly from schema`, () => { @@ -322,11 +331,12 @@ some other text frontmatter { title }`, - (node) => { + node => { expect(node).toMatchSnapshot() expect(console.warn).toBeCalled() expect(node.tableOfContents).toBe(null) - }) + } + ) bootstrapTest( `correctly generates table of contents`, @@ -350,9 +360,10 @@ final text frontmatter { title }`, - (node) => { - expect(node).toMatchSnapshot() - }) + node => { + expect(node).toMatchSnapshot() + } + ) }) describe(`Links are correctly prefixed`, () => { @@ -366,7 +377,7 @@ This is [a reference] [a reference]: /path/to/page2 `, `html`, - (node) => { + node => { expect(node).toMatchSnapshot() expect(node.html).toMatch(``) expect(node.html).toMatch(``) diff --git a/packages/gatsby-transformer-remark/src/extend-node-type.js b/packages/gatsby-transformer-remark/src/extend-node-type.js index fe95b3fd7bc48..07827d42b1458 100644 --- a/packages/gatsby-transformer-remark/src/extend-node-type.js +++ b/packages/gatsby-transformer-remark/src/extend-node-type.js @@ -62,7 +62,7 @@ const withPathPrefix = (url, pathPrefix) => const ASTPromiseMap = new Map() module.exports = ( - { type, store, pathPrefix, getNode, getNodes, cache, reporter }, + { type, store, pathPrefix, getNode, getNodesByType, cache, reporter }, pluginOptions ) => { if (type.name !== `MarkdownRemark`) { @@ -74,7 +74,13 @@ module.exports = ( return new Promise((resolve, reject) => { // Setup Remark. - const { commonmark = true, footnotes = true, pedantic = true, gfm = true, blocks } = pluginOptions + const { + commonmark = true, + footnotes = true, + pedantic = true, + gfm = true, + blocks, + } = pluginOptions const remarkOptions = { gfm, commonmark, @@ -113,7 +119,7 @@ module.exports = ( } else { const ASTGenerationPromise = new Promise(async resolve => { if (process.env.NODE_ENV !== `production` || !fileNodes) { - fileNodes = getNodes().filter(n => n.internal.type === `File`) + fileNodes = getNodesByType(`File`) } const ast = await new Promise((resolve, reject) => { // Use Bluebird's Promise function "each" to run remark plugins serially. @@ -180,7 +186,7 @@ module.exports = ( // typegen plugins just modify the auto-generated types to add derived fields // as well as computationally expensive fields. if (process.env.NODE_ENV !== `production` || !fileNodes) { - fileNodes = getNodes().filter(n => n.internal.type === `File`) + fileNodes = getNodesByType(`File`) } // Use Bluebird's Promise function "each" to run remark plugins serially. Promise.each(pluginOptions.plugins, plugin => { @@ -249,10 +255,16 @@ module.exports = ( const addSlugToUrl = function(node) { if (node.url) { if (_.get(markdownNode, pathToSlugField) === undefined) { - console.warn(`Skipping TableOfContents. Field '${pathToSlugField}' missing from markdown node`) + console.warn( + `Skipping TableOfContents. Field '${pathToSlugField}' missing from markdown node` + ) return null } - node.url = [pathPrefix, _.get(markdownNode, pathToSlugField), node.url] + node.url = [ + pathPrefix, + _.get(markdownNode, pathToSlugField), + node.url, + ] .join(`/`) .replace(/\/\//g, `/`) } diff --git a/packages/gatsby-transformer-screenshot/package.json b/packages/gatsby-transformer-screenshot/package.json index 69bea066974ea..933cc371107dc 100644 --- a/packages/gatsby-transformer-screenshot/package.json +++ b/packages/gatsby-transformer-screenshot/package.json @@ -25,7 +25,7 @@ "license": "MIT", "main": "index.js", "peerDependencies": { - "gatsby": ">2.0.15" + "gatsby": "^2.0.33" }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-transformer-screenshot", "scripts": { diff --git a/packages/gatsby-transformer-screenshot/src/gatsby-node.js b/packages/gatsby-transformer-screenshot/src/gatsby-node.js index 8dfbf597d2c58..7461380add999 100644 --- a/packages/gatsby-transformer-screenshot/src/gatsby-node.js +++ b/packages/gatsby-transformer-screenshot/src/gatsby-node.js @@ -16,13 +16,11 @@ const screenshotQueue = new Queue( ) exports.onPreBootstrap = ( - { store, cache, actions, createNodeId, getNodes, createContentDigest }, + { store, cache, actions, createNodeId, getNodesByType, createContentDigest }, pluginOptions ) => { const { createNode, touchNode } = actions - const screenshotNodes = getNodes().filter( - n => n.internal.type === `Screenshot` - ) + const screenshotNodes = getNodesByType(`Screenshot`) if (screenshotNodes.length === 0) { return null diff --git a/packages/gatsby-transformer-sharp/package.json b/packages/gatsby-transformer-sharp/package.json index d28af3123ab2c..395d2355b9530 100644 --- a/packages/gatsby-transformer-sharp/package.json +++ b/packages/gatsby-transformer-sharp/package.json @@ -29,7 +29,7 @@ ], "license": "MIT", "peerDependencies": { - "gatsby": ">2.0.0-alpha", + "gatsby": "^2.0.33", "gatsby-plugin-sharp": "^2.0.0-beta.3" }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-transformer-sharp", diff --git a/packages/gatsby-transformer-sharp/src/gatsby-node.js b/packages/gatsby-transformer-sharp/src/gatsby-node.js index 2bb6f5ac6ce6d..3f9433c15f639 100644 --- a/packages/gatsby-transformer-sharp/src/gatsby-node.js +++ b/packages/gatsby-transformer-sharp/src/gatsby-node.js @@ -3,14 +3,12 @@ const fs = require(`fs-extra`) exports.onCreateNode = require(`./on-node-create`) exports.setFieldsOnGraphQLNodeType = require(`./extend-node-type`) -exports.onPreExtractQueries = async ({ store, getNodes }) => { +exports.onPreExtractQueries = async ({ store, getNodesByType }) => { const program = store.getState().program // Check if there are any ImageSharp nodes. If so add fragments for ImageSharp. // The fragment will cause an error if there are no ImageSharp nodes. - const nodes = getNodes() - - if (!nodes.some(n => n.internal.type === `ImageSharp`)) { + if (getNodesByType(`ImageSharp`).length == 0) { return } diff --git a/packages/gatsby/src/bootstrap/index.js b/packages/gatsby/src/bootstrap/index.js index 261007bd6d7a5..a302452d098fc 100644 --- a/packages/gatsby/src/bootstrap/index.js +++ b/packages/gatsby/src/bootstrap/index.js @@ -52,6 +52,10 @@ module.exports = async (args: BootstrapArgs) => { const spanArgs = args.parentSpan ? { childOf: args.parentSpan } : {} const bootstrapSpan = tracer.startSpan(`bootstrap`, spanArgs) + // Start plugin runner which listens to the store + // and invokes Gatsby API based on actions. + require(`../redux/plugin-runner`) + const program = { ...args, // Fix program directory path for windows env. diff --git a/packages/gatsby/src/bootstrap/load-plugins/__tests__/__snapshots__/load-plugins.js.snap b/packages/gatsby/src/bootstrap/load-plugins/__tests__/__snapshots__/load-plugins.js.snap index b3ce0d7cae856..3a9a00fb3947b 100644 --- a/packages/gatsby/src/bootstrap/load-plugins/__tests__/__snapshots__/load-plugins.js.snap +++ b/packages/gatsby/src/bootstrap/load-plugins/__tests__/__snapshots__/load-plugins.js.snap @@ -4,7 +4,7 @@ exports[`Load plugins Load plugins for a site 1`] = ` Array [ Object { "browserAPIs": Array [], - "id": "Plugin dev-404-page", + "id": "d48d1c52-fe48-53cb-8f08-aa4b47dde5a5", "name": "dev-404-page", "nodeAPIs": Array [ "createPagesStatefully", @@ -18,7 +18,7 @@ Array [ }, Object { "browserAPIs": Array [], - "id": "Plugin load-babel-config", + "id": "1fc32581-893a-55e8-8927-bcd667e2b700", "name": "load-babel-config", "nodeAPIs": Array [ "onPreBootstrap", @@ -32,7 +32,7 @@ Array [ }, Object { "browserAPIs": Array [], - "id": "Plugin internal-data-bridge", + "id": "a5079d69-ba80-53dc-82f9-0f440bd5448c", "name": "internal-data-bridge", "nodeAPIs": Array [ "sourceNodes", @@ -47,7 +47,7 @@ Array [ }, Object { "browserAPIs": Array [], - "id": "Plugin prod-404", + "id": "f795702c-a3b8-5a88-88ee-5d06019d44fa", "name": "prod-404", "nodeAPIs": Array [ "onCreatePage", @@ -61,7 +61,7 @@ Array [ }, Object { "browserAPIs": Array [], - "id": "Plugin query-runner", + "id": "84dad27f-1d44-51fc-ac56-4db2e5222995", "name": "query-runner", "nodeAPIs": Array [ "onCreatePage", @@ -75,7 +75,7 @@ Array [ }, Object { "browserAPIs": Array [], - "id": "Plugin default-site-plugin", + "id": "7374ebf2-d961-52ee-92a2-c25e7cb387a9", "name": "default-site-plugin", "nodeAPIs": Array [], "pluginOptions": Object { @@ -87,7 +87,7 @@ Array [ }, Object { "browserAPIs": Array [], - "id": "Plugin gatsby-plugin-page-creator", + "id": "d5fb39bc-5b13-5925-86de-f0e18549d272", "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -108,7 +108,7 @@ exports[`Load plugins Loads plugins defined with an object but without an option Array [ Object { "browserAPIs": Array [], - "id": "Plugin dev-404-page", + "id": "d48d1c52-fe48-53cb-8f08-aa4b47dde5a5", "name": "dev-404-page", "nodeAPIs": Array [ "createPagesStatefully", @@ -122,7 +122,7 @@ Array [ }, Object { "browserAPIs": Array [], - "id": "Plugin load-babel-config", + "id": "1fc32581-893a-55e8-8927-bcd667e2b700", "name": "load-babel-config", "nodeAPIs": Array [ "onPreBootstrap", @@ -136,7 +136,7 @@ Array [ }, Object { "browserAPIs": Array [], - "id": "Plugin internal-data-bridge", + "id": "a5079d69-ba80-53dc-82f9-0f440bd5448c", "name": "internal-data-bridge", "nodeAPIs": Array [ "sourceNodes", @@ -151,7 +151,7 @@ Array [ }, Object { "browserAPIs": Array [], - "id": "Plugin prod-404", + "id": "f795702c-a3b8-5a88-88ee-5d06019d44fa", "name": "prod-404", "nodeAPIs": Array [ "onCreatePage", @@ -165,7 +165,7 @@ Array [ }, Object { "browserAPIs": Array [], - "id": "Plugin query-runner", + "id": "84dad27f-1d44-51fc-ac56-4db2e5222995", "name": "query-runner", "nodeAPIs": Array [ "onCreatePage", @@ -190,7 +190,7 @@ Array [ }, Object { "browserAPIs": Array [], - "id": "Plugin default-site-plugin", + "id": "7374ebf2-d961-52ee-92a2-c25e7cb387a9", "name": "default-site-plugin", "nodeAPIs": Array [], "pluginOptions": Object { @@ -202,7 +202,7 @@ Array [ }, Object { "browserAPIs": Array [], - "id": "Plugin gatsby-plugin-page-creator", + "id": "d5fb39bc-5b13-5925-86de-f0e18549d272", "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", diff --git a/packages/gatsby/src/bootstrap/load-plugins/load.js b/packages/gatsby/src/bootstrap/load-plugins/load.js index 89e745e532c04..ef79cb5e00a4a 100644 --- a/packages/gatsby/src/bootstrap/load-plugins/load.js +++ b/packages/gatsby/src/bootstrap/load-plugins/load.js @@ -6,6 +6,7 @@ const crypto = require(`crypto`) const glob = require(`glob`) const { store } = require(`../../redux`) const existsSync = require(`fs-exists-cached`).sync +const createNodeId = require(`../../utils/create-node-id`) function createFileContentHash(root, globPattern) { const hash = crypto.createHash(`md5`) @@ -48,7 +49,7 @@ function resolvePlugin(pluginName) { return { resolve: resolvedPath, name: packageJSON.name || pluginName, - id: `Plugin ${packageJSON.name || pluginName}`, + id: createNodeId(packageJSON.name, `Plugin`), version: packageJSON.version || createFileContentHash(resolvedPath, `**`), } @@ -72,7 +73,7 @@ function resolvePlugin(pluginName) { return { resolve: resolvedPath, - id: `Plugin ${packageJSON.name}`, + id: createNodeId(packageJSON.name, `Plugin`), name: packageJSON.name, version: packageJSON.version, } @@ -126,6 +127,15 @@ module.exports = (config = {}) => { return { ...info, + // Make sure key is unique to plugin options. E.g there could + // be multiple source-filesystem plugins, with different names + // (docs, blogs). + id: createNodeId( + plugin.options + ? plugin.name + JSON.stringify(plugin.options) + : plugin.name, + `Plugin` + ), pluginOptions: _.merge({ plugins: [] }, plugin.options), } } @@ -154,7 +164,7 @@ module.exports = (config = {}) => { // Add the site's default "plugin" i.e. gatsby-x files in root of site. plugins.push({ resolve: slash(process.cwd()), - id: `Plugin default-site-plugin`, + id: createNodeId(`default-site-plugin`, `Plugin`), name: `default-site-plugin`, version: createFileContentHash(process.cwd(), `gatsby-*`), pluginOptions: { diff --git a/packages/gatsby/src/commands/repl.js b/packages/gatsby/src/commands/repl.js index 00ce79e987e31..926f9af76d873 100644 --- a/packages/gatsby/src/commands/repl.js +++ b/packages/gatsby/src/commands/repl.js @@ -1,7 +1,13 @@ const repl = require(`repl`) const { graphql } = require(`graphql`) const bootstrap = require(`../bootstrap`) -const { store, loadNodeContent, getNodes, getNode } = require(`../redux`) +const { + loadNodeContent, + getNodes, + getNode, + getNodesByType, +} = require(`../db/nodes`) +const { store } = require(`../redux`) module.exports = async program => { // run bootstrap @@ -16,9 +22,10 @@ module.exports = async program => { pages, components, staticQueryComponents, - nodes, } = store.getState() + const nodes = getNodes() + const query = async query => { const result = await graphql(schema, query, {}, {}, {}) console.log(`query result: ${JSON.stringify(result)}`) @@ -35,6 +42,7 @@ module.exports = async program => { _.context.dataPaths = jsonDataPaths _.context.getNode = getNode _.context.getNodes = getNodes + _.context.getNodesByType = getNodesByType _.context.loadNodeContent = loadNodeContent _.context.nodes = [...nodes.entries()] _.context.pages = [...pages.entries()] diff --git a/packages/gatsby/src/db/loki/nodes.js b/packages/gatsby/src/db/loki/nodes.js new file mode 100644 index 0000000000000..efe043758165c --- /dev/null +++ b/packages/gatsby/src/db/loki/nodes.js @@ -0,0 +1,12 @@ +function notSupported() { + throw new Error(`Loki not supported yet`) +} + +module.exports = { + getNodes: notSupported(), + getNode: notSupported(), + getNodesByType: notSupported(), + hasNodeChanged: notSupported(), + loadNodeContent: notSupported(), + getNodeAndSavePathDependency: notSupported(), +} diff --git a/packages/gatsby/src/db/nodes.js b/packages/gatsby/src/db/nodes.js new file mode 100644 index 0000000000000..ac4b8fbe394c4 --- /dev/null +++ b/packages/gatsby/src/db/nodes.js @@ -0,0 +1,16 @@ +const backend = process.env.GATSBY_DB_NODES || `redux` +let nodesDb +switch (backend) { + case `redux`: + nodesDb = require(`../redux/nodes`) + break + case `loki`: + nodesDb = require(`./loki/nodes`) + break + default: + throw new Error( + `Unsupported DB nodes backend (value of env var GATSBY_DB_NODES)` + ) +} + +module.exports = nodesDb diff --git a/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js b/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js index a5af127a384de..ce88d751b67b5 100644 --- a/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js +++ b/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js @@ -6,7 +6,7 @@ const _ = require(`lodash`) const { emitter } = require(`../../redux`) const { boundActionCreators } = require(`../../redux/actions`) -const { getNode } = require(`../../redux`) +const { getNode } = require(`../../db/nodes`) function transformPackageJson(json) { const transformDeps = deps => diff --git a/packages/gatsby/src/redux/__tests__/nodes.js b/packages/gatsby/src/redux/__tests__/nodes.js index 3ce2ac9efbd74..95e380749d1ed 100644 --- a/packages/gatsby/src/redux/__tests__/nodes.js +++ b/packages/gatsby/src/redux/__tests__/nodes.js @@ -1,5 +1,6 @@ const { actions } = require(`../actions`) -const { store, getNode } = require(`../index`) +const { store } = require(`../index`) +const { getNode } = require(`../nodes`) const nodeReducer = require(`../reducers/nodes`) const nodeTouchedReducer = require(`../reducers/nodes-touched`) diff --git a/packages/gatsby/src/redux/actions.js b/packages/gatsby/src/redux/actions.js index b54e66a61ffe6..bae7afc761f5e 100644 --- a/packages/gatsby/src/redux/actions.js +++ b/packages/gatsby/src/redux/actions.js @@ -9,7 +9,7 @@ const path = require(`path`) const fs = require(`fs`) const url = require(`url`) const kebabHash = require(`kebab-hash`) -const { hasNodeChanged, getNode } = require(`./index`) +const { hasNodeChanged, getNode } = require(`../db/nodes`) const { trackInlineObjectsInRootNode } = require(`../schema/node-tracking`) const { store } = require(`./index`) const fileExistsSync = require(`fs-exists-cached`).sync @@ -1093,7 +1093,8 @@ actions.createRedirect = ({ // url.parse will not cover protocol-relative urls so do a separate check for those const parsed = url.parse(toPath) const isRelativeProtocol = toPath.startsWith(`//`) - const toPathPrefix = parsed.protocol != null || isRelativeProtocol ? `` : pathPrefix + const toPathPrefix = + parsed.protocol != null || isRelativeProtocol ? `` : pathPrefix return { type: `CREATE_REDIRECT`, diff --git a/packages/gatsby/src/redux/index.js b/packages/gatsby/src/redux/index.js index 0198062671108..203b0438aac81 100644 --- a/packages/gatsby/src/redux/index.js +++ b/packages/gatsby/src/redux/index.js @@ -1,5 +1,4 @@ const Redux = require(`redux`) -const Promise = require(`bluebird`) const _ = require(`lodash`) const fs = require(`fs`) const mitt = require(`mitt`) @@ -119,89 +118,3 @@ exports.emitter = emitter /** Redux store */ exports.store = store - -/** - * Get all nodes from redux store. - * - * @returns {Array} - */ -exports.getNodes = () => { - const nodes = store.getState().nodes - if (nodes) { - return Array.from(nodes.values()) - } else { - return [] - } -} -const getNode = id => store.getState().nodes.get(id) - -/** Get node by id from store. - * - * @param {string} id - * @returns {Object} - */ -exports.getNode = getNode - -/** - * Determine if node has changed. - * - * @param {string} id - * @param {string} digest - * @returns {boolean} - */ -exports.hasNodeChanged = (id, digest) => { - const node = store.getState().nodes.get(id) - if (!node) { - return true - } else { - return node.internal.contentDigest !== digest - } -} - -/** - * Get content for a node from the plugin that created it. - * - * @param {Object} node - * @returns {promise} - */ -exports.loadNodeContent = node => { - if (_.isString(node.internal.content)) { - return Promise.resolve(node.internal.content) - } else { - return new Promise(resolve => { - // Load plugin's loader function - const plugin = store - .getState() - .flattenedPlugins.find(plug => plug.name === node.internal.owner) - const { loadNodeContent } = require(plugin.resolve) - if (!loadNodeContent) { - throw new Error( - `Could not find function loadNodeContent for plugin ${plugin.name}` - ) - } - - return loadNodeContent(node).then(content => { - // TODO update node's content field here. - resolve(content) - }) - }) - } -} - -/** - * Get node and save path dependency. - * - * @param {string} id - * @param {string} path - * @returns {Object} node - */ -exports.getNodeAndSavePathDependency = (id, path) => { - const { createPageDependency } = require(`./actions/add-page-dependency`) - const node = getNode(id) - createPageDependency({ path, nodeId: id }) - return node -} - -// Start plugin runner which listens to the store -// and invokes Gatsby API based on actions. -require(`./plugin-runner`) diff --git a/packages/gatsby/src/redux/nodes.js b/packages/gatsby/src/redux/nodes.js new file mode 100644 index 0000000000000..ad182cb8d406d --- /dev/null +++ b/packages/gatsby/src/redux/nodes.js @@ -0,0 +1,91 @@ +const _ = require(`lodash`) +const Promise = require(`bluebird`) +const { store } = require(`./index`) + +/** + * Get all nodes from redux store. + * + * @returns {Array} + */ +const getNodes = () => { + const nodes = store.getState().nodes + if (nodes) { + return Array.from(nodes.values()) + } else { + return [] + } +} + +exports.getNodes = getNodes + +const getNode = id => store.getState().nodes.get(id) + +/** Get node by id from store. + * + * @param {string} id + * @returns {Object} + */ +exports.getNode = getNode + +exports.getNodesByType = type => + getNodes().filter(node => node.internal.type === type) + +/** + * Determine if node has changed. + * + * @param {string} id + * @param {string} digest + * @returns {boolean} + */ +exports.hasNodeChanged = (id, digest) => { + const node = store.getState().nodes.get(id) + if (!node) { + return true + } else { + return node.internal.contentDigest !== digest + } +} + +/** + * Get content for a node from the plugin that created it. + * + * @param {Object} node + * @returns {promise} + */ +exports.loadNodeContent = node => { + if (_.isString(node.internal.content)) { + return Promise.resolve(node.internal.content) + } else { + return new Promise(resolve => { + // Load plugin's loader function + const plugin = store + .getState() + .flattenedPlugins.find(plug => plug.name === node.internal.owner) + const { loadNodeContent } = require(plugin.resolve) + if (!loadNodeContent) { + throw new Error( + `Could not find function loadNodeContent for plugin ${plugin.name}` + ) + } + + return loadNodeContent(node).then(content => { + // TODO update node's content field here. + resolve(content) + }) + }) + } +} + +/** + * Get node and save path dependency. + * + * @param {string} id + * @param {string} path + * @returns {Object} node + */ +exports.getNodeAndSavePathDependency = (id, path) => { + const { createPageDependency } = require(`./actions/add-page-dependency`) + const node = getNode(id) + createPageDependency({ path, nodeId: id }) + return node +} diff --git a/packages/gatsby/src/redux/plugin-runner.js b/packages/gatsby/src/redux/plugin-runner.js index a12a197a8995a..fb19f43cf3448 100644 --- a/packages/gatsby/src/redux/plugin-runner.js +++ b/packages/gatsby/src/redux/plugin-runner.js @@ -1,10 +1,11 @@ // Invoke plugins for certain actions. -const { store, emitter } = require(`./index`) +const { emitter } = require(`./index`) +const { getNode } = require(`../db/nodes`) const apiRunnerNode = require(`../utils/api-runner-node`) emitter.on(`CREATE_NODE`, action => { - const node = store.getState().nodes.get(action.payload.id) + const node = getNode(action.payload.id) const traceTags = { nodeId: node.id, nodeType: node.internal.type } apiRunnerNode(`onCreateNode`, { node, diff --git a/packages/gatsby/src/schema/__tests__/node-tracking-test.js b/packages/gatsby/src/schema/__tests__/node-tracking-test.js index 9df62557ad2a6..480b7d622e9be 100644 --- a/packages/gatsby/src/schema/__tests__/node-tracking-test.js +++ b/packages/gatsby/src/schema/__tests__/node-tracking-test.js @@ -40,7 +40,7 @@ describe(`Track root nodes`, () => { ` require(`fs`).__setMockFiles(MOCK_FILE_INFO) - const { getNode, getNodes } = require(`../../redux`) + const { getNode, getNodes } = require(`../../db/nodes`) const { findRootNodeAncestor } = require(`../node-tracking`) const runSift = require(`../run-sift`) const buildNodeTypes = require(`../build-node-types`) diff --git a/packages/gatsby/src/schema/__tests__/run-sift.js b/packages/gatsby/src/schema/__tests__/run-sift.js index 4e6050b64b32a..a253412718bb8 100644 --- a/packages/gatsby/src/schema/__tests__/run-sift.js +++ b/packages/gatsby/src/schema/__tests__/run-sift.js @@ -34,7 +34,7 @@ const mockNodes = [ }, ] -jest.mock(`../../redux`, () => { +jest.mock(`../../db/nodes`, () => { return { getNode: id => mockNodes.find(node => node.id === id), } diff --git a/packages/gatsby/src/schema/build-node-connections.js b/packages/gatsby/src/schema/build-node-connections.js index 0ec2825e2bd3f..8b90fa3d59bd8 100644 --- a/packages/gatsby/src/schema/build-node-connections.js +++ b/packages/gatsby/src/schema/build-node-connections.js @@ -10,7 +10,7 @@ const { } = require(`./infer-graphql-input-fields-from-fields`) const createSortField = require(`./create-sort-field`) const buildConnectionFields = require(`./build-connection-fields`) -const { getNodes } = require(`../redux`) +const { getNodesByType } = require(`../db/nodes`) module.exports = (types: any) => { const connections = {} @@ -68,10 +68,7 @@ module.exports = (types: any) => { path = rootValue.path } const runSift = require(`./run-sift`) - const latestNodes = _.filter( - getNodes(), - n => n.internal.type === type.name - ) + const latestNodes = getNodesByType(type.name) return runSift({ args: resolveArgs, nodes: latestNodes, diff --git a/packages/gatsby/src/schema/build-node-types.js b/packages/gatsby/src/schema/build-node-types.js index 7b2e8874b602b..eb92a6915cd67 100644 --- a/packages/gatsby/src/schema/build-node-types.js +++ b/packages/gatsby/src/schema/build-node-types.js @@ -17,7 +17,12 @@ const { inferInputObjectStructureFromNodes, } = require(`./infer-graphql-input-fields`) const { nodeInterface } = require(`./node-interface`) -const { getNodes, getNode, getNodeAndSavePathDependency } = require(`../redux`) +const { + getNodes, + getNodesByType, + getNode, + getNodeAndSavePathDependency, +} = require(`../db/nodes`) const { createPageDependency } = require(`../redux/actions/add-page-dependency`) const { setFileNodeRootType } = require(`./types/type-file`) const { clearTypeExampleValues } = require(`./data-tree-utils`) @@ -196,10 +201,7 @@ module.exports = async ({ parentSpan }) => { ) { latestNodes = nodesCache.get(typeName) } else { - latestNodes = _.filter( - getNodes(), - n => n.internal.type === typeName - ) + latestNodes = getNodesByType(typeName) nodesCache.set(typeName, latestNodes) } if (!_.isObject(args)) { diff --git a/packages/gatsby/src/schema/infer-graphql-input-fields.js b/packages/gatsby/src/schema/infer-graphql-input-fields.js index 1fc0184d40cea..a7c4c9fb89725 100644 --- a/packages/gatsby/src/schema/infer-graphql-input-fields.js +++ b/packages/gatsby/src/schema/infer-graphql-input-fields.js @@ -21,7 +21,7 @@ const { } = require(`./data-tree-utils`) const { findLinkedNode } = require(`./infer-graphql-type`) -const { getNodes } = require(`../redux`) +const { getNodesByType } = require(`../db/nodes`) const is32BitInteger = require(`../utils/is-32-bit-integer`) import type { @@ -277,9 +277,7 @@ export function inferInputObjectStructureFromNodes({ if (linkedNodeCache[linkedNode.internal.type]) { value = linkedNodeCache[linkedNode.internal.type] } else { - const relatedNodes = getNodes().filter( - node => node.internal.type === linkedNode.internal.type - ) + const relatedNodes = getNodesByType(linkedNode.internal.type) value = getExampleValues({ nodes: relatedNodes, typeName: linkedNode.internal.type, diff --git a/packages/gatsby/src/schema/infer-graphql-type.js b/packages/gatsby/src/schema/infer-graphql-type.js index 87e30304c66c9..955df13c4db26 100644 --- a/packages/gatsby/src/schema/infer-graphql-type.js +++ b/packages/gatsby/src/schema/infer-graphql-type.js @@ -12,7 +12,8 @@ const _ = require(`lodash`) const invariant = require(`invariant`) const { oneLine } = require(`common-tags`) -const { store, getNode, getNodes } = require(`../redux`) +const { store } = require(`../redux`) +const { getNode, getNodes, getNodesByType } = require(`../db/nodes`) const { createPageDependency } = require(`../redux/actions/add-page-dependency`) const createTypeName = require(`./create-type-name`) const createKey = require(`./create-key`) @@ -154,9 +155,8 @@ function inferFromMapping( const findNode = (fieldValue, path) => { const linkedNode = _.find( - getNodes(), - n => - n.internal.type === linkedType && _.get(n, linkedField) === fieldValue + getNodesByType(linkedType), + n => _.get(n, linkedField) === fieldValue ) if (linkedNode) { createPageDependency({ path, nodeId: linkedNode.id }) @@ -257,7 +257,10 @@ function inferFromFieldName(value, selector, types): GraphQLFieldConfig<*, *> { let type // If there's more than one type, we'll create a union type. if (fields.length > 1) { - const typeName = `Union_${key}_${fields.map(f => f.name).sort().join(`__`)}` + const typeName = `Union_${key}_${fields + .map(f => f.name) + .sort() + .join(`__`)}` if (unionTypes.has(typeName)) { type = unionTypes.get(typeName) diff --git a/packages/gatsby/src/schema/node-tracking.js b/packages/gatsby/src/schema/node-tracking.js index 495e720d44bab..8d9905cbbfe5c 100644 --- a/packages/gatsby/src/schema/node-tracking.js +++ b/packages/gatsby/src/schema/node-tracking.js @@ -1,5 +1,5 @@ const _ = require(`lodash`) -const { getNode, getNodes } = require(`../redux`) +const { getNode, getNodes } = require(`../db/nodes`) /** * Map containing links between inline objects or arrays diff --git a/packages/gatsby/src/schema/run-sift.js b/packages/gatsby/src/schema/run-sift.js index 76de76e85b73e..2206751b34041 100644 --- a/packages/gatsby/src/schema/run-sift.js +++ b/packages/gatsby/src/schema/run-sift.js @@ -6,7 +6,7 @@ const { createPageDependency } = require(`../redux/actions/add-page-dependency`) const prepareRegex = require(`./prepare-regex`) const Promise = require(`bluebird`) const { trackInlineObjectsInRootNode } = require(`./node-tracking`) -const { getNode } = require(`../redux`) +const { getNode } = require(`../db/nodes`) const resolvedNodesCache = new Map() const enhancedNodeCache = new Map() diff --git a/packages/gatsby/src/schema/types/type-file.js b/packages/gatsby/src/schema/types/type-file.js index 301db11b6fce5..fa40ac1abadfa 100644 --- a/packages/gatsby/src/schema/types/type-file.js +++ b/packages/gatsby/src/schema/types/type-file.js @@ -6,7 +6,7 @@ const isRelativeUrl = require(`is-relative-url`) const normalize = require(`normalize-path`) const systemPath = require(`path`) -const { getNodes } = require(`../../redux`) +const { getNodesByType } = require(`../../db/nodes`) const { findRootNodeAncestor } = require(`../node-tracking`) const { createPageDependency, @@ -108,7 +108,7 @@ function pointsToFile(nodes, key, value) { } const pathToOtherNode = normalize(joinPath(rootNode.dir, value)) - const otherFileExists = getNodes().some( + const otherFileExists = getNodesByType(`File`).some( n => n.absolutePath === pathToOtherNode ) return otherFileExists @@ -148,8 +148,8 @@ function createType(fileNodeRootType, isArray) { // Use that path to find the linked File node. const linkedFileNode = _.find( - getNodes(), - n => n.internal.type === `File` && n.absolutePath === fileLinkPath + getNodesByType(`File`), + n => n.absolutePath === fileLinkPath ) if (linkedFileNode) { createPageDependency({ diff --git a/packages/gatsby/src/utils/api-runner-node.js b/packages/gatsby/src/utils/api-runner-node.js index 906b3393d1025..bfc2fa7574e4e 100644 --- a/packages/gatsby/src/utils/api-runner-node.js +++ b/packages/gatsby/src/utils/api-runner-node.js @@ -68,15 +68,15 @@ const runAPI = (plugin, api, args) => { pluginSpan.setTag(`plugin`, plugin.name) let pathPrefix = `` + const { store, emitter } = require(`../redux`) const { - store, - emitter, loadNodeContent, getNodes, getNode, + getNodesByType, hasNodeChanged, getNodeAndSavePathDependency, - } = require(`../redux`) + } = require(`../db/nodes`) const { boundActionCreators } = require(`../redux/actions`) const doubleBoundActionCreators = doubleBind( @@ -111,6 +111,7 @@ const runAPI = (plugin, api, args) => { emitter, getNodes, getNode, + getNodesByType, hasNodeChanged, reporter, getNodeAndSavePathDependency, diff --git a/packages/gatsby/src/utils/source-nodes.js b/packages/gatsby/src/utils/source-nodes.js index a90fb7e132fb9..ec544feba284b 100644 --- a/packages/gatsby/src/utils/source-nodes.js +++ b/packages/gatsby/src/utils/source-nodes.js @@ -2,7 +2,8 @@ const _ = require(`lodash`) const report = require(`gatsby-cli/lib/reporter`) const apiRunner = require(`./api-runner-node`) -const { store, getNode } = require(`../redux`) +const { store } = require(`../redux`) +const { getNode, getNodes } = require(`../db/nodes`) const { boundActionCreators } = require(`../redux/actions`) const { deleteNode } = boundActionCreators @@ -18,7 +19,7 @@ function discoverPluginsWithoutNodes(storeState) { ) // Find out which plugins own already created nodes const nodeOwners = _.uniq( - Array.from(storeState.nodes.values()).reduce((acc, node) => { + Array.from(getNodes()).reduce((acc, node) => { acc.push(node.internal.owner) return acc }, []) @@ -45,7 +46,7 @@ module.exports = async ({ parentSpan } = {}) => { // Garbage collect stale data nodes const touchedNodes = Object.keys(state.nodesTouched) - const staleNodes = Array.from(state.nodes.values()).filter(node => { + const staleNodes = Array.from(getNodes()).filter(node => { // Find the root node. let rootNode = node let whileCount = 0