feat(gatsby): add lokijs nodes db implementation (#9919)

This PR adds a feature flag `GATSBY_DB_NODES` that can be used to change the storage engine for the gatsby data layer (`nodes`). 

- `redux` (default) which uses the existing redux implementation, and `sift` for querying. Or, you can use 
- `loki` which uses the loki in-memory database to store and query.

This PR re-implements functionality in #9338, but with all tests passing and addressing previous feedback. It should also be easier to review since it builds on several refactorings. 

Some things to note:

1. I [submitted a PR to lokijs](techfort/LokiJS#718) which still hasn't been merged, though the author says he'll start working on it soon. Therefore, in in interim, I've published [@moocar/lokijs](https://www.npmjs.com/package/@moocar/lokijs). 
1. I haven't implemented auto indexing of query fields yet. I'll attack that next.
1. I suggest we ask the community to try out the feature flag once it's merged to get feedback. All the tests pass, but this is such a big change that we'll want to test it gradually
1. While loki uses the same mongo query language as sift, they do have different behavior. Most of my time on this PR was spent ensuring that loki behaves **exactly** like sift. See [db/loki/nodes-query.js](https://github.com/gatsbyjs/gatsby/blob/cddbe893a4ce638babb1cbe5e5da4c13b6f5e57d/packages/gatsby/src/db/loki/nodes-query.js). But there's a chance a few edge cases have slipped through the cracks.
1. the feature flag works with the tests too `GATSBY_DB_NODES=loki yarn test`. We should perhaps look into running this on all PRs

packages/gatsby/package.json

@@ -16,6 +16,7 @@
     "@babel/polyfill": "^7.0.0",
     "@babel/runtime": "^7.0.0",
     "@babel/traverse": "^7.0.0",
+    "@moocar/lokijs": "^1.0.1",
     "@reach/router": "^1.1.1",
     "autoprefixer": "^8.6.5",
     "babel-core": "7.0.0-bridge.0",

packages/gatsby/src/bootstrap/index.js

@@ -20,6 +20,7 @@ const report = require(`gatsby-cli/lib/reporter`)
 const getConfigFile = require(`./get-config-file`)
 const tracer = require(`opentracing`).globalTracer()
 const preferDefault = require(`./prefer-default`)
+const nodeTracking = require(`../db/node-tracking`)
 
 // Show stack trace on unhandled promises.
 process.on(`unhandledRejection`, (reason, p) => {
@@ -221,6 +222,32 @@ module.exports = async (args: BootstrapArgs) => {
 
   activity.end()
 
+  if (process.env.GATSBY_DB_NODES === `loki`) {
+    const loki = require(`../db/loki`)
+    // Start the nodes database (in memory loki js with interval disk
+    // saves). If data was saved from a previous build, it will be
+    // loaded here
+    activity = report.activityTimer(`start nodes db`, {
+      parentSpan: bootstrapSpan,
+    })
+    activity.start()
+    const dbSaveFile = `${program.directory}/.cache/loki/loki.db`
+    try {
+      await loki.start({
+        saveFile: dbSaveFile,
+      })
+    } catch (e) {
+      report.error(
+        `Error starting DB. Perhaps try deleting ${path.dirname(dbSaveFile)}`
+      )
+    }
+    activity.end()
+  }
+
+  // By now, our nodes database has been loaded, so ensure that we
+  // have tracked all inline objects
+  nodeTracking.trackDbNodes()
+
   // Copy our site files to the root of the site.
   activity = report.activityTimer(`copy gatsby files`, {
     parentSpan: bootstrapSpan,

packages/gatsby/src/db/__tests__/fixtures/ensure-loki.js

@@ -0,0 +1,8 @@
+const { backend } = require(`../../nodes`)
+
+module.exports = () => {
+  if (backend === `loki`) {
+    const lokiDb = require(`../../loki`)
+    beforeAll(lokiDb.start)
+  }
+}

packages/gatsby/src/db/__tests__/node-tracking-test.js

@@ -1,72 +1,57 @@
-const { readFile, writeFile } = require(`fs-extra`)
-
-jest.mock(`fs`)
-jest.mock(`fs-extra`, () => {
+const { store } = require(`../../redux`)
+const {
+  boundActionCreators: { createNode },
+} = require(`../../redux/actions`)
+const { getNode } = require(`../../db/nodes`)
+const { findRootNodeAncestor, trackDbNodes } = require(`../node-tracking`)
+const nodeTypes = require(`../../schema/build-node-types`)
+const { run: runQuery } = require(`../nodes-query`)
+require(`./fixtures/ensure-loki`)()
+
+function makeNode() {
   return {
-    readFile: jest.fn(() => `contents`),
-    writeFile: jest.fn(),
-  }
-})
-
-afterEach(() => {
-  readFile.mockClear()
-  writeFile.mockClear()
-})
-
-describe(`Track root nodes`, () => {
-  const reduxStatePath = `${process.cwd()}/.cache/redux-state.json`
-  const MOCK_FILE_INFO = {}
-  MOCK_FILE_INFO[reduxStatePath] = `
-  {
-      "nodes": {
-          "id1": {
-            "id": "id1",
-            "parent": null,
-            "children": [],
-            "inlineObject": {
-              "field": "fieldOfFirstNode"
-            },
-            "inlineArray": [
-              1, 2, 3
-            ],
-            "internal": {
-              "type": "TestNode",
-              "contentDigest": "digest1",
-              "owner": "test"
-            }
-          }
-      }
-  }
-  `
-  require(`fs`).__setMockFiles(MOCK_FILE_INFO)
-
-  const { getNode } = require(`../../db/nodes`)
-  const { findRootNodeAncestor } = require(`../node-tracking`)
-  const { runQuery } = require(`../../db/nodes`)
-  const buildNodeTypes = require(`../../schema/build-node-types`)
-  const {
-    boundActionCreators: { createNode },
-  } = require(`../../redux/actions`)
-
-  createNode(
-    {
-      id: `id2`,
-      parent: null,
-      children: [],
-      inlineObject: {
-        field: `fieldOfSecondNode`,
-      },
-      inlineArray: [1, 2, 3],
-      internal: {
-        type: `TestNode`,
-        contentDigest: `digest2`,
-      },
+    id: `id1`,
+    parent: null,
+    children: [],
+    inlineObject: {
+      field: `fieldOfFirstNode`,
+    },
+    inlineArray: [1, 2, 3],
+    internal: {
+      type: `TestNode`,
+      contentDigest: `digest1`,
+      owner: `test`,
     },
-    {
-      name: `test`,
+  }
+}
+
+describe(`track root nodes`, () => {
+  beforeEach(() => {
+    const nodes = [makeNode()]
+    store.dispatch({ type: `DELETE_CACHE` })
+    for (const node of nodes) {
+      store.dispatch({ type: `CREATE_NODE`, payload: node })
     }
-  )
-
+    trackDbNodes()
+    createNode(
+      {
+        id: `id2`,
+        parent: null,
+        children: [],
+        inlineObject: {
+          field: `fieldOfSecondNode`,
+        },
+        inlineArray: [1, 2, 3],
+        internal: {
+          type: `TestNode`,
+          contentDigest: `digest2`,
+        },
+      },
+      {
+        name: `test`,
+      }
+    )
+  })
   describe(`Tracks nodes read from redux state cache`, () => {
     it(`Tracks inline objects`, () => {
       const node = getNode(`id1`)
@@ -75,15 +60,13 @@ describe(`Track root nodes`, () => {
 
       expect(trackedRootNode).toEqual(node)
     })
-
     it(`Tracks inline arrays`, () => {
       const node = getNode(`id1`)
       const inlineObject = node.inlineArray
       const trackedRootNode = findRootNodeAncestor(inlineObject)
 
       expect(trackedRootNode).toEqual(node)
     })
-
     it(`Doesn't track copied objects`, () => {
       const node = getNode(`id1`)
       const copiedInlineObject = { ...node.inlineObject }
@@ -92,7 +75,6 @@ describe(`Track root nodes`, () => {
       expect(trackedRootNode).not.toEqual(node)
     })
   })
-
   describe(`Tracks nodes created using createNode action`, () => {
     it(`Tracks inline objects`, () => {
       const node = getNode(`id2`)
@@ -103,11 +85,11 @@ describe(`Track root nodes`, () => {
     })
   })
 
-  describe(`Tracks nodes returned by running sift`, () => {
+  describe(`Tracks nodes returned by queries`, () => {
     let type
 
     beforeAll(async () => {
-      type = (await buildNodeTypes({})).testNode.nodeObjectType
+      type = (await nodeTypes.buildAll({})).testNode.nodeObjectType
     })
 
     it(`Tracks objects when running query without filter`, async () => {