Add fileFinder for build sourcing (no file watchers)

packages/gatsby-source-filesystem/package.json

@@ -16,6 +16,7 @@
     "md5-file": "^3.1.1",
     "mime": "^2.2.0",
     "pretty-bytes": "^4.0.2",
+    "readdirp-walk": "^1.6.0",
     "slash": "^1.0.0",
     "valid-url": "^1.0.9",
     "xstate": "^3.1.0"

packages/gatsby-source-filesystem/src/__tests__/fileFinder.js

@@ -0,0 +1,27 @@
+const fileFinder = require(`../file-finder`)
+
+describe(`file-finder`, () => {
+  it(`resolves an array of file paths`, async done => {
+    expect.assertions(3)
+    const queue = await fileFinder(`${__dirname}/fixtures/test-fs`)
+    const expectedQueue = [
+      `${__dirname}/fixtures/test-fs/index.md`,
+      `${__dirname}/fixtures/test-fs/dirA/index.md`,
+      `${__dirname}/fixtures/test-fs/dirB/index.md`,
+      `${__dirname}/fixtures/test-fs/dirA/a/index.md`,
+      `${__dirname}/fixtures/test-fs/dirA/a/A/index.md`,
+      `${__dirname}/fixtures/test-fs/dirA/b/index.md`,
+      `${__dirname}/fixtures/test-fs/dirA/b/A/index.md`,
+      `${__dirname}/fixtures/test-fs/dirA/c/index.md`,
+      `${__dirname}/fixtures/test-fs/dirA/c/A/index.md`,
+    ]
+
+    expect(queue.length).toBeGreaterThan(0)
+    expect(queue.length).toEqual(expectedQueue.length)
+    const hasAllFiles = expectedQueue.every(expected =>
+      queue.some(item => item === expected)
+    )
+    expect(hasAllFiles).toBe(true)
+    done()
+  })
+})

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/.toBeIgnored.md

@@ -0,0 +1 @@
+should never be queued

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/a/A/index.md

@@ -0,0 +1 @@
+"DirA a A" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/a/index.md

@@ -0,0 +1 @@
+"DirA a" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/b/A/index.md

@@ -0,0 +1 @@
+"DirA a A" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/b/index.md

@@ -0,0 +1 @@
+"DirA b" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/c/A/index.md

@@ -0,0 +1 @@
+"DirA a A" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/c/index.md

@@ -0,0 +1 @@
+"DirA c" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/index.md

@@ -0,0 +1 @@
+"DirA" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirB/index.md

@@ -0,0 +1 @@
+"DirB" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/empty/.placeholder.md

@@ -0,0 +1 @@
+This file is only so we can commit an empty folder. It should be ignored during testing.

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/index.md

@@ -0,0 +1 @@
+root dir content

packages/gatsby-source-filesystem/src/file-finder.js

@@ -0,0 +1,34 @@
+const readdirp = require(`readdirp-walk`)
+
+const ignoredRE = /^\./
+const ignored = [`yarn.lock`, `package-lock.json`, `node_modules`, `dist`]
+
+function fileFinder(dir) {
+  return new Promise(resolve => {
+    let fileList = []
+    const stream = readdirp({ root: dir })
+
+    stream.on(`data`, data => {
+      const { name, fullPath, stat } = data
+
+      if (
+        stat.isDirectory() ||
+        ignoredRE.test(name) ||
+        ignored.includes(name)
+      ) {
+        return
+      }
+      if (fullPath.includes(`node_modules`) || fullPath.includes(`build`)) {
+        return
+      }
+      fileList.push(fullPath)
+      return
+    })
+
+    stream.on(`end`, () => {
+      resolve(fileList)
+    })
+  })
+}
+
+module.exports = fileFinder

packages/gatsby-source-filesystem/src/file-watcher.js

@@ -0,0 +1,212 @@
+const path = require(`path`)
+const chokidar = require(`chokidar`)
+const { Machine } = require(`xstate`)
+
+const { createFileNode } = require(`./create-file-node`)
+/**
+ * Create a state machine to manage Chokidar's not-ready/ready states and for
+ * emitting file system events into Gatsby.
+ *
+ * On the latter, this solves the problem where if you call createNode for the
+ * same File node in quick succession, this can leave Gatsby's internal state
+ * in disarray causing queries to fail. The latter state machine tracks when
+ * Gatsby is "processing" a node update or when it's "idle". If updates come in
+ * while Gatsby is processing, we queue them until the system returns to an
+ * "idle" state.
+ */
+const createFSMachine = () =>
+  Machine({
+    key: `emitFSEvents`,
+    parallel: true,
+    strict: true,
+    states: {
+      CHOKIDAR: {
+        initial: `CHOKIDAR_NOT_READY`,
+        states: {
+          CHOKIDAR_NOT_READY: {
+            on: {
+              CHOKIDAR_READY: `CHOKIDAR_WATCHING`,
+              BOOTSTRAP_FINISHED: `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`,
+            },
+          },
+          CHOKIDAR_WATCHING: {
+            on: {
+              BOOTSTRAP_FINISHED: `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`,
+              CHOKIDAR_READY: `CHOKIDAR_WATCHING`,
+            },
+          },
+          CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED: {
+            on: {
+              CHOKIDAR_READY: `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`,
+            },
+          },
+        },
+      },
+      PROCESSING: {
+        initial: `BOOTSTRAPPING`,
+        states: {
+          BOOTSTRAPPING: {
+            on: {
+              BOOTSTRAP_FINISHED: `IDLE`,
+            },
+          },
+          IDLE: {
+            on: {
+              EMIT_FS_EVENT: `PROCESSING`,
+            },
+          },
+          PROCESSING: {
+            on: {
+              QUERY_QUEUE_DRAINED: `IDLE`,
+              TOUCH_NODE: `IDLE`,
+            },
+          },
+        },
+      },
+    },
+  })
+
+function fileWatcher(
+  { actions, getNode, createNodeId, reporter, emitter },
+  pluginOptions
+) {
+  const { createNode, deleteNode } = actions
+  // Validate that the path is absolute.
+  // Absolute paths are required to resolve images correctly.
+  if (!path.isAbsolute(pluginOptions.path)) {
+    pluginOptions.path = path.resolve(process.cwd(), pluginOptions.path)
+  }
+
+  const fsMachine = createFSMachine()
+  let currentState = fsMachine.initialState
+  let pathQueue = []
+  let fileNodeQueue = new Map()
+  const createAndProcessNode = path => {
+    const fileNodePromise = createFileNode(
+      path,
+      createNodeId,
+      pluginOptions
+    ).then(fileNode => {
+      if (currentState.value.PROCESSING === `PROCESSING`) {
+        fileNodeQueue.set(fileNode.id, fileNode)
+      } else {
+        currentState = fsMachine.transition(currentState.value, `EMIT_FS_EVENT`)
+        createNode(fileNode)
+      }
+    })
+    return fileNodePromise
+  }
+
+  const watcher = chokidar.watch(pluginOptions.path, {
+    ignored: [
+      `**/*.un~`,
+      `**/.gitignore`,
+      `**/.npmignore`,
+      `**/.babelrc`,
+      `**/yarn.lock`,
+      `**/node_modules`,
+      `../**/dist/**`,
+    ],
+  })
+
+  emitter.on(`BOOTSTRAP_FINISHED`, () => {
+    currentState = fsMachine.transition(
+      currentState.value,
+      `BOOTSTRAP_FINISHED`
+    )
+  })
+  emitter.on(`TOUCH_NODE`, () => {
+    // If we create a node which is the same as the previous version, createNode
+    // returns TOUCH_NODE and then nothing else happens so we listen to that
+    // to return the state back to IDLE.
+    currentState = fsMachine.transition(currentState.value, `TOUCH_NODE`)
+  })
+
+  emitter.on(`QUERY_QUEUE_DRAINED`, () => {
+    currentState = fsMachine.transition(
+      currentState.value,
+      `QUERY_QUEUE_DRAINED`
+    )
+    // If we have any updates queued, run one of them now.
+    if (fileNodeQueue.size > 0) {
+      const toProcess = fileNodeQueue.get(Array.from(fileNodeQueue.keys())[0])
+      fileNodeQueue.delete(toProcess.id)
+      currentState = fsMachine.transition(currentState.value, `EMIT_FS_EVENT`)
+      createNode(toProcess)
+    }
+  })
+  watcher.on(`add`, path => {
+    if (currentState.value.CHOKIDAR !== `CHOKIDAR_NOT_READY`) {
+      if (
+        currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+      ) {
+        reporter.info(`added file at ${path}`)
+      }
+      createAndProcessNode(path).catch(err => reporter.error(err))
+    } else {
+      pathQueue.push(path)
+    }
+  })
+
+  watcher.on(`change`, path => {
+    if (
+      currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+    ) {
+      reporter.info(`changed file at ${path}`)
+    }
+    createAndProcessNode(path).catch(err => reporter.error(err))
+  })
+
+  watcher.on(`unlink`, path => {
+    if (
+      currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+    ) {
+      reporter.info(`file deleted at ${path}`)
+    }
+    const node = getNode(createNodeId(path))
+    // It's possible the file node was never created as sometimes tools will
+    // write and then immediately delete temporary files to the file system.
+    if (node) {
+      currentState = fsMachine.transition(currentState.value, `EMIT_FS_EVENT`)
+      deleteNode({ node })
+    }
+  })
+
+  watcher.on(`addDir`, path => {
+    if (currentState.value.CHOKIDAR !== `CHOKIDAR_NOT_READY`) {
+      if (
+        currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+      ) {
+        reporter.info(`added directory at ${path}`)
+      }
+      createAndProcessNode(path).catch(err => reporter.error(err))
+    } else {
+      pathQueue.push(path)
+    }
+  })
+
+  watcher.on(`unlinkDir`, path => {
+    if (
+      currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+    ) {
+      reporter.info(`directory deleted at ${path}`)
+    }
+    const node = getNode(createNodeId(path))
+    deleteNode({ node })
+  })
+
+  const flushPathQueue = () => {
+    let queue = pathQueue.slice()
+    pathQueue = []
+    return Promise.all(queue.map(createAndProcessNode))
+  }
+
+  return new Promise((resolve, reject) => {
+    watcher.on(`ready`, () => {
+      currentState = fsMachine.transition(currentState.value, `CHOKIDAR_READY`)
+      flushPathQueue().then(resolve, reject)
+    })
+  })
+}
+
+module.exports = fileWatcher