Feat(source-filesystem): No file watchers on build

packages/gatsby-source-filesystem/package.json

@@ -18,6 +18,7 @@
     "mime": "^2.2.0",
     "pretty-bytes": "^4.0.2",
     "read-chunk": "^3.0.0",
+    "readdirp": "^2.2.1",
     "slash": "^1.0.0",
     "valid-url": "^1.0.9",
     "xstate": "^3.1.0"

packages/gatsby-source-filesystem/src/__tests__/find-files.js

@@ -0,0 +1,35 @@
+const findFiles = require(`../find-files`)
+const path = require(`path`)
+
+describe(`findOnce`, () => {
+  it(`resolves an array of file paths`, async done => {
+    expect.assertions(3)
+    const queue = await findFiles(`${__dirname}/fixtures/test-fs`, {
+      watch: false,
+      ignore: [
+        `.toBeIgnored.md`,
+        `**/.placeholder.md`,
+        `**/notInTheFixture.bar`,
+      ],
+    })
+    const expectedQueue = [
+      path.join(__dirname, `fixtures`, `test-fs`, `index.md`),
+      path.join(__dirname, `fixtures`, `test-fs`, `dirA`, `index.md`),
+      path.join(__dirname, `fixtures`, `test-fs`, `dirB`, `index.md`),
+      path.join(__dirname, `fixtures`, `test-fs`, `dirA`, `a`, `index.md`),
+      path.join(__dirname, `fixtures`, `test-fs`, `dirA`, `a`, `A`, `index.md`),
+      path.join(__dirname, `fixtures`, `test-fs`, `dirA`, `b`, `index.md`),
+      path.join(__dirname, `fixtures`, `test-fs`, `dirA`, `b`, `A`, `index.md`),
+      path.join(__dirname, `fixtures`, `test-fs`, `dirA`, `c`, `index.md`),
+      path.join(__dirname, `fixtures`, `test-fs`, `dirA`, `c`, `A`, `index.md`),
+    ]
+
+    expect(queue.length).toBeGreaterThan(0)
+    expect(queue.length).toEqual(expectedQueue.length)
+    const hasAllFiles = expectedQueue.every(expected =>
+      queue.some(item => item === expected)
+    )
+    expect(hasAllFiles).toBe(true)
+    done()
+  })
+})

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/.toBeIgnored.md

@@ -0,0 +1 @@
+should never be queued

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/a/A/index.md

@@ -0,0 +1 @@
+"DirA a A" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/a/index.md

@@ -0,0 +1 @@
+"DirA a" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/b/A/index.md

@@ -0,0 +1 @@
+"DirA a A" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/b/index.md

@@ -0,0 +1 @@
+"DirA b" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/c/A/index.md

@@ -0,0 +1 @@
+"DirA a A" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/c/index.md

@@ -0,0 +1 @@
+"DirA c" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/index.md

@@ -0,0 +1 @@
+"DirA" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirB/index.md

@@ -0,0 +1 @@
+"DirB" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/empty/.placeholder.md

@@ -0,0 +1 @@
+This file is only so we can commit an empty folder. It should be ignored during testing.

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/index.md

@@ -0,0 +1 @@
+root dir content

packages/gatsby-source-filesystem/src/file-watcher.js

@@ -0,0 +1,159 @@
+const path = require(`path`)
+const { Machine } = require(`xstate`)
+const findFiles = require(`./find-files`)
+const { createFileNode } = require(`./create-file-node`)
+
+/**
+ * Create a state machine to manage Chokidar's not-ready/ready states.
+ */
+const createFSMachine = () =>
+  Machine({
+    key: `emitFSEvents`,
+    parallel: true,
+    strict: true,
+    states: {
+      CHOKIDAR: {
+        initial: `CHOKIDAR_NOT_READY`,
+        states: {
+          CHOKIDAR_NOT_READY: {
+            on: {
+              CHOKIDAR_READY: `CHOKIDAR_WATCHING`,
+              BOOTSTRAP_FINISHED: `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`,
+            },
+          },
+          CHOKIDAR_WATCHING: {
+            on: {
+              BOOTSTRAP_FINISHED: `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`,
+              CHOKIDAR_READY: `CHOKIDAR_WATCHING`,
+            },
+          },
+          CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED: {
+            on: {
+              CHOKIDAR_READY: `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`,
+            },
+          },
+        },
+      },
+    },
+  })
+
+function fileWatcher(
+  { actions, getNode, createNodeId, reporter, emitter },
+  pluginOptions
+) {
+  const { createNode, deleteNode } = actions
+  const fsMachine = createFSMachine()
+  let currentState = fsMachine.initialState
+
+  // Validate that the path is absolute.
+  // Absolute paths are required to resolve images correctly.
+  if (!path.isAbsolute(pluginOptions.path)) {
+    pluginOptions.path = path.resolve(process.cwd(), pluginOptions.path)
+  }
+
+  // Once bootstrap is finished, we only let one File node update go through
+  // the system at a time.
+  emitter.on(`BOOTSTRAP_FINISHED`, () => {
+    currentState = fsMachine.transition(
+      currentState.value,
+      `BOOTSTRAP_FINISHED`
+    )
+  })
+
+  const watcher = findFiles(pluginOptions.path, {
+    watch: process.env.GATSBY_EXECUTING_COMMAND === `develop`,
+    ignore: pluginOptions.ignore,
+  })
+
+  const createAndProcessNode = path => {
+    const fileNodePromise = createFileNode(
+      path,
+      createNodeId,
+      pluginOptions
+    ).then(fileNode => {
+      createNode(fileNode)
+      return null
+    })
+    return fileNodePromise
+  }
+
+  // For every path that is reported before the 'ready' event, we throw them
+  // into a queue and then flush the queue when 'ready' event arrives.
+  // After 'ready', we handle the 'add' event without putting it into a queue.
+  let pathQueue = []
+  const flushPathQueue = () => {
+    let queue = pathQueue.slice()
+    pathQueue = []
+    return Promise.all(queue.map(createAndProcessNode))
+  }
+
+  watcher.on(`add`, path => {
+    if (currentState.value.CHOKIDAR !== `CHOKIDAR_NOT_READY`) {
+      if (
+        currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+      ) {
+        reporter.info(`added file at ${path}`)
+      }
+      createAndProcessNode(path).catch(err => reporter.error(err))
+    } else {
+      pathQueue.push(path)
+    }
+  })
+
+  watcher.on(`change`, path => {
+    if (
+      currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+    ) {
+      reporter.info(`changed file at ${path}`)
+    }
+    createAndProcessNode(path).catch(err => reporter.error(err))
+  })
+
+  watcher.on(`unlink`, path => {
+    if (
+      currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+    ) {
+      reporter.info(`file deleted at ${path}`)
+    }
+    const node = getNode(createNodeId(path))
+    // It's possible the file node was never created as sometimes tools will
+    // write and then immediately delete temporary files to the file system.
+    if (node) {
+      deleteNode({ node })
+    }
+  })
+
+  watcher.on(`addDir`, path => {
+    if (currentState.value.CHOKIDAR !== `CHOKIDAR_NOT_READY`) {
+      if (
+        currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+      ) {
+        reporter.info(`added directory at ${path}`)
+      }
+      createAndProcessNode(path).catch(err => reporter.error(err))
+    } else {
+      pathQueue.push(path)
+    }
+  })
+
+  watcher.on(`unlinkDir`, path => {
+    if (
+      currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+    ) {
+      reporter.info(`directory deleted at ${path}`)
+    }
+    const node = getNode(createNodeId(path))
+    if (node) {
+      deleteNode({ node })
+    }
+  })
+
+  return new Promise((resolve, reject) => {
+    watcher.on(`ready`, () => {
+      currentState = fsMachine.transition(currentState.value, `CHOKIDAR_READY`)
+      flushPathQueue().then(resolve, reject)
+    })
+  })
+}
+
+module.exports = fileWatcher

packages/gatsby-source-filesystem/src/find-files.js

@@ -0,0 +1,44 @@
+const chokidar = require(`chokidar`)
+const readdirp = require(`readdirp`)
+const anymatch = require(`anymatch`)
+
+const findOnce = (path, options) => {
+  const { ignored } = options
+  return new Promise(resolve => {
+    let fileList = []
+    const stream = readdirp({ root: path })
+
+    stream.on(`data`, data => {
+      if (anymatch(ignored, data.path)) return
+      fileList.push(data.fullPath)
+      return
+    })
+
+    stream.on(`end`, () => {
+      resolve(fileList)
+    })
+  })
+}
+
+module.exports = (path, options) => {
+  const { watch, ignore } = options
+
+  const ignored = [
+    `**/*.un~`,
+    `**/.DS_Store`,
+    `**/.gitignore`,
+    `**/.npmignore`,
+    `**/.babelrc`,
+    `**/yarn.lock`,
+    `**/bower_components`,
+    `**/node_modules`,
+    `../**/dist/**`,
+    ...(ignore || []),
+  ]
+
+  if (watch === true) {
+    return chokidar.watch(path, { ignored })
+  } else {
+    return findOnce(path, { ignored })
+  }
+}