diff --git a/packages/gatsby-source-filesystem/package.json b/packages/gatsby-source-filesystem/package.json index 222bc72d0a6c0..4572cd9ea64e1 100644 --- a/packages/gatsby-source-filesystem/package.json +++ b/packages/gatsby-source-filesystem/package.json @@ -18,6 +18,7 @@ "mime": "^2.2.0", "pretty-bytes": "^4.0.2", "read-chunk": "^3.0.0", + "readdirp": "^2.2.1", "slash": "^1.0.0", "valid-url": "^1.0.9", "xstate": "^3.1.0" diff --git a/packages/gatsby-source-filesystem/src/__tests__/find-files.js b/packages/gatsby-source-filesystem/src/__tests__/find-files.js new file mode 100644 index 0000000000000..a0b8a7edfe46c --- /dev/null +++ b/packages/gatsby-source-filesystem/src/__tests__/find-files.js @@ -0,0 +1,35 @@ +const findFiles = require(`../find-files`) +const path = require(`path`) + +describe(`findOnce`, () => { + it(`resolves an array of file paths`, async done => { + expect.assertions(3) + const queue = await findFiles(`${__dirname}/fixtures/test-fs`, { + watch: false, + ignore: [ + `.toBeIgnored.md`, + `**/.placeholder.md`, + `**/notInTheFixture.bar`, + ], + }) + const expectedQueue = [ + path.join(__dirname, `fixtures`, `test-fs`, `index.md`), + path.join(__dirname, `fixtures`, `test-fs`, `dirA`, `index.md`), + path.join(__dirname, `fixtures`, `test-fs`, `dirB`, `index.md`), + path.join(__dirname, `fixtures`, `test-fs`, `dirA`, `a`, `index.md`), + path.join(__dirname, `fixtures`, `test-fs`, `dirA`, `a`, `A`, `index.md`), + path.join(__dirname, `fixtures`, `test-fs`, `dirA`, `b`, `index.md`), + path.join(__dirname, `fixtures`, `test-fs`, `dirA`, `b`, `A`, `index.md`), + path.join(__dirname, `fixtures`, `test-fs`, `dirA`, `c`, `index.md`), + path.join(__dirname, `fixtures`, `test-fs`, `dirA`, `c`, `A`, `index.md`), + ] + + expect(queue.length).toBeGreaterThan(0) + expect(queue.length).toEqual(expectedQueue.length) + const hasAllFiles = expectedQueue.every(expected => + queue.some(item => item === expected) + ) + expect(hasAllFiles).toBe(true) + done() + }) +}) diff --git a/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/.toBeIgnored.md b/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/.toBeIgnored.md new file mode 100644 index 0000000000000..c692f9ee8378f --- /dev/null +++ b/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/.toBeIgnored.md @@ -0,0 +1 @@ +should never be queued diff --git a/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/a/A/index.md b/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/a/A/index.md new file mode 100644 index 0000000000000..10f799646661b --- /dev/null +++ b/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/a/A/index.md @@ -0,0 +1 @@ +"DirA a A" content diff --git a/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/a/index.md b/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/a/index.md new file mode 100644 index 0000000000000..20a1e322b452c --- /dev/null +++ b/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/a/index.md @@ -0,0 +1 @@ +"DirA a" content diff --git a/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/b/A/index.md b/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/b/A/index.md new file mode 100644 index 0000000000000..10f799646661b --- /dev/null +++ b/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/b/A/index.md @@ -0,0 +1 @@ +"DirA a A" content diff --git a/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/b/index.md b/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/b/index.md new file mode 100644 index 0000000000000..3d204ff4c875a --- /dev/null +++ b/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/b/index.md @@ -0,0 +1 @@ +"DirA b" content diff --git a/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/c/A/index.md b/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/c/A/index.md new file mode 100644 index 0000000000000..10f799646661b --- /dev/null +++ b/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/c/A/index.md @@ -0,0 +1 @@ +"DirA a A" content diff --git a/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/c/index.md b/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/c/index.md new file mode 100644 index 0000000000000..ee018a1652f2c --- /dev/null +++ b/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/c/index.md @@ -0,0 +1 @@ +"DirA c" content diff --git a/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/index.md b/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/index.md new file mode 100644 index 0000000000000..5e9dd3ea6f9ee --- /dev/null +++ b/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/index.md @@ -0,0 +1 @@ +"DirA" content diff --git a/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirB/index.md b/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirB/index.md new file mode 100644 index 0000000000000..11256c8a0cf08 --- /dev/null +++ b/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirB/index.md @@ -0,0 +1 @@ +"DirB" content diff --git a/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/empty/.placeholder.md b/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/empty/.placeholder.md new file mode 100644 index 0000000000000..aec7b712a5cf8 --- /dev/null +++ b/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/empty/.placeholder.md @@ -0,0 +1 @@ +This file is only so we can commit an empty folder. It should be ignored during testing. diff --git a/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/index.md b/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/index.md new file mode 100644 index 0000000000000..fff09d2428d0c --- /dev/null +++ b/packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/index.md @@ -0,0 +1 @@ +root dir content diff --git a/packages/gatsby-source-filesystem/src/file-watcher.js b/packages/gatsby-source-filesystem/src/file-watcher.js new file mode 100644 index 0000000000000..f45cc75b81492 --- /dev/null +++ b/packages/gatsby-source-filesystem/src/file-watcher.js @@ -0,0 +1,159 @@ +const path = require(`path`) +const { Machine } = require(`xstate`) +const findFiles = require(`./find-files`) +const { createFileNode } = require(`./create-file-node`) + +/** + * Create a state machine to manage Chokidar's not-ready/ready states. + */ +const createFSMachine = () => + Machine({ + key: `emitFSEvents`, + parallel: true, + strict: true, + states: { + CHOKIDAR: { + initial: `CHOKIDAR_NOT_READY`, + states: { + CHOKIDAR_NOT_READY: { + on: { + CHOKIDAR_READY: `CHOKIDAR_WATCHING`, + BOOTSTRAP_FINISHED: `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`, + }, + }, + CHOKIDAR_WATCHING: { + on: { + BOOTSTRAP_FINISHED: `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`, + CHOKIDAR_READY: `CHOKIDAR_WATCHING`, + }, + }, + CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED: { + on: { + CHOKIDAR_READY: `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`, + }, + }, + }, + }, + }, + }) + +function fileWatcher( + { actions, getNode, createNodeId, reporter, emitter }, + pluginOptions +) { + const { createNode, deleteNode } = actions + const fsMachine = createFSMachine() + let currentState = fsMachine.initialState + + // Validate that the path is absolute. + // Absolute paths are required to resolve images correctly. + if (!path.isAbsolute(pluginOptions.path)) { + pluginOptions.path = path.resolve(process.cwd(), pluginOptions.path) + } + + // Once bootstrap is finished, we only let one File node update go through + // the system at a time. + emitter.on(`BOOTSTRAP_FINISHED`, () => { + currentState = fsMachine.transition( + currentState.value, + `BOOTSTRAP_FINISHED` + ) + }) + + const watcher = findFiles(pluginOptions.path, { + watch: process.env.GATSBY_EXECUTING_COMMAND === `develop`, + ignore: pluginOptions.ignore, + }) + + const createAndProcessNode = path => { + const fileNodePromise = createFileNode( + path, + createNodeId, + pluginOptions + ).then(fileNode => { + createNode(fileNode) + return null + }) + return fileNodePromise + } + + // For every path that is reported before the 'ready' event, we throw them + // into a queue and then flush the queue when 'ready' event arrives. + // After 'ready', we handle the 'add' event without putting it into a queue. + let pathQueue = [] + const flushPathQueue = () => { + let queue = pathQueue.slice() + pathQueue = [] + return Promise.all(queue.map(createAndProcessNode)) + } + + watcher.on(`add`, path => { + if (currentState.value.CHOKIDAR !== `CHOKIDAR_NOT_READY`) { + if ( + currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED` + ) { + reporter.info(`added file at ${path}`) + } + createAndProcessNode(path).catch(err => reporter.error(err)) + } else { + pathQueue.push(path) + } + }) + + watcher.on(`change`, path => { + if ( + currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED` + ) { + reporter.info(`changed file at ${path}`) + } + createAndProcessNode(path).catch(err => reporter.error(err)) + }) + + watcher.on(`unlink`, path => { + if ( + currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED` + ) { + reporter.info(`file deleted at ${path}`) + } + const node = getNode(createNodeId(path)) + // It's possible the file node was never created as sometimes tools will + // write and then immediately delete temporary files to the file system. + if (node) { + deleteNode({ node }) + } + }) + + watcher.on(`addDir`, path => { + if (currentState.value.CHOKIDAR !== `CHOKIDAR_NOT_READY`) { + if ( + currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED` + ) { + reporter.info(`added directory at ${path}`) + } + createAndProcessNode(path).catch(err => reporter.error(err)) + } else { + pathQueue.push(path) + } + }) + + watcher.on(`unlinkDir`, path => { + if ( + currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED` + ) { + reporter.info(`directory deleted at ${path}`) + } + const node = getNode(createNodeId(path)) + if (node) { + deleteNode({ node }) + } + }) + + return new Promise((resolve, reject) => { + watcher.on(`ready`, () => { + currentState = fsMachine.transition(currentState.value, `CHOKIDAR_READY`) + flushPathQueue().then(resolve, reject) + }) + }) +} + +module.exports = fileWatcher diff --git a/packages/gatsby-source-filesystem/src/find-files.js b/packages/gatsby-source-filesystem/src/find-files.js new file mode 100644 index 0000000000000..f597da26e8179 --- /dev/null +++ b/packages/gatsby-source-filesystem/src/find-files.js @@ -0,0 +1,44 @@ +const chokidar = require(`chokidar`) +const readdirp = require(`readdirp`) +const anymatch = require(`anymatch`) + +const findOnce = (path, options) => { + const { ignored } = options + return new Promise(resolve => { + let fileList = [] + const stream = readdirp({ root: path }) + + stream.on(`data`, data => { + if (anymatch(ignored, data.path)) return + fileList.push(data.fullPath) + return + }) + + stream.on(`end`, () => { + resolve(fileList) + }) + }) +} + +module.exports = (path, options) => { + const { watch, ignore } = options + + const ignored = [ + `**/*.un~`, + `**/.DS_Store`, + `**/.gitignore`, + `**/.npmignore`, + `**/.babelrc`, + `**/yarn.lock`, + `**/bower_components`, + `**/node_modules`, + `../**/dist/**`, + ...(ignore || []), + ] + + if (watch === true) { + return chokidar.watch(path, { ignored }) + } else { + return findOnce(path, { ignored }) + } +} diff --git a/packages/gatsby-source-filesystem/src/gatsby-node.js b/packages/gatsby-source-filesystem/src/gatsby-node.js index b74d3a4cab6c8..f04c42e05d40e 100644 --- a/packages/gatsby-source-filesystem/src/gatsby-node.js +++ b/packages/gatsby-source-filesystem/src/gatsby-node.js @@ -1,49 +1,13 @@ -const chokidar = require(`chokidar`) const fs = require(`fs`) -const path = require(`path`) -const { Machine } = require(`xstate`) -const { createFileNode } = require(`./create-file-node`) +const fileWatcher = require(`./file-watcher`) +const findFiles = require(`./find-files`) -/** - * Create a state machine to manage Chokidar's not-ready/ready states. - */ -const createFSMachine = () => - Machine({ - key: `emitFSEvents`, - parallel: true, - strict: true, - states: { - CHOKIDAR: { - initial: `CHOKIDAR_NOT_READY`, - states: { - CHOKIDAR_NOT_READY: { - on: { - CHOKIDAR_READY: `CHOKIDAR_WATCHING`, - BOOTSTRAP_FINISHED: `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`, - }, - }, - CHOKIDAR_WATCHING: { - on: { - BOOTSTRAP_FINISHED: `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`, - CHOKIDAR_READY: `CHOKIDAR_WATCHING`, - }, - }, - CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED: { - on: { - CHOKIDAR_READY: `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`, - }, - }, - }, - }, - }, - }) +const { createFileNode } = require(`./create-file-node`) -exports.sourceNodes = ( - { actions, getNode, createNodeId, hasNodeChanged, reporter, emitter }, - pluginOptions -) => { - const { createNode, deleteNode } = actions +exports.sourceNodes = async (gatsby, pluginOptions) => { + const { actions, createNodeId, reporter } = gatsby + const { createNode } = actions // Validate that the path exists. if (!fs.existsSync(pluginOptions.path)) { @@ -58,128 +22,21 @@ See docs here - https://www.gatsbyjs.org/packages/gatsby-source-filesystem/ `) } - // Validate that the path is absolute. - // Absolute paths are required to resolve images correctly. - if (!path.isAbsolute(pluginOptions.path)) { - pluginOptions.path = path.resolve(process.cwd(), pluginOptions.path) - } - - const fsMachine = createFSMachine() - let currentState = fsMachine.initialState - - // Once bootstrap is finished, we only let one File node update go through - // the system at a time. - emitter.on(`BOOTSTRAP_FINISHED`, () => { - currentState = fsMachine.transition( - currentState.value, - `BOOTSTRAP_FINISHED` - ) - }) - - const watcher = chokidar.watch(pluginOptions.path, { - ignored: [ - `**/*.un~`, - `**/.DS_Store`, - `**/.gitignore`, - `**/.npmignore`, - `**/.babelrc`, - `**/yarn.lock`, - `**/bower_components`, - `**/node_modules`, - `../**/dist/**`, - ...(pluginOptions.ignore || []), - ], - }) - - const createAndProcessNode = path => { - const fileNodePromise = createFileNode( - path, - createNodeId, - pluginOptions - ).then(fileNode => { - createNode(fileNode) - return null + if (process.env.NODE_ENV === `production`) { + const pathQueue = await findFiles(pluginOptions.path, { + watch: false, + ignore: pluginOptions.ignore, }) - return fileNodePromise - } + const createAndProcessNode = path => + createFileNode(path, createNodeId, pluginOptions).then(createNode) - // For every path that is reported before the 'ready' event, we throw them - // into a queue and then flush the queue when 'ready' event arrives. - // After 'ready', we handle the 'add' event without putting it into a queue. - let pathQueue = [] - const flushPathQueue = () => { - let queue = pathQueue.slice() - pathQueue = [] - return Promise.all(queue.map(createAndProcessNode)) + // TODO: investigate batching this work instead of using Promise.all? + return new Promise((resolve, reject) => + Promise.all(pathQueue.map(createAndProcessNode)).then(resolve, reject) + ) } - watcher.on(`add`, path => { - if (currentState.value.CHOKIDAR !== `CHOKIDAR_NOT_READY`) { - if ( - currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED` - ) { - reporter.info(`added file at ${path}`) - } - createAndProcessNode(path).catch(err => reporter.error(err)) - } else { - pathQueue.push(path) - } - }) - - watcher.on(`change`, path => { - if ( - currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED` - ) { - reporter.info(`changed file at ${path}`) - } - createAndProcessNode(path).catch(err => reporter.error(err)) - }) - - watcher.on(`unlink`, path => { - if ( - currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED` - ) { - reporter.info(`file deleted at ${path}`) - } - const node = getNode(createNodeId(path)) - // It's possible the file node was never created as sometimes tools will - // write and then immediately delete temporary files to the file system. - if (node) { - deleteNode({ node }) - } - }) - - watcher.on(`addDir`, path => { - if (currentState.value.CHOKIDAR !== `CHOKIDAR_NOT_READY`) { - if ( - currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED` - ) { - reporter.info(`added directory at ${path}`) - } - createAndProcessNode(path).catch(err => reporter.error(err)) - } else { - pathQueue.push(path) - } - }) - - watcher.on(`unlinkDir`, path => { - if ( - currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED` - ) { - reporter.info(`directory deleted at ${path}`) - } - const node = getNode(createNodeId(path)) - if (node) { - deleteNode({ node }) - } - }) - - return new Promise((resolve, reject) => { - watcher.on(`ready`, () => { - currentState = fsMachine.transition(currentState.value, `CHOKIDAR_READY`) - flushPathQueue().then(resolve, reject) - }) - }) + return fileWatcher(gatsby, pluginOptions) } exports.setFieldsOnGraphQLNodeType = require(`./extend-file-node`) diff --git a/yarn.lock b/yarn.lock index 0693dd16be6e8..6c841fa0b1b2d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15892,7 +15892,7 @@ readdir-scoped-modules@^1.0.0: graceful-fs "^4.1.2" once "^1.3.0" -readdirp@^2.0.0: +readdirp@^2.0.0, readdirp@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-2.2.1.tgz#0e87622a3325aa33e892285caf8b4e846529a525" integrity sha512-1JU/8q+VgFZyxwrJ+SVIOsh+KywWGpds3NTqikiKpDMZWScmAYyKIgqkO+ARvNWJfXeXR1zxz7aHF4u4CyH6vQ==