diff --git a/package.json b/package.json index 3ff36f09575b9..990c7c6e28294 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,8 @@ "mime": "^2.2.0", "pretty-bytes": "^4.0.2", "slash": "^1.0.0", - "valid-url": "^1.0.9" + "valid-url": "^1.0.9", + "xstate": "^3.1.0" }, "devDependencies": { "@babel/cli": "^7.0.0-beta.42", diff --git a/src/create-file-node.js b/src/create-file-node.js index f6d2a82a6cbd5..a13734963f356 100644 --- a/src/create-file-node.js +++ b/src/create-file-node.js @@ -7,13 +7,6 @@ const prettyBytes = require(`pretty-bytes`) const md5File = require(`bluebird`).promisify(require(`md5-file`)) const crypto = require(`crypto`) -const createId = path => { - const slashed = slash(path) - return `${slashed} absPath of file` -} - -exports.createId = createId - exports.createFileNode = async ( pathToFile, createNodeId, diff --git a/src/gatsby-node.js b/src/gatsby-node.js index a43a5c2e4008f..9fe828f017775 100644 --- a/src/gatsby-node.js +++ b/src/gatsby-node.js @@ -1,10 +1,77 @@ const chokidar = require(`chokidar`) const fs = require(`fs`) +const { Machine } = require(`xstate`) const { createId, createFileNode } = require(`./create-file-node`) +/** + * Create a state machine to manage Chokidar's not-ready/ready states and for + * emitting file system events into Gatsby. + * + * On the latter, this solves the problem where if you call createNode for the + * same File node in quick succession, this can leave Gatsby's internal state + * in disarray causing queries to fail. The latter state machine tracks when + * Gatsby is "processing" a node update or when it's "idle". If updates come in + * while Gatsby is processing, we queue them until the system returns to an + * "idle" state. + */ +const fsMachine = Machine({ + key: "emitFSEvents", + parallel: true, + strict: true, + states: { + CHOKIDAR: { + initial: `CHOKIDAR_NOT_READY`, + states: { + CHOKIDAR_NOT_READY: { + on: { + CHOKIDAR_READY: "CHOKIDAR_WATCHING", + BOOTSTRAP_FINISHED: "CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED", + }, + }, + CHOKIDAR_WATCHING: { + on: { + BOOTSTRAP_FINISHED: "CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED", + CHOKIDAR_READY: "CHOKIDAR_WATCHING", + }, + }, + CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED: { + on: { + CHOKIDAR_READY: "CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED", + }, + }, + }, + }, + PROCESSING: { + initial: `BOOTSTRAPPING`, + states: { + BOOTSTRAPPING: { + on: { + BOOTSTRAP_FINISHED: "IDLE", + }, + }, + IDLE: { + on: { + EMIT_FS_EVENT: `PROCESSING`, + }, + }, + PROCESSING: { + on: { + QUERY_QUEUE_DRAINED: `IDLE`, + TOUCH_NODE: `IDLE`, + }, + }, + }, + }, + }, +}) + +let currentState = fsMachine.initialState + +const fileQueue = new Map() + exports.sourceNodes = ( - { actions, getNode, createNodeId, hasNodeChanged, reporter }, + { actions, getNode, createNodeId, hasNodeChanged, reporter, emitter }, pluginOptions ) => { const { createNode, deleteNode } = actions @@ -21,8 +88,36 @@ Please pick a path to an existing directory. See docs here - https://www.gatsbyjs.org/packages/gatsby-source-filesystem/ `) } + let fileNodeQueue = new Map() + + // Once bootstrap is finished, we only let one File node update go through + // the system at a time. + emitter.on(`BOOTSTRAP_FINISHED`, () => { + currentState = fsMachine.transition( + currentState.value, + `BOOTSTRAP_FINISHED` + ) + }) + emitter.on(`TOUCH_NODE`, () => { + // If we create a node which is the same as the previous version, createNode + // returns TOUCH_NODE and then nothing else happens so we listen to that + // to return the state back to IDLE. + currentState = fsMachine.transition(currentState.value, `TOUCH_NODE`) + }) - let ready = false + emitter.on(`QUERY_QUEUE_DRAINED`, () => { + currentState = fsMachine.transition( + currentState.value, + `QUERY_QUEUE_DRAINED` + ) + // If we have any updates queued, run one of them now. + if (fileNodeQueue.size > 0) { + const toProcess = fileNodeQueue.get(Array.from(fileNodeQueue.keys())[0]) + fileNodeQueue.delete(toProcess.id) + currentState = fsMachine.transition(currentState.value, `EMIT_FS_EVENT`) + createNode(toProcess) + } + }) const watcher = chokidar.watch(pluginOptions.path, { ignored: [ @@ -36,8 +131,21 @@ See docs here - https://www.gatsbyjs.org/packages/gatsby-source-filesystem/ ], }) - const createAndProcessNode = path => - createFileNode(path, createNodeId, pluginOptions).then(createNode) + const createAndProcessNode = path => { + const fileNodePromise = createFileNode( + path, + createNodeId, + pluginOptions + ).then(fileNode => { + if (currentState.value.PROCESSING === `PROCESSING`) { + fileNodeQueue.set(fileNode.id, fileNode) + } else { + currentState = fsMachine.transition(currentState.value, `EMIT_FS_EVENT`) + createNode(fileNode) + } + }) + return fileNodePromise + } // For every path that is reported before the 'ready' event, we throw them // into a queue and then flush the queue when 'ready' event arrives. @@ -50,8 +158,12 @@ See docs here - https://www.gatsbyjs.org/packages/gatsby-source-filesystem/ } watcher.on(`add`, path => { - if (ready) { - reporter.info(`added file at ${path}`) + if (currentState.value.CHOKIDAR !== `CHOKIDAR_NOT_READY`) { + if ( + currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED` + ) { + reporter.info(`added file at ${path}`) + } createAndProcessNode(path).catch(err => reporter.error(err)) } else { pathQueue.push(path) @@ -59,23 +171,36 @@ See docs here - https://www.gatsbyjs.org/packages/gatsby-source-filesystem/ }) watcher.on(`change`, path => { - reporter.info(`changed file at ${path}`) + if ( + currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED` + ) { + reporter.info(`changed file at ${path}`) + } createAndProcessNode(path).catch(err => reporter.error(err)) }) watcher.on(`unlink`, path => { - reporter.info(`file deleted at ${path}`) - const node = getNode(createId(path)) + if ( + currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED` + ) { + reporter.info(`file deleted at ${path}`) + } + const node = getNode(createNodeId(path)) // It's possible the file node was never created as sometimes tools will // write and then immediately delete temporary files to the file system. if (node) { + currentState = fsMachine.transition(currentState.value, `EMIT_FS_EVENT`) deleteNode(node.id, node) } }) watcher.on(`addDir`, path => { - if (ready) { - reporter.info(`added directory at ${path}`) + if (currentState.value.CHOKIDAR !== `CHOKIDAR_NOT_READY`) { + if ( + currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED` + ) { + reporter.info(`added directory at ${path}`) + } createAndProcessNode(path).catch(err => reporter.error(err)) } else { pathQueue.push(path) @@ -83,16 +208,18 @@ See docs here - https://www.gatsbyjs.org/packages/gatsby-source-filesystem/ }) watcher.on(`unlinkDir`, path => { - reporter.info(`directory deleted at ${path}`) - const node = getNode(createId(path)) + if ( + currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED` + ) { + reporter.info(`directory deleted at ${path}`) + } + const node = getNode(createNodeId(path)) deleteNode(node.id, node) }) return new Promise((resolve, reject) => { watcher.on(`ready`, () => { - if (ready) return - - ready = true + currentState = fsMachine.transition(currentState.value, `CHOKIDAR_READY`) flushPathQueue().then(resolve, reject) }) })