From 3a3b569d4ff22d2a6223a56ed7adc6d26ff47b25 Mon Sep 17 00:00:00 2001 From: TJ Simons Date: Tue, 27 Mar 2018 16:42:08 -0500 Subject: [PATCH] Revert files to master --- packages/gatsby-source-wordpress/README.md | 4 -- .../src/__tests__/request-in-queue.js | 49 ---------------- packages/gatsby-source-wordpress/src/fetch.js | 29 +++++----- .../src/gatsby-node.js | 2 - .../src/request-in-queue.js | 58 ------------------- 5 files changed, 13 insertions(+), 129 deletions(-) delete mode 100644 packages/gatsby-source-wordpress/src/__tests__/request-in-queue.js delete mode 100644 packages/gatsby-source-wordpress/src/request-in-queue.js diff --git a/packages/gatsby-source-wordpress/README.md b/packages/gatsby-source-wordpress/README.md index a873c1d1fc53e..fb74b438842a6 100644 --- a/packages/gatsby-source-wordpress/README.md +++ b/packages/gatsby-source-wordpress/README.md @@ -88,10 +88,6 @@ plugins: [ sourceUrl: "https://source-url.com", replacementUrl: "https://replacement-url.com", }, - // How many requests should be sent out simultaneously. - // Lower this number if your blog is on a server with - // limitted resources. - concurrentRequests: 50 }, }, ]; diff --git a/packages/gatsby-source-wordpress/src/__tests__/request-in-queue.js b/packages/gatsby-source-wordpress/src/__tests__/request-in-queue.js deleted file mode 100644 index 4c1ffd9a8f9ce..0000000000000 --- a/packages/gatsby-source-wordpress/src/__tests__/request-in-queue.js +++ /dev/null @@ -1,49 +0,0 @@ -jest.mock(`axios`) - -const requestInQueue = require(`../request-in-queue`) -const axios = require(`axios`) - -axios.mockImplementation(opts => { - if (opts.throw) { throw new Error(opts.throw) } - - return opts.url.slice(opts.url.lastIndexOf(`/`) + 1) -}) - -describe(`requestInQueue`, () => { - let requests - - beforeEach(() => { - requests = [ - { method: `get`, url: `https://gatsbyjs.org/1` }, - { method: `get`, url: `https://gatsbyjs.org/2` }, - { method: `get`, url: `https://gatsbyjs.org/3` }, - { method: `get`, url: `https://gatsbyjs.org/4` }, - ] - }) - - afterEach(() => { - axios.mockClear() - }) - - it(`runs all requests in queue`, async () => { - await requestInQueue(requests) - - requests.forEach((req) => { - expect(axios).toHaveBeenCalledWith(req) - }) - }) - - it(`returns the values in the same order they were requested`, async () => { - const responses = await requestInQueue(requests) - expect(responses).toEqual([`1`, `2`, `3`, `4`]) - }) - - it(`stops any requests when one throws an error`, async () => { - try { - await requestInQueue([{ throw: `error` }, ...requests]) - } catch (err) { - expect(err).toBeDefined() - } - expect(axios).toHaveBeenCalledTimes(1) - }) -}) diff --git a/packages/gatsby-source-wordpress/src/fetch.js b/packages/gatsby-source-wordpress/src/fetch.js index 6724019b38518..39ee4a9181d7b 100644 --- a/packages/gatsby-source-wordpress/src/fetch.js +++ b/packages/gatsby-source-wordpress/src/fetch.js @@ -3,7 +3,6 @@ const axios = require(`axios`) const _ = require(`lodash`) const colorized = require(`./output-color`) const httpExceptionHandler = require(`./http-exception-handler`) -const requestInQueue = require(`./request-in-queue`) /** * High-level function to coordinate fetching data from a WordPress @@ -19,7 +18,6 @@ async function fetch({ baseUrl, typePrefix, refactoredEntityTypes, - concurrentRequests, }) { // If the site is hosted on wordpress.com, the API Route differs. // Same entity types are exposed (excepted for medias and users which need auth) @@ -129,7 +127,6 @@ async function fetch({ _hostingWPCOM, _auth, _accessToken, - concurrentRequests, }) ) if (_verbose) console.log(``) @@ -188,7 +185,6 @@ async function fetchData({ _hostingWPCOM, _auth, _accessToken, - concurrentRequests, }) { const type = route.type const url = route.url @@ -204,7 +200,7 @@ async function fetchData({ if (_verbose) console.time(`Fetching the ${type} took`) let routeResponse = await getPages( - { url, _perPage, _hostingWPCOM, _auth, _accessToken, getPages }, + { url, _perPage, _hostingWPCOM, _auth, _accessToken }, 1 ) @@ -267,7 +263,7 @@ async function fetchData({ * @returns */ async function getPages( - { url, _perPage, _hostingWPCOM, _auth, _accessToken, _verbose, concurrentRequests }, + { url, _perPage, _hostingWPCOM, _auth, _accessToken, _verbose }, page = 1 ) { try { @@ -317,18 +313,18 @@ async function getPages( } // We got page 1, now we want pages 2 through totalPages - const pageOptions = _.range(2, totalPages + 1).map(getPage => getOptions(getPage)) - - // using batchSize instead of concurrent for less overhead from better-queue - // the lib doesn't utilize cluster/child_process, so there isn't real concurrency - const pages = await requestInQueue(pageOptions, { batchSize: concurrentRequests }) - - const pageData = pages.map(page => page.data) - pageData.forEach(list => { - result = result.concat(list) + const requests = _.range(2, totalPages + 1).map(getPage => { + const options = getOptions(getPage) + return axios(options) }) - return result + return Promise.all(requests).then(pages => { + const data = pages.map(page => page.data) + data.forEach(list => { + result = result.concat(list) + }) + return result + }) } catch (e) { return httpExceptionHandler(e) } @@ -353,6 +349,7 @@ function getValidRoutes({ refactoredEntityTypes, }) { let validRoutes = [] + for (let key of Object.keys(allRoutes.data.routes)) { if (_verbose) console.log(`Route discovered :`, key) let route = allRoutes.data.routes[key] diff --git a/packages/gatsby-source-wordpress/src/gatsby-node.js b/packages/gatsby-source-wordpress/src/gatsby-node.js index 1707c004bd963..37ad22f50370f 100644 --- a/packages/gatsby-source-wordpress/src/gatsby-node.js +++ b/packages/gatsby-source-wordpress/src/gatsby-node.js @@ -28,7 +28,6 @@ exports.sourceNodes = async ( verboseOutput, perPage = 100, searchAndReplaceContentUrls = {}, - concurrentRequests = 10, } ) => { const { createNode } = boundActionCreators @@ -49,7 +48,6 @@ exports.sourceNodes = async ( _perPage, typePrefix, refactoredEntityTypes, - concurrentRequests, }) // Normalize data & create nodes diff --git a/packages/gatsby-source-wordpress/src/request-in-queue.js b/packages/gatsby-source-wordpress/src/request-in-queue.js deleted file mode 100644 index 79a5c70ab330a..0000000000000 --- a/packages/gatsby-source-wordpress/src/request-in-queue.js +++ /dev/null @@ -1,58 +0,0 @@ -const Queue = require(`better-queue`) -const Promise = require(`bluebird`) -const request = require(`axios`) - -const _defaults = { - id: `url`, -} - -/** - * [handleQueue description] - * @param {[type]} task [description] - * @param {Function} cb [description] - * @return {[type]} [description] - */ -async function handleQueue(task, cb) { - try { - const response = await request(task) - cb(null, response) - } catch (err) { - cb(err) - } -} - -/** - * @typedef {Options} - * @type {Object} - * @see For a detailed descriptions of the tasks, - * see {@link https://www.npmjs.com/package/better-queue#full-documentation|better-queue on Github} - */ - -/** - * Run a series of requests tasks in a queue for better flow control - * - * @param {Object[]} tasks An array of Axios formatted request objects - * @param {Options} opts Options that will be given to better-queue - * @return {Promise} Resolves with the accumulated values from the tasks - */ -module.exports = function requestInQueue (tasks, opts) { - return new Promise((res, rej) => { - const q = new Queue(handleQueue, { ..._defaults, opts }) - - const taskMap = new Map(tasks.map((t) => { - q.push(t) - return [t.url, null] - })) - - q.on(`task_failed`, (id, err) => { - rej(`${id} failed with err: ${err}`) - q.destroy() - }) - - q.on(`task_finish`, (id, response) => { - taskMap.set(id, response) - }) - - q.on(`drain`, () => res(Array.from(taskMap.values()))) - }) -}