diff --git a/benchmarks/source-drupal/package.json b/benchmarks/source-drupal/package.json index 6d5c5c7a9abb7..9e39fb4de9675 100644 --- a/benchmarks/source-drupal/package.json +++ b/benchmarks/source-drupal/package.json @@ -16,21 +16,21 @@ }, "dependencies": { "cross-env": "^7.0.0", - "dotenv": "^8.2.0", - "faker": "^4.1.0", - "gatsby": "^2.19.7", - "gatsby-image": "^2.2.40", + "dotenv": "^9.0.2", + "faker": "^5.5.3", + "gatsby": "^3.5.1", + "gatsby-image": "^3.5.0", "gatsby-plugin-benchmark-reporting": "*", - "gatsby-plugin-sharp": "^2.4.5", - "gatsby-source-drupal": "^3.3.18", - "gatsby-source-filesystem": "^2.1.48", - "gatsby-transformer-sharp": "^2.3.14", + "gatsby-plugin-sharp": "^3.5.0", + "gatsby-source-drupal": "^4.7.2", + "gatsby-source-filesystem": "^3.5.0", + "gatsby-transformer-sharp": "^3.5.0", "lodash.kebabcase": "^4.1.1", "node-fetch": "^2.6.0", - "react": "^16.12.0", - "react-dom": "^16.12.0", - "ts-node": "^8.6.2", - "typescript": "^3.8.3" + "react": "^17.0.2", + "react-dom": "^17.0.2", + "ts-node": "^9.1.1", + "typescript": "^4.2.4" }, "devDependencies": { "prettier": "2.0.4" diff --git a/packages/gatsby-source-drupal/README.md b/packages/gatsby-source-drupal/README.md index 96462d279c4c4..8531d24f66365 100644 --- a/packages/gatsby-source-drupal/README.md +++ b/packages/gatsby-source-drupal/README.md @@ -150,7 +150,7 @@ module.exports = { } ``` -## GET Params +## GET Search Params You can append optional GET request params to the request url using `params` option. @@ -214,9 +214,13 @@ module.exports = { } ``` +## Concurrent API Requests + +You can use the `concurrentAPIRequests` option to change how many simultaneous API requests are made to the server/service. 20 is the default and seems to be the fastest for most sites. + ## Disallowed Link Types -You can use the `disallowedLinkTypes` option to skip link types found in JSON:API documents. By default it skips the `self` and `describedby` links, which do not provide data that can be sourced. You may override the setting to add additional link types to be skipped. +You can use the `disallowedLinkTypes` option to skip link types found in JSON:API documents. By default it skips the `self`, `describedby`, `contact_message--feedback`, and `contact_message--pesonal` links, which do not provide data that can be sourced. You may override the setting to add additional link types to be skipped. ```javascript // In your gatsby-config.js @@ -227,7 +231,12 @@ module.exports = { options: { baseUrl: `https://live-contentacms.pantheonsite.io/`, // skip the action--action resource type. - disallowedLinkTypes: [`self`, `describedby`, `action--action`], + disallowedLinkTypes: [ + `self`, + `describedby`, + `contact_message--feedback`, + `contact_message--personal`, + ], }, }, ], diff --git a/packages/gatsby-source-drupal/package.json b/packages/gatsby-source-drupal/package.json index d1fc285759d67..e60da6eb75265 100644 --- a/packages/gatsby-source-drupal/package.json +++ b/packages/gatsby-source-drupal/package.json @@ -8,7 +8,9 @@ }, "dependencies": { "@babel/runtime": "^7.14.0", - "axios": "^0.21.1", + "got": "^11.8.2", + "agentkeepalive": "^4.1.1", + "fastq": "^1.11.0", "bluebird": "^3.7.2", "body-parser": "^1.19.0", "gatsby-source-filesystem": "^3.8.0-next.1", @@ -23,11 +25,7 @@ "cross-env": "^7.0.3" }, "homepage": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-drupal#readme", - "keywords": [ - "gatsby", - "gatsby-plugin", - "gatsby-source-plugin" - ], + "keywords": ["gatsby", "gatsby-plugin", "gatsby-source-plugin"], "license": "MIT", "peerDependencies": { "gatsby": "^3.0.0-next.0" diff --git a/packages/gatsby-source-drupal/src/__tests__/index.js b/packages/gatsby-source-drupal/src/__tests__/index.js index 2bb194e737253..4111cc2d1e2fa 100644 --- a/packages/gatsby-source-drupal/src/__tests__/index.js +++ b/packages/gatsby-source-drupal/src/__tests__/index.js @@ -1,16 +1,14 @@ -jest.mock(`axios`, () => { - return { - get: jest.fn(path => { - const last = path.split(`/`).pop() - try { - return { data: require(`./fixtures/${last}.json`) } - } catch (e) { - console.log(`Error`, e) - return null - } - }), - } -}) +jest.mock(`got`, () => + jest.fn(path => { + const last = path.split(`/`).pop() + try { + return { body: require(`./fixtures/${last}.json`) } + } catch (e) { + console.log(`Error`, e) + return null + } + }) +) jest.mock(`gatsby-source-filesystem`, () => { return { @@ -506,18 +504,18 @@ describe(`gatsby-source-drupal`, () => { describe(`Error handling`, () => { describe(`Does end activities if error is thrown`, () => { - const axios = require(`axios`) + const got = require(`got`) beforeEach(() => { nodes = {} reporter.activityTimer.mockClear() activity.start.mockClear() activity.end.mockClear() - axios.get.mockClear() + got.mockClear() downloadFileSpy.mockClear() }) it(`during data fetching`, async () => { - axios.get.mockImplementationOnce(() => { + got.mockImplementationOnce(() => { throw new Error(`data fetching failed`) }) expect.assertions(5) diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js index a3364b38bc06b..a75fec25c3c23 100644 --- a/packages/gatsby-source-drupal/src/gatsby-node.js +++ b/packages/gatsby-source-drupal/src/gatsby-node.js @@ -1,6 +1,9 @@ -const axios = require(`axios`) +const got = require(`got`) const _ = require(`lodash`) const urlJoin = require(`url-join`) +import HttpAgent from "agentkeepalive" + +const { HttpsAgent } = HttpAgent const { setOptions, getOptions } = require(`./plugin-options`) @@ -12,6 +15,17 @@ const { } = require(`./normalize`) const { handleReferences, handleWebhookUpdate } = require(`./utils`) +const agent = { + http: new HttpAgent(), + https: new HttpsAgent(), +} + +async function worker([url, options]) { + return got(url, { agent, ...options }) +} + +const requestQueue = require(`fastq`).promise(worker, 20) + const asyncPool = require(`tiny-async-pool`) const bodyParser = require(`body-parser`) @@ -53,12 +67,18 @@ exports.sourceNodes = async ( const { baseUrl, apiBase = `jsonapi`, - basicAuth, + basicAuth = {}, filters, headers, - params, + params = {}, concurrentFileRequests = 20, - disallowedLinkTypes = [`self`, `describedby`], + concurrentAPIRequests = 20, + disallowedLinkTypes = [ + `self`, + `describedby`, + `contact_message--feedback`, + `contact_message--personal`, + ], skipFileDownloads = false, fastBuilds = false, entityReferenceRevisions = [], @@ -70,6 +90,9 @@ exports.sourceNodes = async ( } = pluginOptions const { createNode, setPluginStatus, touchNode } = actions + // Update the concurrency limit from the plugin options + requestQueue.concurrency = concurrentAPIRequests + if (webhookBody && Object.keys(webhookBody).length) { const changesActivity = reporter.activityTimer( `loading Drupal content changes`, @@ -139,19 +162,22 @@ exports.sourceNodes = async ( try { // Hit fastbuilds endpoint with the lastFetched date. - const data = await axios.get( + const res = await requestQueue.push([ urlJoin(baseUrl, `gatsby-fastbuilds/sync/`, lastFetched.toString()), { - auth: basicAuth, + username: basicAuth.username, + password: basicAuth.password, headers, - params, - } - ) + searchParams: params, + responseType: `json`, + cache, + }, + ]) - if (data.data.status === -1) { + if (res.body.status === -1) { // The incremental data is expired or this is the first fetch. reporter.info(`Unable to pull incremental data changes from Drupal`) - setPluginStatus({ lastFetched: data.data.timestamp }) + setPluginStatus({ lastFetched: res.body.timestamp }) requireFullRebuild = true } else { // Touch nodes so they are not garbage collected by Gatsby. @@ -162,7 +188,7 @@ exports.sourceNodes = async ( }) // Process sync data from Drupal. - const nodesToSync = data.data.entities + const nodesToSync = res.body.entities for (const nodeSyncData of nodesToSync) { if (nodeSyncData.action === `delete`) { actions.deleteNode( @@ -208,7 +234,7 @@ exports.sourceNodes = async ( } } - setPluginStatus({ lastFetched: data.data.timestamp }) + setPluginStatus({ lastFetched: res.body.timestamp }) } } catch (e) { gracefullyRethrow(drupalFetchIncrementalActivity, e) @@ -233,13 +259,19 @@ exports.sourceNodes = async ( let allData try { - const data = await axios.get(urlJoin(baseUrl, apiBase), { - auth: basicAuth, - headers, - params, - }) + const res = await requestQueue.push([ + urlJoin(baseUrl, apiBase), + { + username: basicAuth.username, + password: basicAuth.password, + headers, + searchParams: params, + responseType: `json`, + cache, + }, + ]) allData = await Promise.all( - _.map(data.data.links, async (url, type) => { + _.map(res.body.links, async (url, type) => { if (disallowedLinkTypes.includes(type)) return if (!url) return if (!type) return @@ -276,31 +308,36 @@ exports.sourceNodes = async ( let d try { - d = await axios.get(url, { - auth: basicAuth, - headers, - params, - }) + d = await requestQueue.push([ + url, + { + username: basicAuth.username, + password: basicAuth.password, + headers, + responseType: `json`, + cache, + }, + ]) } catch (error) { - if (error.response && error.response.status == 405) { + if (error.response && error.response.statusCode == 405) { // The endpoint doesn't support the GET method, so just skip it. return [] } else { console.error(`Failed to fetch ${url}`, error.message) - console.log(error.data) + console.log(error) throw error } } - data = data.concat(d.data.data) + data = data.concat(d.body.data) // Add support for includes. Includes allow entity data to be expanded // based on relationships. The expanded data is exposed as `included` // in the JSON API response. // See https://www.drupal.org/docs/8/modules/jsonapi/includes - if (d.data.included) { - data = data.concat(d.data.included) + if (d.body.included) { + data = data.concat(d.body.included) } - if (d.data.links && d.data.links.next) { - data = await getNext(d.data.links.next, data) + if (d.body.links && d.body.links.next) { + data = await getNext(d.body.links.next, data) } return data @@ -485,8 +522,8 @@ exports.pluginOptionsSchema = ({ Joi }) => `The path to the root of the JSONAPI — defaults to "jsonapi"` ), basicAuth: Joi.object({ - username: Joi.string(), - password: Joi.string(), + username: Joi.string().required(), + password: Joi.string().required(), }).description(`Enables basicAuth`), filters: Joi.object().description( `Pass filters to the JSON API for specific collections` @@ -496,6 +533,7 @@ exports.pluginOptionsSchema = ({ Joi }) => ), params: Joi.object().description(`Append optional GET params to requests`), concurrentFileRequests: Joi.number().integer().default(20).min(1), + concurrentAPIRequests: Joi.number().integer().default(20).min(1), disallowedLinkTypes: Joi.array().items(Joi.string()), skipFileDownloads: Joi.boolean(), fastBuilds: Joi.boolean(), diff --git a/renovate.json5 b/renovate.json5 index b62f7091cb066..8999660ccae7e 100644 --- a/renovate.json5 +++ b/renovate.json5 @@ -16733,9 +16733,7 @@ ], "groupName": "minor and patch dependencies for gatsby-source-drupal", "groupSlug": "gatsby-source-drupal-prod-minor", - "matchPackageNames": [ - "axios" - ], + "matchPackageNames": [], "matchUpdateTypes": [ "patch" ], @@ -16768,9 +16766,7 @@ ], "groupName": "major dependencies for gatsby-source-drupal", "groupSlug": "gatsby-source-drupal-prod-major", - "matchPackageNames": [ - "axios" - ], + "matchPackageNames": [], "matchUpdateTypes": [ "major", "minor" diff --git a/yarn.lock b/yarn.lock index 75d8806130e30..16b74144ea937 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5764,6 +5764,15 @@ agentkeepalive@^3.4.1: dependencies: humanize-ms "^1.2.1" +agentkeepalive@^4.1.1: + version "4.1.4" + resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-4.1.4.tgz#d928028a4862cb11718e55227872e842a44c945b" + integrity sha512-+V/rGa3EuU74H6wR04plBb7Ks10FbtUQgRj/FQOG7uUIEuaINI+AiqJR1k6t3SVNs7o7ZjIdus6706qqzVq8jQ== + dependencies: + debug "^4.1.0" + depd "^1.1.2" + humanize-ms "^1.2.1" + aggregate-error@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-3.0.1.tgz#db2fe7246e536f40d9b5442a39e117d7dd6a24e0" @@ -10175,7 +10184,7 @@ denque@^1.4.1: resolved "https://registry.yarnpkg.com/denque/-/denque-1.4.1.tgz#6744ff7641c148c3f8a69c307e51235c1f4a37cf" integrity sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ== -depd@~1.1.2: +depd@^1.1.2, depd@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" @@ -12035,6 +12044,13 @@ fastq@^1.10.0, fastq@^1.6.0: dependencies: reusify "^1.0.4" +fastq@^1.11.0: + version "1.11.0" + resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.11.0.tgz#bb9fb955a07130a918eb63c1f5161cc32a5d0858" + integrity sha512-7Eczs8gIPDrVzT+EksYBcupqMyxSHXXrHOLRRxU2/DicV8789MRBRR8+Hc2uWzUupOs4YS4JzBmBxjjCVBxD/g== + dependencies: + reusify "^1.0.4" + fault@^1.0.0, fault@^1.0.2: version "1.0.4" resolved "https://registry.yarnpkg.com/fault/-/fault-1.0.4.tgz#eafcfc0a6d214fc94601e170df29954a4f842f13" @@ -13402,6 +13418,23 @@ got@^11.7.0: p-cancelable "^2.0.0" responselike "^2.0.0" +got@^11.8.2: + version "11.8.2" + resolved "https://registry.yarnpkg.com/got/-/got-11.8.2.tgz#7abb3959ea28c31f3576f1576c1effce23f33599" + integrity sha512-D0QywKgIe30ODs+fm8wMZiAcZjypcCodPNuMz5H9Mny7RJ+IjJ10BdmGW7OM7fHXP+O7r6ZwapQ/YQmMSvB0UQ== + dependencies: + "@sindresorhus/is" "^4.0.0" + "@szmarczak/http-timer" "^4.0.5" + "@types/cacheable-request" "^6.0.1" + "@types/responselike" "^1.0.0" + cacheable-lookup "^5.0.3" + cacheable-request "^7.0.1" + decompress-response "^6.0.0" + http2-wrapper "^1.0.0-beta.5.2" + lowercase-keys "^2.0.0" + p-cancelable "^2.0.0" + responselike "^2.0.0" + got@^6.7.1: version "6.7.1" resolved "http://registry.npmjs.org/got/-/got-6.7.1.tgz#240cd05785a9a18e561dc1b44b41c763ef1e8db0"