diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 964d0e25..f3842324 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,12 +1,15 @@ name: Run Docker build and tests on: + workflow_dispatch: push: branches: - master + - prod pull_request: branches: - master + - prod permissions: contents: read diff --git a/README.md b/README.md index 793bedb9..afe24032 100755 --- a/README.md +++ b/README.md @@ -147,6 +147,17 @@ See `local.env.list`, `dev.env.list` and `prod.env.list` tempate files. - Allowed resource types: Container, Object - Allowed permissions: Read, Write, List, Add, Process +### Running Crawler Queues in separate Azure account + +Crawler can be run with the queues in a different Azure account from the storage +blobs. This is useful where you are running the crawler and submitting results +to `clearlydefinedprod` _but_ you don't want to have queues in the same Azure +account. As anyone with access to `clearlydefinedprod` can get access to your +queues. + +Set env var `CRAWLER_QUEUE_AZURE_CONNECTION_STRING` with a connection string +obtained from Azure. + ## Build and run Docker image locally `docker build --platform linux/amd64 -t cdcrawler:latest .` diff --git a/config/cdConfig.js b/config/cdConfig.js index c1110828..d71b6bd6 100644 --- a/config/cdConfig.js +++ b/config/cdConfig.js @@ -127,7 +127,7 @@ module.exports = { }, storageQueue: { weights: { immediate: 3, soon: 2, normal: 3, later: 2 }, - connectionString: cd_azblob.connection, + connectionString: config.get('CRAWLER_QUEUE_AZURE_CONNECTION_STRING') || cd_azblob.connection, queueName: config.get('CRAWLER_QUEUE_PREFIX') || 'cdcrawlerdev', visibilityTimeout: 8 * 60 * 60, // 8 hours visibilityTimeout_remainLocal: fetchedCacheTtlSeconds, diff --git a/lib/fetch.js b/lib/fetch.js new file mode 100644 index 00000000..056152b8 --- /dev/null +++ b/lib/fetch.js @@ -0,0 +1,48 @@ +// (c) Copyright 2024, SAP SE and ClearlyDefined contributors. Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +const axios = require('axios') + +function buildRequestOptions(request) { + let responseType = 'text' + if (request.json) { + responseType = 'json' + } else if (request.encoding === null) { + responseType = 'stream' + } + + const validateOptions = {} + if (request.simple === false) { + validateOptions.validateStatus = () => true + } + + return { + method: request.method, + url: request.url || request.uri, + responseType, + headers: request.headers, + data: request.body, + ...validateOptions + } +} + +async function callFetch(request, axiosInstance = axios) { + try { + const options = buildRequestOptions(request) + const response = await axiosInstance(options) + if (!request.resolveWithFullResponse) return response.data + response.statusCode = response.status + response.statusMessage = response.statusText + return response + } catch (error) { + error.statusCode = error.response?.status + throw error + } +} + +function withDefaults(opts) { + const axiosInstance = axios.create(opts) + return request => callFetch(request, axiosInstance) +} + +module.exports = { callFetch, withDefaults } diff --git a/package-lock.json b/package-lock.json index 0c371ef0..4886d43d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "clearlydefined-crawler", - "version": "1.1.0", + "version": "1.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "clearlydefined-crawler", - "version": "1.1.0", + "version": "1.2.0", "hasInstallScript": true, "license": "MIT", "dependencies": { @@ -46,7 +46,6 @@ "parse-github-url": "^1.0.2", "patch-package": "^6.5.1", "qlimit": "^0.1.1", - "request-promise-native": "^1.0.5", "requestretry": "^4.0.0", "semver": "^7.6.0", "sha.js": "^2.4.11", @@ -65,6 +64,7 @@ "eslint": "^8.56.0", "eslint-config-prettier": "^9.1.0", "mocha": "^8.2.0", + "mockttp": "^3.13.0", "nyc": "^15.0.0", "prettier": "3.2.4", "proxyquire": "^2.1.3", @@ -564,6 +564,114 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/@graphql-tools/merge": { + "version": "8.3.1", + "resolved": "https://registry.npmjs.org/@graphql-tools/merge/-/merge-8.3.1.tgz", + "integrity": "sha512-BMm99mqdNZbEYeTPK3it9r9S6rsZsQKtlqJsSBknAclXq2pGEfOxjcIZi+kBSkHZKPKCRrYDd5vY0+rUmIHVLg==", + "dev": true, + "dependencies": { + "@graphql-tools/utils": "8.9.0", + "tslib": "^2.4.0" + }, + "peerDependencies": { + "graphql": "^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0" + } + }, + "node_modules/@graphql-tools/merge/node_modules/@graphql-tools/utils": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/@graphql-tools/utils/-/utils-8.9.0.tgz", + "integrity": "sha512-pjJIWH0XOVnYGXCqej8g/u/tsfV4LvLlj0eATKQu5zwnxd/TiTHq7Cg313qUPTFFHZ3PP5wJ15chYVtLDwaymg==", + "dev": true, + "dependencies": { + "tslib": "^2.4.0" + }, + "peerDependencies": { + "graphql": "^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0" + } + }, + "node_modules/@graphql-tools/schema": { + "version": "8.5.1", + "resolved": "https://registry.npmjs.org/@graphql-tools/schema/-/schema-8.5.1.tgz", + "integrity": "sha512-0Esilsh0P/qYcB5DKQpiKeQs/jevzIadNTaT0jeWklPMwNbT7yMX4EqZany7mbeRRlSRwMzNzL5olyFdffHBZg==", + "dev": true, + "dependencies": { + "@graphql-tools/merge": "8.3.1", + "@graphql-tools/utils": "8.9.0", + "tslib": "^2.4.0", + "value-or-promise": "1.0.11" + }, + "peerDependencies": { + "graphql": "^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0" + } + }, + "node_modules/@graphql-tools/schema/node_modules/@graphql-tools/utils": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/@graphql-tools/utils/-/utils-8.9.0.tgz", + "integrity": "sha512-pjJIWH0XOVnYGXCqej8g/u/tsfV4LvLlj0eATKQu5zwnxd/TiTHq7Cg313qUPTFFHZ3PP5wJ15chYVtLDwaymg==", + "dev": true, + "dependencies": { + "tslib": "^2.4.0" + }, + "peerDependencies": { + "graphql": "^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0" + } + }, + "node_modules/@graphql-tools/utils": { + "version": "8.13.1", + "resolved": "https://registry.npmjs.org/@graphql-tools/utils/-/utils-8.13.1.tgz", + "integrity": "sha512-qIh9yYpdUFmctVqovwMdheVNJqFh+DQNWIhX87FJStfXYnmweBUDATok9fWPleKeFwxnW8IapKmY8m8toJEkAw==", + "dev": true, + "dependencies": { + "tslib": "^2.4.0" + }, + "peerDependencies": { + "graphql": "^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0" + } + }, + "node_modules/@httptoolkit/httpolyglot": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/@httptoolkit/httpolyglot/-/httpolyglot-2.2.1.tgz", + "integrity": "sha512-HOS/0zWc3yn7NM0RQFgBeepeTE8eAKtyOkcGL/TV6if5MAfr+3bH9rwCyAhbXbjlLVR3afeBRt8JYKEerDcygA==", + "dev": true, + "dependencies": { + "@types/node": "*" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@httptoolkit/subscriptions-transport-ws": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/@httptoolkit/subscriptions-transport-ws/-/subscriptions-transport-ws-0.11.2.tgz", + "integrity": "sha512-YB+gYYVjgYUeJrGkfS91ABeNWCFU7EVcn9Cflf2UXjsIiPJEI6yPxujPcjKv9wIJpM+33KQW/qVEmc+BdIDK2w==", + "dev": true, + "dependencies": { + "backo2": "^1.0.2", + "eventemitter3": "^3.1.0", + "iterall": "^1.2.1", + "symbol-observable": "^1.0.4", + "ws": "^8.8.0" + }, + "peerDependencies": { + "graphql": "^15.7.2 || ^16.0.0" + } + }, + "node_modules/@httptoolkit/websocket-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@httptoolkit/websocket-stream/-/websocket-stream-6.0.1.tgz", + "integrity": "sha512-A0NOZI+Glp3Xgcz6Na7i7o09+/+xm2m0UCU8gdtM2nIv6/cjLmhMZMqehSpTlgbx9omtLmV8LVqOskPEyWnmZQ==", + "dev": true, + "dependencies": { + "@types/ws": "*", + "duplexify": "^3.5.1", + "inherits": "^2.0.1", + "isomorphic-ws": "^4.0.1", + "readable-stream": "^2.3.3", + "safe-buffer": "^5.1.2", + "ws": "*", + "xtend": "^4.0.0" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.14", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", @@ -782,6 +890,39 @@ "integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==", "dev": true }, + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", + "dev": true + }, + "node_modules/@types/cors": { + "version": "2.8.17", + "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.17.tgz", + "integrity": "sha512-8CGDvrBj1zgo2qE+oS3pOCyYNqCPryMWY2bGfwA0dcfopWGgxs+78df0Rs3rc9THP4JkOhLsAa+15VdpAqkcUA==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/node": { + "version": "20.14.10", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.10.tgz", + "integrity": "sha512-MdiXf+nDuMvY0gJKxyfZ7/6UFsETO7mGKF54MVD/ekJS6HdFtpZFBgrh6Pseu64XTb2MLyFPlbW6hj8HYRQNOQ==", + "dev": true, + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@types/ws": { + "version": "8.5.10", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.10.tgz", + "integrity": "sha512-vmQSUcfalpIq0R9q7uTo2lXs6eGIpt9wtnLdMv9LVpIjCA/+ufZRozlVoVelIYixx1ugCBKDhn89vnsEGOCx9A==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@ungap/promise-all-settled": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz", @@ -845,6 +986,18 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dev": true, + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/aggregate-error": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", @@ -1015,6 +1168,18 @@ "node": "*" } }, + "node_modules/ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "dev": true, + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/async": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", @@ -1051,6 +1216,15 @@ "semver": "bin/semver" } }, + "node_modules/async-mutex": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.5.0.tgz", + "integrity": "sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA==", + "dev": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -1149,11 +1323,26 @@ "sax": "0.5.x" } }, + "node_modules/backo2": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/backo2/-/backo2-1.0.2.tgz", + "integrity": "sha512-zj6Z6M7Eq+PBZ7PQxl5NT665MvJdAkzp0f60nAJ+sLaSCBPMwVak5ZegFbgVCzFcCJTKFoMizvM5Ld7+JrRJHA==", + "dev": true + }, "node_modules/balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, + "node_modules/base64-arraybuffer": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-0.1.5.tgz", + "integrity": "sha512-437oANT9tP582zZMwSvZGy2nmSeAb8DW2me3y+Uv1Wp2Rulr8Mqlyrv3E7MLxmsiaPSMMDmiDVzgE+e8zlMx9g==", + "dev": true, + "engines": { + "node": ">= 0.6.0" + } + }, "node_modules/base64-js": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", @@ -1170,6 +1359,15 @@ "node": ">= 0.8" } }, + "node_modules/basic-ftp": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.5.tgz", + "integrity": "sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==", + "dev": true, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/bcrypt-pbkdf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", @@ -1196,33 +1394,6 @@ "safe-buffer": "^5.1.1" } }, - "node_modules/bl/node_modules/process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" - }, - "node_modules/bl/node_modules/readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/bl/node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, "node_modules/body-parser": { "version": "1.20.2", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", @@ -1317,6 +1488,12 @@ "node": ">=8" } }, + "node_modules/brotli-wasm": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/brotli-wasm/-/brotli-wasm-1.3.1.tgz", + "integrity": "sha512-Vp+v3QXddvy39Ycbmvd3/Y1kUvKhwtnprzeABcKWN4jmyg6W3W5MhGPCfXBMHeSQnizgpV59iWmkSRp7ykOnDQ==", + "dev": true + }, "node_modules/browser-stdout": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", @@ -1377,6 +1554,15 @@ "node": ">= 0.8" } }, + "node_modules/cacheable-lookup": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-6.1.0.tgz", + "integrity": "sha512-KJ/Dmo1lDDhmW2XDPMo+9oiy/CeqosPguPCrgcVzKyZrL6pM1gU2GmPY/xo6OQPTUaA/c0kwHuywB4E6nmT9ww==", + "dev": true, + "engines": { + "node": ">=10.6.0" + } + }, "node_modules/caching-transform": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/caching-transform/-/caching-transform-4.0.0.tgz", @@ -1684,6 +1870,15 @@ "node": ">= 0.6.x" } }, + "node_modules/common-tags": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/common-tags/-/common-tags-1.8.2.tgz", + "integrity": "sha512-gk/Z852D2Wtb//0I+kRFNKKE9dIIVirjoqPoA1wJU+XePVXZfGeBpk45+A1rKO4Q43prqWBNY/MiIeRLbPWUaA==", + "dev": true, + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/commondir": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", @@ -1709,31 +1904,61 @@ "typedarray": "^0.0.6" } }, - "node_modules/concat-stream/node_modules/process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + "node_modules/connect": { + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/connect/-/connect-3.7.0.tgz", + "integrity": "sha512-ZqRXc+tZukToSNmh5C2iWMSoV3X1YUcPbqEM4DkEG5tNQXrQUZCNVGGv3IuicnkMtPfGf3Xtp8WCXs295iQ1pQ==", + "dev": true, + "dependencies": { + "debug": "2.6.9", + "finalhandler": "1.1.2", + "parseurl": "~1.3.3", + "utils-merge": "1.0.1" + }, + "engines": { + "node": ">= 0.10.0" + } }, - "node_modules/concat-stream/node_modules/readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "node_modules/connect/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" + "ms": "2.0.0" } }, - "node_modules/concat-stream/node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "node_modules/connect/node_modules/finalhandler": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", + "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", + "dev": true, "dependencies": { - "safe-buffer": "~5.1.0" + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "statuses": "~1.5.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/connect/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + }, + "node_modules/connect/node_modules/statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==", + "dev": true, + "engines": { + "node": ">= 0.6" } }, "node_modules/console-control-strings": { @@ -1815,6 +2040,34 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, + "node_modules/cors": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "dev": true, + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/cors-gate": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/cors-gate/-/cors-gate-1.1.3.tgz", + "integrity": "sha512-RFqvbbpj02lqKDhqasBEkgzmT3RseCH3DKy5sT2W9S1mhctABKQP3ktKcnKN0h8t4pJ2SneI3hPl3TGNi/VmZA==", + "dev": true + }, + "node_modules/cross-fetch": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz", + "integrity": "sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==", + "dev": true, + "dependencies": { + "node-fetch": "^2.6.12" + } + }, "node_modules/cross-spawn": { "version": "6.0.5", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", @@ -1883,6 +2136,15 @@ "node": ">=0.10" } }, + "node_modules/data-uri-to-buffer": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", + "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", + "dev": true, + "engines": { + "node": ">= 14" + } + }, "node_modules/debug": { "version": "4.3.5", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", @@ -2023,25 +2285,6 @@ "node": ">=6.0.0" } }, - "node_modules/decompress-tarxz/node_modules/process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" - }, - "node_modules/decompress-tarxz/node_modules/readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, "node_modules/decompress-tarxz/node_modules/rimraf": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", @@ -2053,14 +2296,6 @@ "rimraf": "bin.js" } }, - "node_modules/decompress-tarxz/node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, "node_modules/decompress-unzip": { "version": "4.0.1", "resolved": "git+ssh://git@github.com/clearlydefined/decompress-unzip.git#c26c29ca034209ddf46dd2392a54ee29b6818ccf", @@ -2130,6 +2365,20 @@ "abstract-leveldown": "~2.6.0" } }, + "node_modules/degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "dev": true, + "dependencies": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -2160,6 +2409,18 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/destroyable-server": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/destroyable-server/-/destroyable-server-1.0.2.tgz", + "integrity": "sha512-Ln7ZKRq+7kr/3e4FCI8+jAjRbqbdaET8/ZBoUVvn+sDSAD7zDZA5mykkPRcrjBcaGy+LOM4ntMlIp1NMj1kMxw==", + "dev": true, + "dependencies": { + "@types/node": "*" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/detect-libc": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", @@ -2278,6 +2539,18 @@ "dujs": "du.js" } }, + "node_modules/duplexify": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", + "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", + "dev": true, + "dependencies": { + "end-of-stream": "^1.0.0", + "inherits": "^2.0.1", + "readable-stream": "^2.0.0", + "stream-shift": "^1.0.0" + } + }, "node_modules/ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", @@ -2370,6 +2643,27 @@ "node": ">=0.8.0" } }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "dev": true, + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, "node_modules/eslint": { "version": "8.56.0", "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.56.0.tgz", @@ -2887,6 +3181,12 @@ "node": ">= 0.6" } }, + "node_modules/eventemitter3": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.2.tgz", + "integrity": "sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q==", + "dev": true + }, "node_modules/express": { "version": "4.19.2", "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", @@ -3493,20 +3793,6 @@ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, "node_modules/function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", @@ -3644,6 +3930,35 @@ "node": ">=4" } }, + "node_modules/get-uri": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.3.tgz", + "integrity": "sha512-BzUrJBS9EcUb4cFol8r4W3v1cPsSyajLSthNkz5BxbpDcHN5tIrM10E2eNvfnvBn3DaT3DUgx0OpsBKkaOpanw==", + "dev": true, + "dependencies": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.2", + "debug": "^4.3.4", + "fs-extra": "^11.2.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/get-uri/node_modules/fs-extra": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", + "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, "node_modules/getpass": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", @@ -3727,6 +4042,54 @@ "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", "dev": true }, + "node_modules/graphql": { + "version": "15.9.0", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-15.9.0.tgz", + "integrity": "sha512-GCOQdvm7XxV1S4U4CGrsdlEN37245eC8P9zaYCMr6K1BG0IPGy5lUwmJsEOGyl1GD6HXjOtl2keCP9asRBwNvA==", + "dev": true, + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/graphql-http": { + "version": "1.22.1", + "resolved": "https://registry.npmjs.org/graphql-http/-/graphql-http-1.22.1.tgz", + "integrity": "sha512-4Jor+LRbA7SfSaw7dfDUs2UBzvWg3cKrykfHRgKsOIvQaLuf+QOcG2t3Mx5N9GzSNJcuqMqJWz0ta5+BryEmXg==", + "dev": true, + "engines": { + "node": ">=12" + }, + "peerDependencies": { + "graphql": ">=0.11 <=16" + } + }, + "node_modules/graphql-subscriptions": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/graphql-subscriptions/-/graphql-subscriptions-1.2.1.tgz", + "integrity": "sha512-95yD/tKi24q8xYa7Q9rhQN16AYj5wPbrb8tmHGM3WRc9EBmWrG/0kkMl+tQG8wcEuE9ibR4zyOM31p5Sdr2v4g==", + "dev": true, + "dependencies": { + "iterall": "^1.3.0" + }, + "peerDependencies": { + "graphql": "^0.10.5 || ^0.11.3 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0" + } + }, + "node_modules/graphql-tag": { + "version": "2.12.6", + "resolved": "https://registry.npmjs.org/graphql-tag/-/graphql-tag-2.12.6.tgz", + "integrity": "sha512-FdSNcu2QQcWnM2VNvSCCDCVS5PpPqpzgFT8+GXzqJuoDd0CBncxCY278u4mhRO7tMgo2JjgJA5aZ+nWSQ/Z+xg==", + "dev": true, + "dependencies": { + "tslib": "^2.1.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "graphql": "^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0" + } + }, "node_modules/growl": { "version": "1.10.5", "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", @@ -3860,6 +4223,29 @@ "resolved": "https://registry.npmjs.org/htmlencode/-/htmlencode-0.0.4.tgz", "integrity": "sha1-9+LWr74YqHp45jujMI51N2Z0Dj8=" }, + "node_modules/http-encoding": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/http-encoding/-/http-encoding-1.5.1.tgz", + "integrity": "sha512-2m4JnG1Z5RX5pRMdccyp6rX1jVo4LO+ussQzWdwR4AmrWhtX0KP1NyslVAFAspQwMxt2P00CCWXIBKj7ILZLpQ==", + "dev": true, + "dependencies": { + "brotli-wasm": "^1.1.0", + "pify": "^5.0.0", + "zstd-codec": "^0.1.4" + } + }, + "node_modules/http-encoding/node_modules/pify": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-5.0.0.tgz", + "integrity": "sha512-eW/gHNMlxdSP6dmG6uJip6FXN0EQBwm2clYYd8Wul42Cwu/DK8HEftzsapcNdYe2MfLiIwZqsDk2RDEsTE79hA==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/http-errors": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", @@ -3883,6 +4269,31 @@ "node": ">= 0.8" } }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "dev": true, + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/http-proxy-agent/node_modules/agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "dev": true, + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/http-signature": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", @@ -3897,6 +4308,32 @@ "npm": ">=1.3.7" } }, + "node_modules/http2-wrapper": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-2.2.1.tgz", + "integrity": "sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==", + "dev": true, + "dependencies": { + "quick-lru": "^5.1.1", + "resolve-alpn": "^1.2.0" + }, + "engines": { + "node": ">=10.19.0" + } + }, + "node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "dev": true, + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -3988,6 +4425,31 @@ "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" }, + "node_modules/ip-address": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", + "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", + "dev": true, + "dependencies": { + "jsbn": "1.1.0", + "sprintf-js": "^1.1.3" + }, + "engines": { + "node": ">= 12" + } + }, + "node_modules/ip-address/node_modules/jsbn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", + "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==", + "dev": true + }, + "node_modules/ip-address/node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "dev": true + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -4165,6 +4627,15 @@ "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" }, + "node_modules/isomorphic-ws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/isomorphic-ws/-/isomorphic-ws-4.0.1.tgz", + "integrity": "sha512-BhBvN2MBpWTaSHdWRb/bwdZJ1WaehQ2L1KngkCkfLUGF0mAWAT1sQUQacEmQ0jXkFw/czDXPNQSL5u2/Krsz1w==", + "dev": true, + "peerDependencies": { + "ws": "*" + } + }, "node_modules/isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", @@ -4414,15 +4885,6 @@ "node": ">=8" } }, - "node_modules/istanbul-lib-source-maps/node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/istanbul-reports": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.0.2.tgz", @@ -4436,6 +4898,12 @@ "node": ">=8" } }, + "node_modules/iterall": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/iterall/-/iterall-1.3.0.tgz", + "integrity": "sha512-QZ9qOMdF+QLHxy1QIpUHUU1D5pS2CG2P69LF6L6CPjPYA/XMOmKV3PZpawHoAjHNyB0swdVTRxdYT4tbBbxqwg==", + "dev": true + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -4749,6 +5217,15 @@ "integrity": "sha512-l9x0+1offnKKIzYVjyXU2SiwhXDLekRzKyhnbyldPHvC7BvLPVpdNUNR2KeMAiCN2D/kLNttZgQD5WjSxuBx3Q==", "dev": true }, + "node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "dev": true, + "engines": { + "node": ">=12" + } + }, "node_modules/ltgt": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/ltgt/-/ltgt-2.2.1.tgz", @@ -5267,6 +5744,68 @@ "node": ">=10" } }, + "node_modules/mockttp": { + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/mockttp/-/mockttp-3.13.0.tgz", + "integrity": "sha512-ZwRPeufqZLGJQbHX/wouxVJT5+xmhaOOknwcrhOFQPA+W35wOq4qmeACZzBleyOO0+Y78rtp8MuB4HQe8ix4cg==", + "dev": true, + "dependencies": { + "@graphql-tools/schema": "^8.5.0", + "@graphql-tools/utils": "^8.8.0", + "@httptoolkit/httpolyglot": "^2.2.1", + "@httptoolkit/subscriptions-transport-ws": "^0.11.2", + "@httptoolkit/websocket-stream": "^6.0.1", + "@types/cors": "^2.8.6", + "@types/node": "*", + "async-mutex": "^0.5.0", + "base64-arraybuffer": "^0.1.5", + "body-parser": "^1.15.2", + "cacheable-lookup": "^6.0.0", + "common-tags": "^1.8.0", + "connect": "^3.7.0", + "cors": "^2.8.4", + "cors-gate": "^1.1.3", + "cross-fetch": "^3.1.5", + "destroyable-server": "^1.0.2", + "express": "^4.14.0", + "fast-json-patch": "^3.1.1", + "graphql": "^14.0.2 || ^15.5", + "graphql-http": "^1.22.0", + "graphql-subscriptions": "^1.1.0", + "graphql-tag": "^2.12.6", + "http-encoding": "^1.5.1", + "http2-wrapper": "^2.2.1", + "https-proxy-agent": "^5.0.1", + "isomorphic-ws": "^4.0.1", + "lodash": "^4.16.4", + "lru-cache": "^7.14.0", + "native-duplexpair": "^1.0.0", + "node-forge": "^1.2.1", + "pac-proxy-agent": "^7.0.0", + "parse-multipart-data": "^1.4.0", + "performance-now": "^2.1.0", + "portfinder": "^1.0.32", + "read-tls-client-hello": "^1.0.0", + "semver": "^7.5.3", + "socks-proxy-agent": "^7.0.0", + "typed-error": "^3.0.2", + "urlpattern-polyfill": "^8.0.0", + "uuid": "^8.3.2", + "ws": "^8.8.0" + }, + "bin": { + "mockttp": "dist/admin/admin-bin.js" + }, + "engines": { + "node": ">=14.14.0" + } + }, + "node_modules/mockttp/node_modules/fast-json-patch": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/fast-json-patch/-/fast-json-patch-3.1.1.tgz", + "integrity": "sha512-vf6IHUX2SBcA+5/+4883dsIjpBTqmfBjmYiWK1savxQmFk4JfBMLa7ynTYOs1Rolp/T1betJxHiGD3g1Mn8lUQ==", + "dev": true + }, "node_modules/module-not-found-error": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/module-not-found-error/-/module-not-found-error-1.0.1.tgz", @@ -5331,6 +5870,12 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/native-duplexpair": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/native-duplexpair/-/native-duplexpair-1.0.0.tgz", + "integrity": "sha512-E7QQoM+3jvNtlmyfqRZ0/U75VFgCls+fSkbml2MpgWkWyz3ox8Y58gNhfuziuQYGNNQAbFZJQck55LHCnCK6CA==", + "dev": true + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -5379,6 +5924,15 @@ "node": ">= 0.6" } }, + "node_modules/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "dev": true, + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/nice-try": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", @@ -5431,6 +5985,35 @@ "isarray": "0.0.1" } }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dev": true, + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/node-forge": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz", + "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==", + "dev": true, + "engines": { + "node": ">= 6.13.0" + } + }, "node_modules/node-html-parser": { "version": "5.3.3", "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-5.3.3.tgz", @@ -5831,6 +6414,77 @@ "node": ">=6" } }, + "node_modules/pac-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.0.2.tgz", + "integrity": "sha512-BFi3vZnO9X5Qt6NRz7ZOaPja3ic0PhlsmCRYLOpN11+mWBCR6XJDqW5RF3j8jm4WGGQZtBA+bTfxYzeKW73eHg==", + "dev": true, + "dependencies": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.5", + "pac-resolver": "^7.0.1", + "socks-proxy-agent": "^8.0.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "dev": true, + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/https-proxy-agent": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", + "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", + "dev": true, + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/socks-proxy-agent": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.4.tgz", + "integrity": "sha512-GNAq/eg8Udq2x0eNiFkr9gRg5bA7PXEWagQdeRX4cPSG+X/8V38v637gim9bjFptMk1QWsCTr0ttrJEiXbNnRw==", + "dev": true, + "dependencies": { + "agent-base": "^7.1.1", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-resolver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", + "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "dev": true, + "dependencies": { + "degenerator": "^5.0.0", + "netmask": "^2.0.2" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/package-hash": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/package-hash/-/package-hash-4.0.0.tgz", @@ -5891,6 +6545,12 @@ "xtend": "~4.0.0" } }, + "node_modules/parse-multipart-data": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/parse-multipart-data/-/parse-multipart-data-1.5.0.tgz", + "integrity": "sha512-ck5zaMF0ydjGfejNMnlo5YU2oJ+pT+80Jb1y4ybanT27j+zbVP/jkYmCrUGsEln0Ox/hZmuvgy8Ra7AxbXP2Mw==", + "dev": true + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -6107,6 +6767,38 @@ "node": ">=8" } }, + "node_modules/portfinder": { + "version": "1.0.32", + "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.32.tgz", + "integrity": "sha512-on2ZJVVDXRADWE6jnQaX0ioEylzgBpQk8r55NE4wjXW1ZxO+BgDlY6DXwj20i0V8eB4SenDQ00WEaxfiIQPcxg==", + "dev": true, + "dependencies": { + "async": "^2.6.4", + "debug": "^3.2.7", + "mkdirp": "^0.5.6" + }, + "engines": { + "node": ">= 0.12.0" + } + }, + "node_modules/portfinder/node_modules/async": { + "version": "2.6.4", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz", + "integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==", + "dev": true, + "dependencies": { + "lodash": "^4.17.14" + } + }, + "node_modules/portfinder/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "dependencies": { + "ms": "^2.1.1" + } + }, "node_modules/prettier": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.2.4.tgz", @@ -6123,9 +6815,9 @@ } }, "node_modules/process-nextick-args": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", - "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=" + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, "node_modules/process-on-spawn": { "version": "1.0.0", @@ -6239,6 +6931,18 @@ } ] }, + "node_modules/quick-lru": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", + "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/randombytes": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", @@ -6284,19 +6988,40 @@ "rc": "cli.js" } }, + "node_modules/read-tls-client-hello": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/read-tls-client-hello/-/read-tls-client-hello-1.0.1.tgz", + "integrity": "sha512-OvSzfVv6Y656ekUxB7aDhWkLW7y1ck16ChfLFNJhKNADFNweH2fvyiEZkGmmdtXbOtlNuH2zVXZoFCW349M+GA==", + "dev": true, + "dependencies": { + "@types/node": "*" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/readable-stream": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz", - "integrity": "sha1-j5A0HmilPMySh4jaz80Rs265t44=", + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", "dependencies": { "core-util-is": "~1.0.0", - "inherits": "~2.0.1", + "inherits": "~2.0.3", "isarray": "~1.0.0", - "process-nextick-args": "~1.0.6", - "string_decoder": "~0.10.x", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, + "node_modules/readable-stream/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/readdirp": { "version": "3.5.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.5.0.tgz", @@ -6357,37 +7082,6 @@ "node": ">= 6" } }, - "node_modules/request-promise-core": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.4.tgz", - "integrity": "sha512-TTbAfBBRdWD7aNNOoVOBH4pN/KigV6LyapYNNlAPA8JwbovRti1E88m3sYAwsLi5ryhPKsE9APwnjFTgdUjTpw==", - "dependencies": { - "lodash": "^4.17.19" - }, - "engines": { - "node": ">=0.10.0" - }, - "peerDependencies": { - "request": "^2.34" - } - }, - "node_modules/request-promise-native": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/request-promise-native/-/request-promise-native-1.0.9.tgz", - "integrity": "sha512-wcW+sIUiWnKgNY0dqCpOZkUbF/I+YPi+f09JZIDa39Ec+q82CpSYniDp+ISgTTbKmnpJWASeJBPZmoxH84wt3g==", - "deprecated": "request-promise-native has been deprecated because it extends the now deprecated request package, see https://github.com/request/request/issues/3142", - "dependencies": { - "request-promise-core": "1.1.4", - "stealthy-require": "^1.1.1", - "tough-cookie": "^2.3.3" - }, - "engines": { - "node": ">=0.12.0" - }, - "peerDependencies": { - "request": "^2.34" - } - }, "node_modules/request/node_modules/uuid": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", @@ -6425,6 +7119,12 @@ "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", "dev": true }, + "node_modules/resolve-alpn": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz", + "integrity": "sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==", + "dev": true + }, "node_modules/resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", @@ -6682,6 +7382,53 @@ "node": ">=6" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "dev": true, + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.3.tgz", + "integrity": "sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==", + "dev": true, + "dependencies": { + "ip-address": "^9.0.5", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-7.0.0.tgz", + "integrity": "sha512-Fgl0YPZ902wEsAyiQ+idGd1A7rSFx/ayC1CQVMw5P+EQx2V0SgpGtf6OKFhVjPflPUl9YMmEOnmfjCdMUsygww==", + "dev": true, + "dependencies": { + "agent-base": "^6.0.2", + "debug": "^4.3.3", + "socks": "^2.6.2" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/spawn-wrap": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/spawn-wrap/-/spawn-wrap-2.0.0.tgz", @@ -6867,13 +7614,11 @@ "node": ">= 0.8" } }, - "node_modules/stealthy-require": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", - "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=", - "engines": { - "node": ">=0.10.0" - } + "node_modules/stream-shift": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", + "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==", + "dev": true }, "node_modules/string_decoder": { "version": "0.10.31", @@ -6940,6 +7685,15 @@ "node": ">=4" } }, + "node_modules/symbol-observable": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz", + "integrity": "sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/tar": { "version": "4.4.19", "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.19.tgz", @@ -6974,33 +7728,6 @@ "node": ">= 0.8.0" } }, - "node_modules/tar-stream/node_modules/process-nextick-args": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", - "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" - }, - "node_modules/tar-stream/node_modules/readable-stream": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", - "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/tar-stream/node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, "node_modules/tar/node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -7117,6 +7844,18 @@ "node": ">=0.8" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true + }, + "node_modules/tslib": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", + "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==", + "dev": true + }, "node_modules/tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", @@ -7163,6 +7902,16 @@ "node": ">= 0.6" } }, + "node_modules/typed-error": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/typed-error/-/typed-error-3.2.2.tgz", + "integrity": "sha512-Z48LU67/qJ+vyA7lh3ozELqpTp3pvQoY5RtLi5wQ/UGSrEidBhlVSqhjr8B3iqbGpjqAoJYrtSYXWMDtidWGkA==", + "dev": true, + "engines": { + "node": ">=6.0.0", + "npm": ">=3.0.0" + } + }, "node_modules/typedarray": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", @@ -7191,6 +7940,12 @@ "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz", "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==" }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true + }, "node_modules/universalify": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", @@ -7215,6 +7970,12 @@ "punycode": "^2.1.0" } }, + "node_modules/urlpattern-polyfill": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/urlpattern-polyfill/-/urlpattern-polyfill-8.0.2.tgz", + "integrity": "sha512-Qp95D4TPJl1kC9SKigDcqgyM2VDVO4RiJc2d4qe5GrYm+zbIQCWWKAFaJNQ4BhdFeDGwBmAxqJBwWSJDb9T3BQ==", + "dev": true + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -7228,6 +7989,15 @@ "node": ">= 0.4.0" } }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "dev": true, + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/validator": { "version": "13.11.0", "resolved": "https://registry.npmjs.org/validator/-/validator-13.11.0.tgz", @@ -7236,6 +8006,15 @@ "node": ">= 0.10" } }, + "node_modules/value-or-promise": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/value-or-promise/-/value-or-promise-1.0.11.tgz", + "integrity": "sha512-41BrgH+dIbCFXClcSapVs5M6GkENd3gQOJpEfPDNa71LsUGMXDL0jMWpI/Rh7WhX+Aalfz2TTS3Zt5pUsbnhLg==", + "dev": true, + "engines": { + "node": ">=12" + } + }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -7265,6 +8044,22 @@ "node": ">=0.10.0" } }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/when": { "version": "3.7.8", "resolved": "https://registry.npmjs.org/when/-/when-3.7.8.tgz", @@ -7444,6 +8239,27 @@ "typedarray-to-buffer": "^3.1.5" } }, + "node_modules/ws": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", + "dev": true, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/xml2js": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz", @@ -7700,6 +8516,12 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zstd-codec": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/zstd-codec/-/zstd-codec-0.1.4.tgz", + "integrity": "sha512-KYnWoFWgGtWyQEKNnUcb3u8ZtKO8dn5d8u+oGpxPlopqsPyv60U8suDyfk7Z7UtAO6Sk5i1aVcAs9RbaB1n36A==", + "dev": true } } } diff --git a/package.json b/package.json index 1877b9bc..522bcc62 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "clearlydefined-crawler", - "version": "1.1.0", + "version": "1.2.0", "description": "A crawler that walks projects and packages looking for data of interest to the ClearlyDefined project.", "main": "./index.js", "scripts": { @@ -70,7 +70,6 @@ "parse-github-url": "^1.0.2", "patch-package": "^6.5.1", "qlimit": "^0.1.1", - "request-promise-native": "^1.0.5", "requestretry": "^4.0.0", "semver": "^7.6.0", "sha.js": "^2.4.11", @@ -89,6 +88,7 @@ "eslint": "^8.56.0", "eslint-config-prettier": "^9.1.0", "mocha": "^8.2.0", + "mockttp": "^3.13.0", "nyc": "^15.0.0", "prettier": "3.2.4", "proxyquire": "^2.1.3", diff --git a/providers/fetch/cratesioFetch.js b/providers/fetch/cratesioFetch.js index e0a5e9bf..9d9a8e40 100644 --- a/providers/fetch/cratesioFetch.js +++ b/providers/fetch/cratesioFetch.js @@ -3,7 +3,7 @@ const { clone } = require('lodash') const AbstractFetch = require('./abstractFetch') -const request = require('request-promise-native') +const { callFetch: request } = require('../../lib/fetch') const fs = require('fs') const path = require('path') const FetchResult = require('../../lib/fetchResult') @@ -68,13 +68,16 @@ class CratesioFetch extends AbstractFetch { // Example: https://crates.io/api/v1/crates/bitflags/1.0.4/download async _getPackage(zip, version) { + const response = await request({ + url: `https://crates.io${version.dl_path}`, + encoding: null, + headers: { + 'User-Agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)', + Accept: 'text/html' + } + }) return new Promise((resolve, reject) => { - request({ - url: `https://crates.io${version.dl_path}`, - json: false, - encoding: null, - headers: { 'User-Agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)' } - }).pipe( + response.pipe( fs .createWriteStream(zip) .on('finish', () => resolve(null)) diff --git a/providers/fetch/debianFetch.js b/providers/fetch/debianFetch.js index 64c83029..76e660cf 100644 --- a/providers/fetch/debianFetch.js +++ b/providers/fetch/debianFetch.js @@ -11,7 +11,7 @@ const memCache = require('memory-cache') const nodeRequest = require('request') const path = require('path') const { promisify } = require('util') -const requestPromise = require('request-promise-native') +const { callFetch: requestPromise } = require('../../lib/fetch') const tmp = require('tmp') const unixArchive = require('ar-async') const FetchResult = require('../../lib/fetchResult') diff --git a/providers/fetch/goFetch.js b/providers/fetch/goFetch.js index e666168b..8a7431b9 100644 --- a/providers/fetch/goFetch.js +++ b/providers/fetch/goFetch.js @@ -1,5 +1,5 @@ const { clone } = require('lodash') -const requestPromise = require('request-promise-native') +const { callFetch: requestPromise } = require('../../lib/fetch') const AbstractFetch = require('./abstractFetch') const nodeRequest = require('request') const fs = require('fs') diff --git a/providers/fetch/mavenBasedFetch.js b/providers/fetch/mavenBasedFetch.js index a505cbfc..36500455 100644 --- a/providers/fetch/mavenBasedFetch.js +++ b/providers/fetch/mavenBasedFetch.js @@ -2,7 +2,7 @@ // SPDX-License-Identifier: MIT const AbstractFetch = require('./abstractFetch') -const requestPromise = require('request-promise-native') +const { withDefaults } = require('../../lib/fetch') const nodeRequest = require('request') const { clone, get } = require('lodash') const { promisify } = require('util') @@ -29,7 +29,7 @@ class MavenBasedFetch extends AbstractFetch { constructor(providerMap, options) { super(options) this._providerMap = { ...providerMap } - this._handleRequestPromise = options.requestPromise || requestPromise.defaults(defaultHeaders) + this._handleRequestPromise = options.requestPromise || withDefaults(defaultHeaders) this._handleRequestStream = options.requestStream || nodeRequest.defaults(defaultHeaders).get } diff --git a/providers/fetch/npmjsFetch.js b/providers/fetch/npmjsFetch.js index 8eb7861b..394e736b 100644 --- a/providers/fetch/npmjsFetch.js +++ b/providers/fetch/npmjsFetch.js @@ -3,7 +3,7 @@ const AbstractFetch = require('./abstractFetch') const nodeRequest = require('request') -const requestPromise = require('request-promise-native') +const { callFetch: requestPromise } = require('../../lib/fetch') const fs = require('fs') const { clone, get } = require('lodash') const FetchResult = require('../../lib/fetchResult') diff --git a/providers/fetch/podFetch.js b/providers/fetch/podFetch.js index 4dcac567..d61b4f19 100644 --- a/providers/fetch/podFetch.js +++ b/providers/fetch/podFetch.js @@ -3,7 +3,7 @@ const { clone, get } = require('lodash') const AbstractFetch = require('./abstractFetch') -const request = require('request-promise-native') +const { callFetch: request } = require('../../lib/fetch') const fs = require('fs') const path = require('path') const crypto = require('crypto') @@ -89,8 +89,9 @@ class PodFetch extends AbstractFetch { async _getSourceArchive(dir, url, podspec) { const archive = path.join(dir.name, `${podspec.name}-${podspec.version}.archive`) const output = path.join(dir.name, `${podspec.name}-${podspec.version}`) + const response = await request({ url: url, encoding: null }) return new Promise((resolve, reject) => { - request({ url: url, json: false, encoding: null }).pipe( + response.pipe( fs .createWriteStream(archive) .on('finish', async () => { diff --git a/providers/store/webhookDeltaStore.js b/providers/store/webhookDeltaStore.js index baf5f71d..450ee38a 100644 --- a/providers/store/webhookDeltaStore.js +++ b/providers/store/webhookDeltaStore.js @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation and others. Licensed under the MIT license. // SPDX-License-Identifier: MIT -const request = require('request-promise-native') +const { callFetch: request } = require('../../lib/fetch') const { pick } = require('lodash') class WebhookDeltaStore { diff --git a/test/fixtures/fetch/redis-0.1.0.json b/test/fixtures/fetch/redis-0.1.0.json new file mode 100644 index 00000000..98d42202 --- /dev/null +++ b/test/fixtures/fetch/redis-0.1.0.json @@ -0,0 +1,33 @@ +{ + "name": "redis", + "version": "0.1.0", + "description": "Redis client library", + "author": { + "name": "Matt Ranney", + "email": "mjr@ranney.com" + }, + "main": "./index", + "repository": { + "type": "git", + "url": "git://github.com/mranney/node_redis.git" + }, + "_id": "redis@0.1.0", + "engines": { + "node": "*" + }, + "_nodeSupported": true, + "_npmVersion": "0.2.7-2", + "_nodeVersion": "v0.3.1-pre", + "dist": { + "tarball": "https://registry.npmjs.org/redis/-/redis-0.1.0.tgz", + "shasum": "6f1c493aec95638f654058a7001f42b5df6ba517", + "integrity": "sha512-JJCzhwVAfoa9CF26RWjJafrUkNFjQZw+F2l8kiJo0V0GUwfEI1NNbsiy5ARN9Zrfzp7Eif++OtIKfn+gXy6p9w==", + "signatures": [ + { + "keyid": "SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA", + "sig": "MEUCIQDNKjPgRYSLe0f0xhylEWpdOSHflJvXV2R+bhpXzKFm3QIgJYd4VlkHWdeM0HV6Ll9+c/PlV6oecAN6DexxD6sZ0H8=" + } + ] + }, + "directories": {} +} diff --git a/test/unit/lib/fetchTests.js b/test/unit/lib/fetchTests.js new file mode 100644 index 00000000..330b3899 --- /dev/null +++ b/test/unit/lib/fetchTests.js @@ -0,0 +1,210 @@ +const { fail } = require('assert') +const { callFetch, withDefaults } = require('../../../lib/fetch') +const { expect } = require('chai') +const fs = require('fs') +const mockttp = require('mockttp') + +describe('CallFetch', () => { + describe('with mock server', () => { + const mockServer = mockttp.getLocal() + beforeEach(async () => await mockServer.start()) + afterEach(async () => await mockServer.stop()) + + it('checks if the response is JSON while sending GET request', async () => { + const path = '/registry.npmjs.com/redis/0.1.0' + const expected = fs.readFileSync('test/fixtures/fetch/redis-0.1.0.json') + await mockServer.forGet(path).thenReply(200, expected) + + const response = await callFetch({ + url: mockServer.urlFor(path), + method: 'GET', + json: true + }) + expect(response).to.be.deep.equal(JSON.parse(expected)) + }) + + it('checks if the full response is fetched', async () => { + const path = '/registry.npmjs.com/redis/0.1.0' + const expected = fs.readFileSync('test/fixtures/fetch/redis-0.1.0.json') + await mockServer.forGet(path).thenReply(200, expected) + + const response = await callFetch({ + url: mockServer.urlFor(path), + method: 'GET', + resolveWithFullResponse: true + }) + expect(response.statusCode).to.be.equal(200) + expect(response.statusMessage).to.be.equal('OK') + }) + + it('should throw error with error code', async () => { + const path = '/registry.npmjs.com/redis/0.1.' + await mockServer.forGet(path).thenReply(404) + try { + await callFetch({ + url: mockServer.urlFor(path), + method: 'GET', + json: 'true', + resolveWithFullResponse: true + }) + fail('should have thrown') + } catch (err) { + expect(err.statusCode).to.be.equal(404) + } + }) + + it('checks if the response is text while sending GET request', async () => { + const path = '/proxy.golang.org/rsc.io/quote/@v/v1.3.0.mod' + await mockServer.forGet(path).thenReply(200, 'module "rsc.io/quote"\n') + + const response = await callFetch({ + url: mockServer.urlFor(path), + method: 'GET' + }) + expect(response).to.be.equal('module "rsc.io/quote"\n') + }) + + it('should download stream successfully with GET request', async () => { + const expected = JSON.parse(fs.readFileSync('test/fixtures/fetch/redis-0.1.0.json')) + const path = '/registry.npmjs.com/redis/0.1.0' + await mockServer.forGet(path).thenStream(200, fs.createReadStream('test/fixtures/fetch/redis-0.1.0.json')) + + const response = await callFetch({ + url: mockServer.urlFor(path), + encoding: null + }) + const destination = 'test/fixtures/fetch/temp.json' + await new Promise(resolve => { + response.pipe(fs.createWriteStream(destination).on('finish', () => resolve(true))) + }) + const downloaded = JSON.parse(fs.readFileSync(destination)) + expect(downloaded).to.be.deep.equal(expected) + fs.unlinkSync(destination) + }) + + it('should apply default headers ', async () => { + const path = '/registry.npmjs.com/redis/0.1.0' + const url = mockServer.urlFor(path) + const endpointMock = await mockServer.forGet(path).thenReply(200) + + const defaultOptions = { headers: { 'user-agent': 'clearlydefined.io crawler (clearlydefined@outlook.com)' } } + const requestWithDefaults = withDefaults(defaultOptions) + await requestWithDefaults({ url }) + await requestWithDefaults({ url }) + + const requests = await endpointMock.getSeenRequests() + expect(requests.length).to.equal(2) + expect(requests[0].url).to.equal(url) + expect(requests[0].headers).to.include(defaultOptions.headers) + expect(requests[1].url).to.equal(url) + expect(requests[1].headers).to.include(defaultOptions.headers) + }) + + it('checks if the response is text with uri option in GET request', async () => { + const path = '/proxy.golang.org/rsc.io/quote/@v/v1.3.0.mod' + await mockServer.forGet(path).thenReply(200, 'done') + + const response = await callFetch({ + uri: mockServer.urlFor(path), + method: 'GET' + }) + expect(response).to.be.equal('done') + }) + + it('should POST with JSON', async function () { + const path = '/webhook' + const endpointMock = await mockServer.forPost(path).thenReply(200, 'done') + + const response = await callFetch({ + method: 'POST', + uri: mockServer.urlFor(path), + json: true, + body: { test: 'test' }, + headers: { 'x-crawler': 'secret' }, + resolveWithFullResponse: true + }) + expect(response.statusCode).to.be.equal(200) + const requests = await endpointMock.getSeenRequests() + expect(requests.length).to.equal(1) + const json = await requests[0].body.getJson() + expect(json).to.deep.equal({ test: 'test' }) + expect(requests[0].headers).to.include({ 'x-crawler': 'secret' }) + }) + + describe('test simple', () => { + it('should handle 300 when simple is true by default', async () => { + const path = '/registry.npmjs.com/redis/0.1.0' + await mockServer.forGet(path).thenReply(300, 'test') + + try { + await callFetch({ url: mockServer.urlFor(path) }) + fail('should have thrown') + } catch (err) { + expect(err.statusCode).to.be.equal(300) + } + }) + + it('should handle 300 with simple === false', async () => { + const path = '/registry.npmjs.com/redis/0.1.0' + await mockServer.forGet(path).thenReply(300, 'test') + + const response = await callFetch({ + url: mockServer.urlFor(path), + simple: false + }) + expect(response).to.be.equal('test') + }) + + it('should return status 300 with simple === false', async () => { + const path = '/registry.npmjs.com/redis/0.1.0' + await mockServer.forGet(path).thenReply(300, 'test') + + const response = await callFetch({ + url: mockServer.urlFor(path), + simple: false, + resolveWithFullResponse: true + }) + expect(response.statusCode).to.be.equal(300) + expect(response.statusMessage).to.be.equal('Multiple Choices') + }) + }) + }) + + describe('test crate download', () => { + // This test case downloads a crate package + // This URL would send a JSON response if the header is not provided as a part of request. + it('should follow redirect and download the package when responseType is stream', async () => { + const response = await callFetch({ + url: 'https://crates.io/api/v1/crates/bitflags/1.0.4/download', + method: 'GET', + encoding: null, + headers: { + Accept: 'text/html' + } + }) + // Validating the length of the content in order to verify the response is a crate package. + // JSON response would not return this header in response resulting in failing this test case. + expect(response.headers['content-length']).to.be.equal('15282') + return new Promise((resolve, reject) => { + response.on('data', () => {}) + response.on('end', () => resolve(true)) + response.on('error', reject) + }) + }) + + it('should download the package when responseType is stream', async () => { + const response = await callFetch({ + url: 'https://static.crates.io/crates/bitflags/bitflags-1.0.4.crate', + method: 'GET', + encoding: null + }) + // Validating the length of the content inorder to verify the response is a crate package. + expect(response.headers['content-length']).to.be.equal('15282') + return new Promise((resolve, reject) => { + response.on('data', () => {}) + response.on('end', () => resolve(true)) + response.on('error', reject) + }) + }) + }) +}) diff --git a/test/unit/providers/fetch/cratesioFetchTests.js b/test/unit/providers/fetch/cratesioFetchTests.js index 7dac184e..e500b52d 100644 --- a/test/unit/providers/fetch/cratesioFetchTests.js +++ b/test/unit/providers/fetch/cratesioFetchTests.js @@ -33,13 +33,13 @@ describe('crateFetch workflow', () => { const body = fs.readFileSync(`test/fixtures/crates/${pickFile(options.url)}`) if (options && options.json) return JSON.parse(body) const response = new PassThrough() - response.write(fs.readFileSync(`test/fixtures/crates/${pickFile(options.url)}`)) + response.write(body) response.statusCode = 200 response.end() return response } Fetch = proxyquire('../../../../providers/fetch/cratesioFetch', { - 'request-promise-native': requestPromiseStub + '../../lib/fetch': { callFetch: requestPromiseStub } }) }) diff --git a/test/unit/providers/fetch/dispatcherTests.js b/test/unit/providers/fetch/dispatcherTests.js index 20f58c9a..ec0a801c 100644 --- a/test/unit/providers/fetch/dispatcherTests.js +++ b/test/unit/providers/fetch/dispatcherTests.js @@ -213,7 +213,7 @@ describe('fetchDispatcher cache fetch result', () => { beforeEach(() => { const NpmFetch = proxyquire('../../../../providers/fetch/npmjsFetch', { - 'request-promise-native': npmRegistryRequestStub + '../../lib/fetch': { callFetch: npmRegistryRequestStub } }) const npmFetch = NpmFetch({ logger: { log: sinon.stub() } }) npmFetch._getPackage = sinon @@ -288,7 +288,7 @@ describe('fetchDispatcher cache fetch result', () => { beforeEach(() => { const CrateioFetch = proxyquire('../../../../providers/fetch/cratesioFetch', { - 'request-promise-native': requestPromiseStub + '../../lib/fetch': { callFetch: requestPromiseStub } }) const packagistFetch = CrateioFetch({ logger: { log: sinon.stub() } }) fetchDispatcher = setupDispatcher(packagistFetch) @@ -340,7 +340,7 @@ describe('fetchDispatcher cache fetch result', () => { beforeEach(() => { const GoFetch = proxyquire('../../../../providers/fetch/goFetch', { request: { get: createGetStub(fileSupplier) }, - 'request-promise-native': createRequestPromiseStub(fileSupplier) + '../../lib/fetch': { callFetch: createRequestPromiseStub(fileSupplier) } }) const fetch = GoFetch({ logger: { info: sinon.stub() }, http: successHttpStub }) fetchDispatcher = setupDispatcher(fetch) @@ -402,7 +402,7 @@ describe('fetchDispatcher cache fetch result', () => { return { get: sinon.stub().resolves({ body: loadJson('pod/versions.json'), statusCode: 200 }) } } }, - 'request-promise-native': sinon.stub().resolves(loadJson('pod/registryData.json')) + '../../lib/fetch': { callFetch: sinon.stub().resolves(loadJson('pod/registryData.json')) } }) const fetch = PodFetch({ logger: { info: sinon.stub() } }) fetch._getPackage = sinon.stub().resolves('/tmp/cd-pYKk9q/SwiftLCS-1.0') diff --git a/test/unit/providers/fetch/goFetchTests.js b/test/unit/providers/fetch/goFetchTests.js index 7425e708..2b1c0ce7 100644 --- a/test/unit/providers/fetch/goFetchTests.js +++ b/test/unit/providers/fetch/goFetchTests.js @@ -88,7 +88,7 @@ describe('Go Proxy fetching', () => { } Fetch = proxyquire('../../../../providers/fetch/goFetch', { request: { get: getStub }, - 'request-promise-native': requestPromiseStub + '../../lib/fetch': { callFetch: requestPromiseStub } }) }) diff --git a/test/unit/providers/fetch/mavenBasedFetchTests.js b/test/unit/providers/fetch/mavenBasedFetchTests.js index 5b5a826e..5b3995cc 100644 --- a/test/unit/providers/fetch/mavenBasedFetchTests.js +++ b/test/unit/providers/fetch/mavenBasedFetchTests.js @@ -1,5 +1,8 @@ const { expect } = require('chai') const MavenBasedFetch = require('../../../../providers/fetch/mavenBasedFetch') +const mockttp = require('mockttp') +const sinon = require('sinon') +const Request = require('../../../../ghcrawler').request describe('MavenBasedFetch', () => { describe('find contained file stat', () => { @@ -16,4 +19,25 @@ describe('MavenBasedFetch', () => { expect(file.mtime.toISOString().includes('2022-02-24')) }) }) + + describe('Integration test for component not found', function () { + const path = '/remotecontent?filepath=' + const mockServer = mockttp.getLocal() + beforeEach(async () => await mockServer.start()) + afterEach(async () => await mockServer.stop()) + + it('should handle maven components not found', async () => { + const handler = new MavenBasedFetch( + { + mavencentral: mockServer.urlFor(path) + }, + { logger: { log: sinon.stub() } } + ) + await mockServer.forAnyRequest().thenReply(404) + const request = await handler.handle( + new Request('test', 'cd:/maven/mavencentral/org.apache.httpcomponents/httpcore/4.') + ) + expect(request.processControl).to.be.equal('skip') + }) + }) }) diff --git a/test/unit/providers/fetch/npmjsFetchTests.js b/test/unit/providers/fetch/npmjsFetchTests.js index ddc6653a..53230e65 100644 --- a/test/unit/providers/fetch/npmjsFetchTests.js +++ b/test/unit/providers/fetch/npmjsFetchTests.js @@ -73,7 +73,7 @@ describe('', () => { } Fetch = proxyquire('../../../../providers/fetch/npmjsFetch', { request: { get: getStub }, - 'request-promise-native': requestPromiseStub + '../../lib/fetch': { callFetch: requestPromiseStub } }) Fetch._resultBox = resultBox }) diff --git a/test/unit/providers/fetch/packagistFetchTests.js b/test/unit/providers/fetch/packagistFetchTests.js index 43d944f2..b18c07c9 100644 --- a/test/unit/providers/fetch/packagistFetchTests.js +++ b/test/unit/providers/fetch/packagistFetchTests.js @@ -41,7 +41,7 @@ describe('packagistFetch', () => { } Fetch = proxyquire('../../../../providers/fetch/packagistFetch', { request: { get: getStub }, - 'request-promise-native': requestPromiseStub + '../../lib/fetch': { callFetch: requestPromiseStub } }) Fetch._resultBox = resultBox }) diff --git a/test/unit/providers/fetch/podFetchTests.js b/test/unit/providers/fetch/podFetchTests.js index 74e0fd16..db42063b 100644 --- a/test/unit/providers/fetch/podFetchTests.js +++ b/test/unit/providers/fetch/podFetchTests.js @@ -19,7 +19,7 @@ describe('podFetch', () => { } } }, - 'request-promise-native': sinon.stub().resolves(loadJson('registryData.json')) + '../../lib/fetch': { callFetch: sinon.stub().resolves(loadJson('registryData.json')) } }) let fetch