From 56b603ccbd461edbe7f27036ff7741b7f3a4ba05 Mon Sep 17 00:00:00 2001 From: Willie Ruemmele Date: Tue, 23 May 2023 11:53:09 -0600 Subject: [PATCH] chore: use throttledPromise from kit, remove UTs --- package.json | 6 +-- src/commands/project/retrieve/start.ts | 36 ++++++++------ src/utils/promiseQueue.ts | 38 --------------- test/utils/errorCodes.test.ts | 5 ++ test/utils/promiseQueue.test.ts | 62 ------------------------ yarn.lock | 67 ++++++++++++++++++++++++++ 6 files changed, 96 insertions(+), 118 deletions(-) delete mode 100644 src/utils/promiseQueue.ts delete mode 100644 test/utils/promiseQueue.test.ts diff --git a/package.json b/package.json index 5405e08d1..e792d78aa 100644 --- a/package.json +++ b/package.json @@ -5,10 +5,10 @@ "author": "Salesforce", "bugs": "https://github.com/forcedotcom/cli/issues", "dependencies": { - "@oclif/core": "^2.8.2", + "@oclif/core": "^2.8.5", "@salesforce/apex-node": "^1.6.0", - "@salesforce/core": "^3.36.1", - "@salesforce/kit": "^1.9.2", + "@salesforce/core": "^3.36.2", + "@salesforce/kit": "^3.0.1", "@salesforce/sf-plugins-core": "^2.4.2", "@salesforce/source-deploy-retrieve": "^8.4.0", "@salesforce/source-tracking": "^3.1.5", diff --git a/src/commands/project/retrieve/start.ts b/src/commands/project/retrieve/start.ts index 65e78e527..d00f5144f 100644 --- a/src/commands/project/retrieve/start.ts +++ b/src/commands/project/retrieve/start.ts @@ -21,7 +21,7 @@ import { import { SfCommand, toHelpSection, Flags } from '@salesforce/sf-plugins-core'; import { getString } from '@salesforce/ts-types'; import { SourceTracking, SourceConflictError } from '@salesforce/source-tracking'; -import { Duration } from '@salesforce/kit'; +import { Duration, ThrottledPromiseAll } from '@salesforce/kit'; import { Interfaces } from '@oclif/core'; import { DEFAULT_ZIP_FILE_NAME, ensuredDirFlag, zipFileFlag } from '../../../utils/flags'; @@ -30,7 +30,6 @@ import { MetadataRetrieveResultFormatter } from '../../../formatters/metadataRet import { getPackageDirs } from '../../../utils/project'; import { RetrieveResultJson } from '../../../utils/types'; import { writeConflictTable } from '../../../utils/conflicts'; -import { promisesQueue } from '../../../utils/promiseQueue'; Messages.importMessagesDirectory(__dirname); const messages = Messages.loadMessages('@salesforce/plugin-deploy-retrieve', 'retrieve.start'); @@ -243,7 +242,8 @@ export default class RetrieveMetadata extends SfCommand { } private async moveResultsForRetrieveTargetDir(targetDir: string, resolvedTargetDir: string): Promise { - async function mv(src: string): Promise { + async function mv(s: unknown): Promise { + const src = s as string; let directories: string[] = []; let files: string[] = []; const srcStat = await fs.promises.stat(src); @@ -265,17 +265,17 @@ export default class RetrieveMetadata extends SfCommand { } else { files.push(src); } - await promisesQueue( - files, - async (file: string): Promise => { - const dest = join(src.replace(join('main', 'default'), ''), file); - const destDir = dirname(dest); - await fs.promises.mkdir(destDir, { recursive: true }); - await fs.promises.rename(join(src, file), dest); - return dest; - }, - 50 - ); + const throttled = new ThrottledPromiseAll({ concurrency: 50 }); + throttled.add(files, async (file: unknown): Promise => { + // the method type expects (arg: unknown) but we know it's a string + const f = file as string; + const dest = join(src.replace(join('main', 'default'), ''), f); + const destDir = dirname(dest); + await fs.promises.mkdir(destDir, { recursive: true }); + await fs.promises.rename(join(src, f), dest); + return dest; + }); + await throttled.all(); return directories; } // getFileResponses fails once the files have been moved, calculate where they're moved to, and then move them @@ -283,8 +283,14 @@ export default class RetrieveMetadata extends SfCommand { fileResponse.filePath = fileResponse.filePath?.replace(join('main', 'default'), ''); }); // move contents of 'main/default' to 'retrievetargetdir' - await promisesQueue([join(resolvedTargetDir, 'main', 'default')], mv, 5, true); + const throttled = new ThrottledPromiseAll({ concurrency: 50 }); + throttled.add([join(resolvedTargetDir, 'main', 'default')], mv); + await throttled.all(); // remove 'main/default' + + // + + // please clear cache await fs.promises.rm(join(targetDir, 'main'), { recursive: true }); } } diff --git a/src/utils/promiseQueue.ts b/src/utils/promiseQueue.ts deleted file mode 100644 index a1c5b659d..000000000 --- a/src/utils/promiseQueue.ts +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2022, salesforce.com, inc. - * All rights reserved. - * Licensed under the BSD 3-Clause license. - * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause - */ -import { ensureArray } from '@salesforce/kit'; - -/** - * Function to throttle a list of promises. - * - * @param sourceQueue - The list of items to process. - * @param producer - The function to produce a promise from an item. - * @param concurrency - The number of promises to run at a time. - * @param queueResults - Whether to queue the results of the promises. - */ -export async function promisesQueue( - sourceQueue: T[], - producer: (arg0: T) => Promise, - concurrency: number, - queueResults = false -): Promise { - const results: T[] = []; - let queue = [...sourceQueue]; - while (queue.length > 0) { - const next = queue.slice(0, concurrency); - queue = queue.slice(concurrency); - // eslint-disable-next-line no-await-in-loop - const nextResults = (await Promise.all(ensureArray(next.map(producer)))) - .flat(1) - .filter((val) => val !== undefined) as T[]; - if (queueResults) { - queue.push(...nextResults); - } - results.push(...nextResults); - } - return results; -} diff --git a/test/utils/errorCodes.test.ts b/test/utils/errorCodes.test.ts index 690cd35f1..deb14f956 100644 --- a/test/utils/errorCodes.test.ts +++ b/test/utils/errorCodes.test.ts @@ -7,6 +7,11 @@ import { expect } from 'chai'; import { DEPLOY_STATUS_CODES_DESCRIPTIONS, DEPLOY_STATUS_CODES } from '../../src/utils/errorCodes'; +// + +// + +// refact describe('error codes', () => { describe('help descriptions DEPLOY_STATUS_CODES_DESCRIPTIONS', () => { it('creates an object with every status code as a key', () => { diff --git a/test/utils/promiseQueue.test.ts b/test/utils/promiseQueue.test.ts deleted file mode 100644 index c67d49c3d..000000000 --- a/test/utils/promiseQueue.test.ts +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2022, salesforce.com, inc. - * All rights reserved. - * Licensed under the BSD 3-Clause license. - * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause - */ -import { expect } from 'chai'; -import { promisesQueue } from '../../src/utils/promiseQueue'; -describe('promisesQueue', () => { - const numberResolver = (n: number) => Promise.resolve(n); - it('should handle 0 queue entries', async () => { - const results = await promisesQueue([], numberResolver, 1); - expect(results).to.deep.equal([]); - }); - it('should handle many queue entries', async () => { - const results = await promisesQueue([1], numberResolver, 1); - expect(results).to.have.length(1); - expect(results[0]).to.equal(1); - }); - it('should handle 500 queue entry one at a time', async () => { - const a = Array.from({ length: 500 }, (v, i) => i); - const results = await promisesQueue(a, numberResolver, 1); - expect(results).to.have.length(500); - expect(results[499]).to.equal(499); - }); - it('should handle 500 queue entry 10 at a time', async () => { - const a = Array.from({ length: 500 }, (v, i) => i); - const results = await promisesQueue(a, numberResolver, 10); - expect(results).to.have.length(500); - expect(results[499]).to.equal(499); - }); - it('should handle 500 queue entry 500 at a time', async () => { - const a = Array.from({ length: 500 }, (v, i) => i); - const results = await promisesQueue(a, numberResolver, 500); - expect(results).to.have.length(500); - expect(results[499]).to.equal(499); - }); - it('should reject at entry two', async () => { - await promisesQueue( - [1, 2], - (n: number): Promise => (n === 2 ? Promise.reject(n) : Promise.resolve(n)), - 1 - ).catch((e) => expect(e).to.equal(2)); - }); - it('should queue 250 more with a total of 750 promises', async () => { - let count = 0; - const moreResolver = (n: number): Promise => { - const rn = n === 0 && count === 0 ? Array.from({ length: 250 }, (v, i) => i + 500) : n; - count++; - return Promise.resolve(count < 502 ? rn : []); - }; - const a = Array.from({ length: 500 }, (v, i) => i); - const results = await promisesQueue(a, moreResolver, 500, true); - expect(results).to.have.length(750); - }); - it('should handle 5000 queue entry 100 at a time', async () => { - const a = Array.from({ length: 5000 }, (v, i) => i); - const results = await promisesQueue(a, numberResolver, 100); - expect(results).to.have.length(5000); - expect(results[499]).to.equal(499); - }); -}); diff --git a/yarn.lock b/yarn.lock index 807299ef1..dabcd0dbf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1103,6 +1103,27 @@ jsonwebtoken "9.0.0" ts-retry-promise "^0.7.0" +"@salesforce/core@^3.36.2": + version "3.36.2" + resolved "https://registry.yarnpkg.com/@salesforce/core/-/core-3.36.2.tgz#135fa2fb858da6245b9ecff1e7819aeaeead5150" + integrity sha512-IS1rR6Y0tMJYx/+TOAUQ9Gs+Vtum0MHLGfodT7ZJMQZQHEp1S4o0BJ8676uq5sASAnVL64GA+Et/LWCCOWWEuw== + dependencies: + "@salesforce/bunyan" "^2.0.0" + "@salesforce/kit" "^1.9.2" + "@salesforce/schemas" "^1.5.1" + "@salesforce/ts-types" "^1.7.2" + "@types/semver" "^7.3.13" + ajv "^8.12.0" + archiver "^5.3.0" + change-case "^4.1.2" + debug "^3.2.7" + faye "^1.4.0" + form-data "^4.0.0" + js2xmlparser "^4.0.1" + jsforce "^2.0.0-beta.23" + jsonwebtoken "9.0.0" + ts-retry-promise "^0.7.0" + "@salesforce/dev-config@^3.0.0", "@salesforce/dev-config@^3.1.0": version "3.1.0" resolved "https://registry.yarnpkg.com/@salesforce/dev-config/-/dev-config-3.1.0.tgz#8eb5b35860ff60d1c1dc3fd9329b01a28475d5b9" @@ -1157,6 +1178,14 @@ shx "^0.3.3" tslib "^2.5.0" +"@salesforce/kit@^3.0.1": + version "3.0.1" + resolved "https://registry.yarnpkg.com/@salesforce/kit/-/kit-3.0.1.tgz#2375e96b0b22445b52069ca59474652febe8fd2e" + integrity sha512-pH0o7zNKRSjHOBPJeV2nUXrmYD9NsFfxxoVBaa66V7wXQH7OSBlO0/VvEAs1rKZ+MJELgSlpf6h9TajYqTYRrA== + dependencies: + "@salesforce/ts-types" "^2.0.1" + tslib "^2.5.2" + "@salesforce/plugin-command-reference@^2.4.4": version "2.4.5" resolved "https://registry.yarnpkg.com/@salesforce/plugin-command-reference/-/plugin-command-reference-2.4.5.tgz#2b9440306a1e11dce90638a05066378b5caff7ea" @@ -1383,6 +1412,13 @@ dependencies: tslib "^2.5.0" +"@salesforce/ts-types@^2.0.1": + version "2.0.2" + resolved "https://registry.yarnpkg.com/@salesforce/ts-types/-/ts-types-2.0.2.tgz#44c2ab3a765b57a9a285f2be31a661ac5b373ac3" + integrity sha512-FxQnTtdn9mcnGpc6qxQbpP+0CnDY1OcMNM+QL+TgC5uSyxnzo6vBrK7OenrK9jdgfvJZAKH1PjeAj2QahbpBwQ== + dependencies: + tslib "^2.5.2" + "@sindresorhus/is@^4.0.0": version "4.6.0" resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-4.6.0.tgz#3c7c9c46e678feefe7a2e5bb609d3dbd665ffb3f" @@ -5448,6 +5484,32 @@ jsforce@^2.0.0-beta.21: strip-ansi "^6.0.0" xml2js "^0.5.0" +jsforce@^2.0.0-beta.23: + version "2.0.0-beta.23" + resolved "https://registry.yarnpkg.com/jsforce/-/jsforce-2.0.0-beta.23.tgz#5eeb1a2635498ee77f0726ababe089cc12113551" + integrity sha512-lfeLHbCJ40ela1JPu4VqgK1GtIzR0NQOMZ/Rxesv9Ty3PyxK18o01AQ4E+JqGAWDDQhwyUp2A/9V+uOqU3w7Rw== + dependencies: + "@babel/runtime" "^7.12.5" + "@babel/runtime-corejs3" "^7.12.5" + "@types/node" "^12.19.9" + abort-controller "^3.0.0" + base64url "^3.0.1" + commander "^4.0.1" + core-js "^3.6.4" + csv-parse "^4.8.2" + csv-stringify "^5.3.4" + faye "^1.4.0" + form-data "^4.0.0" + fs-extra "^8.1.0" + https-proxy-agent "^5.0.0" + inquirer "^7.0.0" + multistream "^3.1.0" + node-fetch "^2.6.1" + open "^7.0.0" + regenerator-runtime "^0.13.3" + strip-ansi "^6.0.0" + xml2js "^0.5.0" + json-buffer@3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.1.tgz#9338802a30d3b6605fbe0613e094008ca8c05a13" @@ -8502,6 +8564,11 @@ tslib@^2, tslib@^2.0.0, tslib@^2.0.1, tslib@^2.0.3, tslib@^2.1.0, tslib@^2.3.1, resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.5.0.tgz#42bfed86f5787aeb41d031866c8f402429e0fddf" integrity sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg== +tslib@^2.5.2: + version "2.5.2" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.5.2.tgz#1b6f07185c881557b0ffa84b111a0106989e8338" + integrity sha512-5svOrSA2w3iGFDs1HibEVBGbDrAY82bFQ3HZ3ixB+88nsbsWQoKqDRb5UBYAUPEzbBn6dAp5gRNXglySbx1MlA== + tsutils@^3.21.0: version "3.21.0" resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623"