Speedup fetch step by limiting concurrency on network requests only (#…

…1182) * Speedup fetch step by imposing concurrency limit on network requests only * Remove commented out code
yarnpkg · Apr 10, 2020 · aaf0e84 · aaf0e84
1 parent d3859c2
commit aaf0e84
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 4 deletions.
diff --git a/.yarn/versions/f234e0b0.yml b/.yarn/versions/f234e0b0.yml
@@ -0,0 +1,30 @@
+releases:
+  "@yarnpkg/cli": prerelease
+  "@yarnpkg/core": prerelease
+
+declined:
+  - "@yarnpkg/plugin-compat"
+  - "@yarnpkg/plugin-constraints"
+  - "@yarnpkg/plugin-dlx"
+  - "@yarnpkg/plugin-essentials"
+  - "@yarnpkg/plugin-exec"
+  - "@yarnpkg/plugin-file"
+  - "@yarnpkg/plugin-git"
+  - "@yarnpkg/plugin-github"
+  - "@yarnpkg/plugin-http"
+  - "@yarnpkg/plugin-init"
+  - "@yarnpkg/plugin-interactive-tools"
+  - "@yarnpkg/plugin-link"
+  - "@yarnpkg/plugin-node-modules"
+  - "@yarnpkg/plugin-npm"
+  - "@yarnpkg/plugin-npm-cli"
+  - "@yarnpkg/plugin-pack"
+  - "@yarnpkg/plugin-patch"
+  - "@yarnpkg/plugin-pnp"
+  - "@yarnpkg/plugin-stage"
+  - "@yarnpkg/plugin-typescript"
+  - "@yarnpkg/plugin-version"
+  - "@yarnpkg/plugin-workspace-tools"
+  - "@yarnpkg/builder"
+  - "@yarnpkg/doctor"
+  - "@yarnpkg/pnpify"
diff --git a/packages/yarnpkg-core/sources/Project.ts b/packages/yarnpkg-core/sources/Project.ts
@@ -909,13 +909,12 @@ export class Project {
       return structUtils.stringifyLocator(pkg);
     }]);
 
-    const limit = pLimit(5);
     let firstError = false;
 
     const progress = Report.progressViaCounter(locatorHashes.length);
     report.reportProgress(progress);
 
-    await Promise.all(locatorHashes.map(locatorHash => limit(async () => {
+    await Promise.all(locatorHashes.map(locatorHash => (async () => {
       const pkg = this.storedPackages.get(locatorHash);
       if (!pkg)
         throw new Error(`Assertion failed: The locator should have been registered`);
@@ -941,7 +940,7 @@ export class Project {
       if (fetchResult.releaseFs) {
         fetchResult.releaseFs();
       }
-    }).finally(() => {
+    })().finally(() => {
       progress.tick();
     })));
 

diff --git a/packages/yarnpkg-core/sources/httpUtils.ts b/packages/yarnpkg-core/sources/httpUtils.ts
@@ -2,11 +2,16 @@ import got, {GotOptions, NormalizedOptions, Response} from 'got';
 import {Agent as HttpsAgent}                          from 'https';
 import {Agent as HttpAgent}                           from 'http';
 import micromatch                                     from 'micromatch';
+import plimit                                         from 'p-limit';
 import tunnel, {ProxyOptions}                         from 'tunnel';
 import {URL}                                          from 'url';
 
 import {Configuration}                                from './Configuration';
 
+const NETWORK_CONCURRENCY = 8;
+
+const limit = plimit(NETWORK_CONCURRENCY);
+
 const cache = new Map<string, Promise<Response<Buffer>>>();
 
 const globalHttpAgent = new HttpAgent({keepAlive: true});
@@ -101,7 +106,7 @@ export async function request(target: string, body: Body, {configuration, header
     hooks: makeHooks(),
   });
 
-  return gotClient(target) as unknown as Response<any>;
+  return limit(() => gotClient(target) as unknown as Response<any>);
 }
 
 export async function get(target: string, {configuration, json, ...rest}: Options) {