Add p-limit dependency and implement concurrency control in fetchDocs

current limit seems to be a good balance of speed reliability

package.json

@@ -47,6 +47,7 @@
     "fs-extra": "^11.2.0",
     "got": "^14.4.3",
     "jsonpath": "^1.1.1",
+    "p-limit": "^6.1.0",
     "path": "^0.12.7",
     "sqlite": "^5.1.1",
     "sqlite3": "^5.1.7"

src/fetchDocs.ts

@@ -1,13 +1,16 @@
 import fs from 'fs-extra'
 import got from 'got'
 import path from 'path'
+import pLimit from 'p-limit'
 import * as cheerio from 'cheerio'
 
 import { addTocAnchors } from './addTocAnchors.js'
 import { highlightCode } from './highlightCode.js'
 import { TocItem } from './types.js'
 import { template } from './template.html.js'
 
+const limit = pLimit(100)
+
 /**
  * Create a page from the pageBody and add it to the template
  * 
@@ -60,7 +63,7 @@ export async function fetchDocs(
     if (tocItem.contents) {
         await Promise.all(
             tocItem.contents.map(async (content) => {
-                return fetchDocs(content, urlRoot, docsDir)
+                limit(() => fetchDocs(content, urlRoot, docsDir))
             })
         )
     }

yarn.lock

@@ -1889,6 +1889,7 @@ __metadata:
     jest: "npm:^29.7.0"
     jest-fetch-mock: "npm:^3.0.3"
     jsonpath: "npm:^1.1.1"
+    p-limit: "npm:^6.1.0"
     path: "npm:^0.12.7"
     prettier: "npm:^3.3.3"
     sqlite: "npm:^5.1.1"
@@ -4317,6 +4318,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"p-limit@npm:^6.1.0":
+  version: 6.1.0
+  resolution: "p-limit@npm:6.1.0"
+  dependencies:
+    yocto-queue: "npm:^1.1.1"
+  checksum: 10c0/40af29461206185a81bdc971ed499d97ceb344114fd21420db95debd9c979b6c02d66a41c321246d09245a51e68410e13df92622cc8c0130f87c6bd81a15d777
+  languageName: node
+  linkType: hard
+
 "p-locate@npm:^4.1.0":
   version: 4.1.0
   resolution: "p-locate@npm:4.1.0"
@@ -5714,3 +5724,10 @@ __metadata:
   checksum: 10c0/dceb44c28578b31641e13695d200d34ec4ab3966a5729814d5445b194933c096b7ced71494ce53a0e8820685d1d010df8b2422e5bf2cdea7e469d97ffbea306f
   languageName: node
   linkType: hard
+
+"yocto-queue@npm:^1.1.1":
+  version: 1.1.1
+  resolution: "yocto-queue@npm:1.1.1"
+  checksum: 10c0/cb287fe5e6acfa82690acb43c283de34e945c571a78a939774f6eaba7c285bacdf6c90fbc16ce530060863984c906d2b4c6ceb069c94d1e0a06d5f2b458e2a92
+  languageName: node
+  linkType: hard