ci: Validate docs urls for langchain nodes as well (no-changelog)

also, speed up the url validation script

.github/scripts/package.json

@@ -1,9 +1,12 @@
 {
   "dependencies": {
+    "cacheable-lookup": "6.1.0",
     "conventional-changelog": "^4.0.0",
-    "glob": "^10.3.0",
-    "semver": "^7.5.4",
-    "tempfile": "^5.0.0",
+    "debug": "4.3.4",
+    "glob": "10.3.10",
+    "p-limit": "3.1.0",
+    "semver": "7.5.4",
+    "tempfile": "5.0.0",
     "typescript": "*"
   }
 }

scripts/validate-docs-links.js → .github/scripts/validate-docs-links.js

@@ -1,11 +1,19 @@
 #!/usr/bin/env node
 
+const packages = ['nodes-base', '@n8n/nodes-langchain'];
+const concurrency = 20;
+let exitCode = 0;
+
+const debug = require('debug')('n8n');
 const path = require('path');
 const https = require('https');
-const glob = require('fast-glob');
+const glob = require('glob');
 const pLimit = require('p-limit');
+const Lookup = require('cacheable-lookup').default;
 
-const nodesBaseDir = path.resolve(__dirname, '../packages/nodes-base');
+const agent = new https.Agent({ keepAlive: true, keepAliveMsecs: 5000 });
+new Lookup().install(agent);
+const limiter = pLimit(concurrency);
 
 const validateUrl = async (kind, name, documentationUrl) =>
 	new Promise((resolve, reject) => {
@@ -22,21 +30,26 @@ const validateUrl = async (kind, name, documentationUrl) =>
 					port: 443,
 					path: url.pathname,
 					method: 'HEAD',
+					agent,
+				},
+				(res) => {
+					debug('\u2713', kind, name);
+					resolve([name, res.statusCode]);
 				},
-				(res) => resolve([name, res.statusCode]),
 			)
 			.on('error', (e) => reject(e))
 			.end();
 	});
 
-const checkLinks = async (kind) => {
-	let types = require(path.join(nodesBaseDir, `dist/types/${kind}.json`));
+const checkLinks = async (baseDir, kind) => {
+	let types = require(path.join(baseDir, `dist/types/${kind}.json`));
 	if (kind === 'nodes')
 		types = types.filter(({ codex }) => !!codex?.resources?.primaryDocumentation);
-	const limit = pLimit(30);
+	debug(kind, types.length);
+
 	const statuses = await Promise.all(
 		types.map((type) =>
-			limit(() => {
+			limiter(() => {
 				const documentationUrl =
 					kind === 'credentials'
 						? type.documentationUrl
@@ -55,10 +68,13 @@ const checkLinks = async (kind) => {
 
 	if (missingDocs.length) console.log('Documentation URL missing for %s', kind, missingDocs);
 	if (invalidUrls.length) console.log('Documentation URL invalid for %s', kind, invalidUrls);
-	if (missingDocs.length || invalidUrls.length) process.exit(1);
+	if (missingDocs.length || invalidUrls.length) exitCode = 1;
 };
 
 (async () => {
-	await checkLinks('credentials');
-	await checkLinks('nodes');
+	for (const packageName of packages) {
+		const baseDir = path.resolve(__dirname, '../../packages', packageName);
+		await Promise.all([checkLinks(baseDir, 'credentials'), checkLinks(baseDir, 'nodes')]);
+		if (exitCode !== 0) process.exit(exitCode);
+	}
 })();

.github/workflows/check-documentation-urls.yml

@@ -27,10 +27,12 @@ jobs:
         run: pnpm install --frozen-lockfile
 
       - name: Build nodes-base
-        run: pnpm --filter @n8n/client-oauth2 --filter n8n-workflow --filter n8n-core --filter n8n-nodes-base build
+        run: pnpm --filter @n8n/client-oauth2 --filter n8n-workflow --filter n8n-core --filter n8n-nodes-base --filter @n8n/n8n-nodes-langchain build
+
+      - run: npm install --prefix=.github/scripts --no-package-lock
 
       - name: Test URLs
-        run: node scripts/validate-docs-links.js
+        run: node .github/scripts/validate-docs-links.js
 
       - name: Notify Slack on failure
         uses: act10ns/[email protected]