From f208a6e087ede9e1c094c416de543f0898da3d60 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E0=A4=95=E0=A4=BE=E0=A4=B0=E0=A4=A4=E0=A5=8B=E0=A4=AB?=
 =?UTF-8?q?=E0=A5=8D=E0=A4=AB=E0=A5=87=E0=A4=B2=E0=A4=B8=E0=A5=8D=E0=A4=95?=
 =?UTF-8?q?=E0=A5=8D=E0=A4=B0=E0=A4=BF=E0=A4=AA=E0=A5=8D=E0=A4=9F=E2=84=A2?=
 <netroy@users.noreply.github.com>
Date: Tue, 9 Jan 2024 19:17:07 +0100
Subject: [PATCH] ci: Validate docs urls for langchain nodes as well
 (no-changelog) (#8271)

---
 .github/scripts/package.json                  |  9 +++--
 .../scripts}/validate-docs-links.js           | 36 +++++++++++++------
 .../workflows/check-documentation-urls.yml    |  6 ++--
 3 files changed, 36 insertions(+), 15 deletions(-)
 rename {scripts => .github/scripts}/validate-docs-links.js (60%)

diff --git a/.github/scripts/package.json b/.github/scripts/package.json
index 6033dd191c7e8..5798df63219a5 100644
--- a/.github/scripts/package.json
+++ b/.github/scripts/package.json
@@ -1,9 +1,12 @@
 {
   "dependencies": {
+    "cacheable-lookup": "6.1.0",
     "conventional-changelog": "^4.0.0",
-    "glob": "^10.3.0",
-    "semver": "^7.5.4",
-    "tempfile": "^5.0.0",
+    "debug": "4.3.4",
+    "glob": "10.3.10",
+    "p-limit": "3.1.0",
+    "semver": "7.5.4",
+    "tempfile": "5.0.0",
     "typescript": "*"
   }
 }
diff --git a/scripts/validate-docs-links.js b/.github/scripts/validate-docs-links.js
similarity index 60%
rename from scripts/validate-docs-links.js
rename to .github/scripts/validate-docs-links.js
index aab9a23380fd5..2335b62eed4ac 100644
--- a/scripts/validate-docs-links.js
+++ b/.github/scripts/validate-docs-links.js
@@ -1,11 +1,19 @@
 #!/usr/bin/env node
 
+const packages = ['nodes-base', '@n8n/nodes-langchain'];
+const concurrency = 20;
+let exitCode = 0;
+
+const debug = require('debug')('n8n');
 const path = require('path');
 const https = require('https');
-const glob = require('fast-glob');
+const glob = require('glob');
 const pLimit = require('p-limit');
+const Lookup = require('cacheable-lookup').default;
 
-const nodesBaseDir = path.resolve(__dirname, '../packages/nodes-base');
+const agent = new https.Agent({ keepAlive: true, keepAliveMsecs: 5000 });
+new Lookup().install(agent);
+const limiter = pLimit(concurrency);
 
 const validateUrl = async (kind, name, documentationUrl) =>
 	new Promise((resolve, reject) => {
@@ -22,21 +30,26 @@ const validateUrl = async (kind, name, documentationUrl) =>
 					port: 443,
 					path: url.pathname,
 					method: 'HEAD',
+					agent,
+				},
+				(res) => {
+					debug('✓', kind, name);
+					resolve([name, res.statusCode]);
 				},
-				(res) => resolve([name, res.statusCode]),
 			)
 			.on('error', (e) => reject(e))
 			.end();
 	});
 
-const checkLinks = async (kind) => {
-	let types = require(path.join(nodesBaseDir, `dist/types/${kind}.json`));
+const checkLinks = async (baseDir, kind) => {
+	let types = require(path.join(baseDir, `dist/types/${kind}.json`));
 	if (kind === 'nodes')
 		types = types.filter(({ codex }) => !!codex?.resources?.primaryDocumentation);
-	const limit = pLimit(30);
+	debug(kind, types.length);
+
 	const statuses = await Promise.all(
 		types.map((type) =>
-			limit(() => {
+			limiter(() => {
 				const documentationUrl =
 					kind === 'credentials'
 						? type.documentationUrl
@@ -55,10 +68,13 @@ const checkLinks = async (kind) => {
 
 	if (missingDocs.length) console.log('Documentation URL missing for %s', kind, missingDocs);
 	if (invalidUrls.length) console.log('Documentation URL invalid for %s', kind, invalidUrls);
-	if (missingDocs.length || invalidUrls.length) process.exit(1);
+	if (missingDocs.length || invalidUrls.length) exitCode = 1;
 };
 
 (async () => {
-	await checkLinks('credentials');
-	await checkLinks('nodes');
+	for (const packageName of packages) {
+		const baseDir = path.resolve(__dirname, '../../packages', packageName);
+		await Promise.all([checkLinks(baseDir, 'credentials'), checkLinks(baseDir, 'nodes')]);
+		if (exitCode !== 0) process.exit(exitCode);
+	}
 })();
diff --git a/.github/workflows/check-documentation-urls.yml b/.github/workflows/check-documentation-urls.yml
index 4b379499a485e..99bd412105b3e 100644
--- a/.github/workflows/check-documentation-urls.yml
+++ b/.github/workflows/check-documentation-urls.yml
@@ -27,10 +27,12 @@ jobs:
         run: pnpm install --frozen-lockfile
 
       - name: Build nodes-base
-        run: pnpm --filter @n8n/client-oauth2 --filter n8n-workflow --filter n8n-core --filter n8n-nodes-base build
+        run: pnpm --filter @n8n/client-oauth2 --filter n8n-workflow --filter n8n-core --filter n8n-nodes-base --filter @n8n/n8n-nodes-langchain build
+
+      - run: npm install --prefix=.github/scripts --no-package-lock
 
       - name: Test URLs
-        run: node scripts/validate-docs-links.js
+        run: node .github/scripts/validate-docs-links.js
 
       - name: Notify Slack on failure
         uses: act10ns/slack@v2.0.0