feat: Add n8n-benchmark cli (no-changelog)

packages/benchmark/Dockerfile

@@ -0,0 +1,67 @@
+# NOTE:
+# 	This Dockerfile needs to be built in the root of the repository
+# 	`docker build -t n8n-benchmark -f packages/benchmark/Dockerfile .`
+FROM node:20.16.0 AS base
+
+# Install required dependencies
+RUN apt-get update && apt-get install -y gnupg2 curl
+
+# Add k6 GPG key and repository
+RUN mkdir -p /etc/apt/keyrings && \
+	curl -sS https://dl.k6.io/key.gpg | gpg --dearmor --yes -o /etc/apt/keyrings/k6.gpg && \
+	chmod a+x /etc/apt/keyrings/k6.gpg && \
+	echo "deb [signed-by=/etc/apt/keyrings/k6.gpg] https://dl.k6.io/deb stable main" | tee /etc/apt/sources.list.d/k6.list
+
+# Update and install k6
+RUN apt-get update && \
+	apt-get install -y k6 tini && \
+	apt-get clean && \
+	rm -rf /var/lib/apt/lists/*
+
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+RUN corepack enable
+
+#
+# Builder
+FROM base AS builder
+
+WORKDIR /app
+
+COPY --chown=node:node ./pnpm-lock.yaml /app/pnpm-lock.yaml
+COPY --chown=node:node ./pnpm-workspace.yaml /app/pnpm-workspace.yaml
+COPY --chown=node:node ./package.json /app/package.json
+COPY --chown=node:node ./packages/benchmark/package.json /app/packages/benchmark/package.json
+COPY --chown=node:node ./patches /app/patches
+COPY --chown=node:node ./scripts /app/scripts
+
+RUN pnpm install --frozen-lockfile
+
+# Package.json
+COPY --chown=node:node ./packages/benchmark/package.json /app/packages/benchmark/package.json
+
+# TS config files
+COPY --chown=node:node ./tsconfig.json /app/tsconfig.json
+COPY --chown=node:node ./tsconfig.build.json /app/tsconfig.build.json
+COPY --chown=node:node ./tsconfig.backend.json /app/tsconfig.backend.json
+COPY --chown=node:node ./packages/benchmark/tsconfig.json /app/packages/benchmark/tsconfig.json
+COPY --chown=node:node ./packages/benchmark/tsconfig.build.json /app/packages/benchmark/tsconfig.build.json
+
+# Source files
+COPY --chown=node:node ./packages/benchmark/src /app/packages/benchmark/src
+COPY --chown=node:node ./packages/benchmark/bin /app/packages/benchmark/bin
+COPY --chown=node:node ./packages/benchmark/testScenarios /app/packages/benchmark/testScenarios
+
+WORKDIR /app/packages/benchmark
+RUN pnpm build
+
+#
+# Runner
+FROM base AS runner
+
+COPY --from=builder /app /app
+
+WORKDIR /app/packages/benchmark
+USER node
+
+ENTRYPOINT [ "/app/packages/benchmark/bin/n8n-benchmark" ]

packages/benchmark/README.md

@@ -0,0 +1,33 @@
+# n8n benchmarking tool
+
+Tool for executing benchmarks against an n8n instance.
+
+## Requirements
+
+- [k6](https://grafana.com/docs/k6/latest/)
+- Node.js v20 or higher
+
+## Running locally
+
+```sh
+pnpm build
+
+# Run tests against http://localhost:5678 with specified email and password
+[email protected] N8N_USER_PASSWORD=password ./bin/n8n-benchmark run
+
+# If you installed k6 using brew, you might have to specify it explicitly
+K6_PATH=/opt/homebrew/bin/k6 [email protected] N8N_USER_PASSWORD=password ./bin/n8n-benchmark run
+```
+
+## Configuration
+
+The configuration options the cli accepts can be seen from [config.ts](./src/config/config.ts)
+
+## Docker build
+
+Because k6 doesn't have an arm64 build available for linux, we need to build against amd64
+
+```sh
+# In the repository root
+docker build --platform linux/amd64 -f packages/benchmark/Dockerfile -t n8n-benchmark:latest .
+```

packages/benchmark/bin/n8n-benchmark

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+
+// Check if version should be displayed
+const versionFlags = ['-v', '-V', '--version'];
+if (versionFlags.includes(process.argv.slice(-1)[0])) {
+	console.log(require('../package').version);
+	process.exit(0);
+}
+
+(async () => {
+	const oclif = require('@oclif/core');
+	await oclif.execute({ dir: __dirname });
+})();

packages/benchmark/nodemon.json

@@ -0,0 +1,6 @@
+{
+	"ignore": ["**/*.spec.ts", ".git", "node_modules"],
+	"watch": ["commands", "index.ts", "src"],
+	"exec": "npm start",
+	"ext": "ts"
+}

packages/benchmark/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "n8n-benchmark",
+  "version": "1.0.0",
+  "description": "",
+  "main": "dist/index",
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json",
+    "start": "./bin/n8n-benchmark",
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "typecheck": "tsc --noEmit",
+    "watch": "concurrently \"tsc -w -p tsconfig.build.json\" \"tsc-alias -w -p tsconfig.build.json\""
+  },
+  "engines": {
+    "node": ">=20.10"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@oclif/core": "4.0.7",
+    "axios": "catalog:",
+    "convict": "6.2.4",
+    "dotenv": "8.6.0",
+    "zx": "^8.1.4"
+  },
+  "devDependencies": {
+    "@types/convict": "^6.1.1",
+    "@types/k6": "^0.52.0",
+    "@types/node": "^20.14.8",
+    "tsc-alias": "^1.8.7",
+    "typescript": "^5.5.2"
+  },
+  "bin": {
+    "n8n-benchmark": "./bin/n8n-benchmark"
+  },
+  "oclif": {
+    "bin": "n8n-benchmark",
+    "commands": "./dist/commands",
+    "topicSeparator": " "
+  }
+}

packages/benchmark/src/commands/list.ts

@@ -0,0 +1,21 @@
+import { Command } from '@oclif/core';
+import { TestScenarioLoader } from '@/testScenario/testScenarioLoader';
+import { loadConfig } from '@/config/config';
+
+export default class ListCommand extends Command {
+	static description = 'List all available test scenarios';
+
+	async run() {
+		const config = loadConfig();
+		const testScenarioLoader = new TestScenarioLoader();
+
+		const allScenarios = testScenarioLoader.loadAllTestScenarios(config.get('testScenariosPath'));
+
+		console.log('Available test scenarios:');
+		console.log('');
+
+		for (const testCase of allScenarios) {
+			console.log('\t', testCase.name, ':', testCase.description);
+		}
+	}
+}

packages/benchmark/src/commands/run.ts

@@ -0,0 +1,39 @@
+import { Command, Flags } from '@oclif/core';
+import { loadConfig } from '@/config/config';
+import { TestScenarioLoader } from '@/testScenario/testScenarioLoader';
+import { TestScenarioRunner } from '@/testExecution/testScenarioRunner';
+import { N8nApiClient } from '@/n8nApiClient/n8nApiClient';
+import { TestDataFileLoader } from '@/testScenario/testDataLoader';
+import { K6Executor } from '@/testExecution/k6Executor';
+
+export default class RunCommand extends Command {
+	static description = 'Run all (default) or specified test scenarios';
+
+	// TODO: Add support for filtering scenarios
+	static flags = {
+		scenarios: Flags.string({
+			char: 't',
+			description: 'Comma-separated list of test scenarios to run',
+			required: false,
+		}),
+	};
+
+	async run() {
+		const config = loadConfig();
+		const testScenarioLoader = new TestScenarioLoader();
+
+		const testScenarioRunner = new TestScenarioRunner(
+			new N8nApiClient(config.get('n8n.baseUrl')),
+			new TestDataFileLoader(),
+			new K6Executor(config.get('k6ExecutablePath'), config.get('n8n.baseUrl')),
+			{
+				email: config.get('n8n.user.email'),
+				password: config.get('n8n.user.password'),
+			},
+		);
+
+		const allScenarios = testScenarioLoader.loadAllTestScenarios(config.get('testScenariosPath'));
+
+		await testScenarioRunner.runManyTestScenarios(allScenarios);
+	}
+}

packages/benchmark/src/config/config.ts

@@ -0,0 +1,50 @@
+import convict from 'convict';
+import dotenv from 'dotenv';
+
+dotenv.config();
+
+const configSchema = {
+	testScenariosPath: {
+		doc: 'The path to the test scenarios',
+		format: String,
+		default: 'testScenarios',
+	},
+	n8n: {
+		baseUrl: {
+			doc: 'The base URL for the n8n instance',
+			format: String,
+			default: 'http://localhost:5678',
+			env: 'N8N_BASE_URL',
+		},
+		user: {
+			email: {
+				doc: 'The email address of the n8n user',
+				format: String,
+				default: '[email protected]',
+				env: 'N8N_USER_EMAIL',
+			},
+			password: {
+				doc: 'The password of the n8n user',
+				format: String,
+				default: 'VerySecret!123',
+				env: 'N8N_USER_PASSWORD',
+			},
+		},
+	},
+	k6ExecutablePath: {
+		doc: 'The path to the k6 binary',
+		format: String,
+		default: 'k6',
+		env: 'K6_PATH',
+	},
+};
+
+export type Config = ReturnType<typeof loadConfig>;
+
+export function loadConfig() {
+	const config = convict(configSchema);
+
+	config.validate({ allowed: 'strict' });
+
+	return config;
+}

packages/benchmark/src/n8nApiClient/authenticatedN8nApiClient.ts

@@ -0,0 +1,68 @@
+import { strict as assert } from 'node:assert';
+import { N8nApiClient } from './n8nApiClient';
+import { AxiosRequestConfig } from 'axios';
+
+export class AuthenticatedN8nApiClient extends N8nApiClient {
+	constructor(
+		apiBaseUrl: string,
+		private readonly authCookie: string,
+	) {
+		super(apiBaseUrl);
+	}
+
+	static async createUsingUsernameAndPassword(
+		apiClient: N8nApiClient,
+		loginDetails: {
+			email: string;
+			password: string;
+		},
+	) {
+		const response = await apiClient.restApiRequest('/login', {
+			method: 'POST',
+			data: loginDetails,
+		});
+
+		const cookieHeader = response.headers['set-cookie'];
+		const authCookie = Array.isArray(cookieHeader) ? cookieHeader.join('; ') : cookieHeader;
+		assert(authCookie);
+
+		return new AuthenticatedN8nApiClient(apiClient.apiBaseUrl, authCookie);
+	}
+
+	async get<T>(endpoint: string) {
+		return await this.authenticatedRequest<T>(endpoint, {
+			method: 'GET',
+		});
+	}
+
+	async post<T>(endpoint: string, data: unknown) {
+		return await this.authenticatedRequest<T>(endpoint, {
+			method: 'POST',
+			data,
+		});
+	}
+
+	async patch<T>(endpoint: string, data: unknown) {
+		return await this.authenticatedRequest<T>(endpoint, {
+			method: 'PATCH',
+			data,
+		});
+	}
+
+	async delete<T>(endpoint: string, data?: unknown) {
+		return await this.authenticatedRequest<T>(endpoint, {
+			method: 'DELETE',
+			data,
+		});
+	}
+
+	protected async authenticatedRequest<T>(endpoint: string, init: Omit<AxiosRequestConfig, 'url'>) {
+		return await this.restApiRequest<T>(endpoint, {
+			...init,
+			headers: {
+				...init.headers,
+				cookie: this.authCookie,
+			},
+		});
+	}
+}