feat: Add n8n-benchmark cli

Adds a new cli tool for executing benchmark tests against any n8n instance. The
idea is to treat n8n as a black box and use webhook triggers to initiate workflow
executions.

The tool consists of following parts:
- Test scenarios
  - Consists of test data (workflow(s)) and k6 script to run the benchmark
  - In the `testScenarios` folder
- n8n-benchmark cli
  - In the `src` folder
  - Based on oclif
  - Two commands:
    - list : lists all available test scenarios
    - run : runs all available test scenarios (or specified ones in the future)

Test scenario execution works as follows:
- Setup: setup owner if needed, login
- For each test scenario:
  - Import the specified test data using n8n internal API
  - Run the benchmark and print out results

This is the first step in creating an automated benchmark test setup. Next step is to
add the test environment setup and orchestration to run the benchmarks there.
The feasibility of the approach has been validated in [CAT-26](https://linear.app/n8n/issue/CAT-26/spike-select-infra-to-run-the-benchmarks)
See the [linear project](https://linear.app/n8n/project/n8n-benchmarking-suite-3e7679d176b4/overview) for more details

packages/benchmark/Dockerfile

@@ -0,0 +1,67 @@
+# NOTE:
+# 	This Dockerfile needs to be built in the root of the repository
+# 	`docker build -t n8n-benchmark -f packages/benchmark/Dockerfile .`
+FROM node:20.16.0 AS base
+
+# Install required dependencies
+RUN apt-get update && apt-get install -y gnupg2 curl
+
+# Add k6 GPG key and repository
+RUN mkdir -p /etc/apt/keyrings && \
+	curl -sS https://dl.k6.io/key.gpg | gpg --dearmor --yes -o /etc/apt/keyrings/k6.gpg && \
+	chmod a+x /etc/apt/keyrings/k6.gpg && \
+	echo "deb [signed-by=/etc/apt/keyrings/k6.gpg] https://dl.k6.io/deb stable main" | tee /etc/apt/sources.list.d/k6.list
+
+# Update and install k6
+RUN apt-get update && \
+	apt-get install -y k6 tini && \
+	apt-get clean && \
+	rm -rf /var/lib/apt/lists/*
+
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+RUN corepack enable
+
+#
+# Builder
+FROM base AS builder
+
+WORKDIR /app
+
+COPY --chown=node:node ./pnpm-lock.yaml /app/pnpm-lock.yaml
+COPY --chown=node:node ./pnpm-workspace.yaml /app/pnpm-workspace.yaml
+COPY --chown=node:node ./package.json /app/package.json
+COPY --chown=node:node ./packages/benchmark/package.json /app/packages/benchmark/package.json
+COPY --chown=node:node ./patches /app/patches
+COPY --chown=node:node ./scripts /app/scripts
+
+RUN pnpm install --frozen-lockfile
+
+# Package.json
+COPY --chown=node:node ./packages/benchmark/package.json /app/packages/benchmark/package.json
+
+# TS config files
+COPY --chown=node:node ./tsconfig.json /app/tsconfig.json
+COPY --chown=node:node ./tsconfig.build.json /app/tsconfig.build.json
+COPY --chown=node:node ./tsconfig.backend.json /app/tsconfig.backend.json
+COPY --chown=node:node ./packages/benchmark/tsconfig.json /app/packages/benchmark/tsconfig.json
+COPY --chown=node:node ./packages/benchmark/tsconfig.build.json /app/packages/benchmark/tsconfig.build.json
+
+# Source files
+COPY --chown=node:node ./packages/benchmark/src /app/packages/benchmark/src
+COPY --chown=node:node ./packages/benchmark/bin /app/packages/benchmark/bin
+COPY --chown=node:node ./packages/benchmark/testScenarios /app/packages/benchmark/testScenarios
+
+WORKDIR /app/packages/benchmark
+RUN pnpm build
+
+#
+# Runner
+FROM base AS runner
+
+COPY --from=builder /app /app
+
+WORKDIR /app/packages/benchmark
+USER node
+
+ENTRYPOINT [ "/app/packages/benchmark/bin/n8n-benchmark" ]

packages/benchmark/README.md

@@ -0,0 +1,33 @@
+# n8n benchmarking tool
+
+Tool for executing benchmarks against an n8n instance.
+
+## Requirements
+
+- [k6](https://grafana.com/docs/k6/latest/)
+- Node.js v20 or higher
+
+## Running locally
+
+```sh
+pnpm build
+
+# Run tests against http://localhost:5678 with specified email and password
+[email protected] N8N_USER_PASSWORD=password ./bin/n8n-benchmark run
+
+# If you installed k6 using brew, you might have to specify it explicitly
+K6_PATH=/opt/homebrew/bin/k6 [email protected] N8N_USER_PASSWORD=password ./bin/n8n-benchmark run
+```
+
+## Configuration
+
+The configuration options the cli accepts can be seen from [config.ts](./src/config/config.ts)
+
+## Docker build
+
+Because k6 doesn't have an arm64 build available for linux, we need to build against amd64
+
+```sh
+# In the repository root
+docker build --platform linux/amd64 -f packages/benchmark/Dockerfile -t n8n-benchmark:latest .
+```

packages/benchmark/bin/n8n-benchmark

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+
+// Check if version should be displayed
+const versionFlags = ['-v', '-V', '--version'];
+if (versionFlags.includes(process.argv.slice(-1)[0])) {
+	console.log(require('../package').version);
+	process.exit(0);
+}
+
+(async () => {
+	const oclif = require('@oclif/core');
+	await oclif.execute({ dir: __dirname });
+})();

packages/benchmark/nodemon.json

@@ -0,0 +1,6 @@
+{
+	"ignore": ["**/*.spec.ts", ".git", "node_modules"],
+	"watch": ["commands", "index.ts", "src"],
+	"exec": "npm start",
+	"ext": "ts"
+}

packages/benchmark/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "n8n-benchmark",
+  "version": "1.0.0",
+  "description": "",
+  "main": "dist/index",
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json",
+    "start": "./bin/n8n-benchmark",
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "typecheck": "tsc --noEmit",
+    "watch": "concurrently \"tsc -w -p tsconfig.build.json\" \"tsc-alias -w -p tsconfig.build.json\""
+  },
+  "engines": {
+    "node": ">=20.10"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@oclif/core": "4.0.7",
+    "axios": "catalog:",
+    "convict": "6.2.4",
+    "dotenv": "8.6.0",
+    "zx": "^8.1.4"
+  },
+  "devDependencies": {
+    "@types/convict": "^6.1.1",
+    "@types/k6": "^0.52.0",
+    "@types/node": "^20.14.8",
+    "tsc-alias": "^1.8.7",
+    "typescript": "^5.5.2"
+  },
+  "bin": {
+    "n8n-benchmark": "./bin/n8n-benchmark"
+  },
+  "oclif": {
+    "bin": "n8n-benchmark",
+    "commands": "./dist/commands",
+    "topicSeparator": " "
+  }
+}

packages/benchmark/src/commands/list.ts

@@ -0,0 +1,21 @@
+import { Command } from '@oclif/core';
+import { TestScenarioLoader } from '@/testScenario/testScenarioLoader';
+import { loadConfig } from '@/config/config';
+
+export default class ListCommand extends Command {
+	static description = 'List all available test scenarios';
+
+	async run() {
+		const config = loadConfig();
+		const testScenarioLoader = new TestScenarioLoader();
+
+		const allScenarios = testScenarioLoader.loadAllTestScenarios(config.get('testScenariosPath'));
+
+		console.log('Available test scenarios:');
+		console.log('');
+
+		for (const testCase of allScenarios) {
+			console.log('\t', testCase.name, ':', testCase.description);
+		}
+	}
+}

packages/benchmark/src/commands/run.ts

@@ -0,0 +1,39 @@
+import { Command, Flags } from '@oclif/core';
+import { loadConfig } from '@/config/config';
+import { TestScenarioLoader } from '@/testScenario/testScenarioLoader';
+import { TestScenarioRunner } from '@/testExecution/testScenarioRunner';
+import { N8nApiClient } from '@/n8nApiClient/n8nApiClient';
+import { TestDataFileLoader } from '@/testScenario/testDataLoader';
+import { K6Executor } from '@/testExecution/k6Executor';
+
+export default class RunCommand extends Command {
+	static description = 'Run all (default) or specified test scenarios';
+
+	// TODO: Add support for filtering scenarios
+	static flags = {
+		scenarios: Flags.string({
+			char: 't',
+			description: 'Comma-separated list of test scenarios to run',
+			required: false,
+		}),
+	};
+
+	async run() {
+		const config = loadConfig();
+		const testScenarioLoader = new TestScenarioLoader();
+
+		const testScenarioRunner = new TestScenarioRunner(
+			new N8nApiClient(config.get('n8n.baseUrl')),
+			new TestDataFileLoader(),
+			new K6Executor(config.get('k6ExecutablePath'), config.get('n8n.baseUrl')),
+			{
+				email: config.get('n8n.user.email'),
+				password: config.get('n8n.user.password'),
+			},
+		);
+
+		const allScenarios = testScenarioLoader.loadAllTestScenarios(config.get('testScenariosPath'));
+
+		await testScenarioRunner.runManyTestScenarios(allScenarios);
+	}
+}

packages/benchmark/src/config/config.ts

@@ -0,0 +1,50 @@
+import convict from 'convict';
+import dotenv from 'dotenv';
+
+dotenv.config();
+
+const configSchema = {
+	testScenariosPath: {
+		doc: 'The path to the test scenarios',
+		format: String,
+		default: 'testScenarios',
+	},
+	n8n: {
+		baseUrl: {
+			doc: 'The base URL for the n8n instance',
+			format: String,
+			default: 'http://localhost:5678',
+			env: 'N8N_BASE_URL',
+		},
+		user: {
+			email: {
+				doc: 'The email address of the n8n user',
+				format: String,
+				default: '[email protected]',
+				env: 'N8N_USER_EMAIL',
+			},
+			password: {
+				doc: 'The password of the n8n user',
+				format: String,
+				default: 'VerySecret!123',
+				env: 'N8N_USER_PASSWORD',
+			},
+		},
+	},
+	k6ExecutablePath: {
+		doc: 'The path to the k6 binary',
+		format: String,
+		default: 'k6',
+		env: 'K6_PATH',
+	},
+};
+
+export type Config = ReturnType<typeof loadConfig>;
+
+export function loadConfig() {
+	const config = convict(configSchema);
+
+	config.validate({ allowed: 'strict' });
+
+	return config;
+}

packages/benchmark/src/n8nApiClient/authenticatedN8nApiClient.ts

@@ -0,0 +1,68 @@
+import { strict as assert } from 'node:assert';
+import { N8nApiClient } from './n8nApiClient';
+import { AxiosRequestConfig } from 'axios';
+
+export class AuthenticatedN8nApiClient extends N8nApiClient {
+	constructor(
+		apiBaseUrl: string,
+		private readonly authCookie: string,
+	) {
+		super(apiBaseUrl);
+	}
+
+	static async createUsingUsernameAndPassword(
+		apiClient: N8nApiClient,
+		loginDetails: {
+			email: string;
+			password: string;
+		},
+	) {
+		const response = await apiClient.restApiRequest('/login', {
+			method: 'POST',
+			data: loginDetails,
+		});
+
+		const cookieHeader = response.headers['set-cookie'];
+		const authCookie = Array.isArray(cookieHeader) ? cookieHeader.join('; ') : cookieHeader;
+		assert(authCookie);
+
+		return new AuthenticatedN8nApiClient(apiClient.apiBaseUrl, authCookie);
+	}
+
+	async get<T>(endpoint: string) {
+		return await this.authenticatedRequest<T>(endpoint, {
+			method: 'GET',
+		});
+	}
+
+	async post<T>(endpoint: string, data: unknown) {
+		return await this.authenticatedRequest<T>(endpoint, {
+			method: 'POST',
+			data,
+		});
+	}
+
+	async patch<T>(endpoint: string, data: unknown) {
+		return await this.authenticatedRequest<T>(endpoint, {
+			method: 'PATCH',
+			data,
+		});
+	}
+
+	async delete<T>(endpoint: string, data?: unknown) {
+		return await this.authenticatedRequest<T>(endpoint, {
+			method: 'DELETE',
+			data,
+		});
+	}
+
+	protected async authenticatedRequest<T>(endpoint: string, init: Omit<AxiosRequestConfig, 'url'>) {
+		return await this.restApiRequest<T>(endpoint, {
+			...init,
+			headers: {
+				...init.headers,
+				cookie: this.authCookie,
+			},
+		});
+	}
+}