feat: Report benchmark results (no-changelog) (#10534)

.github/workflows/benchmark-nightly.yml

@@ -23,6 +23,7 @@ env:
   ARM_CLIENT_ID: ${{ secrets.BENCHMARK_ARM_CLIENT_ID }}
   ARM_SUBSCRIPTION_ID: ${{ secrets.BENCHMARK_ARM_SUBSCRIPTION_ID }}
   ARM_TENANT_ID: ${{ secrets.BENCHMARK_ARM_TENANT_ID }}
+  K6_API_TOKEN: ${{ secrets.K6_API_TOKEN }}
 
 permissions:
   id-token: write

packages/@n8n/benchmark/.eslintrc.js

@@ -12,7 +12,7 @@ module.exports = {
 		project: './tsconfig.json',
 	},
 
-	ignorePatterns: ['scenarios/**'],
+	ignorePatterns: ['scenarios/**', 'scripts/**'],
 
 	rules: {
 		'n8n-local-rules/no-plain-errors': 'off',

packages/@n8n/benchmark/scripts/runInCloud.mjs

@@ -15,7 +15,7 @@
 // @ts-check
 import fs from 'fs';
 import minimist from 'minimist';
-import { $, sleep, tmpdir, which } from 'zx';
+import { $, sleep, which } from 'zx';
 import path from 'path';
 import { SshClient } from './sshClient.mjs';
 import { TerraformClient } from './terraformClient.mjs';
@@ -124,6 +124,7 @@ function readAvailableN8nSetups() {
  * @property {string} n8nSetupToUse
  * @property {string} n8nTag
  * @property {string} benchmarkTag
+ * @property {string} [k6ApiToken]
  *
  * @returns {Promise<Config>}
  */
@@ -136,12 +137,14 @@ async function parseAndValidateConfig() {
 	const isVerbose = args.debug || false;
 	const n8nTag = args.n8nTag || process.env.N8N_DOCKER_TAG || 'latest';
 	const benchmarkTag = args.benchmarkTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
+	const k6ApiToken = args.k6ApiToken || process.env.K6_API_TOKEN || undefined;
 
 	return {
 		isVerbose,
 		n8nSetupToUse,
 		n8nTag,
 		benchmarkTag,
+		k6ApiToken,
 	};
 }
 
@@ -177,6 +180,9 @@ function printUsage() {
 	console.log('  --debug              Enable verbose output');
 	console.log('  --n8nTag             Docker tag for n8n image. Default is latest');
 	console.log('  --benchmarkTag       Docker tag for benchmark cli image. Default is latest');
+	console.log(
+		'  --k6ApiToken         API token for k6 cloud. Default is read from K6_API_TOKEN env var. If omitted, k6 cloud will not be used.',
+	);
 	console.log('');
 	console.log('Available setups:');
 	console.log(`  ${availableSetups.join(', ')}`);

packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite/docker-compose.yml

@@ -14,3 +14,4 @@ services:
       - n8n
     environment:
       - N8N_BASE_URL=http://n8n:5678
+      - K6_API_TOKEN=${K6_API_TOKEN}

packages/@n8n/benchmark/scripts/runOnVm/runOnVm.mjs

@@ -21,13 +21,15 @@ async function main() {
 	const composeFilePath = path.join(__dirname, 'n8nSetups', n8nSetupToUse);
 	const n8nTag = argv.n8nDockerTag || process.env.N8N_DOCKER_TAG || 'latest';
 	const benchmarkTag = argv.benchmarkDockerTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
+	const k6ApiToken = argv.k6ApiToken || process.env.K6_API_TOKEN || undefined;
 
 	const $$ = $({
 		cwd: composeFilePath,
 		verbose: true,
 		env: {
 			N8N_VERSION: n8nTag,
 			BENCHMARK_VERSION: benchmarkTag,
+			K6_API_TOKEN: k6ApiToken,
 		},
 	});
 

packages/@n8n/benchmark/src/commands/run.ts

@@ -25,7 +25,11 @@ export default class RunCommand extends Command {
 		const scenarioRunner = new ScenarioRunner(
 			new N8nApiClient(config.get('n8n.baseUrl')),
 			new ScenarioDataFileLoader(),
-			new K6Executor(config.get('k6ExecutablePath'), config.get('n8n.baseUrl')),
+			new K6Executor({
+				k6ExecutablePath: config.get('k6.executablePath'),
+				k6ApiToken: config.get('k6.apiToken'),
+				n8nApiBaseUrl: config.get('n8n.baseUrl'),
+			}),
 			{
 				email: config.get('n8n.user.email'),
 				password: config.get('n8n.user.password'),

packages/@n8n/benchmark/src/config/config.ts

@@ -31,11 +31,19 @@ const configSchema = {
 			},
 		},
 	},
-	k6ExecutablePath: {
-		doc: 'The path to the k6 binary',
-		format: String,
-		default: 'k6',
-		env: 'K6_PATH',
+	k6: {
+		executablePath: {
+			doc: 'The path to the k6 binary',
+			format: String,
+			default: 'k6',
+			env: 'K6_PATH',
+		},
+		apiToken: {
+			doc: 'The API token for k6 cloud',
+			format: String,
+			default: undefined,
+			env: 'K6_API_TOKEN',
+		},
 	},
 };
 

packages/@n8n/benchmark/src/testExecution/k6Executor.ts

@@ -1,38 +1,103 @@
-import { $, which } from 'zx';
+import fs from 'fs';
+import path from 'path';
+import { $, which, tmpfile } from 'zx';
 import type { Scenario } from '@/types/scenario';
 
+export type K6ExecutorOpts = {
+	k6ExecutablePath: string;
+	k6ApiToken?: string;
+	n8nApiBaseUrl: string;
+};
+
+/**
+ * Flag for the k6 CLI.
+ * @example ['--duration', '1m']
+ * @example ['--quiet']
+ */
+type K6CliFlag = [string] | [string, string];
+
 /**
  * Executes test scenarios using k6
  */
 export class K6Executor {
-	constructor(
-		private readonly k6ExecutablePath: string,
-		private readonly n8nApiBaseUrl: string,
-	) {}
+	/**
+	 * This script is dynamically injected into the k6 test script to generate
+	 * a summary report of the test execution.
+	 */
+	private readonly handleSummaryScript = `
+import { textSummary } from 'https://jslib.k6.io/k6-summary/0.0.2/index.js';
+export function handleSummary(data) {
+	return {
+		stdout: textSummary(data),
+		'{{scenarioName}}.summary.json': JSON.stringify(data),
+	};
+}
+`;
+
+	constructor(private readonly opts: K6ExecutorOpts) {}
 
 	async executeTestScenario(scenario: Scenario) {
-		// For 1 min with 5 virtual users
-		const stage = '1m:5';
+		const augmentedTestScriptPath = this.augmentSummaryScript(scenario);
+		const runDirPath = path.dirname(augmentedTestScriptPath);
+
+		const flags: K6CliFlag[] = [['--quiet'], ['--duration', '1m'], ['--vus', '5']];
+
+		if (this.opts.k6ApiToken) {
+			flags.push(['--out', 'cloud']);
+		}
+
+		const flattedFlags = flags.flat(2);
 
 		const k6ExecutablePath = await this.resolveK6ExecutablePath();
 
 		const processPromise = $({
-			cwd: scenario.scenarioDirPath,
+			cwd: runDirPath,
 			env: {
-				API_BASE_URL: this.n8nApiBaseUrl,
+				API_BASE_URL: this.opts.n8nApiBaseUrl,
+				K6_CLOUD_TOKEN: this.opts.k6ApiToken,
 			},
-		})`${k6ExecutablePath} run --quiet --stage ${stage} ${scenario.scriptPath}`;
+		})`${k6ExecutablePath} run ${flattedFlags} ${augmentedTestScriptPath}`;
 
 		for await (const chunk of processPromise.stdout) {
 			console.log((chunk as Buffer).toString());
 		}
+
+		this.loadEndOfTestSummary(runDirPath, scenario.name);
+	}
+
+	/**
+	 * Augments the test script with a summary script
+	 *
+	 * @returns Absolute path to the augmented test script
+	 */
+	private augmentSummaryScript(scenario: Scenario) {
+		const fullTestScriptPath = path.join(scenario.scenarioDirPath, scenario.scriptPath);
+		const testScript = fs.readFileSync(fullTestScriptPath, 'utf8');
+		const summaryScript = this.handleSummaryScript.replace('{{scenarioName}}', scenario.name);
+
+		const augmentedTestScript = `${testScript}\n\n${summaryScript}`;
+
+		const tempFilePath = tmpfile(`${scenario.name}.ts`, augmentedTestScript);
+
+		return tempFilePath;
+	}
+
+	private loadEndOfTestSummary(dir: string, scenarioName: string): K6EndOfTestSummary {
+		const summaryReportPath = path.join(dir, `${scenarioName}.summary.json`);
+		const summaryReport = fs.readFileSync(summaryReportPath, 'utf8');
+
+		try {
+			return JSON.parse(summaryReport);
+		} catch (error) {
+			throw new Error(`Failed to parse the summary report at ${summaryReportPath}`);
+		}
 	}
 
 	/**
 	 * @returns Resolved path to the k6 executable
 	 */
 	private async resolveK6ExecutablePath(): Promise<string> {
-		const k6ExecutablePath = await which(this.k6ExecutablePath, { nothrow: true });
+		const k6ExecutablePath = await which(this.opts.k6ExecutablePath, { nothrow: true });
 		if (!k6ExecutablePath) {
 			throw new Error(
 				'Could not find k6 executable based on your `PATH`. Please ensure k6 is available in your system and add it to your `PATH` or specify the path to the k6 executable using the `K6_PATH` environment variable.',