Merge branch 'master' into pay-2096-typeerror-cannot-read-properties-…

…of-undefined-reading
n8n-io · Oct 23, 2024 · 2401a2f · 2401a2f
2 parents 9c2c8fe + 5d0e0e0
commit 2401a2f
Show file tree

Hide file tree

Showing 43 changed files with 397 additions and 245 deletions.
diff --git a/cypress/e2e/17-sharing.cy.ts b/cypress/e2e/17-sharing.cy.ts
@@ -226,6 +226,7 @@ describe('Sharing', { disableAutoLogin: true }, () => {
 			.should('have.length', 1)
 			.click();
 		credentialsModal.getters.saveButton().click();
+		credentialsModal.getters.saveButton().should('have.text', 'Saved');
 		credentialsModal.actions.close();
 
 		projects.getProjectTabWorkflows().click();
@@ -252,6 +253,7 @@ describe('Sharing', { disableAutoLogin: true }, () => {
 		credentialsModal.getters.usersSelect().click();
 		getVisibleSelect().find('li').should('have.length', 4).first().click();
 		credentialsModal.getters.saveButton().click();
+		credentialsModal.getters.saveButton().should('have.text', 'Saved');
 		credentialsModal.actions.close();
 
 		credentialsPage.getters

diff --git a/cypress/e2e/39-projects.cy.ts b/cypress/e2e/39-projects.cy.ts
@@ -15,7 +15,12 @@ import {
 	NDV,
 	MainSidebar,
 } from '../pages';
-import { getVisibleDropdown, getVisibleModalOverlay, getVisibleSelect } from '../utils';
+import {
+	getVisibleDropdown,
+	getVisibleModalOverlay,
+	getVisibleSelect,
+	getVisiblePopper,
+} from '../utils';
 
 const workflowsPage = new WorkflowsPage();
 const workflowPage = new WorkflowPage();
@@ -579,6 +584,9 @@ describe('Projects', { disableAutoLogin: true }, () => {
 			credentialsPage.getters.credentialCardActions('Credential in Project 1').click();
 			credentialsPage.getters.credentialMoveButton().click();
 
+			// wait for all poppers to be gone
+			getVisiblePopper().should('have.length', 0);
+
 			projects
 				.getResourceMoveModal()
 				.should('be.visible')
@@ -602,6 +610,9 @@ describe('Projects', { disableAutoLogin: true }, () => {
 			credentialsPage.getters.credentialCardActions('Credential in Project 1').click();
 			credentialsPage.getters.credentialMoveButton().click();
 
+			// wait for all poppers to be gone
+			getVisiblePopper().should('have.length', 0);
+
 			projects
 				.getResourceMoveModal()
 				.should('be.visible')
@@ -624,6 +635,9 @@ describe('Projects', { disableAutoLogin: true }, () => {
 			credentialsPage.getters.credentialCardActions('Credential in Project 1').click();
 			credentialsPage.getters.credentialMoveButton().click();
 
+			// wait for all poppers to be gone
+			getVisiblePopper().should('have.length', 0);
+
 			projects
 				.getResourceMoveModal()
 				.should('be.visible')
@@ -637,6 +651,9 @@ describe('Projects', { disableAutoLogin: true }, () => {
 				.click();
 			projects.getResourceMoveModal().find('button:contains("Move credential")').click();
 
+			// wait for all poppers to be gone
+			getVisiblePopper().should('have.length', 0);
+
 			credentialsPage.getters
 				.credentialCards()
 				.should('have.length', 3)

diff --git a/docker/images/n8n/n8n-task-runners.json b/docker/images/n8n/n8n-task-runners.json
@@ -9,6 +9,7 @@
 				"PATH",
 				"N8N_RUNNERS_GRANT_TOKEN",
 				"N8N_RUNNERS_N8N_URI",
+				"N8N_RUNNERS_MAX_PAYLOAD",
 				"NODE_FUNCTION_ALLOW_BUILTIN",
 				"NODE_FUNCTION_ALLOW_EXTERNAL"
 			],

diff --git a/packages/@n8n/benchmark/README.md b/packages/@n8n/benchmark/README.md
@@ -1,6 +1,6 @@
 # n8n benchmarking tool
 
-Tool for executing benchmarks against an n8n instance. The tool consists of these components:
+Tool for executing benchmarks against an n8n instance.
 
 ## Directory structure
 
@@ -12,6 +12,39 @@ packages/@n8n/benchmark
 ├── scripts          Orchestration scripts
 ```
 
+## Benchmarking an existing n8n instance
+
+The easiest way to run the existing benchmark scenarios is to use the benchmark docker image:
+
+```sh
+docker pull ghcr.io/n8n-io/n8n-benchmark:latest
+# Print the help to list all available flags
+docker run ghcr.io/n8n-io/n8n-benchmark:latest run --help
+# Run all available benchmark scenarios for 1 minute with 5 concurrent requests
+docker run ghcr.io/n8n-io/n8n-benchmark:latest run \
+	--n8nBaseUrl=https://instance.url \
+	[email protected] \
+	--n8nUserPassword=InstanceOwnerPassword \
+	--vus=5 \
+	--duration=1m \
+	--scenarioFilter SingleWebhook
+```
+
+### Using custom scenarios with the Docker image
+
+It is also possible to create your own [benchmark scenarios](#benchmark-scenarios) and load them using the `--testScenariosPath` flag:
+
+```sh
+# Assuming your scenarios are located in `./scenarios`, mount them into `/scenarios` in the container
+docker run -v ./scenarios:/scenarios ghcr.io/n8n-io/n8n-benchmark:latest run \
+	--n8nBaseUrl=https://instance.url \
+	[email protected] \
+	--n8nUserPassword=InstanceOwnerPassword \
+	--vus=5 \
+	--duration=1m \
+	--testScenariosPath=/scenarios
+```
+
 ## Running the entire benchmark suite
 
 The benchmark suite consists of [benchmark scenarios](#benchmark-scenarios) and different [n8n setups](#n8n-setups).

diff --git a/packages/@n8n/benchmark/src/commands/run.ts b/packages/@n8n/benchmark/src/commands/run.ts
@@ -13,6 +13,10 @@ export default class RunCommand extends Command {
 
 	static flags = {
 		testScenariosPath,
+		scenarioFilter: Flags.string({
+			char: 'f',
+			description: 'Filter scenarios by name',
+		}),
 		scenarioNamePrefix: Flags.string({
 			description: 'Prefix for the scenario name',
 			default: 'Unnamed',
@@ -95,7 +99,7 @@ export default class RunCommand extends Command {
 			flags.scenarioNamePrefix,
 		);
 
-		const allScenarios = scenarioLoader.loadAll(flags.testScenariosPath);
+		const allScenarios = scenarioLoader.loadAll(flags.testScenariosPath, flags.scenarioFilter);
 
 		await scenarioRunner.runManyScenarios(allScenarios);
 	}

diff --git a/packages/@n8n/benchmark/src/scenario/scenario-loader.ts b/packages/@n8n/benchmark/src/scenario/scenario-loader.ts
@@ -8,7 +8,7 @@ export class ScenarioLoader {
 	/**
 	 * Loads all scenarios from the given path
 	 */
-	loadAll(pathToScenarios: string): Scenario[] {
+	loadAll(pathToScenarios: string, filter?: string): Scenario[] {
 		pathToScenarios = path.resolve(pathToScenarios);
 		const scenarioFolders = fs
 			.readdirSync(pathToScenarios, { withFileTypes: true })
@@ -18,6 +18,9 @@ export class ScenarioLoader {
 		const scenarios: Scenario[] = [];
 
 		for (const folder of scenarioFolders) {
+			if (filter && !folder.toLowerCase().includes(filter.toLowerCase())) {
+				continue;
+			}
 			const scenarioPath = path.join(pathToScenarios, folder);
 			const manifestFileName = `${folder}.manifest.json`;
 			const scenarioManifestPath = path.join(pathToScenarios, folder, manifestFileName);

diff --git a/packages/@n8n/config/src/configs/runners.config.ts b/packages/@n8n/config/src/configs/runners.config.ts
@@ -32,6 +32,10 @@ export class TaskRunnersConfig {
 	@Env('N8N_RUNNERS_SERVER_LISTEN_ADDRESS')
 	listenAddress: string = '127.0.0.1';
 
+	/** Maximum size of a payload sent to the runner in bytes, Default 1G */
+	@Env('N8N_RUNNERS_MAX_PAYLOAD')
+	maxPayload: number = 1024 * 1024 * 1024;
+
 	@Env('N8N_RUNNERS_LAUNCHER_PATH')
 	launcherPath: string = '';
 

diff --git a/packages/@n8n/config/test/config.test.ts b/packages/@n8n/config/test/config.test.ts
@@ -227,6 +227,7 @@ describe('GlobalConfig', () => {
 			path: '/runners',
 			authToken: '',
 			listenAddress: '127.0.0.1',
+			maxPayload: 1024 * 1024 * 1024,
 			port: 5679,
 			launcherPath: '',
 			launcherRunner: 'javascript',

diff --git a/packages/@n8n/task-runner/src/js-task-runner/__tests__/js-task-runner.test.ts b/packages/@n8n/task-runner/src/js-task-runner/__tests__/js-task-runner.test.ts
@@ -189,6 +189,25 @@ describe('JsTaskRunner', () => {
 				['{ wf: $workflow }', { wf: { active: true, id: '1', name: 'Test Workflow' } }],
 				['$vars', { var: 'value' }],
 			],
+			'Node.js internal functions': [
+				['typeof Function', 'function'],
+				['typeof eval', 'function'],
+				['typeof setTimeout', 'function'],
+				['typeof setInterval', 'function'],
+				['typeof setImmediate', 'function'],
+				['typeof clearTimeout', 'function'],
+				['typeof clearInterval', 'function'],
+				['typeof clearImmediate', 'function'],
+			],
+			'JS built-ins': [
+				['typeof btoa', 'function'],
+				['typeof atob', 'function'],
+				['typeof TextDecoder', 'function'],
+				['typeof TextDecoderStream', 'function'],
+				['typeof TextEncoder', 'function'],
+				['typeof TextEncoderStream', 'function'],
+				['typeof FormData', 'function'],
+			],
 		};
 
 		for (const [groupName, tests] of Object.entries(testGroups)) {

diff --git a/packages/@n8n/task-runner/src/js-task-runner/js-task-runner.ts b/packages/@n8n/task-runner/src/js-task-runner/js-task-runner.ts
@@ -169,6 +169,11 @@ export class JsTaskRunner extends TaskRunner {
 			// Missing JS natives
 			btoa,
 			atob,
+			TextDecoder,
+			TextDecoderStream,
+			TextEncoder,
+			TextEncoderStream,
+			FormData,
 		};
 	}
 

diff --git a/packages/@n8n/task-runner/src/task-runner.ts b/packages/@n8n/task-runner/src/task-runner.ts
@@ -42,6 +42,8 @@ export interface RPCCallObject {
 const VALID_TIME_MS = 1000;
 const VALID_EXTRA_MS = 100;
 
+const DEFAULT_MAX_PAYLOAD_SIZE = 1024 * 1024 * 1024;
+
 export abstract class TaskRunner {
 	id: string = nanoid();
 
@@ -74,6 +76,9 @@ export abstract class TaskRunner {
 			headers: {
 				authorization: `Bearer ${grantToken}`,
 			},
+			maxPayload: process.env.N8N_RUNNERS_MAX_PAYLOAD
+				? parseInt(process.env.N8N_RUNNERS_MAX_PAYLOAD)
+				: DEFAULT_MAX_PAYLOAD_SIZE,
 		});
 		this.ws.addEventListener('message', this.receiveMessage);
 		this.ws.addEventListener('close', this.stopTaskOffers);

diff --git a/packages/cli/src/runners/runner-ws-server.ts b/packages/cli/src/runners/runner-ws-server.ts
@@ -1,16 +1,7 @@
-import { GlobalConfig } from '@n8n/config';
-import type { Application } from 'express';
-import { ServerResponse, type Server } from 'http';
-import { ApplicationError } from 'n8n-workflow';
-import type { Socket } from 'net';
-import Container, { Service } from 'typedi';
-import { parse as parseUrl } from 'url';
-import { Server as WSServer } from 'ws';
+import { Service } from 'typedi';
 import type WebSocket from 'ws';
 
 import { Logger } from '@/logging/logger.service';
-import { send } from '@/response-helper';
-import { TaskRunnerAuthController } from '@/runners/auth/task-runner-auth.controller';
 
 import type {
 	RunnerMessage,
@@ -24,24 +15,6 @@ function heartbeat(this: WebSocket) {
 	this.isAlive = true;
 }
 
-function getEndpointBasePath(restEndpoint: string) {
-	const globalConfig = Container.get(GlobalConfig);
-
-	let path = globalConfig.taskRunners.path;
-	if (path.startsWith('/')) {
-		path = path.slice(1);
-	}
-	if (path.endsWith('/')) {
-		path = path.slice(-1);
-	}
-
-	return `/${restEndpoint}/${path}`;
-}
-
-function getWsEndpoint(restEndpoint: string) {
-	return `${getEndpointBasePath(restEndpoint)}/_ws`;
-}
-
 @Service()
 export class TaskRunnerService {
 	runnerConnections: Map<TaskRunner['id'], WebSocket> = new Map();
@@ -127,61 +100,3 @@ export class TaskRunnerService {
 		this.add(req.query.id, req.ws);
 	}
 }
-
-// Checks for upgrade requests on the runners path and upgrades the connection
-// then, passes the request back to the app to handle the routing
-export const setupRunnerServer = (restEndpoint: string, server: Server, app: Application) => {
-	const globalConfig = Container.get(GlobalConfig);
-	const { authToken } = globalConfig.taskRunners;
-
-	if (!authToken) {
-		throw new ApplicationError(
-			'Authentication token must be configured when task runners are enabled. Use N8N_RUNNERS_AUTH_TOKEN environment variable to set it.',
-		);
-	}
-
-	const endpoint = getWsEndpoint(restEndpoint);
-	const wsServer = new WSServer({ noServer: true });
-	server.on('upgrade', (request: TaskRunnerServerInitRequest, socket: Socket, head) => {
-		if (parseUrl(request.url).pathname !== endpoint) {
-			// We can't close the connection here since the Push connections
-			// are using the same HTTP server and upgrade requests and this
-			// gets triggered for both
-			return;
-		}
-
-		wsServer.handleUpgrade(request, socket, head, (ws) => {
-			request.ws = ws;
-
-			const response = new ServerResponse(request);
-			response.writeHead = (statusCode) => {
-				if (statusCode > 200) ws.close();
-				return response;
-			};
-
-			// @ts-expect-error Hidden API?
-			// eslint-disable-next-line @typescript-eslint/no-unsafe-call
-			app.handle(request, response);
-		});
-	});
-};
-
-export const setupRunnerHandler = (restEndpoint: string, app: Application) => {
-	const wsEndpoint = getWsEndpoint(restEndpoint);
-	const authEndpoint = `${getEndpointBasePath(restEndpoint)}/auth`;
-
-	const taskRunnerAuthController = Container.get(TaskRunnerAuthController);
-	const taskRunnerService = Container.get(TaskRunnerService);
-	app.use(
-		wsEndpoint,
-		// eslint-disable-next-line @typescript-eslint/unbound-method
-		taskRunnerAuthController.authMiddleware,
-		(req: TaskRunnerServerInitRequest, res: TaskRunnerServerInitResponse) =>
-			taskRunnerService.handleRequest(req, res),
-	);
-
-	app.post(
-		authEndpoint,
-		send(async (req) => await taskRunnerAuthController.createGrantToken(req)),
-	);
-};
diff --git a/packages/cli/src/runners/task-runner-process.ts b/packages/cli/src/runners/task-runner-process.ts
@@ -72,6 +72,7 @@ export class TaskRunnerProcess {
 				PATH: process.env.PATH,
 				N8N_RUNNERS_GRANT_TOKEN: grantToken,
 				N8N_RUNNERS_N8N_URI: n8nUri,
+				N8N_RUNNERS_MAX_PAYLOAD: this.runnerConfig.maxPayload.toString(),
 				NODE_FUNCTION_ALLOW_BUILTIN: process.env.NODE_FUNCTION_ALLOW_BUILTIN,
 				NODE_FUNCTION_ALLOW_EXTERNAL: process.env.NODE_FUNCTION_ALLOW_EXTERNAL,
 			},
@@ -84,6 +85,7 @@ export class TaskRunnerProcess {
 				PATH: process.env.PATH,
 				N8N_RUNNERS_GRANT_TOKEN: grantToken,
 				N8N_RUNNERS_N8N_URI: n8nUri,
+				N8N_RUNNERS_MAX_PAYLOAD: this.runnerConfig.maxPayload.toString(),
 				NODE_FUNCTION_ALLOW_BUILTIN: process.env.NODE_FUNCTION_ALLOW_BUILTIN,
 				NODE_FUNCTION_ALLOW_EXTERNAL: process.env.NODE_FUNCTION_ALLOW_EXTERNAL,
 				// For debug logging if enabled

diff --git a/packages/cli/src/runners/task-runner-server.ts b/packages/cli/src/runners/task-runner-server.ts
@@ -114,7 +114,10 @@ export class TaskRunnerServer {
 		a.ok(authToken);
 		a.ok(this.server);
 
-		this.wsServer = new WSServer({ noServer: true });
+		this.wsServer = new WSServer({
+			noServer: true,
+			maxPayload: this.globalConfig.taskRunners.maxPayload,
+		});
 		this.server.on('upgrade', this.handleUpgradeRequest);
 	}