detect crashes and report them to Sentry

packages/cli/commands/start.ts

@@ -35,6 +35,7 @@ import {
 import { getLogger } from '../src/Logger';
 import { getAllInstalledPackages } from '../src/CommunityNodes/packageModel';
 import { initErrorHandling } from '../src/ErrorReporting';
+import * as CrashJournal from '../src/CrashJournal';
 
 // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-var-requires
 const open = require('open');
@@ -84,14 +85,20 @@ export class Start extends Command {
 	}
 
 	/**
-	 * Stoppes the n8n in a graceful way.
+	 * Stop n8n in a graceful way.
 	 * Make for example sure that all the webhooks from third party services
 	 * get removed.
 	 */
 	// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
 	static async stopProcess() {
 		getLogger().info('\nStopping n8n...');
 
+		const exit = () => {
+			CrashJournal.cleanup().finally(() => {
+				process.exit(processExitCode);
+			});
+		};
+
 		try {
 			// Stop with trying to activate workflows that could not be activated
 			activeWorkflowRunner?.removeAllQueuedWorkflowActivations();
@@ -103,7 +110,7 @@ export class Start extends Command {
 				// In case that something goes wrong with shutdown we
 				// kill after max. 30 seconds no matter what
 				console.log(`process exited after 30s`);
-				process.exit(processExitCode);
+				exit();
 			}, 30000);
 
 			await InternalHooksManager.getInstance().onN8nStop();
@@ -144,26 +151,27 @@ export class Start extends Command {
 			console.error('There was an error shutting down n8n.', error);
 		}
 
-		process.exit(processExitCode);
+		exit();
 	}
 
 	async run() {
 		// Make sure that n8n shuts down gracefully if possible
 		process.once('SIGTERM', Start.stopProcess);
 		process.once('SIGINT', Start.stopProcess);
 
+		const logger = getLogger();
+		LoggerProxy.init(logger);
+		logger.info('Initializing n8n process');
+
 		initErrorHandling();
+		await CrashJournal.init();
 
 		// eslint-disable-next-line @typescript-eslint/no-shadow
 		const { flags } = this.parse(Start);
 
 		// Wrap that the process does not close but we can still use async
 		await (async () => {
 			try {
-				const logger = getLogger();
-				LoggerProxy.init(logger);
-				logger.info('Initializing n8n process');
-
 				// Start directly with the init of the database to improve startup time
 				const startDbInitPromise = Db.init().catch((error: Error) => {
 					logger.error(`There was an error initializing DB: "${error.message}"`);

packages/cli/commands/webhook.ts

@@ -27,9 +27,10 @@ import {
 
 import { getLogger } from '../src/Logger';
 import { initErrorHandling } from '../src/ErrorReporting';
+import * as CrashJournal from '../src/CrashJournal';
 
 let activeWorkflowRunner: ActiveWorkflowRunner.ActiveWorkflowRunner | undefined;
-let processExistCode = 0;
+let processExitCode = 0;
 
 export class Webhook extends Command {
 	static description = 'Starts n8n webhook process. Intercepts only production URLs.';
@@ -41,22 +42,28 @@ export class Webhook extends Command {
 	};
 
 	/**
-	 * Stops the n8n in a graceful way.
+	 * Stops n8n in a graceful way.
 	 * Make for example sure that all the webhooks from third party services
 	 * get removed.
 	 */
 	// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
 	static async stopProcess() {
 		LoggerProxy.info(`\nStopping n8n...`);
 
+		const exit = () => {
+			CrashJournal.cleanup().finally(() => {
+				process.exit(processExitCode);
+			});
+		};
+
 		try {
 			const externalHooks = ExternalHooks();
 			await externalHooks.run('n8n.stop', []);
 
 			setTimeout(() => {
 				// In case that something goes wrong with shutdown we
 				// kill after max. 30 seconds no matter what
-				process.exit(processExistCode);
+				exit();
 			}, 30000);
 
 			// Wait for active workflow executions to finish
@@ -78,7 +85,7 @@ export class Webhook extends Command {
 			LoggerProxy.error('There was an error shutting down n8n.', error);
 		}
 
-		process.exit(processExistCode);
+		exit();
 	}
 
 	// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
@@ -91,6 +98,7 @@ export class Webhook extends Command {
 		process.once('SIGINT', Webhook.stopProcess);
 
 		initErrorHandling();
+		await CrashJournal.init();
 
 		// eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/no-shadow
 		const { flags } = this.parse(Webhook);
@@ -119,7 +127,7 @@ export class Webhook extends Command {
 					// eslint-disable-next-line @typescript-eslint/restrict-template-expressions, @typescript-eslint/no-unsafe-member-access
 					logger.error(`There was an error initializing DB: "${error.message}"`);
 
-					processExistCode = 1;
+					processExitCode = 1;
 					// @ts-ignore
 					process.emit('SIGINT');
 					process.exit(1);
@@ -231,7 +239,7 @@ export class Webhook extends Command {
 				// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
 				logger.error(`Webhook process cannot continue. "${error.message}"`);
 
-				processExistCode = 1;
+				processExitCode = 1;
 				// @ts-ignore
 				process.emit('SIGINT');
 				process.exit(1);

packages/cli/commands/worker.ts

@@ -69,11 +69,11 @@ export class Worker extends Command {
 
 	static jobQueue: Queue.JobQueue;
 
-	static processExistCode = 0;
+	static processExitCode = 0;
 	// static activeExecutions = ActiveExecutions.getInstance();
 
 	/**
-	 * Stoppes the n8n in a graceful way.
+	 * Stop n8n in a graceful way.
 	 * Make for example sure that all the webhooks from third party services
 	 * get removed.
 	 */
@@ -95,7 +95,7 @@ export class Worker extends Command {
 			setTimeout(() => {
 				// In case that something goes wrong with shutdown we
 				// kill after max. 30 seconds no matter what
-				process.exit(Worker.processExistCode);
+				process.exit(Worker.processExitCode);
 			}, maxStopTime);
 
 			// Wait for active workflow executions to finish
@@ -116,7 +116,7 @@ export class Worker extends Command {
 			LoggerProxy.error('There was an error shutting down n8n.', error);
 		}
 
-		process.exit(Worker.processExistCode);
+		process.exit(Worker.processExitCode);
 	}
 
 	async runJob(job: Queue.Job, nodeTypes: INodeTypes): Promise<Queue.JobResponse> {
@@ -275,7 +275,7 @@ export class Worker extends Command {
 				const startDbInitPromise = Db.init().catch((error) => {
 					logger.error(`There was an error initializing DB: "${error.message}"`);
 
-					Worker.processExistCode = 1;
+					Worker.processExitCode = 1;
 					// @ts-ignore
 					process.emit('SIGINT');
 					process.exit(1);
@@ -446,7 +446,7 @@ export class Worker extends Command {
 			} catch (error) {
 				logger.error(`Worker process cannot continue. "${error.message}"`);
 
-				Worker.processExistCode = 1;
+				Worker.processExitCode = 1;
 				// @ts-ignore
 				process.emit('SIGINT');
 				process.exit(1);

packages/cli/src/CrashJournal.ts

@@ -0,0 +1,33 @@
+import { existsSync } from 'fs';
+import { mkdir, utimes, open, rm } from 'fs/promises';
+import { join, dirname } from 'path';
+import { UserSettings } from 'n8n-core';
+import { ErrorReporterProxy as ErrorReporter, LoggerProxy, sleep } from 'n8n-workflow';
+
+export const touchFile = async (filePath: string): Promise<void> => {
+	await mkdir(dirname(filePath), { recursive: true });
+	const time = new Date();
+	try {
+		await utimes(filePath, time, time);
+	} catch {
+		const fd = await open(filePath, 'w');
+		await fd.close();
+	}
+};
+
+const journalFile = join(UserSettings.getUserN8nFolderPath(), 'crash.journal');
+
+export const init = async () => {
+	if (existsSync(journalFile)) {
+		// Crash detected
+		ErrorReporter.warn('Last session crashed');
+		LoggerProxy.error('Last session crashed');
+		// add a 10 seconds pause to slow down crash-looping
+		await sleep(10_000);
+	}
+	await touchFile(journalFile);
+};
+
+export const cleanup = async () => {
+	await rm(journalFile, { force: true });
+};