remove pnpm patches. they aren't available in releases

package.json

@@ -95,8 +95,7 @@
       "[email protected]": "patches/[email protected]",
       "[email protected]": "patches/[email protected]",
       "@sentry/[email protected]": "patches/@[email protected]",
-      "[email protected]": "patches/[email protected]",
-      "[email protected]": "patches/[email protected]"
+      "[email protected]": "patches/[email protected]"
     }
   }
 }

packages/cli/src/databases/migrations/sqlite/1652367743993-AddUserSettings.ts

@@ -1,6 +1,8 @@
 import type { MigrationContext, ReversibleMigration } from '@db/types';
 
 export class AddUserSettings1652367743993 implements ReversibleMigration {
+	transaction = false as const;
+
 	async up({ queryRunner, tablePrefix }: MigrationContext) {
 		await queryRunner.query(
 			`CREATE TABLE "temporary_user" ("id" varchar PRIMARY KEY NOT NULL, "email" varchar(255), "firstName" varchar(32), "lastName" varchar(32), "password" varchar, "resetPasswordToken" varchar, "resetPasswordTokenExpiration" integer DEFAULT NULL, "personalizationAnswers" text, "createdAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), "updatedAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), "globalRoleId" integer NOT NULL, "settings" text, CONSTRAINT "FK_${tablePrefix}f0609be844f9200ff4365b1bb3d" FOREIGN KEY ("globalRoleId") REFERENCES "${tablePrefix}role" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION)`,

packages/cli/src/databases/migrations/sqlite/1652905585850-AddAPIKeyColumn.ts

@@ -1,6 +1,8 @@
 import type { MigrationContext, ReversibleMigration } from '@db/types';
 
 export class AddAPIKeyColumn1652905585850 implements ReversibleMigration {
+	transaction = false as const;
+
 	async up({ queryRunner, tablePrefix }: MigrationContext) {
 		await queryRunner.query(
 			`CREATE TABLE "temporary_user" ("id" varchar PRIMARY KEY NOT NULL, "email" varchar(255), "firstName" varchar(32), "lastName" varchar(32), "password" varchar, "resetPasswordToken" varchar, "resetPasswordTokenExpiration" integer DEFAULT NULL, "personalizationAnswers" text, "createdAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), "updatedAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), "globalRoleId" integer NOT NULL, "settings" text, "apiKey" varchar, CONSTRAINT "FK_${tablePrefix}f0609be844f9200ff4365b1bb3d" FOREIGN KEY ("globalRoleId") REFERENCES "${tablePrefix}role" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION)`,

packages/cli/src/databases/migrations/sqlite/1673268682475-DeleteExecutionsWithWorkflows.ts

@@ -1,6 +1,8 @@
 import type { MigrationContext, ReversibleMigration } from '@db/types';
 
 export class DeleteExecutionsWithWorkflows1673268682475 implements ReversibleMigration {
+	transaction = false as const;
+
 	async up({ queryRunner, tablePrefix }: MigrationContext) {
 		const workflowIds = (await queryRunner.query(`
 			SELECT id FROM "${tablePrefix}workflow_entity"

packages/cli/src/databases/migrations/sqlite/1690000000002-MigrateIntegerKeysToString.ts

@@ -1,20 +1,32 @@
+import { statSync } from 'fs';
+import path from 'path';
+import { UserSettings } from 'n8n-core';
 import type { MigrationContext, IrreversibleMigration } from '@db/types';
+import config from '@/config';
 
 export class MigrateIntegerKeysToString1690000000002 implements IrreversibleMigration {
 	pruneBeforeRunning = true;
 
-	async up({ queryRunner, tablePrefix }: MigrationContext) {
+	transaction = false as const;
+
+	async up(context: MigrationContext) {
+		// eslint-disable-next-line @typescript-eslint/no-use-before-define
+		await pruneExecutionsData(context);
+
+		const { queryRunner, tablePrefix } = context;
+
 		await queryRunner.query(`
-CREATE TABLE "${tablePrefix}TMP_workflow_entity" ("id" varchar(36) PRIMARY KEY NOT NULL, "name" varchar(128) NOT NULL, "active" boolean NOT NULL, "nodes" text, "connections" text NOT NULL, "createdAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), "updatedAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), "settings" text, "staticData" text, "pinData" text, "versionId" varchar(36), "triggerCount" integer NOT NULL DEFAULT 0);`);
+			CREATE TABLE "${tablePrefix}TMP_workflow_entity" ("id" varchar(36) PRIMARY KEY NOT NULL, "name" varchar(128) NOT NULL, "active" boolean NOT NULL, "nodes" text, "connections" text NOT NULL, "createdAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), "updatedAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), "settings" text, "staticData" text, "pinData" text, "versionId" varchar(36), "triggerCount" integer NOT NULL DEFAULT 0);`);
 		await queryRunner.query(
 			`INSERT INTO "${tablePrefix}TMP_workflow_entity" (id, name, active, nodes, connections, createdAt, updatedAt, settings, staticData, pinData, triggerCount, versionId) SELECT id, name, active, nodes, connections, createdAt, updatedAt, settings, staticData, pinData, triggerCount, versionId FROM "${tablePrefix}workflow_entity";`,
 		);
 		await queryRunner.query(`DROP TABLE "${tablePrefix}workflow_entity";`);
-		await queryRunner.query(`ALTER TABLE "${tablePrefix}TMP_workflow_entity" RENAME TO "${tablePrefix}workflow_entity";
-`);
+		await queryRunner.query(
+			`ALTER TABLE "${tablePrefix}TMP_workflow_entity" RENAME TO "${tablePrefix}workflow_entity"`,
+		);
 
 		await queryRunner.query(`
-CREATE TABLE "${tablePrefix}TMP_tag_entity" ("id" varchar(36) PRIMARY KEY NOT NULL, "name" varchar(24) NOT NULL, "createdAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), "updatedAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')));`);
+			CREATE TABLE "${tablePrefix}TMP_tag_entity" ("id" varchar(36) PRIMARY KEY NOT NULL, "name" varchar(24) NOT NULL, "createdAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), "updatedAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')));`);
 		await queryRunner.query(
 			`INSERT INTO "${tablePrefix}TMP_tag_entity" SELECT * FROM "${tablePrefix}tag_entity";`,
 		);
@@ -24,7 +36,7 @@ CREATE TABLE "${tablePrefix}TMP_tag_entity" ("id" varchar(36) PRIMARY KEY NOT NU
 		);
 
 		await queryRunner.query(`
-CREATE TABLE "${tablePrefix}TMP_workflows_tags" ("workflowId" varchar(36) NOT NULL, "tagId" integer NOT NULL, CONSTRAINT "FK_${tablePrefix}workflows_tags_workflow_entity" FOREIGN KEY ("workflowId") REFERENCES "${tablePrefix}workflow_entity" ("id") ON DELETE CASCADE ON UPDATE NO ACTION, CONSTRAINT "FK_${tablePrefix}workflows_tags_tag_entity" FOREIGN KEY ("tagId") REFERENCES "${tablePrefix}tag_entity" ("id") ON DELETE CASCADE ON UPDATE NO ACTION, PRIMARY KEY ("workflowId", "tagId"));`);
+			CREATE TABLE "${tablePrefix}TMP_workflows_tags" ("workflowId" varchar(36) NOT NULL, "tagId" integer NOT NULL, CONSTRAINT "FK_${tablePrefix}workflows_tags_workflow_entity" FOREIGN KEY ("workflowId") REFERENCES "${tablePrefix}workflow_entity" ("id") ON DELETE CASCADE ON UPDATE NO ACTION, CONSTRAINT "FK_${tablePrefix}workflows_tags_tag_entity" FOREIGN KEY ("tagId") REFERENCES "${tablePrefix}tag_entity" ("id") ON DELETE CASCADE ON UPDATE NO ACTION, PRIMARY KEY ("workflowId", "tagId"));`);
 		await queryRunner.query(
 			`INSERT INTO "${tablePrefix}TMP_workflows_tags" SELECT * FROM "${tablePrefix}workflows_tags";`,
 		);
@@ -177,3 +189,50 @@ CREATE TABLE "${tablePrefix}TMP_workflows_tags" ("workflowId" varchar(36) NOT NU
 		);
 	}
 }
+
+const DESIRED_DATABASE_FILE_SIZE = 1 * 1024 * 1024 * 1024; // 1 GB
+const migrationsPruningEnabled = process.env.MIGRATIONS_PRUNING_ENABLED === 'true';
+
+function getSqliteDbFileSize(): number {
+	const filename = path.resolve(
+		UserSettings.getUserN8nFolderPath(),
+		config.getEnv('database.sqlite.database'),
+	);
+	const { size } = statSync(filename);
+	return size;
+}
+
+const pruneExecutionsData = async ({ queryRunner, tablePrefix }: MigrationContext) => {
+	if (migrationsPruningEnabled) {
+		const dbFileSize = getSqliteDbFileSize();
+		if (dbFileSize < DESIRED_DATABASE_FILE_SIZE) {
+			console.log(`DB Size not large enough to prune: ${dbFileSize}`);
+			return;
+		}
+
+		console.time('pruningData');
+		const counting = (await queryRunner.query(
+			`select count(id) as rows from "${tablePrefix}execution_entity";`,
+		)) as Array<{ rows: number }>;
+
+		const averageExecutionSize = dbFileSize / counting[0].rows;
+		const numberOfExecutionsToKeep = Math.floor(DESIRED_DATABASE_FILE_SIZE / averageExecutionSize);
+
+		const query = `
+		SELECT id FROM "${tablePrefix}execution_entity"
+		ORDER BY id DESC limit ${numberOfExecutionsToKeep}, 1;
+	`;
+
+		const idToKeep = (await queryRunner.query(query)) as Array<{ id: number }>;
+
+		const removalQuery = `
+		DELETE FROM "${tablePrefix}execution_entity"
+		WHERE id < ${idToKeep[0].id} and status IN ('success');
+	`;
+
+		await queryRunner.query(removalQuery);
+		console.timeEnd('pruningData');
+	} else {
+		console.log('Pruning was requested, but was not enabled');
+	}
+};

packages/cli/src/databases/types.ts

@@ -12,12 +12,13 @@ export interface MigrationContext {
 	migrationName: string;
 }
 
-type MigrationFn = (ctx: MigrationContext) => Promise<void>;
+export type MigrationFn = (ctx: MigrationContext) => Promise<void>;
 
-interface BaseMigration {
+export interface BaseMigration {
 	up: MigrationFn;
 	down?: MigrationFn | never;
 	pruneBeforeRunning?: boolean;
+	transaction?: false;
 }
 
 export interface ReversibleMigration extends BaseMigration {

packages/cli/src/databases/utils/migrationHelpers.ts

@@ -1,27 +1,15 @@
-import { readFileSync, rmSync, statSync } from 'fs';
-import path from 'path';
+import { readFileSync, rmSync } from 'fs';
 import { UserSettings } from 'n8n-core';
 import type { QueryRunner } from 'typeorm/query-runner/QueryRunner';
 import config from '@/config';
 import { getLogger } from '@/Logger';
 import { inTest } from '@/constants';
-import type { Migration, MigrationContext } from '@db/types';
+import type { BaseMigration, Migration, MigrationContext, MigrationFn } from '@db/types';
 
 const logger = getLogger();
 
 const PERSONALIZATION_SURVEY_FILENAME = 'personalizationSurvey.json';
 
-const DESIRED_DATABASE_FILE_SIZE = 1 * 1024 * 1024 * 1024; // 1 GB
-
-function getSqliteDbFileSize(): number {
-	const filename = path.resolve(
-		UserSettings.getUserN8nFolderPath(),
-		config.getEnv('database.sqlite.database'),
-	);
-	const { size } = statSync(filename);
-	return size;
-}
-
 export function loadSurveyFromDisk(): string | null {
 	const userSettingsPath = UserSettings.getUserN8nFolderPath();
 	try {
@@ -69,6 +57,22 @@ function logMigrationEnd(migrationName: string): void {
 	logger.debug(`Finished migration ${migrationName}`);
 }
 
+const runDisablingForeignKeys = async (
+	migration: BaseMigration,
+	context: MigrationContext,
+	fn: MigrationFn,
+) => {
+	const { dbType, queryRunner } = context;
+	if (dbType !== 'sqlite') throw new Error('Disabling transactions only available in sqlite');
+	await queryRunner.query('PRAGMA foreign_keys=OFF');
+	await queryRunner.startTransaction();
+
+	await fn.call(migration, context);
+
+	await queryRunner.commitTransaction();
+	await queryRunner.query('PRAGMA foreign_keys=ON');
+};
+
 export const wrapMigration = (migration: Migration) => {
 	const dbType = config.getEnv('database.type');
 	const dbName = config.getEnv(`database.${dbType === 'mariadb' ? 'mysqldb' : dbType}.database`);
@@ -82,56 +86,25 @@ export const wrapMigration = (migration: Migration) => {
 		logger,
 	};
 
-	const migrationsPruningEnabled =
-		dbType === 'sqlite' && process.env.MIGRATIONS_PRUNING_ENABLED === 'true';
-
 	const { up, down } = migration.prototype;
 	Object.assign(migration.prototype, {
-		async beforeTransaction(this: typeof migration.prototype, queryRunner: QueryRunner) {
-			if (this.pruneBeforeRunning) {
-				if (migrationsPruningEnabled) {
-					const dbFileSize = getSqliteDbFileSize();
-					if (dbFileSize < DESIRED_DATABASE_FILE_SIZE) {
-						console.log(`DB Size not large enough to prune: ${dbFileSize}`);
-						return;
-					}
-
-					console.time('pruningData');
-					const counting = (await queryRunner.query(
-						`select count(id) as rows from "${tablePrefix}execution_entity";`,
-					)) as Array<{ rows: number }>;
-
-					const averageExecutionSize = dbFileSize / counting[0].rows;
-					const numberOfExecutionsToKeep = Math.floor(
-						DESIRED_DATABASE_FILE_SIZE / averageExecutionSize,
-					);
-
-					const query = `
-					SELECT id FROM "${tablePrefix}execution_entity"
-					ORDER BY id DESC limit ${numberOfExecutionsToKeep}, 1;
-				`;
-
-					const idToKeep = (await queryRunner.query(query)) as Array<{ id: number }>;
-
-					const removalQuery = `
-					DELETE FROM "${tablePrefix}execution_entity"
-					WHERE id < ${idToKeep[0].id} and status IN ('success');
-				`;
-
-					await queryRunner.query(removalQuery);
-					console.timeEnd('pruningData');
-				} else {
-					console.log('Pruning was requested, but was not enabled');
-				}
-			}
-		},
-		async up(queryRunner: QueryRunner) {
+		async up(this: BaseMigration, queryRunner: QueryRunner) {
 			logMigrationStart(migrationName);
-			await up.call(this, { queryRunner, ...context });
+			if (!this.transaction) {
+				await runDisablingForeignKeys(this, { queryRunner, ...context }, up);
+			} else {
+				await up.call(this, { queryRunner, ...context });
+			}
 			logMigrationEnd(migrationName);
 		},
-		async down(queryRunner: QueryRunner) {
-			await down?.call(this, { queryRunner, ...context });
+		async down(this: BaseMigration, queryRunner: QueryRunner) {
+			if (down) {
+				if (!this.transaction) {
+					await runDisablingForeignKeys(this, { queryRunner, ...context }, up);
+				} else {
+					await down.call(this, { queryRunner, ...context });
+				}
+			}
 		},
 	});
 };