From 29f1d11ab96a3dfb5498cfaff901b434bd6aa3be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=A4=95=E0=A4=BE=E0=A4=B0=E0=A4=A4=E0=A5=8B=E0=A4=AB?= =?UTF-8?q?=E0=A5=8D=E0=A4=AB=E0=A5=87=E0=A4=B2=E0=A4=B8=E0=A5=8D=E0=A4=95?= =?UTF-8?q?=E0=A5=8D=E0=A4=B0=E0=A4=BF=E0=A4=AA=E0=A5=8D=E0=A4=9F=E2=84=A2?= Date: Thu, 31 Aug 2023 19:49:37 +0200 Subject: [PATCH 001/103] refactor(core): Simplify executions and binary data pruning --- .../handlers/executions/executions.handler.ts | 4 - .../handlers/executions/executions.service.ts | 4 +- packages/cli/src/Server.ts | 1 - .../cli/src/WorkflowExecuteAdditionalData.ts | 88 +----------- packages/cli/src/WorkflowRunner.ts | 2 +- packages/cli/src/config/schema.ts | 12 -- .../repositories/execution.repository.ts | 127 ++++++++++++++---- .../cli/src/executions/executions.service.ts | 12 +- .../core/src/BinaryDataManager/FileSystem.ts | 56 +------- packages/core/src/BinaryDataManager/index.ts | 16 --- packages/core/src/Interfaces.ts | 3 - .../core/test/NodeExecuteFunctions.test.ts | 2 - packages/nodes-base/test/nodes/Helpers.ts | 1 - 13 files changed, 117 insertions(+), 211 deletions(-) diff --git a/packages/cli/src/PublicApi/v1/handlers/executions/executions.handler.ts b/packages/cli/src/PublicApi/v1/handlers/executions/executions.handler.ts index 03b70556e0970..9a5c419b7deef 100644 --- a/packages/cli/src/PublicApi/v1/handlers/executions/executions.handler.ts +++ b/packages/cli/src/PublicApi/v1/handlers/executions/executions.handler.ts @@ -1,7 +1,5 @@ import type express from 'express'; -import { BinaryDataManager } from 'n8n-core'; - import { getExecutions, getExecutionInWorkflows, @@ -37,8 +35,6 @@ export = { return res.status(404).json({ message: 'Not Found' }); } - await BinaryDataManager.getInstance().deleteBinaryDataByExecutionIds([execution.id!]); - await deleteExecution(execution); execution.id = id; diff --git a/packages/cli/src/PublicApi/v1/handlers/executions/executions.service.ts b/packages/cli/src/PublicApi/v1/handlers/executions/executions.service.ts index 6061df800ca97..6ab0fdfc8519e 100644 --- a/packages/cli/src/PublicApi/v1/handlers/executions/executions.service.ts +++ b/packages/cli/src/PublicApi/v1/handlers/executions/executions.service.ts @@ -110,6 +110,6 @@ export async function getExecutionInWorkflows( }); } -export async function deleteExecution(execution: IExecutionBase): Promise { - return Container.get(ExecutionRepository).deleteExecution(execution.id as string); +export async function deleteExecution(execution: IExecutionBase): Promise { + return Container.get(ExecutionRepository).softDeleteExecution(execution.id as string); } diff --git a/packages/cli/src/Server.ts b/packages/cli/src/Server.ts index 7500ef17c0bd7..09a464589999a 100644 --- a/packages/cli/src/Server.ts +++ b/packages/cli/src/Server.ts @@ -393,7 +393,6 @@ export class Server extends AbstractServer { ), executions_data_prune: config.getEnv('executions.pruneData'), executions_data_max_age: config.getEnv('executions.pruneDataMaxAge'), - executions_data_prune_timeout: config.getEnv('executions.pruneDataTimeout'), }, deploymentType: config.getEnv('deployment.type'), binaryDataMode: binaryDataConfig.mode, diff --git a/packages/cli/src/WorkflowExecuteAdditionalData.ts b/packages/cli/src/WorkflowExecuteAdditionalData.ts index 5760cce3d1a6d..903daba4775c6 100644 --- a/packages/cli/src/WorkflowExecuteAdditionalData.ts +++ b/packages/cli/src/WorkflowExecuteAdditionalData.ts @@ -9,7 +9,7 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import { BinaryDataManager, UserSettings, WorkflowExecute } from 'n8n-core'; +import { UserSettings, WorkflowExecute } from 'n8n-core'; import type { IDataObject, @@ -38,9 +38,6 @@ import { import pick from 'lodash/pick'; import { Container } from 'typedi'; -import type { FindOptionsWhere } from 'typeorm'; -import { LessThanOrEqual, In } from 'typeorm'; -import { DateUtils } from 'typeorm/util/DateUtils'; import config from '@/config'; import * as Db from '@/Db'; import { ActiveExecutions } from '@/ActiveExecutions'; @@ -48,7 +45,6 @@ import { CredentialsHelper } from '@/CredentialsHelper'; import { ExternalHooks } from '@/ExternalHooks'; import type { IExecutionDb, - IExecutionFlattedDb, IPushDataExecutionFinished, IWorkflowExecuteProcess, IWorkflowExecutionDataProcess, @@ -181,77 +177,6 @@ export function executeErrorWorkflow( } } -/** - * Prunes Saved Execution which are older than configured. - * Throttled to be executed just once in configured timeframe. - * TODO: Consider moving this whole function to the repository or at least the queries - */ -let throttling = false; -async function pruneExecutionData(this: WorkflowHooks): Promise { - if (!throttling) { - Logger.verbose('Pruning execution data from database'); - - throttling = true; - const timeout = config.getEnv('executions.pruneDataTimeout'); // in seconds - const maxAge = config.getEnv('executions.pruneDataMaxAge'); // in h - const maxCount = config.getEnv('executions.pruneDataMaxCount'); - const date = new Date(); // today - date.setHours(date.getHours() - maxAge); - - // date reformatting needed - see https://github.com/typeorm/typeorm/issues/2286 - - const utcDate = DateUtils.mixedDateToUtcDatetimeString(date); - - const toPrune: Array> = [ - { stoppedAt: LessThanOrEqual(utcDate) }, - ]; - - if (maxCount > 0) { - const executions = await Db.collections.Execution.find({ - select: ['id'], - skip: maxCount, - take: 1, - order: { id: 'DESC' }, - }); - - if (executions[0]) { - toPrune.push({ id: LessThanOrEqual(executions[0].id) }); - } - } - - try { - setTimeout(() => { - throttling = false; - }, timeout * 1000); - let executionIds: Array; - do { - executionIds = ( - await Db.collections.Execution.find({ - select: ['id'], - where: toPrune, - take: 100, - }) - ).map(({ id }) => id); - await Db.collections.Execution.delete({ id: In(executionIds) }); - // Mark binary data for deletion for all executions - await BinaryDataManager.getInstance().markDataForDeletionByExecutionIds(executionIds); - } while (executionIds.length > 0); - } catch (error) { - ErrorReporter.error(error); - throttling = false; - Logger.error( - `Failed pruning execution data from database for execution ID ${this.executionId} (hookFunctionsSave)`, - { - ...error, - executionId: this.executionId, - sessionId: this.sessionId, - workflowId: this.workflowData.id, - }, - ); - } - } -} - export async function saveExecutionMetadata( executionId: string, executionMetadata: Record, @@ -535,11 +460,6 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks { workflowId: this.workflowData.id, }); - // Prune old execution data - if (config.getEnv('executions.pruneData')) { - await pruneExecutionData.call(this); - } - const isManualMode = [this.mode, parentProcessMode].includes('manual'); try { @@ -567,8 +487,7 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks { } if (isManualMode && !saveManualExecutions && !fullRunData.waitTill) { - // Data is always saved, so we remove from database - await Container.get(ExecutionRepository).deleteExecution(this.executionId, true); + await Container.get(ExecutionRepository).softDeleteExecution(this.executionId); return; } @@ -606,8 +525,7 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks { this.executionId, this.retryOf, ); - // Data is always saved, so we remove from database - await Container.get(ExecutionRepository).deleteExecution(this.executionId); + await Container.get(ExecutionRepository).softDeleteExecution(this.executionId); return; } diff --git a/packages/cli/src/WorkflowRunner.ts b/packages/cli/src/WorkflowRunner.ts index b7e4b4eaacb48..5103e4dafe27e 100644 --- a/packages/cli/src/WorkflowRunner.ts +++ b/packages/cli/src/WorkflowRunner.ts @@ -601,7 +601,7 @@ export class WorkflowRunner { (workflowDidSucceed && saveDataSuccessExecution === 'none') || (!workflowDidSucceed && saveDataErrorExecution === 'none') ) { - await Container.get(ExecutionRepository).deleteExecution(executionId); + await Container.get(ExecutionRepository).softDeleteExecution(executionId); } // eslint-disable-next-line id-denylist } catch (err) { diff --git a/packages/cli/src/config/schema.ts b/packages/cli/src/config/schema.ts index 157e9c2a95723..586fba82b973e 100644 --- a/packages/cli/src/config/schema.ts +++ b/packages/cli/src/config/schema.ts @@ -316,12 +316,6 @@ export const schema = { default: 336, env: 'EXECUTIONS_DATA_MAX_AGE', }, - pruneDataTimeout: { - doc: 'Timeout (seconds) after execution data has been pruned', - format: Number, - default: 3600, - env: 'EXECUTIONS_DATA_PRUNE_TIMEOUT', - }, // Additional pruning option to delete executions if total count exceeds the configured max. // Deletes the oldest entries first @@ -907,12 +901,6 @@ export const schema = { env: 'N8N_BINARY_DATA_STORAGE_PATH', doc: 'Path for binary data storage in "filesystem" mode', }, - binaryDataTTL: { - format: Number, - default: 60, - env: 'N8N_BINARY_DATA_TTL', - doc: 'TTL for binary data of unsaved executions in minutes', - }, }, deployment: { diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 07215d4701908..ce4fb0d1bae5b 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -1,29 +1,33 @@ import { Service } from 'typedi'; import { DataSource, In, LessThanOrEqual, MoreThanOrEqual, Repository } from 'typeorm'; +import { DateUtils } from 'typeorm/util/DateUtils'; import type { FindManyOptions, FindOneOptions, FindOptionsWhere, SelectQueryBuilder, } from 'typeorm'; -import { ExecutionEntity } from '../entities/ExecutionEntity'; import { parse, stringify } from 'flatted'; +import { LoggerProxy as Logger } from 'n8n-workflow'; +import type { IExecutionsSummary, IRunExecutionData } from 'n8n-workflow'; +import { BinaryDataManager } from 'n8n-core'; import type { IExecutionBase, IExecutionDb, IExecutionFlattedDb, IExecutionResponse, } from '@/Interfaces'; -import { LoggerProxy } from 'n8n-workflow'; -import type { IExecutionsSummary, IRunExecutionData } from 'n8n-workflow'; -import { ExecutionDataRepository } from './executionData.repository'; -import type { ExecutionData } from '../entities/ExecutionData'; + +import config from '@/config'; import type { IGetExecutionsQueryFilter } from '@/executions/executions.service'; import { isAdvancedExecutionFiltersEnabled } from '@/executions/executionHelpers'; +import type { ExecutionData } from '../entities/ExecutionData'; +import { ExecutionEntity } from '../entities/ExecutionEntity'; import { ExecutionMetadata } from '../entities/ExecutionMetadata'; -import { DateUtils } from 'typeorm/util/DateUtils'; -import { BinaryDataManager } from 'n8n-core'; -import config from '@/config'; +import { ExecutionDataRepository } from './executionData.repository'; +import { inTest } from '@/constants'; + +const PRUNING_BATCH_SIZE = 100; function parseFiltersToQueryBuilder( qb: SelectQueryBuilder, @@ -71,6 +75,14 @@ export class ExecutionRepository extends Repository { private readonly executionDataRepository: ExecutionDataRepository, ) { super(ExecutionEntity, dataSource.manager); + + if (!inTest) { + if (config.getEnv('executions.pruneData')) { + setInterval(async () => this.pruneOlderExecutions(), 60 * 60 * 1000); // Every hour + } + + setInterval(async () => this.deleteSoftDeletedExecutions(), 15 * 60 * 1000); // Every 15 minutes + } } async findMultipleExecutions( @@ -238,16 +250,6 @@ export class ExecutionRepository extends Repository { } } - async deleteExecution(executionId: string, deferBinaryDataDeletion = false) { - const binaryDataManager = BinaryDataManager.getInstance(); - if (deferBinaryDataDeletion) { - await binaryDataManager.markDataForDeletionByExecutionId(executionId); - } else { - await binaryDataManager.deleteBinaryDataByExecutionIds([executionId]); - } - return this.delete({ id: executionId }); - } - async countExecutions( filters: IGetExecutionsQueryFilter | undefined, accessibleWorkflowIds: string[], @@ -287,7 +289,7 @@ export class ExecutionRepository extends Repository { } } catch (error) { if (error instanceof Error) { - LoggerProxy.warn(`Failed to get executions count from Postgres: ${error.message}`, { + Logger.warn(`Failed to get executions count from Postgres: ${error.message}`, { error, }); } @@ -357,7 +359,15 @@ export class ExecutionRepository extends Repository { }); } - async deleteExecutions( + async softDeleteExecution(executionId: string) { + await this.update({ id: executionId }, { deletedAt: Date.now() }); + } + + async softDeleteExecutions(executionIds: string[]) { + await this.update({ id: In(executionIds) }, { deletedAt: Date.now() }); + } + + async deleteExecutionsByFilter( filters: IGetExecutionsQueryFilter | undefined, accessibleWorkflowIds: string[], deleteConditions: { @@ -389,7 +399,7 @@ export class ExecutionRepository extends Repository { if (!executions.length) { if (deleteConditions.ids) { - LoggerProxy.error('Failed to delete an execution due to insufficient permissions', { + Logger.error('Failed to delete an execution due to insufficient permissions', { executionIds: deleteConditions.ids, }); } @@ -397,13 +407,78 @@ export class ExecutionRepository extends Repository { } const executionIds = executions.map(({ id }) => id); - const binaryDataManager = BinaryDataManager.getInstance(); - await binaryDataManager.deleteBinaryDataByExecutionIds(executionIds); - do { // Delete in batches to avoid "SQLITE_ERROR: Expression tree is too large (maximum depth 1000)" error - const batch = executionIds.splice(0, 500); - await this.delete(batch); + const batch = executionIds.splice(0, PRUNING_BATCH_SIZE); + await this.softDeleteExecutions(batch); } while (executionIds.length > 0); } + + private async pruneOlderExecutions() { + Logger.verbose('Pruning execution data from database'); + + const maxAge = config.getEnv('executions.pruneDataMaxAge'); // in h + const maxCount = config.getEnv('executions.pruneDataMaxCount'); + + // Find ids of all executions that were stopped longer that pruneDataMaxAge ago + const date = new Date(); + date.setHours(date.getHours() - maxAge); + + const toPrune: Array> = [ + // date reformatting needed - see https://github.com/typeorm/typeorm/issues/2286 + { stoppedAt: LessThanOrEqual(DateUtils.mixedDateToUtcDatetimeString(date)) }, + ]; + + if (maxCount > 0) { + const executions = await this.find({ + select: ['id'], + skip: maxCount, + take: 1, + order: { id: 'DESC' }, + }); + + if (executions[0]) { + toPrune.push({ id: LessThanOrEqual(executions[0].id) }); + } + } + + const executionIds = ( + await this.find({ + select: ['id'], + where: toPrune, + take: PRUNING_BATCH_SIZE, + }) + ).map(({ id }) => id); + await this.softDeleteExecutions(executionIds); + + if (executionIds.length === PRUNING_BATCH_SIZE) { + setTimeout(async () => this.pruneOlderExecutions(), 1000); + } + } + + private async deleteSoftDeletedExecutions() { + // Find ids of all executions that were deleted over an hour ago + const date = new Date(); + date.setHours(date.getHours() - 1); + + const executionIds = ( + await this.find({ + select: ['id'], + where: { + deletedAt: LessThanOrEqual(DateUtils.mixedDateToUtcDatetimeString(date)), + }, + take: PRUNING_BATCH_SIZE, + }) + ).map(({ id }) => id); + + const binaryDataManager = BinaryDataManager.getInstance(); + await binaryDataManager.deleteBinaryDataByExecutionIds(executionIds); + + // Actually delete these executions + await this.delete({ id: In(executionIds) }); + + if (executionIds.length === PRUNING_BATCH_SIZE) { + setTimeout(async () => this.deleteSoftDeletedExecutions(), 1000); + } + } } diff --git a/packages/cli/src/executions/executions.service.ts b/packages/cli/src/executions/executions.service.ts index aab68b449d4f5..d13776c5d20d8 100644 --- a/packages/cli/src/executions/executions.service.ts +++ b/packages/cli/src/executions/executions.service.ts @@ -351,9 +351,13 @@ export class ExecutionsService { } } - return Container.get(ExecutionRepository).deleteExecutions(requestFilters, sharedWorkflowIds, { - deleteBefore, - ids, - }); + return Container.get(ExecutionRepository).deleteExecutionsByFilter( + requestFilters, + sharedWorkflowIds, + { + deleteBefore, + ids, + }, + ); } } diff --git a/packages/core/src/BinaryDataManager/FileSystem.ts b/packages/core/src/BinaryDataManager/FileSystem.ts index cf9f1d94de220..6ae70271211b9 100644 --- a/packages/core/src/BinaryDataManager/FileSystem.ts +++ b/packages/core/src/BinaryDataManager/FileSystem.ts @@ -1,4 +1,3 @@ -import glob from 'fast-glob'; import { createReadStream } from 'fs'; import fs from 'fs/promises'; import path from 'path'; @@ -10,32 +9,19 @@ import { jsonParse } from 'n8n-workflow'; import type { IBinaryDataConfig, IBinaryDataManager } from '../Interfaces'; import { FileNotFoundError } from '../errors'; -const PREFIX_METAFILE = 'binarymeta'; - const executionExtractionRegexp = /^(\w+)(?:[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12})$/; export class BinaryDataFileSystem implements IBinaryDataManager { private storagePath: string; - private binaryDataTTL: number; - constructor(config: IBinaryDataConfig) { this.storagePath = config.localStoragePath; - this.binaryDataTTL = config.binaryDataTTL; } - async init(startPurger = false): Promise { - if (startPurger) { - setInterval(async () => { - await this.deleteMarkedFiles(); - }, this.binaryDataTTL * 30000); - } - + async init(): Promise { await this.assertFolder(this.storagePath); await this.assertFolder(this.getBinaryDataMetaPath()); - - await this.deleteMarkedFiles(); } async getFileSize(identifier: string): Promise { @@ -81,47 +67,8 @@ export class BinaryDataFileSystem implements IBinaryDataManager { return this.resolveStoragePath(`${identifier}.metadata`); } - async markDataForDeletionByExecutionId(executionId: string): Promise { - const tt = new Date(new Date().getTime() + this.binaryDataTTL * 60000); - return fs.writeFile( - this.resolveStoragePath('meta', `${PREFIX_METAFILE}_${executionId}_${tt.valueOf()}`), - '', - ); - } - - async deleteMarkedFiles(): Promise { - return this.deleteMarkedFilesByMeta(this.getBinaryDataMetaPath(), PREFIX_METAFILE); - } - - private async deleteMarkedFilesByMeta(metaPath: string, filePrefix: string): Promise { - const currentTimeValue = new Date().valueOf(); - const metaFileNames = await glob(`${filePrefix}_*`, { cwd: metaPath }); - - const executionIds = metaFileNames - .map((f) => f.split('_') as [string, string, string]) - .filter(([prefix, , ts]) => { - if (prefix !== filePrefix) return false; - const execTimestamp = parseInt(ts, 10); - return execTimestamp < currentTimeValue; - }) - .map((e) => e[1]); - - const filesToDelete = []; - const deletedIds = await this.deleteBinaryDataByExecutionIds(executionIds); - for (const executionId of deletedIds) { - filesToDelete.push( - ...(await glob(`${filePrefix}_${executionId}_`, { - absolute: true, - cwd: metaPath, - })), - ); - } - await Promise.all(filesToDelete.map(async (file) => fs.rm(file))); - } - async duplicateBinaryDataByIdentifier(binaryDataId: string, prefix: string): Promise { const newBinaryDataId = this.generateFileName(prefix); - await fs.copyFile( this.resolveStoragePath(binaryDataId), this.resolveStoragePath(newBinaryDataId), @@ -130,6 +77,7 @@ export class BinaryDataFileSystem implements IBinaryDataManager { } async deleteBinaryDataByExecutionIds(executionIds: string[]): Promise { + // TODO: switch over to new folder structure, and delete folders instead const set = new Set(executionIds); const fileNames = await fs.readdir(this.storagePath); const deletedIds = []; diff --git a/packages/core/src/BinaryDataManager/index.ts b/packages/core/src/BinaryDataManager/index.ts index 7bfbf614c997b..eca32f184da26 100644 --- a/packages/core/src/BinaryDataManager/index.ts +++ b/packages/core/src/BinaryDataManager/index.ts @@ -156,22 +156,6 @@ export class BinaryDataManager { throw new Error('Storage mode used to store binary data not available'); } - async markDataForDeletionByExecutionId(executionId: string): Promise { - if (this.managers[this.binaryDataMode]) { - await this.managers[this.binaryDataMode].markDataForDeletionByExecutionId(executionId); - } - } - - async markDataForDeletionByExecutionIds(executionIds: string[]): Promise { - if (this.managers[this.binaryDataMode]) { - await Promise.all( - executionIds.map(async (id) => - this.managers[this.binaryDataMode].markDataForDeletionByExecutionId(id), - ), - ); - } - } - async deleteBinaryDataByExecutionIds(executionIds: string[]): Promise { if (this.managers[this.binaryDataMode]) { await this.managers[this.binaryDataMode].deleteBinaryDataByExecutionIds(executionIds); diff --git a/packages/core/src/Interfaces.ts b/packages/core/src/Interfaces.ts index d01b655bf1e8a..f1f3f4ff7a5bd 100644 --- a/packages/core/src/Interfaces.ts +++ b/packages/core/src/Interfaces.ts @@ -38,7 +38,6 @@ export interface IBinaryDataConfig { mode: 'default' | 'filesystem'; availableModes: string; localStoragePath: string; - binaryDataTTL: number; } export interface IBinaryDataManager { @@ -51,8 +50,6 @@ export interface IBinaryDataManager { retrieveBinaryDataByIdentifier(identifier: string): Promise; getBinaryPath(identifier: string): string; getBinaryStream(identifier: string, chunkSize?: number): Readable; - markDataForDeletionByExecutionId(executionId: string): Promise; - deleteMarkedFiles(): Promise; deleteBinaryDataByIdentifier(identifier: string): Promise; duplicateBinaryDataByIdentifier(binaryDataId: string, prefix: string): Promise; deleteBinaryDataByExecutionIds(executionIds: string[]): Promise; diff --git a/packages/core/test/NodeExecuteFunctions.test.ts b/packages/core/test/NodeExecuteFunctions.test.ts index d981c1d246d98..ec2a32e7f0e54 100644 --- a/packages/core/test/NodeExecuteFunctions.test.ts +++ b/packages/core/test/NodeExecuteFunctions.test.ts @@ -34,7 +34,6 @@ describe('NodeExecuteFunctions', () => { mode: 'default', availableModes: 'default', localStoragePath: temporaryDir, - binaryDataTTL: 1, }); // Set our binary data buffer @@ -84,7 +83,6 @@ describe('NodeExecuteFunctions', () => { mode: 'filesystem', availableModes: 'filesystem', localStoragePath: temporaryDir, - binaryDataTTL: 1, }); // Set our binary data buffer diff --git a/packages/nodes-base/test/nodes/Helpers.ts b/packages/nodes-base/test/nodes/Helpers.ts index ecc87dbb556d9..d14ef99577011 100644 --- a/packages/nodes-base/test/nodes/Helpers.ts +++ b/packages/nodes-base/test/nodes/Helpers.ts @@ -222,7 +222,6 @@ export async function initBinaryDataManager(mode: 'default' | 'filesystem' = 'de mode, availableModes: mode, localStoragePath: temporaryDir, - binaryDataTTL: 1, }); return temporaryDir; } From 05e3fefd0cc19552f01dbc8a4f51a069c5203ffc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Tue, 5 Sep 2023 11:19:53 +0200 Subject: [PATCH 002/103] Remove test code from service --- .../databases/repositories/execution.repository.ts | 11 ++++------- .../cli/test/integration/shared/utils/testServer.ts | 1 + 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 620027075b458..ddb483fd89f76 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -25,7 +25,6 @@ import type { ExecutionData } from '../entities/ExecutionData'; import { ExecutionEntity } from '../entities/ExecutionEntity'; import { ExecutionMetadata } from '../entities/ExecutionMetadata'; import { ExecutionDataRepository } from './executionData.repository'; -import { inTest } from '@/constants'; const PRUNING_BATCH_SIZE = 100; @@ -76,13 +75,11 @@ export class ExecutionRepository extends Repository { ) { super(ExecutionEntity, dataSource.manager); - if (!inTest) { - if (config.getEnv('executions.pruneData')) { - setInterval(async () => this.pruneOlderExecutions(), 60 * 60 * 1000); // Every hour - } - - setInterval(async () => this.deleteSoftDeletedExecutions(), 15 * 60 * 1000); // Every 15 minutes + if (config.getEnv('executions.pruneData')) { + setInterval(async () => this.pruneOlderExecutions(), 60 * 60 * 1000); // Every hour } + + setInterval(async () => this.deleteSoftDeletedExecutions(), 15 * 60 * 1000); // Every 15 minutes } async findMultipleExecutions( diff --git a/packages/cli/test/integration/shared/utils/testServer.ts b/packages/cli/test/integration/shared/utils/testServer.ts index b00273eb66958..54e1af53babb4 100644 --- a/packages/cli/test/integration/shared/utils/testServer.ts +++ b/packages/cli/test/integration/shared/utils/testServer.ts @@ -152,6 +152,7 @@ export const setupTestServer = ({ config.set('userManagement.jwtSecret', 'My JWT secret'); config.set('userManagement.isInstanceOwnerSetUp', true); + config.set('executions.pruneData', false); if (enabledFeatures) { Container.get(License).isFeatureEnabled = (feature) => enabledFeatures.includes(feature); From 9dabbe86a8247edc9864bced3c972062c9c3d7c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Tue, 5 Sep 2023 11:24:40 +0200 Subject: [PATCH 003/103] Use time constants --- packages/cli/src/constants.ts | 6 ++++++ .../cli/src/databases/repositories/execution.repository.ts | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/constants.ts b/packages/cli/src/constants.ts index 8d9bbb9e14f2b..ec66cbdf32062 100644 --- a/packages/cli/src/constants.ts +++ b/packages/cli/src/constants.ts @@ -94,3 +94,9 @@ export const CREDENTIAL_BLANKING_VALUE = '__n8n_BLANK_VALUE_e5362baf-c777-4d57-a export const UM_FIX_INSTRUCTION = 'Please fix the database by running ./packages/cli/bin/n8n user-management:reset'; + +export const TIME = { + SECOND: 1000, + MINUTE: 60 * 1000, + HOUR: 60 * 60 * 1000, +}; diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index ddb483fd89f76..981eacad3f1b7 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -25,6 +25,7 @@ import type { ExecutionData } from '../entities/ExecutionData'; import { ExecutionEntity } from '../entities/ExecutionEntity'; import { ExecutionMetadata } from '../entities/ExecutionMetadata'; import { ExecutionDataRepository } from './executionData.repository'; +import { TIME } from '@/constants'; const PRUNING_BATCH_SIZE = 100; @@ -76,10 +77,10 @@ export class ExecutionRepository extends Repository { super(ExecutionEntity, dataSource.manager); if (config.getEnv('executions.pruneData')) { - setInterval(async () => this.pruneOlderExecutions(), 60 * 60 * 1000); // Every hour + setInterval(async () => this.pruneOlderExecutions(), TIME.HOUR); } - setInterval(async () => this.deleteSoftDeletedExecutions(), 15 * 60 * 1000); // Every 15 minutes + setInterval(async () => this.deleteSoftDeletedExecutions(), 15 * TIME.MINUTE); } async findMultipleExecutions( From 3f7de8e044c8ac9c682c45acb6fd8b25a3dcb069 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Tue, 5 Sep 2023 11:26:38 +0200 Subject: [PATCH 004/103] Improve naming --- .../cli/src/databases/repositories/execution.repository.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 981eacad3f1b7..399999cf6bd4b 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -77,7 +77,7 @@ export class ExecutionRepository extends Repository { super(ExecutionEntity, dataSource.manager); if (config.getEnv('executions.pruneData')) { - setInterval(async () => this.pruneOlderExecutions(), TIME.HOUR); + setInterval(async () => this.pruneBySoftDeleting(), TIME.HOUR); } setInterval(async () => this.deleteSoftDeletedExecutions(), 15 * TIME.MINUTE); @@ -422,7 +422,7 @@ export class ExecutionRepository extends Repository { } while (executionIds.length > 0); } - private async pruneOlderExecutions() { + private async pruneBySoftDeleting() { Logger.verbose('Pruning execution data from database'); const maxAge = config.getEnv('executions.pruneDataMaxAge'); // in h @@ -460,7 +460,7 @@ export class ExecutionRepository extends Repository { await this.softDeleteExecutions(executionIds); if (executionIds.length === PRUNING_BATCH_SIZE) { - setTimeout(async () => this.pruneOlderExecutions(), 1000); + setTimeout(async () => this.pruneBySoftDeleting(), 1000); } } From 5583814938af16a84cd3395a3a3b29224b8dde03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Tue, 5 Sep 2023 11:51:11 +0200 Subject: [PATCH 005/103] Use native typeorm soft deletion --- .../v1/handlers/executions/executions.handler.ts | 12 ++++-------- .../v1/handlers/executions/executions.service.ts | 6 +----- .../cli/src/WorkflowExecuteAdditionalData.ts | 5 ++--- packages/cli/src/WorkflowRunner.ts | 2 +- .../src/databases/entities/ExecutionEntity.ts | 3 ++- .../repositories/execution.repository.ts | 16 ++++------------ 6 files changed, 14 insertions(+), 30 deletions(-) diff --git a/packages/cli/src/PublicApi/v1/handlers/executions/executions.handler.ts b/packages/cli/src/PublicApi/v1/handlers/executions/executions.handler.ts index 9a5c419b7deef..90ed886bfa62a 100644 --- a/packages/cli/src/PublicApi/v1/handlers/executions/executions.handler.ts +++ b/packages/cli/src/PublicApi/v1/handlers/executions/executions.handler.ts @@ -1,11 +1,6 @@ import type express from 'express'; -import { - getExecutions, - getExecutionInWorkflows, - deleteExecution, - getExecutionsCount, -} from './executions.service'; +import { getExecutions, getExecutionInWorkflows, getExecutionsCount } from './executions.service'; import { ActiveExecutions } from '@/ActiveExecutions'; import { authorize, validCursor } from '../../shared/middlewares/global.middleware'; import type { ExecutionRequest } from '../../../types'; @@ -13,6 +8,7 @@ import { getSharedWorkflowIds } from '../workflows/workflows.service'; import { encodeNextCursor } from '../../shared/services/pagination.service'; import { Container } from 'typedi'; import { InternalHooks } from '@/InternalHooks'; +import { ExecutionRepository } from '@/databases/repositories'; export = { deleteExecution: [ @@ -31,11 +27,11 @@ export = { // look for the execution on the workflow the user owns const execution = await getExecutionInWorkflows(id, sharedWorkflowsIds, false); - if (!execution) { + if (!execution?.id) { return res.status(404).json({ message: 'Not Found' }); } - await deleteExecution(execution); + await Container.get(ExecutionRepository).softDelete(execution.id); execution.id = id; diff --git a/packages/cli/src/PublicApi/v1/handlers/executions/executions.service.ts b/packages/cli/src/PublicApi/v1/handlers/executions/executions.service.ts index 6ab0fdfc8519e..de08c9095a601 100644 --- a/packages/cli/src/PublicApi/v1/handlers/executions/executions.service.ts +++ b/packages/cli/src/PublicApi/v1/handlers/executions/executions.service.ts @@ -1,4 +1,4 @@ -import type { DeleteResult, FindOptionsWhere } from 'typeorm'; +import type { FindOptionsWhere } from 'typeorm'; import { In, Not, Raw, LessThan } from 'typeorm'; import { Container } from 'typedi'; import type { ExecutionStatus } from 'n8n-workflow'; @@ -109,7 +109,3 @@ export async function getExecutionInWorkflows( unflattenData: true, }); } - -export async function deleteExecution(execution: IExecutionBase): Promise { - return Container.get(ExecutionRepository).softDeleteExecution(execution.id as string); -} diff --git a/packages/cli/src/WorkflowExecuteAdditionalData.ts b/packages/cli/src/WorkflowExecuteAdditionalData.ts index 59aac1192d002..d19bae6e25009 100644 --- a/packages/cli/src/WorkflowExecuteAdditionalData.ts +++ b/packages/cli/src/WorkflowExecuteAdditionalData.ts @@ -39,7 +39,6 @@ import { import pick from 'lodash/pick'; import { Container } from 'typedi'; import config from '@/config'; -import * as Db from '@/Db'; import { ActiveExecutions } from '@/ActiveExecutions'; import { CredentialsHelper } from '@/CredentialsHelper'; import { ExternalHooks } from '@/ExternalHooks'; @@ -471,7 +470,7 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks { } if (isManualMode && !saveManualExecutions && !fullRunData.waitTill) { - await Container.get(ExecutionRepository).softDeleteExecution(this.executionId); + await Container.get(ExecutionRepository).softDelete(this.executionId); return; } @@ -509,7 +508,7 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks { this.executionId, this.retryOf, ); - await Container.get(ExecutionRepository).softDeleteExecution(this.executionId); + await Container.get(ExecutionRepository).softDelete(this.executionId); return; } diff --git a/packages/cli/src/WorkflowRunner.ts b/packages/cli/src/WorkflowRunner.ts index 5103e4dafe27e..2f730af402480 100644 --- a/packages/cli/src/WorkflowRunner.ts +++ b/packages/cli/src/WorkflowRunner.ts @@ -601,7 +601,7 @@ export class WorkflowRunner { (workflowDidSucceed && saveDataSuccessExecution === 'none') || (!workflowDidSucceed && saveDataErrorExecution === 'none') ) { - await Container.get(ExecutionRepository).softDeleteExecution(executionId); + await Container.get(ExecutionRepository).softDelete(executionId); } // eslint-disable-next-line id-denylist } catch (err) { diff --git a/packages/cli/src/databases/entities/ExecutionEntity.ts b/packages/cli/src/databases/entities/ExecutionEntity.ts index f71bf0ba57988..d73267ee52003 100644 --- a/packages/cli/src/databases/entities/ExecutionEntity.ts +++ b/packages/cli/src/databases/entities/ExecutionEntity.ts @@ -9,6 +9,7 @@ import { OneToOne, PrimaryColumn, Relation, + DeleteDateColumn, } from 'typeorm'; import { datetimeColumnType } from './AbstractEntity'; import { idStringifier } from '../utils/transformers'; @@ -49,7 +50,7 @@ export class ExecutionEntity { @Column({ type: datetimeColumnType, nullable: true }) stoppedAt: Date; - @Column(datetimeColumnType) + @DeleteDateColumn({ type: datetimeColumnType, nullable: true }) deletedAt: Date; @Column({ nullable: true }) diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 399999cf6bd4b..bc29057b828b6 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -367,14 +367,6 @@ export class ExecutionRepository extends Repository { }); } - async softDeleteExecution(executionId: string) { - await this.update({ id: executionId }, { deletedAt: Date.now() }); - } - - async softDeleteExecutions(executionIds: string[]) { - await this.update({ id: In(executionIds) }, { deletedAt: Date.now() }); - } - async deleteExecutionsByFilter( filters: IGetExecutionsQueryFilter | undefined, accessibleWorkflowIds: string[], @@ -418,12 +410,12 @@ export class ExecutionRepository extends Repository { do { // Delete in batches to avoid "SQLITE_ERROR: Expression tree is too large (maximum depth 1000)" error const batch = executionIds.splice(0, PRUNING_BATCH_SIZE); - await this.softDeleteExecutions(batch); + await this.softDelete(batch); } while (executionIds.length > 0); } private async pruneBySoftDeleting() { - Logger.verbose('Pruning execution data from database'); + Logger.verbose('Pruning (soft-deleting) execution data from database'); const maxAge = config.getEnv('executions.pruneDataMaxAge'); // in h const maxCount = config.getEnv('executions.pruneDataMaxCount'); @@ -457,10 +449,10 @@ export class ExecutionRepository extends Repository { take: PRUNING_BATCH_SIZE, }) ).map(({ id }) => id); - await this.softDeleteExecutions(executionIds); + await this.softDelete(executionIds); if (executionIds.length === PRUNING_BATCH_SIZE) { - setTimeout(async () => this.pruneBySoftDeleting(), 1000); + setTimeout(async () => this.pruneBySoftDeleting(), TIME.SECOND); } } From f50108a0cbc7a6d3b8c3479acf57434568ac282e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Tue, 5 Sep 2023 12:00:26 +0200 Subject: [PATCH 006/103] Improve naming --- .../src/databases/repositories/execution.repository.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index bc29057b828b6..518adc8dbe6a4 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -80,7 +80,7 @@ export class ExecutionRepository extends Repository { setInterval(async () => this.pruneBySoftDeleting(), TIME.HOUR); } - setInterval(async () => this.deleteSoftDeletedExecutions(), 15 * TIME.MINUTE); + setInterval(async () => this.hardDelete(), 15 * TIME.MINUTE); } async findMultipleExecutions( @@ -456,7 +456,10 @@ export class ExecutionRepository extends Repository { } } - private async deleteSoftDeletedExecutions() { + /** + * Permanently delete all soft-deleted executions and their binary data, in batches. + */ + private async hardDelete() { // Find ids of all executions that were deleted over an hour ago const date = new Date(); date.setHours(date.getHours() - 1); @@ -478,7 +481,7 @@ export class ExecutionRepository extends Repository { await this.delete({ id: In(executionIds) }); if (executionIds.length === PRUNING_BATCH_SIZE) { - setTimeout(async () => this.deleteSoftDeletedExecutions(), 1000); + setTimeout(async () => this.hardDelete(), 1000); } } } From 85ac53a4316011a98f0c3f036ae77b8d2ca90e9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Tue, 5 Sep 2023 12:01:49 +0200 Subject: [PATCH 007/103] Make batch size class field --- .../databases/repositories/execution.repository.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 518adc8dbe6a4..d1c0feef41345 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -27,8 +27,6 @@ import { ExecutionMetadata } from '../entities/ExecutionMetadata'; import { ExecutionDataRepository } from './executionData.repository'; import { TIME } from '@/constants'; -const PRUNING_BATCH_SIZE = 100; - function parseFiltersToQueryBuilder( qb: SelectQueryBuilder, filters?: IGetExecutionsQueryFilter, @@ -70,6 +68,8 @@ function parseFiltersToQueryBuilder( @Service() export class ExecutionRepository extends Repository { + private deletionBatchSize = 100; + constructor( dataSource: DataSource, private readonly executionDataRepository: ExecutionDataRepository, @@ -409,7 +409,7 @@ export class ExecutionRepository extends Repository { const executionIds = executions.map(({ id }) => id); do { // Delete in batches to avoid "SQLITE_ERROR: Expression tree is too large (maximum depth 1000)" error - const batch = executionIds.splice(0, PRUNING_BATCH_SIZE); + const batch = executionIds.splice(0, this.deletionBatchSize); await this.softDelete(batch); } while (executionIds.length > 0); } @@ -446,12 +446,12 @@ export class ExecutionRepository extends Repository { await this.find({ select: ['id'], where: toPrune, - take: PRUNING_BATCH_SIZE, + take: this.deletionBatchSize, }) ).map(({ id }) => id); await this.softDelete(executionIds); - if (executionIds.length === PRUNING_BATCH_SIZE) { + if (executionIds.length === this.deletionBatchSize) { setTimeout(async () => this.pruneBySoftDeleting(), TIME.SECOND); } } @@ -470,7 +470,7 @@ export class ExecutionRepository extends Repository { where: { deletedAt: LessThanOrEqual(DateUtils.mixedDateToUtcDatetimeString(date)), }, - take: PRUNING_BATCH_SIZE, + take: this.deletionBatchSize, }) ).map(({ id }) => id); @@ -480,7 +480,7 @@ export class ExecutionRepository extends Repository { // Actually delete these executions await this.delete({ id: In(executionIds) }); - if (executionIds.length === PRUNING_BATCH_SIZE) { + if (executionIds.length === this.deletionBatchSize) { setTimeout(async () => this.hardDelete(), 1000); } } From 1d87e9ef9e06e128a747aaee67b313a3d0d2513c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Tue, 5 Sep 2023 12:04:49 +0200 Subject: [PATCH 008/103] Cleanup --- packages/core/src/BinaryDataManager/FileSystem.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/core/src/BinaryDataManager/FileSystem.ts b/packages/core/src/BinaryDataManager/FileSystem.ts index 6ae70271211b9..07eb66c815461 100644 --- a/packages/core/src/BinaryDataManager/FileSystem.ts +++ b/packages/core/src/BinaryDataManager/FileSystem.ts @@ -19,7 +19,7 @@ export class BinaryDataFileSystem implements IBinaryDataManager { this.storagePath = config.localStoragePath; } - async init(): Promise { + async init() { await this.assertFolder(this.storagePath); await this.assertFolder(this.getBinaryDataMetaPath()); } @@ -77,7 +77,6 @@ export class BinaryDataFileSystem implements IBinaryDataManager { } async deleteBinaryDataByExecutionIds(executionIds: string[]): Promise { - // TODO: switch over to new folder structure, and delete folders instead const set = new Set(executionIds); const fileNames = await fs.readdir(this.storagePath); const deletedIds = []; From dd16e458fa0a32946a177635d421fda6aa4fb41a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Tue, 5 Sep 2023 13:04:06 +0200 Subject: [PATCH 009/103] Remove unused method --- packages/core/src/BinaryDataManager/FileSystem.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/core/src/BinaryDataManager/FileSystem.ts b/packages/core/src/BinaryDataManager/FileSystem.ts index 07eb66c815461..c5501add5a180 100644 --- a/packages/core/src/BinaryDataManager/FileSystem.ts +++ b/packages/core/src/BinaryDataManager/FileSystem.ts @@ -21,7 +21,6 @@ export class BinaryDataFileSystem implements IBinaryDataManager { async init() { await this.assertFolder(this.storagePath); - await this.assertFolder(this.getBinaryDataMetaPath()); } async getFileSize(identifier: string): Promise { @@ -107,10 +106,6 @@ export class BinaryDataFileSystem implements IBinaryDataManager { return [prefix, uuid()].join(''); } - private getBinaryDataMetaPath() { - return path.join(this.storagePath, 'meta'); - } - private async deleteFromLocalStorage(identifier: string) { return fs.rm(this.getBinaryPath(identifier)); } From 619bff6d95f32b256872cffcae119ec33bfefe3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Tue, 5 Sep 2023 13:05:45 +0200 Subject: [PATCH 010/103] Cleanup --- packages/core/src/BinaryDataManager/FileSystem.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/src/BinaryDataManager/FileSystem.ts b/packages/core/src/BinaryDataManager/FileSystem.ts index c5501add5a180..5c4bdd32a9a2c 100644 --- a/packages/core/src/BinaryDataManager/FileSystem.ts +++ b/packages/core/src/BinaryDataManager/FileSystem.ts @@ -68,6 +68,7 @@ export class BinaryDataFileSystem implements IBinaryDataManager { async duplicateBinaryDataByIdentifier(binaryDataId: string, prefix: string): Promise { const newBinaryDataId = this.generateFileName(prefix); + await fs.copyFile( this.resolveStoragePath(binaryDataId), this.resolveStoragePath(newBinaryDataId), From 8cace1144e84aaf83eb044918807481f3eedb8f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Tue, 5 Sep 2023 13:21:38 +0200 Subject: [PATCH 011/103] Filter out soft-deleted executions --- .../src/databases/repositories/execution.repository.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index d1c0feef41345..4d76bae08aeb4 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -1,5 +1,5 @@ import { Service } from 'typedi'; -import { DataSource, In, LessThanOrEqual, MoreThanOrEqual, Repository } from 'typeorm'; +import { DataSource, In, IsNull, LessThanOrEqual, MoreThanOrEqual, Repository } from 'typeorm'; import { DateUtils } from 'typeorm/util/DateUtils'; import type { FindManyOptions, @@ -118,6 +118,10 @@ export class ExecutionRepository extends Repository { (queryParams.relations as string[]).push('executionData'); } + if (queryParams.where && !Array.isArray(queryParams.where)) { + queryParams.where.deletedAt = IsNull(); + } + const executions = await this.find(queryParams); if (options?.includeData && options?.unflattenData) { @@ -182,6 +186,7 @@ export class ExecutionRepository extends Repository { where: { id, ...options?.where, + deletedAt: IsNull(), }, }; if (options?.includeData) { @@ -340,7 +345,8 @@ export class ExecutionRepository extends Repository { .limit(limit) // eslint-disable-next-line @typescript-eslint/naming-convention .orderBy({ 'execution.id': 'DESC' }) - .andWhere('execution.workflowId IN (:...accessibleWorkflowIds)', { accessibleWorkflowIds }); + .andWhere('execution.workflowId IN (:...accessibleWorkflowIds)', { accessibleWorkflowIds }) + .andWhere('execution.deletedAt IS NULL'); if (excludedExecutionIds.length > 0) { query.andWhere('execution.id NOT IN (:...excludedExecutionIds)', { excludedExecutionIds }); From c4ffe6d0060fcd20f668715c8dbc354ac6dc3961 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Tue, 5 Sep 2023 17:12:56 +0200 Subject: [PATCH 012/103] Add tests --- .../repositories/execution.repository.ts | 29 +++++-- .../integration/executions.controller.test.ts | 39 ++++++++++ .../integration/publicApi/executions.test.ts | 2 + .../cli/test/integration/shared/testDb.ts | 4 + packages/cli/test/integration/shared/types.ts | 3 +- .../integration/shared/utils/testServer.ts | 11 ++- .../repositories/execution.repository.test.ts | 78 +++++++++++++++++++ 7 files changed, 157 insertions(+), 9 deletions(-) create mode 100644 packages/cli/test/integration/executions.controller.test.ts create mode 100644 packages/cli/test/unit/repositories/execution.repository.test.ts diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 4d76bae08aeb4..1ea6b707fa9e7 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -83,6 +83,14 @@ export class ExecutionRepository extends Repository { setInterval(async () => this.hardDelete(), 15 * TIME.MINUTE); } + setDeletionBatchSize(size: number) { + this.deletionBatchSize = size; + } + + getDeletionBatchSize() { + return this.deletionBatchSize; + } + async findMultipleExecutions( queryParams: FindManyOptions, options?: { @@ -420,19 +428,26 @@ export class ExecutionRepository extends Repository { } while (executionIds.length > 0); } - private async pruneBySoftDeleting() { - Logger.verbose('Pruning (soft-deleting) execution data from database'); - - const maxAge = config.getEnv('executions.pruneDataMaxAge'); // in h - const maxCount = config.getEnv('executions.pruneDataMaxCount'); + /** + * Return the timestamp up to which executions should be pruned. + */ + pruningLimit() { + const maxAge = config.getEnv('executions.pruneDataMaxAge'); // hours - // Find ids of all executions that were stopped longer that pruneDataMaxAge ago const date = new Date(); date.setHours(date.getHours() - maxAge); + return date; + } + + async pruneBySoftDeleting() { + Logger.verbose('Pruning (soft-deleting) execution data from database'); + + const maxCount = config.getEnv('executions.pruneDataMaxCount'); + const toPrune: Array> = [ // date reformatting needed - see https://github.com/typeorm/typeorm/issues/2286 - { stoppedAt: LessThanOrEqual(DateUtils.mixedDateToUtcDatetimeString(date)) }, + { stoppedAt: LessThanOrEqual(DateUtils.mixedDateToUtcDatetimeString(this.pruningLimit())) }, ]; if (maxCount > 0) { diff --git a/packages/cli/test/integration/executions.controller.test.ts b/packages/cli/test/integration/executions.controller.test.ts new file mode 100644 index 0000000000000..fa6a70499cc3f --- /dev/null +++ b/packages/cli/test/integration/executions.controller.test.ts @@ -0,0 +1,39 @@ +import * as testDb from './shared/testDb'; +import { setupTestServer } from './shared/utils'; +import type { User } from '@/databases/entities/User'; + +const testServer = setupTestServer({ endpointGroups: ['executions'] }); + +let owner: User; + +const saveExecution = async ({ belongingTo }: { belongingTo: User }) => { + const workflow = await testDb.createWorkflow({}, belongingTo); + return testDb.createSuccessfulExecution(workflow); +}; + +beforeEach(async () => { + await testDb.truncate(['Execution', 'Workflow', 'SharedWorkflow']); + owner = await testDb.createOwner(); +}); + +describe('POST /executions/delete', () => { + test('should hard-delete an execution', async () => { + await saveExecution({ belongingTo: owner }); + + const response = await testServer.authAgentFor(owner).get('/executions').expect(200); + + expect(response.body.data.count).toBe(1); + + const [execution] = response.body.data.results; + + await testServer + .authAgentFor(owner) + .post('/executions/delete') + .send({ ids: [execution.id] }) + .expect(200); + + const executions = await testDb.getAllExecutions(); + + expect(executions).toHaveLength(0); + }); +}); diff --git a/packages/cli/test/integration/publicApi/executions.test.ts b/packages/cli/test/integration/publicApi/executions.test.ts index 2f5216a8b9198..d42f8ca62ad35 100644 --- a/packages/cli/test/integration/publicApi/executions.test.ts +++ b/packages/cli/test/integration/publicApi/executions.test.ts @@ -168,6 +168,8 @@ describe('DELETE /executions/:id', () => { expect(stoppedAt).not.toBeNull(); expect(workflowId).toBe(execution.workflowId); expect(waitTill).toBeNull(); + + await authOwnerAgent.get(`/executions/${execution.id}`).expect(404); }); }); diff --git a/packages/cli/test/integration/shared/testDb.ts b/packages/cli/test/integration/shared/testDb.ts index 3186c3f64bbe2..27b91c9c8122b 100644 --- a/packages/cli/test/integration/shared/testDb.ts +++ b/packages/cli/test/integration/shared/testDb.ts @@ -528,6 +528,10 @@ export async function getAllWorkflows() { return Db.collections.Workflow.find(); } +export async function getAllExecutions() { + return Db.collections.Execution.find(); +} + // ---------------------------------- // workflow sharing // ---------------------------------- diff --git a/packages/cli/test/integration/shared/types.ts b/packages/cli/test/integration/shared/types.ts index 3fd972a6bd1cd..f552c7e1f7bc9 100644 --- a/packages/cli/test/integration/shared/types.ts +++ b/packages/cli/test/integration/shared/types.ts @@ -28,7 +28,8 @@ export type EndpointGroup = | 'tags' | 'externalSecrets' | 'mfa' - | 'metrics'; + | 'metrics' + | 'executions'; export interface SetupProps { applyAuth?: boolean; diff --git a/packages/cli/test/integration/shared/utils/testServer.ts b/packages/cli/test/integration/shared/utils/testServer.ts index b89e6ab7bddd0..b33c3afb04387 100644 --- a/packages/cli/test/integration/shared/utils/testServer.ts +++ b/packages/cli/test/integration/shared/utils/testServer.ts @@ -64,6 +64,7 @@ import { import { JwtService } from '@/services/jwt.service'; import { RoleService } from '@/services/role.service'; import { UserService } from '@/services/user.service'; +import { executionsController } from '@/executions/executions.controller'; /** * Plugin to prefix a path segment into a request URL pathname. @@ -93,7 +94,14 @@ const classifyEndpointGroups = (endpointGroups: EndpointGroup[]) => { const routerEndpoints: EndpointGroup[] = []; const functionEndpoints: EndpointGroup[] = []; - const ROUTER_GROUP = ['credentials', 'workflows', 'publicApi', 'license', 'variables']; + const ROUTER_GROUP = [ + 'credentials', + 'workflows', + 'publicApi', + 'license', + 'variables', + 'executions', + ]; endpointGroups.forEach((group) => (ROUTER_GROUP.includes(group) ? routerEndpoints : functionEndpoints).push(group), @@ -176,6 +184,7 @@ export const setupTestServer = ({ workflows: { controller: workflowsController, path: 'workflows' }, license: { controller: licenseController, path: 'license' }, variables: { controller: variablesController, path: 'variables' }, + executions: { controller: executionsController, path: 'executions' }, }; if (enablePublicAPI) { diff --git a/packages/cli/test/unit/repositories/execution.repository.test.ts b/packages/cli/test/unit/repositories/execution.repository.test.ts new file mode 100644 index 0000000000000..d00c0610ee68e --- /dev/null +++ b/packages/cli/test/unit/repositories/execution.repository.test.ts @@ -0,0 +1,78 @@ +import { Container } from 'typedi'; +import { DataSource, EntityManager } from 'typeorm'; +import { mock } from 'jest-mock-extended'; +import { mockInstance } from '../../integration/shared/utils/'; +import { ExecutionRepository } from '@/databases/repositories'; +import config from '@/config'; +import { LoggerProxy } from 'n8n-workflow'; +import { getLogger } from '@/Logger'; +import { TIME } from '@/constants'; + +const { objectContaining } = expect; + +describe('ExecutionRepository', () => { + const entityManager = mockInstance(EntityManager); + const dataSource = mockInstance(DataSource, { manager: entityManager }); + dataSource.getMetadata.mockReturnValue(mock()); + Object.assign(entityManager, { connection: dataSource }); + + const executionRepository = Container.get(ExecutionRepository); + + beforeAll(() => { + Container.set(ExecutionRepository, executionRepository); + LoggerProxy.init(getLogger()); + }); + + beforeEach(() => { + config.load(config.default); + + jest.clearAllMocks(); + }); + + describe('pruneBySoftDeleting()', () => { + test('should soft-delete executions based on batch size', async () => { + config.set('executions.pruneDataMaxCount', 0); // disable path + + executionRepository.setDeletionBatchSize(5); + + const find = jest.spyOn(ExecutionRepository.prototype, 'find'); + entityManager.find.mockResolvedValueOnce([]); + + await executionRepository.pruneBySoftDeleting(); + + expect(find.mock.calls[0][0]).toEqual( + objectContaining({ take: executionRepository.getDeletionBatchSize() }), + ); + }); + + test('should limit pruning based on EXECUTIONS_DATA_PRUNE_MAX_COUNT', async () => { + const maxCount = 1; + + config.set('executions.pruneDataMaxCount', maxCount); + + const find = jest.spyOn(ExecutionRepository.prototype, 'find'); + entityManager.find.mockResolvedValue([]); + + await executionRepository.pruneBySoftDeleting(); + + expect(find.mock.calls[0][0]).toEqual(objectContaining({ skip: maxCount })); + }); + }); + + describe('pruningLimit()', () => { + test('should limit pruning based on EXECUTIONS_DATA_MAX_AGE', async () => { + config.set('executions.pruneDataMaxCount', 0); // disable path + + const maxAge = 5; // hours + + config.set('executions.pruneDataMaxAge', maxAge); + + const now = Date.now(); + const limit = executionRepository.pruningLimit(); + + const difference = now - limit.valueOf(); + + expect(difference / TIME.HOUR).toBe(maxAge); + }); + }); +}); From 738eb1b6eb940fa2ad83d65d5608bb9de88b8cc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 6 Sep 2023 11:53:05 +0200 Subject: [PATCH 013/103] Improve test --- .../repositories/execution.repository.ts | 29 +++++-------------- .../repositories/execution.repository.test.ts | 24 +++++++++------ 2 files changed, 22 insertions(+), 31 deletions(-) diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 1ea6b707fa9e7..75773a2e97fd6 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -68,7 +68,7 @@ function parseFiltersToQueryBuilder( @Service() export class ExecutionRepository extends Repository { - private deletionBatchSize = 100; + deletionBatchSize = 100; constructor( dataSource: DataSource, @@ -83,14 +83,6 @@ export class ExecutionRepository extends Repository { setInterval(async () => this.hardDelete(), 15 * TIME.MINUTE); } - setDeletionBatchSize(size: number) { - this.deletionBatchSize = size; - } - - getDeletionBatchSize() { - return this.deletionBatchSize; - } - async findMultipleExecutions( queryParams: FindManyOptions, options?: { @@ -428,26 +420,19 @@ export class ExecutionRepository extends Repository { } while (executionIds.length > 0); } - /** - * Return the timestamp up to which executions should be pruned. - */ - pruningLimit() { - const maxAge = config.getEnv('executions.pruneDataMaxAge'); // hours - - const date = new Date(); - date.setHours(date.getHours() - maxAge); - - return date; - } - async pruneBySoftDeleting() { Logger.verbose('Pruning (soft-deleting) execution data from database'); + const maxAge = config.getEnv('executions.pruneDataMaxAge'); // in h const maxCount = config.getEnv('executions.pruneDataMaxCount'); + // Find ids of all executions that were stopped longer that pruneDataMaxAge ago + const date = new Date(); + date.setHours(date.getHours() - maxAge); + const toPrune: Array> = [ // date reformatting needed - see https://github.com/typeorm/typeorm/issues/2286 - { stoppedAt: LessThanOrEqual(DateUtils.mixedDateToUtcDatetimeString(this.pruningLimit())) }, + { stoppedAt: LessThanOrEqual(DateUtils.mixedDateToUtcDatetimeString(date)) }, ]; if (maxCount > 0) { diff --git a/packages/cli/test/unit/repositories/execution.repository.test.ts b/packages/cli/test/unit/repositories/execution.repository.test.ts index d00c0610ee68e..a1307de31e80b 100644 --- a/packages/cli/test/unit/repositories/execution.repository.test.ts +++ b/packages/cli/test/unit/repositories/execution.repository.test.ts @@ -7,6 +7,9 @@ import config from '@/config'; import { LoggerProxy } from 'n8n-workflow'; import { getLogger } from '@/Logger'; import { TIME } from '@/constants'; +import { DateUtils } from 'typeorm/util/DateUtils'; + +jest.mock('typeorm/util/DateUtils'); const { objectContaining } = expect; @@ -33,7 +36,7 @@ describe('ExecutionRepository', () => { test('should soft-delete executions based on batch size', async () => { config.set('executions.pruneDataMaxCount', 0); // disable path - executionRepository.setDeletionBatchSize(5); + executionRepository.deletionBatchSize = 5; const find = jest.spyOn(ExecutionRepository.prototype, 'find'); entityManager.find.mockResolvedValueOnce([]); @@ -41,7 +44,7 @@ describe('ExecutionRepository', () => { await executionRepository.pruneBySoftDeleting(); expect(find.mock.calls[0][0]).toEqual( - objectContaining({ take: executionRepository.getDeletionBatchSize() }), + objectContaining({ take: executionRepository.deletionBatchSize }), ); }); @@ -57,22 +60,25 @@ describe('ExecutionRepository', () => { expect(find.mock.calls[0][0]).toEqual(objectContaining({ skip: maxCount })); }); - }); - describe('pruningLimit()', () => { test('should limit pruning based on EXECUTIONS_DATA_MAX_AGE', async () => { + const maxAge = 5; // hours + config.set('executions.pruneDataMaxCount', 0); // disable path + config.set('executions.pruneDataMaxAge', 5); - const maxAge = 5; // hours + entityManager.find.mockResolvedValue([]); - config.set('executions.pruneDataMaxAge', maxAge); + const dateFormat = jest.spyOn(DateUtils, 'mixedDateToUtcDatetimeString'); const now = Date.now(); - const limit = executionRepository.pruningLimit(); - const difference = now - limit.valueOf(); + await executionRepository.pruneBySoftDeleting(); + + const argDate = dateFormat.mock.calls[0][0]; + const difference = now - argDate.valueOf(); - expect(difference / TIME.HOUR).toBe(maxAge); + expect(Math.round(difference / TIME.HOUR)).toBe(maxAge); }); }); }); From 259fe75b8608754e3c41337dca022dbddbb1dd50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 15:20:02 +0200 Subject: [PATCH 014/103] Soft-delete in single pass --- .../repositories/execution.repository.ts | 39 ++++++++++++------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 75773a2e97fd6..0f54e54c8bf34 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -1,5 +1,13 @@ import { Service } from 'typedi'; -import { DataSource, In, IsNull, LessThanOrEqual, MoreThanOrEqual, Repository } from 'typeorm'; +import { + Brackets, + DataSource, + In, + IsNull, + LessThanOrEqual, + MoreThanOrEqual, + Repository, +} from 'typeorm'; import { DateUtils } from 'typeorm/util/DateUtils'; import type { FindManyOptions, @@ -77,7 +85,7 @@ export class ExecutionRepository extends Repository { super(ExecutionEntity, dataSource.manager); if (config.getEnv('executions.pruneData')) { - setInterval(async () => this.pruneBySoftDeleting(), TIME.HOUR); + setInterval(async () => this.pruneBySoftDeleting(), 10 * TIME.SECOND); } setInterval(async () => this.hardDelete(), 15 * TIME.MINUTE); @@ -430,7 +438,7 @@ export class ExecutionRepository extends Repository { const date = new Date(); date.setHours(date.getHours() - maxAge); - const toPrune: Array> = [ + const toPrune: Array> = [ // date reformatting needed - see https://github.com/typeorm/typeorm/issues/2286 { stoppedAt: LessThanOrEqual(DateUtils.mixedDateToUtcDatetimeString(date)) }, ]; @@ -448,18 +456,19 @@ export class ExecutionRepository extends Repository { } } - const executionIds = ( - await this.find({ - select: ['id'], - where: toPrune, - take: this.deletionBatchSize, - }) - ).map(({ id }) => id); - await this.softDelete(executionIds); - - if (executionIds.length === this.deletionBatchSize) { - setTimeout(async () => this.pruneBySoftDeleting(), TIME.SECOND); - } + const [timeBasedWhere, countBasedWhere] = toPrune; + + await this.createQueryBuilder() + .update(ExecutionEntity) + .set({ deletedAt: new Date() }) + .where( + new Brackets((qb) => + countBasedWhere + ? qb.where(timeBasedWhere).orWhere(countBasedWhere) + : qb.where(timeBasedWhere), + ), + ) + .execute(); } /** From f3f5b271b8d655f611db2352186d8fa20858a341 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 15:23:58 +0200 Subject: [PATCH 015/103] Restore value --- packages/cli/src/databases/repositories/execution.repository.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 0f54e54c8bf34..1670dab32360d 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -85,7 +85,7 @@ export class ExecutionRepository extends Repository { super(ExecutionEntity, dataSource.manager); if (config.getEnv('executions.pruneData')) { - setInterval(async () => this.pruneBySoftDeleting(), 10 * TIME.SECOND); + setInterval(async () => this.pruneBySoftDeleting(), 10 * TIME.HOUR); } setInterval(async () => this.hardDelete(), 15 * TIME.MINUTE); From 65d17cba684ac36d4a1882424b2f6615fe4923af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 15:24:44 +0200 Subject: [PATCH 016/103] Restore from debug value --- packages/cli/src/databases/repositories/execution.repository.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 1670dab32360d..6cd7561026b3a 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -85,7 +85,7 @@ export class ExecutionRepository extends Repository { super(ExecutionEntity, dataSource.manager); if (config.getEnv('executions.pruneData')) { - setInterval(async () => this.pruneBySoftDeleting(), 10 * TIME.HOUR); + setInterval(async () => this.pruneBySoftDeleting(), 1 * TIME.HOUR); } setInterval(async () => this.hardDelete(), 15 * TIME.MINUTE); From 96ae7b464d6f50ecf0255dca2e982a49685b6117 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 15:30:21 +0200 Subject: [PATCH 017/103] Add clarifying comments --- .../cli/test/unit/repositories/execution.repository.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cli/test/unit/repositories/execution.repository.test.ts b/packages/cli/test/unit/repositories/execution.repository.test.ts index a1307de31e80b..d6a77e678e898 100644 --- a/packages/cli/test/unit/repositories/execution.repository.test.ts +++ b/packages/cli/test/unit/repositories/execution.repository.test.ts @@ -34,7 +34,7 @@ describe('ExecutionRepository', () => { describe('pruneBySoftDeleting()', () => { test('should soft-delete executions based on batch size', async () => { - config.set('executions.pruneDataMaxCount', 0); // disable path + config.set('executions.pruneDataMaxCount', 0); // disable prune-by-count path executionRepository.deletionBatchSize = 5; @@ -64,7 +64,7 @@ describe('ExecutionRepository', () => { test('should limit pruning based on EXECUTIONS_DATA_MAX_AGE', async () => { const maxAge = 5; // hours - config.set('executions.pruneDataMaxCount', 0); // disable path + config.set('executions.pruneDataMaxCount', 0); // disable prune-by-count path config.set('executions.pruneDataMaxAge', 5); entityManager.find.mockResolvedValue([]); From 21040b5ac86cd49e0bf14e0c68b1e444a7f64749 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 16:40:44 +0200 Subject: [PATCH 018/103] Update tests --- .../repositories/execution.repository.test.ts | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/packages/cli/test/unit/repositories/execution.repository.test.ts b/packages/cli/test/unit/repositories/execution.repository.test.ts index d6a77e678e898..ac15979504a6f 100644 --- a/packages/cli/test/unit/repositories/execution.repository.test.ts +++ b/packages/cli/test/unit/repositories/execution.repository.test.ts @@ -13,6 +13,14 @@ jest.mock('typeorm/util/DateUtils'); const { objectContaining } = expect; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const qb: any = { + update: jest.fn().mockReturnThis(), + set: jest.fn().mockReturnThis(), + where: jest.fn().mockReturnThis(), + execute: jest.fn().mockReturnThis(), +}; + describe('ExecutionRepository', () => { const entityManager = mockInstance(EntityManager); const dataSource = mockInstance(DataSource, { manager: entityManager }); @@ -33,21 +41,6 @@ describe('ExecutionRepository', () => { }); describe('pruneBySoftDeleting()', () => { - test('should soft-delete executions based on batch size', async () => { - config.set('executions.pruneDataMaxCount', 0); // disable prune-by-count path - - executionRepository.deletionBatchSize = 5; - - const find = jest.spyOn(ExecutionRepository.prototype, 'find'); - entityManager.find.mockResolvedValueOnce([]); - - await executionRepository.pruneBySoftDeleting(); - - expect(find.mock.calls[0][0]).toEqual( - objectContaining({ take: executionRepository.deletionBatchSize }), - ); - }); - test('should limit pruning based on EXECUTIONS_DATA_PRUNE_MAX_COUNT', async () => { const maxCount = 1; @@ -56,6 +49,8 @@ describe('ExecutionRepository', () => { const find = jest.spyOn(ExecutionRepository.prototype, 'find'); entityManager.find.mockResolvedValue([]); + jest.spyOn(ExecutionRepository.prototype, 'createQueryBuilder').mockReturnValueOnce(qb); + await executionRepository.pruneBySoftDeleting(); expect(find.mock.calls[0][0]).toEqual(objectContaining({ skip: maxCount })); @@ -69,6 +64,8 @@ describe('ExecutionRepository', () => { entityManager.find.mockResolvedValue([]); + jest.spyOn(ExecutionRepository.prototype, 'createQueryBuilder').mockReturnValueOnce(qb); + const dateFormat = jest.spyOn(DateUtils, 'mixedDateToUtcDatetimeString'); const now = Date.now(); From aa4d63344c700acc20879bd772fced759969da02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 17:51:58 +0200 Subject: [PATCH 019/103] Add `typedi` to core --- pnpm-lock.yaml | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 22df3e77bb104..8bda6a0b0a436 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -630,6 +630,9 @@ importers: qs: specifier: ^6.10.1 version: 6.11.0 + typedi: + specifier: ^0.10.0 + version: 0.10.0(patch_hash=62r6bc2crgimafeyruodhqlgo4) uuid: specifier: ^8.3.2 version: 8.3.2 @@ -6759,7 +6762,7 @@ packages: ts-dedent: 2.2.0 type-fest: 3.13.1 vue: 3.3.4 - vue-component-type-helpers: 1.8.8 + vue-component-type-helpers: 1.8.11 transitivePeerDependencies: - encoding - supports-color @@ -9069,7 +9072,7 @@ packages: /axios@0.21.4: resolution: {integrity: sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==} dependencies: - follow-redirects: 1.15.2(debug@4.3.4) + follow-redirects: 1.15.2(debug@3.2.7) transitivePeerDependencies: - debug dev: false @@ -9098,11 +9101,12 @@ packages: form-data: 4.0.0 transitivePeerDependencies: - debug + dev: true /axios@1.4.0: resolution: {integrity: sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==} dependencies: - follow-redirects: 1.15.2(debug@4.3.4) + follow-redirects: 1.15.2(debug@3.2.7) form-data: 4.0.0 proxy-from-env: 1.1.0 transitivePeerDependencies: @@ -12674,6 +12678,7 @@ packages: optional: true dependencies: debug: 4.3.4(supports-color@8.1.1) + dev: true /for-each@0.3.3: resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==} @@ -18003,7 +18008,7 @@ packages: resolution: {integrity: sha512-aXYe/D+28kF63W8Cz53t09ypEORz+ULeDCahdAqhVrRm2scbOXFbtnn0GGhvMpYe45grepLKuwui9KxrZ2ZuMw==} engines: {node: '>=14.17.0'} dependencies: - axios: 0.27.2(debug@4.3.4) + axios: 0.27.2(debug@3.2.7) transitivePeerDependencies: - debug dev: false @@ -21734,12 +21739,12 @@ packages: vue: 3.3.4 dev: false - /vue-component-type-helpers@1.8.4: - resolution: {integrity: sha512-6bnLkn8O0JJyiFSIF0EfCogzeqNXpnjJ0vW/SZzNHfe6sPx30lTtTXlE5TFs2qhJlAtDFybStVNpL73cPe3OMQ==} + /vue-component-type-helpers@1.8.11: + resolution: {integrity: sha512-CWItFzuEWjkSXDeMGwQEc5cFH4FaueyPQHfi1mBDe+wA2JABqNjFxFUtmZJ9WHkb0HpEwqgBg1umiXrWYXkXHw==} dev: true - /vue-component-type-helpers@1.8.8: - resolution: {integrity: sha512-Ohv9HQY92nSbpReC6WhY0X4YkOszHzwUHaaN/lev5tHQLM1AEw+LrLeB2bIGIyKGDU7ZVrncXcv/oBny4rjbYg==} + /vue-component-type-helpers@1.8.4: + resolution: {integrity: sha512-6bnLkn8O0JJyiFSIF0EfCogzeqNXpnjJ0vW/SZzNHfe6sPx30lTtTXlE5TFs2qhJlAtDFybStVNpL73cPe3OMQ==} dev: true /vue-demi@0.14.5(vue@3.3.4): From e54becb0f21f7bfd6440da05102a049a03209db0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 17:52:10 +0200 Subject: [PATCH 020/103] Update package.json --- packages/core/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/core/package.json b/packages/core/package.json index a0886df9010b3..d5e4ca1997dfa 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -44,8 +44,8 @@ "@types/uuid": "^8.3.2" }, "dependencies": { - "axios": "^0.21.1", "@n8n/client-oauth2": "workspace:*", + "axios": "^0.21.1", "concat-stream": "^2.0.0", "cron": "~1.7.2", "crypto-js": "~4.1.1", @@ -60,6 +60,7 @@ "p-cancelable": "^2.0.0", "pretty-bytes": "^5.6.0", "qs": "^6.10.1", + "typedi": "^0.10.0", "uuid": "^8.3.2" } } From 80e7258570ef5e0c11840393052738bbeb648212 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 17:52:52 +0200 Subject: [PATCH 021/103] Turn `BinaryDataManager` into service --- packages/core/src/BinaryDataManager/index.ts | 34 ++++++-------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/packages/core/src/BinaryDataManager/index.ts b/packages/core/src/BinaryDataManager/index.ts index eca32f184da26..3c64aed2977fb 100644 --- a/packages/core/src/BinaryDataManager/index.ts +++ b/packages/core/src/BinaryDataManager/index.ts @@ -6,47 +6,31 @@ import { BINARY_ENCODING } from 'n8n-workflow'; import type { IBinaryDataConfig, IBinaryDataManager } from '../Interfaces'; import { BinaryDataFileSystem } from './FileSystem'; import { binaryToBuffer } from './utils'; +import { Service } from 'typedi'; +@Service() export class BinaryDataManager { - static instance: BinaryDataManager | undefined; - private managers: { [key: string]: IBinaryDataManager; - }; + } = {}; - private binaryDataMode: string; + private binaryDataMode = ''; - private availableModes: string[]; + private availableModes: string[] = []; - constructor(config: IBinaryDataConfig) { + async init(config: IBinaryDataConfig, mainManager = false) { this.binaryDataMode = config.mode; this.availableModes = config.availableModes.split(','); this.managers = {}; - } - - static async init(config: IBinaryDataConfig, mainManager = false): Promise { - if (BinaryDataManager.instance) { - throw new Error('Binary Data Manager already initialized'); - } - - BinaryDataManager.instance = new BinaryDataManager(config); - if (BinaryDataManager.instance.availableModes.includes('filesystem')) { - BinaryDataManager.instance.managers.filesystem = new BinaryDataFileSystem(config); - await BinaryDataManager.instance.managers.filesystem.init(mainManager); + if (this.availableModes.includes('filesystem')) { + this.managers.filesystem = new BinaryDataFileSystem(config); + await this.managers.filesystem.init(mainManager); } return undefined; } - static getInstance(): BinaryDataManager { - if (!BinaryDataManager.instance) { - throw new Error('Binary Data Manager not initialized'); - } - - return BinaryDataManager.instance; - } - async copyBinaryFile( binaryData: IBinaryData, filePath: string, From 3a8960fcdf1489153fabbfe333f4869e3da62ea4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 17:53:23 +0200 Subject: [PATCH 022/103] Add `object` to binary data schema options --- packages/cli/src/config/schema.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/config/schema.ts b/packages/cli/src/config/schema.ts index 2782fa0f02245..cf9ea86c4b740 100644 --- a/packages/cli/src/config/schema.ts +++ b/packages/cli/src/config/schema.ts @@ -885,12 +885,12 @@ export const schema = { binaryDataManager: { availableModes: { format: String, - default: 'filesystem', + default: 'filesystem,object', env: 'N8N_AVAILABLE_BINARY_DATA_MODES', doc: 'Available modes of binary data storage, as comma separated strings', }, mode: { - format: ['default', 'filesystem'] as const, + format: ['default', 'filesystem', 'object'] as const, default: 'default', env: 'N8N_DEFAULT_BINARY_DATA_MODE', doc: 'Storage mode for binary data', From 64b1a960e6bd9eb7611348428d178a5e6182d8f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 17:54:55 +0200 Subject: [PATCH 023/103] Use service throughout core and cli --- packages/cli/src/Server.ts | 8 +++++--- packages/cli/src/WebhookHelpers.ts | 4 ++-- packages/cli/src/WorkflowRunnerProcess.ts | 2 +- packages/cli/src/commands/BaseCommand.ts | 2 +- .../repositories/execution.repository.ts | 4 ++-- packages/core/src/NodeExecuteFunctions.ts | 15 ++++++++------- 6 files changed, 19 insertions(+), 16 deletions(-) diff --git a/packages/cli/src/Server.ts b/packages/cli/src/Server.ts index 08d5df488e00a..dcfafbe3dec58 100644 --- a/packages/cli/src/Server.ts +++ b/packages/cli/src/Server.ts @@ -201,6 +201,8 @@ export class Server extends AbstractServer { push: Push; + binaryDataManager: BinaryDataManager; + constructor() { super('main'); @@ -356,6 +358,7 @@ export class Server extends AbstractServer { this.endpointPresetCredentials = config.getEnv('credentials.overwrite.endpoint'); this.push = Container.get(Push); + this.binaryDataManager = Container.get(BinaryDataManager); await super.start(); LoggerProxy.debug(`Server ID: ${this.uniqueInstanceId}`); @@ -1421,13 +1424,12 @@ export class Server extends AbstractServer { async (req: BinaryDataRequest, res: express.Response): Promise => { // TODO UM: check if this needs permission check for UM const identifier = req.params.path; - const binaryDataManager = BinaryDataManager.getInstance(); try { - const binaryPath = binaryDataManager.getBinaryPath(identifier); + const binaryPath = this.binaryDataManager.getBinaryPath(identifier); let { mode, fileName, mimeType } = req.query; if (!fileName || !mimeType) { try { - const metadata = await binaryDataManager.getBinaryMetadata(identifier); + const metadata = await this.binaryDataManager.getBinaryMetadata(identifier); fileName = metadata.fileName; mimeType = metadata.mimeType; res.setHeader('Content-Length', metadata.fileSize); diff --git a/packages/cli/src/WebhookHelpers.ts b/packages/cli/src/WebhookHelpers.ts index c5c05f4215726..aa84e798ce2bb 100644 --- a/packages/cli/src/WebhookHelpers.ts +++ b/packages/cli/src/WebhookHelpers.ts @@ -514,7 +514,7 @@ export async function executeWebhook( const binaryData = (response.body as IDataObject)?.binaryData as IBinaryData; if (binaryData?.id) { res.header(response.headers); - const stream = BinaryDataManager.getInstance().getBinaryStream(binaryData.id); + const stream = Container.get(BinaryDataManager).getBinaryStream(binaryData.id); void pipeline(stream, res).then(() => responseCallback(null, { noWebhookResponse: true }), ); @@ -732,7 +732,7 @@ export async function executeWebhook( // Send the webhook response manually res.setHeader('Content-Type', binaryData.mimeType); if (binaryData.id) { - const stream = BinaryDataManager.getInstance().getBinaryStream(binaryData.id); + const stream = Container.get(BinaryDataManager).getBinaryStream(binaryData.id); await pipeline(stream, res); } else { res.end(Buffer.from(binaryData.data, BINARY_ENCODING)); diff --git a/packages/cli/src/WorkflowRunnerProcess.ts b/packages/cli/src/WorkflowRunnerProcess.ts index d1b009a77ecb8..642a7cb98f727 100644 --- a/packages/cli/src/WorkflowRunnerProcess.ts +++ b/packages/cli/src/WorkflowRunnerProcess.ts @@ -123,7 +123,7 @@ class WorkflowRunnerProcess { await Container.get(InternalHooks).init(instanceId); const binaryDataConfig = config.getEnv('binaryDataManager'); - await BinaryDataManager.init(binaryDataConfig); + await Container.get(BinaryDataManager).init(binaryDataConfig); const license = Container.get(License); await license.init(instanceId); diff --git a/packages/cli/src/commands/BaseCommand.ts b/packages/cli/src/commands/BaseCommand.ts index 78be7cc23f04c..012d581be2f25 100644 --- a/packages/cli/src/commands/BaseCommand.ts +++ b/packages/cli/src/commands/BaseCommand.ts @@ -105,7 +105,7 @@ export abstract class BaseCommand extends Command { protected async initBinaryManager() { const binaryDataConfig = config.getEnv('binaryDataManager'); - await BinaryDataManager.init(binaryDataConfig, true); + await Container.get(BinaryDataManager).init(binaryDataConfig, true); } protected async initExternalHooks() { diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 6cd7561026b3a..3d08483a4507d 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -81,6 +81,7 @@ export class ExecutionRepository extends Repository { constructor( dataSource: DataSource, private readonly executionDataRepository: ExecutionDataRepository, + private readonly binaryDataManager: BinaryDataManager, ) { super(ExecutionEntity, dataSource.manager); @@ -489,8 +490,7 @@ export class ExecutionRepository extends Repository { }) ).map(({ id }) => id); - const binaryDataManager = BinaryDataManager.getInstance(); - await binaryDataManager.deleteBinaryDataByExecutionIds(executionIds); + await this.binaryDataManager.deleteBinaryDataByExecutionIds(executionIds); // Actually delete these executions await this.delete({ id: In(executionIds) }); diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index b6757384bb51c..daee20685ed7a 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -138,6 +138,7 @@ import { } from './WorkflowExecutionMetadata'; import { getSecretsProxy } from './Secrets'; import { getUserN8nFolderPath } from './UserSettings'; +import Container from 'typedi'; axios.defaults.timeout = 300000; // Prevent axios from adding x-form-www-urlencoded headers by default @@ -864,21 +865,21 @@ async function httpRequest( } export function getBinaryPath(binaryDataId: string): string { - return BinaryDataManager.getInstance().getBinaryPath(binaryDataId); + return Container.get(BinaryDataManager).getBinaryPath(binaryDataId); } /** * Returns binary file metadata */ export async function getBinaryMetadata(binaryDataId: string): Promise { - return BinaryDataManager.getInstance().getBinaryMetadata(binaryDataId); + return Container.get(BinaryDataManager).getBinaryMetadata(binaryDataId); } /** * Returns binary file stream for piping */ export function getBinaryStream(binaryDataId: string, chunkSize?: number): Readable { - return BinaryDataManager.getInstance().getBinaryStream(binaryDataId, chunkSize); + return Container.get(BinaryDataManager).getBinaryStream(binaryDataId, chunkSize); } export function assertBinaryData( @@ -915,7 +916,7 @@ export async function getBinaryDataBuffer( inputIndex: number, ): Promise { const binaryData = inputData.main[inputIndex]![itemIndex]!.binary![propertyName]!; - return BinaryDataManager.getInstance().getBinaryDataBuffer(binaryData); + return Container.get(BinaryDataManager).getBinaryDataBuffer(binaryData); } /** @@ -931,7 +932,7 @@ export async function setBinaryDataBuffer( binaryData: Buffer | Readable, executionId: string, ): Promise { - return BinaryDataManager.getInstance().storeBinaryData(data, binaryData, executionId); + return Container.get(BinaryDataManager).storeBinaryData(data, binaryData, executionId); } export async function copyBinaryFile( @@ -984,7 +985,7 @@ export async function copyBinaryFile( returnData.fileName = path.parse(filePath).base; } - return BinaryDataManager.getInstance().copyBinaryFile(returnData, filePath, executionId); + return Container.get(BinaryDataManager).copyBinaryFile(returnData, filePath, executionId); } /** @@ -2563,7 +2564,7 @@ export function getExecuteFunctions( parentWorkflowSettings: workflow.settings, }) .then(async (result) => - BinaryDataManager.getInstance().duplicateBinaryData( + Container.get(BinaryDataManager).duplicateBinaryData( result, additionalData.executionId!, ), From f974b55461d886782841baaf1564944e34fd6235 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 17:55:00 +0200 Subject: [PATCH 024/103] Update tests --- .../core/test/NodeExecuteFunctions.test.ts | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/packages/core/test/NodeExecuteFunctions.test.ts b/packages/core/test/NodeExecuteFunctions.test.ts index ec2a32e7f0e54..27525a43311c1 100644 --- a/packages/core/test/NodeExecuteFunctions.test.ts +++ b/packages/core/test/NodeExecuteFunctions.test.ts @@ -18,23 +18,23 @@ import { proxyRequestToAxios, } from '@/NodeExecuteFunctions'; import { initLogger } from './helpers/utils'; +import Container from 'typedi'; +import type { IBinaryDataConfig } from '@/Interfaces'; const temporaryDir = mkdtempSync(join(tmpdir(), 'n8n')); describe('NodeExecuteFunctions', () => { describe('test binary data helper methods', () => { - // Reset BinaryDataManager for each run. This is a dirty operation, as individual managers are not cleaned. - beforeEach(() => { - BinaryDataManager.instance = undefined; - }); - test("test getBinaryDataBuffer(...) & setBinaryDataBuffer(...) methods in 'default' mode", async () => { // Setup a 'default' binary data manager instance - await BinaryDataManager.init({ + const config: IBinaryDataConfig = { mode: 'default', availableModes: 'default', localStoragePath: temporaryDir, - }); + }; + Container.set(BinaryDataManager, new BinaryDataManager()); + + await Container.get(BinaryDataManager).init(config); // Set our binary data buffer const inputData: Buffer = Buffer.from('This is some binary data', 'utf8'); @@ -78,12 +78,15 @@ describe('NodeExecuteFunctions', () => { }); test("test getBinaryDataBuffer(...) & setBinaryDataBuffer(...) methods in 'filesystem' mode", async () => { - // Setup a 'filesystem' binary data manager instance - await BinaryDataManager.init({ + const config: IBinaryDataConfig = { mode: 'filesystem', availableModes: 'filesystem', localStoragePath: temporaryDir, - }); + }; + Container.set(BinaryDataManager, new BinaryDataManager()); + + // Setup a 'filesystem' binary data manager instance + await Container.get(BinaryDataManager).init(config); // Set our binary data buffer const inputData: Buffer = Buffer.from('This is some binary data', 'utf8'); From 36d02439df1745c67ce0f76b85eb3c851a8ba198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 18:00:32 +0200 Subject: [PATCH 025/103] Reduce diff --- .../core/test/NodeExecuteFunctions.test.ts | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/packages/core/test/NodeExecuteFunctions.test.ts b/packages/core/test/NodeExecuteFunctions.test.ts index 27525a43311c1..9fb7890f7081f 100644 --- a/packages/core/test/NodeExecuteFunctions.test.ts +++ b/packages/core/test/NodeExecuteFunctions.test.ts @@ -19,7 +19,6 @@ import { } from '@/NodeExecuteFunctions'; import { initLogger } from './helpers/utils'; import Container from 'typedi'; -import type { IBinaryDataConfig } from '@/Interfaces'; const temporaryDir = mkdtempSync(join(tmpdir(), 'n8n')); @@ -27,14 +26,13 @@ describe('NodeExecuteFunctions', () => { describe('test binary data helper methods', () => { test("test getBinaryDataBuffer(...) & setBinaryDataBuffer(...) methods in 'default' mode", async () => { // Setup a 'default' binary data manager instance - const config: IBinaryDataConfig = { + Container.set(BinaryDataManager, new BinaryDataManager()); + + await Container.get(BinaryDataManager).init({ mode: 'default', availableModes: 'default', localStoragePath: temporaryDir, - }; - Container.set(BinaryDataManager, new BinaryDataManager()); - - await Container.get(BinaryDataManager).init(config); + }); // Set our binary data buffer const inputData: Buffer = Buffer.from('This is some binary data', 'utf8'); @@ -78,15 +76,14 @@ describe('NodeExecuteFunctions', () => { }); test("test getBinaryDataBuffer(...) & setBinaryDataBuffer(...) methods in 'filesystem' mode", async () => { - const config: IBinaryDataConfig = { - mode: 'filesystem', - availableModes: 'filesystem', - localStoragePath: temporaryDir, - }; Container.set(BinaryDataManager, new BinaryDataManager()); // Setup a 'filesystem' binary data manager instance - await Container.get(BinaryDataManager).init(config); + await Container.get(BinaryDataManager).init({ + mode: 'filesystem', + availableModes: 'filesystem', + localStoragePath: temporaryDir, + }); // Set our binary data buffer const inputData: Buffer = Buffer.from('This is some binary data', 'utf8'); From 5e209e9d147bf9b34d233848270df648e3778ba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 18:02:15 +0200 Subject: [PATCH 026/103] Update default --- packages/core/src/BinaryDataManager/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/BinaryDataManager/index.ts b/packages/core/src/BinaryDataManager/index.ts index 3c64aed2977fb..9e0d9dab2b629 100644 --- a/packages/core/src/BinaryDataManager/index.ts +++ b/packages/core/src/BinaryDataManager/index.ts @@ -14,7 +14,7 @@ export class BinaryDataManager { [key: string]: IBinaryDataManager; } = {}; - private binaryDataMode = ''; + private binaryDataMode = 'default'; private availableModes: string[] = []; From 0c3a6aab28deeabb68c34584f532ab69dc52043c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 18:04:06 +0200 Subject: [PATCH 027/103] Add docline --- packages/core/src/BinaryDataManager/index.ts | 28 ++++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/packages/core/src/BinaryDataManager/index.ts b/packages/core/src/BinaryDataManager/index.ts index 9e0d9dab2b629..44e549b3d115a 100644 --- a/packages/core/src/BinaryDataManager/index.ts +++ b/packages/core/src/BinaryDataManager/index.ts @@ -14,12 +14,18 @@ export class BinaryDataManager { [key: string]: IBinaryDataManager; } = {}; - private binaryDataMode = 'default'; + /** + * Mode for storing binary data: + * - `default` (in memory) + * - `filesystem` (on disk) + * - `object` (S3) + */ + private mode = 'default'; private availableModes: string[] = []; async init(config: IBinaryDataConfig, mainManager = false) { - this.binaryDataMode = config.mode; + this.mode = config.mode; this.availableModes = config.availableModes.split(','); this.managers = {}; @@ -37,14 +43,14 @@ export class BinaryDataManager { executionId: string, ): Promise { // If a manager handles this binary, copy over the binary file and return its reference id. - const manager = this.managers[this.binaryDataMode]; + const manager = this.managers[this.mode]; if (manager) { const identifier = await manager.copyBinaryFile(filePath, executionId); // Add data manager reference id. binaryData.id = this.generateBinaryId(identifier); // Prevent preserving data in memory if handled by a data manager. - binaryData.data = this.binaryDataMode; + binaryData.data = this.mode; const fileSize = await manager.getFileSize(identifier); binaryData.fileSize = prettyBytes(fileSize); @@ -69,7 +75,7 @@ export class BinaryDataManager { executionId: string, ): Promise { // If a manager handles this binary, return the binary data with its reference id. - const manager = this.managers[this.binaryDataMode]; + const manager = this.managers[this.mode]; if (manager) { const identifier = await manager.storeBinaryData(input, executionId); @@ -77,7 +83,7 @@ export class BinaryDataManager { binaryData.id = this.generateBinaryId(identifier); // Prevent preserving data in memory if handled by a data manager. - binaryData.data = this.binaryDataMode; + binaryData.data = this.mode; const fileSize = await manager.getFileSize(identifier); binaryData.fileSize = prettyBytes(fileSize); @@ -141,8 +147,8 @@ export class BinaryDataManager { } async deleteBinaryDataByExecutionIds(executionIds: string[]): Promise { - if (this.managers[this.binaryDataMode]) { - await this.managers[this.binaryDataMode].deleteBinaryDataByExecutionIds(executionIds); + if (this.managers[this.mode]) { + await this.managers[this.mode].deleteBinaryDataByExecutionIds(executionIds); } } @@ -150,7 +156,7 @@ export class BinaryDataManager { inputData: Array, executionId: string, ): Promise { - if (inputData && this.managers[this.binaryDataMode]) { + if (inputData && this.managers[this.mode]) { const returnInputData = (inputData as INodeExecutionData[][]).map( async (executionDataArray) => { if (executionDataArray) { @@ -176,7 +182,7 @@ export class BinaryDataManager { } private generateBinaryId(filename: string) { - return `${this.binaryDataMode}:${filename}`; + return `${this.mode}:${filename}`; } private splitBinaryModeFileId(fileId: string): { mode: string; id: string } { @@ -188,7 +194,7 @@ export class BinaryDataManager { executionData: INodeExecutionData, executionId: string, ): Promise { - const binaryManager = this.managers[this.binaryDataMode]; + const binaryManager = this.managers[this.mode]; if (executionData.binary) { const binaryDataKeys = Object.keys(executionData.binary); From 3d4beba381ca0295025c6e9e595fb1e6592218c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 18:05:16 +0200 Subject: [PATCH 028/103] Remove redundant output types --- packages/core/src/BinaryDataManager/index.ts | 27 ++++++-------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/packages/core/src/BinaryDataManager/index.ts b/packages/core/src/BinaryDataManager/index.ts index 44e549b3d115a..43bd9f1537d85 100644 --- a/packages/core/src/BinaryDataManager/index.ts +++ b/packages/core/src/BinaryDataManager/index.ts @@ -37,11 +37,7 @@ export class BinaryDataManager { return undefined; } - async copyBinaryFile( - binaryData: IBinaryData, - filePath: string, - executionId: string, - ): Promise { + async copyBinaryFile(binaryData: IBinaryData, filePath: string, executionId: string) { // If a manager handles this binary, copy over the binary file and return its reference id. const manager = this.managers[this.mode]; if (manager) { @@ -69,11 +65,7 @@ export class BinaryDataManager { return binaryData; } - async storeBinaryData( - binaryData: IBinaryData, - input: Buffer | Readable, - executionId: string, - ): Promise { + async storeBinaryData(binaryData: IBinaryData, input: Buffer | Readable, executionId: string) { // If a manager handles this binary, return the binary data with its reference id. const manager = this.managers[this.mode]; if (manager) { @@ -102,7 +94,7 @@ export class BinaryDataManager { return binaryData; } - getBinaryStream(identifier: string, chunkSize?: number): Readable { + getBinaryStream(identifier: string, chunkSize?: number) { const { mode, id } = this.splitBinaryModeFileId(identifier); if (this.managers[mode]) { return this.managers[mode].getBinaryStream(id, chunkSize); @@ -128,7 +120,7 @@ export class BinaryDataManager { throw new Error('Storage mode used to store binary data not available'); } - getBinaryPath(identifier: string): string { + getBinaryPath(identifier: string) { const { mode, id } = this.splitBinaryModeFileId(identifier); if (this.managers[mode]) { return this.managers[mode].getBinaryPath(id); @@ -137,7 +129,7 @@ export class BinaryDataManager { throw new Error('Storage mode used to store binary data not available'); } - async getBinaryMetadata(identifier: string): Promise { + async getBinaryMetadata(identifier: string) { const { mode, id } = this.splitBinaryModeFileId(identifier); if (this.managers[mode]) { return this.managers[mode].getBinaryMetadata(id); @@ -146,16 +138,13 @@ export class BinaryDataManager { throw new Error('Storage mode used to store binary data not available'); } - async deleteBinaryDataByExecutionIds(executionIds: string[]): Promise { + async deleteBinaryDataByExecutionIds(executionIds: string[]) { if (this.managers[this.mode]) { await this.managers[this.mode].deleteBinaryDataByExecutionIds(executionIds); } } - async duplicateBinaryData( - inputData: Array, - executionId: string, - ): Promise { + async duplicateBinaryData(inputData: Array, executionId: string) { if (inputData && this.managers[this.mode]) { const returnInputData = (inputData as INodeExecutionData[][]).map( async (executionDataArray) => { @@ -193,7 +182,7 @@ export class BinaryDataManager { private async duplicateBinaryDataInExecData( executionData: INodeExecutionData, executionId: string, - ): Promise { + ) { const binaryManager = this.managers[this.mode]; if (executionData.binary) { From 1bf7a3de9aa8bc76e226ca08adf352b085ce801e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 18:05:32 +0200 Subject: [PATCH 029/103] Remove unused type --- packages/core/src/BinaryDataManager/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/BinaryDataManager/index.ts b/packages/core/src/BinaryDataManager/index.ts index 43bd9f1537d85..24d85058b420d 100644 --- a/packages/core/src/BinaryDataManager/index.ts +++ b/packages/core/src/BinaryDataManager/index.ts @@ -1,5 +1,5 @@ import { readFile, stat } from 'fs/promises'; -import type { BinaryMetadata, IBinaryData, INodeExecutionData } from 'n8n-workflow'; +import type { IBinaryData, INodeExecutionData } from 'n8n-workflow'; import prettyBytes from 'pretty-bytes'; import type { Readable } from 'stream'; import { BINARY_ENCODING } from 'n8n-workflow'; From 115209253dc41367748924a3ff27290eeba4842a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 18:08:31 +0200 Subject: [PATCH 030/103] Rename to submanagers to client --- .../core/src/BinaryDataManager/FileSystem.ts | 4 +- packages/core/src/BinaryDataManager/index.ts | 72 +++++++++---------- packages/core/src/Interfaces.ts | 2 +- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/packages/core/src/BinaryDataManager/FileSystem.ts b/packages/core/src/BinaryDataManager/FileSystem.ts index 5c4bdd32a9a2c..d8580c3c7e631 100644 --- a/packages/core/src/BinaryDataManager/FileSystem.ts +++ b/packages/core/src/BinaryDataManager/FileSystem.ts @@ -6,13 +6,13 @@ import type { Readable } from 'stream'; import type { BinaryMetadata } from 'n8n-workflow'; import { jsonParse } from 'n8n-workflow'; -import type { IBinaryDataConfig, IBinaryDataManager } from '../Interfaces'; +import type { IBinaryDataConfig, BinaryDataClient } from '../Interfaces'; import { FileNotFoundError } from '../errors'; const executionExtractionRegexp = /^(\w+)(?:[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12})$/; -export class BinaryDataFileSystem implements IBinaryDataManager { +export class BinaryDataFileSystem implements BinaryDataClient { private storagePath: string; constructor(config: IBinaryDataConfig) { diff --git a/packages/core/src/BinaryDataManager/index.ts b/packages/core/src/BinaryDataManager/index.ts index 24d85058b420d..96b0df694477c 100644 --- a/packages/core/src/BinaryDataManager/index.ts +++ b/packages/core/src/BinaryDataManager/index.ts @@ -3,15 +3,15 @@ import type { IBinaryData, INodeExecutionData } from 'n8n-workflow'; import prettyBytes from 'pretty-bytes'; import type { Readable } from 'stream'; import { BINARY_ENCODING } from 'n8n-workflow'; -import type { IBinaryDataConfig, IBinaryDataManager } from '../Interfaces'; +import type { IBinaryDataConfig, BinaryDataClient } from '../Interfaces'; import { BinaryDataFileSystem } from './FileSystem'; import { binaryToBuffer } from './utils'; import { Service } from 'typedi'; @Service() export class BinaryDataManager { - private managers: { - [key: string]: IBinaryDataManager; + private clients: { + [key: string]: BinaryDataClient; } = {}; /** @@ -24,34 +24,34 @@ export class BinaryDataManager { private availableModes: string[] = []; - async init(config: IBinaryDataConfig, mainManager = false) { + async init(config: IBinaryDataConfig, mainClient = false) { this.mode = config.mode; this.availableModes = config.availableModes.split(','); - this.managers = {}; + this.clients = {}; if (this.availableModes.includes('filesystem')) { - this.managers.filesystem = new BinaryDataFileSystem(config); - await this.managers.filesystem.init(mainManager); + this.clients.filesystem = new BinaryDataFileSystem(config); + await this.clients.filesystem.init(mainClient); } return undefined; } async copyBinaryFile(binaryData: IBinaryData, filePath: string, executionId: string) { - // If a manager handles this binary, copy over the binary file and return its reference id. - const manager = this.managers[this.mode]; - if (manager) { - const identifier = await manager.copyBinaryFile(filePath, executionId); - // Add data manager reference id. + // If a client handles this binary, copy over the binary file and return its reference id. + const client = this.clients[this.mode]; + if (client) { + const identifier = await client.copyBinaryFile(filePath, executionId); + // Add client reference id. binaryData.id = this.generateBinaryId(identifier); - // Prevent preserving data in memory if handled by a data manager. + // Prevent preserving data in memory if handled by a client. binaryData.data = this.mode; - const fileSize = await manager.getFileSize(identifier); + const fileSize = await client.getFileSize(identifier); binaryData.fileSize = prettyBytes(fileSize); - await manager.storeBinaryMetadata(identifier, { + await client.storeBinaryMetadata(identifier, { fileName: binaryData.fileName, mimeType: binaryData.mimeType, fileSize, @@ -66,21 +66,21 @@ export class BinaryDataManager { } async storeBinaryData(binaryData: IBinaryData, input: Buffer | Readable, executionId: string) { - // If a manager handles this binary, return the binary data with its reference id. - const manager = this.managers[this.mode]; - if (manager) { - const identifier = await manager.storeBinaryData(input, executionId); + // If a client handles this binary, return the binary data with its reference id. + const client = this.clients[this.mode]; + if (client) { + const identifier = await client.storeBinaryData(input, executionId); - // Add data manager reference id. + // Add client reference id. binaryData.id = this.generateBinaryId(identifier); - // Prevent preserving data in memory if handled by a data manager. + // Prevent preserving data in memory if handled by a client. binaryData.data = this.mode; - const fileSize = await manager.getFileSize(identifier); + const fileSize = await client.getFileSize(identifier); binaryData.fileSize = prettyBytes(fileSize); - await manager.storeBinaryMetadata(identifier, { + await client.storeBinaryMetadata(identifier, { fileName: binaryData.fileName, mimeType: binaryData.mimeType, fileSize, @@ -96,8 +96,8 @@ export class BinaryDataManager { getBinaryStream(identifier: string, chunkSize?: number) { const { mode, id } = this.splitBinaryModeFileId(identifier); - if (this.managers[mode]) { - return this.managers[mode].getBinaryStream(id, chunkSize); + if (this.clients[mode]) { + return this.clients[mode].getBinaryStream(id, chunkSize); } throw new Error('Storage mode used to store binary data not available'); @@ -113,8 +113,8 @@ export class BinaryDataManager { async retrieveBinaryDataByIdentifier(identifier: string): Promise { const { mode, id } = this.splitBinaryModeFileId(identifier); - if (this.managers[mode]) { - return this.managers[mode].retrieveBinaryDataByIdentifier(id); + if (this.clients[mode]) { + return this.clients[mode].retrieveBinaryDataByIdentifier(id); } throw new Error('Storage mode used to store binary data not available'); @@ -122,8 +122,8 @@ export class BinaryDataManager { getBinaryPath(identifier: string) { const { mode, id } = this.splitBinaryModeFileId(identifier); - if (this.managers[mode]) { - return this.managers[mode].getBinaryPath(id); + if (this.clients[mode]) { + return this.clients[mode].getBinaryPath(id); } throw new Error('Storage mode used to store binary data not available'); @@ -131,21 +131,21 @@ export class BinaryDataManager { async getBinaryMetadata(identifier: string) { const { mode, id } = this.splitBinaryModeFileId(identifier); - if (this.managers[mode]) { - return this.managers[mode].getBinaryMetadata(id); + if (this.clients[mode]) { + return this.clients[mode].getBinaryMetadata(id); } throw new Error('Storage mode used to store binary data not available'); } async deleteBinaryDataByExecutionIds(executionIds: string[]) { - if (this.managers[this.mode]) { - await this.managers[this.mode].deleteBinaryDataByExecutionIds(executionIds); + if (this.clients[this.mode]) { + await this.clients[this.mode].deleteBinaryDataByExecutionIds(executionIds); } } async duplicateBinaryData(inputData: Array, executionId: string) { - if (inputData && this.managers[this.mode]) { + if (inputData && this.clients[this.mode]) { const returnInputData = (inputData as INodeExecutionData[][]).map( async (executionDataArray) => { if (executionDataArray) { @@ -183,7 +183,7 @@ export class BinaryDataManager { executionData: INodeExecutionData, executionId: string, ) { - const binaryManager = this.managers[this.mode]; + const client = this.clients[this.mode]; if (executionData.binary) { const binaryDataKeys = Object.keys(executionData.binary); @@ -197,7 +197,7 @@ export class BinaryDataManager { return { key, newId: undefined }; } - return binaryManager + return client ?.duplicateBinaryDataByIdentifier( this.splitBinaryModeFileId(binaryDataId).id, executionId, diff --git a/packages/core/src/Interfaces.ts b/packages/core/src/Interfaces.ts index f1f3f4ff7a5bd..fabce5c841a7a 100644 --- a/packages/core/src/Interfaces.ts +++ b/packages/core/src/Interfaces.ts @@ -40,7 +40,7 @@ export interface IBinaryDataConfig { localStoragePath: string; } -export interface IBinaryDataManager { +export interface BinaryDataClient { init(startPurger: boolean): Promise; getFileSize(filePath: string): Promise; copyBinaryFile(filePath: string, executionId: string): Promise; From 5627b4cef127f6d2f96c783dcc286cddbd19f1aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 18:09:33 +0200 Subject: [PATCH 031/103] Rename `BinaryDataFileSystem` to `FileSystemClient` --- .../src/BinaryDataManager/{FileSystem.ts => fs.client.ts} | 2 +- packages/core/src/BinaryDataManager/index.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename packages/core/src/BinaryDataManager/{FileSystem.ts => fs.client.ts} (98%) diff --git a/packages/core/src/BinaryDataManager/FileSystem.ts b/packages/core/src/BinaryDataManager/fs.client.ts similarity index 98% rename from packages/core/src/BinaryDataManager/FileSystem.ts rename to packages/core/src/BinaryDataManager/fs.client.ts index d8580c3c7e631..f23dcd9b2abe7 100644 --- a/packages/core/src/BinaryDataManager/FileSystem.ts +++ b/packages/core/src/BinaryDataManager/fs.client.ts @@ -12,7 +12,7 @@ import { FileNotFoundError } from '../errors'; const executionExtractionRegexp = /^(\w+)(?:[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12})$/; -export class BinaryDataFileSystem implements BinaryDataClient { +export class FileSystemClient implements BinaryDataClient { private storagePath: string; constructor(config: IBinaryDataConfig) { diff --git a/packages/core/src/BinaryDataManager/index.ts b/packages/core/src/BinaryDataManager/index.ts index 96b0df694477c..bc3e4178e32aa 100644 --- a/packages/core/src/BinaryDataManager/index.ts +++ b/packages/core/src/BinaryDataManager/index.ts @@ -4,7 +4,7 @@ import prettyBytes from 'pretty-bytes'; import type { Readable } from 'stream'; import { BINARY_ENCODING } from 'n8n-workflow'; import type { IBinaryDataConfig, BinaryDataClient } from '../Interfaces'; -import { BinaryDataFileSystem } from './FileSystem'; +import { FileSystemClient } from './fs.client'; import { binaryToBuffer } from './utils'; import { Service } from 'typedi'; @@ -30,7 +30,7 @@ export class BinaryDataManager { this.clients = {}; if (this.availableModes.includes('filesystem')) { - this.clients.filesystem = new BinaryDataFileSystem(config); + this.clients.filesystem = new FileSystemClient(config); await this.clients.filesystem.init(mainClient); } From 64517504a45a3de361391b0fcb5b22dff7fc580a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 18:22:56 +0200 Subject: [PATCH 032/103] Clean up types --- packages/cli/src/config/types.ts | 4 +- .../core/src/BinaryDataManager/fs.client.ts | 6 +-- packages/core/src/BinaryDataManager/index.ts | 23 ++++------- packages/core/src/BinaryDataManager/types.ts | 38 +++++++++++++++++++ packages/core/src/Interfaces.ts | 23 ----------- packages/core/src/index.ts | 1 + 6 files changed, 51 insertions(+), 44 deletions(-) create mode 100644 packages/core/src/BinaryDataManager/types.ts diff --git a/packages/cli/src/config/types.ts b/packages/cli/src/config/types.ts index 28dee1e73fba0..b5646ff5ae8fb 100644 --- a/packages/cli/src/config/types.ts +++ b/packages/cli/src/config/types.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ /* eslint-disable @typescript-eslint/no-unused-vars */ -import type { IBinaryDataConfig } from 'n8n-core'; +import type { BinaryData } from 'n8n-core'; import type { schema } from './schema'; // ----------------------------------- @@ -76,7 +76,7 @@ type ToReturnType = T extends NumericPath type ExceptionPaths = { 'queue.bull.redis': object; - binaryDataManager: IBinaryDataConfig; + binaryDataManager: BinaryData.Config; 'nodes.exclude': string[] | undefined; 'nodes.include': string[] | undefined; 'userManagement.isInstanceOwnerSetUp': boolean; diff --git a/packages/core/src/BinaryDataManager/fs.client.ts b/packages/core/src/BinaryDataManager/fs.client.ts index f23dcd9b2abe7..dfd4d315b1a2a 100644 --- a/packages/core/src/BinaryDataManager/fs.client.ts +++ b/packages/core/src/BinaryDataManager/fs.client.ts @@ -6,16 +6,16 @@ import type { Readable } from 'stream'; import type { BinaryMetadata } from 'n8n-workflow'; import { jsonParse } from 'n8n-workflow'; -import type { IBinaryDataConfig, BinaryDataClient } from '../Interfaces'; +import type { BinaryData } from './types'; import { FileNotFoundError } from '../errors'; const executionExtractionRegexp = /^(\w+)(?:[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12})$/; -export class FileSystemClient implements BinaryDataClient { +export class FileSystemClient implements BinaryData.Client { private storagePath: string; - constructor(config: IBinaryDataConfig) { + constructor(config: BinaryData.FileSystemConfig) { this.storagePath = config.localStoragePath; } diff --git a/packages/core/src/BinaryDataManager/index.ts b/packages/core/src/BinaryDataManager/index.ts index bc3e4178e32aa..20c4bd1a6310e 100644 --- a/packages/core/src/BinaryDataManager/index.ts +++ b/packages/core/src/BinaryDataManager/index.ts @@ -3,34 +3,25 @@ import type { IBinaryData, INodeExecutionData } from 'n8n-workflow'; import prettyBytes from 'pretty-bytes'; import type { Readable } from 'stream'; import { BINARY_ENCODING } from 'n8n-workflow'; -import type { IBinaryDataConfig, BinaryDataClient } from '../Interfaces'; +import type { BinaryData } from './types'; import { FileSystemClient } from './fs.client'; import { binaryToBuffer } from './utils'; import { Service } from 'typedi'; @Service() export class BinaryDataManager { - private clients: { - [key: string]: BinaryDataClient; - } = {}; + private availableModes: BinaryData.Mode[] = []; - /** - * Mode for storing binary data: - * - `default` (in memory) - * - `filesystem` (on disk) - * - `object` (S3) - */ - private mode = 'default'; + private mode: BinaryData.Mode = 'default'; - private availableModes: string[] = []; + private clients: Record = {}; - async init(config: IBinaryDataConfig, mainClient = false) { + async init(config: BinaryData.Config, mainClient = false) { + this.availableModes = config.availableModes.split(',') as BinaryData.Mode[]; // @TODO: Remove assertion this.mode = config.mode; - this.availableModes = config.availableModes.split(','); - this.clients = {}; if (this.availableModes.includes('filesystem')) { - this.clients.filesystem = new FileSystemClient(config); + this.clients.filesystem = new FileSystemClient(config as BinaryData.FileSystemConfig); // @TODO: Remove assertion await this.clients.filesystem.init(mainClient); } diff --git a/packages/core/src/BinaryDataManager/types.ts b/packages/core/src/BinaryDataManager/types.ts new file mode 100644 index 0000000000000..f5a20ef3db96d --- /dev/null +++ b/packages/core/src/BinaryDataManager/types.ts @@ -0,0 +1,38 @@ +import type { Readable } from 'stream'; +import type { BinaryMetadata } from 'n8n-workflow'; + +export namespace BinaryData { + /** + * Mode for storing binary data: + * - `default` (in memory) + * - `filesystem` (on disk) + * - `object` (S3) + */ + export type Mode = 'default' | 'filesystem' | 'object'; + + type ConfigBase = { + mode: Mode; + availableModes: string; // comma-separated list + }; + + type InMemoryConfig = ConfigBase & { mode: 'default' }; + + export type FileSystemConfig = ConfigBase & { mode: 'filesystem'; localStoragePath: string }; + + export type Config = InMemoryConfig | FileSystemConfig; + + export interface Client { + init(startPurger: boolean): Promise; + getFileSize(filePath: string): Promise; + copyBinaryFile(filePath: string, executionId: string): Promise; + storeBinaryMetadata(identifier: string, metadata: BinaryMetadata): Promise; + getBinaryMetadata(identifier: string): Promise; + storeBinaryData(binaryData: Buffer | Readable, executionId: string): Promise; + retrieveBinaryDataByIdentifier(identifier: string): Promise; + getBinaryPath(identifier: string): string; + getBinaryStream(identifier: string, chunkSize?: number): Readable; + deleteBinaryDataByIdentifier(identifier: string): Promise; + duplicateBinaryDataByIdentifier(binaryDataId: string, prefix: string): Promise; + deleteBinaryDataByExecutionIds(executionIds: string[]): Promise; + } +} diff --git a/packages/core/src/Interfaces.ts b/packages/core/src/Interfaces.ts index fabce5c841a7a..aa723045fdcd1 100644 --- a/packages/core/src/Interfaces.ts +++ b/packages/core/src/Interfaces.ts @@ -1,9 +1,7 @@ -import type { Readable } from 'stream'; import type { IPollResponse, ITriggerResponse, IWorkflowSettings as IWorkflowSettingsWorkflow, - BinaryMetadata, ValidationResult, } from 'n8n-workflow'; @@ -34,27 +32,6 @@ export interface IWorkflowData { triggerResponses?: ITriggerResponse[]; } -export interface IBinaryDataConfig { - mode: 'default' | 'filesystem'; - availableModes: string; - localStoragePath: string; -} - -export interface BinaryDataClient { - init(startPurger: boolean): Promise; - getFileSize(filePath: string): Promise; - copyBinaryFile(filePath: string, executionId: string): Promise; - storeBinaryMetadata(identifier: string, metadata: BinaryMetadata): Promise; - getBinaryMetadata(identifier: string): Promise; - storeBinaryData(binaryData: Buffer | Readable, executionId: string): Promise; - retrieveBinaryDataByIdentifier(identifier: string): Promise; - getBinaryPath(identifier: string): string; - getBinaryStream(identifier: string, chunkSize?: number): Readable; - deleteBinaryDataByIdentifier(identifier: string): Promise; - duplicateBinaryDataByIdentifier(binaryDataId: string, prefix: string): Promise; - deleteBinaryDataByExecutionIds(executionIds: string[]): Promise; -} - export namespace n8n { export interface PackageJson { name: string; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 2b441605eeb0d..fb79dee6deff3 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -15,3 +15,4 @@ export * from './NodeExecuteFunctions'; export * from './WorkflowExecute'; export { NodeExecuteFunctions, UserSettings }; export * from './errors'; +export * from './BinaryDataManager/types'; From 7a7ad54f2b8e3466be5304670f08b9171b0ee59a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 18:28:52 +0200 Subject: [PATCH 033/103] Move `binaryToBuffer` to manager --- packages/core/src/BinaryDataManager/index.ts | 13 ++++++++++--- packages/core/src/BinaryDataManager/utils.ts | 8 -------- packages/core/src/NodeExecuteFunctions.ts | 13 +++++++------ 3 files changed, 17 insertions(+), 17 deletions(-) delete mode 100644 packages/core/src/BinaryDataManager/utils.ts diff --git a/packages/core/src/BinaryDataManager/index.ts b/packages/core/src/BinaryDataManager/index.ts index 20c4bd1a6310e..c61c4b6a462cc 100644 --- a/packages/core/src/BinaryDataManager/index.ts +++ b/packages/core/src/BinaryDataManager/index.ts @@ -5,8 +5,8 @@ import type { Readable } from 'stream'; import { BINARY_ENCODING } from 'n8n-workflow'; import type { BinaryData } from './types'; import { FileSystemClient } from './fs.client'; -import { binaryToBuffer } from './utils'; import { Service } from 'typedi'; +import concatStream from 'concat-stream'; @Service() export class BinaryDataManager { @@ -77,7 +77,7 @@ export class BinaryDataManager { fileSize, }); } else { - const buffer = await binaryToBuffer(input); + const buffer = await this.binaryToBuffer(input); binaryData.data = buffer.toString(BINARY_ENCODING); binaryData.fileSize = prettyBytes(buffer.length); } @@ -85,6 +85,13 @@ export class BinaryDataManager { return binaryData; } + async binaryToBuffer(body: Buffer | Readable) { + return new Promise((resolve) => { + if (Buffer.isBuffer(body)) resolve(body); + else body.pipe(concatStream(resolve)); + }); + } + getBinaryStream(identifier: string, chunkSize?: number) { const { mode, id } = this.splitBinaryModeFileId(identifier); if (this.clients[mode]) { @@ -94,7 +101,7 @@ export class BinaryDataManager { throw new Error('Storage mode used to store binary data not available'); } - async getBinaryDataBuffer(binaryData: IBinaryData): Promise { + async getBinaryDataBuffer(binaryData: IBinaryData) { if (binaryData.id) { return this.retrieveBinaryDataByIdentifier(binaryData.id); } diff --git a/packages/core/src/BinaryDataManager/utils.ts b/packages/core/src/BinaryDataManager/utils.ts deleted file mode 100644 index 85fb05f6870bd..0000000000000 --- a/packages/core/src/BinaryDataManager/utils.ts +++ /dev/null @@ -1,8 +0,0 @@ -import concatStream from 'concat-stream'; -import type { Readable } from 'stream'; - -export const binaryToBuffer = async (body: Buffer | Readable) => - new Promise((resolve) => { - if (Buffer.isBuffer(body)) resolve(body); - else body.pipe(concatStream(resolve)); - }); diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index daee20685ed7a..74293a9414fb3 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -129,7 +129,6 @@ import { UM_EMAIL_TEMPLATES_INVITE, UM_EMAIL_TEMPLATES_PWRESET, } from './Constants'; -import { binaryToBuffer } from './BinaryDataManager/utils'; import { getAllWorkflowExecutionMetadata, getWorkflowExecutionMetadata, @@ -698,9 +697,9 @@ export async function proxyRequestToAxios( let responseData = response.data; if (Buffer.isBuffer(responseData) || responseData instanceof Readable) { - responseData = await binaryToBuffer(responseData).then((buffer) => - buffer.toString('utf-8'), - ); + responseData = await Container.get(BinaryDataManager) + .binaryToBuffer(responseData) + .then((buffer) => buffer.toString('utf-8')); } if (configObject.simple === false) { @@ -2376,7 +2375,8 @@ const getBinaryHelperFunctions = ({ getBinaryPath, getBinaryStream, getBinaryMetadata, - binaryToBuffer, + binaryToBuffer: async (body: Buffer | Readable) => + Container.get(BinaryDataManager).binaryToBuffer(body), prepareBinaryData: async (binaryData, filePath, mimeType) => prepareBinaryData(binaryData, executionId!, filePath, mimeType), setBinaryDataBuffer: async (data, binaryData) => @@ -2636,7 +2636,8 @@ export function getExecuteFunctions( ); return dataProxy.getDataProxy(); }, - binaryToBuffer, + binaryToBuffer: async (body: Buffer | Readable) => + Container.get(BinaryDataManager).binaryToBuffer(body), async putExecutionToWait(waitTill: Date): Promise { runExecutionData.waitTill = waitTill; if (additionalData.setExecutionStatus) { From cdfe2fce2a9b5054a908b55995d35a6ee4719edb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 18:31:57 +0200 Subject: [PATCH 034/103] Rename `BinaryDataManager` to `BinaryDataService` --- packages/core/src/NodeExecuteFunctions.ts | 22 +++++++++---------- .../binaryData.service.ts} | 2 +- .../fs.client.ts | 0 .../types.ts | 0 packages/core/src/index.ts | 4 ++-- .../core/test/NodeExecuteFunctions.test.ts | 11 +++++----- 6 files changed, 19 insertions(+), 20 deletions(-) rename packages/core/src/{BinaryDataManager/index.ts => binaryData/binaryData.service.ts} (99%) rename packages/core/src/{BinaryDataManager => binaryData}/fs.client.ts (100%) rename packages/core/src/{BinaryDataManager => binaryData}/types.ts (100%) diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index 74293a9414fb3..38a2118a0504c 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -115,7 +115,7 @@ import { Readable } from 'stream'; import { access as fsAccess, writeFile as fsWriteFile } from 'fs/promises'; import { createReadStream } from 'fs'; -import { BinaryDataManager } from './BinaryDataManager'; +import { BinaryDataService } from './binaryData/binaryData.service'; import type { ExtendedValidationResult, IResponseError, IWorkflowSettings } from './Interfaces'; import { extractValue } from './ExtractValue'; import { getClientCredentialsToken } from './OAuth2Helper'; @@ -697,7 +697,7 @@ export async function proxyRequestToAxios( let responseData = response.data; if (Buffer.isBuffer(responseData) || responseData instanceof Readable) { - responseData = await Container.get(BinaryDataManager) + responseData = await Container.get(BinaryDataService) .binaryToBuffer(responseData) .then((buffer) => buffer.toString('utf-8')); } @@ -864,21 +864,21 @@ async function httpRequest( } export function getBinaryPath(binaryDataId: string): string { - return Container.get(BinaryDataManager).getBinaryPath(binaryDataId); + return Container.get(BinaryDataService).getBinaryPath(binaryDataId); } /** * Returns binary file metadata */ export async function getBinaryMetadata(binaryDataId: string): Promise { - return Container.get(BinaryDataManager).getBinaryMetadata(binaryDataId); + return Container.get(BinaryDataService).getBinaryMetadata(binaryDataId); } /** * Returns binary file stream for piping */ export function getBinaryStream(binaryDataId: string, chunkSize?: number): Readable { - return Container.get(BinaryDataManager).getBinaryStream(binaryDataId, chunkSize); + return Container.get(BinaryDataService).getBinaryStream(binaryDataId, chunkSize); } export function assertBinaryData( @@ -915,7 +915,7 @@ export async function getBinaryDataBuffer( inputIndex: number, ): Promise { const binaryData = inputData.main[inputIndex]![itemIndex]!.binary![propertyName]!; - return Container.get(BinaryDataManager).getBinaryDataBuffer(binaryData); + return Container.get(BinaryDataService).getBinaryDataBuffer(binaryData); } /** @@ -931,7 +931,7 @@ export async function setBinaryDataBuffer( binaryData: Buffer | Readable, executionId: string, ): Promise { - return Container.get(BinaryDataManager).storeBinaryData(data, binaryData, executionId); + return Container.get(BinaryDataService).storeBinaryData(data, binaryData, executionId); } export async function copyBinaryFile( @@ -984,7 +984,7 @@ export async function copyBinaryFile( returnData.fileName = path.parse(filePath).base; } - return Container.get(BinaryDataManager).copyBinaryFile(returnData, filePath, executionId); + return Container.get(BinaryDataService).copyBinaryFile(returnData, filePath, executionId); } /** @@ -2376,7 +2376,7 @@ const getBinaryHelperFunctions = ({ getBinaryStream, getBinaryMetadata, binaryToBuffer: async (body: Buffer | Readable) => - Container.get(BinaryDataManager).binaryToBuffer(body), + Container.get(BinaryDataService).binaryToBuffer(body), prepareBinaryData: async (binaryData, filePath, mimeType) => prepareBinaryData(binaryData, executionId!, filePath, mimeType), setBinaryDataBuffer: async (data, binaryData) => @@ -2564,7 +2564,7 @@ export function getExecuteFunctions( parentWorkflowSettings: workflow.settings, }) .then(async (result) => - Container.get(BinaryDataManager).duplicateBinaryData( + Container.get(BinaryDataService).duplicateBinaryData( result, additionalData.executionId!, ), @@ -2637,7 +2637,7 @@ export function getExecuteFunctions( return dataProxy.getDataProxy(); }, binaryToBuffer: async (body: Buffer | Readable) => - Container.get(BinaryDataManager).binaryToBuffer(body), + Container.get(BinaryDataService).binaryToBuffer(body), async putExecutionToWait(waitTill: Date): Promise { runExecutionData.waitTill = waitTill; if (additionalData.setExecutionStatus) { diff --git a/packages/core/src/BinaryDataManager/index.ts b/packages/core/src/binaryData/binaryData.service.ts similarity index 99% rename from packages/core/src/BinaryDataManager/index.ts rename to packages/core/src/binaryData/binaryData.service.ts index c61c4b6a462cc..d0e81bdfa6444 100644 --- a/packages/core/src/BinaryDataManager/index.ts +++ b/packages/core/src/binaryData/binaryData.service.ts @@ -9,7 +9,7 @@ import { Service } from 'typedi'; import concatStream from 'concat-stream'; @Service() -export class BinaryDataManager { +export class BinaryDataService { private availableModes: BinaryData.Mode[] = []; private mode: BinaryData.Mode = 'default'; diff --git a/packages/core/src/BinaryDataManager/fs.client.ts b/packages/core/src/binaryData/fs.client.ts similarity index 100% rename from packages/core/src/BinaryDataManager/fs.client.ts rename to packages/core/src/binaryData/fs.client.ts diff --git a/packages/core/src/BinaryDataManager/types.ts b/packages/core/src/binaryData/types.ts similarity index 100% rename from packages/core/src/BinaryDataManager/types.ts rename to packages/core/src/binaryData/types.ts diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index fb79dee6deff3..c595e56458de0 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -2,7 +2,8 @@ import * as NodeExecuteFunctions from './NodeExecuteFunctions'; import * as UserSettings from './UserSettings'; export * from './ActiveWorkflows'; -export * from './BinaryDataManager'; +export * from './binaryData/binaryData.service'; +export * from './binaryData/types'; export * from './ClassLoader'; export * from './Constants'; export * from './Credentials'; @@ -15,4 +16,3 @@ export * from './NodeExecuteFunctions'; export * from './WorkflowExecute'; export { NodeExecuteFunctions, UserSettings }; export * from './errors'; -export * from './BinaryDataManager/types'; diff --git a/packages/core/test/NodeExecuteFunctions.test.ts b/packages/core/test/NodeExecuteFunctions.test.ts index 9fb7890f7081f..f9b25a46ccbe8 100644 --- a/packages/core/test/NodeExecuteFunctions.test.ts +++ b/packages/core/test/NodeExecuteFunctions.test.ts @@ -11,7 +11,7 @@ import type { Workflow, WorkflowHooks, } from 'n8n-workflow'; -import { BinaryDataManager } from '@/BinaryDataManager'; +import { BinaryDataService } from '@/binaryData/binaryData.service'; import { setBinaryDataBuffer, getBinaryDataBuffer, @@ -26,12 +26,11 @@ describe('NodeExecuteFunctions', () => { describe('test binary data helper methods', () => { test("test getBinaryDataBuffer(...) & setBinaryDataBuffer(...) methods in 'default' mode", async () => { // Setup a 'default' binary data manager instance - Container.set(BinaryDataManager, new BinaryDataManager()); + Container.set(BinaryDataService, new BinaryDataService()); - await Container.get(BinaryDataManager).init({ + await Container.get(BinaryDataService).init({ mode: 'default', availableModes: 'default', - localStoragePath: temporaryDir, }); // Set our binary data buffer @@ -76,10 +75,10 @@ describe('NodeExecuteFunctions', () => { }); test("test getBinaryDataBuffer(...) & setBinaryDataBuffer(...) methods in 'filesystem' mode", async () => { - Container.set(BinaryDataManager, new BinaryDataManager()); + Container.set(BinaryDataService, new BinaryDataService()); // Setup a 'filesystem' binary data manager instance - await Container.get(BinaryDataManager).init({ + await Container.get(BinaryDataService).init({ mode: 'filesystem', availableModes: 'filesystem', localStoragePath: temporaryDir, From b5bc390d8aae4e00463d7910661a7623df938cc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 18:37:41 +0200 Subject: [PATCH 035/103] Unify storage path naming --- packages/cli/src/config/schema.ts | 2 +- packages/core/src/binaryData/fs.client.ts | 2 +- packages/core/src/binaryData/types.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/config/schema.ts b/packages/cli/src/config/schema.ts index cf9ea86c4b740..059e650d5776d 100644 --- a/packages/cli/src/config/schema.ts +++ b/packages/cli/src/config/schema.ts @@ -895,7 +895,7 @@ export const schema = { env: 'N8N_DEFAULT_BINARY_DATA_MODE', doc: 'Storage mode for binary data', }, - localStoragePath: { + storagePath: { format: String, default: path.join(UserSettings.getUserN8nFolderPath(), 'binaryData'), env: 'N8N_BINARY_DATA_STORAGE_PATH', diff --git a/packages/core/src/binaryData/fs.client.ts b/packages/core/src/binaryData/fs.client.ts index dfd4d315b1a2a..78887d79780f5 100644 --- a/packages/core/src/binaryData/fs.client.ts +++ b/packages/core/src/binaryData/fs.client.ts @@ -16,7 +16,7 @@ export class FileSystemClient implements BinaryData.Client { private storagePath: string; constructor(config: BinaryData.FileSystemConfig) { - this.storagePath = config.localStoragePath; + this.storagePath = config.storagePath; } async init() { diff --git a/packages/core/src/binaryData/types.ts b/packages/core/src/binaryData/types.ts index f5a20ef3db96d..36ce9cdae1738 100644 --- a/packages/core/src/binaryData/types.ts +++ b/packages/core/src/binaryData/types.ts @@ -17,7 +17,7 @@ export namespace BinaryData { type InMemoryConfig = ConfigBase & { mode: 'default' }; - export type FileSystemConfig = ConfigBase & { mode: 'filesystem'; localStoragePath: string }; + export type FileSystemConfig = ConfigBase & { mode: 'filesystem'; storagePath: string }; export type Config = InMemoryConfig | FileSystemConfig; From c876697859f3bb278679d17902ef6ce151c0dabd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 18:37:56 +0200 Subject: [PATCH 036/103] Set binary data service in nodes-base test helper --- .../core/test/NodeExecuteFunctions.test.ts | 2 +- packages/nodes-base/package.json | 1 + packages/nodes-base/test/nodes/Helpers.ts | 13 ++++------ pnpm-lock.yaml | 26 +++++++++++-------- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/packages/core/test/NodeExecuteFunctions.test.ts b/packages/core/test/NodeExecuteFunctions.test.ts index f9b25a46ccbe8..7ea8dbc289724 100644 --- a/packages/core/test/NodeExecuteFunctions.test.ts +++ b/packages/core/test/NodeExecuteFunctions.test.ts @@ -81,7 +81,7 @@ describe('NodeExecuteFunctions', () => { await Container.get(BinaryDataService).init({ mode: 'filesystem', availableModes: 'filesystem', - localStoragePath: temporaryDir, + storagePath: temporaryDir, }); // Set our binary data buffer diff --git a/packages/nodes-base/package.json b/packages/nodes-base/package.json index 5cf314d1b9e16..9c4e6321c02d6 100644 --- a/packages/nodes-base/package.json +++ b/packages/nodes-base/package.json @@ -870,6 +870,7 @@ "snowflake-sdk": "^1.8.0", "ssh2-sftp-client": "^7.0.0", "tmp-promise": "^3.0.2", + "typedi": "^0.10.0", "uuid": "^8.3.2", "xlsx": "https://cdn.sheetjs.com/xlsx-0.19.3/xlsx-0.19.3.tgz", "xml2js": "^0.5.0" diff --git a/packages/nodes-base/test/nodes/Helpers.ts b/packages/nodes-base/test/nodes/Helpers.ts index d14ef99577011..8775d0b54dcb7 100644 --- a/packages/nodes-base/test/nodes/Helpers.ts +++ b/packages/nodes-base/test/nodes/Helpers.ts @@ -3,7 +3,8 @@ import path from 'path'; import { tmpdir } from 'os'; import { isEmpty } from 'lodash'; import { get } from 'lodash'; -import { BinaryDataManager, Credentials, constructExecutionMetaData } from 'n8n-core'; +import { BinaryDataService, Credentials, constructExecutionMetaData } from 'n8n-core'; +import { Container } from 'typedi'; import type { CredentialLoadingDetails, ICredentialDataDecryptedObject, @@ -217,13 +218,9 @@ export function createTemporaryDir(prefix = 'n8n') { } export async function initBinaryDataManager(mode: 'default' | 'filesystem' = 'default') { - const temporaryDir = createTemporaryDir(); - await BinaryDataManager.init({ - mode, - availableModes: mode, - localStoragePath: temporaryDir, - }); - return temporaryDir; + const binaryDataService = new BinaryDataService(); + await binaryDataService.init({ mode: 'default', availableModes: mode }); + Container.set(BinaryDataService, binaryDataService); } const credentialTypes = new CredentialType(); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8bda6a0b0a436..667eb59b12779 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -139,7 +139,7 @@ importers: dependencies: axios: specifier: ^0.21.1 - version: 0.21.4 + version: 0.21.4(debug@4.3.2) packages/@n8n_io/eslint-config: devDependencies: @@ -220,7 +220,7 @@ importers: version: 7.28.1 axios: specifier: ^0.21.1 - version: 0.21.4 + version: 0.21.4(debug@4.3.2) basic-auth: specifier: ^2.0.1 version: 2.0.1 @@ -587,7 +587,7 @@ importers: version: link:../@n8n/client-oauth2 axios: specifier: ^0.21.1 - version: 0.21.4 + version: 0.21.4(debug@4.3.2) concat-stream: specifier: ^2.0.0 version: 2.0.0 @@ -850,7 +850,7 @@ importers: version: 10.2.0(vue@3.3.4) axios: specifier: ^0.21.1 - version: 0.21.4 + version: 0.21.4(debug@4.3.2) codemirror-lang-html-n8n: specifier: ^1.0.0 version: 1.0.0 @@ -1183,6 +1183,9 @@ importers: tmp-promise: specifier: ^3.0.2 version: 3.0.3 + typedi: + specifier: ^0.10.0 + version: 0.10.0(patch_hash=62r6bc2crgimafeyruodhqlgo4) uuid: specifier: ^8.3.2 version: 8.3.2 @@ -5009,7 +5012,7 @@ packages: dependencies: '@segment/loosely-validate-event': 2.0.0 auto-changelog: 1.16.4 - axios: 0.21.4 + axios: 0.21.4(debug@4.3.2) axios-retry: 3.3.1 bull: 3.29.3 lodash.clonedeep: 4.5.0 @@ -9069,18 +9072,19 @@ packages: is-retry-allowed: 2.2.0 dev: false - /axios@0.21.4: + /axios@0.21.4(debug@4.3.2): resolution: {integrity: sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==} dependencies: - follow-redirects: 1.15.2(debug@3.2.7) + follow-redirects: 1.15.2(debug@4.3.2) transitivePeerDependencies: - debug dev: false - /axios@0.21.4(debug@4.3.2): - resolution: {integrity: sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==} + /axios@0.27.2: + resolution: {integrity: sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==} dependencies: follow-redirects: 1.15.2(debug@4.3.2) + form-data: 4.0.0 transitivePeerDependencies: - debug dev: false @@ -9106,7 +9110,7 @@ packages: /axios@1.4.0: resolution: {integrity: sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==} dependencies: - follow-redirects: 1.15.2(debug@3.2.7) + follow-redirects: 1.15.2(debug@4.3.2) form-data: 4.0.0 proxy-from-env: 1.1.0 transitivePeerDependencies: @@ -18008,7 +18012,7 @@ packages: resolution: {integrity: sha512-aXYe/D+28kF63W8Cz53t09ypEORz+ULeDCahdAqhVrRm2scbOXFbtnn0GGhvMpYe45grepLKuwui9KxrZ2ZuMw==} engines: {node: '>=14.17.0'} dependencies: - axios: 0.27.2(debug@3.2.7) + axios: 0.27.2 transitivePeerDependencies: - debug dev: false From 25a796cd545e584f9675d279a89645bb06c58578 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 18:38:41 +0200 Subject: [PATCH 037/103] Rename manager in nodes-base --- packages/nodes-base/nodes/Aws/S3/test/V1/AwsS3.node.test.ts | 4 ++-- packages/nodes-base/nodes/Aws/S3/test/V2/AwsS3.node.test.ts | 4 ++-- packages/nodes-base/nodes/Code/test/Code.node.test.ts | 4 ++-- .../nodes/Compression/test/node/Compression.test.ts | 4 ++-- packages/nodes-base/nodes/Crypto/test/Crypto.test.ts | 4 ++-- .../nodes-base/nodes/ICalendar/test/node/ICalendar.test.ts | 4 ++-- .../nodes/MoveBinaryData/test/MoveBinaryData.test.ts | 2 +- .../nodes-base/nodes/QuickChart/test/QuickChart.node.test.ts | 2 +- .../nodes/ReadBinaryFile/test/ReadBinaryFile.test.ts | 2 +- .../nodes/ReadBinaryFiles/test/ReadBinaryFiles.test.ts | 2 +- packages/nodes-base/nodes/ReadPdf/test/ReadPDF.test.ts | 4 ++-- .../nodes/SpreadsheetFile/test/SpreadsheetFile.test.ts | 2 +- .../nodes/WriteBinaryFile/test/WriteBinaryFile.test.ts | 2 +- packages/nodes-base/test/nodes/Helpers.ts | 2 +- 14 files changed, 21 insertions(+), 21 deletions(-) diff --git a/packages/nodes-base/nodes/Aws/S3/test/V1/AwsS3.node.test.ts b/packages/nodes-base/nodes/Aws/S3/test/V1/AwsS3.node.test.ts index 91150c8c86d2d..c13c572ef2758 100644 --- a/packages/nodes-base/nodes/Aws/S3/test/V1/AwsS3.node.test.ts +++ b/packages/nodes-base/nodes/Aws/S3/test/V1/AwsS3.node.test.ts @@ -1,5 +1,5 @@ import nock from 'nock'; -import { getWorkflowFilenames, initBinaryDataManager, testWorkflows } from '@test/nodes/Helpers'; +import { getWorkflowFilenames, initBinaryDataService, testWorkflows } from '@test/nodes/Helpers'; const workflows = getWorkflowFilenames(__dirname); @@ -11,7 +11,7 @@ describe('Test S3 V1 Node', () => { beforeAll(async () => { jest.useFakeTimers({ doNotFake: ['nextTick'], now }); - await initBinaryDataManager(); + await initBinaryDataService(); nock.disableNetConnect(); mock = nock('https://bucket.s3.eu-central-1.amazonaws.com'); diff --git a/packages/nodes-base/nodes/Aws/S3/test/V2/AwsS3.node.test.ts b/packages/nodes-base/nodes/Aws/S3/test/V2/AwsS3.node.test.ts index b8a9a3d6c0a22..2f3b0516c8cbc 100644 --- a/packages/nodes-base/nodes/Aws/S3/test/V2/AwsS3.node.test.ts +++ b/packages/nodes-base/nodes/Aws/S3/test/V2/AwsS3.node.test.ts @@ -1,5 +1,5 @@ import nock from 'nock'; -import { getWorkflowFilenames, initBinaryDataManager, testWorkflows } from '@test/nodes/Helpers'; +import { getWorkflowFilenames, initBinaryDataService, testWorkflows } from '@test/nodes/Helpers'; const workflows = getWorkflowFilenames(__dirname); @@ -11,7 +11,7 @@ describe('Test S3 V2 Node', () => { beforeAll(async () => { jest.useFakeTimers({ doNotFake: ['nextTick'], now }); - await initBinaryDataManager(); + await initBinaryDataService(); nock.disableNetConnect(); mock = nock('https://bucket.s3.eu-central-1.amazonaws.com'); diff --git a/packages/nodes-base/nodes/Code/test/Code.node.test.ts b/packages/nodes-base/nodes/Code/test/Code.node.test.ts index 7c47a0a5632f0..da798db3a8206 100644 --- a/packages/nodes-base/nodes/Code/test/Code.node.test.ts +++ b/packages/nodes-base/nodes/Code/test/Code.node.test.ts @@ -3,7 +3,7 @@ import { NodeVM } from '@n8n/vm2'; import type { IExecuteFunctions, IWorkflowDataProxyData } from 'n8n-workflow'; import { NodeHelpers } from 'n8n-workflow'; import { normalizeItems } from 'n8n-core'; -import { testWorkflows, getWorkflowFilenames, initBinaryDataManager } from '@test/nodes/Helpers'; +import { testWorkflows, getWorkflowFilenames, initBinaryDataService } from '@test/nodes/Helpers'; import { Code } from '../Code.node'; import { ValidationError } from '../ValidationError'; @@ -11,7 +11,7 @@ describe('Test Code Node', () => { const workflows = getWorkflowFilenames(__dirname); beforeAll(async () => { - await initBinaryDataManager(); + await initBinaryDataService(); }); testWorkflows(workflows); diff --git a/packages/nodes-base/nodes/Compression/test/node/Compression.test.ts b/packages/nodes-base/nodes/Compression/test/node/Compression.test.ts index 5fbe41796346d..ea7ff18819ebd 100644 --- a/packages/nodes-base/nodes/Compression/test/node/Compression.test.ts +++ b/packages/nodes-base/nodes/Compression/test/node/Compression.test.ts @@ -5,7 +5,7 @@ import type { IDataObject } from 'n8n-workflow'; import { getResultNodeData, setup, - initBinaryDataManager, + initBinaryDataService, readJsonFileSync, } from '@test/nodes/Helpers'; import { executeWorkflow } from '@test/nodes/ExecuteWorkflow'; @@ -16,7 +16,7 @@ import os from 'node:os'; if (os.platform() !== 'win32') { describe('Execute Compression Node', () => { beforeEach(async () => { - await initBinaryDataManager(); + await initBinaryDataService(); }); const workflowData = readJsonFileSync('nodes/Compression/test/node/workflow.compression.json'); diff --git a/packages/nodes-base/nodes/Crypto/test/Crypto.test.ts b/packages/nodes-base/nodes/Crypto/test/Crypto.test.ts index ab4742b70dd62..b321d4e852b67 100644 --- a/packages/nodes-base/nodes/Crypto/test/Crypto.test.ts +++ b/packages/nodes-base/nodes/Crypto/test/Crypto.test.ts @@ -1,7 +1,7 @@ import fs from 'fs'; import fsPromises from 'fs/promises'; import { Readable } from 'stream'; -import { testWorkflows, getWorkflowFilenames, initBinaryDataManager } from '@test/nodes/Helpers'; +import { testWorkflows, getWorkflowFilenames, initBinaryDataService } from '@test/nodes/Helpers'; const workflows = getWorkflowFilenames(__dirname); @@ -13,7 +13,7 @@ describe('Test Crypto Node', () => { fs.createReadStream = () => Readable.from(Buffer.from('test')) as fs.ReadStream; beforeEach(async () => { - await initBinaryDataManager(); + await initBinaryDataService(); }); testWorkflows(workflows); diff --git a/packages/nodes-base/nodes/ICalendar/test/node/ICalendar.test.ts b/packages/nodes-base/nodes/ICalendar/test/node/ICalendar.test.ts index f9adbf508f22f..9757cd5f2a479 100644 --- a/packages/nodes-base/nodes/ICalendar/test/node/ICalendar.test.ts +++ b/packages/nodes-base/nodes/ICalendar/test/node/ICalendar.test.ts @@ -5,13 +5,13 @@ import { getResultNodeData, setup, readJsonFileSync, - initBinaryDataManager, + initBinaryDataService, } from '@test/nodes/Helpers'; import { executeWorkflow } from '@test/nodes/ExecuteWorkflow'; describe('Execute iCalendar Node', () => { beforeEach(async () => { - await initBinaryDataManager(); + await initBinaryDataService(); }); const workflowData = readJsonFileSync('nodes/ICalendar/test/node/workflow.iCalendar.json'); diff --git a/packages/nodes-base/nodes/MoveBinaryData/test/MoveBinaryData.test.ts b/packages/nodes-base/nodes/MoveBinaryData/test/MoveBinaryData.test.ts index e0aef1e046fc1..8abf8e2db53bf 100644 --- a/packages/nodes-base/nodes/MoveBinaryData/test/MoveBinaryData.test.ts +++ b/packages/nodes-base/nodes/MoveBinaryData/test/MoveBinaryData.test.ts @@ -6,7 +6,7 @@ import path from 'path'; describe('Test Move Binary Data Node', () => { beforeEach(async () => { - await Helpers.initBinaryDataManager(); + await Helpers.initBinaryDataService(); }); const workflow = Helpers.readJsonFileSync( diff --git a/packages/nodes-base/nodes/QuickChart/test/QuickChart.node.test.ts b/packages/nodes-base/nodes/QuickChart/test/QuickChart.node.test.ts index eeaa22449341d..94ea8cc6bd688 100644 --- a/packages/nodes-base/nodes/QuickChart/test/QuickChart.node.test.ts +++ b/packages/nodes-base/nodes/QuickChart/test/QuickChart.node.test.ts @@ -6,7 +6,7 @@ import nock from 'nock'; describe('Test QuickChart Node', () => { beforeEach(async () => { - await Helpers.initBinaryDataManager(); + await Helpers.initBinaryDataService(); nock.disableNetConnect(); nock('https://quickchart.io') .persist() diff --git a/packages/nodes-base/nodes/ReadBinaryFile/test/ReadBinaryFile.test.ts b/packages/nodes-base/nodes/ReadBinaryFile/test/ReadBinaryFile.test.ts index 637b1d6b07847..8590bf61fca05 100644 --- a/packages/nodes-base/nodes/ReadBinaryFile/test/ReadBinaryFile.test.ts +++ b/packages/nodes-base/nodes/ReadBinaryFile/test/ReadBinaryFile.test.ts @@ -6,7 +6,7 @@ import path from 'path'; describe('Test Read Binary File Node', () => { beforeEach(async () => { - await Helpers.initBinaryDataManager(); + await Helpers.initBinaryDataService(); }); const workflow = Helpers.readJsonFileSync( diff --git a/packages/nodes-base/nodes/ReadBinaryFiles/test/ReadBinaryFiles.test.ts b/packages/nodes-base/nodes/ReadBinaryFiles/test/ReadBinaryFiles.test.ts index 78f021c504318..d4bf87f04533f 100644 --- a/packages/nodes-base/nodes/ReadBinaryFiles/test/ReadBinaryFiles.test.ts +++ b/packages/nodes-base/nodes/ReadBinaryFiles/test/ReadBinaryFiles.test.ts @@ -6,7 +6,7 @@ import path from 'path'; describe('Test Read Binary Files Node', () => { beforeEach(async () => { - await Helpers.initBinaryDataManager(); + await Helpers.initBinaryDataService(); }); const workflow = Helpers.readJsonFileSync( diff --git a/packages/nodes-base/nodes/ReadPdf/test/ReadPDF.test.ts b/packages/nodes-base/nodes/ReadPdf/test/ReadPDF.test.ts index cd91f69a1df06..8a394c0ce9ba4 100644 --- a/packages/nodes-base/nodes/ReadPdf/test/ReadPDF.test.ts +++ b/packages/nodes-base/nodes/ReadPdf/test/ReadPDF.test.ts @@ -1,10 +1,10 @@ -import { getWorkflowFilenames, initBinaryDataManager, testWorkflows } from '@test/nodes/Helpers'; +import { getWorkflowFilenames, initBinaryDataService, testWorkflows } from '@test/nodes/Helpers'; describe('Test Read PDF Node', () => { const workflows = getWorkflowFilenames(__dirname); beforeAll(async () => { - await initBinaryDataManager(); + await initBinaryDataService(); }); testWorkflows(workflows); diff --git a/packages/nodes-base/nodes/SpreadsheetFile/test/SpreadsheetFile.test.ts b/packages/nodes-base/nodes/SpreadsheetFile/test/SpreadsheetFile.test.ts index ab7313deda2b9..ddbe08c271611 100644 --- a/packages/nodes-base/nodes/SpreadsheetFile/test/SpreadsheetFile.test.ts +++ b/packages/nodes-base/nodes/SpreadsheetFile/test/SpreadsheetFile.test.ts @@ -6,7 +6,7 @@ import path from 'path'; describe('Execute Spreadsheet File Node', () => { beforeEach(async () => { - await Helpers.initBinaryDataManager(); + await Helpers.initBinaryDataService(); }); // replace workflow json 'Read Binary File' node's filePath to local file diff --git a/packages/nodes-base/nodes/WriteBinaryFile/test/WriteBinaryFile.test.ts b/packages/nodes-base/nodes/WriteBinaryFile/test/WriteBinaryFile.test.ts index 147d4318f3aa5..9cd6df1da5fa7 100644 --- a/packages/nodes-base/nodes/WriteBinaryFile/test/WriteBinaryFile.test.ts +++ b/packages/nodes-base/nodes/WriteBinaryFile/test/WriteBinaryFile.test.ts @@ -6,7 +6,7 @@ import path from 'path'; describe('Test Write Binary File Node', () => { beforeEach(async () => { - await Helpers.initBinaryDataManager(); + await Helpers.initBinaryDataService(); }); const temporaryDir = Helpers.createTemporaryDir(); diff --git a/packages/nodes-base/test/nodes/Helpers.ts b/packages/nodes-base/test/nodes/Helpers.ts index 8775d0b54dcb7..7973221217569 100644 --- a/packages/nodes-base/test/nodes/Helpers.ts +++ b/packages/nodes-base/test/nodes/Helpers.ts @@ -217,7 +217,7 @@ export function createTemporaryDir(prefix = 'n8n') { return mkdtempSync(path.join(tmpdir(), prefix)); } -export async function initBinaryDataManager(mode: 'default' | 'filesystem' = 'default') { +export async function initBinaryDataService(mode: 'default' | 'filesystem' = 'default') { const binaryDataService = new BinaryDataService(); await binaryDataService.init({ mode: 'default', availableModes: mode }); Container.set(BinaryDataService, binaryDataService); From 538d31892c7795eb10c72fbc7e056044db1b7f9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 18:43:42 +0200 Subject: [PATCH 038/103] More renamings --- packages/cli/src/Server.ts | 12 ++++++------ packages/cli/src/WebhookHelpers.ts | 6 +++--- packages/cli/src/WorkflowRunnerProcess.ts | 6 +++--- packages/cli/src/commands/BaseCommand.ts | 6 +++--- packages/cli/src/config/schema.ts | 2 +- packages/cli/src/config/types.ts | 2 +- .../databases/repositories/execution.repository.ts | 6 +++--- .../test/integration/publicApi/executions.test.ts | 2 +- packages/cli/test/integration/shared/utils/index.ts | 11 ++++++++--- 9 files changed, 29 insertions(+), 24 deletions(-) diff --git a/packages/cli/src/Server.ts b/packages/cli/src/Server.ts index dcfafbe3dec58..22c685214e8ae 100644 --- a/packages/cli/src/Server.ts +++ b/packages/cli/src/Server.ts @@ -26,7 +26,7 @@ import type { RequestOptions } from 'oauth-1.0a'; import clientOAuth1 from 'oauth-1.0a'; import { - BinaryDataManager, + BinaryDataService, Credentials, LoadMappingOptions, LoadNodeParameterOptions, @@ -201,7 +201,7 @@ export class Server extends AbstractServer { push: Push; - binaryDataManager: BinaryDataManager; + binaryDataService: BinaryDataService; constructor() { super('main'); @@ -358,13 +358,13 @@ export class Server extends AbstractServer { this.endpointPresetCredentials = config.getEnv('credentials.overwrite.endpoint'); this.push = Container.get(Push); - this.binaryDataManager = Container.get(BinaryDataManager); + this.binaryDataService = Container.get(BinaryDataService); await super.start(); LoggerProxy.debug(`Server ID: ${this.uniqueInstanceId}`); const cpus = os.cpus(); - const binaryDataConfig = config.getEnv('binaryDataManager'); + const binaryDataConfig = config.getEnv('binaryDataService'); const diagnosticInfo: IDiagnosticInfo = { databaseType: config.getEnv('database.type'), disableProductionWebhooksOnMainProcess: config.getEnv( @@ -1425,11 +1425,11 @@ export class Server extends AbstractServer { // TODO UM: check if this needs permission check for UM const identifier = req.params.path; try { - const binaryPath = this.binaryDataManager.getBinaryPath(identifier); + const binaryPath = this.binaryDataService.getBinaryPath(identifier); let { mode, fileName, mimeType } = req.query; if (!fileName || !mimeType) { try { - const metadata = await this.binaryDataManager.getBinaryMetadata(identifier); + const metadata = await this.binaryDataService.getBinaryMetadata(identifier); fileName = metadata.fileName; mimeType = metadata.mimeType; res.setHeader('Content-Length', metadata.fileSize); diff --git a/packages/cli/src/WebhookHelpers.ts b/packages/cli/src/WebhookHelpers.ts index aa84e798ce2bb..fcefd8dc9741c 100644 --- a/packages/cli/src/WebhookHelpers.ts +++ b/packages/cli/src/WebhookHelpers.ts @@ -14,7 +14,7 @@ import stream from 'stream'; import { promisify } from 'util'; import formidable from 'formidable'; -import { BinaryDataManager, NodeExecuteFunctions } from 'n8n-core'; +import { BinaryDataService, NodeExecuteFunctions } from 'n8n-core'; import type { IBinaryData, @@ -514,7 +514,7 @@ export async function executeWebhook( const binaryData = (response.body as IDataObject)?.binaryData as IBinaryData; if (binaryData?.id) { res.header(response.headers); - const stream = Container.get(BinaryDataManager).getBinaryStream(binaryData.id); + const stream = Container.get(BinaryDataService).getBinaryStream(binaryData.id); void pipeline(stream, res).then(() => responseCallback(null, { noWebhookResponse: true }), ); @@ -732,7 +732,7 @@ export async function executeWebhook( // Send the webhook response manually res.setHeader('Content-Type', binaryData.mimeType); if (binaryData.id) { - const stream = Container.get(BinaryDataManager).getBinaryStream(binaryData.id); + const stream = Container.get(BinaryDataService).getBinaryStream(binaryData.id); await pipeline(stream, res); } else { res.end(Buffer.from(binaryData.data, BINARY_ENCODING)); diff --git a/packages/cli/src/WorkflowRunnerProcess.ts b/packages/cli/src/WorkflowRunnerProcess.ts index 642a7cb98f727..80f877e58a560 100644 --- a/packages/cli/src/WorkflowRunnerProcess.ts +++ b/packages/cli/src/WorkflowRunnerProcess.ts @@ -10,7 +10,7 @@ import 'source-map-support/register'; import 'reflect-metadata'; import { Container } from 'typedi'; import type { IProcessMessage } from 'n8n-core'; -import { BinaryDataManager, UserSettings, WorkflowExecute } from 'n8n-core'; +import { BinaryDataService, UserSettings, WorkflowExecute } from 'n8n-core'; import type { ExecutionError, @@ -122,8 +122,8 @@ class WorkflowRunnerProcess { await Container.get(PostHogClient).init(instanceId); await Container.get(InternalHooks).init(instanceId); - const binaryDataConfig = config.getEnv('binaryDataManager'); - await Container.get(BinaryDataManager).init(binaryDataConfig); + const binaryDataConfig = config.getEnv('binaryDataService'); + await Container.get(BinaryDataService).init(binaryDataConfig); const license = Container.get(License); await license.init(instanceId); diff --git a/packages/cli/src/commands/BaseCommand.ts b/packages/cli/src/commands/BaseCommand.ts index 012d581be2f25..ebb4d2c51a5c0 100644 --- a/packages/cli/src/commands/BaseCommand.ts +++ b/packages/cli/src/commands/BaseCommand.ts @@ -3,7 +3,7 @@ import { ExitError } from '@oclif/errors'; import { Container } from 'typedi'; import { LoggerProxy, ErrorReporterProxy as ErrorReporter, sleep } from 'n8n-workflow'; import type { IUserSettings } from 'n8n-core'; -import { BinaryDataManager, UserSettings } from 'n8n-core'; +import { BinaryDataService, UserSettings } from 'n8n-core'; import type { AbstractServer } from '@/AbstractServer'; import { getLogger } from '@/Logger'; import config from '@/config'; @@ -104,8 +104,8 @@ export abstract class BaseCommand extends Command { } protected async initBinaryManager() { - const binaryDataConfig = config.getEnv('binaryDataManager'); - await Container.get(BinaryDataManager).init(binaryDataConfig, true); + const binaryDataConfig = config.getEnv('binaryDataService'); + await Container.get(BinaryDataService).init(binaryDataConfig, true); } protected async initExternalHooks() { diff --git a/packages/cli/src/config/schema.ts b/packages/cli/src/config/schema.ts index 059e650d5776d..e791f41cd7661 100644 --- a/packages/cli/src/config/schema.ts +++ b/packages/cli/src/config/schema.ts @@ -882,7 +882,7 @@ export const schema = { }, }, - binaryDataManager: { + binaryDataService: { availableModes: { format: String, default: 'filesystem,object', diff --git a/packages/cli/src/config/types.ts b/packages/cli/src/config/types.ts index b5646ff5ae8fb..6b87c8016e3fd 100644 --- a/packages/cli/src/config/types.ts +++ b/packages/cli/src/config/types.ts @@ -76,7 +76,7 @@ type ToReturnType = T extends NumericPath type ExceptionPaths = { 'queue.bull.redis': object; - binaryDataManager: BinaryData.Config; + binaryDataService: BinaryData.Config; 'nodes.exclude': string[] | undefined; 'nodes.include': string[] | undefined; 'userManagement.isInstanceOwnerSetUp': boolean; diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 3d08483a4507d..2b5b3c24725fb 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -18,7 +18,7 @@ import type { import { parse, stringify } from 'flatted'; import { LoggerProxy as Logger } from 'n8n-workflow'; import type { IExecutionsSummary, IRunExecutionData } from 'n8n-workflow'; -import { BinaryDataManager } from 'n8n-core'; +import { BinaryDataService } from 'n8n-core'; import type { IExecutionBase, IExecutionDb, @@ -81,7 +81,7 @@ export class ExecutionRepository extends Repository { constructor( dataSource: DataSource, private readonly executionDataRepository: ExecutionDataRepository, - private readonly binaryDataManager: BinaryDataManager, + private readonly binaryDataService: BinaryDataService, ) { super(ExecutionEntity, dataSource.manager); @@ -490,7 +490,7 @@ export class ExecutionRepository extends Repository { }) ).map(({ id }) => id); - await this.binaryDataManager.deleteBinaryDataByExecutionIds(executionIds); + await this.binaryDataService.deleteBinaryDataByExecutionIds(executionIds); // Actually delete these executions await this.delete({ id: In(executionIds) }); diff --git a/packages/cli/test/integration/publicApi/executions.test.ts b/packages/cli/test/integration/publicApi/executions.test.ts index d42f8ca62ad35..cb8ae69c41fd0 100644 --- a/packages/cli/test/integration/publicApi/executions.test.ts +++ b/packages/cli/test/integration/publicApi/executions.test.ts @@ -23,7 +23,7 @@ beforeAll(async () => { user1 = await testDb.createUser({ globalRole: globalUserRole, apiKey: randomApiKey() }); user2 = await testDb.createUser({ globalRole: globalUserRole, apiKey: randomApiKey() }); - // TODO: mock BinaryDataManager instead + // TODO: mock BinaryDataService instead await utils.initBinaryManager(); await utils.initNodeTypes(); diff --git a/packages/cli/test/integration/shared/utils/index.ts b/packages/cli/test/integration/shared/utils/index.ts index 6033422db23fc..039e2ee7ce198 100644 --- a/packages/cli/test/integration/shared/utils/index.ts +++ b/packages/cli/test/integration/shared/utils/index.ts @@ -3,7 +3,7 @@ import { randomBytes } from 'crypto'; import { existsSync } from 'fs'; import { CronJob } from 'cron'; import set from 'lodash/set'; -import { BinaryDataManager, UserSettings } from 'n8n-core'; +import { BinaryDataService, UserSettings } from 'n8n-core'; import type { ICredentialType, IExecuteFunctions, @@ -388,8 +388,13 @@ export async function initNodeTypes() { * Initialize a BinaryManager for test runs. */ export async function initBinaryManager() { - const binaryDataConfig = config.getEnv('binaryDataManager'); - await BinaryDataManager.init(binaryDataConfig); + const binaryDataConfig = config.getEnv('binaryDataService'); + + const binaryDataService = new BinaryDataService(); + await binaryDataService.init(binaryDataConfig); + Container.set(BinaryDataService, binaryDataService); + + await binaryDataService.init(binaryDataConfig); } /** From c7cf4b185bc39873fb61a6ccc4179cc9d6d58866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 19:01:21 +0200 Subject: [PATCH 039/103] Generalize interface --- .../core/src/binaryData/binaryData.service.ts | 37 +++++++------- packages/core/src/binaryData/fs.client.ts | 49 ++++++++++--------- packages/core/src/binaryData/types.ts | 27 +++++----- 3 files changed, 59 insertions(+), 54 deletions(-) diff --git a/packages/core/src/binaryData/binaryData.service.ts b/packages/core/src/binaryData/binaryData.service.ts index d0e81bdfa6444..7b0b9a7f9484a 100644 --- a/packages/core/src/binaryData/binaryData.service.ts +++ b/packages/core/src/binaryData/binaryData.service.ts @@ -28,29 +28,30 @@ export class BinaryDataService { return undefined; } - async copyBinaryFile(binaryData: IBinaryData, filePath: string, executionId: string) { + async copyBinaryFile(binaryData: IBinaryData, path: string, executionId: string) { // If a client handles this binary, copy over the binary file and return its reference id. const client = this.clients[this.mode]; + if (client) { - const identifier = await client.copyBinaryFile(filePath, executionId); + const identifier = await client.copyByPath(path, executionId); // Add client reference id. binaryData.id = this.generateBinaryId(identifier); // Prevent preserving data in memory if handled by a client. binaryData.data = this.mode; - const fileSize = await client.getFileSize(identifier); + const fileSize = await client.getSize(identifier); binaryData.fileSize = prettyBytes(fileSize); - await client.storeBinaryMetadata(identifier, { + await client.storeMetadata(identifier, { fileName: binaryData.fileName, mimeType: binaryData.mimeType, fileSize, }); } else { - const { size } = await stat(filePath); + const { size } = await stat(path); binaryData.fileSize = prettyBytes(size); - binaryData.data = await readFile(filePath, { encoding: BINARY_ENCODING }); + binaryData.data = await readFile(path, { encoding: BINARY_ENCODING }); } return binaryData; @@ -60,7 +61,7 @@ export class BinaryDataService { // If a client handles this binary, return the binary data with its reference id. const client = this.clients[this.mode]; if (client) { - const identifier = await client.storeBinaryData(input, executionId); + const identifier = await client.store(input, executionId); // Add client reference id. binaryData.id = this.generateBinaryId(identifier); @@ -68,10 +69,10 @@ export class BinaryDataService { // Prevent preserving data in memory if handled by a client. binaryData.data = this.mode; - const fileSize = await client.getFileSize(identifier); + const fileSize = await client.getSize(identifier); binaryData.fileSize = prettyBytes(fileSize); - await client.storeBinaryMetadata(identifier, { + await client.storeMetadata(identifier, { fileName: binaryData.fileName, mimeType: binaryData.mimeType, fileSize, @@ -95,7 +96,7 @@ export class BinaryDataService { getBinaryStream(identifier: string, chunkSize?: number) { const { mode, id } = this.splitBinaryModeFileId(identifier); if (this.clients[mode]) { - return this.clients[mode].getBinaryStream(id, chunkSize); + return this.clients[mode].toStream(id, chunkSize); } throw new Error('Storage mode used to store binary data not available'); @@ -112,7 +113,7 @@ export class BinaryDataService { async retrieveBinaryDataByIdentifier(identifier: string): Promise { const { mode, id } = this.splitBinaryModeFileId(identifier); if (this.clients[mode]) { - return this.clients[mode].retrieveBinaryDataByIdentifier(id); + return this.clients[mode].toBuffer(id); } throw new Error('Storage mode used to store binary data not available'); @@ -121,7 +122,7 @@ export class BinaryDataService { getBinaryPath(identifier: string) { const { mode, id } = this.splitBinaryModeFileId(identifier); if (this.clients[mode]) { - return this.clients[mode].getBinaryPath(id); + return this.clients[mode].getPath(id); } throw new Error('Storage mode used to store binary data not available'); @@ -130,15 +131,16 @@ export class BinaryDataService { async getBinaryMetadata(identifier: string) { const { mode, id } = this.splitBinaryModeFileId(identifier); if (this.clients[mode]) { - return this.clients[mode].getBinaryMetadata(id); + return this.clients[mode].getMetadata(id); } throw new Error('Storage mode used to store binary data not available'); } async deleteBinaryDataByExecutionIds(executionIds: string[]) { - if (this.clients[this.mode]) { - await this.clients[this.mode].deleteBinaryDataByExecutionIds(executionIds); + const client = this.clients[this.mode]; + if (client) { + await client.deleteManyByExecutionIds(executionIds); } } @@ -196,10 +198,7 @@ export class BinaryDataService { } return client - ?.duplicateBinaryDataByIdentifier( - this.splitBinaryModeFileId(binaryDataId).id, - executionId, - ) + ?.copyByIdentifier(this.splitBinaryModeFileId(binaryDataId).id, executionId) .then((filename) => ({ newId: this.generateBinaryId(filename), key, diff --git a/packages/core/src/binaryData/fs.client.ts b/packages/core/src/binaryData/fs.client.ts index 78887d79780f5..12e3b9daa9d7e 100644 --- a/packages/core/src/binaryData/fs.client.ts +++ b/packages/core/src/binaryData/fs.client.ts @@ -23,42 +23,36 @@ export class FileSystemClient implements BinaryData.Client { await this.assertFolder(this.storagePath); } - async getFileSize(identifier: string): Promise { - const stats = await fs.stat(this.getBinaryPath(identifier)); + async getSize(identifier: string): Promise { + const stats = await fs.stat(this.getPath(identifier)); return stats.size; } - async copyBinaryFile(filePath: string, executionId: string): Promise { - const binaryDataId = this.generateFileName(executionId); - await this.copyFileToLocalStorage(filePath, binaryDataId); - return binaryDataId; - } - - async storeBinaryMetadata(identifier: string, metadata: BinaryMetadata) { + async storeMetadata(identifier: string, metadata: BinaryMetadata) { await fs.writeFile(this.getMetadataPath(identifier), JSON.stringify(metadata), { encoding: 'utf-8', }); } - async getBinaryMetadata(identifier: string): Promise { + async getMetadata(identifier: string): Promise { return jsonParse(await fs.readFile(this.getMetadataPath(identifier), { encoding: 'utf-8' })); } - async storeBinaryData(binaryData: Buffer | Readable, executionId: string): Promise { + async store(binaryData: Buffer | Readable, executionId: string): Promise { const binaryDataId = this.generateFileName(executionId); await this.saveToLocalStorage(binaryData, binaryDataId); return binaryDataId; } - getBinaryStream(identifier: string, chunkSize?: number): Readable { - return createReadStream(this.getBinaryPath(identifier), { highWaterMark: chunkSize }); + toStream(identifier: string, chunkSize?: number): Readable { + return createReadStream(this.getPath(identifier), { highWaterMark: chunkSize }); } - async retrieveBinaryDataByIdentifier(identifier: string): Promise { + async toBuffer(identifier: string): Promise { return this.retrieveFromLocalStorage(identifier); } - getBinaryPath(identifier: string): string { + getPath(identifier: string): string { return this.resolveStoragePath(identifier); } @@ -66,17 +60,23 @@ export class FileSystemClient implements BinaryData.Client { return this.resolveStoragePath(`${identifier}.metadata`); } - async duplicateBinaryDataByIdentifier(binaryDataId: string, prefix: string): Promise { + async copyByPath(filePath: string, executionId: string): Promise { + const binaryDataId = this.generateFileName(executionId); + await this.copyFileToLocalStorage(filePath, binaryDataId); + return binaryDataId; + } + + async copyByIdentifier(identifier: string, prefix: string): Promise { const newBinaryDataId = this.generateFileName(prefix); await fs.copyFile( - this.resolveStoragePath(binaryDataId), + this.resolveStoragePath(identifier), this.resolveStoragePath(newBinaryDataId), ); return newBinaryDataId; } - async deleteBinaryDataByExecutionIds(executionIds: string[]): Promise { + async deleteManyByExecutionIds(executionIds: string[]): Promise { const set = new Set(executionIds); const fileNames = await fs.readdir(this.storagePath); const deletedIds = []; @@ -91,7 +91,7 @@ export class FileSystemClient implements BinaryData.Client { return deletedIds; } - async deleteBinaryDataByIdentifier(identifier: string): Promise { + async deleteOne(identifier: string): Promise { return this.deleteFromLocalStorage(identifier); } @@ -108,19 +108,19 @@ export class FileSystemClient implements BinaryData.Client { } private async deleteFromLocalStorage(identifier: string) { - return fs.rm(this.getBinaryPath(identifier)); + return fs.rm(this.getPath(identifier)); } private async copyFileToLocalStorage(source: string, identifier: string): Promise { - await fs.cp(source, this.getBinaryPath(identifier)); + await fs.cp(source, this.getPath(identifier)); } private async saveToLocalStorage(binaryData: Buffer | Readable, identifier: string) { - await fs.writeFile(this.getBinaryPath(identifier), binaryData); + await fs.writeFile(this.getPath(identifier), binaryData); } private async retrieveFromLocalStorage(identifier: string): Promise { - const filePath = this.getBinaryPath(identifier); + const filePath = this.getPath(identifier); try { return await fs.readFile(filePath); } catch (e) { @@ -130,8 +130,9 @@ export class FileSystemClient implements BinaryData.Client { private resolveStoragePath(...args: string[]) { const returnPath = path.join(this.storagePath, ...args); - if (path.relative(this.storagePath, returnPath).startsWith('..')) + if (path.relative(this.storagePath, returnPath).startsWith('..')) { throw new FileNotFoundError('Invalid path detected'); + } return returnPath; } } diff --git a/packages/core/src/binaryData/types.ts b/packages/core/src/binaryData/types.ts index 36ce9cdae1738..0a914aa3d33d8 100644 --- a/packages/core/src/binaryData/types.ts +++ b/packages/core/src/binaryData/types.ts @@ -23,16 +23,21 @@ export namespace BinaryData { export interface Client { init(startPurger: boolean): Promise; - getFileSize(filePath: string): Promise; - copyBinaryFile(filePath: string, executionId: string): Promise; - storeBinaryMetadata(identifier: string, metadata: BinaryMetadata): Promise; - getBinaryMetadata(identifier: string): Promise; - storeBinaryData(binaryData: Buffer | Readable, executionId: string): Promise; - retrieveBinaryDataByIdentifier(identifier: string): Promise; - getBinaryPath(identifier: string): string; - getBinaryStream(identifier: string, chunkSize?: number): Readable; - deleteBinaryDataByIdentifier(identifier: string): Promise; - duplicateBinaryDataByIdentifier(binaryDataId: string, prefix: string): Promise; - deleteBinaryDataByExecutionIds(executionIds: string[]): Promise; + + store(binaryData: Buffer | Readable, executionId: string): Promise; + getPath(identifier: string): string; + getSize(path: string): Promise; + + storeMetadata(identifier: string, metadata: BinaryMetadata): Promise; + getMetadata(identifier: string): Promise; + + toBuffer(identifier: string): Promise; + toStream(identifier: string, chunkSize?: number): Readable; + + copyByPath(path: string, executionId: string): Promise; + copyByIdentifier(identifier: string, prefix: string): Promise; + + deleteOne(identifier: string): Promise; + deleteManyByExecutionIds(executionIds: string[]): Promise; } } From 307bb1b595a56e3d51ccfecefd920651fd1a8c7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 19:33:37 +0200 Subject: [PATCH 040/103] Continue generalizing --- .../core/src/binaryData/binaryData.service.ts | 8 +- packages/core/src/binaryData/fs.client.ts | 144 +++++++++--------- packages/core/src/binaryData/s3.client.ts | 55 +++++++ packages/core/src/binaryData/types.ts | 7 +- 4 files changed, 135 insertions(+), 79 deletions(-) create mode 100644 packages/core/src/binaryData/s3.client.ts diff --git a/packages/core/src/binaryData/binaryData.service.ts b/packages/core/src/binaryData/binaryData.service.ts index 7b0b9a7f9484a..38b44b5ee5b96 100644 --- a/packages/core/src/binaryData/binaryData.service.ts +++ b/packages/core/src/binaryData/binaryData.service.ts @@ -21,7 +21,9 @@ export class BinaryDataService { this.mode = config.mode; if (this.availableModes.includes('filesystem')) { - this.clients.filesystem = new FileSystemClient(config as BinaryData.FileSystemConfig); // @TODO: Remove assertion + this.clients.filesystem = new FileSystemClient( + (config as BinaryData.FileSystemConfig).storagePath, + ); // @TODO: Remove assertion await this.clients.filesystem.init(mainClient); } @@ -96,7 +98,7 @@ export class BinaryDataService { getBinaryStream(identifier: string, chunkSize?: number) { const { mode, id } = this.splitBinaryModeFileId(identifier); if (this.clients[mode]) { - return this.clients[mode].toStream(id, chunkSize); + return this.clients[mode].getAsStream(id, chunkSize); } throw new Error('Storage mode used to store binary data not available'); @@ -113,7 +115,7 @@ export class BinaryDataService { async retrieveBinaryDataByIdentifier(identifier: string): Promise { const { mode, id } = this.splitBinaryModeFileId(identifier); if (this.clients[mode]) { - return this.clients[mode].toBuffer(id); + return this.clients[mode].getAsBuffer(id); } throw new Error('Storage mode used to store binary data not available'); diff --git a/packages/core/src/binaryData/fs.client.ts b/packages/core/src/binaryData/fs.client.ts index 12e3b9daa9d7e..f849b2490f744 100644 --- a/packages/core/src/binaryData/fs.client.ts +++ b/packages/core/src/binaryData/fs.client.ts @@ -2,137 +2,137 @@ import { createReadStream } from 'fs'; import fs from 'fs/promises'; import path from 'path'; import { v4 as uuid } from 'uuid'; -import type { Readable } from 'stream'; -import type { BinaryMetadata } from 'n8n-workflow'; import { jsonParse } from 'n8n-workflow'; +import { FileNotFoundError } from '../errors'; +import type { Readable } from 'stream'; +import type { BinaryMetadata } from 'n8n-workflow'; import type { BinaryData } from './types'; -import { FileNotFoundError } from '../errors'; -const executionExtractionRegexp = +const EXECUTION_EXTRACTOR = /^(\w+)(?:[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12})$/; export class FileSystemClient implements BinaryData.Client { - private storagePath: string; + constructor(private storagePath: string) {} - constructor(config: BinaryData.FileSystemConfig) { - this.storagePath = config.storagePath; + async init() { + await this.ensureDirExists(this.storagePath); } - async init() { - await this.assertFolder(this.storagePath); + getPath(identifier: string) { + return this.resolvePath(identifier); } - async getSize(identifier: string): Promise { - const stats = await fs.stat(this.getPath(identifier)); + async getSize(identifier: string) { + const filePath = this.getPath(identifier); + const stats = await fs.stat(filePath); + return stats.size; } - async storeMetadata(identifier: string, metadata: BinaryMetadata) { - await fs.writeFile(this.getMetadataPath(identifier), JSON.stringify(metadata), { - encoding: 'utf-8', - }); - } + getAsStream(identifier: string, chunkSize?: number) { + const filePath = this.getPath(identifier); - async getMetadata(identifier: string): Promise { - return jsonParse(await fs.readFile(this.getMetadataPath(identifier), { encoding: 'utf-8' })); + return createReadStream(filePath, { highWaterMark: chunkSize }); } - async store(binaryData: Buffer | Readable, executionId: string): Promise { - const binaryDataId = this.generateFileName(executionId); - await this.saveToLocalStorage(binaryData, binaryDataId); - return binaryDataId; - } + async getAsBuffer(identifier: string) { + const filePath = this.getPath(identifier); - toStream(identifier: string, chunkSize?: number): Readable { - return createReadStream(this.getPath(identifier), { highWaterMark: chunkSize }); + try { + return await fs.readFile(filePath); + } catch { + throw new Error(`Error finding file: ${filePath}`); + } } - async toBuffer(identifier: string): Promise { - return this.retrieveFromLocalStorage(identifier); - } + async storeMetadata(identifier: string, metadata: BinaryMetadata) { + const filePath = this.resolvePath(`${identifier}.metadata`); - getPath(identifier: string): string { - return this.resolveStoragePath(identifier); + await fs.writeFile(filePath, JSON.stringify(metadata), { encoding: 'utf-8' }); } - getMetadataPath(identifier: string): string { - return this.resolveStoragePath(`${identifier}.metadata`); + async getMetadata(identifier: string): Promise { + const filePath = this.resolvePath(`${identifier}.metadata`); + + return jsonParse(await fs.readFile(filePath, { encoding: 'utf-8' })); } - async copyByPath(filePath: string, executionId: string): Promise { - const binaryDataId = this.generateFileName(executionId); - await this.copyFileToLocalStorage(filePath, binaryDataId); - return binaryDataId; + async store(binaryData: Buffer | Readable, executionId: string) { + const identifier = this.createIdentifier(executionId); + const filePath = this.getPath(identifier); + + await fs.writeFile(filePath, binaryData); + + return identifier; } - async copyByIdentifier(identifier: string, prefix: string): Promise { - const newBinaryDataId = this.generateFileName(prefix); + async deleteOne(identifier: string) { + const filePath = this.getPath(identifier); - await fs.copyFile( - this.resolveStoragePath(identifier), - this.resolveStoragePath(newBinaryDataId), - ); - return newBinaryDataId; + return fs.rm(filePath); } - async deleteManyByExecutionIds(executionIds: string[]): Promise { + async deleteManyByExecutionIds(executionIds: string[]) { const set = new Set(executionIds); const fileNames = await fs.readdir(this.storagePath); const deletedIds = []; + for (const fileName of fileNames) { - const executionId = fileName.match(executionExtractionRegexp)?.[1]; + const executionId = fileName.match(EXECUTION_EXTRACTOR)?.[1]; + if (executionId && set.has(executionId)) { - const filePath = this.resolveStoragePath(fileName); + const filePath = this.resolvePath(fileName); + await Promise.all([fs.rm(filePath), fs.rm(`${filePath}.metadata`)]); + deletedIds.push(executionId); } } + return deletedIds; } - async deleteOne(identifier: string): Promise { - return this.deleteFromLocalStorage(identifier); - } + async copyByPath(filePath: string, executionId: string) { + const identifier = this.createIdentifier(executionId); - private async assertFolder(folder: string): Promise { - try { - await fs.access(folder); - } catch { - await fs.mkdir(folder, { recursive: true }); - } - } + await fs.cp(filePath, this.getPath(identifier)); - private generateFileName(prefix: string): string { - return [prefix, uuid()].join(''); + return identifier; } - private async deleteFromLocalStorage(identifier: string) { - return fs.rm(this.getPath(identifier)); - } + async copyByIdentifier(identifier: string, executionId: string) { + const newIdentifier = this.createIdentifier(executionId); - private async copyFileToLocalStorage(source: string, identifier: string): Promise { - await fs.cp(source, this.getPath(identifier)); - } + await fs.copyFile(this.resolvePath(identifier), this.resolvePath(newIdentifier)); - private async saveToLocalStorage(binaryData: Buffer | Readable, identifier: string) { - await fs.writeFile(this.getPath(identifier), binaryData); + return newIdentifier; } - private async retrieveFromLocalStorage(identifier: string): Promise { - const filePath = this.getPath(identifier); + // ---------------------------------- + // private methods + // ---------------------------------- + + private async ensureDirExists(dir: string) { try { - return await fs.readFile(filePath); - } catch (e) { - throw new Error(`Error finding file: ${filePath}`); + await fs.access(dir); + } catch { + await fs.mkdir(dir, { recursive: true }); } } - private resolveStoragePath(...args: string[]) { + private createIdentifier(executionId: string) { + return [executionId, uuid()].join(''); + } + + // @TODO: Variadic needed? + private resolvePath(...args: string[]) { const returnPath = path.join(this.storagePath, ...args); + if (path.relative(this.storagePath, returnPath).startsWith('..')) { throw new FileNotFoundError('Invalid path detected'); } + return returnPath; } } diff --git a/packages/core/src/binaryData/s3.client.ts b/packages/core/src/binaryData/s3.client.ts new file mode 100644 index 0000000000000..de69464d2afd4 --- /dev/null +++ b/packages/core/src/binaryData/s3.client.ts @@ -0,0 +1,55 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ + +import type { BinaryMetadata } from 'n8n-workflow'; +import type { Readable } from 'stream'; +import type { BinaryData } from './types'; + +export class S3Client implements BinaryData.Client { + async init() { + throw new Error('TODO'); + } + + async store(binaryData: Buffer | Readable, executionId: string): Promise { + throw new Error('TODO'); + } + + getPath(identifier: string): string { + throw new Error('TODO'); + } + + async getSize(path: string): Promise { + throw new Error('TODO'); + } + + async getAsBuffer(identifier: string): Promise { + throw new Error('TODO'); + } + + getAsStream(identifier: string, chunkSize?: number): Readable { + throw new Error('TODO'); + } + + async storeMetadata(identifier: string, metadata: BinaryMetadata): Promise { + throw new Error('TODO'); + } + + async getMetadata(identifier: string): Promise { + throw new Error('TODO'); + } + + async copyByPath(path: string, executionId: string): Promise { + throw new Error('TODO'); + } + + async copyByIdentifier(identifier: string, executionId: string): Promise { + throw new Error('TODO'); + } + + async deleteOne(identifier: string): Promise { + throw new Error('TODO'); + } + + async deleteManyByExecutionIds(executionIds: string[]): Promise { + throw new Error('TODO'); + } +} diff --git a/packages/core/src/binaryData/types.ts b/packages/core/src/binaryData/types.ts index 0a914aa3d33d8..41fd65457d28b 100644 --- a/packages/core/src/binaryData/types.ts +++ b/packages/core/src/binaryData/types.ts @@ -26,14 +26,13 @@ export namespace BinaryData { store(binaryData: Buffer | Readable, executionId: string): Promise; getPath(identifier: string): string; - getSize(path: string): Promise; + getSize(path: string): Promise; // @TODO: Refactor to use identifier? + getAsBuffer(identifier: string): Promise; + getAsStream(identifier: string, chunkSize?: number): Readable; storeMetadata(identifier: string, metadata: BinaryMetadata): Promise; getMetadata(identifier: string): Promise; - toBuffer(identifier: string): Promise; - toStream(identifier: string, chunkSize?: number): Readable; - copyByPath(path: string, executionId: string): Promise; copyByIdentifier(identifier: string, prefix: string): Promise; From a2528d707a6f4922720b601a2dadd2c05aa0ee3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 19:38:09 +0200 Subject: [PATCH 041/103] Fix test --- .../nodes/HttpRequest/test/node/HttpRequest.test.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/nodes-base/nodes/HttpRequest/test/node/HttpRequest.test.ts b/packages/nodes-base/nodes/HttpRequest/test/node/HttpRequest.test.ts index 9be90520ca86d..49617225e96f6 100644 --- a/packages/nodes-base/nodes/HttpRequest/test/node/HttpRequest.test.ts +++ b/packages/nodes-base/nodes/HttpRequest/test/node/HttpRequest.test.ts @@ -1,4 +1,10 @@ -import { setup, equalityTest, workflowToTests, getWorkflowFilenames } from '@test/nodes/Helpers'; +import { + initBinaryDataService, + setup, + equalityTest, + workflowToTests, + getWorkflowFilenames, +} from '@test/nodes/Helpers'; import nock from 'nock'; @@ -8,7 +14,8 @@ describe('Test HTTP Request Node', () => { const baseUrl = 'https://dummyjson.com'; - beforeAll(() => { + beforeAll(async () => { + await initBinaryDataService(); nock.disableNetConnect(); //GET From d0cd85e37e54eed747a7fa7fbc2ccced8f31f53b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 19:48:48 +0200 Subject: [PATCH 042/103] Better naming --- packages/cli/src/Server.ts | 4 +-- packages/cli/src/WebhookHelpers.ts | 4 +-- .../repositories/execution.repository.ts | 2 +- packages/core/src/NodeExecuteFunctions.ts | 8 +++--- .../core/src/binaryData/binaryData.service.ts | 26 ++++++++++++------- packages/core/src/binaryData/fs.client.ts | 2 +- 6 files changed, 27 insertions(+), 19 deletions(-) diff --git a/packages/cli/src/Server.ts b/packages/cli/src/Server.ts index 22c685214e8ae..a69dbc1c7b1be 100644 --- a/packages/cli/src/Server.ts +++ b/packages/cli/src/Server.ts @@ -1425,11 +1425,11 @@ export class Server extends AbstractServer { // TODO UM: check if this needs permission check for UM const identifier = req.params.path; try { - const binaryPath = this.binaryDataService.getBinaryPath(identifier); + const binaryPath = this.binaryDataService.getPath(identifier); let { mode, fileName, mimeType } = req.query; if (!fileName || !mimeType) { try { - const metadata = await this.binaryDataService.getBinaryMetadata(identifier); + const metadata = await this.binaryDataService.getMetadata(identifier); fileName = metadata.fileName; mimeType = metadata.mimeType; res.setHeader('Content-Length', metadata.fileSize); diff --git a/packages/cli/src/WebhookHelpers.ts b/packages/cli/src/WebhookHelpers.ts index fcefd8dc9741c..ea052f507ec18 100644 --- a/packages/cli/src/WebhookHelpers.ts +++ b/packages/cli/src/WebhookHelpers.ts @@ -514,7 +514,7 @@ export async function executeWebhook( const binaryData = (response.body as IDataObject)?.binaryData as IBinaryData; if (binaryData?.id) { res.header(response.headers); - const stream = Container.get(BinaryDataService).getBinaryStream(binaryData.id); + const stream = Container.get(BinaryDataService).getAsStream(binaryData.id); void pipeline(stream, res).then(() => responseCallback(null, { noWebhookResponse: true }), ); @@ -732,7 +732,7 @@ export async function executeWebhook( // Send the webhook response manually res.setHeader('Content-Type', binaryData.mimeType); if (binaryData.id) { - const stream = Container.get(BinaryDataService).getBinaryStream(binaryData.id); + const stream = Container.get(BinaryDataService).getAsStream(binaryData.id); await pipeline(stream, res); } else { res.end(Buffer.from(binaryData.data, BINARY_ENCODING)); diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 2b5b3c24725fb..1c243276ed13b 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -490,7 +490,7 @@ export class ExecutionRepository extends Repository { }) ).map(({ id }) => id); - await this.binaryDataService.deleteBinaryDataByExecutionIds(executionIds); + await this.binaryDataService.deleteManyByExecutionIds(executionIds); // Actually delete these executions await this.delete({ id: In(executionIds) }); diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index 38a2118a0504c..cae93405729b8 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -864,21 +864,21 @@ async function httpRequest( } export function getBinaryPath(binaryDataId: string): string { - return Container.get(BinaryDataService).getBinaryPath(binaryDataId); + return Container.get(BinaryDataService).getPath(binaryDataId); } /** * Returns binary file metadata */ export async function getBinaryMetadata(binaryDataId: string): Promise { - return Container.get(BinaryDataService).getBinaryMetadata(binaryDataId); + return Container.get(BinaryDataService).getMetadata(binaryDataId); } /** * Returns binary file stream for piping */ export function getBinaryStream(binaryDataId: string, chunkSize?: number): Readable { - return Container.get(BinaryDataService).getBinaryStream(binaryDataId, chunkSize); + return Container.get(BinaryDataService).getAsStream(binaryDataId, chunkSize); } export function assertBinaryData( @@ -931,7 +931,7 @@ export async function setBinaryDataBuffer( binaryData: Buffer | Readable, executionId: string, ): Promise { - return Container.get(BinaryDataService).storeBinaryData(data, binaryData, executionId); + return Container.get(BinaryDataService).store(data, binaryData, executionId); } export async function copyBinaryFile( diff --git a/packages/core/src/binaryData/binaryData.service.ts b/packages/core/src/binaryData/binaryData.service.ts index 38b44b5ee5b96..7fc6b6b609b64 100644 --- a/packages/core/src/binaryData/binaryData.service.ts +++ b/packages/core/src/binaryData/binaryData.service.ts @@ -37,7 +37,7 @@ export class BinaryDataService { if (client) { const identifier = await client.copyByPath(path, executionId); // Add client reference id. - binaryData.id = this.generateBinaryId(identifier); + binaryData.id = this.createIdentifier(identifier); // Prevent preserving data in memory if handled by a client. binaryData.data = this.mode; @@ -59,14 +59,14 @@ export class BinaryDataService { return binaryData; } - async storeBinaryData(binaryData: IBinaryData, input: Buffer | Readable, executionId: string) { + async store(binaryData: IBinaryData, input: Buffer | Readable, executionId: string) { // If a client handles this binary, return the binary data with its reference id. const client = this.clients[this.mode]; if (client) { const identifier = await client.store(input, executionId); // Add client reference id. - binaryData.id = this.generateBinaryId(identifier); + binaryData.id = this.createIdentifier(identifier); // Prevent preserving data in memory if handled by a client. binaryData.data = this.mode; @@ -95,8 +95,9 @@ export class BinaryDataService { }); } - getBinaryStream(identifier: string, chunkSize?: number) { + getAsStream(identifier: string, chunkSize?: number) { const { mode, id } = this.splitBinaryModeFileId(identifier); + if (this.clients[mode]) { return this.clients[mode].getAsStream(id, chunkSize); } @@ -114,6 +115,7 @@ export class BinaryDataService { async retrieveBinaryDataByIdentifier(identifier: string): Promise { const { mode, id } = this.splitBinaryModeFileId(identifier); + if (this.clients[mode]) { return this.clients[mode].getAsBuffer(id); } @@ -121,8 +123,9 @@ export class BinaryDataService { throw new Error('Storage mode used to store binary data not available'); } - getBinaryPath(identifier: string) { + getPath(identifier: string) { const { mode, id } = this.splitBinaryModeFileId(identifier); + if (this.clients[mode]) { return this.clients[mode].getPath(id); } @@ -130,7 +133,7 @@ export class BinaryDataService { throw new Error('Storage mode used to store binary data not available'); } - async getBinaryMetadata(identifier: string) { + async getMetadata(identifier: string) { const { mode, id } = this.splitBinaryModeFileId(identifier); if (this.clients[mode]) { return this.clients[mode].getMetadata(id); @@ -139,7 +142,7 @@ export class BinaryDataService { throw new Error('Storage mode used to store binary data not available'); } - async deleteBinaryDataByExecutionIds(executionIds: string[]) { + async deleteManyByExecutionIds(executionIds: string[]) { const client = this.clients[this.mode]; if (client) { await client.deleteManyByExecutionIds(executionIds); @@ -172,12 +175,17 @@ export class BinaryDataService { return inputData as INodeExecutionData[][]; } - private generateBinaryId(filename: string) { + // ---------------------------------- + // private methods + // ---------------------------------- + + private createIdentifier(filename: string) { return `${this.mode}:${filename}`; } private splitBinaryModeFileId(fileId: string): { mode: string; id: string } { const [mode, id] = fileId.split(':'); + return { mode, id }; } @@ -202,7 +210,7 @@ export class BinaryDataService { return client ?.copyByIdentifier(this.splitBinaryModeFileId(binaryDataId).id, executionId) .then((filename) => ({ - newId: this.generateBinaryId(filename), + newId: this.createIdentifier(filename), key, })); }); diff --git a/packages/core/src/binaryData/fs.client.ts b/packages/core/src/binaryData/fs.client.ts index f849b2490f744..aeb0d26fd2294 100644 --- a/packages/core/src/binaryData/fs.client.ts +++ b/packages/core/src/binaryData/fs.client.ts @@ -110,7 +110,7 @@ export class FileSystemClient implements BinaryData.Client { } // ---------------------------------- - // private methods + // private methods // ---------------------------------- private async ensureDirExists(dir: string) { From bfdbdf5c37160f3dbd8b17ff3ddbc1f5912edacd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 19:50:54 +0200 Subject: [PATCH 043/103] More renamings --- packages/cli/src/commands/BaseCommand.ts | 2 +- packages/cli/src/commands/execute.ts | 2 +- packages/cli/src/commands/executeBatch.ts | 2 +- packages/cli/src/commands/start.ts | 2 +- packages/cli/src/commands/webhook.ts | 2 +- packages/cli/src/commands/worker.ts | 2 +- packages/cli/test/integration/publicApi/executions.test.ts | 2 +- packages/cli/test/integration/shared/utils/index.ts | 4 ++-- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/cli/src/commands/BaseCommand.ts b/packages/cli/src/commands/BaseCommand.ts index ebb4d2c51a5c0..9950af15c7e36 100644 --- a/packages/cli/src/commands/BaseCommand.ts +++ b/packages/cli/src/commands/BaseCommand.ts @@ -103,7 +103,7 @@ export abstract class BaseCommand extends Command { process.exit(1); } - protected async initBinaryManager() { + protected async initBinaryDataService() { const binaryDataConfig = config.getEnv('binaryDataService'); await Container.get(BinaryDataService).init(binaryDataConfig, true); } diff --git a/packages/cli/src/commands/execute.ts b/packages/cli/src/commands/execute.ts index c708e664decbe..f50d6c7f53d51 100644 --- a/packages/cli/src/commands/execute.ts +++ b/packages/cli/src/commands/execute.ts @@ -33,7 +33,7 @@ export class Execute extends BaseCommand { async init() { await super.init(); - await this.initBinaryManager(); + await this.initBinaryDataService(); await this.initExternalHooks(); } diff --git a/packages/cli/src/commands/executeBatch.ts b/packages/cli/src/commands/executeBatch.ts index 4822747478da2..a20348c92e5da 100644 --- a/packages/cli/src/commands/executeBatch.ts +++ b/packages/cli/src/commands/executeBatch.ts @@ -180,7 +180,7 @@ export class ExecuteBatch extends BaseCommand { async init() { await super.init(); - await this.initBinaryManager(); + await this.initBinaryDataService(); await this.initExternalHooks(); } diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index 15000ab56ae2d..371e3cf35e736 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -198,7 +198,7 @@ export class Start extends BaseCommand { this.activeWorkflowRunner = Container.get(ActiveWorkflowRunner); await this.initLicense(); - await this.initBinaryManager(); + await this.initBinaryDataService(); await this.initExternalHooks(); await this.initExternalSecrets(); diff --git a/packages/cli/src/commands/webhook.ts b/packages/cli/src/commands/webhook.ts index 7d5bf4630232e..a4b7fe5ae0752 100644 --- a/packages/cli/src/commands/webhook.ts +++ b/packages/cli/src/commands/webhook.ts @@ -78,7 +78,7 @@ export class Webhook extends BaseCommand { await super.init(); await this.initLicense(); - await this.initBinaryManager(); + await this.initBinaryDataService(); await this.initExternalHooks(); await this.initExternalSecrets(); } diff --git a/packages/cli/src/commands/worker.ts b/packages/cli/src/commands/worker.ts index dfaf0e34c0c62..ad8691b3a63a2 100644 --- a/packages/cli/src/commands/worker.ts +++ b/packages/cli/src/commands/worker.ts @@ -237,7 +237,7 @@ export class Worker extends BaseCommand { this.logger.debug('Starting n8n worker...'); await this.initLicense(); - await this.initBinaryManager(); + await this.initBinaryDataService(); await this.initExternalHooks(); await this.initExternalSecrets(); } diff --git a/packages/cli/test/integration/publicApi/executions.test.ts b/packages/cli/test/integration/publicApi/executions.test.ts index cb8ae69c41fd0..0eece7838d851 100644 --- a/packages/cli/test/integration/publicApi/executions.test.ts +++ b/packages/cli/test/integration/publicApi/executions.test.ts @@ -24,7 +24,7 @@ beforeAll(async () => { user2 = await testDb.createUser({ globalRole: globalUserRole, apiKey: randomApiKey() }); // TODO: mock BinaryDataService instead - await utils.initBinaryManager(); + await utils.initBinaryDataService(); await utils.initNodeTypes(); workflowRunner = await utils.initActiveWorkflowRunner(); diff --git a/packages/cli/test/integration/shared/utils/index.ts b/packages/cli/test/integration/shared/utils/index.ts index 039e2ee7ce198..96594b533a3b6 100644 --- a/packages/cli/test/integration/shared/utils/index.ts +++ b/packages/cli/test/integration/shared/utils/index.ts @@ -385,9 +385,9 @@ export async function initNodeTypes() { } /** - * Initialize a BinaryManager for test runs. + * Initialize a BinaryDataService for test runs. */ -export async function initBinaryManager() { +export async function initBinaryDataService() { const binaryDataConfig = config.getEnv('binaryDataService'); const binaryDataService = new BinaryDataService(); From 25e80b90c0a0fe0a68aa8752b0c97a8727dd741c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 19:54:40 +0200 Subject: [PATCH 044/103] Minor cleanup --- .../core/src/binaryData/binaryData.service.ts | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/packages/core/src/binaryData/binaryData.service.ts b/packages/core/src/binaryData/binaryData.service.ts index 7fc6b6b609b64..e6980c85961b5 100644 --- a/packages/core/src/binaryData/binaryData.service.ts +++ b/packages/core/src/binaryData/binaryData.service.ts @@ -98,27 +98,25 @@ export class BinaryDataService { getAsStream(identifier: string, chunkSize?: number) { const { mode, id } = this.splitBinaryModeFileId(identifier); - if (this.clients[mode]) { - return this.clients[mode].getAsStream(id, chunkSize); - } + const client = this.clients[mode]; + + if (client) return client.getAsStream(id, chunkSize); throw new Error('Storage mode used to store binary data not available'); } async getBinaryDataBuffer(binaryData: IBinaryData) { - if (binaryData.id) { - return this.retrieveBinaryDataByIdentifier(binaryData.id); - } + if (binaryData.id) return this.retrieveBinaryDataByIdentifier(binaryData.id); return Buffer.from(binaryData.data, BINARY_ENCODING); } - async retrieveBinaryDataByIdentifier(identifier: string): Promise { + async retrieveBinaryDataByIdentifier(identifier: string) { const { mode, id } = this.splitBinaryModeFileId(identifier); - if (this.clients[mode]) { - return this.clients[mode].getAsBuffer(id); - } + const client = this.clients[mode]; + + if (client) return client.getAsBuffer(id); throw new Error('Storage mode used to store binary data not available'); } @@ -126,27 +124,27 @@ export class BinaryDataService { getPath(identifier: string) { const { mode, id } = this.splitBinaryModeFileId(identifier); - if (this.clients[mode]) { - return this.clients[mode].getPath(id); - } + const client = this.clients[mode]; + + if (client) return client.getPath(id); throw new Error('Storage mode used to store binary data not available'); } async getMetadata(identifier: string) { const { mode, id } = this.splitBinaryModeFileId(identifier); - if (this.clients[mode]) { - return this.clients[mode].getMetadata(id); - } + + const client = this.clients[mode]; + + if (client) return client.getMetadata(id); throw new Error('Storage mode used to store binary data not available'); } async deleteManyByExecutionIds(executionIds: string[]) { const client = this.clients[this.mode]; - if (client) { - await client.deleteManyByExecutionIds(executionIds); - } + + if (client) await client.deleteManyByExecutionIds(executionIds); } async duplicateBinaryData(inputData: Array, executionId: string) { From 3476f6c9ae02674027eb837c6b519b7984c62e19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 20:07:10 +0200 Subject: [PATCH 045/103] More minor cleanup --- .../core/src/binaryData/binaryData.service.ts | 116 ++++++++---------- packages/core/src/binaryData/fs.client.ts | 1 + 2 files changed, 53 insertions(+), 64 deletions(-) diff --git a/packages/core/src/binaryData/binaryData.service.ts b/packages/core/src/binaryData/binaryData.service.ts index e6980c85961b5..9a43b5495923c 100644 --- a/packages/core/src/binaryData/binaryData.service.ts +++ b/packages/core/src/binaryData/binaryData.service.ts @@ -1,12 +1,14 @@ import { readFile, stat } from 'fs/promises'; -import type { IBinaryData, INodeExecutionData } from 'n8n-workflow'; +import concatStream from 'concat-stream'; import prettyBytes from 'pretty-bytes'; -import type { Readable } from 'stream'; +import { Service } from 'typedi'; import { BINARY_ENCODING } from 'n8n-workflow'; -import type { BinaryData } from './types'; + import { FileSystemClient } from './fs.client'; -import { Service } from 'typedi'; -import concatStream from 'concat-stream'; + +import type { Readable } from 'stream'; +import type { BinaryData } from './types'; +import type { IBinaryData, INodeExecutionData } from 'n8n-workflow'; @Service() export class BinaryDataService { @@ -31,60 +33,56 @@ export class BinaryDataService { } async copyBinaryFile(binaryData: IBinaryData, path: string, executionId: string) { - // If a client handles this binary, copy over the binary file and return its reference id. const client = this.clients[this.mode]; - if (client) { - const identifier = await client.copyByPath(path, executionId); - // Add client reference id. - binaryData.id = this.createIdentifier(identifier); - - // Prevent preserving data in memory if handled by a client. - binaryData.data = this.mode; - - const fileSize = await client.getSize(identifier); - binaryData.fileSize = prettyBytes(fileSize); - - await client.storeMetadata(identifier, { - fileName: binaryData.fileName, - mimeType: binaryData.mimeType, - fileSize, - }); - } else { + if (!client) { const { size } = await stat(path); binaryData.fileSize = prettyBytes(size); binaryData.data = await readFile(path, { encoding: BINARY_ENCODING }); + + return binaryData; } + const identifier = await client.copyByPath(path, executionId); + binaryData.id = this.createIdentifier(identifier); + binaryData.data = this.mode; // clear from memory + + const fileSize = await client.getSize(identifier); + binaryData.fileSize = prettyBytes(fileSize); + + await client.storeMetadata(identifier, { + fileName: binaryData.fileName, + mimeType: binaryData.mimeType, + fileSize, + }); + return binaryData; } async store(binaryData: IBinaryData, input: Buffer | Readable, executionId: string) { - // If a client handles this binary, return the binary data with its reference id. const client = this.clients[this.mode]; - if (client) { - const identifier = await client.store(input, executionId); - - // Add client reference id. - binaryData.id = this.createIdentifier(identifier); - // Prevent preserving data in memory if handled by a client. - binaryData.data = this.mode; - - const fileSize = await client.getSize(identifier); - binaryData.fileSize = prettyBytes(fileSize); - - await client.storeMetadata(identifier, { - fileName: binaryData.fileName, - mimeType: binaryData.mimeType, - fileSize, - }); - } else { + if (!client) { const buffer = await this.binaryToBuffer(input); binaryData.data = buffer.toString(BINARY_ENCODING); binaryData.fileSize = prettyBytes(buffer.length); + + return binaryData; } + const identifier = await client.store(input, executionId); + binaryData.id = this.createIdentifier(identifier); + binaryData.data = this.mode; // clear from memory + + const fileSize = await client.getSize(identifier); + binaryData.fileSize = prettyBytes(fileSize); + + await client.storeMetadata(identifier, { + fileName: binaryData.fileName, + mimeType: binaryData.mimeType, + fileSize, + }); + return binaryData; } @@ -98,11 +96,7 @@ export class BinaryDataService { getAsStream(identifier: string, chunkSize?: number) { const { mode, id } = this.splitBinaryModeFileId(identifier); - const client = this.clients[mode]; - - if (client) return client.getAsStream(id, chunkSize); - - throw new Error('Storage mode used to store binary data not available'); + return this.getClient(mode).getAsStream(id, chunkSize); } async getBinaryDataBuffer(binaryData: IBinaryData) { @@ -114,37 +108,23 @@ export class BinaryDataService { async retrieveBinaryDataByIdentifier(identifier: string) { const { mode, id } = this.splitBinaryModeFileId(identifier); - const client = this.clients[mode]; - - if (client) return client.getAsBuffer(id); - - throw new Error('Storage mode used to store binary data not available'); + return this.getClient(mode).getAsBuffer(id); } getPath(identifier: string) { const { mode, id } = this.splitBinaryModeFileId(identifier); - const client = this.clients[mode]; - - if (client) return client.getPath(id); - - throw new Error('Storage mode used to store binary data not available'); + return this.getClient(mode).getPath(id); } async getMetadata(identifier: string) { const { mode, id } = this.splitBinaryModeFileId(identifier); - const client = this.clients[mode]; - - if (client) return client.getMetadata(id); - - throw new Error('Storage mode used to store binary data not available'); + return this.getClient(mode).getMetadata(id); } async deleteManyByExecutionIds(executionIds: string[]) { - const client = this.clients[this.mode]; - - if (client) await client.deleteManyByExecutionIds(executionIds); + await this.getClient(this.mode).deleteManyByExecutionIds(executionIds); } async duplicateBinaryData(inputData: Array, executionId: string) { @@ -226,4 +206,12 @@ export class BinaryDataService { return executionData; } + + private getClient(mode: string) { + const client = this.clients[mode]; + + if (!client) throw new Error('This method is not supported by in-memory storage mode'); + + return client; + } } diff --git a/packages/core/src/binaryData/fs.client.ts b/packages/core/src/binaryData/fs.client.ts index aeb0d26fd2294..7a9b052991fee 100644 --- a/packages/core/src/binaryData/fs.client.ts +++ b/packages/core/src/binaryData/fs.client.ts @@ -3,6 +3,7 @@ import fs from 'fs/promises'; import path from 'path'; import { v4 as uuid } from 'uuid'; import { jsonParse } from 'n8n-workflow'; + import { FileNotFoundError } from '../errors'; import type { Readable } from 'stream'; From 9a25301ee96535d1eed8b5223bc5f0af56e7e3dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 13 Sep 2023 20:12:03 +0200 Subject: [PATCH 046/103] Add clarifying comment --- packages/cli/src/constants.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/cli/src/constants.ts b/packages/cli/src/constants.ts index ec66cbdf32062..c94af1f5c64b1 100644 --- a/packages/cli/src/constants.ts +++ b/packages/cli/src/constants.ts @@ -95,6 +95,9 @@ export const CREDENTIAL_BLANKING_VALUE = '__n8n_BLANK_VALUE_e5362baf-c777-4d57-a export const UM_FIX_INSTRUCTION = 'Please fix the database by running ./packages/cli/bin/n8n user-management:reset'; +/** + * Units of time in milliseconds + */ export const TIME = { SECOND: 1000, MINUTE: 60 * 1000, From 0e075c257d26f5f34fb722fdd2f5120a96f09023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 14 Sep 2023 10:51:50 +0200 Subject: [PATCH 047/103] Speed up pruning if high volume --- .../repositories/execution.repository.ts | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 6cd7561026b3a..6998d9292cff7 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -78,6 +78,8 @@ function parseFiltersToQueryBuilder( export class ExecutionRepository extends Repository { deletionBatchSize = 100; + hardDeletionInterval: NodeJS.Timer | null = null; + constructor( dataSource: DataSource, private readonly executionDataRepository: ExecutionDataRepository, @@ -88,7 +90,15 @@ export class ExecutionRepository extends Repository { setInterval(async () => this.pruneBySoftDeleting(), 1 * TIME.HOUR); } - setInterval(async () => this.hardDelete(), 15 * TIME.MINUTE); + this.setHardDeletionInterval(); + } + + setHardDeletionInterval() { + this.hardDeletionInterval = setInterval(async () => this.hardDelete(), 15 * TIME.MINUTE); + } + + clearHardDeletionInterval() { + if (this.hardDeletionInterval) clearInterval(this.hardDeletionInterval); } async findMultipleExecutions( @@ -495,8 +505,22 @@ export class ExecutionRepository extends Repository { // Actually delete these executions await this.delete({ id: In(executionIds) }); + /** + * If the volume of executions to prune is as high as the batch size, there is a risk + * that the pruning process is unable to catch up to the creation of new executions, + * with high concurrency possibly leading to errors from duplicate deletions. + * + * Therefore, in this high-volume case we speed up the hard deletion cycle, until + * the number of executions to prune is low enough to fit in a single batch. + */ if (executionIds.length === this.deletionBatchSize) { - setTimeout(async () => this.hardDelete(), 1000); + this.clearHardDeletionInterval(); + + setTimeout(async () => this.hardDelete(), 1 * TIME.SECOND); + } else { + if (this.hardDeletionInterval) return; + + this.setHardDeletionInterval(); } } } From 2071c7fb677b8e4818356d55d095e056704f8573 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 14 Sep 2023 11:08:04 +0200 Subject: [PATCH 048/103] Remove call from hook --- packages/cli/src/WorkflowExecuteAdditionalData.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/cli/src/WorkflowExecuteAdditionalData.ts b/packages/cli/src/WorkflowExecuteAdditionalData.ts index 193acce3a53aa..6ed7d4bb0d38f 100644 --- a/packages/cli/src/WorkflowExecuteAdditionalData.ts +++ b/packages/cli/src/WorkflowExecuteAdditionalData.ts @@ -599,11 +599,6 @@ function hookFunctionsSaveWorker(): IWorkflowExecuteHooks { workflowId: this.workflowData.id, }); try { - // Prune old execution data - if (config.getEnv('executions.pruneData')) { - await pruneExecutionData.call(this); - } - if (isWorkflowIdValid(this.workflowData.id) && newStaticData) { // Workflow is saved so update in database try { From 1d50b61f89622f0d48889c6237fb50ea8938126a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 14 Sep 2023 15:37:06 +0200 Subject: [PATCH 049/103] Refix conflict --- packages/cli/src/WebhookHelpers.ts | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/packages/cli/src/WebhookHelpers.ts b/packages/cli/src/WebhookHelpers.ts index 2c9bf77ab4b8d..703e4b63b800d 100644 --- a/packages/cli/src/WebhookHelpers.ts +++ b/packages/cli/src/WebhookHelpers.ts @@ -734,7 +734,7 @@ export async function executeWebhook( // Send the webhook response manually res.setHeader('Content-Type', binaryData.mimeType); if (binaryData.id) { - const stream = BinaryDataManager.getInstance().getBinaryStream(binaryData.id); + const stream = Container.get(BinaryDataService).getAsStream(binaryData.id); await pipeline(stream, res); } else { res.end(Buffer.from(binaryData.data, BINARY_ENCODING)); @@ -756,17 +756,9 @@ export async function executeWebhook( } if (!didSendResponse) { - // Send the webhook response manually - res.setHeader('Content-Type', binaryData.mimeType); - if (binaryData.id) { - const stream = Container.get(BinaryDataService).getAsStream(binaryData.id); - await pipeline(stream, res); - } else { - res.end(Buffer.from(binaryData.data, BINARY_ENCODING)); - } - responseCallback(null, { - noWebhookResponse: true, + data, + responseCode, }); } } From e042e91eaefad4008859353a543e426cc4160cbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 14 Sep 2023 17:09:10 +0200 Subject: [PATCH 050/103] Add formats to schema --- packages/cli/src/config/schema.ts | 37 +++++++++++++------------------ 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/packages/cli/src/config/schema.ts b/packages/cli/src/config/schema.ts index fbca60deefaad..0ff74c7207fb2 100644 --- a/packages/cli/src/config/schema.ts +++ b/packages/cli/src/config/schema.ts @@ -2,28 +2,21 @@ import path from 'path'; import convict from 'convict'; import { UserSettings } from 'n8n-core'; import { jsonParse } from 'n8n-workflow'; +import { ensureStringArray } from './utils'; convict.addFormat({ - name: 'nodes-list', - // @ts-ignore - validate(values: string[], { env }: { env: string }): void { - try { - if (!Array.isArray(values)) { - throw new Error(); - } + name: 'json-string-array', + coerce: (rawStr: string) => + jsonParse(rawStr, { + errorMessage: `Expected this value "${rawStr}" to be valid JSON`, + }), + validate: ensureStringArray, +}); - for (const value of values) { - if (typeof value !== 'string') { - throw new Error(); - } - } - } catch (error) { - throw new TypeError(`${env} is not a valid Array of strings.`); - } - }, - coerce(rawValue: string): string[] { - return jsonParse(rawValue, { errorMessage: 'nodes-list needs to be valid JSON' }); - }, +convict.addFormat({ + name: 'comma-separated-list', + coerce: (rawStr: string) => rawStr.split(','), + validate: ensureStringArray, }); export const schema = { @@ -782,13 +775,13 @@ export const schema = { nodes: { include: { doc: 'Nodes to load', - format: 'nodes-list', + format: 'json-string-array', default: undefined, env: 'NODES_INCLUDE', }, exclude: { doc: 'Nodes not to load', - format: 'nodes-list', + format: 'json-string-array', default: undefined, env: 'NODES_EXCLUDE', }, @@ -896,7 +889,7 @@ export const schema = { binaryDataService: { availableModes: { - format: String, + format: 'comma-separated-list', default: 'filesystem,object', env: 'N8N_AVAILABLE_BINARY_DATA_MODES', doc: 'Available modes of binary data storage, as comma separated strings', From e948f8f2c2e9a36ce336e2dc19087e00fb90bbe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 14 Sep 2023 17:09:20 +0200 Subject: [PATCH 051/103] Add `ensureStringArray` util --- packages/cli/src/config/utils.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 packages/cli/src/config/utils.ts diff --git a/packages/cli/src/config/utils.ts b/packages/cli/src/config/utils.ts new file mode 100644 index 0000000000000..3432e3a78242c --- /dev/null +++ b/packages/cli/src/config/utils.ts @@ -0,0 +1,18 @@ +import type { SchemaObj } from 'convict'; + +class NotStringArrayError extends Error { + constructor(env: string) { + super(); + this.message = `${env} is not a string array.`; + } +} + +export const ensureStringArray = (values: string[], { env }: SchemaObj) => { + if (!env) throw new Error(`Missing env: ${env}`); + + if (!Array.isArray(values)) throw new NotStringArrayError(env); + + for (const value of values) { + if (typeof value !== 'string') throw new NotStringArrayError(env); + } +}; From 0feb926334c452eca8700531c6bc2646c70eea6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 14 Sep 2023 17:09:28 +0200 Subject: [PATCH 052/103] Adjust types based on parsing --- packages/core/src/binaryData/types.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/core/src/binaryData/types.ts b/packages/core/src/binaryData/types.ts index 41fd65457d28b..ce4185ff66456 100644 --- a/packages/core/src/binaryData/types.ts +++ b/packages/core/src/binaryData/types.ts @@ -1,18 +1,13 @@ import type { Readable } from 'stream'; import type { BinaryMetadata } from 'n8n-workflow'; +import type { BINARY_DATA_MODES } from './utils'; export namespace BinaryData { - /** - * Mode for storing binary data: - * - `default` (in memory) - * - `filesystem` (on disk) - * - `object` (S3) - */ - export type Mode = 'default' | 'filesystem' | 'object'; + export type Mode = (typeof BINARY_DATA_MODES)[number]; type ConfigBase = { mode: Mode; - availableModes: string; // comma-separated list + availableModes: string[]; }; type InMemoryConfig = ConfigBase & { mode: 'default' }; From c73f18c2e2dbac85c9ba4b90dc609c1a8dcd4ab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 14 Sep 2023 17:09:37 +0200 Subject: [PATCH 053/103] Add type guard --- packages/core/src/binaryData/utils.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 packages/core/src/binaryData/utils.ts diff --git a/packages/core/src/binaryData/utils.ts b/packages/core/src/binaryData/utils.ts new file mode 100644 index 0000000000000..a3dd1f5ab640a --- /dev/null +++ b/packages/core/src/binaryData/utils.ts @@ -0,0 +1,15 @@ +import type { BinaryData } from './types'; + +export class InvalidModeError extends Error {} + +/** + * Modes for storing binary data: + * - `default` (in memory) + * - `filesystem` (on disk) + * - `object` (S3) + */ +export const BINARY_DATA_MODES = ['default', 'filesystem', 'object'] as const; + +export function areValidModes(modes: string[]): modes is BinaryData.Mode[] { + return modes.every((m) => BINARY_DATA_MODES.includes(m as BinaryData.Mode)); +} From f8fec927b420048be1e389775df2fbeb3bb65f79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 14 Sep 2023 17:09:48 +0200 Subject: [PATCH 054/103] Clean up types in `init()` --- packages/core/src/binaryData/binaryData.service.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/core/src/binaryData/binaryData.service.ts b/packages/core/src/binaryData/binaryData.service.ts index 9a43b5495923c..875cea8ae9925 100644 --- a/packages/core/src/binaryData/binaryData.service.ts +++ b/packages/core/src/binaryData/binaryData.service.ts @@ -5,6 +5,7 @@ import { Service } from 'typedi'; import { BINARY_ENCODING } from 'n8n-workflow'; import { FileSystemClient } from './fs.client'; +import { InvalidModeError, areValidModes } from './utils'; import type { Readable } from 'stream'; import type { BinaryData } from './types'; @@ -19,13 +20,14 @@ export class BinaryDataService { private clients: Record = {}; async init(config: BinaryData.Config, mainClient = false) { - this.availableModes = config.availableModes.split(',') as BinaryData.Mode[]; // @TODO: Remove assertion + if (!areValidModes(config.availableModes)) throw new InvalidModeError(); + + this.availableModes = config.availableModes; this.mode = config.mode; - if (this.availableModes.includes('filesystem')) { - this.clients.filesystem = new FileSystemClient( - (config as BinaryData.FileSystemConfig).storagePath, - ); // @TODO: Remove assertion + if (this.availableModes.includes('filesystem') && config.mode === 'filesystem') { + this.clients.filesystem = new FileSystemClient(config.storagePath); + await this.clients.filesystem.init(mainClient); } From 93ea57ceb60a02fe3eebbc5f69d2a9335ffec4f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 14 Sep 2023 18:42:02 +0200 Subject: [PATCH 055/103] Fix test --- packages/core/test/NodeExecuteFunctions.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/test/NodeExecuteFunctions.test.ts b/packages/core/test/NodeExecuteFunctions.test.ts index 7ea8dbc289724..0f5d42ed6e4ce 100644 --- a/packages/core/test/NodeExecuteFunctions.test.ts +++ b/packages/core/test/NodeExecuteFunctions.test.ts @@ -30,7 +30,7 @@ describe('NodeExecuteFunctions', () => { await Container.get(BinaryDataService).init({ mode: 'default', - availableModes: 'default', + availableModes: ['default'], }); // Set our binary data buffer @@ -80,7 +80,7 @@ describe('NodeExecuteFunctions', () => { // Setup a 'filesystem' binary data manager instance await Container.get(BinaryDataService).init({ mode: 'filesystem', - availableModes: 'filesystem', + availableModes: ['filesystem'], storagePath: temporaryDir, }); From c8676befdb043fffa38b00a0480242f67dd40d96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 14 Sep 2023 18:46:12 +0200 Subject: [PATCH 056/103] Remove unneeded modifier --- packages/cli/src/commands/BaseCommand.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/commands/BaseCommand.ts b/packages/cli/src/commands/BaseCommand.ts index 8f29db79505e3..cf3a7bbb2028f 100644 --- a/packages/cli/src/commands/BaseCommand.ts +++ b/packages/cli/src/commands/BaseCommand.ts @@ -103,7 +103,7 @@ export abstract class BaseCommand extends Command { process.exit(1); } - protected async initBinaryDataService() { + async initBinaryDataService() { const binaryDataConfig = config.getEnv('binaryDataService'); await Container.get(BinaryDataService).init(binaryDataConfig, true); } From 25006e8899e046ee5c6af81cf5779a2131791d77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 14 Sep 2023 18:50:02 +0200 Subject: [PATCH 057/103] Cleanup --- packages/cli/test/integration/shared/utils/index.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/cli/test/integration/shared/utils/index.ts b/packages/cli/test/integration/shared/utils/index.ts index b2f51fca22771..c4412119b6416 100644 --- a/packages/cli/test/integration/shared/utils/index.ts +++ b/packages/cli/test/integration/shared/utils/index.ts @@ -75,13 +75,11 @@ export async function initNodeTypes() { * Initialize a BinaryDataService for test runs. */ export async function initBinaryDataService() { - const binaryDataConfig = config.getEnv('binaryDataService'); - const binaryDataService = new BinaryDataService(); - await binaryDataService.init(binaryDataConfig); - Container.set(BinaryDataService, binaryDataService); - await binaryDataService.init(binaryDataConfig); + await binaryDataService.init(config.getEnv('binaryDataService')); + + Container.set(BinaryDataService, binaryDataService); } /** From a7156f845e8ee92fd12ec5f800134accf5e8e816 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 14 Sep 2023 19:02:56 +0200 Subject: [PATCH 058/103] Fix node tests --- packages/nodes-base/test/nodes/Helpers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nodes-base/test/nodes/Helpers.ts b/packages/nodes-base/test/nodes/Helpers.ts index 7973221217569..6735223e3fdb3 100644 --- a/packages/nodes-base/test/nodes/Helpers.ts +++ b/packages/nodes-base/test/nodes/Helpers.ts @@ -219,7 +219,7 @@ export function createTemporaryDir(prefix = 'n8n') { export async function initBinaryDataService(mode: 'default' | 'filesystem' = 'default') { const binaryDataService = new BinaryDataService(); - await binaryDataService.init({ mode: 'default', availableModes: mode }); + await binaryDataService.init({ mode: 'default', availableModes: [mode] }); Container.set(BinaryDataService, binaryDataService); } From e0e90b6b36446755db8a8f6102ba82ac88e4d24f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 14 Sep 2023 19:03:45 +0200 Subject: [PATCH 059/103] Remove unneeded line --- packages/core/src/binaryData/binaryData.service.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/core/src/binaryData/binaryData.service.ts b/packages/core/src/binaryData/binaryData.service.ts index 875cea8ae9925..5fc543c3bc9cc 100644 --- a/packages/core/src/binaryData/binaryData.service.ts +++ b/packages/core/src/binaryData/binaryData.service.ts @@ -30,8 +30,6 @@ export class BinaryDataService { await this.clients.filesystem.init(mainClient); } - - return undefined; } async copyBinaryFile(binaryData: IBinaryData, path: string, executionId: string) { From 219addba31d1bb02df000af11fe6d1906e8950d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 14 Sep 2023 19:12:08 +0200 Subject: [PATCH 060/103] Rename constant --- packages/core/src/binaryData/fs.client.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/binaryData/fs.client.ts b/packages/core/src/binaryData/fs.client.ts index 7a9b052991fee..9914aa18fd2d0 100644 --- a/packages/core/src/binaryData/fs.client.ts +++ b/packages/core/src/binaryData/fs.client.ts @@ -10,7 +10,7 @@ import type { Readable } from 'stream'; import type { BinaryMetadata } from 'n8n-workflow'; import type { BinaryData } from './types'; -const EXECUTION_EXTRACTOR = +const EXECUTION_ID_EXTRACTOR = /^(\w+)(?:[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12})$/; export class FileSystemClient implements BinaryData.Client { @@ -80,7 +80,7 @@ export class FileSystemClient implements BinaryData.Client { const deletedIds = []; for (const fileName of fileNames) { - const executionId = fileName.match(EXECUTION_EXTRACTOR)?.[1]; + const executionId = fileName.match(EXECUTION_ID_EXTRACTOR)?.[1]; if (executionId && set.has(executionId)) { const filePath = this.resolvePath(fileName); From d7c7f5fb5145ee4232bdfc79767bfc74fafb86ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 14 Sep 2023 19:12:45 +0200 Subject: [PATCH 061/103] Remove comment --- packages/core/src/binaryData/fs.client.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/core/src/binaryData/fs.client.ts b/packages/core/src/binaryData/fs.client.ts index 9914aa18fd2d0..36d2f476be3ef 100644 --- a/packages/core/src/binaryData/fs.client.ts +++ b/packages/core/src/binaryData/fs.client.ts @@ -126,7 +126,6 @@ export class FileSystemClient implements BinaryData.Client { return [executionId, uuid()].join(''); } - // @TODO: Variadic needed? private resolvePath(...args: string[]) { const returnPath = path.join(this.storagePath, ...args); From 1ea380debf77cb74be2d4807b65091f8e13d2435 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 14 Sep 2023 19:21:31 +0200 Subject: [PATCH 062/103] Fix tests --- packages/cli/test/integration/commands/worker.cmd.test.ts | 8 ++++---- .../nodes/HttpRequest/test/binaryData/HttpRequest.test.ts | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/cli/test/integration/commands/worker.cmd.test.ts b/packages/cli/test/integration/commands/worker.cmd.test.ts index d860579ee392b..2d9191c124db7 100644 --- a/packages/cli/test/integration/commands/worker.cmd.test.ts +++ b/packages/cli/test/integration/commands/worker.cmd.test.ts @@ -5,7 +5,7 @@ import { LoggerProxy } from 'n8n-workflow'; import { Telemetry } from '@/telemetry'; import { getLogger } from '@/Logger'; import { ExternalSecretsManager } from '@/ExternalSecrets/ExternalSecretsManager.ee'; -import { BinaryDataManager } from 'n8n-core'; +import { BinaryDataService } from 'n8n-core'; import { CacheService } from '@/services/cache.service'; import { RedisServicePubSubPublisher } from '@/services/redis/RedisServicePubSubPublisher'; import { RedisServicePubSubSubscriber } from '@/services/redis/RedisServicePubSubSubscriber'; @@ -26,7 +26,7 @@ beforeAll(async () => { mockInstance(InternalHooks); mockInstance(CacheService); mockInstance(ExternalSecretsManager); - mockInstance(BinaryDataManager); + mockInstance(BinaryDataService); mockInstance(MessageEventBus); mockInstance(LoadNodesAndCredentials); mockInstance(CredentialTypes); @@ -41,7 +41,7 @@ test('worker initializes all its components', async () => { jest.spyOn(worker, 'init'); jest.spyOn(worker, 'initLicense').mockImplementation(async () => {}); - jest.spyOn(worker, 'initBinaryManager').mockImplementation(async () => {}); + jest.spyOn(worker, 'initBinaryDataService').mockImplementation(async () => {}); jest.spyOn(worker, 'initExternalHooks').mockImplementation(async () => {}); jest.spyOn(worker, 'initExternalSecrets').mockImplementation(async () => {}); jest.spyOn(worker, 'initEventBus').mockImplementation(async () => {}); @@ -64,7 +64,7 @@ test('worker initializes all its components', async () => { expect(worker.uniqueInstanceId).toContain('worker'); expect(worker.uniqueInstanceId.length).toBeGreaterThan(15); expect(worker.initLicense).toHaveBeenCalled(); - expect(worker.initBinaryManager).toHaveBeenCalled(); + expect(worker.initBinaryDataService).toHaveBeenCalled(); expect(worker.initExternalHooks).toHaveBeenCalled(); expect(worker.initExternalSecrets).toHaveBeenCalled(); expect(worker.initEventBus).toHaveBeenCalled(); diff --git a/packages/nodes-base/nodes/HttpRequest/test/binaryData/HttpRequest.test.ts b/packages/nodes-base/nodes/HttpRequest/test/binaryData/HttpRequest.test.ts index c539f4bc44273..4ef4a1720189a 100644 --- a/packages/nodes-base/nodes/HttpRequest/test/binaryData/HttpRequest.test.ts +++ b/packages/nodes-base/nodes/HttpRequest/test/binaryData/HttpRequest.test.ts @@ -4,7 +4,7 @@ import { equalityTest, workflowToTests, getWorkflowFilenames, - initBinaryDataManager, + initBinaryDataService, } from '@test/nodes/Helpers'; describe('Test Binary Data Download', () => { @@ -14,7 +14,7 @@ describe('Test Binary Data Download', () => { const baseUrl = 'https://dummy.domain'; beforeAll(async () => { - await initBinaryDataManager(); + await initBinaryDataService(); nock.disableNetConnect(); From a5b388b80a899bbe881a8886dc5b77e33bba3196 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 14 Sep 2023 19:28:04 +0200 Subject: [PATCH 063/103] Cleanup --- packages/cli/src/config/utils.ts | 3 +-- packages/core/src/binaryData/utils.ts | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/config/utils.ts b/packages/cli/src/config/utils.ts index 3432e3a78242c..4fee1a41108fd 100644 --- a/packages/cli/src/config/utils.ts +++ b/packages/cli/src/config/utils.ts @@ -2,8 +2,7 @@ import type { SchemaObj } from 'convict'; class NotStringArrayError extends Error { constructor(env: string) { - super(); - this.message = `${env} is not a string array.`; + super(`${env} is not a string array.`); } } diff --git a/packages/core/src/binaryData/utils.ts b/packages/core/src/binaryData/utils.ts index a3dd1f5ab640a..ff48d9b8c0e81 100644 --- a/packages/core/src/binaryData/utils.ts +++ b/packages/core/src/binaryData/utils.ts @@ -1,6 +1,6 @@ import type { BinaryData } from './types'; -export class InvalidModeError extends Error {} +export class InvalidBinaryModeError extends Error {} /** * Modes for storing binary data: From 92047547c936a8b283a140551ef4d3d160501612 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 14 Sep 2023 19:32:12 +0200 Subject: [PATCH 064/103] Rename error --- packages/core/src/binaryData/binaryData.service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/binaryData/binaryData.service.ts b/packages/core/src/binaryData/binaryData.service.ts index 5fc543c3bc9cc..1800b86915548 100644 --- a/packages/core/src/binaryData/binaryData.service.ts +++ b/packages/core/src/binaryData/binaryData.service.ts @@ -5,7 +5,7 @@ import { Service } from 'typedi'; import { BINARY_ENCODING } from 'n8n-workflow'; import { FileSystemClient } from './fs.client'; -import { InvalidModeError, areValidModes } from './utils'; +import { InvalidBinaryModeError, areValidModes } from './utils'; import type { Readable } from 'stream'; import type { BinaryData } from './types'; @@ -20,7 +20,7 @@ export class BinaryDataService { private clients: Record = {}; async init(config: BinaryData.Config, mainClient = false) { - if (!areValidModes(config.availableModes)) throw new InvalidModeError(); + if (!areValidModes(config.availableModes)) throw new InvalidBinaryModeError(); this.availableModes = config.availableModes; this.mode = config.mode; From 9597508e2f574466fd4aef5229197f0480045362 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 14 Sep 2023 19:35:14 +0200 Subject: [PATCH 065/103] Improve error message --- packages/core/src/binaryData/binaryData.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/binaryData/binaryData.service.ts b/packages/core/src/binaryData/binaryData.service.ts index 1800b86915548..78640461513cc 100644 --- a/packages/core/src/binaryData/binaryData.service.ts +++ b/packages/core/src/binaryData/binaryData.service.ts @@ -210,7 +210,7 @@ export class BinaryDataService { private getClient(mode: string) { const client = this.clients[mode]; - if (!client) throw new Error('This method is not supported by in-memory storage mode'); + if (!client) throw new Error('No binary data client found'); return client; } From 8fde8be2b3afc0757f8ef896a0727644a9044250 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 14 Sep 2023 19:38:14 +0200 Subject: [PATCH 066/103] Better error --- packages/core/src/binaryData/utils.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/core/src/binaryData/utils.ts b/packages/core/src/binaryData/utils.ts index ff48d9b8c0e81..2392aed312c0e 100644 --- a/packages/core/src/binaryData/utils.ts +++ b/packages/core/src/binaryData/utils.ts @@ -1,7 +1,5 @@ import type { BinaryData } from './types'; -export class InvalidBinaryModeError extends Error {} - /** * Modes for storing binary data: * - `default` (in memory) @@ -13,3 +11,12 @@ export const BINARY_DATA_MODES = ['default', 'filesystem', 'object'] as const; export function areValidModes(modes: string[]): modes is BinaryData.Mode[] { return modes.every((m) => BINARY_DATA_MODES.includes(m as BinaryData.Mode)); } + +export class InvalidBinaryModeError extends Error { + constructor() { + const validModes = BINARY_DATA_MODES.join(', '); + super( + `Invalid binary data mode. Set N8N_AVAILABLE_BINARY_DATA_MODES using only valid modes: ${validModes}`, + ); + } +} From ada012e1abbb7138356e7cd4dcba14ad20849e74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 14 Sep 2023 19:39:25 +0200 Subject: [PATCH 067/103] Better message --- packages/core/src/binaryData/utils.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/core/src/binaryData/utils.ts b/packages/core/src/binaryData/utils.ts index 2392aed312c0e..3e59792469add 100644 --- a/packages/core/src/binaryData/utils.ts +++ b/packages/core/src/binaryData/utils.ts @@ -14,9 +14,6 @@ export function areValidModes(modes: string[]): modes is BinaryData.Mode[] { export class InvalidBinaryModeError extends Error { constructor() { - const validModes = BINARY_DATA_MODES.join(', '); - super( - `Invalid binary data mode. Set N8N_AVAILABLE_BINARY_DATA_MODES using only valid modes: ${validModes}`, - ); + super(`Invalid binary data mode. Valid modes: ${BINARY_DATA_MODES.join(', ')}`); } } From 67e163fb240f505de42248bcb8924fa69fd9d09e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Fri, 15 Sep 2023 10:46:33 +0200 Subject: [PATCH 068/103] Make execution ID non-nullable --- packages/cli/src/Interfaces.ts | 2 +- .../PublicApi/v1/handlers/executions/executions.handler.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/Interfaces.ts b/packages/cli/src/Interfaces.ts index d7718efa90b81..40d6a354c3dda 100644 --- a/packages/cli/src/Interfaces.ts +++ b/packages/cli/src/Interfaces.ts @@ -171,7 +171,7 @@ export type ICredentialsDecryptedResponse = ICredentialsDecryptedDb; export type SaveExecutionDataType = 'all' | 'none'; export interface IExecutionBase { - id?: string; + id: string; mode: WorkflowExecuteMode; startedAt: Date; stoppedAt?: Date; // empty value means execution is still running diff --git a/packages/cli/src/PublicApi/v1/handlers/executions/executions.handler.ts b/packages/cli/src/PublicApi/v1/handlers/executions/executions.handler.ts index 90ed886bfa62a..a6b42c96ee38d 100644 --- a/packages/cli/src/PublicApi/v1/handlers/executions/executions.handler.ts +++ b/packages/cli/src/PublicApi/v1/handlers/executions/executions.handler.ts @@ -27,7 +27,7 @@ export = { // look for the execution on the workflow the user owns const execution = await getExecutionInWorkflows(id, sharedWorkflowsIds, false); - if (!execution?.id) { + if (!execution) { return res.status(404).json({ message: 'Not Found' }); } @@ -103,7 +103,7 @@ export = { const executions = await getExecutions(filters); - const newLastId = !executions.length ? '0' : (executions.slice(-1)[0].id as string); + const newLastId = !executions.length ? '0' : executions.slice(-1)[0].id; filters.lastId = newLastId; From 12636dd7406ed1ecc11202e97d10828153708c10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Fri, 15 Sep 2023 10:52:16 +0200 Subject: [PATCH 069/103] Readability improvements --- .../repositories/execution.repository.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 6998d9292cff7..a892e370cb096 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -86,19 +86,25 @@ export class ExecutionRepository extends Repository { ) { super(ExecutionEntity, dataSource.manager); - if (config.getEnv('executions.pruneData')) { - setInterval(async () => this.pruneBySoftDeleting(), 1 * TIME.HOUR); - } + if (config.getEnv('executions.pruneData')) this.setPruningInterval(); this.setHardDeletionInterval(); } + setPruningInterval() { + setInterval(async () => this.pruneBySoftDeleting(), 1 * TIME.HOUR); + } + setHardDeletionInterval() { + if (this.hardDeletionInterval) return; + this.hardDeletionInterval = setInterval(async () => this.hardDelete(), 15 * TIME.MINUTE); } clearHardDeletionInterval() { - if (this.hardDeletionInterval) clearInterval(this.hardDeletionInterval); + if (!this.hardDeletionInterval) return; + + clearInterval(this.hardDeletionInterval); } async findMultipleExecutions( @@ -518,8 +524,6 @@ export class ExecutionRepository extends Repository { setTimeout(async () => this.hardDelete(), 1 * TIME.SECOND); } else { - if (this.hardDeletionInterval) return; - this.setHardDeletionInterval(); } } From e5c8c7296ffce6df822cdb665e3a13475b0ceedf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Fri, 15 Sep 2023 11:01:28 +0200 Subject: [PATCH 070/103] Adjust types, followup to 67e163f --- packages/cli/src/ActiveExecutions.ts | 3 ++- packages/cli/src/GenericHelpers.ts | 4 ++-- packages/cli/src/Interfaces.ts | 5 +++++ packages/cli/src/WorkflowExecuteAdditionalData.ts | 3 ++- packages/cli/src/WorkflowHelpers.ts | 8 ++++++-- .../src/databases/repositories/execution.repository.ts | 3 ++- .../executionLifecycleHooks/shared/sharedHookFunctions.ts | 6 +++--- 7 files changed, 22 insertions(+), 10 deletions(-) diff --git a/packages/cli/src/ActiveExecutions.ts b/packages/cli/src/ActiveExecutions.ts index 26feeee589b28..f794335204773 100644 --- a/packages/cli/src/ActiveExecutions.ts +++ b/packages/cli/src/ActiveExecutions.ts @@ -12,6 +12,7 @@ import { createDeferredPromise, LoggerProxy } from 'n8n-workflow'; import type { ChildProcess } from 'child_process'; import type PCancelable from 'p-cancelable'; import type { + ExecutionPayload, IExecutingWorkflowData, IExecutionDb, IExecutionsCurrentSummary, @@ -38,7 +39,7 @@ export class ActiveExecutions { if (executionId === undefined) { // Is a new execution so save in DB - const fullExecutionData: IExecutionDb = { + const fullExecutionData: ExecutionPayload = { data: executionData.executionData!, mode: executionData.executionMode, finished: false, diff --git a/packages/cli/src/GenericHelpers.ts b/packages/cli/src/GenericHelpers.ts index 0034ebea90074..ef9e82a9fdbd6 100644 --- a/packages/cli/src/GenericHelpers.ts +++ b/packages/cli/src/GenericHelpers.ts @@ -11,7 +11,7 @@ import { Container } from 'typedi'; import { Like } from 'typeorm'; import config from '@/config'; import * as Db from '@/Db'; -import type { ICredentialsDb, IExecutionDb, IWorkflowDb } from '@/Interfaces'; +import type { ExecutionPayload, ICredentialsDb, IWorkflowDb } from '@/Interfaces'; import * as ResponseHelper from '@/ResponseHelper'; import type { WorkflowEntity } from '@db/entities/WorkflowEntity'; import type { CredentialsEntity } from '@db/entities/CredentialsEntity'; @@ -178,7 +178,7 @@ export async function createErrorExecution( }, }; - const fullExecutionData: IExecutionDb = { + const fullExecutionData: ExecutionPayload = { data: executionData, mode, finished: false, diff --git a/packages/cli/src/Interfaces.ts b/packages/cli/src/Interfaces.ts index 40d6a354c3dda..3f373e9d9f750 100644 --- a/packages/cli/src/Interfaces.ts +++ b/packages/cli/src/Interfaces.ts @@ -189,6 +189,11 @@ export interface IExecutionDb extends IExecutionBase { workflowData?: IWorkflowBase; } +/** + * Payload for creating or updating an execution. + */ +export type ExecutionPayload = Omit; + export interface IExecutionPushResponse { executionId?: string; waitingForWebhook?: boolean; diff --git a/packages/cli/src/WorkflowExecuteAdditionalData.ts b/packages/cli/src/WorkflowExecuteAdditionalData.ts index 6ed7d4bb0d38f..9c2d309733dfd 100644 --- a/packages/cli/src/WorkflowExecuteAdditionalData.ts +++ b/packages/cli/src/WorkflowExecuteAdditionalData.ts @@ -47,6 +47,7 @@ import type { IWorkflowExecuteProcess, IWorkflowExecutionDataProcess, IWorkflowErrorData, + ExecutionPayload, } from '@/Interfaces'; import { NodeTypes } from '@/NodeTypes'; import { Push } from '@/push'; @@ -885,7 +886,7 @@ async function executeWorkflow( // Therefore, database might not contain finished errors. // Force an update to db as there should be no harm doing this - const fullExecutionData: IExecutionDb = { + const fullExecutionData: ExecutionPayload = { data: fullRunData.data, mode: fullRunData.mode, finished: fullRunData.finished ? fullRunData.finished : false, diff --git a/packages/cli/src/WorkflowHelpers.ts b/packages/cli/src/WorkflowHelpers.ts index e1a6e7057f09c..093f713e4ef3b 100644 --- a/packages/cli/src/WorkflowHelpers.ts +++ b/packages/cli/src/WorkflowHelpers.ts @@ -23,7 +23,11 @@ import { } from 'n8n-workflow'; import { v4 as uuid } from 'uuid'; import * as Db from '@/Db'; -import type { IExecutionDb, IWorkflowErrorData, IWorkflowExecutionDataProcess } from '@/Interfaces'; +import type { + ExecutionPayload, + IWorkflowErrorData, + IWorkflowExecutionDataProcess, +} from '@/Interfaces'; import { NodeTypes } from '@/NodeTypes'; // eslint-disable-next-line import/no-cycle import { WorkflowRunner } from '@/WorkflowRunner'; @@ -186,7 +190,7 @@ export async function executeErrorWorkflow( initialNode, ); - const fullExecutionData: IExecutionDb = { + const fullExecutionData: ExecutionPayload = { data: fakeExecution.data, mode: fakeExecution.mode, finished: false, diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index a892e370cb096..6706ceaefba0d 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -20,6 +20,7 @@ import { LoggerProxy as Logger } from 'n8n-workflow'; import type { IExecutionsSummary, IRunExecutionData } from 'n8n-workflow'; import { BinaryDataManager } from 'n8n-core'; import type { + ExecutionPayload, IExecutionBase, IExecutionDb, IExecutionFlattedDb, @@ -242,7 +243,7 @@ export class ExecutionRepository extends Repository { return rest; } - async createNewExecution(execution: IExecutionDb) { + async createNewExecution(execution: ExecutionPayload) { const { data, workflowData, ...rest } = execution; const newExecution = await this.save(rest); diff --git a/packages/cli/src/executionLifecycleHooks/shared/sharedHookFunctions.ts b/packages/cli/src/executionLifecycleHooks/shared/sharedHookFunctions.ts index cf68a1930bed2..113bb2980a9af 100644 --- a/packages/cli/src/executionLifecycleHooks/shared/sharedHookFunctions.ts +++ b/packages/cli/src/executionLifecycleHooks/shared/sharedHookFunctions.ts @@ -1,5 +1,5 @@ import type { ExecutionStatus, IRun, IWorkflowBase } from 'n8n-workflow'; -import type { IExecutionDb } from '@/Interfaces'; +import type { ExecutionPayload, IExecutionDb } from '@/Interfaces'; import pick from 'lodash/pick'; import { isWorkflowIdValid } from '@/utils'; import { LoggerProxy } from 'n8n-workflow'; @@ -24,7 +24,7 @@ export function prepareExecutionDataForDbUpdate(parameters: { workflowData: IWorkflowBase; workflowStatusFinal: ExecutionStatus; retryOf?: string; -}): IExecutionDb { +}) { const { runData, workflowData, workflowStatusFinal, retryOf } = parameters; // Although it is treated as IWorkflowBase here, it's being instantiated elsewhere with properties that may be sensitive // As a result, we should create an IWorkflowBase object with only the data we want to save in it. @@ -41,7 +41,7 @@ export function prepareExecutionDataForDbUpdate(parameters: { 'pinData', ]); - const fullExecutionData: IExecutionDb = { + const fullExecutionData: ExecutionPayload = { data: runData.data, mode: runData.mode, finished: runData.finished ? runData.finished : false, From b7062e59120307f1ed051201715e80f7f754a182 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Fri, 15 Sep 2023 11:25:58 +0200 Subject: [PATCH 071/103] Fix lint --- packages/cli/src/WorkflowExecuteAdditionalData.ts | 1 - packages/cli/src/databases/repositories/execution.repository.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/packages/cli/src/WorkflowExecuteAdditionalData.ts b/packages/cli/src/WorkflowExecuteAdditionalData.ts index 9c2d309733dfd..08bfd71031c8e 100644 --- a/packages/cli/src/WorkflowExecuteAdditionalData.ts +++ b/packages/cli/src/WorkflowExecuteAdditionalData.ts @@ -42,7 +42,6 @@ import { ActiveExecutions } from '@/ActiveExecutions'; import { CredentialsHelper } from '@/CredentialsHelper'; import { ExternalHooks } from '@/ExternalHooks'; import type { - IExecutionDb, IPushDataExecutionFinished, IWorkflowExecuteProcess, IWorkflowExecutionDataProcess, diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 6706ceaefba0d..e1d4c3840e2bb 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -22,7 +22,6 @@ import { BinaryDataManager } from 'n8n-core'; import type { ExecutionPayload, IExecutionBase, - IExecutionDb, IExecutionFlattedDb, IExecutionResponse, } from '@/Interfaces'; From 79ecaa96c8ae1aadf1be3aa05f14b15552f8d2d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Fri, 15 Sep 2023 11:23:18 +0200 Subject: [PATCH 072/103] Fix lint --- packages/cli/src/WorkflowExecuteAdditionalData.ts | 1 - packages/cli/src/databases/repositories/execution.repository.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/packages/cli/src/WorkflowExecuteAdditionalData.ts b/packages/cli/src/WorkflowExecuteAdditionalData.ts index 9c2d309733dfd..08bfd71031c8e 100644 --- a/packages/cli/src/WorkflowExecuteAdditionalData.ts +++ b/packages/cli/src/WorkflowExecuteAdditionalData.ts @@ -42,7 +42,6 @@ import { ActiveExecutions } from '@/ActiveExecutions'; import { CredentialsHelper } from '@/CredentialsHelper'; import { ExternalHooks } from '@/ExternalHooks'; import type { - IExecutionDb, IPushDataExecutionFinished, IWorkflowExecuteProcess, IWorkflowExecutionDataProcess, diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index a45b71e594af0..4cc026b98620d 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -22,7 +22,6 @@ import { BinaryDataService } from 'n8n-core'; import type { ExecutionPayload, IExecutionBase, - IExecutionDb, IExecutionFlattedDb, IExecutionResponse, } from '@/Interfaces'; From 02b6d864d93cd0cb80b840f550bf08d163b56c40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Mon, 18 Sep 2023 09:06:00 +0200 Subject: [PATCH 073/103] Cleanup --- packages/cli/src/config/schema.ts | 4 +- .../BinaryData.service.ts} | 64 +++++++++---------- .../FileSystem.manager.ts} | 6 +- .../ObjectStore.manager.ts} | 6 +- packages/core/src/NodeExecuteFunctions.ts | 2 +- packages/core/src/binaryData/types.ts | 20 ++++-- packages/core/src/binaryData/utils.ts | 8 ++- packages/core/src/index.ts | 4 +- .../core/test/NodeExecuteFunctions.test.ts | 2 +- 9 files changed, 67 insertions(+), 49 deletions(-) rename packages/core/src/{binaryData/binaryData.service.ts => BinaryData/BinaryData.service.ts} (73%) rename packages/core/src/{binaryData/fs.client.ts => BinaryData/FileSystem.manager.ts} (95%) rename packages/core/src/{binaryData/s3.client.ts => BinaryData/ObjectStore.manager.ts} (86%) diff --git a/packages/cli/src/config/schema.ts b/packages/cli/src/config/schema.ts index ee548386a73ea..f3c83d0e93b85 100644 --- a/packages/cli/src/config/schema.ts +++ b/packages/cli/src/config/schema.ts @@ -890,12 +890,12 @@ export const schema = { binaryDataService: { availableModes: { format: 'comma-separated-list', - default: 'filesystem,object', + default: 'filesystem', env: 'N8N_AVAILABLE_BINARY_DATA_MODES', doc: 'Available modes of binary data storage, as comma separated strings', }, mode: { - format: ['default', 'filesystem', 'object'] as const, + format: ['default', 'filesystem'] as const, default: 'default', env: 'N8N_DEFAULT_BINARY_DATA_MODE', doc: 'Storage mode for binary data', diff --git a/packages/core/src/binaryData/binaryData.service.ts b/packages/core/src/BinaryData/BinaryData.service.ts similarity index 73% rename from packages/core/src/binaryData/binaryData.service.ts rename to packages/core/src/BinaryData/BinaryData.service.ts index 78640461513cc..d08e670254abb 100644 --- a/packages/core/src/binaryData/binaryData.service.ts +++ b/packages/core/src/BinaryData/BinaryData.service.ts @@ -4,8 +4,8 @@ import prettyBytes from 'pretty-bytes'; import { Service } from 'typedi'; import { BINARY_ENCODING } from 'n8n-workflow'; -import { FileSystemClient } from './fs.client'; -import { InvalidBinaryModeError, areValidModes } from './utils'; +import { FileSystemManager } from './FileSystem.manager'; +import { InvalidBinaryDataManagerError, InvalidBinaryDataModeError, areValidModes } from './utils'; import type { Readable } from 'stream'; import type { BinaryData } from './types'; @@ -17,25 +17,25 @@ export class BinaryDataService { private mode: BinaryData.Mode = 'default'; - private clients: Record = {}; + private managers: Record = {}; - async init(config: BinaryData.Config, mainClient = false) { - if (!areValidModes(config.availableModes)) throw new InvalidBinaryModeError(); + async init(config: BinaryData.Config, mainManager = false) { + if (!areValidModes(config.availableModes)) throw new InvalidBinaryDataModeError(); this.availableModes = config.availableModes; this.mode = config.mode; if (this.availableModes.includes('filesystem') && config.mode === 'filesystem') { - this.clients.filesystem = new FileSystemClient(config.storagePath); + this.managers.filesystem = new FileSystemManager(config.storagePath); - await this.clients.filesystem.init(mainClient); + await this.managers.filesystem.init(mainManager); } } async copyBinaryFile(binaryData: IBinaryData, path: string, executionId: string) { - const client = this.clients[this.mode]; + const manager = this.managers[this.mode]; - if (!client) { + if (!manager) { const { size } = await stat(path); binaryData.fileSize = prettyBytes(size); binaryData.data = await readFile(path, { encoding: BINARY_ENCODING }); @@ -43,14 +43,14 @@ export class BinaryDataService { return binaryData; } - const identifier = await client.copyByPath(path, executionId); + const identifier = await manager.copyByPath(path, executionId); binaryData.id = this.createIdentifier(identifier); - binaryData.data = this.mode; // clear from memory + binaryData.data = this.mode; // clear binary data from memory - const fileSize = await client.getSize(identifier); + const fileSize = await manager.getSize(identifier); binaryData.fileSize = prettyBytes(fileSize); - await client.storeMetadata(identifier, { + await manager.storeMetadata(identifier, { fileName: binaryData.fileName, mimeType: binaryData.mimeType, fileSize, @@ -60,9 +60,9 @@ export class BinaryDataService { } async store(binaryData: IBinaryData, input: Buffer | Readable, executionId: string) { - const client = this.clients[this.mode]; + const manager = this.managers[this.mode]; - if (!client) { + if (!manager) { const buffer = await this.binaryToBuffer(input); binaryData.data = buffer.toString(BINARY_ENCODING); binaryData.fileSize = prettyBytes(buffer.length); @@ -70,14 +70,14 @@ export class BinaryDataService { return binaryData; } - const identifier = await client.store(input, executionId); + const identifier = await manager.store(input, executionId); binaryData.id = this.createIdentifier(identifier); - binaryData.data = this.mode; // clear from memory + binaryData.data = this.mode; // clear binary data from memory - const fileSize = await client.getSize(identifier); + const fileSize = await manager.getSize(identifier); binaryData.fileSize = prettyBytes(fileSize); - await client.storeMetadata(identifier, { + await manager.storeMetadata(identifier, { fileName: binaryData.fileName, mimeType: binaryData.mimeType, fileSize, @@ -96,7 +96,7 @@ export class BinaryDataService { getAsStream(identifier: string, chunkSize?: number) { const { mode, id } = this.splitBinaryModeFileId(identifier); - return this.getClient(mode).getAsStream(id, chunkSize); + return this.getManager(mode).getStream(id, chunkSize); } async getBinaryDataBuffer(binaryData: IBinaryData) { @@ -108,27 +108,27 @@ export class BinaryDataService { async retrieveBinaryDataByIdentifier(identifier: string) { const { mode, id } = this.splitBinaryModeFileId(identifier); - return this.getClient(mode).getAsBuffer(id); + return this.getManager(mode).getBuffer(id); } getPath(identifier: string) { const { mode, id } = this.splitBinaryModeFileId(identifier); - return this.getClient(mode).getPath(id); + return this.getManager(mode).getPath(id); } async getMetadata(identifier: string) { const { mode, id } = this.splitBinaryModeFileId(identifier); - return this.getClient(mode).getMetadata(id); + return this.getManager(mode).getMetadata(id); } async deleteManyByExecutionIds(executionIds: string[]) { - await this.getClient(this.mode).deleteManyByExecutionIds(executionIds); + await this.getManager(this.mode).deleteManyByExecutionIds(executionIds); } async duplicateBinaryData(inputData: Array, executionId: string) { - if (inputData && this.clients[this.mode]) { + if (inputData && this.managers[this.mode]) { const returnInputData = (inputData as INodeExecutionData[][]).map( async (executionDataArray) => { if (executionDataArray) { @@ -161,7 +161,7 @@ export class BinaryDataService { return `${this.mode}:${filename}`; } - private splitBinaryModeFileId(fileId: string): { mode: string; id: string } { + private splitBinaryModeFileId(fileId: string) { const [mode, id] = fileId.split(':'); return { mode, id }; @@ -171,7 +171,7 @@ export class BinaryDataService { executionData: INodeExecutionData, executionId: string, ) { - const client = this.clients[this.mode]; + const manager = this.managers[this.mode]; if (executionData.binary) { const binaryDataKeys = Object.keys(executionData.binary); @@ -185,7 +185,7 @@ export class BinaryDataService { return { key, newId: undefined }; } - return client + return manager ?.copyByIdentifier(this.splitBinaryModeFileId(binaryDataId).id, executionId) .then((filename) => ({ newId: this.createIdentifier(filename), @@ -207,11 +207,11 @@ export class BinaryDataService { return executionData; } - private getClient(mode: string) { - const client = this.clients[mode]; + private getManager(mode: string) { + const manager = this.managers[mode]; - if (!client) throw new Error('No binary data client found'); + if (manager) return manager; - return client; + throw new InvalidBinaryDataManagerError(); } } diff --git a/packages/core/src/binaryData/fs.client.ts b/packages/core/src/BinaryData/FileSystem.manager.ts similarity index 95% rename from packages/core/src/binaryData/fs.client.ts rename to packages/core/src/BinaryData/FileSystem.manager.ts index 36d2f476be3ef..1e00774382fd0 100644 --- a/packages/core/src/binaryData/fs.client.ts +++ b/packages/core/src/BinaryData/FileSystem.manager.ts @@ -13,7 +13,7 @@ import type { BinaryData } from './types'; const EXECUTION_ID_EXTRACTOR = /^(\w+)(?:[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12})$/; -export class FileSystemClient implements BinaryData.Client { +export class FileSystemManager implements BinaryData.Manager { constructor(private storagePath: string) {} async init() { @@ -31,13 +31,13 @@ export class FileSystemClient implements BinaryData.Client { return stats.size; } - getAsStream(identifier: string, chunkSize?: number) { + getStream(identifier: string, chunkSize?: number) { const filePath = this.getPath(identifier); return createReadStream(filePath, { highWaterMark: chunkSize }); } - async getAsBuffer(identifier: string) { + async getBuffer(identifier: string) { const filePath = this.getPath(identifier); try { diff --git a/packages/core/src/binaryData/s3.client.ts b/packages/core/src/BinaryData/ObjectStore.manager.ts similarity index 86% rename from packages/core/src/binaryData/s3.client.ts rename to packages/core/src/BinaryData/ObjectStore.manager.ts index de69464d2afd4..685f7a967a06b 100644 --- a/packages/core/src/binaryData/s3.client.ts +++ b/packages/core/src/BinaryData/ObjectStore.manager.ts @@ -4,7 +4,7 @@ import type { BinaryMetadata } from 'n8n-workflow'; import type { Readable } from 'stream'; import type { BinaryData } from './types'; -export class S3Client implements BinaryData.Client { +export class ObjectStoreManager implements BinaryData.Manager { async init() { throw new Error('TODO'); } @@ -21,11 +21,11 @@ export class S3Client implements BinaryData.Client { throw new Error('TODO'); } - async getAsBuffer(identifier: string): Promise { + async getBuffer(identifier: string): Promise { throw new Error('TODO'); } - getAsStream(identifier: string, chunkSize?: number): Readable { + getStream(identifier: string, chunkSize?: number): Readable { throw new Error('TODO'); } diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index ae8ba47d09568..fa7dede9ec37b 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -115,7 +115,7 @@ import { Readable } from 'stream'; import { access as fsAccess, writeFile as fsWriteFile } from 'fs/promises'; import { createReadStream } from 'fs'; -import { BinaryDataService } from './binaryData/binaryData.service'; +import { BinaryDataService } from './BinaryData/BinaryData.service'; import type { ExtendedValidationResult, IResponseError, IWorkflowSettings } from './Interfaces'; import { extractValue } from './ExtractValue'; import { getClientCredentialsToken } from './OAuth2Helper'; diff --git a/packages/core/src/binaryData/types.ts b/packages/core/src/binaryData/types.ts index ce4185ff66456..1f0c668ce9a4d 100644 --- a/packages/core/src/binaryData/types.ts +++ b/packages/core/src/binaryData/types.ts @@ -16,22 +16,34 @@ export namespace BinaryData { export type Config = InMemoryConfig | FileSystemConfig; - export interface Client { + export interface Manager { init(startPurger: boolean): Promise; store(binaryData: Buffer | Readable, executionId: string): Promise; getPath(identifier: string): string; - getSize(path: string): Promise; // @TODO: Refactor to use identifier? - getAsBuffer(identifier: string): Promise; - getAsStream(identifier: string, chunkSize?: number): Readable; + // @TODO: Refactor to use identifier + getSize(path: string): Promise; + + getBuffer(identifier: string): Promise; + getStream(identifier: string, chunkSize?: number): Readable; + + // @TODO: Refactor out - not needed for object storage storeMetadata(identifier: string, metadata: BinaryMetadata): Promise; + + // @TODO: Refactor out - not needed for object storage getMetadata(identifier: string): Promise; + // @TODO: Refactor to also use `workflowId` to support full path-like identifier: + // `workflows/{workflowId}/executions/{executionId}/binary_data/{fileId}` copyByPath(path: string, executionId: string): Promise; + copyByIdentifier(identifier: string, prefix: string): Promise; deleteOne(identifier: string): Promise; + + // @TODO: Refactor to also receive `workflowId` to support full path-like identifier: + // `workflows/{workflowId}/executions/{executionId}/binary_data/{fileId}` deleteManyByExecutionIds(executionIds: string[]): Promise; } } diff --git a/packages/core/src/binaryData/utils.ts b/packages/core/src/binaryData/utils.ts index 3e59792469add..05dc84dc6308c 100644 --- a/packages/core/src/binaryData/utils.ts +++ b/packages/core/src/binaryData/utils.ts @@ -12,8 +12,14 @@ export function areValidModes(modes: string[]): modes is BinaryData.Mode[] { return modes.every((m) => BINARY_DATA_MODES.includes(m as BinaryData.Mode)); } -export class InvalidBinaryModeError extends Error { +export class InvalidBinaryDataModeError extends Error { constructor() { super(`Invalid binary data mode. Valid modes: ${BINARY_DATA_MODES.join(', ')}`); } } + +export class InvalidBinaryDataManagerError extends Error { + constructor() { + super('No binary data manager found'); + } +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c595e56458de0..9d6b5bfeebda3 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -2,8 +2,8 @@ import * as NodeExecuteFunctions from './NodeExecuteFunctions'; import * as UserSettings from './UserSettings'; export * from './ActiveWorkflows'; -export * from './binaryData/binaryData.service'; -export * from './binaryData/types'; +export * from './BinaryData/BinaryData.service'; +export * from './BinaryData/types'; export * from './ClassLoader'; export * from './Constants'; export * from './Credentials'; diff --git a/packages/core/test/NodeExecuteFunctions.test.ts b/packages/core/test/NodeExecuteFunctions.test.ts index 0f5d42ed6e4ce..96b45e907f478 100644 --- a/packages/core/test/NodeExecuteFunctions.test.ts +++ b/packages/core/test/NodeExecuteFunctions.test.ts @@ -11,7 +11,7 @@ import type { Workflow, WorkflowHooks, } from 'n8n-workflow'; -import { BinaryDataService } from '@/binaryData/binaryData.service'; +import { BinaryDataService } from '@/BinaryData/BinaryData.service'; import { setBinaryDataBuffer, getBinaryDataBuffer, From 96ea9eab6de78a9b8e494beacdcf413161373e17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Mon, 18 Sep 2023 11:07:56 +0200 Subject: [PATCH 074/103] Comment out cache keys --- .github/workflows/ci-pull-requests.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci-pull-requests.yml b/.github/workflows/ci-pull-requests.yml index e87b1a6d925f6..0fb9809582f7e 100644 --- a/.github/workflows/ci-pull-requests.yml +++ b/.github/workflows/ci-pull-requests.yml @@ -18,7 +18,7 @@ jobs: uses: actions/setup-node@v3.7.0 with: node-version: 18.x - cache: pnpm + # cache: pnpm - name: Install dependencies run: pnpm install --frozen-lockfile @@ -26,19 +26,19 @@ jobs: - name: Build run: pnpm build - - name: Cache build artifacts - uses: actions/cache/save@v3.3.1 - with: - path: ./packages/**/dist - key: ${{ github.sha }}-base:18-test-lint + # - name: Cache build artifacts + # uses: actions/cache/save@v3.3.1 + # with: + # path: ./packages/**/dist + # key: ${{ github.sha }}-base:18-test-lint unit-test: name: Unit tests uses: ./.github/workflows/units-tests-reusable.yml needs: install - with: - ref: ${{ github.event.pull_request.head.ref }} - cacheKey: ${{ github.sha }}-base:18-test-lint + # with: + # ref: ${{ github.event.pull_request.head.ref }} + # cacheKey: ${{ github.sha }}-base:18-test-lint lint: name: Lint changes @@ -63,9 +63,9 @@ jobs: - name: Restore cached build artifacts uses: actions/cache/restore@v3.3.1 - with: - path: ./packages/**/dist - key: ${{ github.sha }}-base:18-test-lint + # with: + # path: ./packages/**/dist + # key: ${{ github.sha }}-base:18-test-lint - name: Lint run: pnpm lint From 3d76b21999b5f71a3eb916b3bf390223e6df2ea6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Mon, 18 Sep 2023 11:11:22 +0200 Subject: [PATCH 075/103] Revert "Comment out cache keys" This reverts commit 96ea9eab6de78a9b8e494beacdcf413161373e17. --- .github/workflows/ci-pull-requests.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci-pull-requests.yml b/.github/workflows/ci-pull-requests.yml index 0fb9809582f7e..e87b1a6d925f6 100644 --- a/.github/workflows/ci-pull-requests.yml +++ b/.github/workflows/ci-pull-requests.yml @@ -18,7 +18,7 @@ jobs: uses: actions/setup-node@v3.7.0 with: node-version: 18.x - # cache: pnpm + cache: pnpm - name: Install dependencies run: pnpm install --frozen-lockfile @@ -26,19 +26,19 @@ jobs: - name: Build run: pnpm build - # - name: Cache build artifacts - # uses: actions/cache/save@v3.3.1 - # with: - # path: ./packages/**/dist - # key: ${{ github.sha }}-base:18-test-lint + - name: Cache build artifacts + uses: actions/cache/save@v3.3.1 + with: + path: ./packages/**/dist + key: ${{ github.sha }}-base:18-test-lint unit-test: name: Unit tests uses: ./.github/workflows/units-tests-reusable.yml needs: install - # with: - # ref: ${{ github.event.pull_request.head.ref }} - # cacheKey: ${{ github.sha }}-base:18-test-lint + with: + ref: ${{ github.event.pull_request.head.ref }} + cacheKey: ${{ github.sha }}-base:18-test-lint lint: name: Lint changes @@ -63,9 +63,9 @@ jobs: - name: Restore cached build artifacts uses: actions/cache/restore@v3.3.1 - # with: - # path: ./packages/**/dist - # key: ${{ github.sha }}-base:18-test-lint + with: + path: ./packages/**/dist + key: ${{ github.sha }}-base:18-test-lint - name: Lint run: pnpm lint From 455c327caaf6d28852663f721037f528352d6d53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Mon, 18 Sep 2023 11:20:33 +0200 Subject: [PATCH 076/103] Rename dir to `TempBinaryData` --- packages/core/src/NodeExecuteFunctions.ts | 2 +- .../src/{BinaryData => TempBinaryData}/BinaryData.service.ts | 0 .../src/{BinaryData => TempBinaryData}/FileSystem.manager.ts | 0 .../src/{BinaryData => TempBinaryData}/ObjectStore.manager.ts | 0 packages/core/src/{binaryData => TempBinaryData}/types.ts | 0 packages/core/src/{binaryData => TempBinaryData}/utils.ts | 0 packages/core/src/index.ts | 4 ++-- packages/core/test/NodeExecuteFunctions.test.ts | 2 +- 8 files changed, 4 insertions(+), 4 deletions(-) rename packages/core/src/{BinaryData => TempBinaryData}/BinaryData.service.ts (100%) rename packages/core/src/{BinaryData => TempBinaryData}/FileSystem.manager.ts (100%) rename packages/core/src/{BinaryData => TempBinaryData}/ObjectStore.manager.ts (100%) rename packages/core/src/{binaryData => TempBinaryData}/types.ts (100%) rename packages/core/src/{binaryData => TempBinaryData}/utils.ts (100%) diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index fa7dede9ec37b..f48caeadaeef3 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -115,7 +115,7 @@ import { Readable } from 'stream'; import { access as fsAccess, writeFile as fsWriteFile } from 'fs/promises'; import { createReadStream } from 'fs'; -import { BinaryDataService } from './BinaryData/BinaryData.service'; +import { BinaryDataService } from './TempBinaryData/BinaryData.service'; import type { ExtendedValidationResult, IResponseError, IWorkflowSettings } from './Interfaces'; import { extractValue } from './ExtractValue'; import { getClientCredentialsToken } from './OAuth2Helper'; diff --git a/packages/core/src/BinaryData/BinaryData.service.ts b/packages/core/src/TempBinaryData/BinaryData.service.ts similarity index 100% rename from packages/core/src/BinaryData/BinaryData.service.ts rename to packages/core/src/TempBinaryData/BinaryData.service.ts diff --git a/packages/core/src/BinaryData/FileSystem.manager.ts b/packages/core/src/TempBinaryData/FileSystem.manager.ts similarity index 100% rename from packages/core/src/BinaryData/FileSystem.manager.ts rename to packages/core/src/TempBinaryData/FileSystem.manager.ts diff --git a/packages/core/src/BinaryData/ObjectStore.manager.ts b/packages/core/src/TempBinaryData/ObjectStore.manager.ts similarity index 100% rename from packages/core/src/BinaryData/ObjectStore.manager.ts rename to packages/core/src/TempBinaryData/ObjectStore.manager.ts diff --git a/packages/core/src/binaryData/types.ts b/packages/core/src/TempBinaryData/types.ts similarity index 100% rename from packages/core/src/binaryData/types.ts rename to packages/core/src/TempBinaryData/types.ts diff --git a/packages/core/src/binaryData/utils.ts b/packages/core/src/TempBinaryData/utils.ts similarity index 100% rename from packages/core/src/binaryData/utils.ts rename to packages/core/src/TempBinaryData/utils.ts diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 9d6b5bfeebda3..ea351ffbbc7e1 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -2,8 +2,8 @@ import * as NodeExecuteFunctions from './NodeExecuteFunctions'; import * as UserSettings from './UserSettings'; export * from './ActiveWorkflows'; -export * from './BinaryData/BinaryData.service'; -export * from './BinaryData/types'; +export * from './TempBinaryData/BinaryData.service'; +export * from './TempBinaryData/types'; export * from './ClassLoader'; export * from './Constants'; export * from './Credentials'; diff --git a/packages/core/test/NodeExecuteFunctions.test.ts b/packages/core/test/NodeExecuteFunctions.test.ts index 96b45e907f478..6a4aeab8bbd8a 100644 --- a/packages/core/test/NodeExecuteFunctions.test.ts +++ b/packages/core/test/NodeExecuteFunctions.test.ts @@ -11,7 +11,7 @@ import type { Workflow, WorkflowHooks, } from 'n8n-workflow'; -import { BinaryDataService } from '@/BinaryData/BinaryData.service'; +import { BinaryDataService } from '@/TempBinaryData/BinaryData.service'; import { setBinaryDataBuffer, getBinaryDataBuffer, From 65b4fb0e699c25186a1f997b0922837ddf9488b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Mon, 18 Sep 2023 11:41:49 +0200 Subject: [PATCH 077/103] Rename back to `BinaryData` --- .../src/{TempBinaryData => BinaryData}/BinaryData.service.ts | 0 .../src/{TempBinaryData => BinaryData}/FileSystem.manager.ts | 0 .../src/{TempBinaryData => BinaryData}/ObjectStore.manager.ts | 0 packages/core/src/{TempBinaryData => BinaryData}/types.ts | 0 packages/core/src/{TempBinaryData => BinaryData}/utils.ts | 0 packages/core/src/NodeExecuteFunctions.ts | 2 +- packages/core/src/index.ts | 4 ++-- packages/core/test/NodeExecuteFunctions.test.ts | 2 +- 8 files changed, 4 insertions(+), 4 deletions(-) rename packages/core/src/{TempBinaryData => BinaryData}/BinaryData.service.ts (100%) rename packages/core/src/{TempBinaryData => BinaryData}/FileSystem.manager.ts (100%) rename packages/core/src/{TempBinaryData => BinaryData}/ObjectStore.manager.ts (100%) rename packages/core/src/{TempBinaryData => BinaryData}/types.ts (100%) rename packages/core/src/{TempBinaryData => BinaryData}/utils.ts (100%) diff --git a/packages/core/src/TempBinaryData/BinaryData.service.ts b/packages/core/src/BinaryData/BinaryData.service.ts similarity index 100% rename from packages/core/src/TempBinaryData/BinaryData.service.ts rename to packages/core/src/BinaryData/BinaryData.service.ts diff --git a/packages/core/src/TempBinaryData/FileSystem.manager.ts b/packages/core/src/BinaryData/FileSystem.manager.ts similarity index 100% rename from packages/core/src/TempBinaryData/FileSystem.manager.ts rename to packages/core/src/BinaryData/FileSystem.manager.ts diff --git a/packages/core/src/TempBinaryData/ObjectStore.manager.ts b/packages/core/src/BinaryData/ObjectStore.manager.ts similarity index 100% rename from packages/core/src/TempBinaryData/ObjectStore.manager.ts rename to packages/core/src/BinaryData/ObjectStore.manager.ts diff --git a/packages/core/src/TempBinaryData/types.ts b/packages/core/src/BinaryData/types.ts similarity index 100% rename from packages/core/src/TempBinaryData/types.ts rename to packages/core/src/BinaryData/types.ts diff --git a/packages/core/src/TempBinaryData/utils.ts b/packages/core/src/BinaryData/utils.ts similarity index 100% rename from packages/core/src/TempBinaryData/utils.ts rename to packages/core/src/BinaryData/utils.ts diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index f48caeadaeef3..fa7dede9ec37b 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -115,7 +115,7 @@ import { Readable } from 'stream'; import { access as fsAccess, writeFile as fsWriteFile } from 'fs/promises'; import { createReadStream } from 'fs'; -import { BinaryDataService } from './TempBinaryData/BinaryData.service'; +import { BinaryDataService } from './BinaryData/BinaryData.service'; import type { ExtendedValidationResult, IResponseError, IWorkflowSettings } from './Interfaces'; import { extractValue } from './ExtractValue'; import { getClientCredentialsToken } from './OAuth2Helper'; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index ea351ffbbc7e1..9d6b5bfeebda3 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -2,8 +2,8 @@ import * as NodeExecuteFunctions from './NodeExecuteFunctions'; import * as UserSettings from './UserSettings'; export * from './ActiveWorkflows'; -export * from './TempBinaryData/BinaryData.service'; -export * from './TempBinaryData/types'; +export * from './BinaryData/BinaryData.service'; +export * from './BinaryData/types'; export * from './ClassLoader'; export * from './Constants'; export * from './Credentials'; diff --git a/packages/core/test/NodeExecuteFunctions.test.ts b/packages/core/test/NodeExecuteFunctions.test.ts index 6a4aeab8bbd8a..96b45e907f478 100644 --- a/packages/core/test/NodeExecuteFunctions.test.ts +++ b/packages/core/test/NodeExecuteFunctions.test.ts @@ -11,7 +11,7 @@ import type { Workflow, WorkflowHooks, } from 'n8n-workflow'; -import { BinaryDataService } from '@/TempBinaryData/BinaryData.service'; +import { BinaryDataService } from '@/BinaryData/BinaryData.service'; import { setBinaryDataBuffer, getBinaryDataBuffer, From abe6d9df1d298dd9d602fb6198b824e87193beaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Mon, 18 Sep 2023 15:11:57 +0200 Subject: [PATCH 078/103] Clear timers on shutdown --- packages/cli/src/commands/start.ts | 3 +++ .../databases/repositories/execution.repository.ts | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index b0a9b0b73988b..0cebdb633ec8e 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -29,6 +29,7 @@ import { eventBus } from '@/eventbus'; import { BaseCommand } from './BaseCommand'; import { InternalHooks } from '@/InternalHooks'; import { License } from '@/License'; +import { ExecutionRepository } from '@/databases/repositories/execution.repository'; // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-var-requires const open = require('open'); @@ -103,6 +104,8 @@ export class Start extends BaseCommand { // Note: While this saves a new license cert to DB, the previous entitlements are still kept in memory so that the shutdown process can complete await Container.get(License).shutdown(); + Container.get(ExecutionRepository).clearTimers(); + await Container.get(InternalHooks).onN8nStop(); const skipWebhookDeregistration = config.getEnv( diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index e1d4c3840e2bb..7b2e436416902 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -78,7 +78,9 @@ function parseFiltersToQueryBuilder( export class ExecutionRepository extends Repository { deletionBatchSize = 100; - hardDeletionInterval: NodeJS.Timer | null = null; + private pruningInterval: NodeJS.Timer | null = null; + + private hardDeletionInterval: NodeJS.Timer | null = null; constructor( dataSource: DataSource, @@ -91,8 +93,13 @@ export class ExecutionRepository extends Repository { this.setHardDeletionInterval(); } + clearTimers() { + if (this.hardDeletionInterval) clearInterval(this.hardDeletionInterval); + if (this.pruningInterval) clearInterval(this.pruningInterval); + } + setPruningInterval() { - setInterval(async () => this.pruneBySoftDeleting(), 1 * TIME.HOUR); + this.pruningInterval = setInterval(async () => this.pruneBySoftDeleting(), 1 * TIME.HOUR); } setHardDeletionInterval() { From 2682b3d596dcbfa3aca8fd697f1f5af589217aef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Mon, 18 Sep 2023 16:20:03 +0200 Subject: [PATCH 079/103] Set timers only on main instance --- packages/cli/src/commands/BaseCommand.ts | 2 ++ packages/cli/src/config/schema.ts | 6 ++++++ .../cli/src/databases/repositories/execution.repository.ts | 3 +++ 3 files changed, 11 insertions(+) diff --git a/packages/cli/src/commands/BaseCommand.ts b/packages/cli/src/commands/BaseCommand.ts index 9b2658ac7f7ff..6aa1e12176999 100644 --- a/packages/cli/src/commands/BaseCommand.ts +++ b/packages/cli/src/commands/BaseCommand.ts @@ -114,6 +114,8 @@ export abstract class BaseCommand extends Command { } async initLicense(instanceType: N8nInstanceType = 'main'): Promise { + config.set('generic.instanceType', instanceType); + const license = Container.get(License); await license.init(this.instanceId, instanceType); diff --git a/packages/cli/src/config/schema.ts b/packages/cli/src/config/schema.ts index 05ed7000ee186..1704be42ba8b2 100644 --- a/packages/cli/src/config/schema.ts +++ b/packages/cli/src/config/schema.ts @@ -432,6 +432,12 @@ export const schema = { default: 'America/New_York', env: 'GENERIC_TIMEZONE', }, + + instanceType: { + doc: 'Type of n8n instance', + format: ['main', 'webhook', 'worker'] as const, + default: 'main', + }, }, // How n8n can be reached (Editor & REST-API) diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 7b2e436416902..11177325c4835 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -87,7 +87,10 @@ export class ExecutionRepository extends Repository { private readonly executionDataRepository: ExecutionDataRepository, ) { super(ExecutionEntity, dataSource.manager); + if (config.get('generic.instanceType') === 'main') this.setTimers(); + } + setTimers() { if (config.getEnv('executions.pruneData')) this.setPruningInterval(); this.setHardDeletionInterval(); From 1c788fe9e7620fd98dcc832a665daa195acf2780 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Mon, 18 Sep 2023 16:21:57 +0200 Subject: [PATCH 080/103] Also for clearing timers --- .../cli/src/databases/repositories/execution.repository.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 11177325c4835..9b3dd338370e2 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -87,6 +87,7 @@ export class ExecutionRepository extends Repository { private readonly executionDataRepository: ExecutionDataRepository, ) { super(ExecutionEntity, dataSource.manager); + if (config.get('generic.instanceType') === 'main') this.setTimers(); } @@ -97,6 +98,8 @@ export class ExecutionRepository extends Repository { } clearTimers() { + if (config.get('generic.instanceType') !== 'main') return; + if (this.hardDeletionInterval) clearInterval(this.hardDeletionInterval); if (this.pruningInterval) clearInterval(this.pruningInterval); } From 04f4b838840ea3e40c261a3d38d618d55ade3be8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Mon, 18 Sep 2023 16:26:38 +0200 Subject: [PATCH 081/103] Rename back to `localStoragePath` --- packages/cli/src/config/schema.ts | 2 +- packages/core/src/BinaryData/BinaryData.service.ts | 2 +- packages/core/src/BinaryData/types.ts | 2 +- packages/core/test/NodeExecuteFunctions.test.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/config/schema.ts b/packages/cli/src/config/schema.ts index c03a70d469b0a..2529354c15334 100644 --- a/packages/cli/src/config/schema.ts +++ b/packages/cli/src/config/schema.ts @@ -906,7 +906,7 @@ export const schema = { env: 'N8N_DEFAULT_BINARY_DATA_MODE', doc: 'Storage mode for binary data', }, - storagePath: { + localStoragePath: { format: String, default: path.join(UserSettings.getUserN8nFolderPath(), 'binaryData'), env: 'N8N_BINARY_DATA_STORAGE_PATH', diff --git a/packages/core/src/BinaryData/BinaryData.service.ts b/packages/core/src/BinaryData/BinaryData.service.ts index d08e670254abb..fe7e5c46a771a 100644 --- a/packages/core/src/BinaryData/BinaryData.service.ts +++ b/packages/core/src/BinaryData/BinaryData.service.ts @@ -26,7 +26,7 @@ export class BinaryDataService { this.mode = config.mode; if (this.availableModes.includes('filesystem') && config.mode === 'filesystem') { - this.managers.filesystem = new FileSystemManager(config.storagePath); + this.managers.filesystem = new FileSystemManager(config.localStoragePath); await this.managers.filesystem.init(mainManager); } diff --git a/packages/core/src/BinaryData/types.ts b/packages/core/src/BinaryData/types.ts index 1f0c668ce9a4d..a691bbb3b83ba 100644 --- a/packages/core/src/BinaryData/types.ts +++ b/packages/core/src/BinaryData/types.ts @@ -12,7 +12,7 @@ export namespace BinaryData { type InMemoryConfig = ConfigBase & { mode: 'default' }; - export type FileSystemConfig = ConfigBase & { mode: 'filesystem'; storagePath: string }; + export type FileSystemConfig = ConfigBase & { mode: 'filesystem'; localStoragePath: string }; export type Config = InMemoryConfig | FileSystemConfig; diff --git a/packages/core/test/NodeExecuteFunctions.test.ts b/packages/core/test/NodeExecuteFunctions.test.ts index 96b45e907f478..d59c976a52cb5 100644 --- a/packages/core/test/NodeExecuteFunctions.test.ts +++ b/packages/core/test/NodeExecuteFunctions.test.ts @@ -81,7 +81,7 @@ describe('NodeExecuteFunctions', () => { await Container.get(BinaryDataService).init({ mode: 'filesystem', availableModes: ['filesystem'], - storagePath: temporaryDir, + localStoragePath: temporaryDir, }); // Set our binary data buffer From 03264f941d04a2ca98279d6c456a701a21b33088 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Mon, 18 Sep 2023 16:28:48 +0200 Subject: [PATCH 082/103] Remove unused arg --- packages/core/src/BinaryData/BinaryData.service.ts | 4 ++-- packages/core/src/BinaryData/types.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/core/src/BinaryData/BinaryData.service.ts b/packages/core/src/BinaryData/BinaryData.service.ts index fe7e5c46a771a..629022ac36fd8 100644 --- a/packages/core/src/BinaryData/BinaryData.service.ts +++ b/packages/core/src/BinaryData/BinaryData.service.ts @@ -19,7 +19,7 @@ export class BinaryDataService { private managers: Record = {}; - async init(config: BinaryData.Config, mainManager = false) { + async init(config: BinaryData.Config) { if (!areValidModes(config.availableModes)) throw new InvalidBinaryDataModeError(); this.availableModes = config.availableModes; @@ -28,7 +28,7 @@ export class BinaryDataService { if (this.availableModes.includes('filesystem') && config.mode === 'filesystem') { this.managers.filesystem = new FileSystemManager(config.localStoragePath); - await this.managers.filesystem.init(mainManager); + await this.managers.filesystem.init(); } } diff --git a/packages/core/src/BinaryData/types.ts b/packages/core/src/BinaryData/types.ts index a691bbb3b83ba..c843f14d30188 100644 --- a/packages/core/src/BinaryData/types.ts +++ b/packages/core/src/BinaryData/types.ts @@ -17,7 +17,7 @@ export namespace BinaryData { export type Config = InMemoryConfig | FileSystemConfig; export interface Manager { - init(startPurger: boolean): Promise; + init(): Promise; store(binaryData: Buffer | Readable, executionId: string): Promise; getPath(identifier: string): string; From a6e33d498ff8ab15331cffecc20181437646cbf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Mon, 18 Sep 2023 16:34:56 +0200 Subject: [PATCH 083/103] Fix lint --- packages/cli/src/commands/BaseCommand.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/commands/BaseCommand.ts b/packages/cli/src/commands/BaseCommand.ts index eeca8f751075d..17a95a155f6fe 100644 --- a/packages/cli/src/commands/BaseCommand.ts +++ b/packages/cli/src/commands/BaseCommand.ts @@ -105,7 +105,7 @@ export abstract class BaseCommand extends Command { async initBinaryDataService() { const binaryDataConfig = config.getEnv('binaryDataService'); - await Container.get(BinaryDataService).init(binaryDataConfig, true); + await Container.get(BinaryDataService).init(binaryDataConfig); } async initExternalHooks() { From 88235544adec7f55718d2dba72244e218d10af09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Tue, 19 Sep 2023 10:12:25 +0200 Subject: [PATCH 084/103] Add logging, refactor for readability --- .../repositories/execution.repository.ts | 59 ++++++++++++------- .../repositories/execution.repository.test.ts | 4 +- 2 files changed, 40 insertions(+), 23 deletions(-) diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 9b3dd338370e2..53d66d5ef1a4a 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -76,11 +76,23 @@ function parseFiltersToQueryBuilder( @Service() export class ExecutionRepository extends Repository { + private logger = Logger; + deletionBatchSize = 100; - private pruningInterval: NodeJS.Timer | null = null; + private intervals: Record = { + softDeletion: undefined, + hardDeletion: undefined, + }; + + private rates: Record = { + softDeletion: 1 * TIME.HOUR, + hardDeletion: 15 * TIME.MINUTE, + }; + + private isMainInstance = config.get('generic.instanceType') === 'main'; - private hardDeletionInterval: NodeJS.Timer | null = null; + private isPruningEnabled = config.getEnv('executions.pruneData'); constructor( dataSource: DataSource, @@ -88,36 +100,35 @@ export class ExecutionRepository extends Repository { ) { super(ExecutionEntity, dataSource.manager); - if (config.get('generic.instanceType') === 'main') this.setTimers(); - } + if (!this.isMainInstance) return; - setTimers() { - if (config.getEnv('executions.pruneData')) this.setPruningInterval(); + if (this.isPruningEnabled) this.setSoftDeletionInterval(); this.setHardDeletionInterval(); } clearTimers() { - if (config.get('generic.instanceType') !== 'main') return; + if (!this.isMainInstance) return; - if (this.hardDeletionInterval) clearInterval(this.hardDeletionInterval); - if (this.pruningInterval) clearInterval(this.pruningInterval); - } + this.logger.info('Clearing soft-deletion and hard-deletion intervals for executions'); - setPruningInterval() { - this.pruningInterval = setInterval(async () => this.pruneBySoftDeleting(), 1 * TIME.HOUR); + clearInterval(this.intervals.softDeletion); + clearInterval(this.intervals.hardDeletion); } - setHardDeletionInterval() { - if (this.hardDeletionInterval) return; + setSoftDeletionInterval() { + this.logger.info('Setting soft-deletion interval (pruning) for executions'); - this.hardDeletionInterval = setInterval(async () => this.hardDelete(), 15 * TIME.MINUTE); + this.intervals.softDeletion = setInterval(async () => this.prune(), this.rates.hardDeletion); } - clearHardDeletionInterval() { - if (!this.hardDeletionInterval) return; + setHardDeletionInterval() { + this.logger.info('Setting hard-deletion interval for executions'); - clearInterval(this.hardDeletionInterval); + this.intervals.hardDeletion = setInterval( + async () => this.hardDelete(), + this.rates.hardDeletion, + ); } async findMultipleExecutions( @@ -457,8 +468,8 @@ export class ExecutionRepository extends Repository { } while (executionIds.length > 0); } - async pruneBySoftDeleting() { - Logger.verbose('Pruning (soft-deleting) execution data from database'); + async prune() { + Logger.verbose('Soft-deleting (pruning) execution data from database'); const maxAge = config.getEnv('executions.pruneDataMaxAge'); // in h const maxCount = config.getEnv('executions.pruneDataMaxCount'); @@ -521,6 +532,10 @@ export class ExecutionRepository extends Repository { const binaryDataManager = BinaryDataManager.getInstance(); await binaryDataManager.deleteBinaryDataByExecutionIds(executionIds); + this.logger.info(`Hard-deleting ${executionIds.length} executions from database`, { + executionIds, + }); + // Actually delete these executions await this.delete({ id: In(executionIds) }); @@ -533,10 +548,12 @@ export class ExecutionRepository extends Repository { * the number of executions to prune is low enough to fit in a single batch. */ if (executionIds.length === this.deletionBatchSize) { - this.clearHardDeletionInterval(); + clearInterval(this.intervals.hardDeletion); setTimeout(async () => this.hardDelete(), 1 * TIME.SECOND); } else { + if (this.intervals.hardDeletion) return; + this.setHardDeletionInterval(); } } diff --git a/packages/cli/test/unit/repositories/execution.repository.test.ts b/packages/cli/test/unit/repositories/execution.repository.test.ts index ac15979504a6f..500e1ae0b6a33 100644 --- a/packages/cli/test/unit/repositories/execution.repository.test.ts +++ b/packages/cli/test/unit/repositories/execution.repository.test.ts @@ -51,7 +51,7 @@ describe('ExecutionRepository', () => { jest.spyOn(ExecutionRepository.prototype, 'createQueryBuilder').mockReturnValueOnce(qb); - await executionRepository.pruneBySoftDeleting(); + await executionRepository.prune(); expect(find.mock.calls[0][0]).toEqual(objectContaining({ skip: maxCount })); }); @@ -70,7 +70,7 @@ describe('ExecutionRepository', () => { const now = Date.now(); - await executionRepository.pruneBySoftDeleting(); + await executionRepository.prune(); const argDate = dateFormat.mock.calls[0][0]; const difference = now - argDate.valueOf(); From a5def408265a59a800a772fb17e25b887cfcf2a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Tue, 19 Sep 2023 10:18:00 +0200 Subject: [PATCH 085/103] Ensure hard-deletion select includes soft-deleted rows --- .../cli/src/databases/repositories/execution.repository.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 53d66d5ef1a4a..0e1814fbe65d8 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -526,6 +526,12 @@ export class ExecutionRepository extends Repository { deletedAt: LessThanOrEqual(DateUtils.mixedDateToUtcDatetimeString(date)), }, take: this.deletionBatchSize, + + /** + * @important This ensures soft-deleted executions are included, + * else `@DeleteDateColumn()` at `deletedAt` will exclude them. + */ + withDeleted: true, }) ).map(({ id }) => id); From 77955e4ea0fe74e9ac9edef82b2f07ddd5a6a3e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Tue, 19 Sep 2023 10:18:51 +0200 Subject: [PATCH 086/103] Switch `info` to `debug` --- .../src/databases/repositories/execution.repository.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 0e1814fbe65d8..4e2def452fdb5 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -110,20 +110,20 @@ export class ExecutionRepository extends Repository { clearTimers() { if (!this.isMainInstance) return; - this.logger.info('Clearing soft-deletion and hard-deletion intervals for executions'); + this.logger.debug('Clearing soft-deletion and hard-deletion intervals for executions'); clearInterval(this.intervals.softDeletion); clearInterval(this.intervals.hardDeletion); } setSoftDeletionInterval() { - this.logger.info('Setting soft-deletion interval (pruning) for executions'); + this.logger.debug('Setting soft-deletion interval (pruning) for executions'); this.intervals.softDeletion = setInterval(async () => this.prune(), this.rates.hardDeletion); } setHardDeletionInterval() { - this.logger.info('Setting hard-deletion interval for executions'); + this.logger.debug('Setting hard-deletion interval for executions'); this.intervals.hardDeletion = setInterval( async () => this.hardDelete(), @@ -538,7 +538,7 @@ export class ExecutionRepository extends Repository { const binaryDataManager = BinaryDataManager.getInstance(); await binaryDataManager.deleteBinaryDataByExecutionIds(executionIds); - this.logger.info(`Hard-deleting ${executionIds.length} executions from database`, { + this.logger.debug(`Hard-deleting ${executionIds.length} executions from database`, { executionIds, }); From b930e3e12d986a497708774ad3764b44e27ff92f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Tue, 19 Sep 2023 10:30:30 +0200 Subject: [PATCH 087/103] Fix tests --- packages/cli/test/integration/executions.controller.test.ts | 6 +++++- packages/cli/test/integration/publicApi/executions.test.ts | 4 ++++ .../cli/test/unit/repositories/execution.repository.test.ts | 2 ++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/cli/test/integration/executions.controller.test.ts b/packages/cli/test/integration/executions.controller.test.ts index fa6a70499cc3f..bc2f783674a1c 100644 --- a/packages/cli/test/integration/executions.controller.test.ts +++ b/packages/cli/test/integration/executions.controller.test.ts @@ -1,8 +1,12 @@ import * as testDb from './shared/testDb'; import { setupTestServer } from './shared/utils'; import type { User } from '@/databases/entities/User'; +import { LoggerProxy } from 'n8n-workflow'; +import { getLogger } from '@/Logger'; -const testServer = setupTestServer({ endpointGroups: ['executions'] }); +LoggerProxy.init(getLogger()); + +let testServer = setupTestServer({ endpointGroups: ['executions'] }); let owner: User; diff --git a/packages/cli/test/integration/publicApi/executions.test.ts b/packages/cli/test/integration/publicApi/executions.test.ts index d42f8ca62ad35..9b208548a5b05 100644 --- a/packages/cli/test/integration/publicApi/executions.test.ts +++ b/packages/cli/test/integration/publicApi/executions.test.ts @@ -5,6 +5,8 @@ import type { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner'; import { randomApiKey } from '../shared/random'; import * as utils from '../shared/utils/'; import * as testDb from '../shared/testDb'; +import { LoggerProxy } from 'n8n-workflow'; +import { getLogger } from '@/Logger'; let owner: User; let user1: User; @@ -14,6 +16,8 @@ let authUser1Agent: SuperAgentTest; let authUser2Agent: SuperAgentTest; let workflowRunner: ActiveWorkflowRunner; +LoggerProxy.init(getLogger()); + const testServer = utils.setupTestServer({ endpointGroups: ['publicApi'] }); beforeAll(async () => { diff --git a/packages/cli/test/unit/repositories/execution.repository.test.ts b/packages/cli/test/unit/repositories/execution.repository.test.ts index 500e1ae0b6a33..7d2d0350dbe93 100644 --- a/packages/cli/test/unit/repositories/execution.repository.test.ts +++ b/packages/cli/test/unit/repositories/execution.repository.test.ts @@ -11,6 +11,8 @@ import { DateUtils } from 'typeorm/util/DateUtils'; jest.mock('typeorm/util/DateUtils'); +LoggerProxy.init(getLogger()); + const { objectContaining } = expect; // eslint-disable-next-line @typescript-eslint/no-explicit-any From e91f3de30a60e7c463af5c002cb1411e8d6e02f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Tue, 19 Sep 2023 10:32:25 +0200 Subject: [PATCH 088/103] Remove redundant checks for `deletedAt` being `NULL` The column `deletedAt` is marked `@DeleteDateColumn()` so all reads from the repository automatically add a `WHERE` clause checking that the column `IS NULL`. --- .../src/databases/repositories/execution.repository.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 4e2def452fdb5..d9a0839f33b08 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -166,10 +166,6 @@ export class ExecutionRepository extends Repository { (queryParams.relations as string[]).push('executionData'); } - if (queryParams.where && !Array.isArray(queryParams.where)) { - queryParams.where.deletedAt = IsNull(); - } - const executions = await this.find(queryParams); if (options?.includeData && options?.unflattenData) { @@ -234,7 +230,6 @@ export class ExecutionRepository extends Repository { where: { id, ...options?.where, - deletedAt: IsNull(), }, }; if (options?.includeData) { @@ -393,9 +388,7 @@ export class ExecutionRepository extends Repository { .limit(limit) // eslint-disable-next-line @typescript-eslint/naming-convention .orderBy({ 'execution.id': 'DESC' }) - .andWhere('execution.workflowId IN (:...accessibleWorkflowIds)', { accessibleWorkflowIds }) - .andWhere('execution.deletedAt IS NULL'); - + .andWhere('execution.workflowId IN (:...accessibleWorkflowIds)', { accessibleWorkflowIds }); if (excludedExecutionIds.length > 0) { query.andWhere('execution.id NOT IN (:...excludedExecutionIds)', { excludedExecutionIds }); } From 2c3704d0e3aec8fe16ec5775fbe3decd16e48a3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Tue, 19 Sep 2023 10:43:41 +0200 Subject: [PATCH 089/103] Fix lint --- .../src/databases/repositories/execution.repository.ts | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index d9a0839f33b08..651e6d7bfc6d7 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -1,13 +1,5 @@ import { Service } from 'typedi'; -import { - Brackets, - DataSource, - In, - IsNull, - LessThanOrEqual, - MoreThanOrEqual, - Repository, -} from 'typeorm'; +import { Brackets, DataSource, In, LessThanOrEqual, MoreThanOrEqual, Repository } from 'typeorm'; import { DateUtils } from 'typeorm/util/DateUtils'; import type { FindManyOptions, From 215f58e7ba1bc3c2312746d1ededf412c1cb6c66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Tue, 19 Sep 2023 11:23:02 +0200 Subject: [PATCH 090/103] Fix last test --- packages/cli/test/integration/publicApi/workflows.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/cli/test/integration/publicApi/workflows.test.ts b/packages/cli/test/integration/publicApi/workflows.test.ts index 910bfbb3970c8..980986cf96d44 100644 --- a/packages/cli/test/integration/publicApi/workflows.test.ts +++ b/packages/cli/test/integration/publicApi/workflows.test.ts @@ -10,6 +10,8 @@ import * as utils from '../shared/utils/'; import * as testDb from '../shared/testDb'; import type { INode } from 'n8n-workflow'; import { STARTING_NODES } from '@/constants'; +import { LoggerProxy } from 'n8n-workflow'; +import { getLogger } from '@/Logger'; let workflowOwnerRole: Role; let owner: User; @@ -18,6 +20,8 @@ let authOwnerAgent: SuperAgentTest; let authMemberAgent: SuperAgentTest; let workflowRunner: ActiveWorkflowRunner; +LoggerProxy.init(getLogger()); + const testServer = utils.setupTestServer({ endpointGroups: ['publicApi'] }); beforeAll(async () => { From c33d164913ba3c231b6489de55b20b8f725a4e94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Tue, 19 Sep 2023 11:54:50 +0200 Subject: [PATCH 091/103] More missing loggers in tests --- .../cli/test/integration/workflows.controller.ee.test.ts | 5 +++++ packages/cli/test/integration/workflows.controller.test.ts | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/packages/cli/test/integration/workflows.controller.ee.test.ts b/packages/cli/test/integration/workflows.controller.ee.test.ts index 6c63bbc8ae9ff..4833c36c1e2e6 100644 --- a/packages/cli/test/integration/workflows.controller.ee.test.ts +++ b/packages/cli/test/integration/workflows.controller.ee.test.ts @@ -13,6 +13,9 @@ import type { SaveCredentialFunction } from './shared/types'; import { makeWorkflow } from './shared/utils/'; import { randomCredentialPayload } from './shared/random'; +import { LoggerProxy } from 'n8n-workflow'; +import { getLogger } from '@/Logger'; + let owner: User; let member: User; let anotherMember: User; @@ -21,6 +24,8 @@ let authMemberAgent: SuperAgentTest; let authAnotherMemberAgent: SuperAgentTest; let saveCredential: SaveCredentialFunction; +LoggerProxy.init(getLogger()); + const sharingSpy = jest.spyOn(UserManagementHelpers, 'isSharingEnabled').mockReturnValue(true); const testServer = utils.setupTestServer({ endpointGroups: ['workflows'], diff --git a/packages/cli/test/integration/workflows.controller.test.ts b/packages/cli/test/integration/workflows.controller.test.ts index fb56237fd7a6b..db4298b933503 100644 --- a/packages/cli/test/integration/workflows.controller.test.ts +++ b/packages/cli/test/integration/workflows.controller.test.ts @@ -12,9 +12,14 @@ import { RoleService } from '@/services/role.service'; import Container from 'typedi'; import type { ListQuery } from '@/requests'; +import { LoggerProxy } from 'n8n-workflow'; +import { getLogger } from '@/Logger'; + let owner: User; let authOwnerAgent: SuperAgentTest; +LoggerProxy.init(getLogger()); + jest.spyOn(UserManagementHelpers, 'isSharingEnabled').mockReturnValue(false); const testServer = utils.setupTestServer({ endpointGroups: ['workflows'] }); From 69764f7b9f1d0f913c0cca95b840b227cf65c67f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Tue, 19 Sep 2023 12:39:59 +0200 Subject: [PATCH 092/103] Add logger to even more test --- packages/cli/test/integration/audit/credentials.risk.test.ts | 5 +++++ packages/cli/test/integration/credentials.controller.test.ts | 5 +++++ packages/cli/test/integration/credentials.ee.test.ts | 5 +++++ packages/cli/test/integration/credentials.test.ts | 4 ++++ packages/cli/test/integration/publicApi/credentials.test.ts | 5 +++++ 5 files changed, 24 insertions(+) diff --git a/packages/cli/test/integration/audit/credentials.risk.test.ts b/packages/cli/test/integration/audit/credentials.risk.test.ts index 10d1d9ecbd042..49a399841870a 100644 --- a/packages/cli/test/integration/audit/credentials.risk.test.ts +++ b/packages/cli/test/integration/audit/credentials.risk.test.ts @@ -7,6 +7,11 @@ import { getRiskSection } from './utils'; import * as testDb from '../shared/testDb'; import { generateNanoId } from '@db/utils/generators'; +import { LoggerProxy } from 'n8n-workflow'; +import { getLogger } from '@/Logger'; + +LoggerProxy.init(getLogger()); + beforeAll(async () => { await testDb.init(); }); diff --git a/packages/cli/test/integration/credentials.controller.test.ts b/packages/cli/test/integration/credentials.controller.test.ts index 1b55c71d46c74..cab87149b12f3 100644 --- a/packages/cli/test/integration/credentials.controller.test.ts +++ b/packages/cli/test/integration/credentials.controller.test.ts @@ -12,6 +12,11 @@ const testServer = utils.setupTestServer({ endpointGroups: ['credentials'] }); let owner: User; let member: User; +import { LoggerProxy } from 'n8n-workflow'; +import { getLogger } from '@/Logger'; + +LoggerProxy.init(getLogger()); + beforeEach(async () => { await testDb.truncate(['SharedCredentials', 'Credentials']); diff --git a/packages/cli/test/integration/credentials.ee.test.ts b/packages/cli/test/integration/credentials.ee.test.ts index a1dc4f7be00b5..e9fa4f088f268 100644 --- a/packages/cli/test/integration/credentials.ee.test.ts +++ b/packages/cli/test/integration/credentials.ee.test.ts @@ -14,6 +14,11 @@ import * as testDb from './shared/testDb'; import type { SaveCredentialFunction } from './shared/types'; import * as utils from './shared/utils/'; +import { LoggerProxy } from 'n8n-workflow'; +import { getLogger } from '@/Logger'; + +LoggerProxy.init(getLogger()); + const sharingSpy = jest.spyOn(UserManagementHelpers, 'isSharingEnabled').mockReturnValue(true); const testServer = utils.setupTestServer({ endpointGroups: ['credentials'] }); diff --git a/packages/cli/test/integration/credentials.test.ts b/packages/cli/test/integration/credentials.test.ts index 526d3bb28c77b..72c0aea730c2e 100644 --- a/packages/cli/test/integration/credentials.test.ts +++ b/packages/cli/test/integration/credentials.test.ts @@ -12,6 +12,10 @@ import { randomCredentialPayload, randomName, randomString } from './shared/rand import * as testDb from './shared/testDb'; import type { SaveCredentialFunction } from './shared/types'; import * as utils from './shared/utils/'; +import { LoggerProxy } from 'n8n-workflow'; +import { getLogger } from '@/Logger'; + +LoggerProxy.init(getLogger()); // mock that credentialsSharing is not enabled jest.spyOn(UserManagementHelpers, 'isSharingEnabled').mockReturnValue(false); diff --git a/packages/cli/test/integration/publicApi/credentials.test.ts b/packages/cli/test/integration/publicApi/credentials.test.ts index bc5c414142a17..fc576a9f7eed7 100644 --- a/packages/cli/test/integration/publicApi/credentials.test.ts +++ b/packages/cli/test/integration/publicApi/credentials.test.ts @@ -10,6 +10,11 @@ import * as utils from '../shared/utils/'; import type { CredentialPayload, SaveCredentialFunction } from '../shared/types'; import * as testDb from '../shared/testDb'; +import { LoggerProxy } from 'n8n-workflow'; +import { getLogger } from '@/Logger'; + +LoggerProxy.init(getLogger()); + let globalMemberRole: Role; let credentialOwnerRole: Role; let owner: User; From 7c3e58db2f8cb80983d885fc6096207089287941 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Tue, 19 Sep 2023 13:28:53 +0200 Subject: [PATCH 093/103] Refactor logging for tests --- packages/cli/test/integration/audit/database.risk.test.ts | 5 +++++ packages/cli/test/integration/audit/filesystem.risk.test.ts | 5 +++++ packages/cli/test/integration/audit/instance.risk.test.ts | 5 +++++ packages/cli/test/integration/audit/nodes.risk.test.ts | 5 +++++ packages/cli/test/integration/commands/import.cmd.test.ts | 5 +++++ .../cli/test/integration/credentials.controller.test.ts | 5 ----- packages/cli/test/integration/credentials.ee.test.ts | 5 ----- packages/cli/test/integration/credentials.test.ts | 4 ---- packages/cli/test/integration/executions.controller.test.ts | 4 ---- packages/cli/test/integration/ldap/ldap.api.test.ts | 5 +++++ packages/cli/test/integration/publicApi/credentials.test.ts | 5 ----- packages/cli/test/integration/publicApi/workflows.test.ts | 4 ---- packages/cli/test/integration/shared/utils/testServer.ts | 6 +++--- .../cli/test/integration/workflows.controller.ee.test.ts | 5 ----- packages/cli/test/integration/workflows.controller.test.ts | 5 ----- packages/cli/test/unit/PermissionChecker.test.ts | 5 +++++ 16 files changed, 38 insertions(+), 40 deletions(-) diff --git a/packages/cli/test/integration/audit/database.risk.test.ts b/packages/cli/test/integration/audit/database.risk.test.ts index a4068f5d77704..b2e485663feb4 100644 --- a/packages/cli/test/integration/audit/database.risk.test.ts +++ b/packages/cli/test/integration/audit/database.risk.test.ts @@ -10,6 +10,11 @@ import { getRiskSection, saveManualTriggerWorkflow } from './utils'; import * as testDb from '../shared/testDb'; import { generateNanoId } from '@db/utils/generators'; +import { LoggerProxy } from 'n8n-workflow'; +import { getLogger } from '@/Logger'; + +LoggerProxy.init(getLogger()); + beforeAll(async () => { await testDb.init(); }); diff --git a/packages/cli/test/integration/audit/filesystem.risk.test.ts b/packages/cli/test/integration/audit/filesystem.risk.test.ts index d8f3e711e7b55..33d0ef8fa5782 100644 --- a/packages/cli/test/integration/audit/filesystem.risk.test.ts +++ b/packages/cli/test/integration/audit/filesystem.risk.test.ts @@ -5,6 +5,11 @@ import { FILESYSTEM_INTERACTION_NODE_TYPES, FILESYSTEM_REPORT } from '@/audit/co import { getRiskSection, saveManualTriggerWorkflow } from './utils'; import * as testDb from '../shared/testDb'; +import { LoggerProxy } from 'n8n-workflow'; +import { getLogger } from '@/Logger'; + +LoggerProxy.init(getLogger()); + beforeAll(async () => { await testDb.init(); }); diff --git a/packages/cli/test/integration/audit/instance.risk.test.ts b/packages/cli/test/integration/audit/instance.risk.test.ts index aa186c4d7cfc3..349f5de38eb46 100644 --- a/packages/cli/test/integration/audit/instance.risk.test.ts +++ b/packages/cli/test/integration/audit/instance.risk.test.ts @@ -14,6 +14,11 @@ import { toReportTitle } from '@/audit/utils'; import config from '@/config'; import { generateNanoId } from '@db/utils/generators'; +import { LoggerProxy } from 'n8n-workflow'; +import { getLogger } from '@/Logger'; + +LoggerProxy.init(getLogger()); + beforeAll(async () => { await testDb.init(); diff --git a/packages/cli/test/integration/audit/nodes.risk.test.ts b/packages/cli/test/integration/audit/nodes.risk.test.ts index fb966d23697fb..dbe414a951930 100644 --- a/packages/cli/test/integration/audit/nodes.risk.test.ts +++ b/packages/cli/test/integration/audit/nodes.risk.test.ts @@ -11,6 +11,11 @@ import { NodeTypes } from '@/NodeTypes'; import { CommunityPackageService } from '@/services/communityPackage.service'; import Container from 'typedi'; +import { LoggerProxy } from 'n8n-workflow'; +import { getLogger } from '@/Logger'; + +LoggerProxy.init(getLogger()); + const nodesAndCredentials = mockInstance(LoadNodesAndCredentials); nodesAndCredentials.getCustomDirectories.mockReturnValue([]); mockInstance(NodeTypes); diff --git a/packages/cli/test/integration/commands/import.cmd.test.ts b/packages/cli/test/integration/commands/import.cmd.test.ts index 750ed86b440f7..a191267f622be 100644 --- a/packages/cli/test/integration/commands/import.cmd.test.ts +++ b/packages/cli/test/integration/commands/import.cmd.test.ts @@ -4,6 +4,11 @@ import { InternalHooks } from '@/InternalHooks'; import { ImportWorkflowsCommand } from '@/commands/import/workflow'; import * as Config from '@oclif/config'; +import { LoggerProxy } from 'n8n-workflow'; +import { getLogger } from '@/Logger'; + +LoggerProxy.init(getLogger()); + beforeAll(async () => { mockInstance(InternalHooks); await testDb.init(); diff --git a/packages/cli/test/integration/credentials.controller.test.ts b/packages/cli/test/integration/credentials.controller.test.ts index cab87149b12f3..1b55c71d46c74 100644 --- a/packages/cli/test/integration/credentials.controller.test.ts +++ b/packages/cli/test/integration/credentials.controller.test.ts @@ -12,11 +12,6 @@ const testServer = utils.setupTestServer({ endpointGroups: ['credentials'] }); let owner: User; let member: User; -import { LoggerProxy } from 'n8n-workflow'; -import { getLogger } from '@/Logger'; - -LoggerProxy.init(getLogger()); - beforeEach(async () => { await testDb.truncate(['SharedCredentials', 'Credentials']); diff --git a/packages/cli/test/integration/credentials.ee.test.ts b/packages/cli/test/integration/credentials.ee.test.ts index e9fa4f088f268..a1dc4f7be00b5 100644 --- a/packages/cli/test/integration/credentials.ee.test.ts +++ b/packages/cli/test/integration/credentials.ee.test.ts @@ -14,11 +14,6 @@ import * as testDb from './shared/testDb'; import type { SaveCredentialFunction } from './shared/types'; import * as utils from './shared/utils/'; -import { LoggerProxy } from 'n8n-workflow'; -import { getLogger } from '@/Logger'; - -LoggerProxy.init(getLogger()); - const sharingSpy = jest.spyOn(UserManagementHelpers, 'isSharingEnabled').mockReturnValue(true); const testServer = utils.setupTestServer({ endpointGroups: ['credentials'] }); diff --git a/packages/cli/test/integration/credentials.test.ts b/packages/cli/test/integration/credentials.test.ts index 72c0aea730c2e..526d3bb28c77b 100644 --- a/packages/cli/test/integration/credentials.test.ts +++ b/packages/cli/test/integration/credentials.test.ts @@ -12,10 +12,6 @@ import { randomCredentialPayload, randomName, randomString } from './shared/rand import * as testDb from './shared/testDb'; import type { SaveCredentialFunction } from './shared/types'; import * as utils from './shared/utils/'; -import { LoggerProxy } from 'n8n-workflow'; -import { getLogger } from '@/Logger'; - -LoggerProxy.init(getLogger()); // mock that credentialsSharing is not enabled jest.spyOn(UserManagementHelpers, 'isSharingEnabled').mockReturnValue(false); diff --git a/packages/cli/test/integration/executions.controller.test.ts b/packages/cli/test/integration/executions.controller.test.ts index bc2f783674a1c..2af9ec5b8597d 100644 --- a/packages/cli/test/integration/executions.controller.test.ts +++ b/packages/cli/test/integration/executions.controller.test.ts @@ -1,10 +1,6 @@ import * as testDb from './shared/testDb'; import { setupTestServer } from './shared/utils'; import type { User } from '@/databases/entities/User'; -import { LoggerProxy } from 'n8n-workflow'; -import { getLogger } from '@/Logger'; - -LoggerProxy.init(getLogger()); let testServer = setupTestServer({ endpointGroups: ['executions'] }); diff --git a/packages/cli/test/integration/ldap/ldap.api.test.ts b/packages/cli/test/integration/ldap/ldap.api.test.ts index 8b00566d1187d..1a3e23cd8ccf8 100644 --- a/packages/cli/test/integration/ldap/ldap.api.test.ts +++ b/packages/cli/test/integration/ldap/ldap.api.test.ts @@ -17,6 +17,9 @@ import { randomEmail, randomName, uniqueId } from './../shared/random'; import * as testDb from './../shared/testDb'; import * as utils from '../shared/utils/'; +import { LoggerProxy } from 'n8n-workflow'; +import { getLogger } from '@/Logger'; + jest.mock('@/telemetry'); jest.mock('@/UserManagement/email/NodeMailer'); @@ -24,6 +27,8 @@ let globalMemberRole: Role; let owner: User; let authOwnerAgent: SuperAgentTest; +LoggerProxy.init(getLogger()); + const defaultLdapConfig = { ...LDAP_DEFAULT_CONFIGURATION, loginEnabled: true, diff --git a/packages/cli/test/integration/publicApi/credentials.test.ts b/packages/cli/test/integration/publicApi/credentials.test.ts index fc576a9f7eed7..bc5c414142a17 100644 --- a/packages/cli/test/integration/publicApi/credentials.test.ts +++ b/packages/cli/test/integration/publicApi/credentials.test.ts @@ -10,11 +10,6 @@ import * as utils from '../shared/utils/'; import type { CredentialPayload, SaveCredentialFunction } from '../shared/types'; import * as testDb from '../shared/testDb'; -import { LoggerProxy } from 'n8n-workflow'; -import { getLogger } from '@/Logger'; - -LoggerProxy.init(getLogger()); - let globalMemberRole: Role; let credentialOwnerRole: Role; let owner: User; diff --git a/packages/cli/test/integration/publicApi/workflows.test.ts b/packages/cli/test/integration/publicApi/workflows.test.ts index 980986cf96d44..910bfbb3970c8 100644 --- a/packages/cli/test/integration/publicApi/workflows.test.ts +++ b/packages/cli/test/integration/publicApi/workflows.test.ts @@ -10,8 +10,6 @@ import * as utils from '../shared/utils/'; import * as testDb from '../shared/testDb'; import type { INode } from 'n8n-workflow'; import { STARTING_NODES } from '@/constants'; -import { LoggerProxy } from 'n8n-workflow'; -import { getLogger } from '@/Logger'; let workflowOwnerRole: Role; let owner: User; @@ -20,8 +18,6 @@ let authOwnerAgent: SuperAgentTest; let authMemberAgent: SuperAgentTest; let workflowRunner: ActiveWorkflowRunner; -LoggerProxy.init(getLogger()); - const testServer = utils.setupTestServer({ endpointGroups: ['publicApi'] }); beforeAll(async () => { diff --git a/packages/cli/test/integration/shared/utils/testServer.ts b/packages/cli/test/integration/shared/utils/testServer.ts index b33c3afb04387..21e381ff830ee 100644 --- a/packages/cli/test/integration/shared/utils/testServer.ts +++ b/packages/cli/test/integration/shared/utils/testServer.ts @@ -141,6 +141,9 @@ export const setupTestServer = ({ app.use(rawBodyReader); app.use(cookieParser()); + const logger = getLogger(); + LoggerProxy.init(logger); + const testServer: TestServer = { app, httpServer: app.listen(0), @@ -152,9 +155,6 @@ export const setupTestServer = ({ beforeAll(async () => { await testDb.init(); - const logger = getLogger(); - LoggerProxy.init(logger); - // Mock all telemetry. mockInstance(InternalHooks); mockInstance(PostHogClient); diff --git a/packages/cli/test/integration/workflows.controller.ee.test.ts b/packages/cli/test/integration/workflows.controller.ee.test.ts index 4833c36c1e2e6..6c63bbc8ae9ff 100644 --- a/packages/cli/test/integration/workflows.controller.ee.test.ts +++ b/packages/cli/test/integration/workflows.controller.ee.test.ts @@ -13,9 +13,6 @@ import type { SaveCredentialFunction } from './shared/types'; import { makeWorkflow } from './shared/utils/'; import { randomCredentialPayload } from './shared/random'; -import { LoggerProxy } from 'n8n-workflow'; -import { getLogger } from '@/Logger'; - let owner: User; let member: User; let anotherMember: User; @@ -24,8 +21,6 @@ let authMemberAgent: SuperAgentTest; let authAnotherMemberAgent: SuperAgentTest; let saveCredential: SaveCredentialFunction; -LoggerProxy.init(getLogger()); - const sharingSpy = jest.spyOn(UserManagementHelpers, 'isSharingEnabled').mockReturnValue(true); const testServer = utils.setupTestServer({ endpointGroups: ['workflows'], diff --git a/packages/cli/test/integration/workflows.controller.test.ts b/packages/cli/test/integration/workflows.controller.test.ts index db4298b933503..fb56237fd7a6b 100644 --- a/packages/cli/test/integration/workflows.controller.test.ts +++ b/packages/cli/test/integration/workflows.controller.test.ts @@ -12,14 +12,9 @@ import { RoleService } from '@/services/role.service'; import Container from 'typedi'; import type { ListQuery } from '@/requests'; -import { LoggerProxy } from 'n8n-workflow'; -import { getLogger } from '@/Logger'; - let owner: User; let authOwnerAgent: SuperAgentTest; -LoggerProxy.init(getLogger()); - jest.spyOn(UserManagementHelpers, 'isSharingEnabled').mockReturnValue(false); const testServer = utils.setupTestServer({ endpointGroups: ['workflows'] }); diff --git a/packages/cli/test/unit/PermissionChecker.test.ts b/packages/cli/test/unit/PermissionChecker.test.ts index ab1443c0c7ee3..139cf0571003b 100644 --- a/packages/cli/test/unit/PermissionChecker.test.ts +++ b/packages/cli/test/unit/PermissionChecker.test.ts @@ -25,6 +25,11 @@ import type { SaveCredentialFunction } from '../integration/shared/types'; import { mockInstance } from '../integration/shared/utils/'; import { OwnershipService } from '@/services/ownership.service'; +import { LoggerProxy } from 'n8n-workflow'; +import { getLogger } from '@/Logger'; + +LoggerProxy.init(getLogger()); + let mockNodeTypes: INodeTypes; let credentialOwnerRole: Role; let workflowOwnerRole: Role; From 80a4fc509adf65cb17b7a5b76bb6d5a7a03649cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 20 Sep 2023 19:39:31 +0200 Subject: [PATCH 094/103] Fix misresolved conflict --- .../cli/src/databases/repositories/execution.repository.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 23ea9d30ab7e4..051d206b54dd8 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -521,8 +521,7 @@ export class ExecutionRepository extends Repository { }) ).map(({ id }) => id); - const binaryDataManager = BinaryDataManager.getInstance(); - await binaryDataManager.deleteBinaryDataByExecutionIds(executionIds); + await this.binaryDataService.deleteManyByExecutionIds(executionIds); this.logger.debug(`Hard-deleting ${executionIds.length} executions from database`, { executionIds, From 20d5ea78494d75f4283eeba7c8f7d40f5a3465f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 20 Sep 2023 19:44:09 +0200 Subject: [PATCH 095/103] Remove excess check on init --- packages/core/src/BinaryData/BinaryData.service.ts | 2 +- packages/core/src/BinaryData/types.ts | 11 +++-------- packages/core/test/NodeExecuteFunctions.test.ts | 1 + 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/packages/core/src/BinaryData/BinaryData.service.ts b/packages/core/src/BinaryData/BinaryData.service.ts index 629022ac36fd8..5540e2c2c4061 100644 --- a/packages/core/src/BinaryData/BinaryData.service.ts +++ b/packages/core/src/BinaryData/BinaryData.service.ts @@ -25,7 +25,7 @@ export class BinaryDataService { this.availableModes = config.availableModes; this.mode = config.mode; - if (this.availableModes.includes('filesystem') && config.mode === 'filesystem') { + if (this.availableModes.includes('filesystem')) { this.managers.filesystem = new FileSystemManager(config.localStoragePath); await this.managers.filesystem.init(); diff --git a/packages/core/src/BinaryData/types.ts b/packages/core/src/BinaryData/types.ts index c843f14d30188..e6bc3f6cedc43 100644 --- a/packages/core/src/BinaryData/types.ts +++ b/packages/core/src/BinaryData/types.ts @@ -5,17 +5,12 @@ import type { BINARY_DATA_MODES } from './utils'; export namespace BinaryData { export type Mode = (typeof BINARY_DATA_MODES)[number]; - type ConfigBase = { - mode: Mode; + export type Config = { + mode: 'default' | 'filesystem'; availableModes: string[]; + localStoragePath: string; }; - type InMemoryConfig = ConfigBase & { mode: 'default' }; - - export type FileSystemConfig = ConfigBase & { mode: 'filesystem'; localStoragePath: string }; - - export type Config = InMemoryConfig | FileSystemConfig; - export interface Manager { init(): Promise; diff --git a/packages/core/test/NodeExecuteFunctions.test.ts b/packages/core/test/NodeExecuteFunctions.test.ts index fa421392327c6..fc15729362807 100644 --- a/packages/core/test/NodeExecuteFunctions.test.ts +++ b/packages/core/test/NodeExecuteFunctions.test.ts @@ -33,6 +33,7 @@ describe('NodeExecuteFunctions', () => { await Container.get(BinaryDataService).init({ mode: 'default', availableModes: ['default'], + localStoragePath: temporaryDir, }); // Set our binary data buffer From 0f84eb689a2ab3713534d0ddd016fe0c0ea3c785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 21 Sep 2023 09:16:41 +0200 Subject: [PATCH 096/103] Remove `ObjectStore.manager.ts` stub --- .../src/BinaryData/ObjectStore.manager.ts | 55 ------------------- 1 file changed, 55 deletions(-) delete mode 100644 packages/core/src/BinaryData/ObjectStore.manager.ts diff --git a/packages/core/src/BinaryData/ObjectStore.manager.ts b/packages/core/src/BinaryData/ObjectStore.manager.ts deleted file mode 100644 index 685f7a967a06b..0000000000000 --- a/packages/core/src/BinaryData/ObjectStore.manager.ts +++ /dev/null @@ -1,55 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ - -import type { BinaryMetadata } from 'n8n-workflow'; -import type { Readable } from 'stream'; -import type { BinaryData } from './types'; - -export class ObjectStoreManager implements BinaryData.Manager { - async init() { - throw new Error('TODO'); - } - - async store(binaryData: Buffer | Readable, executionId: string): Promise { - throw new Error('TODO'); - } - - getPath(identifier: string): string { - throw new Error('TODO'); - } - - async getSize(path: string): Promise { - throw new Error('TODO'); - } - - async getBuffer(identifier: string): Promise { - throw new Error('TODO'); - } - - getStream(identifier: string, chunkSize?: number): Readable { - throw new Error('TODO'); - } - - async storeMetadata(identifier: string, metadata: BinaryMetadata): Promise { - throw new Error('TODO'); - } - - async getMetadata(identifier: string): Promise { - throw new Error('TODO'); - } - - async copyByPath(path: string, executionId: string): Promise { - throw new Error('TODO'); - } - - async copyByIdentifier(identifier: string, executionId: string): Promise { - throw new Error('TODO'); - } - - async deleteOne(identifier: string): Promise { - throw new Error('TODO'); - } - - async deleteManyByExecutionIds(executionIds: string[]): Promise { - throw new Error('TODO'); - } -} From 605d02b91a47598087061d4caf1fd1658b4ccd4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 21 Sep 2023 10:29:57 +0200 Subject: [PATCH 097/103] Revert rename to `BinaryDataManager` --- packages/cli/src/Server.ts | 2 +- packages/cli/src/WorkflowRunnerProcess.ts | 2 +- packages/cli/src/commands/BaseCommand.ts | 2 +- packages/cli/src/config/schema.ts | 2 +- packages/cli/src/config/types.ts | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/Server.ts b/packages/cli/src/Server.ts index 9e0bbec7b9cf0..ee3a67d7aaec5 100644 --- a/packages/cli/src/Server.ts +++ b/packages/cli/src/Server.ts @@ -366,7 +366,7 @@ export class Server extends AbstractServer { LoggerProxy.debug(`Server ID: ${this.uniqueInstanceId}`); const cpus = os.cpus(); - const binaryDataConfig = config.getEnv('binaryDataService'); + const binaryDataConfig = config.getEnv('binaryDataManager'); const diagnosticInfo: IDiagnosticInfo = { databaseType: config.getEnv('database.type'), disableProductionWebhooksOnMainProcess: config.getEnv( diff --git a/packages/cli/src/WorkflowRunnerProcess.ts b/packages/cli/src/WorkflowRunnerProcess.ts index ee1b95932d714..4266f0facf6fe 100644 --- a/packages/cli/src/WorkflowRunnerProcess.ts +++ b/packages/cli/src/WorkflowRunnerProcess.ts @@ -123,7 +123,7 @@ class WorkflowRunnerProcess { await Container.get(PostHogClient).init(instanceId); await Container.get(InternalHooks).init(instanceId); - const binaryDataConfig = config.getEnv('binaryDataService'); + const binaryDataConfig = config.getEnv('binaryDataManager'); await Container.get(BinaryDataService).init(binaryDataConfig); const license = Container.get(License); diff --git a/packages/cli/src/commands/BaseCommand.ts b/packages/cli/src/commands/BaseCommand.ts index 17a95a155f6fe..17cc38c1f08c7 100644 --- a/packages/cli/src/commands/BaseCommand.ts +++ b/packages/cli/src/commands/BaseCommand.ts @@ -104,7 +104,7 @@ export abstract class BaseCommand extends Command { } async initBinaryDataService() { - const binaryDataConfig = config.getEnv('binaryDataService'); + const binaryDataConfig = config.getEnv('binaryDataManager'); await Container.get(BinaryDataService).init(binaryDataConfig); } diff --git a/packages/cli/src/config/schema.ts b/packages/cli/src/config/schema.ts index 2529354c15334..bf7e7750c179d 100644 --- a/packages/cli/src/config/schema.ts +++ b/packages/cli/src/config/schema.ts @@ -893,7 +893,7 @@ export const schema = { }, }, - binaryDataService: { + binaryDataManager: { availableModes: { format: 'comma-separated-list', default: 'filesystem', diff --git a/packages/cli/src/config/types.ts b/packages/cli/src/config/types.ts index 6b87c8016e3fd..b5646ff5ae8fb 100644 --- a/packages/cli/src/config/types.ts +++ b/packages/cli/src/config/types.ts @@ -76,7 +76,7 @@ type ToReturnType = T extends NumericPath type ExceptionPaths = { 'queue.bull.redis': object; - binaryDataService: BinaryData.Config; + binaryDataManager: BinaryData.Config; 'nodes.exclude': string[] | undefined; 'nodes.include': string[] | undefined; 'userManagement.isInstanceOwnerSetUp': boolean; From c430f22a09e46566456b8cbfd9b75389730ea689 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 21 Sep 2023 10:59:15 +0200 Subject: [PATCH 098/103] Missing renaming in test --- packages/cli/test/integration/shared/utils/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/test/integration/shared/utils/index.ts b/packages/cli/test/integration/shared/utils/index.ts index c4412119b6416..ee4b1e57509f3 100644 --- a/packages/cli/test/integration/shared/utils/index.ts +++ b/packages/cli/test/integration/shared/utils/index.ts @@ -77,7 +77,7 @@ export async function initNodeTypes() { export async function initBinaryDataService() { const binaryDataService = new BinaryDataService(); - await binaryDataService.init(config.getEnv('binaryDataService')); + await binaryDataService.init(config.getEnv('binaryDataManager')); Container.set(BinaryDataService, binaryDataService); } From 0cdfb7901f49bcc4cec7ae428885b6fafaef6de0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Fri, 22 Sep 2023 11:31:19 +0200 Subject: [PATCH 099/103] Rename constant --- packages/core/src/BinaryData/utils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/BinaryData/utils.ts b/packages/core/src/BinaryData/utils.ts index 05dc84dc6308c..7398683e32e9b 100644 --- a/packages/core/src/BinaryData/utils.ts +++ b/packages/core/src/BinaryData/utils.ts @@ -4,9 +4,9 @@ import type { BinaryData } from './types'; * Modes for storing binary data: * - `default` (in memory) * - `filesystem` (on disk) - * - `object` (S3) + * - `s3` (S3-compatible storage) */ -export const BINARY_DATA_MODES = ['default', 'filesystem', 'object'] as const; +export const BINARY_DATA_MODES = ['default', 'filesystem', 's3'] as const; export function areValidModes(modes: string[]): modes is BinaryData.Mode[] { return modes.every((m) => BINARY_DATA_MODES.includes(m as BinaryData.Mode)); From 61c2f5a25345cee129102f54da7316dc301b9fbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Fri, 22 Sep 2023 12:05:58 +0200 Subject: [PATCH 100/103] Restore `LogCatch` `LogCatch` was merged into master at a different file: `/packages/core/binaryData/index.ts`, which is now at `BinaryData.service.ts` --- packages/core/src/BinaryData/BinaryData.service.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/core/src/BinaryData/BinaryData.service.ts b/packages/core/src/BinaryData/BinaryData.service.ts index 5540e2c2c4061..f16850c600821 100644 --- a/packages/core/src/BinaryData/BinaryData.service.ts +++ b/packages/core/src/BinaryData/BinaryData.service.ts @@ -2,14 +2,15 @@ import { readFile, stat } from 'fs/promises'; import concatStream from 'concat-stream'; import prettyBytes from 'pretty-bytes'; import { Service } from 'typedi'; -import { BINARY_ENCODING } from 'n8n-workflow'; +import { BINARY_ENCODING, LoggerProxy as Logger, IBinaryData } from 'n8n-workflow'; import { FileSystemManager } from './FileSystem.manager'; import { InvalidBinaryDataManagerError, InvalidBinaryDataModeError, areValidModes } from './utils'; import type { Readable } from 'stream'; import type { BinaryData } from './types'; -import type { IBinaryData, INodeExecutionData } from 'n8n-workflow'; +import type { INodeExecutionData } from 'n8n-workflow'; +import { LogCatch } from '../decorators/LogCatch.decorator'; @Service() export class BinaryDataService { @@ -32,6 +33,7 @@ export class BinaryDataService { } } + @LogCatch((error) => Logger.error('Failed to copy binary data file', { error })) async copyBinaryFile(binaryData: IBinaryData, path: string, executionId: string) { const manager = this.managers[this.mode]; @@ -59,6 +61,7 @@ export class BinaryDataService { return binaryData; } + @LogCatch((error) => Logger.error('Failed to write binary data file', { error })) async store(binaryData: IBinaryData, input: Buffer | Readable, executionId: string) { const manager = this.managers[this.mode]; @@ -127,6 +130,9 @@ export class BinaryDataService { await this.getManager(this.mode).deleteManyByExecutionIds(executionIds); } + @LogCatch((error) => + Logger.error('Failed to copy all binary data files for execution', { error }), + ) async duplicateBinaryData(inputData: Array, executionId: string) { if (inputData && this.managers[this.mode]) { const returnInputData = (inputData as INodeExecutionData[][]).map( From 74adb24ba7d1f4c0cebb4fcf20c1713b6b23697b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Fri, 22 Sep 2023 12:35:44 +0200 Subject: [PATCH 101/103] Add file not found error to `getSize` --- packages/core/src/BinaryData/FileSystem.manager.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/core/src/BinaryData/FileSystem.manager.ts b/packages/core/src/BinaryData/FileSystem.manager.ts index 1e00774382fd0..84a86716ddbad 100644 --- a/packages/core/src/BinaryData/FileSystem.manager.ts +++ b/packages/core/src/BinaryData/FileSystem.manager.ts @@ -26,9 +26,13 @@ export class FileSystemManager implements BinaryData.Manager { async getSize(identifier: string) { const filePath = this.getPath(identifier); - const stats = await fs.stat(filePath); - return stats.size; + try { + const stats = await fs.stat(filePath); + return stats.size; + } catch (error) { + throw new Error('Failed to find binary data file in filesystem', { cause: error }); + } } getStream(identifier: string, chunkSize?: number) { From 0bc2457d76f373c9c008a10b0b36e0fd2ef9b8e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Fri, 22 Sep 2023 13:14:27 +0200 Subject: [PATCH 102/103] Add `mode` to `InvalidBinaryDataManagerError` --- packages/core/src/BinaryData/BinaryData.service.ts | 2 +- packages/core/src/BinaryData/utils.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/core/src/BinaryData/BinaryData.service.ts b/packages/core/src/BinaryData/BinaryData.service.ts index f16850c600821..42859ea5964eb 100644 --- a/packages/core/src/BinaryData/BinaryData.service.ts +++ b/packages/core/src/BinaryData/BinaryData.service.ts @@ -218,6 +218,6 @@ export class BinaryDataService { if (manager) return manager; - throw new InvalidBinaryDataManagerError(); + throw new InvalidBinaryDataManagerError(mode); } } diff --git a/packages/core/src/BinaryData/utils.ts b/packages/core/src/BinaryData/utils.ts index 7398683e32e9b..c2bea73850f85 100644 --- a/packages/core/src/BinaryData/utils.ts +++ b/packages/core/src/BinaryData/utils.ts @@ -19,7 +19,7 @@ export class InvalidBinaryDataModeError extends Error { } export class InvalidBinaryDataManagerError extends Error { - constructor() { - super('No binary data manager found'); + constructor(mode: string) { + super('No binary data manager found for mode: ' + mode); } } From d27e4c4467c2a264fdede5c0e01d691f679fa24b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Fri, 22 Sep 2023 15:20:41 +0200 Subject: [PATCH 103/103] Delete many by execution IDs only if manager available --- packages/core/src/BinaryData/BinaryData.service.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/core/src/BinaryData/BinaryData.service.ts b/packages/core/src/BinaryData/BinaryData.service.ts index 42859ea5964eb..cd5b65816c9cc 100644 --- a/packages/core/src/BinaryData/BinaryData.service.ts +++ b/packages/core/src/BinaryData/BinaryData.service.ts @@ -127,7 +127,11 @@ export class BinaryDataService { } async deleteManyByExecutionIds(executionIds: string[]) { - await this.getManager(this.mode).deleteManyByExecutionIds(executionIds); + const manager = this.getManager(this.mode); + + if (!manager) return; + + await manager.deleteManyByExecutionIds(executionIds); } @LogCatch((error) =>