Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(core): Port pruning config #11507

Merged
merged 3 commits into from
Nov 4, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions packages/@n8n/config/src/configs/pruning.config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
import { Config, Env } from '../decorators';

@Config
export class PruningConfig {
/** Whether to delete past executions on a rolling basis. */
@Env('EXECUTIONS_DATA_PRUNE')
isEnabled: boolean = true;

/** How old (hours) a finished execution must be to qualify for soft-deletion. */
@Env('EXECUTIONS_DATA_MAX_AGE')
maxAge: number = 336;

/**
* Max number of finished executions to keep in database. Does not necessarily
* prune to the exact max number. `0` for unlimited.
*/
@Env('EXECUTIONS_DATA_PRUNE_MAX_COUNT')
maxCount: number = 10_000;

/**
* How old (hours) a finished execution must be to qualify for hard-deletion.
* This buffer by default excludes recent executions as the user may need
* them while building a workflow.
*/
@Env('EXECUTIONS_DATA_HARD_DELETE_BUFFER')
hardDeleteBuffer: number = 1;

/** How often (minutes) execution data should be hard-deleted. */
@Env('EXECUTIONS_DATA_PRUNE_HARD_DELETE_INTERVAL')
hardDeleteInterval: number = 15;

/** How often (minutes) execution data should be soft-deleted */
@Env('EXECUTIONS_DATA_PRUNE_SOFT_DELETE_INTERVAL')
softDeleteInterval: number = 60;
}
5 changes: 5 additions & 0 deletions packages/@n8n/config/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { LicenseConfig } from './configs/license.config';
import { LoggingConfig } from './configs/logging.config';
import { MultiMainSetupConfig } from './configs/multi-main-setup.config';
import { NodesConfig } from './configs/nodes.config';
import { PruningConfig } from './configs/pruning.config';
import { PublicApiConfig } from './configs/public-api.config';
import { TaskRunnersConfig } from './configs/runners.config';
import { ScalingModeConfig } from './configs/scaling-mode.config';
Expand All @@ -24,6 +25,7 @@ import { Config, Env, Nested } from './decorators';
export { Config, Env, Nested } from './decorators';
export { TaskRunnersConfig } from './configs/runners.config';
export { SecurityConfig } from './configs/security.config';
export { PruningConfig } from './configs/pruning.config';
export { FrontendBetaFeatures, FrontendConfig } from './configs/frontend.config';
export { LOG_SCOPES } from './configs/logging.config';
export type { LogScope } from './configs/logging.config';
Expand Down Expand Up @@ -112,4 +114,7 @@ export class GlobalConfig {

@Nested
security: SecurityConfig;

@Nested
pruning: PruningConfig;
}
8 changes: 8 additions & 0 deletions packages/@n8n/config/test/config.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,14 @@ describe('GlobalConfig', () => {
blockFileAccessToN8nFiles: true,
daysAbandonedWorkflow: 90,
},
pruning: {
isEnabled: true,
maxAge: 336,
maxCount: 10_000,
hardDeleteBuffer: 1,
hardDeleteInterval: 15,
softDeleteInterval: 60,
},
};

it('should use all default values when no env variables are defined', () => {
Expand Down
48 changes: 0 additions & 48 deletions packages/cli/src/config/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,54 +98,6 @@ export const schema = {
env: 'EXECUTIONS_DATA_SAVE_MANUAL_EXECUTIONS',
},

// To not exceed the database's capacity and keep its size moderate
// the execution data gets pruned regularly (default: 15 minute interval).
// All saved execution data older than the max age will be deleted.
// Pruning is currently not activated by default, which will change in
// a future version.
pruneData: {
doc: 'Delete data of past executions on a rolling basis',
format: Boolean,
default: true,
env: 'EXECUTIONS_DATA_PRUNE',
},
pruneDataMaxAge: {
doc: 'How old (hours) the finished execution data has to be to get soft-deleted',
format: Number,
default: 336,
env: 'EXECUTIONS_DATA_MAX_AGE',
},
pruneDataHardDeleteBuffer: {
doc: 'How old (hours) the finished execution data has to be to get hard-deleted. By default, this buffer excludes recent executions as the user may need them while building a workflow.',
format: Number,
default: 1,
env: 'EXECUTIONS_DATA_HARD_DELETE_BUFFER',
},
pruneDataIntervals: {
hardDelete: {
doc: 'How often (minutes) execution data should be hard-deleted',
format: Number,
default: 15,
env: 'EXECUTIONS_DATA_PRUNE_HARD_DELETE_INTERVAL',
},
softDelete: {
doc: 'How often (minutes) execution data should be soft-deleted',
format: Number,
default: 60,
env: 'EXECUTIONS_DATA_PRUNE_SOFT_DELETE_INTERVAL',
},
},

// Additional pruning option to delete executions if total count exceeds the configured max.
// Deletes the oldest entries first
// Set to 0 for No limit
pruneDataMaxCount: {
doc: "Maximum number of finished executions to keep in DB. Doesn't necessarily prune exactly to max number. 0 = no limit",
format: Number,
default: 10000,
env: 'EXECUTIONS_DATA_PRUNE_MAX_COUNT',
},

queueRecovery: {
interval: {
doc: 'How often (minutes) to check for queue recovery',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ import type {
} from 'n8n-workflow';
import { Service } from 'typedi';

import config from '@/config';
import { AnnotationTagEntity } from '@/databases/entities/annotation-tag-entity.ee';
import { AnnotationTagMapping } from '@/databases/entities/annotation-tag-mapping.ee';
import { ExecutionAnnotation } from '@/databases/entities/execution-annotation.ee';
Expand Down Expand Up @@ -460,8 +459,7 @@ export class ExecutionRepository extends Repository<ExecutionEntity> {
}

async softDeletePrunableExecutions() {
const maxAge = config.getEnv('executions.pruneDataMaxAge'); // in h
const maxCount = config.getEnv('executions.pruneDataMaxCount');
const { maxAge, maxCount } = this.globalConfig.pruning;

// Sub-query to exclude executions having annotations
const annotatedExecutionsSubQuery = this.manager
Expand Down Expand Up @@ -517,7 +515,7 @@ export class ExecutionRepository extends Repository<ExecutionEntity> {

async hardDeleteSoftDeletedExecutions() {
const date = new Date();
date.setHours(date.getHours() - config.getEnv('executions.pruneDataHardDeleteBuffer'));
date.setHours(date.getHours() - this.globalConfig.pruning.hardDeleteBuffer);

const workflowIdsAndExecutionIds = (
await this.find({
Expand Down
4 changes: 2 additions & 2 deletions packages/cli/src/events/relays/telemetry.event-relay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -771,8 +771,8 @@ export class TelemetryEventRelay extends EventRelay {
executions_data_save_manual_executions: config.getEnv(
'executions.saveDataManualExecutions',
),
executions_data_prune: config.getEnv('executions.pruneData'),
executions_data_max_age: config.getEnv('executions.pruneDataMaxAge'),
executions_data_prune: this.globalConfig.pruning.isEnabled,
executions_data_max_age: this.globalConfig.pruning.maxAge,
},
n8n_deployment_type: config.getEnv('deployment.type'),
n8n_binary_data_mode: binaryDataConfig.mode,
Expand Down
6 changes: 3 additions & 3 deletions packages/cli/src/services/frontend.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -222,9 +222,9 @@ export class FrontendService {
licensePruneTime: -1,
},
pruning: {
isEnabled: config.getEnv('executions.pruneData'),
maxAge: config.getEnv('executions.pruneDataMaxAge'),
maxCount: config.getEnv('executions.pruneDataMaxCount'),
isEnabled: this.globalConfig.pruning.isEnabled,
maxAge: this.globalConfig.pruning.maxAge,
maxCount: this.globalConfig.pruning.maxCount,
},
security: {
blockFileAccessToN8nFiles: this.securityConfig.blockFileAccessToN8nFiles,
Expand Down
7 changes: 3 additions & 4 deletions packages/cli/src/services/pruning.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import { BinaryDataService, InstanceSettings } from 'n8n-core';
import { jsonStringify } from 'n8n-workflow';
import { Service } from 'typedi';

import config from '@/config';
import { inTest, TIME } from '@/constants';
import { ExecutionRepository } from '@/databases/repositories/execution.repository';
import { OnShutdown } from '@/decorators/on-shutdown';
Expand All @@ -16,8 +15,8 @@ export class PruningService {
private hardDeletionBatchSize = 100;

private rates: Record<string, number> = {
softDeletion: config.getEnv('executions.pruneDataIntervals.softDelete') * TIME.MINUTE,
hardDeletion: config.getEnv('executions.pruneDataIntervals.hardDelete') * TIME.MINUTE,
softDeletion: this.globalConfig.pruning.softDeleteInterval * TIME.MINUTE,
hardDeletion: this.globalConfig.pruning.hardDeleteInterval * TIME.MINUTE,
};

public softDeletionInterval: NodeJS.Timer | undefined;
Expand Down Expand Up @@ -52,7 +51,7 @@ export class PruningService {

private isPruningEnabled() {
const { instanceType, isFollower } = this.instanceSettings;
if (!config.getEnv('executions.pruneData') || inTest || instanceType !== 'main') {
if (!this.globalConfig.pruning.isEnabled || inTest || instanceType !== 'main') {
return false;
}

Expand Down
22 changes: 10 additions & 12 deletions packages/cli/test/integration/pruning.service.test.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import { GlobalConfig } from '@n8n/config';
import { mock } from 'jest-mock-extended';
import { BinaryDataService, InstanceSettings } from 'n8n-core';
import type { ExecutionStatus } from 'n8n-workflow';
import Container from 'typedi';

import config from '@/config';
import { TIME } from '@/constants';
import type { ExecutionEntity } from '@/databases/entities/execution-entity';
import type { WorkflowEntity } from '@/databases/entities/workflow-entity';
Expand All @@ -28,17 +28,19 @@ describe('softDeleteOnPruningCycle()', () => {
const now = new Date();
const yesterday = new Date(Date.now() - TIME.DAY);
let workflow: WorkflowEntity;
let globalConfig: GlobalConfig;

beforeAll(async () => {
await testDb.init();

globalConfig = Container.get(GlobalConfig);
pruningService = new PruningService(
mockInstance(Logger),
instanceSettings,
Container.get(ExecutionRepository),
mockInstance(BinaryDataService),
mock(),
mock(),
globalConfig,
);

workflow = await createWorkflow();
Expand All @@ -52,10 +54,6 @@ describe('softDeleteOnPruningCycle()', () => {
await testDb.terminate();
});

afterEach(() => {
config.load(config.default);
});

async function findAllExecutions() {
return await Container.get(ExecutionRepository).find({
order: { id: 'asc' },
Expand All @@ -64,9 +62,9 @@ describe('softDeleteOnPruningCycle()', () => {
}

describe('when EXECUTIONS_DATA_PRUNE_MAX_COUNT is set', () => {
beforeEach(() => {
config.set('executions.pruneDataMaxCount', 1);
config.set('executions.pruneDataMaxAge', 336);
beforeAll(() => {
globalConfig.pruning.maxAge = 336;
globalConfig.pruning.maxCount = 1;
});

test('should mark as deleted based on EXECUTIONS_DATA_PRUNE_MAX_COUNT', async () => {
Expand Down Expand Up @@ -165,9 +163,9 @@ describe('softDeleteOnPruningCycle()', () => {
});

describe('when EXECUTIONS_DATA_MAX_AGE is set', () => {
beforeEach(() => {
config.set('executions.pruneDataMaxAge', 1); // 1h
config.set('executions.pruneDataMaxCount', 0);
beforeAll(() => {
globalConfig.pruning.maxAge = 1;
globalConfig.pruning.maxCount = 0;
});

test('should mark as deleted based on EXECUTIONS_DATA_MAX_AGE', async () => {
Expand Down
Loading