Skip to content

Commit

Permalink
feat: Add variables feature (#5602)
Browse files Browse the repository at this point in the history
* feat: add variables db models and migrations

* feat: variables api endpoints

* feat: add $variables to expressions

* test: fix ActiveWorkflowRunner tests failing

* test: a different fix for the tests broken by $variables

* feat: variables licensing

* fix: could create one extra variable than licensed for

* feat: Add Variables UI page and $vars global property (#5750)

* feat: add support for row slot to datatable

* feat: add variables create, read, update, delete

* feat: add vars autocomplete

* chore: remove alert

* feat: add variables autocomplete for code and expressions

* feat: add tests for variable components

* feat: add variables search and sort

* test: update tests for variables view

* chore: fix test and linting issue

* refactor: review changes

* feat: add variable creation telemetry

* fix: Improve variables listing and disabled case, fix resource sorting (no-changelog) (#5903)

* fix: Improve variables disabled experience and fix sorting

* fix: update action box margin

* test: update tests for variables row and datatable

* fix: Add ee controller to base controller

* fix: variables.ee routes not being added

* feat: add variables validation

* fix: fix vue-fragment bug that breaks everything

* chore: Update lock

* feat: Add variables input validation and permissions (no-changelog) (#5910)

* feat: add input validation

* feat: handle variables view for non-instance-owner users

* test: update variables tests

* fix: fix data-testid pattern

* feat: improve overflow styles

* test: fix variables row snapshot

* feat: update sorting to take newly created variables into account

* fix: fix list layout overflow

* fix: fix adding variables on page other than 1. fix validation

* feat: add docs link

* fix: fix default displayName function for resource-list-layout

* feat: improve vars expressions ux, cm-tooltip

* test: fix datatable test

* feat: add MATCH_REGEX validation rule

* fix: overhaul how datatable pagination selector works

* feat: update  completer description

* fix: conditionally update usage syntax based on key validation

* test: update datatable snapshot

* fix: fix variables-row button margins

* fix: fix pagination overflow

* test: Fix broken test

* test: Update snapshot

* fix: Remove duplicate declaration

* feat: add custom variables icon

---------

Co-authored-by: Alex Grozav <[email protected]>
Co-authored-by: Omar Ajoue <[email protected]>
  • Loading branch information
3 people authored Apr 18, 2023
1 parent 1555387 commit 1bb9871
Show file tree
Hide file tree
Showing 94 changed files with 2,925 additions and 200 deletions.
2 changes: 2 additions & 0 deletions packages/cli/src/Db.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import {
SharedWorkflowRepository,
TagRepository,
UserRepository,
VariablesRepository,
WebhookRepository,
WorkflowRepository,
WorkflowStatisticsRepository,
Expand Down Expand Up @@ -178,6 +179,7 @@ export async function init(
collections.SharedWorkflow = Container.get(SharedWorkflowRepository);
collections.Tag = Container.get(TagRepository);
collections.User = Container.get(UserRepository);
collections.Variables = Container.get(VariablesRepository);
collections.Webhook = Container.get(WebhookRepository);
collections.Workflow = Container.get(WorkflowRepository);
collections.WorkflowStatistics = Container.get(WorkflowStatisticsRepository);
Expand Down
7 changes: 7 additions & 0 deletions packages/cli/src/Interfaces.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ import type {
SharedWorkflowRepository,
TagRepository,
UserRepository,
VariablesRepository,
WebhookRepository,
WorkflowRepository,
WorkflowStatisticsRepository,
Expand Down Expand Up @@ -99,6 +100,7 @@ export interface IDatabaseCollections {
SharedWorkflow: SharedWorkflowRepository;
Tag: TagRepository;
User: UserRepository;
Variables: VariablesRepository;
Webhook: WebhookRepository;
Workflow: WorkflowRepository;
WorkflowStatistics: WorkflowStatisticsRepository;
Expand Down Expand Up @@ -458,6 +460,7 @@ export interface IInternalHooksClass {
}): Promise<void>;
onApiKeyCreated(apiKeyDeletedData: { user: User; public_api: boolean }): Promise<void>;
onApiKeyDeleted(apiKeyDeletedData: { user: User; public_api: boolean }): Promise<void>;
onVariableCreated(createData: { variable_type: string }): Promise<void>;
}

export interface IVersionNotificationSettings {
Expand Down Expand Up @@ -538,11 +541,15 @@ export interface IN8nUISettings {
saml: boolean;
logStreaming: boolean;
advancedExecutionFilters: boolean;
variables: boolean;
};
hideUsagePage: boolean;
license: {
environment: 'production' | 'staging';
};
variables: {
limit: number;
};
}

export interface IPersonalizationSurveyAnswers {
Expand Down
4 changes: 4 additions & 0 deletions packages/cli/src/InternalHooks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -981,4 +981,8 @@ export class InternalHooks implements IInternalHooksClass {
async onAuditGeneratedViaCli() {
return this.telemetry.track('Instance generated security audit via CLI command');
}

async onVariableCreated(createData: { variable_type: string }): Promise<void> {
return this.telemetry.track('User created variable', createData);
}
}
17 changes: 15 additions & 2 deletions packages/cli/src/License.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,12 @@ import type { ILogger } from 'n8n-workflow';
import { getLogger } from './Logger';
import config from '@/config';
import * as Db from '@/Db';
import { LICENSE_FEATURES, N8N_VERSION, SETTINGS_LICENSE_CERT_KEY } from './constants';
import {
LICENSE_FEATURES,
LICENSE_QUOTAS,
N8N_VERSION,
SETTINGS_LICENSE_CERT_KEY,
} from './constants';
import { Service } from 'typedi';

async function loadCertStr(): Promise<TLicenseBlock> {
Expand Down Expand Up @@ -119,6 +124,10 @@ export class License {
return this.isFeatureEnabled(LICENSE_FEATURES.ADVANCED_EXECUTION_FILTERS);
}

isVariablesEnabled() {
return this.isFeatureEnabled(LICENSE_FEATURES.VARIABLES);
}

getCurrentEntitlements() {
return this.manager?.getCurrentEntitlements() ?? [];
}
Expand Down Expand Up @@ -162,7 +171,11 @@ export class License {

// Helper functions for computed data
getTriggerLimit(): number {
return (this.getFeatureValue('quota:activeWorkflows') ?? -1) as number;
return (this.getFeatureValue(LICENSE_QUOTAS.TRIGGER_LIMIT) ?? -1) as number;
}

getVariablesLimit(): number {
return (this.getFeatureValue(LICENSE_QUOTAS.VARIABLES_LIMIT) ?? -1) as number;
}

getPlanName(): string {
Expand Down
18 changes: 18 additions & 0 deletions packages/cli/src/Server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,9 @@ import {
import { getSamlLoginLabel, isSamlLoginEnabled, isSamlLicensed } from './sso/saml/samlHelpers';
import { SamlController } from './sso/saml/routes/saml.controller.ee';
import { SamlService } from './sso/saml/saml.service.ee';
import { variablesController } from './environments/variables.controller';
import { LdapManager } from './Ldap/LdapManager.ee';
import { getVariablesLimit, isVariablesEnabled } from '@/environments/enviromentHelpers';
import { getCurrentAuthenticationMethod } from './sso/ssoHelpers';

const exec = promisify(callbackExec);
Expand Down Expand Up @@ -317,11 +319,15 @@ class Server extends AbstractServer {
saml: false,
logStreaming: false,
advancedExecutionFilters: false,
variables: false,
},
hideUsagePage: config.getEnv('hideUsagePage'),
license: {
environment: config.getEnv('license.tenantId') === 1 ? 'production' : 'staging',
},
variables: {
limit: 0,
},
};
}

Expand All @@ -347,6 +353,7 @@ class Server extends AbstractServer {
ldap: isLdapEnabled(),
saml: isSamlLicensed(),
advancedExecutionFilters: isAdvancedExecutionFiltersEnabled(),
variables: isVariablesEnabled(),
});

if (isLdapEnabled()) {
Expand All @@ -363,6 +370,10 @@ class Server extends AbstractServer {
});
}

if (isVariablesEnabled()) {
this.frontendSettings.variables.limit = getVariablesLimit();
}

if (config.get('nodes.packagesMissing').length > 0) {
this.frontendSettings.missingPackages = true;
}
Expand Down Expand Up @@ -540,6 +551,13 @@ class Server extends AbstractServer {
}

// ----------------------------------------
// Variables
// ----------------------------------------

this.app.use(`/${this.restEndpoint}/variables`, variablesController);

// ----------------------------------------

// Returns parameter values which normally get loaded from an external API or
// get generated dynamically
this.app.get(
Expand Down
6 changes: 5 additions & 1 deletion packages/cli/src/WorkflowExecuteAdditionalData.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1164,7 +1164,10 @@ export async function getBase(
const webhookWaitingBaseUrl = urlBaseWebhook + config.getEnv('endpoints.webhookWaiting');
const webhookTestBaseUrl = urlBaseWebhook + config.getEnv('endpoints.webhookTest');

const encryptionKey = await UserSettings.getEncryptionKey();
const [encryptionKey, variables] = await Promise.all([
UserSettings.getEncryptionKey(),
WorkflowHelpers.getVariables(),
]);

return {
credentialsHelper: new CredentialsHelper(encryptionKey),
Expand All @@ -1179,6 +1182,7 @@ export async function getBase(
executionTimeoutTimestamp,
userId,
setExecutionStatus,
variables,
};
}

Expand Down
9 changes: 9 additions & 0 deletions packages/cli/src/WorkflowHelpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -562,3 +562,12 @@ export function validateWorkflowCredentialUsage(

return newWorkflowVersion;
}

export async function getVariables(): Promise<IDataObject> {
return Object.freeze(
(await Db.collections.Variables.find()).reduce((prev, curr) => {
prev[curr.key] = curr.value;
return prev;
}, {} as IDataObject),
);
}
6 changes: 6 additions & 0 deletions packages/cli/src/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,12 @@ export enum LICENSE_FEATURES {
SAML = 'feat:saml',
LOG_STREAMING = 'feat:logStreaming',
ADVANCED_EXECUTION_FILTERS = 'feat:advancedExecutionFilters',
VARIABLES = 'feat:variables',
}

export enum LICENSE_QUOTAS {
TRIGGER_LIMIT = 'quota:activeWorkflows',
VARIABLES_LIMIT = 'quota:maxVariables',
}

export const CREDENTIAL_BLANKING_VALUE = '__n8n_BLANK_VALUE_e5362baf-c777-4d57-a609-6eaf1f9e87f6';
16 changes: 16 additions & 0 deletions packages/cli/src/databases/entities/Variables.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
import { Column, Entity, PrimaryGeneratedColumn } from 'typeorm';

@Entity()
export class Variables {
@PrimaryGeneratedColumn()
id: number;

@Column('text')
key: string;

@Column('text', { default: 'string' })
type: string;

@Column('text')
value: string;
}
2 changes: 2 additions & 0 deletions packages/cli/src/databases/entities/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { SharedCredentials } from './SharedCredentials';
import { SharedWorkflow } from './SharedWorkflow';
import { TagEntity } from './TagEntity';
import { User } from './User';
import { Variables } from './Variables';
import { WebhookEntity } from './WebhookEntity';
import { WorkflowEntity } from './WorkflowEntity';
import { WorkflowTagMapping } from './WorkflowTagMapping';
Expand All @@ -32,6 +33,7 @@ export const entities = {
SharedWorkflow,
TagEntity,
User,
Variables,
WebhookEntity,
WorkflowEntity,
WorkflowTagMapping,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
import { logMigrationEnd, logMigrationStart, getTablePrefix } from '@db/utils/migrationHelpers';
import config from '@/config';

export class CreateVariables1677501636753 implements MigrationInterface {
name = 'CreateVariables1677501636753';
public async up(queryRunner: QueryRunner): Promise<void> {
logMigrationStart(this.name);
const tablePrefix = getTablePrefix();

await queryRunner.query(`
CREATE TABLE ${tablePrefix}variables (
id int(11) auto_increment NOT NULL PRIMARY KEY,
\`key\` VARCHAR(50) NOT NULL,
\`type\` VARCHAR(50) DEFAULT 'string' NOT NULL,
value VARCHAR(255) NULL,
UNIQUE (\`key\`)
)
ENGINE=InnoDB;
`);

logMigrationEnd(this.name);
}

public async down(queryRunner: QueryRunner): Promise<void> {
logMigrationStart(this.name);
const tablePrefix = getTablePrefix();

await queryRunner.query(`DROP TABLE ${tablePrefix}variables;`);

logMigrationEnd(this.name);
}
}
2 changes: 2 additions & 0 deletions packages/cli/src/databases/migrations/mysqldb/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import { AddStatusToExecutions1674138566000 } from './1674138566000-AddStatusToE
import { MigrateExecutionStatus1676996103000 } from './1676996103000-MigrateExecutionStatus';
import { UpdateRunningExecutionStatus1677236788851 } from './1677236788851-UpdateRunningExecutionStatus';
import { CreateExecutionMetadataTable1679416281779 } from './1679416281779-CreateExecutionMetadataTable';
import { CreateVariables1677501636753 } from './1677501636753-CreateVariables';

export const mysqlMigrations = [
InitialMigration1588157391238,
Expand Down Expand Up @@ -74,4 +75,5 @@ export const mysqlMigrations = [
MigrateExecutionStatus1676996103000,
UpdateRunningExecutionStatus1677236788851,
CreateExecutionMetadataTable1679416281779,
CreateVariables1677501636753,
];
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
import { logMigrationEnd, logMigrationStart, getTablePrefix } from '@db/utils/migrationHelpers';
import config from '@/config';

export class CreateVariables1677501636754 implements MigrationInterface {
name = 'CreateVariables1677501636754';
public async up(queryRunner: QueryRunner): Promise<void> {
logMigrationStart(this.name);
const tablePrefix = getTablePrefix();

await queryRunner.query(`
CREATE TABLE public.variables (
id serial4 NOT NULL PRIMARY KEY,
"key" varchar(50) NOT NULL,
"type" varchar(50) NOT NULL DEFAULT 'string',
value varchar(255) NULL,
UNIQUE ("key")
);
`);

logMigrationEnd(this.name);
}

public async down(queryRunner: QueryRunner): Promise<void> {
logMigrationStart(this.name);
const tablePrefix = getTablePrefix();

await queryRunner.query(`DROP TABLE ${tablePrefix}variables;`);

logMigrationEnd(this.name);
}
}
2 changes: 2 additions & 0 deletions packages/cli/src/databases/migrations/postgresdb/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import { AddStatusToExecutions1674138566000 } from './1674138566000-AddStatusToE
import { MigrateExecutionStatus1676996103000 } from './1676996103000-MigrateExecutionStatus';
import { UpdateRunningExecutionStatus1677236854063 } from './1677236854063-UpdateRunningExecutionStatus';
import { CreateExecutionMetadataTable1679416281778 } from './1679416281778-CreateExecutionMetadataTable';
import { CreateVariables1677501636754 } from './1677501636754-CreateVariables';

export const postgresMigrations = [
InitialMigration1587669153312,
Expand Down Expand Up @@ -70,4 +71,5 @@ export const postgresMigrations = [
MigrateExecutionStatus1676996103000,
UpdateRunningExecutionStatus1677236854063,
CreateExecutionMetadataTable1679416281778,
CreateVariables1677501636754,
];
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
import { logMigrationEnd, logMigrationStart, getTablePrefix } from '@db/utils/migrationHelpers';
import config from '@/config';

export class CreateVariables1677501636752 implements MigrationInterface {
name = 'CreateVariables1677501636752';
public async up(queryRunner: QueryRunner): Promise<void> {
logMigrationStart(this.name);
const tablePrefix = getTablePrefix();

await queryRunner.query(`
CREATE TABLE ${tablePrefix}variables (
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
"key" TEXT NOT NULL,
"type" TEXT NOT NULL DEFAULT ('string'),
value TEXT,
UNIQUE("key")
);
`);

logMigrationEnd(this.name);
}

public async down(queryRunner: QueryRunner): Promise<void> {
logMigrationStart(this.name);
const tablePrefix = getTablePrefix();

await queryRunner.query(`DROP TABLE ${tablePrefix}variables;`);

logMigrationEnd(this.name);
}
}
2 changes: 2 additions & 0 deletions packages/cli/src/databases/migrations/sqlite/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import { AddStatusToExecutions1674138566000 } from './1674138566000-AddStatusToE
import { MigrateExecutionStatus1676996103000 } from './1676996103000-MigrateExecutionStatus';
import { UpdateRunningExecutionStatus1677237073720 } from './1677237073720-UpdateRunningExecutionStatus';
import { CreateExecutionMetadataTable1679416281777 } from './1679416281777-CreateExecutionMetadataTable';
import { CreateVariables1677501636752 } from './1677501636752-CreateVariables';

const sqliteMigrations = [
InitialMigration1588102412422,
Expand Down Expand Up @@ -67,6 +68,7 @@ const sqliteMigrations = [
AddStatusToExecutions1674138566000,
MigrateExecutionStatus1676996103000,
UpdateRunningExecutionStatus1677237073720,
CreateVariables1677501636752,
CreateExecutionMetadataTable1679416281777,
];

Expand Down
1 change: 1 addition & 0 deletions packages/cli/src/databases/repositories/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ export { SharedCredentialsRepository } from './sharedCredentials.repository';
export { SharedWorkflowRepository } from './sharedWorkflow.repository';
export { TagRepository } from './tag.repository';
export { UserRepository } from './user.repository';
export { VariablesRepository } from './variables.repository';
export { WebhookRepository } from './webhook.repository';
export { WorkflowRepository } from './workflow.repository';
export { WorkflowStatisticsRepository } from './workflowStatistics.repository';
Expand Down
Loading

0 comments on commit 1bb9871

Please sign in to comment.