feat: Version control mvp (#6271)

* implement basic git service

* cleanup connected prop

* add skeleton of git functions

* initial import/export setup

* split out export service

* refactor and improve export

* begin import

* more commands and basic import

* clean up imports with transactions

* work folder import functions

* reintroduce versionid

* add missing import to pull workfolder

* add get-status endpoint

* add cleanup to disconnect

* add initRepo options

* add more checks and cleanup

* minor cleanup

* refactor prefs

* fix server.ts

* fix sending deleted files

* rename files to ee

* add variable override and fix critical cred import bug

* fix mkdir race condition

* make initRepo default to true

* fix front back integration

* improve connect flow

* add comment to generated ssh key

* fix(editor): use useToast composable

* fix buttons size

* commenting out repo init for now

* fix(editor): update UI logic

* fix(editor): remove console.log

* fix(editor): remove unused ref

* adjust endpoints for improved UI

* fix(editor): add push and pull buttons

* keep or not ssh key

* switching file name to id

* fix(editor): add success messages, fix save button

* fixed faulty diff preventing pull

* fix build

* fix(editor): adding loader to VC components

* removing duplicate exports

* improve conflict finding on push pull

* manage pull conflict

* alternate push pull

* fix pull confirmation

* fix rm and credential export/import

* switch to alternative pull implementation

* fix initial commit

* fix(editor): subscribing to VC store action to refresh lists

* fix(editor): wrap VC store actions with try

* feat: add fine-grained file selection for push action

* fix: close modal after successful push

* fix(editor): VC preferences validation

* fix confirm

* fix: update endpoint to /get-status

* feat: update pull modal override changes message

* fix missing wf error

* undo

* removing connect endpoint

* fix(editor): add button titles

* fix(editor): cleaning up store action

* add version-control/set-read-only protection

* fix(editor): adding set branch readonly

* fix(editor): remove Push button if branch set to readonly

* fix(editor): fix some styles

* fix(editor): remove duplicate and delete actions in WF list when branch is readonly

* fix: load status before opening selective push modal

* fix(editor): extend readonly logic

* add cleanup after failed initRepo

* fix deleted files crashing get-status

* fix n8n-checkbox in staging dialog

* fix(editor): fix loading

* fix(editor): resize buttons

* fix(editor): fix translation

* fix(editor): fix copy text size

* fix(editor): fix copy text size

* fix(editor): add disconnection confirmation

* fix(editor): add disconnection confirmation

* fix(editor): set large buttons

* add public api Pull endpoint

* feat: add refresh ssh key

* return prefs when new keys are generated

* fix(editor): adding readOnly mode to main header

* fix(editor): adding readOnly mode to workflow settings

* improve credential owner import

* add middleware to endpoints

* improve public api error/doc

* do not create branch if one already exists

* update wordings for connect toasts

* fix(editor): updating and separating readonly modes

* fix(editor): fix readonly mode in WF list

* fix(editor): disable elements dragging on canvas in readonly mode (WIP: not working when NodeView page is loaded first)

* fix(editor): fix canvas draggables in readonly env

* fix(editor): remove unused variables

* fix(editor): hide actions in node connections when readonly

* fix(editor): hide actions in node connections when readonly

* fix(editor): disable Save button when readonly

* fix(editor): disable Save settings if no branch is selected

* fix(editor): lint fix

* fix(editor): update snapshots

* fix(editor): replace Loading... text

* fix(editor): reset Loading... text

* reset branchname on disconnect

* fix(editor): adding some translations

* fix(editor): fix unit test

* fix(editor): fix loading

* fix(editor): set settings saved message

* fix(editor): update connection flag

* fix branchName not returning after connect

* temporary (but still breaking) fix for postgres

* fix(editor): adding tooltip to Push/Pull buttons when they're collapsed

* fix(editor): enabled activator in readonly mode

* fix test

* fix(editor): disabling new item addition for workflows in readonly mode

* fix(editor): modify Pull/Push button tooltips

* do not commit empty variables file

---------

Co-authored-by: Michael Auerswald <[email protected]>
Co-authored-by: Romain Minaud <[email protected]>
Co-authored-by: Alex Grozav <[email protected]>

packages/cli/package.json

@@ -78,6 +78,7 @@
     "@types/jsonwebtoken": "^9.0.1",
     "@types/localtunnel": "^1.9.0",
     "@types/lodash.debounce": "^4.0.7",
+    "@types/lodash.difference": "^4",
     "@types/lodash.get": "^4.4.6",
     "@types/lodash.intersection": "^4.4.7",
     "@types/lodash.iteratee": "^4.7.7",
@@ -91,6 +92,7 @@
     "@types/lodash.uniq": "^4.5.7",
     "@types/lodash.uniqby": "^4.7.7",
     "@types/lodash.unset": "^4.5.7",
+    "@types/lodash.without": "^4.4.7",
     "@types/parseurl": "^1.3.1",
     "@types/passport-jwt": "^3.0.6",
     "@types/psl": "^1.1.0",
@@ -159,6 +161,7 @@
     "jwks-rsa": "^3.0.1",
     "ldapts": "^4.2.6",
     "localtunnel": "^2.0.0",
+    "lodash.difference": "^4",
     "lodash.get": "^4.4.2",
     "lodash.intersection": "^4.4.0",
     "lodash.iteratee": "^4.7.0",
@@ -172,6 +175,7 @@
     "lodash.uniq": "^4.5.0",
     "lodash.uniqby": "^4.7.0",
     "lodash.unset": "^4.5.2",
+    "lodash.without": "^4.4.0",
     "luxon": "^3.3.0",
     "mysql2": "~2.3.3",
     "n8n-core": "workspace:*",
@@ -197,6 +201,7 @@
     "samlify": "^2.8.9",
     "semver": "^7.3.8",
     "shelljs": "^0.8.5",
+    "simple-git": "^3.17.0",
     "source-map-support": "^0.5.21",
     "sqlite3": "^5.1.6",
     "sse-channel": "^4.0.0",

packages/cli/src/PublicApi/types.d.ts

@@ -163,6 +163,16 @@ export interface IJsonSchema {
 	required: string[];
 }
 
+export class VersionControlPull {
+	force?: boolean;
+
+	variables?: { [key: string]: string };
+}
+
+export declare namespace PublicVersionControlRequest {
+	type Pull = AuthenticatedRequest<{}, {}, VersionControlPull, {}>;
+}
+
 // ----------------------------------
 //           /audit
 // ----------------------------------

packages/cli/src/PublicApi/v1/handlers/versionControl/spec/paths/versionControl.yml

@@ -0,0 +1,25 @@
+post:
+  x-eov-operation-id: pull
+  x-eov-operation-handler: v1/handlers/versionControl/versionControl.handler
+  tags:
+    - VersionControl
+  summary: Pull changes from the remote repository
+  description: Requires the Version Control feature to be licensed and connected to a repository.
+  requestBody:
+    description: Pull options
+    required: true
+    content:
+      application/json:
+        schema:
+          $ref: "../schemas/pull.yml"
+  responses:
+    "200":
+      description: Import result
+      content:
+        application/json:
+          schema:
+            $ref: "../schemas/importResult.yml"
+    "400":
+      $ref: "../../../../shared/spec/responses/badRequest.yml"
+    "409":
+      $ref: "../../../../shared/spec/responses/conflict.yml"

packages/cli/src/PublicApi/v1/handlers/versionControl/spec/schemas/importResult.yml

@@ -0,0 +1,55 @@
+type: object
+additionalProperties: true
+properties:
+  variables:
+    type: object
+    properties:
+      added:
+        type: array
+        items:
+          type: string
+      changed:
+        type: array
+        items:
+          type: string
+  credentials:
+    type: array
+    items:
+      type: object
+      properties:
+        id:
+          type: string
+        name:
+          type: string
+        type:
+          type: string
+  workflows:
+    type: array
+    items:
+      type: object
+      properties:
+        id:
+          type: string
+        name:
+          type: string
+  tags:
+    type: object
+    properties:
+      tags:
+        type: array
+        items:
+          type: object
+          properties:
+            id:
+              type: string
+            name:
+              type: string
+      mappings:
+        type: array
+        items:
+          type: object
+          properties:
+            workflowId:
+              type: string
+            tagId:
+              type: string

packages/cli/src/PublicApi/v1/handlers/versionControl/spec/schemas/pull.yml

@@ -0,0 +1,8 @@
+type: object
+properties:
+  force:
+    type: boolean
+    example: true
+  variables:
+    type: object
+    example: { "foo": "bar" }

packages/cli/src/PublicApi/v1/handlers/versionControl/versionControl.handler.ts

@@ -0,0 +1,47 @@
+import type express from 'express';
+import type { StatusResult } from 'simple-git';
+import type { PublicVersionControlRequest } from '../../../types';
+import { authorize } from '../../shared/middlewares/global.middleware';
+import type { ImportResult } from '@/environments/versionControl/types/importResult';
+import Container from 'typedi';
+import { VersionControlService } from '@/environments/versionControl/versionControl.service.ee';
+import { VersionControlPreferencesService } from '@/environments/versionControl/versionControlPreferences.service.ee';
+import { isVersionControlLicensed } from '@/environments/versionControl/versionControlHelper.ee';
+
+export = {
+	pull: [
+		authorize(['owner', 'member']),
+		async (
+			req: PublicVersionControlRequest.Pull,
+			res: express.Response,
+		): Promise<ImportResult | StatusResult | Promise<express.Response>> => {
+			const versionControlPreferencesService = Container.get(VersionControlPreferencesService);
+			if (!isVersionControlLicensed()) {
+				return res
+					.status(401)
+					.json({ status: 'Error', message: 'Version Control feature is not licensed' });
+			}
+			if (!versionControlPreferencesService.isVersionControlConnected()) {
+				return res
+					.status(400)
+					.json({ status: 'Error', message: 'Version Control is not connected to a repository' });
+			}
+			try {
+				const versionControlService = Container.get(VersionControlService);
+				const result = await versionControlService.pullWorkfolder({
+					force: req.body.force,
+					variables: req.body.variables,
+					userId: req.user.id,
+					importAfterPull: true,
+				});
+				if ((result as ImportResult)?.workflows) {
+					return res.status(200).send(result as ImportResult);
+				} else {
+					return res.status(409).send(result);
+				}
+			} catch (error) {
+				return res.status(400).send((error as { message: string }).message);
+			}
+		},
+	],
+};

packages/cli/src/PublicApi/v1/openapi.yml

@@ -1,7 +1,7 @@
 ---
 openapi: 3.0.0
 info:
-  title: n8n Public API
+  title: n8n Public API11
   description: n8n Public API
   termsOfService: https://n8n.io/legal/terms
   contact:
@@ -24,35 +24,39 @@ tags:
     description: Operations about workflows
   - name: Credential
     description: Operations about credentials
+  - name: VersionControl
+    description: Operations about version control
 
 paths:
   /audit:
-    $ref: './handlers/audit/spec/paths/audit.yml'
+    $ref: "./handlers/audit/spec/paths/audit.yml"
   /credentials:
-    $ref: './handlers/credentials/spec/paths/credentials.yml'
+    $ref: "./handlers/credentials/spec/paths/credentials.yml"
   /credentials/{id}:
-    $ref: './handlers/credentials/spec/paths/credentials.id.yml'
+    $ref: "./handlers/credentials/spec/paths/credentials.id.yml"
   /credentials/schema/{credentialTypeName}:
-    $ref: './handlers/credentials/spec/paths/credentials.schema.id.yml'
+    $ref: "./handlers/credentials/spec/paths/credentials.schema.id.yml"
   /executions:
-    $ref: './handlers/executions/spec/paths/executions.yml'
+    $ref: "./handlers/executions/spec/paths/executions.yml"
   /executions/{id}:
-    $ref: './handlers/executions/spec/paths/executions.id.yml'
+    $ref: "./handlers/executions/spec/paths/executions.id.yml"
   /workflows:
-    $ref: './handlers/workflows/spec/paths/workflows.yml'
+    $ref: "./handlers/workflows/spec/paths/workflows.yml"
   /workflows/{id}:
-    $ref: './handlers/workflows/spec/paths/workflows.id.yml'
+    $ref: "./handlers/workflows/spec/paths/workflows.id.yml"
   /workflows/{id}/activate:
-    $ref: './handlers/workflows/spec/paths/workflows.id.activate.yml'
+    $ref: "./handlers/workflows/spec/paths/workflows.id.activate.yml"
   /workflows/{id}/deactivate:
-    $ref: './handlers/workflows/spec/paths/workflows.id.deactivate.yml'
+    $ref: "./handlers/workflows/spec/paths/workflows.id.deactivate.yml"
+  /version-control/pull:
+    $ref: "./handlers/versionControl/spec/paths/versionControl.yml"
 components:
   schemas:
-    $ref: './shared/spec/schemas/_index.yml'
+    $ref: "./shared/spec/schemas/_index.yml"
   responses:
-    $ref: './shared/spec/responses/_index.yml'
+    $ref: "./shared/spec/responses/_index.yml"
   parameters:
-    $ref: './shared/spec/parameters/_index.yml'
+    $ref: "./shared/spec/parameters/_index.yml"
   securitySchemes:
     ApiKeyAuth:
       type: apiKey

packages/cli/src/PublicApi/v1/shared/spec/responses/_index.yml

@@ -1,6 +1,8 @@
 NotFound:
-  $ref: './notFound.yml'
+  $ref: "./notFound.yml"
 Unauthorized:
-  $ref: './unauthorized.yml'
+  $ref: "./unauthorized.yml"
 BadRequest:
-  $ref: './badRequest.yml'
+  $ref: "./badRequest.yml"
+Conflict:
+  $ref: "./conflict.yml"

packages/cli/src/PublicApi/v1/shared/spec/responses/conflict.yml

@@ -0,0 +1 @@
+description: Conflict

packages/cli/src/PublicApi/v1/shared/spec/schemas/_index.yml

@@ -1,22 +1,26 @@
 Error:
-  $ref: './error.yml'
+  $ref: "./error.yml"
 Execution:
-  $ref: './../../../handlers/executions/spec/schemas/execution.yml'
+  $ref: "./../../../handlers/executions/spec/schemas/execution.yml"
 Node:
-  $ref: './../../../handlers/workflows/spec/schemas/node.yml'
+  $ref: "./../../../handlers/workflows/spec/schemas/node.yml"
 Tag:
-  $ref: './../../../handlers/workflows/spec/schemas/tag.yml'
+  $ref: "./../../../handlers/workflows/spec/schemas/tag.yml"
 Workflow:
-  $ref: './../../../handlers/workflows/spec/schemas/workflow.yml'
+  $ref: "./../../../handlers/workflows/spec/schemas/workflow.yml"
 WorkflowSettings:
-  $ref: './../../../handlers/workflows/spec/schemas/workflowSettings.yml'
+  $ref: "./../../../handlers/workflows/spec/schemas/workflowSettings.yml"
 ExecutionList:
-  $ref: './../../../handlers/executions/spec/schemas/executionList.yml'
+  $ref: "./../../../handlers/executions/spec/schemas/executionList.yml"
 WorkflowList:
-  $ref: './../../../handlers/workflows/spec/schemas/workflowList.yml'
+  $ref: "./../../../handlers/workflows/spec/schemas/workflowList.yml"
 Credential:
-  $ref: './../../../handlers/credentials/spec/schemas/credential.yml'
+  $ref: "./../../../handlers/credentials/spec/schemas/credential.yml"
 CredentialType:
-  $ref: './../../../handlers/credentials/spec/schemas/credentialType.yml'
+  $ref: "./../../../handlers/credentials/spec/schemas/credentialType.yml"
 Audit:
-  $ref: './../../../handlers/audit/spec/schemas/audit.yml'
+  $ref: "./../../../handlers/audit/spec/schemas/audit.yml"
+Pull:
+  $ref: "./../../../handlers/versionControl/spec/schemas/pull.yml"
+ImportResult:
+  $ref: "./../../../handlers/versionControl/spec/schemas/importResult.yml"

packages/cli/src/Server.ts

@@ -167,9 +167,10 @@ import {
 	isLdapCurrentAuthenticationMethod,
 	isSamlCurrentAuthenticationMethod,
 } from './sso/ssoHelpers';
-import { isVersionControlLicensed } from '@/environments/versionControl/versionControlHelper';
+import { isVersionControlLicensed } from '@/environments/versionControl/versionControlHelper.ee';
 import { VersionControlService } from '@/environments/versionControl/versionControl.service.ee';
 import { VersionControlController } from '@/environments/versionControl/versionControl.controller.ee';
+import { VersionControlPreferencesService } from './environments/versionControl/versionControlPreferences.service.ee';
 
 const exec = promisify(callbackExec);
 
@@ -468,6 +469,7 @@ export class Server extends AbstractServer {
 		const postHog = this.postHog;
 		const samlService = Container.get(SamlService);
 		const versionControlService = Container.get(VersionControlService);
+		const versionControlPreferencesService = Container.get(VersionControlPreferencesService);
 
 		const controllers: object[] = [
 			new EventBusController(),
@@ -496,7 +498,7 @@ export class Server extends AbstractServer {
 				postHog,
 			}),
 			new SamlController(samlService),
-			new VersionControlController(versionControlService),
+			new VersionControlController(versionControlService, versionControlPreferencesService),
 		];
 
 		if (isLdapEnabled()) {

packages/cli/src/environments/versionControl/constants.ts

@@ -1 +1,15 @@
 export const VERSION_CONTROL_PREFERENCES_DB_KEY = 'features.versionControl';
+export const VERSION_CONTROL_GIT_FOLDER = 'git';
+export const VERSION_CONTROL_GIT_KEY_COMMENT = 'n8n deploy key';
+export const VERSION_CONTROL_WORKFLOW_EXPORT_FOLDER = 'workflows';
+export const VERSION_CONTROL_CREDENTIAL_EXPORT_FOLDER = 'credentials';
+export const VERSION_CONTROL_VARIABLES_EXPORT_FILE = 'variables.json';
+export const VERSION_CONTROL_TAGS_EXPORT_FILE = 'tags.json';
+export const VERSION_CONTROL_SSH_FOLDER = 'ssh';
+export const VERSION_CONTROL_SSH_KEY_NAME = 'key';
+export const VERSION_CONTROL_DEFAULT_BRANCH = 'main';
+export const VERSION_CONTROL_ORIGIN = 'origin';
+export const VERSION_CONTROL_API_ROOT = 'version-control';
+export const VERSION_CONTROL_README = `
+# n8n Version Control
+`;

packages/cli/src/environments/versionControl/middleware/versionControlEnabledMiddleware.ts → packages/cli/src/environments/versionControl/middleware/versionControlEnabledMiddleware.ee.ts

@@ -1,11 +1,11 @@
 import type { RequestHandler } from 'express';
-import {
-	isVersionControlLicensed,
-	isVersionControlLicensedAndEnabled,
-} from '../versionControlHelper';
+import { isVersionControlLicensed } from '../versionControlHelper.ee';
+import Container from 'typedi';
+import { VersionControlPreferencesService } from '../versionControlPreferences.service.ee';
 
 export const versionControlLicensedAndEnabledMiddleware: RequestHandler = (req, res, next) => {
-	if (isVersionControlLicensedAndEnabled()) {
+	const versionControlPreferencesService = Container.get(VersionControlPreferencesService);
+	if (versionControlPreferencesService.isVersionControlLicensedAndEnabled()) {
 		next();
 	} else {
 		res.status(401).json({ status: 'error', message: 'Unauthorized' });

packages/cli/src/environments/versionControl/types/exportResult.ts

@@ -0,0 +1,9 @@
+export interface ExportResult {
+	count: number;
+	folder: string;
+	files: Array<{
+		id: string;
+		name: string;
+	}>;
+	removedFiles?: string[];
+}

packages/cli/src/environments/versionControl/types/exportableCredential.ts

@@ -0,0 +1,8 @@
+import type { ICredentialDataDecryptedObject } from 'n8n-workflow';
+
+export interface ExportableCredential {
+	id: string;
+	name: string;
+	type: string;
+	data: ICredentialDataDecryptedObject;
+}