Skip to content

Commit

Permalink
Merge branch 'master' into pay-318-integrate-ldap-with-autenticationm…
Browse files Browse the repository at this point in the history
…ethod
  • Loading branch information
flipswitchingmonkey authored Mar 24, 2023
2 parents f3925d8 + 161de11 commit 27e9a98
Show file tree
Hide file tree
Showing 9 changed files with 74 additions and 138 deletions.
1 change: 0 additions & 1 deletion packages/cli/src/CredentialsHelper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -451,7 +451,6 @@ export class CredentialsHelper extends ICredentialsHelper {
type: string,
data: ICredentialDataDecryptedObject,
): Promise<void> {
// eslint-disable-next-line @typescript-eslint/await-thenable
const credentials = await this.getCredentials(nodeCredentials, type);

if (!Db.isInitialized) {
Expand Down
191 changes: 61 additions & 130 deletions packages/cli/src/LoadNodesAndCredentials.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import uniq from 'lodash.uniq';
import glob from 'fast-glob';
import type { DirectoryLoader, Types } from 'n8n-core';
import {
CUSTOM_EXTENSION_ENV,
Expand All @@ -18,18 +19,18 @@ import type {
import { LoggerProxy, ErrorReporterProxy as ErrorReporter } from 'n8n-workflow';

import { createWriteStream } from 'fs';
import { access as fsAccess, mkdir, readdir as fsReaddir, stat as fsStat } from 'fs/promises';
import { mkdir } from 'fs/promises';
import path from 'path';
import config from '@/config';
import type { InstalledPackages } from '@db/entities/InstalledPackages';
import { executeCommand } from '@/CommunityNodes/helpers';
import {
CLI_DIR,
GENERATED_STATIC_DIR,
RESPONSE_ERROR_MESSAGES,
CUSTOM_API_CALL_KEY,
CUSTOM_API_CALL_NAME,
inTest,
CLI_DIR,
} from '@/constants';
import { CredentialsOverwrites } from '@/CredentialsOverwrites';
import { Service } from 'typedi';
Expand All @@ -52,6 +53,8 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {

logger: ILogger;

private downloadFolder: string;

async init() {
// Make sure the imported modules can resolve dependencies fine.
const delimiter = process.platform === 'win32' ? ';' : ':';
Expand All @@ -61,8 +64,13 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
if (!inTest) module.constructor._initPaths();

await this.loadNodesFromBasePackages();
await this.loadNodesFromDownloadedPackages();
this.downloadFolder = UserSettings.getUserN8nFolderDownloadedNodesPath();

// Load nodes from `n8n-nodes-base` and any other `n8n-nodes-*` package in the main `node_modules`
await this.loadNodesFromNodeModules(CLI_DIR);
// Load nodes from installed community packages
await this.loadNodesFromNodeModules(this.downloadFolder);

await this.loadNodesFromCustomDirectories();
await this.postProcessLoaders();
this.injectCustomApiCallOptions();
Expand Down Expand Up @@ -109,32 +117,20 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
await writeStaticJSON('credentials', this.types.credentials);
}

private async loadNodesFromBasePackages() {
const nodeModulesPath = await this.getNodeModulesPath();
const nodePackagePaths = await this.getN8nNodePackages(nodeModulesPath);

for (const packagePath of nodePackagePaths) {
await this.runDirectoryLoader(LazyPackageDirectoryLoader, packagePath);
}
}

private async loadNodesFromDownloadedPackages(): Promise<void> {
const nodePackages = [];
try {
// Read downloaded nodes and credentials
const downloadedNodesDir = UserSettings.getUserN8nFolderDownloadedNodesPath();
const downloadedNodesDirModules = path.join(downloadedNodesDir, 'node_modules');
await fsAccess(downloadedNodesDirModules);
const downloadedPackages = await this.getN8nNodePackages(downloadedNodesDirModules);
nodePackages.push(...downloadedPackages);
} catch (error) {
// Folder does not exist so ignore and return
return;
}
private async loadNodesFromNodeModules(scanDir: string): Promise<void> {
const nodeModulesDir = path.join(scanDir, 'node_modules');
const globOptions = { cwd: nodeModulesDir, onlyDirectories: true };
const installedPackagePaths = [
...(await glob('n8n-nodes-*', { ...globOptions, deep: 1 })),
...(await glob('@*/n8n-nodes-*', { ...globOptions, deep: 2 })),
];

for (const packagePath of nodePackages) {
for (const packagePath of installedPackagePaths) {
try {
await this.runDirectoryLoader(PackageDirectoryLoader, packagePath);
await this.runDirectoryLoader(
LazyPackageDirectoryLoader,
path.join(nodeModulesDir, packagePath),
);
} catch (error) {
ErrorReporter.error(error);
}
Expand All @@ -158,49 +154,45 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
}
}

/**
* Returns all the names of the packages which could contain n8n nodes
*/
private async getN8nNodePackages(baseModulesPath: string): Promise<string[]> {
const getN8nNodePackagesRecursive = async (relativePath: string): Promise<string[]> => {
const results: string[] = [];
const nodeModulesPath = `${baseModulesPath}/${relativePath}`;
const nodeModules = await fsReaddir(nodeModulesPath);
for (const nodeModule of nodeModules) {
const isN8nNodesPackage = nodeModule.indexOf('n8n-nodes-') === 0;
const isNpmScopedPackage = nodeModule.indexOf('@') === 0;
if (!isN8nNodesPackage && !isNpmScopedPackage) {
continue;
}
if (!(await fsStat(nodeModulesPath)).isDirectory()) {
continue;
}
if (isN8nNodesPackage) {
results.push(`${baseModulesPath}/${relativePath}${nodeModule}`);
}
if (isNpmScopedPackage) {
results.push(...(await getN8nNodePackagesRecursive(`${relativePath}${nodeModule}/`)));
}
}
return results;
};
return getN8nNodePackagesRecursive('');
}

async loadNpmModule(packageName: string, version?: string): Promise<InstalledPackages> {
const downloadFolder = UserSettings.getUserN8nFolderDownloadedNodesPath();
const command = `npm install ${packageName}${version ? `@${version}` : ''}`;
private async installOrUpdateNpmModule(
packageName: string,
options: { version?: string } | { installedPackage: InstalledPackages },
) {
const isUpdate = 'installedPackage' in options;
const command = isUpdate
? `npm update ${packageName}`
: `npm install ${packageName}${options.version ? `@${options.version}` : ''}`;

await executeCommand(command);
try {
await executeCommand(command);
} catch (error) {
if (error instanceof Error && error.message === RESPONSE_ERROR_MESSAGES.PACKAGE_NOT_FOUND) {
throw new Error(`The npm package "${packageName}" could not be found.`);
}
throw error;
}

const finalNodeUnpackedPath = path.join(downloadFolder, 'node_modules', packageName);
const finalNodeUnpackedPath = path.join(this.downloadFolder, 'node_modules', packageName);

const loader = await this.runDirectoryLoader(PackageDirectoryLoader, finalNodeUnpackedPath);
let loader: PackageDirectoryLoader;
try {
loader = await this.runDirectoryLoader(PackageDirectoryLoader, finalNodeUnpackedPath);
} catch (error) {
// Remove this package since loading it failed
const removeCommand = `npm remove ${packageName}`;
try {
await executeCommand(removeCommand);
} catch {}
throw new Error(RESPONSE_ERROR_MESSAGES.PACKAGE_LOADING_FAILED, { cause: error });
}

if (loader.loadedNodes.length > 0) {
// Save info to DB
try {
const { persistInstalledPackageData } = await import('@/CommunityNodes/packageModel');
const { persistInstalledPackageData, removePackageFromDatabase } = await import(
'@/CommunityNodes/packageModel'
);
if (isUpdate) await removePackageFromDatabase(options.installedPackage);
const installedPackage = await persistInstalledPackageData(loader);
await this.postProcessLoaders();
await this.generateTypesForFrontend();
Expand All @@ -223,6 +215,10 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
}
}

async installNpmModule(packageName: string, version?: string): Promise<InstalledPackages> {
return this.installOrUpdateNpmModule(packageName, { version });
}

async removeNpmModule(packageName: string, installedPackage: InstalledPackages): Promise<void> {
const command = `npm remove ${packageName}`;

Expand All @@ -244,49 +240,7 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
packageName: string,
installedPackage: InstalledPackages,
): Promise<InstalledPackages> {
const downloadFolder = UserSettings.getUserN8nFolderDownloadedNodesPath();

const command = `npm i ${packageName}@latest`;

try {
await executeCommand(command);
} catch (error) {
if (error instanceof Error && error.message === RESPONSE_ERROR_MESSAGES.PACKAGE_NOT_FOUND) {
throw new Error(`The npm package "${packageName}" could not be found.`);
}
throw error;
}

const finalNodeUnpackedPath = path.join(downloadFolder, 'node_modules', packageName);

const loader = await this.runDirectoryLoader(PackageDirectoryLoader, finalNodeUnpackedPath);

if (loader.loadedNodes.length > 0) {
// Save info to DB
try {
const { persistInstalledPackageData, removePackageFromDatabase } = await import(
'@/CommunityNodes/packageModel'
);
await removePackageFromDatabase(installedPackage);
const newlyInstalledPackage = await persistInstalledPackageData(loader);
await this.postProcessLoaders();
await this.generateTypesForFrontend();
return newlyInstalledPackage;
} catch (error) {
LoggerProxy.error('Failed to save installed packages and nodes', {
error: error as Error,
packageName,
});
throw error;
}
} else {
// Remove this package since it contains no loadable nodes
const removeCommand = `npm remove ${packageName}`;
try {
await executeCommand(removeCommand);
} catch {}
throw new Error(RESPONSE_ERROR_MESSAGES.PACKAGE_DOES_NOT_CONTAIN_NODES);
}
return this.installOrUpdateNpmModule(packageName, { installedPackage });
}

/**
Expand Down Expand Up @@ -399,27 +353,4 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
}
}
}

private async getNodeModulesPath(): Promise<string> {
// Get the path to the node-modules folder to be later able
// to load the credentials and nodes
const checkPaths = [
// In case "n8n" package is in same node_modules folder.
path.join(CLI_DIR, '..', 'n8n-workflow'),
// In case "n8n" package is the root and the packages are
// in the "node_modules" folder underneath it.
path.join(CLI_DIR, 'node_modules', 'n8n-workflow'),
// In case "n8n" package is installed using npm/yarn workspaces
// the node_modules folder is in the root of the workspace.
path.join(CLI_DIR, '..', '..', 'node_modules', 'n8n-workflow'),
];
for (const checkPath of checkPaths) {
try {
await fsAccess(checkPath);
// Folder exists, so use it.
return path.dirname(checkPath);
} catch {} // Folder does not exist so get next one
}
throw new Error('Could not find "node_modules" folder!');
}
}
5 changes: 2 additions & 3 deletions packages/cli/src/commands/start.ts
Original file line number Diff line number Diff line change
Expand Up @@ -267,11 +267,10 @@ export class Start extends BaseCommand {
// Optimistic approach - stop if any installation fails
// eslint-disable-next-line no-restricted-syntax
for (const missingPackage of missingPackages) {
// eslint-disable-next-line no-await-in-loop
void (await this.loadNodesAndCredentials.loadNpmModule(
await this.loadNodesAndCredentials.installNpmModule(
missingPackage.packageName,
missingPackage.version,
));
);
missingPackages.delete(missingPackage);
}
LoggerProxy.info('Packages reinstalled successfully. Resuming regular initialization.');
Expand Down
3 changes: 2 additions & 1 deletion packages/cli/src/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ export const CUSTOM_API_CALL_KEY = '__CUSTOM_API_CALL__';

export const CLI_DIR = resolve(__dirname, '..');
export const TEMPLATES_DIR = join(CLI_DIR, 'templates');
export const NODES_BASE_DIR = join(CLI_DIR, '..', 'nodes-base');
export const NODES_BASE_DIR = dirname(require.resolve('n8n-nodes-base'));
export const GENERATED_STATIC_DIR = join(UserSettings.getUserHome(), '.cache/n8n/public');
export const EDITOR_UI_DIST_DIR = join(dirname(require.resolve('n8n-editor-ui')), 'dist');

Expand All @@ -45,6 +45,7 @@ export const RESPONSE_ERROR_MESSAGES = {
PACKAGE_NOT_FOUND: 'Package not found in npm',
PACKAGE_VERSION_NOT_FOUND: 'The specified package version was not found',
PACKAGE_DOES_NOT_CONTAIN_NODES: 'The specified package does not contain any nodes',
PACKAGE_LOADING_FAILED: 'The specified package could not be loaded',
DISK_IS_FULL: 'There appears to be insufficient disk space',
};

Expand Down
7 changes: 5 additions & 2 deletions packages/cli/src/controllers/nodes.controller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ export class NodesController {

let installedPackage: InstalledPackages;
try {
installedPackage = await this.loadNodesAndCredentials.loadNpmModule(
installedPackage = await this.loadNodesAndCredentials.installNpmModule(
parsed.packageName,
parsed.version,
);
Expand All @@ -125,7 +125,10 @@ export class NodesController {
failure_reason: errorMessage,
});

const message = [`Error loading package "${name}"`, errorMessage].join(':');
let message = [`Error loading package "${name}" `, errorMessage].join(':');
if (error instanceof Error && error.cause instanceof Error) {
message += `\nCause: ${error.cause.message}`;
}

const clientError = error instanceof Error ? isClientError(error) : false;
throw new (clientError ? BadRequestError : InternalServerError)(message);
Expand Down
2 changes: 1 addition & 1 deletion packages/cli/test/integration/nodes.api.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ describe('POST /nodes', () => {
mocked(hasPackageLoaded).mockReturnValueOnce(false);
mocked(checkNpmPackageStatus).mockResolvedValueOnce({ status: 'OK' });

mockLoadNodesAndCredentials.loadNpmModule.mockImplementationOnce(mockedEmptyPackage);
mockLoadNodesAndCredentials.installNpmModule.mockImplementationOnce(mockedEmptyPackage);

const { statusCode } = await authOwnerShellAgent.post('/nodes').send({
name: utils.installedPackagePayload().packageName,
Expand Down
2 changes: 2 additions & 0 deletions packages/nodes-base/.eslintrc.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ module.exports = {

...sharedOptions(__dirname),

ignorePatterns: ['index.js'],

rules: {
'@typescript-eslint/consistent-type-imports': 'error',

Expand Down
Empty file added packages/nodes-base/index.js
Empty file.
1 change: 1 addition & 0 deletions packages/nodes-base/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
"name": "Jan Oberhauser",
"email": "[email protected]"
},
"main": "index.js",
"repository": {
"type": "git",
"url": "git+https://github.com/n8n-io/n8n.git"
Expand Down

0 comments on commit 27e9a98

Please sign in to comment.