diff --git a/dev-packages/cli/package.json b/dev-packages/cli/package.json index 37bfc761030b1..700c3e051312e 100644 --- a/dev-packages/cli/package.json +++ b/dev-packages/cli/package.json @@ -43,9 +43,11 @@ "chai": "^4.2.0", "chalk": "4.0.0", "decompress": "^4.2.1", + "glob": "^8.0.3", + "log-update": "^4.0.0", "mocha": "^7.0.0", - "puppeteer": "^2.0.0", "puppeteer-to-istanbul": "^1.2.2", + "puppeteer": "^2.0.0", "temp": "^0.9.1", "yargs": "^15.3.1" }, diff --git a/dev-packages/cli/src/check-dependencies.ts b/dev-packages/cli/src/check-dependencies.ts new file mode 100644 index 0000000000000..00d9e852b4fb9 --- /dev/null +++ b/dev-packages/cli/src/check-dependencies.ts @@ -0,0 +1,310 @@ +// ***************************************************************************** +// Copyright (C) 2022 STMicroelectronics and others. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0. +// +// This Source Code may also be made available under the following Secondary +// Licenses when the conditions for such availability set forth in the Eclipse +// Public License v. 2.0 are satisfied: GNU General Public License, version 2 +// with the GNU Classpath Exception which is available at +// https://www.gnu.org/software/classpath/license.html. +// +// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 +// ***************************************************************************** + +import * as fs from 'fs'; +import * as path from 'path'; +import { glob } from 'glob'; +import { create as logUpdater } from 'log-update'; +import * as chalk from 'chalk'; + +const NODE_MODULES = 'node_modules'; +const PACKAGE_JSON = 'package.json'; + +const logUpdate = logUpdater(process.stdout); + +interface CheckDependenciesOptions { + workspaces: string[] | undefined, + include: string[], + exclude: string[], + skipHoisted: boolean, + skipUniqueness: boolean, + skipSingleTheiaVersion: boolean, + suppress: boolean +} + +/** NPM package */ +interface Package { + /** Name of the package, e.g. `@theia/core`. */ + name: string, + /** Actual resolved version of the package, e.g. `1.27.0`. */ + version: string, + /** Path of the package relative to the workspace, e.g. `node_modules/@theia/core`. */ + path: string, + /** Whether the package is hoisted or not, i.e., whether it is contained in the root `node_modules`. */ + hoisted: boolean, + /** Workspace location in which the package was found. */ + dependent: string | undefined +} + +/** Issue found with a specific package. */ +interface DependencyIssue { + /** Type of the issue. */ + issueType: 'not-hoisted' | 'multiple-versions' | 'theia-version-mix', + /** Package with issue. */ + package: Package, + /** Packages related to this issue. */ + relatedPackages: Package[], + /** Severity */ + severity: 'warning' | 'error' +} + +export default function checkDependencies(options: CheckDependenciesOptions): void { + const workspaces = deriveWorkspaces(options); + logUpdate(`✅ Found ${workspaces.length} workspaces`); + + console.log('🔍 Collecting dependencies...'); + const dependencies = findAllDependencies(workspaces, options); + logUpdate(`✅ Found ${dependencies.length} dependencies`); + + console.log('🔍 Analyzing dependencies...'); + const issues = analyzeDependencies(dependencies, options); + if (issues.length <= 0) { + logUpdate('✅ No issues were found'); + process.exit(0); + } + + logUpdate('🟠 Found ' + issues.length + ' issues'); + printIssues(issues); + printHints(issues); + process.exit(options.suppress ? 0 : 1); +} + +function deriveWorkspaces(options: CheckDependenciesOptions): string[] { + const wsGlobs = options.workspaces ?? readWorkspaceGlobsFromPackageJson(); + const workspaces: string[] = []; + for (const wsGlob of wsGlobs) { + workspaces.push(...glob.sync(wsGlob + '/')); + } + return workspaces; +} + +function readWorkspaceGlobsFromPackageJson(): string[] { + const rootPackageJson = path.join(process.cwd(), PACKAGE_JSON); + if (!fs.existsSync(rootPackageJson)) { + console.error('Directory does not contain a package.json with defined workspaces'); + console.info('Run in the root of a Theia project or specify them via --workspaces'); + process.exit(1); + } + return require(rootPackageJson).workspaces ?? []; +} + +function findAllDependencies(workspaces: string[], options: CheckDependenciesOptions): Package[] { + const dependencies: Package[] = []; + dependencies.push(...findDependencies('.', options)); + for (const workspace of workspaces) { + dependencies.push(...findDependencies(workspace, options)); + } + return dependencies; +} + +function findDependencies(workspace: string, options: CheckDependenciesOptions): Package[] { + const dependent = getPackageName(path.join(process.cwd(), workspace, PACKAGE_JSON)); + const nodeModulesDir = path.join(workspace, NODE_MODULES); + const matchingPackageJsons: Package[] = []; + options.include.forEach(include => + glob.sync(`${include}/${PACKAGE_JSON}`, { + cwd: nodeModulesDir, + ignore: [ + `**/${NODE_MODULES}/**`, // node_modules folders within dependencies + `[^@]*/*/**/${PACKAGE_JSON}`, // package.json that isn't at the package root (and not in an @org) + `@*/*/*/**/${PACKAGE_JSON}`, // package.json that isn't at the package root (and in an @org) + ...options.exclude] // user-specified exclude patterns + }).forEach(packageJson => + matchingPackageJsons.push(toDependency(packageJson, nodeModulesDir, dependent)) + ) + ); + return matchingPackageJsons; +} + +function toDependency(packageJsonPath: string, nodeModulesDir: string, dependent?: string): Package { + const fullPackageJsonPath = path.join(process.cwd(), nodeModulesDir, packageJsonPath); + const name = getPackageName(fullPackageJsonPath); + const version = getPackageVersion(fullPackageJsonPath); + return { + name: name ?? packageJsonPath.replace('/' + PACKAGE_JSON, ''), + version: version ?? 'unknown', + path: path.relative(process.cwd(), fullPackageJsonPath), + hoisted: nodeModulesDir === NODE_MODULES, + dependent: dependent + }; +} + +function getPackageVersion(fullPackageJsonPath: string): string | undefined { + try { + return require(fullPackageJsonPath).version; + } catch (error) { + return undefined; + } +} + +function getPackageName(fullPackageJsonPath: string): string | undefined { + try { + return require(fullPackageJsonPath).name; + } catch (error) { + return undefined; + } +} + +function analyzeDependencies(packages: Package[], options: CheckDependenciesOptions): DependencyIssue[] { + const issues: DependencyIssue[] = []; + if (!options.skipHoisted) { + issues.push(...findNotHoistedDependencies(packages, options)); + } + if (!options.skipUniqueness) { + issues.push(...findDuplicateDependencies(packages, options)); + } + if (!options.skipSingleTheiaVersion) { + issues.push(...findTheiaVersionMix(packages, options)); + } + return issues; +} + +function findNotHoistedDependencies(packages: Package[], options: CheckDependenciesOptions): DependencyIssue[] { + const issues: DependencyIssue[] = []; + const nonHoistedPackages = packages.filter(p => p.hoisted === false); + for (const nonHoistedPackage of nonHoistedPackages) { + issues.push(createNonHoistedPackageIssue(nonHoistedPackage, options)); + } + return issues; +} + +function createNonHoistedPackageIssue(nonHoistedPackage: Package, options: CheckDependenciesOptions): DependencyIssue { + return { + issueType: 'not-hoisted', + package: nonHoistedPackage, + relatedPackages: [getHoistedPackageByName(nonHoistedPackage.name)], + severity: options.suppress ? 'warning' : 'error' + }; +} + +function getHoistedPackageByName(name: string): Package { + return toDependency(path.join(name, PACKAGE_JSON), NODE_MODULES); +} + +function findDuplicateDependencies(packages: Package[], options: CheckDependenciesOptions): DependencyIssue[] { + const duplicates: string[] = []; + const packagesGroupedByName = new Map(); + for (const currentPackage of packages) { + const name = currentPackage.name; + if (!packagesGroupedByName.has(name)) { + packagesGroupedByName.set(name, []); + } + const currentPackages = packagesGroupedByName.get(name)!; + currentPackages.push(currentPackage); + if (currentPackages.length > 1 && duplicates.indexOf(name) === -1) { + duplicates.push(name); + } + } + + duplicates.sort(); + const issues: DependencyIssue[] = []; + for (const duplicate of duplicates) { + const duplicatePackages = packagesGroupedByName.get(duplicate); + if (duplicatePackages && duplicatePackages.length > 0) { + issues.push({ + issueType: 'multiple-versions', + package: duplicatePackages.pop()!, + relatedPackages: duplicatePackages, + severity: options.suppress ? 'warning' : 'error' + }); + } + } + + return issues; +} + +function findTheiaVersionMix(packages: Package[], options: CheckDependenciesOptions): DependencyIssue[] { + // @theia/monaco-editor-core is following the versions of Monaco so it can't be part of this check + const theiaPackages = packages.filter(p => p.name.startsWith('@theia/') && !p.name.startsWith('@theia/monaco-editor-core')); + let theiaVersion = undefined; + let referenceTheiaPackage = undefined; + const packagesWithOtherVersion: Package[] = []; + for (const theiaPackage of theiaPackages) { + if (!theiaVersion && theiaPackage.version) { + theiaVersion = theiaPackage.version; + referenceTheiaPackage = theiaPackage; + } else if (theiaVersion !== theiaPackage.version) { + packagesWithOtherVersion.push(theiaPackage); + } + } + + if (referenceTheiaPackage && packagesWithOtherVersion.length > 0) { + return [{ + issueType: 'theia-version-mix', + package: referenceTheiaPackage, + relatedPackages: packagesWithOtherVersion, + severity: 'error' + }]; + } + return []; +} + +function printIssues(issues: DependencyIssue[]): void { + console.log(); + const indent = issues.length.toString().length; + issues.forEach((issue, index) => { + printIssue(issue, index + 1, indent); + }); +} + +function printIssue(issue: DependencyIssue, issueNumber: number, indent: number): void { + const remainingIndent = indent - issueNumber.toString().length; + const indentString = ' '.repeat(remainingIndent + 1); + console.log(issueTitle(issue, issueNumber, indentString)); + console.log(issueDetails(issue, ' ' + ' '.repeat(indent))); + console.log(); +} + +function issueTitle(issue: DependencyIssue, issueNumber: number, indent: string): string { + const dependent = issue.package.dependent ? ` in ${chalk.blueBright(issue.package.dependent ?? 'unknown')}` : ''; + return chalk.bgWhiteBright.bold.black(`#${issueNumber}${indent}`) + ' ' + chalk.cyanBright(issue.package.name) + + dependent + chalk.dim(` [${issue.issueType}]`); +} + +function issueDetails(issue: DependencyIssue, indent: string): string { + return indent + severity(issue) + ' ' + issueMessage(issue) + '\n' + + indent + versionLine(issue.package) + '\n' + + issue.relatedPackages.map(p => indent + versionLine(p)).join('\n'); +} + +function issueMessage(issue: DependencyIssue): string { + if (issue.issueType === 'multiple-versions') { + return `Multiple versions of dependency ${chalk.bold(issue.package.name)} found.`; + } else if (issue.issueType === 'theia-version-mix') { + return `Mix of ${chalk.bold('@theia/*')} versions found.`; + } else { + return `Dependency ${chalk.bold(issue.package.name)} is not hoisted.`; + } +} + +function severity(issue: DependencyIssue): string { + return issue.severity === 'error' ? chalk.red('error') : chalk.yellow('warning'); +} + +function versionLine(pckg: Package): string { + return chalk.bold(pckg.version) + ' in ' + pckg.path; +} + +function printHints(issues: DependencyIssue[]): void { + console.log(); + if (issues.find(i => i.issueType === 'theia-version-mix')) { + console.log('⛔ A mix of Theia versions is very likely leading to a broken application.'); + } + console.log(`ℹī¸ Use ${chalk.bold('yarn why ')} to find out why those multiple versions of a package are pulled.`); + console.log('ℹī¸ Try to resolve those issues by finding package versions along the dependency chain that depend on compatible versions.'); + console.log(`ℹī¸ Use ${chalk.bold('resolutions')} in your root package.json to force specific versions as a last resort.`); + console.log(); +} diff --git a/dev-packages/cli/src/check-hoisting.ts b/dev-packages/cli/src/check-hoisting.ts deleted file mode 100644 index 7db4aea8aba71..0000000000000 --- a/dev-packages/cli/src/check-hoisting.ts +++ /dev/null @@ -1,140 +0,0 @@ -// ***************************************************************************** -// Copyright (C) 2018-2019 TypeFox and others -// -// This program and the accompanying materials are made available under the -// terms of the Eclipse Public License v. 2.0 which is available at -// http://www.eclipse.org/legal/epl-2.0. -// -// This Source Code may also be made available under the following Secondary -// Licenses when the conditions for such availability set forth in the Eclipse -// Public License v. 2.0 are satisfied: GNU General Public License, version 2 -// with the GNU Classpath Exception which is available at -// https://www.gnu.org/software/classpath/license.html. -// -// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 -// ***************************************************************************** - -import * as fs from 'fs'; -import * as path from 'path'; - -/** - * This script makes sure all the dependencies are hoisted into the root `node_modules` after running `yarn`. - * - https://github.com/eclipse-theia/theia/pull/2994#issuecomment-425447650 - * - https://github.com/eclipse-theia/theia/pull/2994#issuecomment-425649817 - * - * If you do not want to bail the execution: `theia check:hoisted -s` - */ - -type DiagnosticType = 'error' | 'warn'; - -interface Diagnostic { - severity: number, - message: string, -} - -type DiagnosticMap = Map; - -/** - * Folders to skip inside the `node_modules` when checking the hoisted dependencies. Such as the `.bin` and `.cache` folders. - */ -const toSkip = ['.bin', '.cache']; - -function collectIssues(): DiagnosticMap { - console.log('🔍 Analyzing hoisted dependencies in the Theia extensions...'); - const root = process.cwd(); - const rootNodeModules = path.join(root, 'node_modules'); - const packages = path.join(root, 'packages'); - - const issues = new Map(); - for (const extension of fs.readdirSync(packages)) { - const extensionPath = path.join(packages, extension); - const nodeModulesPath = path.join(extensionPath, 'node_modules'); - if (fs.existsSync(nodeModulesPath)) { - for (const dependency of fs.readdirSync(nodeModulesPath).filter(name => toSkip.indexOf(name) === -1)) { - const dependencyPath = path.join(nodeModulesPath, dependency); - const version = versionOf(dependencyPath); - let message = `Dependency '${dependency}' ${version ? `[${version}] ` : ''}was not hoisted to the root 'node_modules' folder.`; - const existingDependency = path.join(rootNodeModules, dependency); - if (fs.existsSync(existingDependency)) { - const otherVersion = versionOf(existingDependency); - if (otherVersion) { - message += ` The same dependency already exists with version ${otherVersion} at '${existingDependency}'.`; - } - } - error(issues, extension, message); - } - } else { - warn(issues, extension, "Does not have 'node_modules' folder."); - } - } - return issues; -} - -function versionOf(npmPackagePath: string): string { - const packageJsonPath = path.join(npmPackagePath, 'package.json'); - if (fs.existsSync(packageJsonPath)) { - return require(packageJsonPath).version || ''; - } - return ''; -} - -function warn(issues: DiagnosticMap, extension: string, message: string): void { - log(issues, extension, message, 'warn'); -} - -function error(issues: DiagnosticMap, extension: string, message: string): void { - log(issues, extension, message, 'error'); -} - -function log(issues: DiagnosticMap, extension: string, message: string, type: DiagnosticType): void { - const key = `@theia/${extension}`; - if (!issues.has(key)) { - issues.set(key, []); - } - const severity = toSeverity(type); - issues.get(key)!.push({ severity, message }); -} - -function toSeverity(type: DiagnosticType): number { - switch (type) { - case 'error': return 0; - case 'warn': return 1; - default: throw new Error(`Unexpected type: ${type}.`); - } -} - -function toType(severity: number): DiagnosticType { - switch (severity) { - case 0: return 'error'; - case 1: return 'warn'; - default: throw new Error(`Unexpected severity: ${severity}.`); - } -} - -export default function assert({ suppress }: { suppress: boolean }): void { - const issues = collectIssues(); - console.log('📖 Summary:'); - let code = 0; - if (issues.size > 0) { - for (const [extension, issuesPerExtension] of issues.entries()) { - issuesPerExtension.sort((left, right) => left.severity - right.severity); - if (issuesPerExtension) { - console.log(`The following dependency issues were detected in '${extension}':`); - for (const { severity, message } of issuesPerExtension) { - const type = toType(severity); - console.log(` - ${type}: ${message}`); - if (type === 'error') { - code = 1; - } - } - } - } - } else { - console.log('🎉 No dependency issues were detected.'); - } - if (code !== 0 && suppress) { - console.log('⚠ī¸ This is a reminder to fix the dependency issues.'); - process.exit(0); - } - process.exit(code); -} diff --git a/dev-packages/cli/src/theia.ts b/dev-packages/cli/src/theia.ts index 6d266c519fe8f..8488cdc1b0787 100644 --- a/dev-packages/cli/src/theia.ts +++ b/dev-packages/cli/src/theia.ts @@ -22,7 +22,7 @@ import yargsFactory = require('yargs/yargs'); import { ApplicationPackageManager, rebuild } from '@theia/application-manager'; import { ApplicationProps, DEFAULT_SUPPORTED_API_VERSION } from '@theia/application-package'; import * as ffmpeg from '@theia/ffmpeg'; -import checkHoisted from './check-hoisting'; +import checkDependencies from './check-dependencies'; import downloadPlugins from './download-plugins'; import runTest from './run-test'; import { LocalizationManager, extract } from '@theia/localization-manager'; @@ -193,7 +193,117 @@ async function theiaCli(): Promise { } }, handler: ({ suppress }) => { - checkHoisted({ suppress }); + checkDependencies({ + workspaces: ['packages/*'], + include: ['**'], + exclude: ['.bin/**', '.cache/**'], + skipHoisted: false, + skipUniqueness: true, + skipSingleTheiaVersion: true, + suppress + }); + } + }) + .command<{ + suppress: boolean + }>({ + command: 'check:theia-version', + describe: 'Check that all dependencies have been resolved to the same Theia version', + builder: { + 'suppress': { + alias: 's', + describe: 'Suppress exiting with failure code', + boolean: true, + default: false + } + }, + handler: ({ suppress }) => { + checkDependencies({ + workspaces: undefined, + include: ['@theia/**'], + exclude: [], + skipHoisted: true, + skipUniqueness: false, + skipSingleTheiaVersion: false, + suppress + }); + } + }) + .command<{ + workspaces: string[] | undefined, + include: string[], + exclude: string[], + skipHoisted: boolean, + skipUniqueness: boolean, + skipSingleTheiaVersion: boolean, + suppress: boolean + }>({ + command: 'check:dependencies', + describe: 'Check uniqueness of dependency versions or whether they are hoisted', + builder: { + 'workspaces': { + alias: 'w', + describe: 'Glob patterns of workspaces to analyze, relative to `cwd`', + array: true, + defaultDescription: 'All glob patterns listed in the package.json\'s workspaces', + demandOption: false + }, + 'include': { + alias: 'i', + describe: 'Glob pattern of dependencies\' package names to be included, e.g. -i "@theia/**"', + array: true, + default: ['**'] + }, + 'exclude': { + alias: 'e', + describe: 'Glob pattern of dependencies\' package names to be excluded', + array: true, + defaultDescription: 'None', + default: [] + }, + 'skip-hoisted': { + alias: 'h', + describe: 'Skip checking whether dependencies are hoisted', + boolean: true, + default: false + }, + 'skip-uniqueness': { + alias: 'u', + describe: 'Skip checking whether all dependencies are resolved to a unique version', + boolean: true, + default: false + }, + 'skip-single-theia-version': { + alias: 't', + describe: 'Skip checking whether all @theia/* dependencies are resolved to a single version', + boolean: true, + default: false + }, + 'suppress': { + alias: 's', + describe: 'Suppress exiting with failure code', + boolean: true, + default: false + } + }, + handler: ({ + workspaces, + include, + exclude, + skipHoisted, + skipUniqueness, + skipSingleTheiaVersion, + suppress + }) => { + checkDependencies({ + workspaces, + include, + exclude, + skipHoisted, + skipUniqueness, + skipSingleTheiaVersion, + suppress + }); } }) .command<{ diff --git a/yarn.lock b/yarn.lock index a05751759db62..b978f3ad561b8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3152,7 +3152,7 @@ ansi-colors@^4.1.1: resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-4.1.3.tgz#37611340eb2243e70cc604cad35d63270d48781b" integrity sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw== -ansi-escapes@^4.2.1: +ansi-escapes@^4.2.1, ansi-escapes@^4.3.0: version "4.3.2" resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-4.3.2.tgz#6b2291d1db7d98b6521d5f1efa42d0f3a9feb65e" integrity sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ== @@ -7516,6 +7516,16 @@ log-symbols@^4.1.0: chalk "^4.1.0" is-unicode-supported "^0.1.0" +log-update@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/log-update/-/log-update-4.0.0.tgz#589ecd352471f2a1c0c570287543a64dfd20e0a1" + integrity sha512-9fkkDevMefjg0mmzWFBW8YkFP91OrizzkW3diF7CpG+S2EYdy4+TVfGwz1zeF8x7hCx1ovSPTOE9Ngib74qqUg== + dependencies: + ansi-escapes "^4.3.0" + cli-cursor "^3.1.0" + slice-ansi "^4.0.0" + wrap-ansi "^6.2.0" + loose-envify@^1.1.0, loose-envify@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf"