From b451380bb048086d90bf7a9c34c4d9c056afbb1e Mon Sep 17 00:00:00 2001 From: David Festal Date: Tue, 2 Apr 2024 13:22:03 +0200 Subject: [PATCH] refactor: experimental `embed-as-dependencies` option for the backend (#1418) Experimental `embed-as-dependencies` option... Signed-off-by: David Festal --- packages/cli/package.json | 3 +- .../{backend.ts => backend-embed-as-code.ts} | 143 +--- .../backend-embed-as-dependencies.ts | 739 ++++++++++++++++++ .../export-dynamic-plugin/backend-utils.ts | 161 ++++ .../commands/export-dynamic-plugin/command.ts | 41 +- .../src/commands/export-dynamic-plugin/dev.ts | 98 +++ .../export-dynamic-plugin/frontend.ts | 4 +- packages/cli/src/commands/index.ts | 14 +- yarn.lock | 5 + 9 files changed, 1080 insertions(+), 128 deletions(-) rename packages/cli/src/commands/export-dynamic-plugin/{backend.ts => backend-embed-as-code.ts} (69%) create mode 100644 packages/cli/src/commands/export-dynamic-plugin/backend-embed-as-dependencies.ts create mode 100644 packages/cli/src/commands/export-dynamic-plugin/backend-utils.ts create mode 100644 packages/cli/src/commands/export-dynamic-plugin/dev.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index 9cb5f5abfc..66c7b4cbc0 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -87,7 +87,8 @@ "webpack": "^5.89.0", "webpack-dev-server": "^4.15.1", "yml-loader": "^2.1.0", - "yn": "^4.0.0" + "yn": "^4.0.0", + "is-native-module": "^1.1.3" }, "devDependencies": { "@backstage/backend-common": "0.21.3", diff --git a/packages/cli/src/commands/export-dynamic-plugin/backend.ts b/packages/cli/src/commands/export-dynamic-plugin/backend-embed-as-code.ts similarity index 69% rename from packages/cli/src/commands/export-dynamic-plugin/backend.ts rename to packages/cli/src/commands/export-dynamic-plugin/backend-embed-as-code.ts index b39713d5ce..edeea5233b 100644 --- a/packages/cli/src/commands/export-dynamic-plugin/backend.ts +++ b/packages/cli/src/commands/export-dynamic-plugin/backend-embed-as-code.ts @@ -15,17 +15,14 @@ */ import { BackstagePackageJson, PackageRoleInfo } from '@backstage/cli-node'; -import { ConfigReader } from '@backstage/config'; -import { loadConfig } from '@backstage/config-loader'; import { getPackages } from '@manypkg/get-packages'; import { OptionValues } from 'commander'; import fs from 'fs-extra'; import { InteropType, rollup } from 'rollup'; -import * as semver from 'semver'; import { execSync } from 'child_process'; -import path, { basename } from 'path'; +import path from 'path'; import { Output } from '../../lib/builder'; import { makeRollupConfigs } from '../../lib/builder/config'; @@ -35,11 +32,15 @@ import { readEntryPoints } from '../../lib/entryPoints'; import { productionPack } from '../../lib/packager/productionPack'; import { paths } from '../../lib/paths'; import { Task } from '../../lib/tasks'; +import { + addToDependenciesForModule, + addToMainDependencies, +} from './backend-utils'; export async function backend( roleInfo: PackageRoleInfo, opts: OptionValues, -): Promise { +): Promise { if (!fs.existsSync(paths.resolveTarget('src', 'dynamic'))) { console.warn( `Package doesn't seem to provide dynamic loading entrypoints. You might want to add dynamic loading entrypoints in a src/dynamic folder.`, @@ -166,46 +167,15 @@ export async function backend( rollupConfig.plugins?.push( embedModules({ filter: filter, - addDependency(embeddedModule, dependencyName, newDependencyVersion) { - const existingDependencyVersion = dependenciesToAdd[dependencyName]; - if (existingDependencyVersion === undefined) { - dependenciesToAdd[dependencyName] = newDependencyVersion; - return; - } - - if (existingDependencyVersion === newDependencyVersion) { - return; - } - - const existingDependencyMinVersion = semver.minVersion( - existingDependencyVersion, - ); - if ( - existingDependencyMinVersion && - semver.satisfies(existingDependencyMinVersion, newDependencyVersion) - ) { - console.log( - `Several compatible versions ('${existingDependencyVersion}', '${newDependencyVersion}') of the same transitive dependency ('${dependencyName}') for embedded module ('${embeddedModule}'): keeping '${existingDependencyVersion}'`, - ); - return; - } - - const newDependencyMinVersion = semver.minVersion(newDependencyVersion); - if ( - newDependencyMinVersion && - semver.satisfies(newDependencyMinVersion, existingDependencyVersion) - ) { - dependenciesToAdd[dependencyName] = newDependencyVersion; - console.log( - `Several compatible versions ('${existingDependencyVersion}', '${newDependencyVersion}') of the same transitive dependency ('${dependencyName}') for embedded module ('${embeddedModule}'): keeping '${newDependencyVersion}'`, - ); - return; - } - - throw new Error( - `Several incompatible versions ('${existingDependencyVersion}', '${newDependencyVersion}') of the same transitive dependency ('${dependencyName}') for embedded module ('${embeddedModule}')`, - ); - }, + addDependency: (embeddedModule, dependencyName, newDependencyVersion) => + addToDependenciesForModule( + { + name: dependencyName, + version: newDependencyVersion, + }, + dependenciesToAdd, + embeddedModule, + ), }), ); @@ -267,33 +237,19 @@ export async function backend( f => !f.startsWith('dist-dynamic/'), ); - for (const dep in dependenciesToAdd) { - if (!Object.prototype.hasOwnProperty.call(dependenciesToAdd, dep)) { - continue; - } - pkgToCustomize.dependencies ||= {}; - const existingVersion = pkgToCustomize.dependencies[dep]; - if (existingVersion === undefined) { - pkgToCustomize.dependencies[dep] = dependenciesToAdd[dep]; - continue; - } - if (existingVersion !== dependenciesToAdd[dep]) { - const existingMinVersion = semver.minVersion(existingVersion); - - if ( - existingMinVersion && - semver.satisfies(existingMinVersion, dependenciesToAdd[dep]) - ) { - console.log( - `The version of a dependency ('${dep}') of an embedded module differs from the main module's dependencies: '${dependenciesToAdd[dep]}', '${existingVersion}': keeping it as it is compatible`, - ); - } else { - throw new Error( - `The version of a dependency ('${dep}') of an embedded module conflicts with main module dependencies: '${dependenciesToAdd[dep]}', '${existingVersion}': cannot proceed!`, - ); - } - } + // We remove scripts, because they do not make sense for this derived package. + // They even bring errors, especially the pre-pack and post-pack ones: + // we want to be able to use npm pack on this derived package to distribute it as a dynamic plugin, + // and obviously this should not trigger the backstage pre-pack or post-pack actions + // which are related to the packaging of the original static package. + pkgToCustomize.scripts = {}; + + const pkgDependencies = pkgToCustomize.dependencies || {}; + addToMainDependencies(dependenciesToAdd, pkgDependencies); + if (Object.keys(pkgDependencies).length > 0) { + pkgToCustomize.dependencies = pkgDependencies; } + if (pkgToCustomize.dependencies) { for (const monoRepoPackage of monoRepoPackages.packages) { if (pkgToCustomize.dependencies[monoRepoPackage.packageJson.name]) { @@ -376,10 +332,13 @@ export async function backend( } if (opts.install) { - const version = execSync('yarn --version').toString().trim(); + const yarn = 'yarn'; + const version = execSync(`${yarn} --version`).toString().trim(); const yarnInstall = version.startsWith('1.') - ? `yarn install --production${yarnLockExists ? ' --frozen-lockfile' : ''}` - : `yarn install${yarnLockExists ? ' --immutable' : ''}`; + ? `${yarn} install --production${ + yarnLockExists ? ' --frozen-lockfile' : '' + }` + : `${yarn} install${yarnLockExists ? ' --immutable' : ''}`; await Task.forCommand(yarnInstall, { cwd: target, optional: false }); await fs.remove(paths.resolveTarget('dist-dynamic', '.yarn')); @@ -393,39 +352,5 @@ export async function backend( minify: Boolean(opts.minify), }); - if (opts.dev) { - const appConfigs = await loadConfig({ - configRoot: paths.targetRoot, - configTargets: [], - }); - const fullConfig = ConfigReader.fromConfigs(appConfigs.appConfigs); - - const dynamicPlugins = fullConfig.getOptional('dynamicPlugins'); - if ( - typeof dynamicPlugins === 'object' && - dynamicPlugins !== null && - 'rootDirectory' in dynamicPlugins && - typeof dynamicPlugins.rootDirectory === 'string' - ) { - await fs.ensureSymlink( - paths.resolveTarget('src'), - path.resolve(target, 'src'), - 'dir', - ); - const dynamicPluginsRootPath = path.isAbsolute( - dynamicPlugins.rootDirectory, - ) - ? dynamicPlugins.rootDirectory - : paths.resolveTargetRoot(dynamicPlugins.rootDirectory); - await fs.ensureSymlink( - target, - path.resolve(dynamicPluginsRootPath, basename(paths.targetDir)), - 'dir', - ); - } else { - throw new Error( - `'dynamicPlugins.rootDirectory' should be configured in the app config in order to use the --dev option.`, - ); - } - } + return target; } diff --git a/packages/cli/src/commands/export-dynamic-plugin/backend-embed-as-dependencies.ts b/packages/cli/src/commands/export-dynamic-plugin/backend-embed-as-dependencies.ts new file mode 100644 index 0000000000..3ca4090148 --- /dev/null +++ b/packages/cli/src/commands/export-dynamic-plugin/backend-embed-as-dependencies.ts @@ -0,0 +1,739 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BackstagePackageJson } from '@backstage/cli-node'; + +import { getPackages, Packages } from '@manypkg/get-packages'; +import { parseSyml } from '@yarnpkg/parsers'; +import chalk from 'chalk'; +import { OptionValues } from 'commander'; +import * as fs from 'fs-extra'; +import * as semver from 'semver'; + +import { execSync } from 'child_process'; +import { createRequire } from 'node:module'; +import * as path from 'path'; + +import { productionPack } from '../../lib/packager/productionPack'; +import { paths } from '../../lib/paths'; +import { Task } from '../../lib/tasks'; +import { Lockfile } from '../../lib/versioning'; +import { + addToDependenciesForModule, + addToMainDependencies, + gatherNativeModules, + isValidPluginModule, +} from './backend-utils'; + +export async function backend(opts: OptionValues): Promise { + const target = path.join(paths.targetDir, 'dist-dynamic'); + const yarn = 'yarn'; + + const pkgContent = await fs.readFile( + paths.resolveTarget('package.json'), + 'utf8', + ); + const pkg = JSON.parse(pkgContent) as BackstagePackageJson; + if (pkg.bundled) { + throw new Error( + `Packages exported as dynamic backend plugins should not have the ${chalk.cyan( + 'bundled', + )} field set to ${chalk.cyan('true')}`, + ); + } + + const packagesToEmbed = (opts.embedPackage || []) as string[]; + const monoRepoPackages = await getPackages(paths.targetDir); + const embeddedResolvedPackages = await searchEmbedded( + pkg, + packagesToEmbed, + monoRepoPackages, + createRequire(paths.targetDir), + [], + ); + const embeddedPackages = embeddedResolvedPackages.map(e => e.packageName); + const unusedEmbeddedPackages = packagesToEmbed.filter( + e => !embeddedPackages.includes(e), + ); + if (unusedEmbeddedPackages.length > 0) { + Task.log( + chalk.yellow( + `Some embedded packages are not part of the plugin transitive dependencies:${chalk.cyan( + ['', ...unusedEmbeddedPackages].join('\n- '), + )}`, + ), + ); + } + + const stringOrRegexp = (s: string) => + s.startsWith('/') && s.endsWith('/') ? new RegExp(s.slice(1, -1)) : s; + + const moveToPeerDependencies = [ + /@backstage\//, + ...((opts.sharedPackage || []) as string[]) + .filter(p => !p.startsWith('!')) + .map(stringOrRegexp), + ]; + + const dontMoveToPeerDependencies = ((opts.sharedPackage || []) as string[]) + .filter(p => p.startsWith('!')) + .map(p => p.slice(1)) + .map(stringOrRegexp); + dontMoveToPeerDependencies.push(...embeddedPackages); + + const sharedPackagesRules: SharedPackagesRules = { + include: moveToPeerDependencies, + exclude: dontMoveToPeerDependencies, + }; + + if (opts.clean) { + await fs.remove(target); + } + + await fs.mkdirs(target); + await fs.writeFile( + path.join(target, '.gitignore'), + ` +* +!package.json +!yarn.lock +`, + ); + + const embeddedPeerDependencies: { + [key: string]: string; + } = {}; + + function embeddedPackageRelativePath(p: ResolvedEmbedded): string { + return path.join( + 'embedded', + p.packageName.replace(/^@/, '').replace(/\//, '-'), + ); + } + for (const embedded of embeddedResolvedPackages) { + const customizeManifest = customizeForDynamicUse({ + embedded: embeddedResolvedPackages, + monoRepoPackages, + sharedPackages: sharedPackagesRules, + overridding: { + private: true, + version: `${embedded.version}+embedded`, + }, + after: embeddedPkg => { + if (embeddedPkg.peerDependencies) { + Object.entries(embeddedPkg.peerDependencies).forEach( + ([name, version]) => { + addToDependenciesForModule( + { name, version }, + embeddedPeerDependencies, + embeddedPkg.name, + ); + }, + ); + } + }, + }); + const embeddedDestDir = path.join( + target, + embeddedPackageRelativePath(embedded), + ); + if (!embedded.alreadyPacked) { + if (opts.build) { + Task.log(`Building embedded package ${chalk.cyan(embedded.dir)}`); + await Task.forCommand(`${yarn} build`, { + cwd: embedded.dir, + optional: false, + }); + } + Task.log(`Packing embedded package ${chalk.cyan(embedded.dir)}`); + await productionPack({ + packageDir: embedded.dir, + targetDir: embeddedDestDir, + customizeManifest, + }); + } else { + Task.log(`Packing embedded package ${chalk.cyan(embedded.dir)}`); + fs.rmSync(embeddedDestDir, { force: true, recursive: true }); + fs.cpSync(embedded.dir, embeddedDestDir, { recursive: true }); + const embeddedPkgPath = path.join(embeddedDestDir, 'package.json'); + const embeddedPkgContent = await fs.readFile(embeddedPkgPath, 'utf8'); + const embeddedPkg = JSON.parse( + embeddedPkgContent, + ) as BackstagePackageJson; + customizeManifest(embeddedPkg); + await fs.writeJson(embeddedPkgPath, embeddedPkg, { + encoding: 'utf8', + spaces: 2, + }); + } + } + + const embeddedDependenciesResolutions: { [key: string]: any } = {}; + embeddedResolvedPackages.map(ep => { + embeddedDependenciesResolutions[ + ep.packageName + ] = `file:./${embeddedPackageRelativePath(ep)}`; + }); + + if (opts.build) { + Task.log(`Building main package`); + await Task.forCommand(`${yarn} build`, { + cwd: paths.targetDir, + optional: false, + }); + } + + Task.log(`Packing main package`); + await productionPack({ + packageDir: '', + targetDir: target, + customizeManifest: customizeForDynamicUse({ + embedded: embeddedResolvedPackages, + monoRepoPackages, + sharedPackages: sharedPackagesRules, + overridding: { + name: `${pkg.name}-dynamic`, + bundleDependencies: true, + // We remove scripts, because they do not make sense for this derived package. + // They even bring errors, especially the pre-pack and post-pack ones: + // we want to be able to use npm pack on this derived package to distribute it as a dynamic plugin, + // and obviously this should not trigger the backstage pre-pack or post-pack actions + // which are related to the packaging of the original static package. + scripts: {}, + }, + additionalResolutions: embeddedDependenciesResolutions, + after(mainPkg) { + if (Object.keys(embeddedPeerDependencies).length === 0) { + return; + } + Task.log( + `Hoisting peer dependencies of embedded packages to the main package`, + ); + const mainPeerDependencies = mainPkg.peerDependencies || {}; + addToMainDependencies(embeddedPeerDependencies, mainPeerDependencies); + if (Object.keys(mainPeerDependencies).length > 0) { + mainPkg.peerDependencies = mainPeerDependencies; + } + }, + }), + }); + + const yarnLock = path.resolve(target, 'yarn.lock'); + const yarnLockExists = await fs.pathExists(yarnLock); + + if (!yarnLockExists) { + // Search the yarn.lock of the static plugin, possibly at the root of the monorepo. + + let staticPluginYarnLock: string | undefined; + if (await fs.pathExists(path.join(paths.targetDir, 'yarn.lock'))) { + staticPluginYarnLock = path.join(paths.targetDir, 'yarn.lock'); + } else if (await fs.pathExists(path.join(paths.targetRoot, 'yarn.lock'))) { + staticPluginYarnLock = path.join(paths.targetRoot, 'yarn.lock'); + } + + if (!staticPluginYarnLock) { + throw new Error( + `Could not find the static plugin ${chalk.cyan( + 'yarn.lock', + )} file in either the local folder or the monorepo root (${chalk.cyan( + paths.targetRoot, + )})`, + ); + } + + await fs.copyFile(staticPluginYarnLock, yarnLock); + + if (!opts.install) { + Task.log( + chalk.yellow( + `Last export step (${chalk.cyan( + 'yarn install', + )} has been disabled: the dynamic plugin package ${chalk.cyan( + 'yarn.lock', + )} file will be inconsistent until ${chalk.cyan( + 'yarn install', + )} is run manually`, + ), + ); + } + } + + if (opts.install) { + Task.log(`Installing private dependencies of the main package`); + + const version = execSync(`${yarn} --version`).toString().trim(); + const yarnInstall = version.startsWith('1.') + ? `${yarn} install --production${ + yarnLockExists ? ' --frozen-lockfile' : '' + }` + : `${yarn} install${yarnLockExists ? ' --immutable' : ''}`; + + await Task.forCommand(yarnInstall, { cwd: target, optional: false }); + await fs.remove(paths.resolveTarget('dist-dynamic', '.yarn')); + + // Checking if some shared dependencies have been included inside the private dependencies + const lockFile = await Lockfile.load(yarnLock); + const sharedPackagesInPrivateDeps: string[] = []; + for (const key of lockFile.keys()) { + const entry = lockFile.get(key); + if (!entry) { + continue; + } + if (`${pkg.name}-dynamic` === key) { + continue; + } + if (embeddedPackages.includes(key)) { + continue; + } + if (isPackageShared(key, sharedPackagesRules)) { + sharedPackagesInPrivateDeps.push(key); + } + } + if (sharedPackagesInPrivateDeps.length > 0) { + // Some shared dependencies have been included inside the private dependencies + // => analyze the yarn.lock file to guess from which direct dependencies they + // were imported. + + const dynamicPkgContent = await fs.readFile( + path.resolve(target, 'package.json'), + 'utf8', + ); + + const dynamicPkg = JSON.parse(dynamicPkgContent) as BackstagePackageJson; + const lockfileContents = await fs.readFile(yarnLock, 'utf8'); + let data: any; + try { + data = parseSyml(lockfileContents); + } catch (err) { + throw new Error(`Failed parsing ${chalk.cyan(yarnLock)}: ${err}`); + } + + const packagesToProbablyEmbed: string[] = []; + for (const dep in dynamicPkg.dependencies || []) { + if ( + !Object.prototype.hasOwnProperty.call(dynamicPkg.dependencies, dep) + ) { + continue; + } + const matchingEntry = Object.entries(data).find(([q, _]) => { + return ( + q.startsWith(`${dep}@`) && + (q.includes(`@${dynamicPkg.dependencies![dep]}`) || + q.includes(`@npm:${dynamicPkg.dependencies![dep]}`)) + ); + }); + + if (matchingEntry) { + const yarnEntry = matchingEntry[1] as any; + if (yarnEntry.dependencies) { + if ( + Object.keys(yarnEntry.dependencies).some(d => { + return isPackageShared(d, sharedPackagesRules); + }) + ) { + packagesToProbablyEmbed.push(dep); + } + } + } + } + + throw new Error( + `Following shared package(s) should not be part of the plugin private dependencies:${chalk.cyan( + ['', ...sharedPackagesInPrivateDeps].join('\n- '), + )}\n\nEither unshare them with the ${chalk.cyan( + '--shared-package !', + )} option, or use the ${chalk.cyan( + '--embed-package', + )} to embed the following packages which use shared dependencies:${chalk.cyan( + ['', ...packagesToProbablyEmbed].join('\n- '), + )}`, + ); + } + + // Check whether private dependencies contain native modules, and fail for now (not supported). + + const nativePackages: string[] = []; + for await (const n of gatherNativeModules(target)) { + nativePackages.push(n); + } + if (nativePackages.length > 0) { + throw new Error( + `Dynamic plugins do not support native plugins. the following native modules have been transitively detected:${chalk.cyan( + ['', ...nativePackages].join('\n- '), + )}`, + ); + } + + // Check that the backend plugin provides expected entrypoints. + const dynamicPluginRequire = createRequire(target); + if ( + ![`${target}/alpha`, target].some(root => { + try { + const pluginModule = dynamicPluginRequire(root); + return isValidPluginModule(pluginModule); + } catch (_) { + return false; + } + }) + ) { + throw new Error( + `Backend plugin is not valid for dynamic loading: it should either export a ${chalk.cyan( + 'BackendFeature', + )} or ${chalk.cyan( + 'BackendFeatureFactory', + )} as default export, or export a ${chalk.cyan( + 'const dynamicPluginInstaller: BackendDynamicPluginInstaller', + )} field as dynamic loading entrypoint`, + ); + } + } + return target; +} + +type ResolvedEmbedded = { + packageName: string; + version: string; + dir: string; + parentPackageName: string; + alreadyPacked: boolean; +}; + +async function searchEmbedded( + pkg: BackstagePackageJson, + packagesToEmbed: string[], + monoRepoPackages: Packages, + req: NodeRequire, + alreadyResolved: ResolvedEmbedded[], +): Promise { + const embedded = [...packagesToEmbed]; + let regex: RegExp | undefined = undefined; + switch (pkg.backstage?.role) { + case 'backend-plugin': + regex = /-backend$/; + break; + case 'backend-plugin-module': + regex = /-backend-module-.+$/; + break; + case 'node-library': + regex = /-node$/; + break; + default: + } + if (regex) { + const commonPackage = pkg.name.replace(regex, '-common'); + if ( + commonPackage !== pkg.name && + !alreadyResolved.find(r => r.packageName === commonPackage) + ) { + embedded.push(commonPackage); + } + const nodePackage = pkg.name.replace(regex, '-node'); + if ( + nodePackage !== pkg.name && + !alreadyResolved.find(r => r.packageName === nodePackage) + ) { + embedded.push(nodePackage); + } + } + + const resolved: ResolvedEmbedded[] = []; + if (pkg.dependencies) { + for (const dep in pkg.dependencies || []) { + if (!Object.prototype.hasOwnProperty.call(pkg.dependencies, dep)) { + continue; + } + + if (embedded.includes(dep)) { + const dependencyVersion = pkg.dependencies[dep]; + + const relatedMonoRepoPackages = monoRepoPackages.packages.filter( + p => p.packageJson.name === dep, + ); + if (relatedMonoRepoPackages.length > 1) { + throw new Error( + `Two packages named '${dep}' exist in the monorepo structure: this is not supported.`, + ); + } + + if ( + relatedMonoRepoPackages.length === 0 && + dependencyVersion.startsWith('workspace:') + ) { + throw new Error( + `No package named '${dep}' exist in the monorepo structure.`, + ); + } + + let resolvedPackage: BackstagePackageJson | undefined; + let resolvedPackageDir: string; + if (relatedMonoRepoPackages.length === 1) { + const monoRepoPackage = relatedMonoRepoPackages[0]; + + let isResolved: boolean = false; + if (dependencyVersion.startsWith('workspace:')) { + isResolved = checkWorkspacePackageVersion(dependencyVersion, { + dir: monoRepoPackage.dir, + version: monoRepoPackage.packageJson.version, + }); + } else if ( + semver.satisfies( + monoRepoPackage.packageJson.version, + dependencyVersion, + ) + ) { + isResolved = true; + } + + if (!isResolved) { + throw new Error( + `Monorepo package named '${dep}' at '${monoRepoPackage.dir}' doesn't satisfy dependency version requirement in parent package '${pkg.name}'.`, + ); + } + resolvedPackage = JSON.parse( + await fs.readFile( + paths.resolveTarget( + path.join(monoRepoPackage.dir, 'package.json'), + ), + 'utf8', + ), + ) as BackstagePackageJson; + resolvedPackageDir = monoRepoPackage.dir; + } else { + // Not found as a source package in the monorepo (if any). + // Let's try to find the package through a require call. + const resolvedPackageJson = req.resolve( + path.join(dep, 'package.json'), + ); + resolvedPackageDir = path.dirname(resolvedPackageJson); + resolvedPackage = JSON.parse( + await fs.readFile(resolvedPackageJson, 'utf8'), + ) as BackstagePackageJson; + + if (!semver.satisfies(resolvedPackage.version, dependencyVersion)) { + throw new Error( + `Resolved package named '${dep}' at '${resolvedPackageDir}' doesn't satisfy dependency version requirement in parent package '${pkg.name}': '${resolvedPackage.version}', '${dependencyVersion}'.`, + ); + } + } + + if (resolvedPackage.bundled) { + throw new Error( + 'Packages embedded inside dynamic backend plugins should not have the "bundled" field set to true', + ); + } + + if ( + ![...alreadyResolved, ...resolved].find( + p => p.dir === resolvedPackageDir, + ) + ) { + resolved.push({ + dir: resolvedPackageDir, + packageName: resolvedPackage.name, + version: resolvedPackage.version ?? '0.0.0', + parentPackageName: pkg.name, + alreadyPacked: resolvedPackage.main?.endsWith('.cjs.js') || false, + }); + + resolved.push( + ...(await searchEmbedded( + resolvedPackage, + embedded, + monoRepoPackages, + req, + [...alreadyResolved, ...resolved], + )), + ); + } + } + } + } + return resolved; +} + +function checkWorkspacePackageVersion( + requiredVersionSpec: string, + pkg: { version: string; dir: string }, +): boolean { + const versionDetail = requiredVersionSpec.replace(/^workspace:/, ''); + + return ( + pkg.dir === versionDetail || + versionDetail === '*' || + versionDetail === '~' || + versionDetail === '^' || + semver.satisfies(pkg.version, versionDetail) + ); +} + +function customizeForDynamicUse(options: { + embedded: ResolvedEmbedded[]; + monoRepoPackages: Packages | undefined; + sharedPackages?: SharedPackagesRules | undefined; + overridding?: + | (Partial & { + bundleDependencies?: boolean; + }) + | undefined; + additionalOverrides?: { [key: string]: any } | undefined; + additionalResolutions?: { [key: string]: any } | undefined; + after?: ((pkg: BackstagePackageJson) => void) | undefined; +}): (pkg: BackstagePackageJson) => void { + return (pkgToCustomize: BackstagePackageJson) => { + for (const field in options.overridding || {}) { + if (!Object.prototype.hasOwnProperty.call(options.overridding, field)) { + continue; + } + (pkgToCustomize as any)[field] = (options.overridding as any)[field]; + } + + pkgToCustomize.files = pkgToCustomize.files?.filter( + f => !f.startsWith('dist-dynamic/'), + ); + + if (pkgToCustomize.dependencies) { + for (const dep in pkgToCustomize.dependencies) { + if ( + !Object.prototype.hasOwnProperty.call( + pkgToCustomize.dependencies, + dep, + ) + ) { + continue; + } + + const dependencyVersionSpec = pkgToCustomize.dependencies[dep]; + if (dependencyVersionSpec.startsWith('workspace:')) { + let resolvedVersion: string | undefined; + const embeddedDep = options.embedded.find( + e => + e.packageName === dep && + checkWorkspacePackageVersion(dependencyVersionSpec, e), + ); + if (embeddedDep) { + resolvedVersion = embeddedDep.version; + } else if (options.monoRepoPackages) { + const relatedMonoRepoPackages = + options.monoRepoPackages.packages.filter( + p => p.packageJson.name === dep, + ); + if (relatedMonoRepoPackages.length > 1) { + throw new Error( + `Two packages named ${chalk.cyan( + dep, + )} exist in the monorepo structure: this is not supported.`, + ); + } + if ( + relatedMonoRepoPackages.length === 1 && + checkWorkspacePackageVersion(dependencyVersionSpec, { + dir: relatedMonoRepoPackages[0].dir, + version: relatedMonoRepoPackages[0].packageJson.version, + }) + ) { + resolvedVersion = relatedMonoRepoPackages[0].packageJson.version; + } + } + + if (!resolvedVersion) { + throw new Error( + `Workspace dependency ${chalk.cyan(dep)} of package ${chalk.cyan( + pkgToCustomize.name, + )} doesn't exist in the monorepo structure: maybe you should embed it ?`, + ); + } + + pkgToCustomize.dependencies[dep] = resolvedVersion; + } + + if (isPackageShared(dep, options.sharedPackages)) { + Task.log(` moving ${chalk.cyan(dep)} to peerDependencies`); + + pkgToCustomize.peerDependencies ||= {}; + pkgToCustomize.peerDependencies[dep] = + pkgToCustomize.dependencies[dep]; + delete pkgToCustomize.dependencies[dep]; + } + } + } + + // We remove devDependencies here since we want the dynamic plugin derived package + // to get only production dependencies, and no transitive dependencies, in both + // the node_modules sub-folder and yarn.lock file in `dist-dynamic`. + // + // And it happens that `yarn install --production` (yarn 1) doesn't completely + // remove devDependencies as needed. + // + // See https://github.com/yarnpkg/yarn/issues/6373#issuecomment-760068356 + pkgToCustomize.devDependencies = {}; + + const overrides = (pkgToCustomize as any).overrides || {}; + (pkgToCustomize as any).overrides = { + // The following lines are a workaround for the fact that the @aws-sdk/util-utf8-browser package + // is not compatible with the NPM 9+, so that `npm pack` would not grab the Javascript files. + // This package has been deprecated in favor of @smithy/util-utf8. + // + // See https://github.com/aws/aws-sdk-js-v3/issues/5305. + '@aws-sdk/util-utf8-browser': { + '@smithy/util-utf8': '^2.0.0', + }, + ...(options.additionalOverrides || {}), + ...overrides, + }; + const resolutions = (pkgToCustomize as any).resolutions || {}; + (pkgToCustomize as any).resolutions = { + // The following lines are a workaround for the fact that the @aws-sdk/util-utf8-browser package + // is not compatible with the NPM 9+, so that `npm pack` would not grab the Javascript files. + // This package has been deprecated in favor of @smithy/util-utf8. + // + // See https://github.com/aws/aws-sdk-js-v3/issues/5305. + '@aws-sdk/util-utf8-browser': 'npm:@smithy/util-utf8@~2', + ...(options.additionalResolutions || {}), + ...resolutions, + }; + + if (options.after) { + options.after(pkgToCustomize); + } + }; +} + +type SharedPackagesRules = { + include: (string | RegExp)[]; + exclude: (string | RegExp)[]; +}; + +function isPackageShared( + pkgName: string, + rules: SharedPackagesRules | undefined, +) { + function test(str: string, expr: string | RegExp): boolean { + if (typeof expr === 'string') { + return str === expr; + } + return expr.test(str); + } + + if ((rules?.exclude || []).some(dontMove => test(pkgName, dontMove))) { + return false; + } + + if ((rules?.include || []).some(move => test(pkgName, move))) { + return true; + } + + return false; +} diff --git a/packages/cli/src/commands/export-dynamic-plugin/backend-utils.ts b/packages/cli/src/commands/export-dynamic-plugin/backend-utils.ts new file mode 100644 index 0000000000..a9d83172fe --- /dev/null +++ b/packages/cli/src/commands/export-dynamic-plugin/backend-utils.ts @@ -0,0 +1,161 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as fs from 'fs-extra'; +// @ts-ignore +import isNative from 'is-native-module'; +import semver from 'semver'; + +import path from 'path'; + +export function addToDependenciesForModule( + dependency: { name: string; version: string }, + dependencies: { [key: string]: string }, + module: string, +): void { + const existingDependencyVersion = dependencies[dependency.name]; + if (existingDependencyVersion === undefined) { + dependencies[dependency.name] = dependency.version; + return; + } + + if (existingDependencyVersion === dependency.version) { + return; + } + + const existingDependencyMinVersion = semver.minVersion( + existingDependencyVersion, + ); + if ( + existingDependencyMinVersion && + semver.satisfies(existingDependencyMinVersion, dependency.version) + ) { + console.log( + `Several compatible versions ('${existingDependencyVersion}', '${dependency.version}') of the same transitive dependency ('${dependency.name}') for embedded module ('${module}'): keeping '${existingDependencyVersion}'`, + ); + return; + } + + const newDependencyMinVersion = semver.minVersion(dependency.version); + if ( + newDependencyMinVersion && + semver.satisfies(newDependencyMinVersion, existingDependencyVersion) + ) { + dependencies[dependency.name] = dependency.version; + console.log( + `Several compatible versions ('${existingDependencyVersion}', '${dependency.version}') of the same transitive dependency ('${dependency.name}') for embedded module ('${module}'): keeping '${dependency.version}'`, + ); + return; + } + + throw new Error( + `Several incompatible versions ('${existingDependencyVersion}', '${dependency.version}') of the same transitive dependency ('${dependency.name}') for embedded module ('${module}')`, + ); +} + +export function addToMainDependencies( + dependenciesToAdd: { [key: string]: string }, + mainDependencies: { [key: string]: string }, +): void { + for (const dep in dependenciesToAdd) { + if (!Object.prototype.hasOwnProperty.call(dependenciesToAdd, dep)) { + continue; + } + const existingVersion = mainDependencies[dep]; + if (existingVersion === undefined) { + mainDependencies[dep] = dependenciesToAdd[dep]; + continue; + } + if (existingVersion !== dependenciesToAdd[dep]) { + const existingMinVersion = semver.minVersion(existingVersion); + + if ( + existingMinVersion && + semver.satisfies(existingMinVersion, dependenciesToAdd[dep]) + ) { + console.log( + `The version of a dependency ('${dep}') of an embedded module differs from the main module's dependencies: '${dependenciesToAdd[dep]}', '${existingVersion}': keeping it as it is compatible`, + ); + } else { + throw new Error( + `The version of a dependency ('${dep}') of an embedded module conflicts with main module dependencies: '${dependenciesToAdd[dep]}', '${existingVersion}': cannot proceed!`, + ); + } + } + } +} + +export function isValidPluginModule(pluginModule: any): boolean { + return ( + isBackendFeature(pluginModule.default) || + isBackendFeatureFactory(pluginModule.default) || + isBackendDynamicPluginInstaller(pluginModule.dynamicPluginInstaller) + ); +} + +function isBackendFeature(value: unknown): boolean { + return ( + !!value && + typeof value === 'object' && + (value as any).$$type === '@backstage/BackendFeature' + ); +} + +function isBackendFeatureFactory(value: unknown): boolean { + return ( + !!value && + typeof value === 'function' && + (value as any).$$type === '@backstage/BackendFeatureFactory' + ); +} + +function isBackendDynamicPluginInstaller(obj: unknown): boolean { + return ( + obj !== undefined && + typeof obj === 'object' && + obj !== null && + 'kind' in obj && + obj.kind === 'new' && + 'install' in obj && + typeof obj.install === 'function' + ); +} + +export async function* gatherNativeModules( + pkgPath: string, +): AsyncGenerator { + if (await fs.pathExists(path.join(pkgPath, 'package.json'))) { + yield* (async function* anaylzePackageJson() { + const pkg = JSON.parse( + (await fs.readFile(path.join(pkgPath, 'package.json'))).toString( + 'utf8', + ), + ); + if (isNative(pkg)) { + yield pkg.name || pkgPath; + } + })(); + if (await fs.pathExists(path.join(pkgPath, 'node_modules'))) { + yield* gatherNativeModules(path.join(pkgPath, 'node_modules')); + } + } else { + for (const file of await fs.readdir(pkgPath)) { + if ((await fs.stat(path.join(pkgPath, file))).isDirectory()) { + yield* gatherNativeModules(path.join(pkgPath, file)); + } + } + } +} diff --git a/packages/cli/src/commands/export-dynamic-plugin/command.ts b/packages/cli/src/commands/export-dynamic-plugin/command.ts index 98fe600278..a49ddc2582 100644 --- a/packages/cli/src/commands/export-dynamic-plugin/command.ts +++ b/packages/cli/src/commands/export-dynamic-plugin/command.ts @@ -16,12 +16,16 @@ import { PackageRoles } from '@backstage/cli-node'; +import chalk from 'chalk'; import { OptionValues } from 'commander'; import fs from 'fs-extra'; import { paths } from '../../lib/paths'; import { getConfigSchema } from '../../lib/schema/collect'; -import { backend } from './backend'; +import { Task } from '../../lib/tasks'; +import { backend as backendEmbedAsCode } from './backend-embed-as-code'; +import { backend as backendEmbedAsDependencies } from './backend-embed-as-dependencies'; +import { applyDevOptions } from './dev'; import { frontend } from './frontend'; const saveSchema = async (packageName: string, path: string) => { @@ -39,25 +43,30 @@ export async function command(opts: OptionValues): Promise { throw new Error(`Target package must have 'backstage.role' set`); } + let targetPath: string; const roleInfo = PackageRoles.getRoleInfo(role); + let configSchemaPath: string; if (role === 'backend-plugin' || role === 'backend-plugin-module') { - await backend(roleInfo, opts); - - await saveSchema(rawPkg.name, 'dist-dynamic/dist/configSchema.json'); - - return; + if (opts.embedAsDependencies) { + targetPath = await backendEmbedAsDependencies(opts); + } else { + targetPath = await backendEmbedAsCode(roleInfo, opts); + } + configSchemaPath = 'dist-dynamic/dist/configSchema.json'; + } else if (role === 'frontend-plugin') { + targetPath = await frontend(roleInfo, opts); + configSchemaPath = 'dist-scalprum/configSchema.json'; + } else { + throw new Error( + 'Only packages with the "backend-plugin", "backend-plugin-module" or "frontend-plugin" roles can be exported as dynamic backend plugins', + ); } - if (role === 'frontend-plugin') { - await frontend(roleInfo, opts); - - await saveSchema(rawPkg.name, 'dist-scalprum/configSchema.json'); - - return; - } - - throw new Error( - 'Only packages with the "backend-plugin", "backend-plugin-module" or "frontend-plugin" roles can be exported as dynamic backend plugins', + Task.log( + `Saving self-contained config schema in ${chalk.cyan(configSchemaPath)}`, ); + await saveSchema(rawPkg.name, configSchemaPath); + + await applyDevOptions(opts, rawPkg.name, roleInfo, targetPath); } diff --git a/packages/cli/src/commands/export-dynamic-plugin/dev.ts b/packages/cli/src/commands/export-dynamic-plugin/dev.ts new file mode 100644 index 0000000000..5cf9befe05 --- /dev/null +++ b/packages/cli/src/commands/export-dynamic-plugin/dev.ts @@ -0,0 +1,98 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PackageRoleInfo } from '@backstage/cli-node'; +import { ConfigReader } from '@backstage/config'; +import { loadConfig } from '@backstage/config-loader'; + +import chalk from 'chalk'; +import { OptionValues } from 'commander'; +import * as fs from 'fs-extra'; + +import path from 'path'; + +import { paths } from '../../lib/paths'; +import { Task } from '../../lib/tasks'; + +export async function applyDevOptions( + opts: OptionValues, + pkgName: string, + role: PackageRoleInfo, + target: string, +) { + if (opts.dev) { + if (role.platform === 'node') { + await fs.ensureSymlink( + paths.resolveTarget('src'), + path.resolve(target, 'src'), + 'dir', + ); + } + + let dynamicPluginsRootPath = opts.dynamicPluginsRoot as string | undefined; + let shouldSymlink = false; + if (!dynamicPluginsRootPath) { + shouldSymlink = true; + const appConfigs = await loadConfig({ + configRoot: paths.targetRoot, + configTargets: [], + }); + const fullConfig = ConfigReader.fromConfigs(appConfigs.appConfigs); + + const dynamicPlugins = fullConfig.getOptional('dynamicPlugins'); + if ( + typeof dynamicPlugins === 'object' && + dynamicPlugins !== null && + 'rootDirectory' in dynamicPlugins && + typeof dynamicPlugins.rootDirectory === 'string' + ) { + dynamicPluginsRootPath = path.isAbsolute(dynamicPlugins.rootDirectory) + ? dynamicPlugins.rootDirectory + : paths.resolveTargetRoot(dynamicPlugins.rootDirectory); + } else { + throw new Error( + `${chalk.cyan( + 'dynamicPlugins.rootDirectory', + )} should be configured in the app config in order to use the ${chalk.cyan( + '--dev', + )} option`, + ); + } + } + const destFolderPath = path.resolve( + dynamicPluginsRootPath, + pkgName.replace(/^@/, '').replace(/\//, '-') + + (role.platform === 'node' ? '-dynamic' : ''), + ); + if (shouldSymlink) { + Task.log( + `Linking to the dynamic plugin folder in the dynamic plugins root: ${chalk.cyan( + path.dirname(destFolderPath), + )}`, + ); + fs.rmSync(destFolderPath, { force: true, recursive: true }); + await fs.ensureSymlink(target, destFolderPath, 'dir'); + } else { + Task.log( + `Copying the dynamic plugin folder to the dynamic plugins root: ${chalk.cyan( + path.dirname(destFolderPath), + )}`, + ); + fs.rmSync(destFolderPath, { force: true, recursive: true }); + fs.cpSync(target, destFolderPath, { recursive: true }); + } + } +} diff --git a/packages/cli/src/commands/export-dynamic-plugin/frontend.ts b/packages/cli/src/commands/export-dynamic-plugin/frontend.ts index 23eb2805b9..edfa5e245f 100644 --- a/packages/cli/src/commands/export-dynamic-plugin/frontend.ts +++ b/packages/cli/src/commands/export-dynamic-plugin/frontend.ts @@ -25,7 +25,7 @@ import { paths } from '../../lib/paths'; export async function frontend( _: PackageRoleInfo, __: OptionValues, -): Promise { +): Promise { const { name, version, @@ -66,4 +66,6 @@ export async function frontend( }, fromPackage: name, }); + + return paths.targetDir; } diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index 1e3e941a5d..91c109c953 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -105,13 +105,25 @@ export function registerScriptCommand(program: Command) { '--no-install', 'Do not run `yarn install` to fill the dynamic plugin `node_modules` folder (backend plugin only).', ) + .option( + '--no-build', + 'Do not run `yarn build` on the main and embedded packages before exporting (backend plugin only).', + ) .option( '--clean', 'Remove the dynamic plugin output before exporting again.', ) .option( '--dev', - 'Allow testing/debugging a backend plugin dynamic loading locally. This installs the dynamic plugin content (symlink) into the dynamic plugins root folder configured in the app config. This also creates a link from the dynamic plugin content to the plugin package `src` folder, to enable the use of source maps (backend plugin only).', + 'Allow testing/debugging a dynamic plugin locally. This creates a link from the dynamic plugin content to the plugin package `src` folder, to enable the use of source maps (backend plugin only). This also installs the dynamic plugin content (symlink) into the dynamic plugins root folder configured in the app config (or copies the plugin content to the location explicitely provided by the `--dynamic-plugins-root` argument).', + ) + .option( + '--dynamic-plugins-root ', + 'Provides the dynamic plugins root folder when the dynamic plugins content should be copied when using the `--dev` argument.', + ) + .option( + '--embed-as-dependencies', + 'Include embedded packages as private dependencies, instead of merging the with the generated code. Experimental for now, but expected to become the default.', ) .action(lazy(() => import('./export-dynamic-plugin').then(m => m.command))); diff --git a/yarn.lock b/yarn.lock index bbf4cf9d7e..9ddb74dc20 100644 --- a/yarn.lock +++ b/yarn.lock @@ -23454,6 +23454,11 @@ is-nan@^1.3.2: call-bind "^1.0.0" define-properties "^1.1.3" +is-native-module@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/is-native-module/-/is-native-module-1.1.3.tgz#79e4bf7005570e99aed2c9e8d3d7b323d016f3fc" + integrity sha512-AmRtvnEkwv5XTWSMARUdGs6Gi8S0/MGNCOBFPJ5Pvm3poHiqxxNAoPwWbcgGF3yXq01U9WW593VvT9B86iOqJw== + is-negative-zero@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.2.tgz#7bf6f03a28003b8b3965de3ac26f664d765f3150"