feat(cli): generate plugin specific schema for dynamic plugins (#912)

Signed-off-by: Tomas Coufal <[email protected]>

packages/cli/package.json

@@ -38,8 +38,8 @@
     "@backstage/errors": "^1.2.3",
     "@backstage/eslint-plugin": "^0.1.3",
     "@backstage/types": "^1.1.1",
-    "@openshift/dynamic-plugin-sdk-webpack": "^3.0.0",
     "@manypkg/get-packages": "^1.1.3",
+    "@openshift/dynamic-plugin-sdk-webpack": "^3.0.0",
     "@pmmmwh/react-refresh-webpack-plugin": "^0.5.7",
     "@rollup/plugin-commonjs": "^25.0.4",
     "@rollup/plugin-json": "^6.0.0",
@@ -83,6 +83,7 @@
     "semver": "^7.5.4",
     "style-loader": "^3.3.1",
     "swc-loader": "^0.2.3",
+    "typescript-json-schema": "^0.62.0",
     "webpack": "^5.89.0",
     "webpack-dev-server": "^4.15.1",
     "yml-loader": "^2.1.0",

packages/cli/src/commands/export-dynamic-plugin/command.ts

@@ -20,9 +20,18 @@ import { OptionValues } from 'commander';
 import fs from 'fs-extra';
 
 import { paths } from '../../lib/paths';
+import { getConfigSchema } from '../../lib/schema/collect';
 import { backend } from './backend';
 import { frontend } from './frontend';
 
+const saveSchema = async (packageName: string, path: string) => {
+  const configSchema = await getConfigSchema(packageName);
+  await fs.writeJson(paths.resolveTarget(path), configSchema, {
+    encoding: 'utf8',
+    spaces: 2,
+  });
+};
+
 export async function command(opts: OptionValues): Promise<void> {
   const rawPkg = await fs.readJson(paths.resolveTarget('package.json'));
   const role = PackageRoles.getRoleFromPackage(rawPkg);
@@ -33,11 +42,19 @@ export async function command(opts: OptionValues): Promise<void> {
   const roleInfo = PackageRoles.getRoleInfo(role);
 
   if (role === 'backend-plugin' || role === 'backend-plugin-module') {
-    return backend(roleInfo, opts);
+    await backend(roleInfo, opts);
+
+    await saveSchema(rawPkg.name, 'dist-dynamic/dist/configSchema.json');
+
+    return;
   }
 
   if (role === 'frontend-plugin') {
-    return frontend(roleInfo, opts);
+    await frontend(roleInfo, opts);
+
+    await saveSchema(rawPkg.name, 'dist-scalprum/configSchema.json');
+
+    return;
   }
 
   throw new Error(

packages/cli/src/commands/index.ts

@@ -100,6 +100,11 @@ export function registerScriptCommand(program: Command) {
       'Allow testing/debugging a backend plugin dynamic loading locally. This installs the dynamic plugin content (symlink) into the dynamic plugins root folder configured in the app config. This also creates a link from the dynamic plugin content to the plugin package `src` folder, to enable the use of source maps (backend plugin only).',
     )
     .action(lazy(() => import('./export-dynamic-plugin').then(m => m.command)));
+
+  command
+    .command('schema')
+    .description('Print configuration schema for a package')
+    .action(lazy(() => import('./schema').then(m => m.default)));
 }
 
 export function registerCommands(program: Command) {

packages/cli/src/commands/schema/command.ts

@@ -0,0 +1,11 @@
+import fs from 'fs-extra';
+
+import { paths } from '../../lib/paths';
+import { getConfigSchema } from '../../lib/schema/collect';
+
+export default async () => {
+  const { name } = await fs.readJson(paths.resolveTarget('package.json'));
+  const configSchema = await getConfigSchema(name);
+
+  process.stdout.write(`${JSON.stringify(configSchema, null, 2)}\n`);
+};

packages/cli/src/commands/schema/index.ts

@@ -0,0 +1 @@
+export { default } from './command';

packages/cli/src/lib/schema/collect.ts

@@ -0,0 +1,266 @@
+import { mergeConfigSchemas } from '@backstage/config-loader';
+import { assertError } from '@backstage/errors';
+import { JsonObject } from '@backstage/types';
+
+import fs from 'fs-extra';
+
+import { EOL } from 'os';
+import {
+  dirname,
+  relative as relativePath,
+  resolve as resolvePath,
+  sep,
+} from 'path';
+
+type ConfigSchemaPackageEntry = {
+  /**
+   * The configuration schema itself.
+   */
+  value: JsonObject;
+  /**
+   * The relative path that the configuration schema was discovered at.
+   */
+  path: string;
+};
+
+type Item = {
+  name?: string;
+  parentPath?: string;
+  packagePath?: string;
+};
+
+/**
+ * Filter out Backstage core packages from crawled dependencies based on their package name
+ * @param depName Package name to crawl
+ * @returns True if package is useful to crawl, false otherwise
+ */
+const filterPackages = (depName: string) => {
+  // reject all core dependencies
+  if (depName.startsWith('@backstage')) {
+    // make an exception for Backstage core plugins (used in plugin wrappers) unless they are common to all Backstage instances
+    if (depName.startsWith('@backstage/plugin-')) {
+      if (
+        depName.startsWith('@backstage/plugin-catalog-') ||
+        depName.startsWith('@backstage/plugin-permission-') ||
+        depName.startsWith('@backstage/plugin-search-') ||
+        depName.startsWith('@backstage/plugin-scaffolder-')
+      ) {
+        return false;
+      }
+      return true;
+    }
+    return false;
+  } else if (depName === '@janus-idp/cli') {
+    // reject CLI schema
+    return false;
+  }
+  // all other packages should be included in the schema
+  return true;
+};
+
+const req =
+  typeof __non_webpack_require__ === 'undefined'
+    ? require
+    : __non_webpack_require__;
+
+/**
+ * This collects all known config schemas across all dependencies of the app.
+ * Inspired by https://github.com/backstage/backstage/blob/a957d4654f35fb5ba6cc3450bcdb2634dcbb7724/packages/config-loader/src/schema/collect.ts#L43
+ * All unrelated logic removed from ^, only collection code is left
+ *
+ * @param packageName Package name to collect schema for
+ */
+async function collectConfigSchemas(
+  packageName: string,
+): Promise<ConfigSchemaPackageEntry[]> {
+  const schemas = new Array<ConfigSchemaPackageEntry>();
+  const tsSchemaPaths = new Array<string>();
+  const visitedPackageVersions = new Map<string, Set<string>>(); // pkgName: [versions...]
+
+  const currentDir = await fs.realpath(process.cwd());
+
+  async function processItem(item: Item) {
+    let pkgPath = item.packagePath;
+
+    if (pkgPath) {
+      const pkgExists = await fs.pathExists(pkgPath);
+      if (!pkgExists) {
+        return;
+      }
+    } else if (item.name) {
+      const { name, parentPath } = item;
+
+      try {
+        pkgPath = req.resolve(
+          `${name}/package.json`,
+          parentPath && {
+            paths: [parentPath],
+          },
+        );
+      } catch {
+        // We can somewhat safely ignore packages that don't export package.json,
+        // as they are likely not part of the Backstage ecosystem anyway.
+      }
+    }
+    if (!pkgPath) {
+      return;
+    }
+
+    const pkg = await fs.readJson(pkgPath);
+
+    // Ensures that we only process the same version of each package once.
+    let versions = visitedPackageVersions.get(pkg.name);
+    if (versions?.has(pkg.version)) {
+      return;
+    }
+    if (!versions) {
+      versions = new Set();
+      visitedPackageVersions.set(pkg.name, versions);
+    }
+    versions.add(pkg.version);
+
+    const depNames = [
+      ...Object.keys(pkg.dependencies ?? {}),
+      ...Object.keys(pkg.devDependencies ?? {}),
+      ...Object.keys(pkg.optionalDependencies ?? {}),
+      ...Object.keys(pkg.peerDependencies ?? {}),
+    ];
+
+    const hasSchema = 'configSchema' in pkg;
+    if (hasSchema) {
+      if (typeof pkg.configSchema === 'string') {
+        const isJson = pkg.configSchema.endsWith('.json');
+        const isDts = pkg.configSchema.endsWith('.d.ts');
+        if (!isJson && !isDts) {
+          throw new Error(
+            `Config schema files must be .json or .d.ts, got ${pkg.configSchema}`,
+          );
+        }
+        if (isDts) {
+          tsSchemaPaths.push(
+            relativePath(
+              currentDir,
+              resolvePath(dirname(pkgPath), pkg.configSchema),
+            ),
+          );
+        } else {
+          const path = resolvePath(dirname(pkgPath), pkg.configSchema);
+          const value = await fs.readJson(path);
+          schemas.push({
+            value,
+            path: relativePath(currentDir, path),
+          });
+        }
+      } else {
+        schemas.push({
+          value: pkg.configSchema,
+          path: relativePath(currentDir, pkgPath),
+        });
+      }
+    }
+
+    await Promise.all(
+      depNames
+        .filter(filterPackages)
+        .map(depName => processItem({ name: depName, parentPath: pkgPath })),
+    );
+  }
+
+  await processItem({ name: packageName, parentPath: currentDir });
+
+  const tsSchemas = await compileTsSchemas(tsSchemaPaths);
+
+  return schemas.concat(tsSchemas);
+}
+
+// This handles the support of TypeScript .d.ts config schema declarations.
+// We collect all typescript schema definition and compile them all in one go.
+// This is much faster than compiling them separately.
+// Copy-pasted from: https://github.com/backstage/backstage/blob/a957d4654f35fb5ba6cc3450bcdb2634dcbb7724/packages/config-loader/src/schema/collect.ts#L160
+async function compileTsSchemas(paths: string[]) {
+  if (paths.length === 0) {
+    return [];
+  }
+
+  // Lazy loaded, because this brings up all of TypeScript and we don't
+  // want that eagerly loaded in tests
+  const { getProgramFromFiles, buildGenerator } = await import(
+    'typescript-json-schema'
+  );
+
+  const program = getProgramFromFiles(paths, {
+    incremental: false,
+    isolatedModules: true,
+    lib: ['ES5'], // Skipping most libs speeds processing up a lot, we just need the primitive types anyway
+    noEmit: true,
+    noResolve: true,
+    skipLibCheck: true, // Skipping lib checks speeds things up
+    skipDefaultLibCheck: true,
+    strict: true,
+    typeRoots: [], // Do not include any additional types
+    types: [],
+  });
+
+  const tsSchemas = paths.map(path => {
+    let value;
+    try {
+      const generator = buildGenerator(
+        program,
+        // This enables the use of these tags in TSDoc comments
+        {
+          required: true,
+          validationKeywords: ['visibility', 'deepVisibility', 'deprecated'],
+        },
+        [path.split(sep).join('/')], // Unix paths are expected for all OSes here
+      );
+
+      // All schemas should export a `Config` symbol
+      value = generator?.getSchemaForSymbol('Config') as JsonObject | null;
+
+      // This makes sure that no additional symbols are defined in the schema. We don't allow
+      // this because they share a global namespace and will be merged together, leading to
+      // unpredictable behavior.
+      const userSymbols = new Set(generator?.getUserSymbols());
+      userSymbols.delete('Config');
+      if (userSymbols.size !== 0) {
+        const names = Array.from(userSymbols).join("', '");
+        throw new Error(
+          `Invalid configuration schema in ${path}, additional symbol definitions are not allowed, found '${names}'`,
+        );
+      }
+
+      // This makes sure that no unsupported types are used in the schema, for example `Record<,>`.
+      // The generator will extract these as a schema reference, which will in turn be broken for our usage.
+      const reffedDefs = Object.keys(generator?.ReffedDefinitions ?? {});
+      if (reffedDefs.length !== 0) {
+        const lines = reffedDefs.join(`${EOL}  `);
+        throw new Error(
+          `Invalid configuration schema in ${path}, the following definitions are not supported:${EOL}${EOL}  ${lines}`,
+        );
+      }
+    } catch (error) {
+      assertError(error);
+      if (error.message !== 'type Config not found') {
+        throw error;
+      }
+    }
+
+    if (!value) {
+      throw new Error(`Invalid schema in ${path}, missing Config export`);
+    }
+    return { path, value };
+  });
+
+  return tsSchemas;
+}
+
+/**
+ * Collect JSON schema for given plugin package (without core Backstage schema)
+ * @param packageName Name of the package for which it is needed to collect schema
+ * @returns JSON Schema object
+ */
+export const getConfigSchema = async (packageName: string) => {
+  const schemas = await collectConfigSchemas(packageName);
+
+  return mergeConfigSchemas((schemas as JsonObject[]).map(_ => _.value as any));
+};

yarn.lock

@@ -25153,6 +25153,20 @@ typescript-json-schema@^0.61.0:
     typescript "~5.1.0"
     yargs "^17.1.1"
 
+typescript-json-schema@^0.62.0:
+  version "0.62.0"
+  resolved "https://registry.yarnpkg.com/typescript-json-schema/-/typescript-json-schema-0.62.0.tgz#774b06b0c9d86d7f3580ea9136363a6eafae1470"
+  integrity sha512-qRO6pCgyjKJ230QYdOxDRpdQrBeeino4v5p2rYmSD72Jf4rD3O+cJcROv46sQukm46CLWoeusqvBgKpynEv25g==
+  dependencies:
+    "@types/json-schema" "^7.0.9"
+    "@types/node" "^16.9.2"
+    glob "^7.1.7"
+    path-equal "^1.2.5"
+    safe-stable-stringify "^2.2.0"
+    ts-node "^10.9.1"
+    typescript "~5.1.0"
+    yargs "^17.1.1"
+
 [email protected]:
   version "5.2.2"
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.2.2.tgz#5ebb5e5a5b75f085f22bc3f8460fba308310fa78"