Add script to identify plugin dependencies for TS project references …

…migration (#80463)

* move kbn-dev-utils plugin helpers under a dedicated folder

* use getPluginSearchPaths in kbn-config & kbn-optimizer

* add a script to find plugin dependencies not migrated to TS project refs

* update docs

* add a script reporting all circular deps between plugins

based on kibana.json declaration, so it doesn't provide all the cases

* fix optimizer scan logic. removed by mistake

* revert changes. fails on CI

* remove prod depenedency on kbn/dev-utils

* remove last export

* only run plugin discovery once to speed up circular dep detection

* address comments

* address comments

* update fixtures

Co-authored-by: spalger <[email protected]>

docs/developer/best-practices/typescript.asciidoc

@@ -28,7 +28,7 @@ This architecture imposes several limitations to which we must comply:
 [discrete]
 ==== Prerequisites
 Since project refs rely on generated `d.ts` files, the migration order does matter. You can migrate your plugin only when all the plugin dependencies already have migrated. It creates a situation where commonly used plugins (such as `data` or `kibana_react`) have to migrate first.
-https://github.com/elastic/kibana/issues/79343 is going to provide a tool for identifying a plugin dependency tree.
+Run `node scripts/find_plugins_without_ts_refs.js --id your_plugin_id` to get a list of plugins that should be switched to TS project refs to unblock your plugin migration.
 
 [discrete]
 ==== Implementation

packages/kbn-config/src/env.ts

@@ -19,6 +19,7 @@
 
 import { resolve, join } from 'path';
 import loadJsonFile from 'load-json-file';
+import { getPluginSearchPaths } from './plugins';
 import { PackageInfo, EnvironmentMode } from './types';
 
 /** @internal */
@@ -114,21 +115,11 @@ export class Env {
     this.binDir = resolve(this.homeDir, 'bin');
     this.logDir = resolve(this.homeDir, 'log');
 
-    /**
-     * BEWARE: this needs to stay roughly synchronized with the @kbn/optimizer
-     * `packages/kbn-optimizer/src/optimizer_config.ts` determines the paths
-     * that should be searched for plugins to build
-     */
-    this.pluginSearchPaths = [
-      resolve(this.homeDir, 'src', 'plugins'),
-      ...(options.cliArgs.oss ? [] : [resolve(this.homeDir, 'x-pack', 'plugins')]),
-      resolve(this.homeDir, 'plugins'),
-      ...(options.cliArgs.runExamples ? [resolve(this.homeDir, 'examples')] : []),
-      ...(options.cliArgs.runExamples && !options.cliArgs.oss
-        ? [resolve(this.homeDir, 'x-pack', 'examples')]
-        : []),
-      resolve(this.homeDir, '..', 'kibana-extra'),
-    ];
+    this.pluginSearchPaths = getPluginSearchPaths({
+      rootDir: this.homeDir,
+      oss: options.cliArgs.oss,
+      examples: options.cliArgs.runExamples,
+    });
 
     this.cliArgs = Object.freeze(options.cliArgs);
     this.configs = Object.freeze(options.configs);

packages/kbn-config/src/index.ts

@@ -35,3 +35,4 @@ export { ObjectToConfigAdapter } from './object_to_config_adapter';
 export { CliArgs, Env, RawPackageInfo } from './env';
 export { EnvironmentMode, PackageInfo } from './types';
 export { LegacyObjectToConfigAdapter, LegacyLoggingConfig } from './legacy';
+export { getPluginSearchPaths } from './plugins';

packages/kbn-config/src/plugins/index.ts

@@ -0,0 +1,19 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+export { getPluginSearchPaths } from './plugin_search_paths';

packages/kbn-config/src/plugins/plugin_search_paths.ts

@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { resolve } from 'path';
+
+interface SearchOptions {
+  rootDir: string;
+  oss: boolean;
+  examples: boolean;
+}
+
+export function getPluginSearchPaths({ rootDir, oss, examples }: SearchOptions) {
+  return [
+    resolve(rootDir, 'src', 'plugins'),
+    ...(oss ? [] : [resolve(rootDir, 'x-pack', 'plugins')]),
+    resolve(rootDir, 'plugins'),
+    ...(examples ? [resolve(rootDir, 'examples')] : []),
+    ...(examples && !oss ? [resolve(rootDir, 'x-pack', 'examples')] : []),
+    resolve(rootDir, '..', 'kibana-extra'),
+  ];
+}

packages/kbn-dev-utils/src/index.ts

@@ -40,7 +40,6 @@ export * from './axios';
 export * from './stdio';
 export * from './ci_stats_reporter';
 export * from './plugin_list';
-export * from './simple_kibana_platform_plugin_discovery';
+export * from './plugins';
 export * from './streams';
 export * from './babel';
-export * from './parse_kibana_platform_plugin';

packages/kbn-dev-utils/src/plugin_list/discover_plugins.ts

@@ -24,7 +24,7 @@ import MarkdownIt from 'markdown-it';
 import cheerio from 'cheerio';
 import { REPO_ROOT } from '@kbn/utils';
 
-import { simpleKibanaPlatformPluginDiscovery } from '../simple_kibana_platform_plugin_discovery';
+import { simpleKibanaPlatformPluginDiscovery } from '../plugins';
 import { extractAsciidocInfo } from './extract_asciidoc_info';
 
 export interface Plugin {

packages/kbn-dev-utils/src/plugins/index.ts

@@ -0,0 +1,21 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export * from './parse_kibana_platform_plugin';
+export * from './simple_kibana_platform_plugin_discovery';

packages/kbn-dev-utils/src/parse_kibana_platform_plugin.ts → packages/kbn-dev-utils/src/plugins/parse_kibana_platform_plugin.ts

@@ -23,20 +23,35 @@ import loadJsonFile from 'load-json-file';
 export interface KibanaPlatformPlugin {
   readonly directory: string;
   readonly manifestPath: string;
-  readonly manifest: {
-    id: string;
-    ui: boolean;
-    server: boolean;
-    [key: string]: unknown;
-  };
+  readonly manifest: Manifest;
+}
+
+function isValidDepsDeclaration(input: unknown, type: string): string[] {
+  if (typeof input === 'undefined') return [];
+  if (Array.isArray(input) && input.every((i) => typeof i === 'string')) {
+    return input;
+  }
+  throw new TypeError(`The "${type}" in plugin manifest should be an array of strings.`);
+}
+
+interface Manifest {
+  id: string;
+  ui: boolean;
+  server: boolean;
+  kibanaVersion: string;
+  version: string;
+  requiredPlugins: readonly string[];
+  optionalPlugins: readonly string[];
+  requiredBundles: readonly string[];
+  extraPublicDirs: readonly string[];
 }
 
 export function parseKibanaPlatformPlugin(manifestPath: string): KibanaPlatformPlugin {
   if (!Path.isAbsolute(manifestPath)) {
     throw new TypeError('expected new platform manifest path to be absolute');
   }
 
-  const manifest = loadJsonFile.sync(manifestPath);
+  const manifest: Partial<Manifest> = loadJsonFile.sync(manifestPath);
   if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) {
     throw new TypeError('expected new platform plugin manifest to be a JSON encoded object');
   }
@@ -45,6 +60,10 @@ export function parseKibanaPlatformPlugin(manifestPath: string): KibanaPlatformP
     throw new TypeError('expected new platform plugin manifest to have a string id');
   }
 
+  if (typeof manifest.version !== 'string') {
+    throw new TypeError('expected new platform plugin manifest to have a string version');
+  }
+
   return {
     directory: Path.dirname(manifestPath),
     manifestPath,
@@ -54,6 +73,12 @@ export function parseKibanaPlatformPlugin(manifestPath: string): KibanaPlatformP
       ui: !!manifest.ui,
       server: !!manifest.server,
       id: manifest.id,
+      version: manifest.version,
+      kibanaVersion: manifest.kibanaVersion || manifest.version,
+      requiredPlugins: isValidDepsDeclaration(manifest.requiredPlugins, 'requiredPlugins'),
+      optionalPlugins: isValidDepsDeclaration(manifest.optionalPlugins, 'optionalPlugins'),
+      requiredBundles: isValidDepsDeclaration(manifest.requiredBundles, 'requiredBundles'),
+      extraPublicDirs: isValidDepsDeclaration(manifest.extraPublicDirs, 'extraPublicDirs'),
     },
   };
 }

packages/kbn-optimizer/package.json

@@ -14,6 +14,7 @@
     "@babel/core": "^7.11.6",
     "@kbn/babel-preset": "1.0.0",
     "@kbn/dev-utils": "1.0.0",
+    "@kbn/config": "1.0.0",
     "@kbn/std": "1.0.0",
     "@kbn/ui-shared-deps": "1.0.0",
     "autoprefixer": "^9.7.4",

packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/kibana.json

@@ -1,5 +1,6 @@
 {
   "id": "bar",
   "ui": true,
-  "requiredBundles": ["foo"]
+  "requiredBundles": ["foo"],
+  "version": "8.0.0"
 }

packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/foo/kibana.json

@@ -1,4 +1,5 @@
 {
   "id": "foo",
-  "ui": true
+  "ui": true,
+  "version": "8.0.0"
 }

packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/nested/baz/kibana.json

@@ -1,3 +1,4 @@
 {
-  "id": "baz"
+  "id": "baz",
+  "version": "8.0.0"
 }

packages/kbn-optimizer/src/__fixtures__/mock_repo/test_plugins/test_baz/kibana.json

@@ -1,3 +1,4 @@
 {
-  "id": "test_baz"
+  "id": "test_baz",
+  "version": "8.0.0"
 }

packages/kbn-optimizer/src/__fixtures__/mock_repo/x-pack/baz/kibana.json

@@ -1,4 +1,5 @@
 {
   "id": "baz",
-  "ui": true
+  "ui": true,
+  "version": "8.0.0"
 }

packages/kbn-optimizer/src/optimizer/optimizer_config.ts

@@ -19,6 +19,7 @@
 
 import Path from 'path';
 import Os from 'os';
+import { getPluginSearchPaths } from '@kbn/config';
 
 import {
   Bundle,
@@ -167,19 +168,14 @@ export class OptimizerConfig {
       throw new TypeError('outputRoot must be an absolute path');
     }
 
-    /**
-     * BEWARE: this needs to stay roughly synchronized with
-     * `src/core/server/config/env.ts` which determines which paths
-     * should be searched for plugins to load
-     */
-    const pluginScanDirs = options.pluginScanDirs || [
-      Path.resolve(repoRoot, 'src/plugins'),
-      ...(oss ? [] : [Path.resolve(repoRoot, 'x-pack/plugins')]),
-      Path.resolve(repoRoot, 'plugins'),
-      ...(examples ? [Path.resolve('examples')] : []),
-      ...(examples && !oss ? [Path.resolve('x-pack/examples')] : []),
-      Path.resolve(repoRoot, '../kibana-extra'),
-    ];
+    const pluginScanDirs =
+      options.pluginScanDirs ||
+      getPluginSearchPaths({
+        rootDir: repoRoot,
+        oss,
+        examples,
+      });
+
     if (!pluginScanDirs.every((p) => Path.isAbsolute(p))) {
       throw new TypeError('pluginScanDirs must all be absolute paths');
     }

scripts/find_plugin_circular_deps.js

@@ -0,0 +1,21 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+require('../src/setup_node_env');
+require('../src/dev/run_find_plugin_circular_deps');

scripts/find_plugins_without_ts_refs.js

@@ -0,0 +1,21 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+require('../src/setup_node_env');
+require('../src/dev/run_find_plugins_without_ts_refs');

src/dev/plugin_discovery/find_plugins.ts

@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import Path from 'path';
+import { REPO_ROOT } from '@kbn/dev-utils';
+import { getPluginSearchPaths } from '@kbn/config';
+import { KibanaPlatformPlugin, simpleKibanaPlatformPluginDiscovery } from '@kbn/dev-utils';
+
+export interface SearchOptions {
+  oss: boolean;
+  examples: boolean;
+  extraPluginScanDirs: string[];
+}
+
+export function findPlugins({
+  oss,
+  examples,
+  extraPluginScanDirs,
+}: SearchOptions): Map<string, KibanaPlatformPlugin> {
+  const pluginSearchPaths = getPluginSearchPaths({
+    rootDir: REPO_ROOT,
+    oss,
+    examples,
+  });
+
+  for (const extraScanDir of extraPluginScanDirs) {
+    if (!Path.isAbsolute(extraScanDir)) {
+      throw new TypeError('extraPluginScanDirs must all be absolute paths');
+    }
+    pluginSearchPaths.push(extraScanDir);
+  }
+
+  const plugins = simpleKibanaPlatformPluginDiscovery(pluginSearchPaths, []);
+  return new Map(plugins.map((p) => [p.manifest.id, p]));
+}