diff --git a/.eslintrc.js b/.eslintrc.js index 6bd6c3cb8ee0d..0a1e2ed49854b 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -472,6 +472,7 @@ module.exports = { { files: [ 'test/functional/services/lib/web_element_wrapper/scroll_into_view_if_necessary.js', + 'src/legacy/ui/ui_render/bootstrap/kbn_bundles_loader_source.js', '**/browser_exec_scripts/**/*.js', ], rules: { diff --git a/NOTICE.txt b/NOTICE.txt index 33c1d535d7df3..946b328b8766c 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -21,6 +21,11 @@ used. Logarithmic ticks are places at powers of ten and at half those values if there are not to many ticks already (e.g. [1, 5, 10, 50, 100]). For details, see https://github.com/flot/flot/pull/1328 +--- +This module was heavily inspired by the externals plugin that ships with webpack@97d58d31 +MIT License http://www.opensource.org/licenses/mit-license.php +Author Tobias Koppers @sokra + --- This product has relied on ASTExplorer that is licensed under MIT. diff --git a/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.extrapublicdirs.md b/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.extrapublicdirs.md new file mode 100644 index 0000000000000..c46e60f2ecf6d --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.extrapublicdirs.md @@ -0,0 +1,17 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [PluginManifest](./kibana-plugin-core-server.pluginmanifest.md) > [extraPublicDirs](./kibana-plugin-core-server.pluginmanifest.extrapublicdirs.md) + +## PluginManifest.extraPublicDirs property + +> Warning: This API is now obsolete. +> +> + +Specifies directory names that can be imported by other ui-plugins built using the same instance of the @kbn/optimizer. A temporary measure we plan to replace with better mechanisms for sharing static code between plugins + +Signature: + +```typescript +readonly extraPublicDirs?: string[]; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md b/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md index fe0ca476bbcb2..5edee51d6c523 100644 --- a/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md +++ b/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md @@ -21,6 +21,7 @@ Should never be used in code outside of Core but is exported for documentation p | Property | Type | Description | | --- | --- | --- | | [configPath](./kibana-plugin-core-server.pluginmanifest.configpath.md) | ConfigPath | Root [configuration path](./kibana-plugin-core-server.configpath.md) used by the plugin, defaults to "id" in snake\_case format. | +| [extraPublicDirs](./kibana-plugin-core-server.pluginmanifest.extrapublicdirs.md) | string[] | Specifies directory names that can be imported by other ui-plugins built using the same instance of the @kbn/optimizer. A temporary measure we plan to replace with better mechanisms for sharing static code between plugins | | [id](./kibana-plugin-core-server.pluginmanifest.id.md) | PluginName | Identifier of the plugin. Must be a string in camelCase. Part of a plugin public contract. Other plugins leverage it to access plugin API, navigate to the plugin, etc. | | [kibanaVersion](./kibana-plugin-core-server.pluginmanifest.kibanaversion.md) | string | The version of Kibana the plugin is compatible with, defaults to "version". | | [optionalPlugins](./kibana-plugin-core-server.pluginmanifest.optionalplugins.md) | readonly PluginName[] | An optional list of the other plugins that if installed and enabled \*\*may be\*\* leveraged by this plugin for some additional functionality but otherwise are not required for this plugin to work properly. | diff --git a/examples/demo_search/kibana.json b/examples/demo_search/kibana.json index cb73274ed23f7..08a0af36eae43 100644 --- a/examples/demo_search/kibana.json +++ b/examples/demo_search/kibana.json @@ -6,5 +6,6 @@ "server": true, "ui": true, "requiredPlugins": ["data"], - "optionalPlugins": [] + "optionalPlugins": [], + "extraPublicDirs": ["common"] } diff --git a/examples/embeddable_examples/kibana.json b/examples/embeddable_examples/kibana.json index f446e7f31ac8e..34fa5aafb7aac 100644 --- a/examples/embeddable_examples/kibana.json +++ b/examples/embeddable_examples/kibana.json @@ -6,5 +6,6 @@ "server": true, "ui": true, "requiredPlugins": ["embeddable"], - "optionalPlugins": [] + "optionalPlugins": [], + "extraPublicDirs": ["public/todo"] } diff --git a/examples/url_generators_examples/kibana.json b/examples/url_generators_examples/kibana.json index 0767018e3bb98..83b992e0abbfd 100644 --- a/examples/url_generators_examples/kibana.json +++ b/examples/url_generators_examples/kibana.json @@ -6,5 +6,6 @@ "server": false, "ui": true, "requiredPlugins": ["share"], - "optionalPlugins": [] + "optionalPlugins": [], + "extraPublicDirs": ["public/url_generator"] } diff --git a/packages/kbn-optimizer/README.md b/packages/kbn-optimizer/README.md index c7f50c6af8dfd..9ff0f56344274 100644 --- a/packages/kbn-optimizer/README.md +++ b/packages/kbn-optimizer/README.md @@ -30,6 +30,18 @@ Bundles built by the the optimizer include a cache file which describes the info When a bundle is determined to be up-to-date a worker is not started for the bundle. If running the optimizer with the `--dev/--watch` flag, then all the files referenced by cached bundles are watched for changes. Once a change is detected in any of the files referenced by the built bundle a worker is started. If a file is changed that is referenced by several bundles then workers will be started for each bundle, combining workers together to respect the worker limit. +## Bundle Refs + +In order to dramatically reduce the size of our bundles, and the time it takes to build them, bundles will "ref" other bundles being built at the same time. When the optimizer starts it creates a list of "refs" that could be had from the list of bundles being built. Each worker uses that list to determine which import statements in a bundle should be replaced with a runtime reference to the output of another bundle. + +At runtime the bundles share a set of entry points via the `__kbnBundles__` global. By default a plugin shares `public` so that other code can use relative imports to access that directory. To expose additional directories they must be listed in the plugin's kibana.json "extraPublicDirs" field. The directories listed there will **also** be exported from the plugins bundle so that any other plugin can import that directory. "common" is commonly in the list of "extraPublicDirs". + +> NOTE: We plan to replace the `extraPublicDirs` functionality soon with better mechanisms for statically sharing code between bundles. + +When a directory is listed in the "extraPublicDirs" it will always be included in the bundle so that other plugins have access to it. The worker building the bundle has no way of knowing whether another plugin is using the directory, so be careful of adding test code or unnecessary directories to that list. + +Any import in a bundle which resolves into another bundles "context" directory, ie `src/plugins/*`, must map explicitly to a "public dir" exported by that plugin. If the resolved import is not in the list of public dirs an error will be thrown and the optimizer will fail to build that bundle until the error is fixed. + ## API To run the optimizer from code, you can import the [`OptimizerConfig`][OptimizerConfig] class and [`runOptimizer`][Optimizer] function. Create an [`OptimizerConfig`][OptimizerConfig] instance by calling it's static `create()` method with some options, then pass it to the [`runOptimizer`][Optimizer] function. `runOptimizer()` returns an observable of update objects, which are summaries of the optimizer state plus an optional `event` property which describes the internal events occuring and may be of use. You can use the [`logOptimizerState()`][LogOptimizerState] helper to write the relevant bits of state to a tooling log or checkout it's implementation to see how the internal events like [`WorkerStdio`][ObserveWorker] and [`WorkerStarted`][ObserveWorker] are used. diff --git a/packages/kbn-optimizer/package.json b/packages/kbn-optimizer/package.json index b7c9a63897bf9..c9ac85f7046fe 100644 --- a/packages/kbn-optimizer/package.json +++ b/packages/kbn-optimizer/package.json @@ -28,6 +28,7 @@ "cpy": "^8.0.0", "css-loader": "^3.4.2", "del": "^5.1.0", + "execa": "^4.0.2", "file-loader": "^4.2.0", "istanbul-instrumenter-loader": "^3.0.1", "jest-diff": "^25.1.0", @@ -44,6 +45,7 @@ "terser-webpack-plugin": "^2.1.2", "tinymath": "1.2.1", "url-loader": "^2.2.0", + "val-loader": "^1.1.1", "watchpack": "^1.6.0", "webpack": "^4.41.5", "webpack-merge": "^4.2.2" diff --git a/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/index.ts b/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/index.ts index 7ddd10f4a388f..c881a15eac5b5 100644 --- a/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/index.ts +++ b/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/index.ts @@ -19,6 +19,6 @@ import './legacy/styles.scss'; import './index.scss'; -import { fooLibFn } from '../../foo/public/index'; +import { fooLibFn } from '../../foo/public'; export * from './lib'; export { fooLibFn }; diff --git a/packages/kbn-optimizer/src/common/bundle.test.ts b/packages/kbn-optimizer/src/common/bundle.test.ts index ec78a1bdf020e..b209bbca25ac4 100644 --- a/packages/kbn-optimizer/src/common/bundle.test.ts +++ b/packages/kbn-optimizer/src/common/bundle.test.ts @@ -23,7 +23,7 @@ jest.mock('fs'); const SPEC: BundleSpec = { contextDir: '/foo/bar', - entry: 'entry', + publicDirNames: ['public'], id: 'bar', outputDir: '/foo/bar/target', sourceRoot: '/foo', @@ -49,9 +49,11 @@ it('creates cache keys', () => { }, "spec": Object { "contextDir": "/foo/bar", - "entry": "entry", "id": "bar", "outputDir": "/foo/bar/target", + "publicDirNames": Array [ + "public", + ], "sourceRoot": "/foo", "type": "plugin", }, @@ -82,9 +84,11 @@ it('parses bundles from JSON specs', () => { "state": undefined, }, "contextDir": "/foo/bar", - "entry": "entry", "id": "bar", "outputDir": "/foo/bar/target", + "publicDirNames": Array [ + "public", + ], "sourceRoot": "/foo", "type": "plugin", }, diff --git a/packages/kbn-optimizer/src/common/bundle.ts b/packages/kbn-optimizer/src/common/bundle.ts index 9e2ad186ba40c..80af94c30f8da 100644 --- a/packages/kbn-optimizer/src/common/bundle.ts +++ b/packages/kbn-optimizer/src/common/bundle.ts @@ -29,8 +29,8 @@ export interface BundleSpec { readonly type: typeof VALID_BUNDLE_TYPES[0]; /** Unique id for this bundle */ readonly id: string; - /** Webpack entry request for this plugin, relative to the contextDir */ - readonly entry: string; + /** directory names relative to the contextDir that can be imported from */ + readonly publicDirNames: string[]; /** Absolute path to the plugin source directory */ readonly contextDir: string; /** Absolute path to the root of the repository */ @@ -44,8 +44,8 @@ export class Bundle { public readonly type: BundleSpec['type']; /** Unique identifier for this bundle */ public readonly id: BundleSpec['id']; - /** Path, relative to `contextDir`, to the entry file for the Webpack bundle */ - public readonly entry: BundleSpec['entry']; + /** directory names relative to the contextDir that can be imported from */ + public readonly publicDirNames: BundleSpec['publicDirNames']; /** * Absolute path to the root of the bundle context (plugin directory) * where the entry is resolved relative to and the default output paths @@ -62,7 +62,7 @@ export class Bundle { constructor(spec: BundleSpec) { this.type = spec.type; this.id = spec.id; - this.entry = spec.entry; + this.publicDirNames = spec.publicDirNames; this.contextDir = spec.contextDir; this.sourceRoot = spec.sourceRoot; this.outputDir = spec.outputDir; @@ -73,8 +73,6 @@ export class Bundle { /** * Calculate the cache key for this bundle based from current * mtime values. - * - * @param mtimes pre-fetched mtimes (ms || undefined) for all referenced files */ createCacheKey(files: string[], mtimes: Map): unknown { return { @@ -94,7 +92,7 @@ export class Bundle { return { type: this.type, id: this.id, - entry: this.entry, + publicDirNames: this.publicDirNames, contextDir: this.contextDir, sourceRoot: this.sourceRoot, outputDir: this.outputDir, @@ -134,9 +132,9 @@ export function parseBundles(json: string) { throw new Error('`bundles[]` must have a string `id` property'); } - const { entry } = spec; - if (!(typeof entry === 'string')) { - throw new Error('`bundles[]` must have a string `entry` property'); + const { publicDirNames } = spec; + if (!Array.isArray(publicDirNames) || !publicDirNames.every((d) => typeof d === 'string')) { + throw new Error('`bundles[]` must have an array of strings `publicDirNames` property'); } const { contextDir } = spec; @@ -157,7 +155,7 @@ export function parseBundles(json: string) { return new Bundle({ type, id, - entry, + publicDirNames, contextDir, sourceRoot, outputDir, diff --git a/packages/kbn-optimizer/src/common/bundle_cache.ts b/packages/kbn-optimizer/src/common/bundle_cache.ts index 1dbc7f1d1b6b0..5ae3e4c28a201 100644 --- a/packages/kbn-optimizer/src/common/bundle_cache.ts +++ b/packages/kbn-optimizer/src/common/bundle_cache.ts @@ -25,6 +25,7 @@ export interface State { cacheKey?: unknown; moduleCount?: number; files?: string[]; + bundleRefExportIds?: string[]; } const DEFAULT_STATE: State = {}; @@ -87,6 +88,10 @@ export class BundleCache { return this.get().files; } + public getBundleRefExportIds() { + return this.get().bundleRefExportIds; + } + public getCacheKey() { return this.get().cacheKey; } diff --git a/packages/kbn-optimizer/src/common/bundle_refs.ts b/packages/kbn-optimizer/src/common/bundle_refs.ts new file mode 100644 index 0000000000000..a5c60f2031c0b --- /dev/null +++ b/packages/kbn-optimizer/src/common/bundle_refs.ts @@ -0,0 +1,130 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import Path from 'path'; + +import { Bundle } from './bundle'; +import { UnknownVals } from './ts_helpers'; + +export interface BundleRef { + bundleId: string; + contextDir: string; + contextPrefix: string; + entry: string; + exportId: string; +} + +export class BundleRefs { + static fromBundles(bundles: Bundle[]) { + return new BundleRefs( + bundles.reduce( + (acc: BundleRef[], b) => [ + ...acc, + ...b.publicDirNames.map( + (name): BundleRef => ({ + bundleId: b.id, + contextDir: b.contextDir, + // Path.resolve converts separators and strips the final separator + contextPrefix: Path.resolve(b.contextDir) + Path.sep, + entry: name, + exportId: `${b.type}/${b.id}/${name}`, + }) + ), + ], + [] + ) + ); + } + + static parseSpec(json: unknown) { + if (typeof json !== 'string') { + throw new Error('expected `bundleRefs` spec to be a JSON string'); + } + + let spec; + try { + spec = JSON.parse(json); + } catch (error) { + throw new Error('`bundleRefs` spec must be valid JSON'); + } + + if (!Array.isArray(spec)) { + throw new Error('`bundleRefs` spec must be an array'); + } + + return new BundleRefs( + spec.map( + (refSpec: UnknownVals): BundleRef => { + if (typeof refSpec !== 'object' || !refSpec) { + throw new Error('`bundleRefs[]` must be an object'); + } + + const { bundleId } = refSpec; + if (typeof bundleId !== 'string') { + throw new Error('`bundleRefs[].bundleId` must be a string'); + } + + const { contextDir } = refSpec; + if (typeof contextDir !== 'string' || !Path.isAbsolute(contextDir)) { + throw new Error('`bundleRefs[].contextDir` must be an absolute directory'); + } + + const { contextPrefix } = refSpec; + if (typeof contextPrefix !== 'string' || !Path.isAbsolute(contextPrefix)) { + throw new Error('`bundleRefs[].contextPrefix` must be an absolute directory'); + } + + const { entry } = refSpec; + if (typeof entry !== 'string') { + throw new Error('`bundleRefs[].entry` must be a string'); + } + + const { exportId } = refSpec; + if (typeof exportId !== 'string') { + throw new Error('`bundleRefs[].exportId` must be a string'); + } + + return { + bundleId, + contextDir, + contextPrefix, + entry, + exportId, + }; + } + ) + ); + } + + constructor(private readonly refs: BundleRef[]) {} + + public filterByExportIds(exportIds: string[]) { + return this.refs.filter((r) => exportIds.includes(r.exportId)); + } + + public filterByContextPrefix(bundle: Bundle, absolutePath: string) { + return this.refs.filter( + (ref) => ref.bundleId !== bundle.id && absolutePath.startsWith(ref.contextPrefix) + ); + } + + public toSpecJson() { + return JSON.stringify(this.refs); + } +} diff --git a/packages/kbn-optimizer/src/common/index.ts b/packages/kbn-optimizer/src/common/index.ts index c51905be04565..7d021a5ee7847 100644 --- a/packages/kbn-optimizer/src/common/index.ts +++ b/packages/kbn-optimizer/src/common/index.ts @@ -19,6 +19,7 @@ export * from './bundle'; export * from './bundle_cache'; +export * from './bundle_refs'; export * from './worker_config'; export * from './worker_messages'; export * from './compiler_messages'; diff --git a/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap b/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap index 436a6843d5188..1418656c15efb 100644 --- a/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap +++ b/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`builds expected bundles, saves bundle counts to metadata: 1 async bundle 1`] = `"(window[\\"foo_bundle_jsonpfunction\\"]=window[\\"foo_bundle_jsonpfunction\\"]||[]).push([[1],{4:function(module,exports,__webpack_require__){\\"use strict\\";Object.defineProperty(exports,\\"__esModule\\",{value:true});exports.foo=foo;function foo(){}}}]);"`; +exports[`builds expected bundles, saves bundle counts to metadata: 1 async bundle 1`] = `"(window[\\"foo_bundle_jsonpfunction\\"]=window[\\"foo_bundle_jsonpfunction\\"]||[]).push([[1],{5:function(module,exports,__webpack_require__){\\"use strict\\";Object.defineProperty(exports,\\"__esModule\\",{value:true});exports.foo=foo;function foo(){}}}]);"`; exports[`builds expected bundles, saves bundle counts to metadata: OptimizerConfig 1`] = ` OptimizerConfig { @@ -11,9 +11,11 @@ OptimizerConfig { "state": undefined, }, "contextDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar, - "entry": "./public/index", "id": "bar", "outputDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/target/public, + "publicDirNames": Array [ + "public", + ], "sourceRoot": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo, "type": "plugin", }, @@ -23,9 +25,11 @@ OptimizerConfig { "state": undefined, }, "contextDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo, - "entry": "./public/index", "id": "foo", "outputDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/target/public, + "publicDirNames": Array [ + "public", + ], "sourceRoot": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo, "type": "plugin", }, @@ -37,16 +41,19 @@ OptimizerConfig { "plugins": Array [ Object { "directory": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar, + "extraPublicDirs": Array [], "id": "bar", "isUiPlugin": true, }, Object { "directory": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/baz, + "extraPublicDirs": Array [], "id": "baz", "isUiPlugin": false, }, Object { "directory": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo, + "extraPublicDirs": Array [], "id": "foo", "isUiPlugin": true, }, @@ -57,6 +64,6 @@ OptimizerConfig { } `; -exports[`builds expected bundles, saves bundle counts to metadata: bar bundle 1`] = `"var __kbnBundles__=typeof __kbnBundles__===\\"object\\"?__kbnBundles__:{};__kbnBundles__[\\"plugin/bar\\"]=function(modules){function webpackJsonpCallback(data){var chunkIds=data[0];var moreModules=data[1];var moduleId,chunkId,i=0,resolves=[];for(;i { const foo = config.bundles.find((b) => b.id === 'foo')!; expect(foo).toBeTruthy(); foo.cache.refresh(); - expect(foo.cache.getModuleCount()).toBe(5); + expect(foo.cache.getModuleCount()).toBe(6); expect(foo.cache.getReferencedFiles()).toMatchInlineSnapshot(` Array [ /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/async_import.ts, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/ext.ts, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/index.ts, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/lib.ts, + /packages/kbn-optimizer/target/worker/entry_point_creator.js, /packages/kbn-ui-shared-deps/public_path_module_creator.js, ] `); @@ -154,7 +155,7 @@ it('builds expected bundles, saves bundle counts to metadata', async () => { bar.cache.refresh(); expect(bar.cache.getModuleCount()).toBe( // code + styles + style/css-loader runtimes + public path updater - 21 + 18 ); expect(bar.cache.getReferencedFiles()).toMatchInlineSnapshot(` @@ -165,11 +166,8 @@ it('builds expected bundles, saves bundle counts to metadata', async () => { /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/index.ts, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/legacy/styles.scss, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/lib.ts, - /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/async_import.ts, - /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/ext.ts, - /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/index.ts, - /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/lib.ts, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/src/legacy/ui/public/icon.svg, + /packages/kbn-optimizer/target/worker/entry_point_creator.js, /packages/kbn-ui-shared-deps/public_path_module_creator.js, ] `); diff --git a/packages/kbn-optimizer/src/integration_tests/bundle_cache.test.ts b/packages/kbn-optimizer/src/integration_tests/bundle_cache.test.ts index 39064c64062e8..48cab508954a0 100644 --- a/packages/kbn-optimizer/src/integration_tests/bundle_cache.test.ts +++ b/packages/kbn-optimizer/src/integration_tests/bundle_cache.test.ts @@ -75,6 +75,7 @@ it('emits "bundle cached" event when everything is updated', async () => { optimizerCacheKey, files, moduleCount: files.length, + bundleRefExportIds: [], }); const cacheEvents = await getBundleCacheEvent$(config, optimizerCacheKey) @@ -115,6 +116,7 @@ it('emits "bundle not cached" event when cacheKey is up to date but caching is d optimizerCacheKey, files, moduleCount: files.length, + bundleRefExportIds: [], }); const cacheEvents = await getBundleCacheEvent$(config, optimizerCacheKey) @@ -155,6 +157,7 @@ it('emits "bundle not cached" event when optimizerCacheKey is missing', async () optimizerCacheKey: undefined, files, moduleCount: files.length, + bundleRefExportIds: [], }); const cacheEvents = await getBundleCacheEvent$(config, optimizerCacheKey) @@ -195,6 +198,7 @@ it('emits "bundle not cached" event when optimizerCacheKey is outdated, includes optimizerCacheKey: 'old', files, moduleCount: files.length, + bundleRefExportIds: [], }); const cacheEvents = await getBundleCacheEvent$(config, optimizerCacheKey) @@ -217,6 +221,53 @@ it('emits "bundle not cached" event when optimizerCacheKey is outdated, includes `); }); +it('emits "bundle not cached" event when bundleRefExportIds is outdated, includes diff', async () => { + const config = OptimizerConfig.create({ + repoRoot: MOCK_REPO_DIR, + pluginScanDirs: [], + pluginPaths: [Path.resolve(MOCK_REPO_DIR, 'plugins/foo')], + maxWorkerCount: 1, + }); + const [bundle] = config.bundles; + + const optimizerCacheKey = 'optimizerCacheKey'; + const files = [ + Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/ext.ts'), + Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/index.ts'), + Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/lib.ts'), + ]; + const mtimes = await getMtimes(files); + const cacheKey = bundle.createCacheKey(files, mtimes); + + bundle.cache.set({ + cacheKey, + optimizerCacheKey, + files, + moduleCount: files.length, + bundleRefExportIds: ['plugin/bar/public'], + }); + + const cacheEvents = await getBundleCacheEvent$(config, optimizerCacheKey) + .pipe(toArray()) + .toPromise(); + + expect(cacheEvents).toMatchInlineSnapshot(` + Array [ + Object { + "bundle": , + "diff": "- Expected + + Received + +  [ + + \\"plugin/bar/public\\" +  ]", + "reason": "bundle references outdated", + "type": "bundle not cached", + }, + ] + `); +}); + it('emits "bundle not cached" event when cacheKey is missing', async () => { const config = OptimizerConfig.create({ repoRoot: MOCK_REPO_DIR, @@ -238,6 +289,7 @@ it('emits "bundle not cached" event when cacheKey is missing', async () => { optimizerCacheKey, files, moduleCount: files.length, + bundleRefExportIds: [], }); const cacheEvents = await getBundleCacheEvent$(config, optimizerCacheKey) @@ -276,6 +328,7 @@ it('emits "bundle not cached" event when cacheKey is outdated', async () => { optimizerCacheKey, files, moduleCount: files.length, + bundleRefExportIds: [], }); jest.spyOn(bundle, 'createCacheKey').mockImplementation(() => 'new'); diff --git a/packages/kbn-optimizer/src/integration_tests/watch_bundles_for_changes.test.ts b/packages/kbn-optimizer/src/integration_tests/watch_bundles_for_changes.test.ts index 91d0f308e0ef6..176b17c979da9 100644 --- a/packages/kbn-optimizer/src/integration_tests/watch_bundles_for_changes.test.ts +++ b/packages/kbn-optimizer/src/integration_tests/watch_bundles_for_changes.test.ts @@ -29,14 +29,14 @@ jest.mock('fs'); jest.mock('watchpack'); const MockWatchPack: jest.MockedClass = jest.requireMock('watchpack'); -const bundleEntryPath = (bundle: Bundle) => `${bundle.contextDir}/${bundle.entry}`; +const bundleEntryPath = (bundle: Bundle) => `${bundle.contextDir}/public/index.ts`; const makeTestBundle = (id: string) => { const bundle = new Bundle({ type: 'plugin', id, contextDir: `/repo/plugins/${id}/public`, - entry: 'index.ts', + publicDirNames: ['public'], outputDir: `/repo/plugins/${id}/target/public`, sourceRoot: `/repo`, }); diff --git a/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.test.ts b/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.test.ts index 4671276797049..ca50a49e26913 100644 --- a/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.test.ts +++ b/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.test.ts @@ -57,7 +57,7 @@ const assertReturnVal = (workers: Assignments[]) => { const testBundle = (id: string) => new Bundle({ contextDir: `/repo/plugin/${id}/public`, - entry: 'index.ts', + publicDirNames: ['public'], id, outputDir: `/repo/plugins/${id}/target/public`, sourceRoot: `/repo`, diff --git a/packages/kbn-optimizer/src/optimizer/bundle_cache.ts b/packages/kbn-optimizer/src/optimizer/bundle_cache.ts index 55e8e1d3fd084..83db8570bd408 100644 --- a/packages/kbn-optimizer/src/optimizer/bundle_cache.ts +++ b/packages/kbn-optimizer/src/optimizer/bundle_cache.ts @@ -20,7 +20,7 @@ import * as Rx from 'rxjs'; import { mergeAll } from 'rxjs/operators'; -import { Bundle } from '../common'; +import { Bundle, BundleRefs } from '../common'; import { OptimizerConfig } from './optimizer_config'; import { getMtimes } from './get_mtimes'; @@ -35,7 +35,9 @@ export interface BundleNotCachedEvent { | 'optimizer cache key mismatch' | 'missing cache key' | 'cache key mismatch' - | 'cache disabled'; + | 'cache disabled' + | 'bundle references missing' + | 'bundle references outdated'; diff?: string; bundle: Bundle; } @@ -52,6 +54,7 @@ export function getBundleCacheEvent$( return Rx.defer(async () => { const events: BundleCacheEvent[] = []; const eligibleBundles: Bundle[] = []; + const bundleRefs = BundleRefs.fromBundles(config.bundles); for (const bundle of config.bundles) { if (!config.cache) { @@ -93,6 +96,32 @@ export function getBundleCacheEvent$( continue; } + const bundleRefExportIds = bundle.cache.getBundleRefExportIds(); + if (!bundleRefExportIds) { + events.push({ + type: 'bundle not cached', + reason: 'bundle references missing', + bundle, + }); + continue; + } + + const refs = bundleRefs.filterByExportIds(bundleRefExportIds); + + const bundleRefsDiff = diffCacheKey( + refs.map((r) => r.exportId).sort((a, b) => a.localeCompare(b)), + bundleRefExportIds + ); + if (bundleRefsDiff) { + events.push({ + type: 'bundle not cached', + reason: 'bundle references outdated', + diff: bundleRefsDiff, + bundle, + }); + continue; + } + eligibleBundles.push(bundle); } diff --git a/packages/kbn-optimizer/src/optimizer/cache_keys.ts b/packages/kbn-optimizer/src/optimizer/cache_keys.ts index 2766f6d63702b..d0aaad979485d 100644 --- a/packages/kbn-optimizer/src/optimizer/cache_keys.ts +++ b/packages/kbn-optimizer/src/optimizer/cache_keys.ts @@ -37,16 +37,6 @@ import { OptimizerConfig } from './optimizer_config'; const OPTIMIZER_DIR = Path.dirname(require.resolve('../../package.json')); const RELATIVE_DIR = Path.relative(REPO_ROOT, OPTIMIZER_DIR); -function omit(obj: T, keys: K[]): Omit { - const result: any = {}; - for (const [key, value] of Object.entries(obj) as any) { - if (!keys.includes(key)) { - result[key] = value; - } - } - return result as Omit; -} - export function diffCacheKey(expected?: unknown, actual?: unknown) { const expectedJson = jsonStable(expected, { space: ' ', @@ -185,7 +175,7 @@ export async function getOptimizerCacheKey(config: OptimizerConfig) { bootstrap, deletedPaths, modifiedTimes: {} as Record, - workerConfig: omit(config.getWorkerConfig('♻'), ['watch', 'profileWebpack', 'cache']), + workerConfig: config.getCacheableWorkerConfig(), }; const mtimes = await getMtimes(modifiedPaths); diff --git a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts index 2174c488ad6cc..bbd3ddc11f448 100644 --- a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts +++ b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts @@ -31,16 +31,19 @@ it('returns a bundle for core and each plugin', () => { directory: '/repo/plugins/foo', id: 'foo', isUiPlugin: true, + extraPublicDirs: [], }, { directory: '/repo/plugins/bar', id: 'bar', isUiPlugin: false, + extraPublicDirs: [], }, { directory: '/outside/of/repo/plugins/baz', id: 'baz', isUiPlugin: true, + extraPublicDirs: [], }, ], '/repo' @@ -49,17 +52,21 @@ it('returns a bundle for core and each plugin', () => { Array [ Object { "contextDir": /plugins/foo, - "entry": "./public/index", "id": "foo", "outputDir": /plugins/foo/target/public, + "publicDirNames": Array [ + "public", + ], "sourceRoot": , "type": "plugin", }, Object { "contextDir": "/outside/of/repo/plugins/baz", - "entry": "./public/index", "id": "baz", "outputDir": "/outside/of/repo/plugins/baz/target/public", + "publicDirNames": Array [ + "public", + ], "sourceRoot": , "type": "plugin", }, diff --git a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts index b75a8a6edc264..2635289088725 100644 --- a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts +++ b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts @@ -31,7 +31,7 @@ export function getPluginBundles(plugins: KibanaPlatformPlugin[], repoRoot: stri new Bundle({ type: 'plugin', id: p.id, - entry: './public/index', + publicDirNames: ['public', ...p.extraPublicDirs], sourceRoot: repoRoot, contextDir: p.directory, outputDir: Path.resolve(p.directory, 'target/public'), diff --git a/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.test.ts b/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.test.ts index e047b6d1e44cf..0961881df461c 100644 --- a/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.test.ts +++ b/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.test.ts @@ -37,21 +37,25 @@ it('parses kibana.json files of plugins found in pluginDirs', () => { Array [ Object { "directory": /packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar, + "extraPublicDirs": Array [], "id": "bar", "isUiPlugin": true, }, Object { "directory": /packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/baz, + "extraPublicDirs": Array [], "id": "baz", "isUiPlugin": false, }, Object { "directory": /packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/foo, + "extraPublicDirs": Array [], "id": "foo", "isUiPlugin": true, }, Object { "directory": /packages/kbn-optimizer/src/__fixtures__/mock_repo/test_plugins/test_baz, + "extraPublicDirs": Array [], "id": "test_baz", "isUiPlugin": false, }, diff --git a/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.ts b/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.ts index 992feab6cd364..bfc60a29efa27 100644 --- a/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.ts +++ b/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.ts @@ -26,6 +26,7 @@ export interface KibanaPlatformPlugin { readonly directory: string; readonly id: string; readonly isUiPlugin: boolean; + readonly extraPublicDirs: string[]; } /** @@ -64,9 +65,24 @@ function readKibanaPlatformPlugin(manifestPath: string): KibanaPlatformPlugin { throw new TypeError('expected new platform plugin manifest to have a string id'); } + let extraPublicDirs: string[] | undefined; + if (manifest.extraPublicDirs) { + if ( + !Array.isArray(manifest.extraPublicDirs) || + !manifest.extraPublicDirs.every((p) => typeof p === 'string') + ) { + throw new TypeError( + 'expected new platform plugin manifest to have an array of strings `extraPublicDirs` property' + ); + } + + extraPublicDirs = manifest.extraPublicDirs as string[]; + } + return { directory: Path.dirname(manifestPath), id: manifest.id, isUiPlugin: !!manifest.ui, + extraPublicDirs: extraPublicDirs || [], }; } diff --git a/packages/kbn-optimizer/src/optimizer/observe_worker.ts b/packages/kbn-optimizer/src/optimizer/observe_worker.ts index f5c944cefb76f..fef3efc13a516 100644 --- a/packages/kbn-optimizer/src/optimizer/observe_worker.ts +++ b/packages/kbn-optimizer/src/optimizer/observe_worker.ts @@ -17,14 +17,14 @@ * under the License. */ -import { fork, ChildProcess } from 'child_process'; import { Readable } from 'stream'; import { inspect } from 'util'; +import execa from 'execa'; import * as Rx from 'rxjs'; -import { map, takeUntil } from 'rxjs/operators'; +import { map, takeUntil, first, ignoreElements } from 'rxjs/operators'; -import { isWorkerMsg, WorkerConfig, WorkerMsg, Bundle } from '../common'; +import { isWorkerMsg, WorkerConfig, WorkerMsg, Bundle, BundleRefs } from '../common'; import { OptimizerConfig } from './optimizer_config'; @@ -42,7 +42,7 @@ export interface WorkerStarted { export type WorkerStatus = WorkerStdio | WorkerStarted; interface ProcResource extends Rx.Unsubscribable { - proc: ChildProcess; + proc: execa.ExecaChildProcess; } const isNumeric = (input: any) => String(input).match(/^[0-9]+$/); @@ -68,22 +68,20 @@ if (inspectFlagIndex !== -1) { function usingWorkerProc( config: OptimizerConfig, - workerConfig: WorkerConfig, - bundles: Bundle[], - fn: (proc: ChildProcess) => Rx.Observable + fn: (proc: execa.ExecaChildProcess) => Rx.Observable ) { return Rx.using( (): ProcResource => { - const args = [JSON.stringify(workerConfig), JSON.stringify(bundles.map((b) => b.toSpec()))]; - - const proc = fork(require.resolve('../worker/run_worker'), args, { - stdio: ['ignore', 'pipe', 'pipe', 'ipc'], - execArgv: [ + const proc = execa.node(require.resolve('../worker/run_worker'), [], { + nodeOptions: [ ...(inspectFlag && config.inspectWorkers ? [`${inspectFlag}=${inspectPortCounter++}`] : []), ...(config.maxWorkerCount <= 3 ? ['--max-old-space-size=2048'] : []), ], + buffer: false, + stderr: 'pipe', + stdout: 'pipe', }); return { @@ -123,6 +121,51 @@ function observeStdio$(stream: Readable, name: WorkerStdio['stream']) { ); } +/** + * We used to pass configuration to the worker as JSON encoded arguments, but they + * grew too large for argv, especially on Windows, so we had to move to an async init + * where we send the args over IPC. To keep the logic simple we basically mock the + * argv behavior and don't use complicated messages or anything so that state can + * be initialized in the worker before most of the code is run. + */ +function initWorker( + proc: execa.ExecaChildProcess, + config: OptimizerConfig, + workerConfig: WorkerConfig, + bundles: Bundle[] +) { + const msg$ = Rx.fromEvent<[unknown]>(proc, 'message').pipe( + // validate the initialization messages from the process + map(([msg]) => { + if (typeof msg === 'string') { + switch (msg) { + case 'init': + return 'init' as const; + case 'ready': + return 'ready' as const; + } + } + + throw new Error(`unexpected message from worker while initializing: [${inspect(msg)}]`); + }) + ); + + return Rx.concat( + msg$.pipe(first((msg) => msg === 'init')), + Rx.defer(() => { + proc.send({ + args: [ + JSON.stringify(workerConfig), + JSON.stringify(bundles.map((b) => b.toSpec())), + BundleRefs.fromBundles(config.bundles).toSpecJson(), + ], + }); + return []; + }), + msg$.pipe(first((msg) => msg === 'ready')) + ).pipe(ignoreElements()); +} + /** * Start a worker process with the specified `workerConfig` and * `bundles` and return an observable of the events related to @@ -134,10 +177,11 @@ export function observeWorker( workerConfig: WorkerConfig, bundles: Bundle[] ): Rx.Observable { - return usingWorkerProc(config, workerConfig, bundles, (proc) => { - let lastMsg: WorkerMsg; + return usingWorkerProc(config, (proc) => { + const init$ = initWorker(proc, config, workerConfig, bundles); - return Rx.merge( + let lastMsg: WorkerMsg; + const worker$: Rx.Observable = Rx.merge( Rx.of({ type: 'worker started', bundles, @@ -195,5 +239,7 @@ export function observeWorker( ) ) ); + + return Rx.concat(init$, worker$); }); } diff --git a/packages/kbn-optimizer/src/optimizer/optimizer_config.ts b/packages/kbn-optimizer/src/optimizer/optimizer_config.ts index 37d8a4f5eb8ae..04e0714d74c3e 100644 --- a/packages/kbn-optimizer/src/optimizer/optimizer_config.ts +++ b/packages/kbn-optimizer/src/optimizer/optimizer_config.ts @@ -20,7 +20,7 @@ import Path from 'path'; import Os from 'os'; -import { Bundle, WorkerConfig } from '../common'; +import { Bundle, WorkerConfig, CacheableWorkerConfig } from '../common'; import { findKibanaPlatformPlugins, KibanaPlatformPlugin } from './kibana_platform_plugins'; import { getPluginBundles } from './get_plugin_bundles'; @@ -34,6 +34,16 @@ function pickMaxWorkerCount(dist: boolean) { return Math.max(maxWorkers, 2); } +function omit(obj: T, keys: K[]): Omit { + const result: any = {}; + for (const [key, value] of Object.entries(obj) as any) { + if (!keys.includes(key)) { + result[key] = value; + } + } + return result as Omit; +} + interface Options { /** absolute path to root of the repo/build */ repoRoot: string; @@ -152,7 +162,7 @@ export class OptimizerConfig { new Bundle({ type: 'entry', id: 'core', - entry: './public/index', + publicDirNames: ['public', 'public/utils', 'utils'], sourceRoot: options.repoRoot, contextDir: Path.resolve(options.repoRoot, 'src/core'), outputDir: Path.resolve(options.repoRoot, 'src/core/target/public'), @@ -198,4 +208,14 @@ export class OptimizerConfig { browserslistEnv: this.dist ? 'production' : process.env.BROWSERSLIST_ENV || 'dev', }; } + + getCacheableWorkerConfig(): CacheableWorkerConfig { + return omit(this.getWorkerConfig('♻'), [ + // these config options don't change the output of the bundles, so + // should not invalidate caches when they change + 'watch', + 'profileWebpack', + 'cache', + ]); + } } diff --git a/packages/kbn-optimizer/src/worker/bundle_ref_module.ts b/packages/kbn-optimizer/src/worker/bundle_ref_module.ts new file mode 100644 index 0000000000000..cde25564cf528 --- /dev/null +++ b/packages/kbn-optimizer/src/worker/bundle_ref_module.ts @@ -0,0 +1,67 @@ +/* eslint-disable @kbn/eslint/require-license-header */ + +/** + * @notice + * + * This module was heavily inspired by the externals plugin that ships with webpack@97d58d31 + * MIT License http://www.opensource.org/licenses/mit-license.php + * Author Tobias Koppers @sokra + */ + +// @ts-ignore not typed by @types/webpack +import Module from 'webpack/lib/Module'; + +export class BundleRefModule extends Module { + public built = false; + public buildMeta?: any; + public buildInfo?: any; + public exportsArgument = '__webpack_exports__'; + + constructor(public readonly exportId: string) { + super('kbn/bundleRef', null); + } + + libIdent() { + return this.exportId; + } + + chunkCondition(chunk: any) { + return chunk.hasEntryModule(); + } + + identifier() { + return '@kbn/bundleRef ' + JSON.stringify(this.exportId); + } + + readableIdentifier() { + return this.identifier(); + } + + needRebuild() { + return false; + } + + build(_: any, __: any, ___: any, ____: any, callback: () => void) { + this.built = true; + this.buildMeta = {}; + this.buildInfo = {}; + callback(); + } + + source() { + return ` + __webpack_require__.r(__webpack_exports__); + var ns = __kbnBundles__.get('${this.exportId}'); + Object.defineProperties(__webpack_exports__, Object.getOwnPropertyDescriptors(ns)) + `; + } + + size() { + return 42; + } + + updateHash(hash: any) { + hash.update(this.identifier()); + super.updateHash(hash); + } +} diff --git a/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts b/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts new file mode 100644 index 0000000000000..9c4d5ed7f8a98 --- /dev/null +++ b/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts @@ -0,0 +1,192 @@ +/* eslint-disable @kbn/eslint/require-license-header */ + +/** + * @notice + * + * This module was heavily inspired by the externals plugin that ships with webpack@97d58d31 + * MIT License http://www.opensource.org/licenses/mit-license.php + * Author Tobias Koppers @sokra + */ + +import Path from 'path'; +import Fs from 'fs'; + +import webpack from 'webpack'; + +import { Bundle, BundleRefs, BundleRef } from '../common'; +import { BundleRefModule } from './bundle_ref_module'; + +const RESOLVE_EXTENSIONS = ['.js', '.ts', '.tsx']; + +function safeStat(path: string): Promise { + return new Promise((resolve, reject) => { + Fs.stat(path, (error, stat) => { + if (error?.code === 'ENOENT') { + resolve(undefined); + } else if (error) { + reject(error); + } else { + resolve(stat); + } + }); + }); +} + +interface RequestData { + context: string; + dependencies: Array<{ request: string }>; +} + +type Callback = (error?: any, result?: T) => void; +type ModuleFactory = (data: RequestData, callback: Callback) => void; + +export class BundleRefsPlugin { + private readonly resolvedRefEntryCache = new Map>(); + private readonly resolvedRequestCache = new Map>(); + private readonly ignorePrefix = Path.resolve(this.bundle.contextDir) + Path.sep; + + constructor(private readonly bundle: Bundle, private readonly bundleRefs: BundleRefs) {} + + /** + * Called by webpack when the plugin is passed in the webpack config + */ + public apply(compiler: webpack.Compiler) { + // called whenever the compiler starts to compile, passed the params + // that will be used to create the compilation + compiler.hooks.compile.tap('BundleRefsPlugin', (compilationParams: any) => { + // clear caches because a new compilation is starting, meaning that files have + // changed and we should re-run resolutions + this.resolvedRefEntryCache.clear(); + this.resolvedRequestCache.clear(); + + // hook into the creation of NormalModule instances in webpack, if the import + // statement leading to the creation of the module is pointing to a bundleRef + // entry then create a BundleRefModule instead of a NormalModule. + compilationParams.normalModuleFactory.hooks.factory.tap( + 'BundleRefsPlugin/normalModuleFactory/factory', + (wrappedFactory: ModuleFactory): ModuleFactory => (data, callback) => { + const context = data.context; + const dep = data.dependencies[0]; + + this.maybeReplaceImport(context, dep.request).then( + (module) => { + if (!module) { + wrappedFactory(data, callback); + } else { + callback(undefined, module); + } + }, + (error) => callback(error) + ); + } + ); + }); + } + + private cachedResolveRefEntry(ref: BundleRef) { + const cached = this.resolvedRefEntryCache.get(ref); + + if (cached) { + return cached; + } + + const absoluteRequest = Path.resolve(ref.contextDir, ref.entry); + const promise = this.cachedResolveRequest(absoluteRequest).then((resolved) => { + if (!resolved) { + throw new Error(`Unable to resolve request [${ref.entry}] relative to [${ref.contextDir}]`); + } + + return resolved; + }); + this.resolvedRefEntryCache.set(ref, promise); + return promise; + } + + private cachedResolveRequest(absoluteRequest: string) { + const cached = this.resolvedRequestCache.get(absoluteRequest); + + if (cached) { + return cached; + } + + const promise = this.resolveRequest(absoluteRequest); + this.resolvedRequestCache.set(absoluteRequest, promise); + return promise; + } + + private async resolveRequest(absoluteRequest: string) { + const stats = await safeStat(absoluteRequest); + if (stats && stats.isFile()) { + return absoluteRequest; + } + + // look for an index file in directories + if (stats?.isDirectory()) { + for (const ext of RESOLVE_EXTENSIONS) { + const indexPath = Path.resolve(absoluteRequest, `index${ext}`); + const indexStats = await safeStat(indexPath); + if (indexStats?.isFile()) { + return indexPath; + } + } + } + + // look for a file with one of the supported extensions + for (const ext of RESOLVE_EXTENSIONS) { + const filePath = `${absoluteRequest}${ext}`; + const fileStats = await safeStat(filePath); + if (fileStats?.isFile()) { + return filePath; + } + } + + return; + } + + /** + * Determine if an import request resolves to a bundleRef export id. If the + * request resolves to a bundle ref context but none of the exported directories + * then an error is thrown. If the request does not resolve to a bundleRef then + * undefined is returned. Otherwise it returns the referenced bundleRef. + */ + private async maybeReplaceImport(context: string, request: string) { + // ignore imports that have loaders defined or are not relative seeming + if (request.includes('!') || !request.startsWith('.')) { + return; + } + + const requestExt = Path.extname(request); + if (requestExt && !RESOLVE_EXTENSIONS.includes(requestExt)) { + return; + } + + const absoluteRequest = Path.resolve(context, request); + if (absoluteRequest.startsWith(this.ignorePrefix)) { + return; + } + + const resolved = await this.cachedResolveRequest(absoluteRequest); + if (!resolved) { + return; + } + + const eligibleRefs = this.bundleRefs.filterByContextPrefix(this.bundle, resolved); + if (!eligibleRefs.length) { + // import doesn't match a bundle context + return; + } + + for (const ref of eligibleRefs) { + const resolvedEntry = await this.cachedResolveRefEntry(ref); + if (resolved === resolvedEntry) { + return new BundleRefModule(ref.exportId); + } + } + + const bundleId = Array.from(new Set(eligibleRefs.map((r) => r.bundleId))).join(', '); + const publicDir = eligibleRefs.map((r) => r.entry).join(', '); + throw new Error( + `import [${request}] references a non-public export of the [${bundleId}] bundle and must point to one of the public directories: [${publicDir}]` + ); + } +} diff --git a/packages/kbn-optimizer/src/worker/entry_point_creator.ts b/packages/kbn-optimizer/src/worker/entry_point_creator.ts new file mode 100644 index 0000000000000..f8f41b2e13422 --- /dev/null +++ b/packages/kbn-optimizer/src/worker/entry_point_creator.ts @@ -0,0 +1,32 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +module.exports = function ({ + entries, +}: { + entries: Array<{ importId: string; requirePath: string }>; +}) { + const lines = entries.map(({ importId, requirePath }) => [ + `__kbnBundles__.define('${importId}', __webpack_require__, require.resolve('${requirePath}'))`, + ]); + + return { + code: lines.join('\n'), + }; +}; diff --git a/packages/kbn-optimizer/src/worker/run_compilers.ts b/packages/kbn-optimizer/src/worker/run_compilers.ts index 4ab289d031d72..de5e9372e9e7a 100644 --- a/packages/kbn-optimizer/src/worker/run_compilers.ts +++ b/packages/kbn-optimizer/src/worker/run_compilers.ts @@ -35,7 +35,9 @@ import { WorkerConfig, ascending, parseFilePath, + BundleRefs, } from '../common'; +import { BundleRefModule } from './bundle_ref_module'; import { getWebpackConfig } from './webpack.config'; import { isFailureStats, failedStatsToErrorMessage } from './webpack_helpers'; import { @@ -43,7 +45,6 @@ import { isNormalModule, isIgnoredModule, isConcatenatedModule, - WebpackNormalModule, getModulePath, } from './webpack_helpers'; @@ -98,40 +99,43 @@ const observeCompiler = ( }); } - const normalModules = stats.compilation.modules.filter( - (module): module is WebpackNormalModule => { - if (isNormalModule(module)) { - return true; - } + const bundleRefExportIds: string[] = []; + const referencedFiles = new Set(); + let normalModuleCount = 0; + + for (const module of stats.compilation.modules) { + if (isNormalModule(module)) { + normalModuleCount += 1; + const path = getModulePath(module); + const parsedPath = parseFilePath(path); - if (isExternalModule(module) || isIgnoredModule(module) || isConcatenatedModule(module)) { - return false; + if (!parsedPath.dirs.includes('node_modules')) { + referencedFiles.add(path); + continue; } - throw new Error(`Unexpected module type: ${inspect(module)}`); + const nmIndex = parsedPath.dirs.lastIndexOf('node_modules'); + const isScoped = parsedPath.dirs[nmIndex + 1].startsWith('@'); + referencedFiles.add( + Path.join( + parsedPath.root, + ...parsedPath.dirs.slice(0, nmIndex + 1 + (isScoped ? 2 : 1)), + 'package.json' + ) + ); + continue; } - ); - - const referencedFiles = new Set(); - for (const module of normalModules) { - const path = getModulePath(module); - const parsedPath = parseFilePath(path); + if (module instanceof BundleRefModule) { + bundleRefExportIds.push(module.exportId); + continue; + } - if (!parsedPath.dirs.includes('node_modules')) { - referencedFiles.add(path); + if (isExternalModule(module) || isIgnoredModule(module) || isConcatenatedModule(module)) { continue; } - const nmIndex = parsedPath.dirs.lastIndexOf('node_modules'); - const isScoped = parsedPath.dirs[nmIndex + 1].startsWith('@'); - referencedFiles.add( - Path.join( - parsedPath.root, - ...parsedPath.dirs.slice(0, nmIndex + 1 + (isScoped ? 2 : 1)), - 'package.json' - ) - ); + throw new Error(`Unexpected module type: ${inspect(module)}`); } const files = Array.from(referencedFiles).sort(ascending((p) => p)); @@ -150,14 +154,15 @@ const observeCompiler = ( ); bundle.cache.set({ + bundleRefExportIds, optimizerCacheKey: workerConfig.optimizerCacheKey, cacheKey: bundle.createCacheKey(files, mtimes), - moduleCount: normalModules.length, + moduleCount: normalModuleCount, files, }); return compilerMsgs.compilerSuccess({ - moduleCount: normalModules.length, + moduleCount: normalModuleCount, }); }) ); @@ -185,8 +190,14 @@ const observeCompiler = ( /** * Run webpack compilers */ -export const runCompilers = (workerConfig: WorkerConfig, bundles: Bundle[]) => { - const multiCompiler = webpack(bundles.map((def) => getWebpackConfig(def, workerConfig))); +export const runCompilers = ( + workerConfig: WorkerConfig, + bundles: Bundle[], + bundleRefs: BundleRefs +) => { + const multiCompiler = webpack( + bundles.map((def) => getWebpackConfig(def, bundleRefs, workerConfig)) + ); return Rx.merge( /** diff --git a/packages/kbn-optimizer/src/worker/run_worker.ts b/packages/kbn-optimizer/src/worker/run_worker.ts index f83c69477f471..781cf83624a1e 100644 --- a/packages/kbn-optimizer/src/worker/run_worker.ts +++ b/packages/kbn-optimizer/src/worker/run_worker.ts @@ -17,9 +17,19 @@ * under the License. */ +import { inspect } from 'util'; + import * as Rx from 'rxjs'; +import { take, mergeMap } from 'rxjs/operators'; -import { parseBundles, parseWorkerConfig, WorkerMsg, isWorkerMsg, WorkerMsgs } from '../common'; +import { + parseBundles, + parseWorkerConfig, + WorkerMsg, + isWorkerMsg, + WorkerMsgs, + BundleRefs, +} from '../common'; import { runCompilers } from './run_compilers'; @@ -73,14 +83,38 @@ setInterval(() => { } }, 1000).unref(); +function assertInitMsg(msg: unknown): asserts msg is { args: string[] } { + if (typeof msg !== 'object' || !msg) { + throw new Error(`expected init message to be an object: ${inspect(msg)}`); + } + + const { args } = msg as Record; + if (!args || !Array.isArray(args) || !args.every((a) => typeof a === 'string')) { + throw new Error( + `expected init message to have an 'args' property that's an array of strings: ${inspect(msg)}` + ); + } +} + Rx.defer(() => { - const workerConfig = parseWorkerConfig(process.argv[2]); - const bundles = parseBundles(process.argv[3]); + process.send!('init'); + + return Rx.fromEvent<[unknown]>(process as any, 'message').pipe( + take(1), + mergeMap(([msg]) => { + assertInitMsg(msg); + process.send!('ready'); + + const workerConfig = parseWorkerConfig(msg.args[0]); + const bundles = parseBundles(msg.args[1]); + const bundleRefs = BundleRefs.parseSpec(msg.args[2]); - // set BROWSERSLIST_ENV so that style/babel loaders see it before running compilers - process.env.BROWSERSLIST_ENV = workerConfig.browserslistEnv; + // set BROWSERSLIST_ENV so that style/babel loaders see it before running compilers + process.env.BROWSERSLIST_ENV = workerConfig.browserslistEnv; - return runCompilers(workerConfig, bundles); + return runCompilers(workerConfig, bundles, bundleRefs); + }) + ); }).subscribe( (msg) => { send(msg); diff --git a/packages/kbn-optimizer/src/worker/webpack.config.ts b/packages/kbn-optimizer/src/worker/webpack.config.ts index 5086bf0368374..2230750985070 100644 --- a/packages/kbn-optimizer/src/worker/webpack.config.ts +++ b/packages/kbn-optimizer/src/worker/webpack.config.ts @@ -17,10 +17,8 @@ * under the License. */ -import Fs from 'fs'; import Path from 'path'; -import normalizePath from 'normalize-path'; import { stringifyRequest } from 'loader-utils'; import webpack from 'webpack'; // @ts-ignore @@ -32,86 +30,20 @@ import { CleanWebpackPlugin } from 'clean-webpack-plugin'; import CompressionPlugin from 'compression-webpack-plugin'; import * as UiSharedDeps from '@kbn/ui-shared-deps'; -import { Bundle, WorkerConfig, parseDirPath, DisallowedSyntaxPlugin } from '../common'; +import { Bundle, BundleRefs, WorkerConfig, parseDirPath, DisallowedSyntaxPlugin } from '../common'; +import { BundleRefsPlugin } from './bundle_refs_plugin'; const BABEL_PRESET_PATH = require.resolve('@kbn/babel-preset/webpack_preset'); -const SHARED_BUNDLES = [ - { - type: 'entry', - id: 'core', - rootRelativeDir: 'src/core/public', - }, - { - type: 'plugin', - id: 'data', - rootRelativeDir: 'src/plugins/data/public', - }, - { - type: 'plugin', - id: 'kibanaReact', - rootRelativeDir: 'src/plugins/kibana_react/public', - }, - { - type: 'plugin', - id: 'kibanaUtils', - rootRelativeDir: 'src/plugins/kibana_utils/public', - }, - { - type: 'plugin', - id: 'esUiShared', - rootRelativeDir: 'src/plugins/es_ui_shared/public', - }, -]; - -/** - * Determine externals statements for require/import statements by looking - * for requests resolving to the primary public export of the data, kibanaReact, - * amd kibanaUtils plugins. If this module is being imported then rewrite - * the import to access the global `__kbnBundles__` variables and access - * the relavent properties from that global object. - * - * @param bundle - * @param context the directory containing the module which made `request` - * @param request the request for a module from a commonjs require() call or import statement - */ -function dynamicExternals(bundle: Bundle, context: string, request: string) { - // ignore imports that have loaders defined or are not relative seeming - if (request.includes('!') || !request.startsWith('.')) { - return; - } - - // determine the most acurate resolution string we can without running full resolution - const rootRelative = normalizePath( - Path.relative(bundle.sourceRoot, Path.resolve(context, request)) - ); - for (const sharedBundle of SHARED_BUNDLES) { - if ( - rootRelative !== sharedBundle.rootRelativeDir || - `${bundle.type}/${bundle.id}` === `${sharedBundle.type}/${sharedBundle.id}` - ) { - continue; - } - - return `__kbnBundles__['${sharedBundle.type}/${sharedBundle.id}']`; - } - - // import doesn't match a root public import - return undefined; -} - -export function getWebpackConfig(bundle: Bundle, worker: WorkerConfig) { - const extensions = ['.js', '.ts', '.tsx', '.json']; - const entryExtension = extensions.find((ext) => - Fs.existsSync(Path.resolve(bundle.contextDir, bundle.entry) + ext) - ); +export function getWebpackConfig(bundle: Bundle, bundleRefs: BundleRefs, worker: WorkerConfig) { + const ENTRY_CREATOR = require.resolve('./entry_point_creator'); const commonConfig: webpack.Configuration = { node: { fs: 'empty' }, context: bundle.contextDir, cache: true, entry: { - [bundle.id]: `${bundle.entry}${entryExtension}`, + [bundle.id]: ENTRY_CREATOR, }, devtool: worker.dist ? false : '#cheap-source-map', @@ -126,27 +58,19 @@ export function getWebpackConfig(bundle: Bundle, worker: WorkerConfig) { info.absoluteResourcePath )}${info.query}`, jsonpFunction: `${bundle.id}_bundle_jsonpfunction`, - // When the entry point is loaded, assign it's default export - // to a key on the global `__kbnBundles__` object. - library: ['__kbnBundles__', `${bundle.type}/${bundle.id}`], }, optimization: { noEmitOnErrors: true, }, - externals: [ - UiSharedDeps.externals, - function (context, request, cb) { - try { - cb(undefined, dynamicExternals(bundle, context, request)); - } catch (error) { - cb(error, undefined); - } - }, - ], + externals: [UiSharedDeps.externals], - plugins: [new CleanWebpackPlugin(), new DisallowedSyntaxPlugin()], + plugins: [ + new CleanWebpackPlugin(), + new DisallowedSyntaxPlugin(), + new BundleRefsPlugin(bundle, bundleRefs), + ], module: { // no parse rules for a few known large packages which have no require() statements @@ -159,6 +83,36 @@ export function getWebpackConfig(bundle: Bundle, worker: WorkerConfig) { ], rules: [ + { + include: [ENTRY_CREATOR], + use: [ + { + loader: UiSharedDeps.publicPathLoader, + options: { + key: bundle.id, + }, + }, + { + loader: require.resolve('val-loader'), + options: { + entries: bundle.publicDirNames.map((name) => { + const absolute = Path.resolve(bundle.contextDir, name); + const newContext = Path.dirname(ENTRY_CREATOR); + const importId = `${bundle.type}/${bundle.id}/${name}`; + + // relative path from context of the ENTRY_CREATOR, with linux path separators + let requirePath = Path.relative(newContext, absolute).split('\\').join('/'); + if (!requirePath.startsWith('.')) { + // ensure requirePath is identified by node as relative + requirePath = `./${requirePath}`; + } + + return { importId, requirePath }; + }), + }, + }, + ], + }, { test: /\.css$/, include: /node_modules/, @@ -289,13 +243,6 @@ export function getWebpackConfig(bundle: Bundle, worker: WorkerConfig) { }, }, }, - { - include: [`${Path.resolve(bundle.contextDir, bundle.entry)}${entryExtension}`], - loader: UiSharedDeps.publicPathLoader, - options: { - key: bundle.id, - }, - }, { test: /\.(html|md|txt|tmpl)$/, use: { @@ -306,7 +253,7 @@ export function getWebpackConfig(bundle: Bundle, worker: WorkerConfig) { }, resolve: { - extensions, + extensions: ['.js', '.ts', '.tsx', 'json'], mainFields: ['browser', 'main'], alias: { tinymath: require.resolve('tinymath/lib/tinymath.es5.js'), diff --git a/src/core/public/plugins/plugin_reader.test.ts b/src/core/public/plugins/plugin_reader.test.ts index d4324f81de8e6..b3bc84f30daac 100644 --- a/src/core/public/plugins/plugin_reader.test.ts +++ b/src/core/public/plugins/plugin_reader.test.ts @@ -17,25 +17,37 @@ * under the License. */ -import { CoreWindow, read, UnknownPluginInitializer } from './plugin_reader'; +import { CoreWindow, read } from './plugin_reader'; + +const coreWindow: CoreWindow & { + __kbnBundles__: { stub(key: string, value: any): void }; +} = window as any; -const coreWindow: CoreWindow = window as any; beforeEach(() => { - coreWindow.__kbnBundles__ = {}; + const stubs = new Map(); + coreWindow.__kbnBundles__ = { + get(key) { + return stubs.get(key); + }, + has(key) { + return stubs.has(key); + }, + stub(key, value) { + stubs.set(key, value); + }, + }; }); it('handles undefined plugin exports', () => { - coreWindow.__kbnBundles__['plugin/foo'] = undefined; - expect(() => { read('foo'); }).toThrowError(`Definition of plugin "foo" not found and may have failed to load.`); }); it('handles plugin exports with a "plugin" export that is not a function', () => { - coreWindow.__kbnBundles__['plugin/foo'] = { + coreWindow.__kbnBundles__.stub('plugin/foo/public', { plugin: 1234, - } as any; + }); expect(() => { read('foo'); @@ -43,11 +55,8 @@ it('handles plugin exports with a "plugin" export that is not a function', () => }); it('returns the plugin initializer when the "plugin" named export is a function', () => { - const plugin: UnknownPluginInitializer = () => { - return undefined as any; - }; - - coreWindow.__kbnBundles__['plugin/foo'] = { plugin }; + const plugin = () => {}; + coreWindow.__kbnBundles__.stub('plugin/foo/public', { plugin }); expect(read('foo')).toBe(plugin); }); diff --git a/src/core/public/plugins/plugin_reader.ts b/src/core/public/plugins/plugin_reader.ts index 1907dfa6a3e99..d80bda7483775 100644 --- a/src/core/public/plugins/plugin_reader.ts +++ b/src/core/public/plugins/plugin_reader.ts @@ -31,7 +31,8 @@ export type UnknownPluginInitializer = PluginInitializer { optionalPlugins: true, ui: true, server: true, + extraPublicDirs: true, }; return new Set(Object.keys(manifestFields)); @@ -70,7 +71,11 @@ const KNOWN_MANIFEST_FIELDS = (() => { * @param packageInfo Kibana package info. * @internal */ -export async function parseManifest(pluginPath: string, packageInfo: PackageInfo, log: Logger) { +export async function parseManifest( + pluginPath: string, + packageInfo: PackageInfo, + log: Logger +): Promise { const manifestPath = resolve(pluginPath, MANIFEST_FILE_NAME); let manifestContent; @@ -130,6 +135,19 @@ export async function parseManifest(pluginPath: string, packageInfo: PackageInfo ); } + if ( + manifest.extraPublicDirs && + (!Array.isArray(manifest.extraPublicDirs) || + !manifest.extraPublicDirs.every((dir) => typeof dir === 'string')) + ) { + throw PluginDiscoveryError.invalidManifest( + manifestPath, + new Error( + `The "extraPublicDirs" in plugin manifest for "${manifest.id}" should be an array of strings.` + ) + ); + } + const expectedKibanaVersion = typeof manifest.kibanaVersion === 'string' && manifest.kibanaVersion ? manifest.kibanaVersion @@ -175,6 +193,7 @@ export async function parseManifest(pluginPath: string, packageInfo: PackageInfo optionalPlugins: Array.isArray(manifest.optionalPlugins) ? manifest.optionalPlugins : [], ui: includesUiPlugin, server: includesServerPlugin, + extraPublicDirs: manifest.extraPublicDirs, }; } diff --git a/src/core/server/plugins/types.ts b/src/core/server/plugins/types.ts index 4fa4e1780e596..2ca5c9f6ed3c5 100644 --- a/src/core/server/plugins/types.ts +++ b/src/core/server/plugins/types.ts @@ -153,6 +153,14 @@ export interface PluginManifest { * Specifies whether plugin includes some server-side specific functionality. */ readonly server: boolean; + + /** + * Specifies directory names that can be imported by other ui-plugins built + * using the same instance of the @kbn/optimizer. A temporary measure we plan + * to replace with better mechanisms for sharing static code between plugins + * @deprecated + */ + readonly extraPublicDirs?: string[]; } /** diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index bfdb74a7bf082..75ba090e76512 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -1521,6 +1521,8 @@ export interface PluginInitializerContext { // @public export interface PluginManifest { readonly configPath: ConfigPath; + // @deprecated + readonly extraPublicDirs?: string[]; readonly id: PluginName; readonly kibanaVersion: string; readonly optionalPlugins: readonly PluginName[]; @@ -2544,8 +2546,8 @@ export const validBodyOutput: readonly ["data", "stream"]; // src/core/server/legacy/types.ts:165:3 - (ae-forgotten-export) The symbol "LegacyNavLinkSpec" needs to be exported by the entry point index.d.ts // src/core/server/legacy/types.ts:166:3 - (ae-forgotten-export) The symbol "LegacyAppSpec" needs to be exported by the entry point index.d.ts // src/core/server/legacy/types.ts:167:16 - (ae-forgotten-export) The symbol "LegacyPluginSpec" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:230:3 - (ae-forgotten-export) The symbol "KibanaConfigType" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:230:3 - (ae-forgotten-export) The symbol "SharedGlobalConfigKeys" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:232:3 - (ae-forgotten-export) The symbol "PathConfigType" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:238:3 - (ae-forgotten-export) The symbol "KibanaConfigType" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:238:3 - (ae-forgotten-export) The symbol "SharedGlobalConfigKeys" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:240:3 - (ae-forgotten-export) The symbol "PathConfigType" needs to be exported by the entry point index.d.ts ``` diff --git a/src/legacy/ui/ui_render/bootstrap/app_bootstrap.js b/src/legacy/ui/ui_render/bootstrap/app_bootstrap.js index 0e6936dd64a15..19f75317883d7 100644 --- a/src/legacy/ui/ui_render/bootstrap/app_bootstrap.js +++ b/src/legacy/ui/ui_render/bootstrap/app_bootstrap.js @@ -22,9 +22,11 @@ import { createHash } from 'crypto'; import { readFile } from 'fs'; import { resolve } from 'path'; +import { kbnBundlesLoaderSource } from './kbn_bundles_loader_source'; + export class AppBootstrap { constructor({ templateData }) { - this.templateData = templateData; + this.templateData = { ...templateData, kbnBundlesLoaderSource }; this._rawTemplate = undefined; } diff --git a/src/legacy/ui/ui_render/bootstrap/kbn_bundles_loader_source.js b/src/legacy/ui/ui_render/bootstrap/kbn_bundles_loader_source.js new file mode 100644 index 0000000000000..cb5488118cc77 --- /dev/null +++ b/src/legacy/ui/ui_render/bootstrap/kbn_bundles_loader_source.js @@ -0,0 +1,51 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +module.exports = { + kbnBundlesLoaderSource: `(${kbnBundlesLoader.toString()})();`, +}; + +function kbnBundlesLoader() { + var modules = {}; + + function has(prop) { + return Object.prototype.hasOwnProperty.call(modules, prop); + } + + function define(key, bundleRequire, bundleModuleKey) { + if (has(key)) { + throw new Error('__kbnBundles__ already has a module defined for "' + key + '"'); + } + + modules[key] = { + bundleRequire, + bundleModuleKey, + }; + } + + function get(key) { + if (!has(key)) { + throw new Error('__kbnBundles__ does not have a module defined for "' + key + '"'); + } + + return modules[key].bundleRequire(modules[key].bundleModuleKey); + } + + return { has: has, define: define, get: get }; +} diff --git a/src/legacy/ui/ui_render/bootstrap/template.js.hbs b/src/legacy/ui/ui_render/bootstrap/template.js.hbs index fb9c5643a8804..4739b31eec6ba 100644 --- a/src/legacy/ui/ui_render/bootstrap/template.js.hbs +++ b/src/legacy/ui/ui_render/bootstrap/template.js.hbs @@ -2,6 +2,7 @@ var kbnCsp = JSON.parse(document.querySelector('kbn-csp').getAttribute('data')); window.__kbnStrictCsp__ = kbnCsp.strictCsp; window.__kbnDarkMode__ = {{darkMode}}; window.__kbnPublicPath__ = {{publicPathMap}}; +window.__kbnBundles__ = {{kbnBundlesLoaderSource}} if (window.__kbnStrictCsp__ && window.__kbnCspNotEnforced__) { var legacyBrowserError = document.getElementById('kbn_legacy_browser_error'); @@ -77,12 +78,7 @@ if (window.__kbnStrictCsp__ && window.__kbnCspNotEnforced__) { {{/each}} ], function () { {{#unless legacyBundlePath}} - if (!__kbnBundles__ || !__kbnBundles__['entry/core'] || typeof __kbnBundles__['entry/core'].__kbnBootstrap__ !== 'function') { - console.error('entry/core bundle did not load correctly'); - failure(); - } else { - __kbnBundles__['entry/core'].__kbnBootstrap__() - } + __kbnBundles__.get('entry/core/public').__kbnBootstrap__(); {{/unless}} load([ diff --git a/src/legacy/ui/ui_render/ui_render_mixin.js b/src/legacy/ui/ui_render/ui_render_mixin.js index 0f38b6d7b40bd..b181c71b972d5 100644 --- a/src/legacy/ui/ui_render/ui_render_mixin.js +++ b/src/legacy/ui/ui_render/ui_render_mixin.js @@ -26,8 +26,6 @@ import { AppBootstrap } from './bootstrap'; import { getApmConfig } from '../apm'; import { DllCompiler } from '../../../optimize/dynamic_dll_plugin'; -const uniq = (...items) => Array.from(new Set(items)); - /** * @typedef {import('../../server/kbn_server').default} KbnServer * @typedef {import('../../server/kbn_server').ResponseToolkit} ResponseToolkit @@ -141,15 +139,7 @@ export function uiRenderMixin(kbnServer, server, config) { ]), ]; - const kpPluginIds = uniq( - // load these plugins first, they are "shared" and other bundles access their - // public/index exports without considering topographic sorting by plugin deps (for now) - 'kibanaUtils', - 'kibanaReact', - 'data', - 'esUiShared', - ...kbnServer.newPlatform.__internals.uiPlugins.public.keys() - ); + const kpPluginIds = Array.from(kbnServer.newPlatform.__internals.uiPlugins.public.keys()); const jsDependencyPaths = [ ...UiSharedDeps.jsDepFilenames.map( diff --git a/src/plugins/data/kibana.json b/src/plugins/data/kibana.json index f5df747f17e1e..7c703596edda0 100644 --- a/src/plugins/data/kibana.json +++ b/src/plugins/data/kibana.json @@ -7,5 +7,11 @@ "expressions", "uiActions" ], - "optionalPlugins": ["usageCollection"] + "optionalPlugins": ["usageCollection"], + "extraPublicDirs": [ + "common", + "common/types", + "common/utils/abort_utils", + "public/actions/value_click_action" + ] } diff --git a/src/plugins/es_ui_shared/kibana.json b/src/plugins/es_ui_shared/kibana.json index 5a3db3b344090..28a461d7e3777 100644 --- a/src/plugins/es_ui_shared/kibana.json +++ b/src/plugins/es_ui_shared/kibana.json @@ -1,5 +1,14 @@ { "id": "esUiShared", "version": "kibana", - "ui": true + "ui": true, + "extraPublicDirs": [ + "public/request/np_ready_request", + "static/ace_x_json/hooks", + "static/forms/components", + "static/forms/helpers", + "static/forms/helpers/field_validators/types", + "static/forms/hook_form_lib", + "static/validators/string" + ] } diff --git a/src/plugins/expressions/kibana.json b/src/plugins/expressions/kibana.json index 5d2112103e94d..4774c69cc29ff 100644 --- a/src/plugins/expressions/kibana.json +++ b/src/plugins/expressions/kibana.json @@ -5,5 +5,6 @@ "ui": true, "requiredPlugins": [ "bfetch" - ] + ], + "extraPublicDirs": ["common", "common/fonts"] } diff --git a/src/plugins/inspector/kibana.json b/src/plugins/inspector/kibana.json index 39d3ff65eed53..e77723b44fcef 100644 --- a/src/plugins/inspector/kibana.json +++ b/src/plugins/inspector/kibana.json @@ -2,5 +2,9 @@ "id": "inspector", "version": "kibana", "server": false, - "ui": true + "ui": true, + "extraPublicDirs": [ + "common", + "common/adapters/request" + ] } diff --git a/src/plugins/kibana_legacy/kibana.json b/src/plugins/kibana_legacy/kibana.json index e96b4859a36d0..606acd8b88b05 100644 --- a/src/plugins/kibana_legacy/kibana.json +++ b/src/plugins/kibana_legacy/kibana.json @@ -2,5 +2,6 @@ "id": "kibanaLegacy", "version": "kibana", "server": true, - "ui": true + "ui": true, + "extraPublicDirs": ["common/kbn_base_url"] } diff --git a/src/plugins/kibana_utils/kibana.json b/src/plugins/kibana_utils/kibana.json index 6fa39d82d1021..8dbcee776af18 100644 --- a/src/plugins/kibana_utils/kibana.json +++ b/src/plugins/kibana_utils/kibana.json @@ -1,5 +1,6 @@ { "id": "kibanaUtils", "version": "kibana", - "ui": true + "ui": true, + "extraPublicDirs": ["common", "common/state_containers", "demos/state_containers/todomvc"] } diff --git a/src/plugins/saved_objects_management/kibana.json b/src/plugins/saved_objects_management/kibana.json index 22135ce4558ae..6184d890c415c 100644 --- a/src/plugins/saved_objects_management/kibana.json +++ b/src/plugins/saved_objects_management/kibana.json @@ -4,5 +4,6 @@ "server": true, "ui": true, "requiredPlugins": ["home", "management", "data"], - "optionalPlugins": ["dashboard", "visualizations", "discover"] + "optionalPlugins": ["dashboard", "visualizations", "discover"], + "extraPublicDirs": ["public/lib"] } diff --git a/src/plugins/telemetry/kibana.json b/src/plugins/telemetry/kibana.json index f623f4f2a565d..a497597762520 100644 --- a/src/plugins/telemetry/kibana.json +++ b/src/plugins/telemetry/kibana.json @@ -6,5 +6,8 @@ "requiredPlugins": [ "telemetryCollectionManager", "usageCollection" + ], + "extraPublicDirs": [ + "common/constants" ] } diff --git a/src/plugins/telemetry_management_section/kibana.json b/src/plugins/telemetry_management_section/kibana.json index 3364833def4d6..de257d0f0b044 100644 --- a/src/plugins/telemetry_management_section/kibana.json +++ b/src/plugins/telemetry_management_section/kibana.json @@ -6,5 +6,6 @@ "requiredPlugins": [ "advancedSettings", "telemetry" - ] + ], + "extraPublicDirs": ["public/components"] } diff --git a/test/interpreter_functional/plugins/kbn_tp_run_pipeline/public/np_ready/app/components/main.tsx b/test/interpreter_functional/plugins/kbn_tp_run_pipeline/public/np_ready/app/components/main.tsx index a50248a5b6fa3..5e7305e1279ca 100644 --- a/test/interpreter_functional/plugins/kbn_tp_run_pipeline/public/np_ready/app/components/main.tsx +++ b/test/interpreter_functional/plugins/kbn_tp_run_pipeline/public/np_ready/app/components/main.tsx @@ -21,7 +21,7 @@ import React from 'react'; import { EuiPage, EuiPageBody, EuiPageContent, EuiPageContentHeader } from '@elastic/eui'; import { first } from 'rxjs/operators'; import { IInterpreterRenderHandlers, ExpressionValue } from 'src/plugins/expressions'; -import { RequestAdapter, DataAdapter } from '../../../../../../../../src/plugins/inspector'; +import { RequestAdapter, DataAdapter } from '../../../../../../../../src/plugins/inspector/public'; import { Adapters, ExpressionRenderHandler } from '../../types'; import { getExpressions } from '../../services'; diff --git a/test/scripts/jenkins_build_kibana.sh b/test/scripts/jenkins_build_kibana.sh index efef91b8581a3..9fba4cc75958a 100755 --- a/test/scripts/jenkins_build_kibana.sh +++ b/test/scripts/jenkins_build_kibana.sh @@ -2,9 +2,16 @@ source src/dev/ci_setup/setup_env.sh -echo " -> building kibana platform plugins" +echo " -> building examples separate from test plugins" node scripts/build_kibana_platform_plugins \ --oss \ + --examples \ + --verbose; + +echo " -> building test plugins" +node scripts/build_kibana_platform_plugins \ + --oss \ + --no-examples \ --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \ --verbose; diff --git a/test/scripts/jenkins_xpack_build_kibana.sh b/test/scripts/jenkins_xpack_build_kibana.sh index 8d3ef525e60e5..e887c00eb0ae9 100755 --- a/test/scripts/jenkins_xpack_build_kibana.sh +++ b/test/scripts/jenkins_xpack_build_kibana.sh @@ -3,8 +3,15 @@ cd "$KIBANA_DIR" source src/dev/ci_setup/setup_env.sh -echo " -> building kibana platform plugins" +echo " -> building examples separate from test plugins" node scripts/build_kibana_platform_plugins \ + --examples \ + --verbose; + +echo " -> building test plugins" +node scripts/build_kibana_platform_plugins \ + --no-examples \ + --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \ --scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \ --scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \ --scan-dir "$XPACK_DIR/test/plugin_api_integration/plugins" \ diff --git a/x-pack/plugins/advanced_ui_actions/public/dynamic_actions/dynamic_action_manager.ts b/x-pack/plugins/advanced_ui_actions/public/dynamic_actions/dynamic_action_manager.ts index 3e37bac0a17af..58344026079e7 100644 --- a/x-pack/plugins/advanced_ui_actions/public/dynamic_actions/dynamic_action_manager.ts +++ b/x-pack/plugins/advanced_ui_actions/public/dynamic_actions/dynamic_action_manager.ts @@ -12,7 +12,10 @@ import { UiActionsActionDefinition as ActionDefinition, } from '../../../../../src/plugins/ui_actions/public'; import { defaultState, transitions, selectors, State } from './dynamic_action_manager_state'; -import { StateContainer, createStateContainer } from '../../../../../src/plugins/kibana_utils'; +import { + StateContainer, + createStateContainer, +} from '../../../../../src/plugins/kibana_utils/common'; import { StartContract } from '../plugin'; import { SerializedAction, SerializedEvent } from './types'; diff --git a/x-pack/plugins/alerting/kibana.json b/x-pack/plugins/alerting/kibana.json index 59c4bb2221b0a..4c7c10a11cf1f 100644 --- a/x-pack/plugins/alerting/kibana.json +++ b/x-pack/plugins/alerting/kibana.json @@ -6,5 +6,6 @@ "kibanaVersion": "kibana", "configPath": ["xpack", "alerting"], "requiredPlugins": ["licensing", "taskManager", "encryptedSavedObjects", "actions", "eventLog"], - "optionalPlugins": ["usageCollection", "spaces", "security"] + "optionalPlugins": ["usageCollection", "spaces", "security"], + "extraPublicDirs": ["common", "common/parse_duration"] } diff --git a/x-pack/plugins/apm/kibana.json b/x-pack/plugins/apm/kibana.json index a6afd7ede3453..f140002099410 100644 --- a/x-pack/plugins/apm/kibana.json +++ b/x-pack/plugins/apm/kibana.json @@ -25,5 +25,6 @@ "configPath": [ "xpack", "apm" - ] + ], + "extraPublicDirs": ["public/style/variables"] } diff --git a/x-pack/plugins/canvas/common/lib/autocomplete.ts b/x-pack/plugins/canvas/common/lib/autocomplete.ts index 982aee1ea19c8..51ce6a953238c 100644 --- a/x-pack/plugins/canvas/common/lib/autocomplete.ts +++ b/x-pack/plugins/canvas/common/lib/autocomplete.ts @@ -14,7 +14,7 @@ import { ExpressionFunction, ExpressionFunctionParameter, getByAlias, -} from '../../../../../src/plugins/expressions'; +} from '../../../../../src/plugins/expressions/common'; const MARKER = 'CANVAS_SUGGESTION_MARKER'; diff --git a/x-pack/plugins/data_enhanced/kibana.json b/x-pack/plugins/data_enhanced/kibana.json index 443bb63a27799..1be55d2b7a635 100644 --- a/x-pack/plugins/data_enhanced/kibana.json +++ b/x-pack/plugins/data_enhanced/kibana.json @@ -8,6 +8,7 @@ "requiredPlugins": [ "data" ], + "optionalPlugins": ["kibanaReact", "kibanaUtils"], "server": true, "ui": true } diff --git a/x-pack/plugins/features/kibana.json b/x-pack/plugins/features/kibana.json index 6e51f3b650710..29e78d75c948e 100644 --- a/x-pack/plugins/features/kibana.json +++ b/x-pack/plugins/features/kibana.json @@ -5,5 +5,6 @@ "optionalPlugins": ["visTypeTimelion"], "configPath": ["xpack", "features"], "server": true, - "ui": true + "ui": true, + "extraPublicDirs": ["common"] } diff --git a/x-pack/plugins/ingest_manager/kibana.json b/x-pack/plugins/ingest_manager/kibana.json index 382ea0444093d..35447139607a6 100644 --- a/x-pack/plugins/ingest_manager/kibana.json +++ b/x-pack/plugins/ingest_manager/kibana.json @@ -5,5 +5,6 @@ "ui": true, "configPath": ["xpack", "ingestManager"], "requiredPlugins": ["licensing", "data", "encryptedSavedObjects"], - "optionalPlugins": ["security", "features", "cloud"] + "optionalPlugins": ["security", "features", "cloud"], + "extraPublicDirs": ["common"] } diff --git a/x-pack/plugins/lens/kibana.json b/x-pack/plugins/lens/kibana.json index ce544b31b88ef..73c9986b7f746 100644 --- a/x-pack/plugins/lens/kibana.json +++ b/x-pack/plugins/lens/kibana.json @@ -13,5 +13,6 @@ "visualizations" ], "optionalPlugins": ["embeddable", "usageCollection", "taskManager"], - "configPath": ["xpack", "lens"] + "configPath": ["xpack", "lens"], + "extraPublicDirs": ["common/constants"] } diff --git a/x-pack/plugins/license_management/kibana.json b/x-pack/plugins/license_management/kibana.json index be28c8e978d8a..6da923c5cff5a 100644 --- a/x-pack/plugins/license_management/kibana.json +++ b/x-pack/plugins/license_management/kibana.json @@ -5,5 +5,6 @@ "ui": true, "requiredPlugins": ["home", "licensing", "management"], "optionalPlugins": ["telemetry"], - "configPath": ["xpack", "license_management"] + "configPath": ["xpack", "license_management"], + "extraPublicDirs": ["common/constants"] } diff --git a/x-pack/plugins/maps/kibana.json b/x-pack/plugins/maps/kibana.json index b8bad47327f22..995913333084a 100644 --- a/x-pack/plugins/maps/kibana.json +++ b/x-pack/plugins/maps/kibana.json @@ -14,5 +14,6 @@ "visualizations", "embeddable" ], - "ui": true + "ui": true, + "extraPublicDirs": ["common/constants"] } diff --git a/x-pack/plugins/ml/kibana.json b/x-pack/plugins/ml/kibana.json index e9d4aff3484b1..d54f1941c7a25 100644 --- a/x-pack/plugins/ml/kibana.json +++ b/x-pack/plugins/ml/kibana.json @@ -24,5 +24,13 @@ "licenseManagement" ], "server": true, - "ui": true + "ui": true, + "extraPublicDirs": [ + "common/constants/anomalies", + "common/util/anomaly_utils", + "common/util/errors", + "common/util/validators", + "public/application/components/data_grid", + "public/application/data_frame_analytics/common" + ] } diff --git a/x-pack/plugins/observability/kibana.json b/x-pack/plugins/observability/kibana.json index 712a46f76bb74..bab78c2a9ee11 100644 --- a/x-pack/plugins/observability/kibana.json +++ b/x-pack/plugins/observability/kibana.json @@ -10,5 +10,6 @@ "licensing" ], "ui": true, - "server": true + "server": true, + "extraPublicDirs": ["public/typings/eui_styled_components"] } diff --git a/x-pack/plugins/spaces/kibana.json b/x-pack/plugins/spaces/kibana.json index 7e6d0425f2ae0..9483cb67392c4 100644 --- a/x-pack/plugins/spaces/kibana.json +++ b/x-pack/plugins/spaces/kibana.json @@ -13,5 +13,6 @@ "savedObjectsManagement" ], "server": true, - "ui": true + "ui": true, + "extraPublicDirs": ["common"] } diff --git a/x-pack/plugins/triggers_actions_ui/kibana.json b/x-pack/plugins/triggers_actions_ui/kibana.json index 11cbacee08afc..1abe7218176ac 100644 --- a/x-pack/plugins/triggers_actions_ui/kibana.json +++ b/x-pack/plugins/triggers_actions_ui/kibana.json @@ -5,5 +5,6 @@ "ui": true, "optionalPlugins": ["alerting", "alertingBuiltins"], "requiredPlugins": ["management", "charts", "data"], - "configPath": ["xpack", "trigger_actions_ui"] + "configPath": ["xpack", "trigger_actions_ui"], + "extraPublicDirs": ["public/common", "public/common/constants"] }