From 306892d746a1af211f8a559dfb2f1a2689e97393 Mon Sep 17 00:00:00 2001 From: spalger Date: Thu, 11 Jun 2020 22:36:15 -0700 Subject: [PATCH 01/31] add support for bundle references --- examples/demo_search/kibana.json | 3 +- examples/embeddable_examples/kibana.json | 3 +- examples/url_generators_examples/kibana.json | 5 +- packages/kbn-optimizer/package.json | 1 + .../kbn-optimizer/src/common/bundle.test.ts | 17 +- packages/kbn-optimizer/src/common/bundle.ts | 35 ++- .../kbn-optimizer/src/common/bundle_cache.ts | 5 + .../kbn-optimizer/src/common/bundle_refs.ts | 233 ++++++++++++++++++ packages/kbn-optimizer/src/common/index.ts | 1 + .../assign_bundles_to_workers.test.ts | 3 +- .../src/optimizer/bundle_cache.ts | 33 ++- .../kbn-optimizer/src/optimizer/cache_keys.ts | 12 +- .../src/optimizer/get_plugin_bundles.ts | 3 +- .../src/optimizer/kibana_platform_plugins.ts | 48 ++++ .../src/optimizer/observe_worker.ts | 8 +- .../src/optimizer/optimizer_config.ts | 27 +- .../src/worker/entry_point_creator.ts | 28 +++ .../kbn-optimizer/src/worker/run_compilers.ts | 15 +- .../kbn-optimizer/src/worker/run_worker.ts | 12 +- .../src/worker/webpack.config.ts | 118 +++------ .../discovery/plugin_manifest_parser.ts | 2 + src/core/server/plugins/types.ts | 6 + src/plugins/data/kibana.json | 3 +- src/plugins/es_ui_shared/kibana.json | 10 +- src/plugins/expressions/kibana.json | 3 +- src/plugins/inspector/kibana.json | 3 +- src/plugins/kibana_legacy/kibana.json | 3 +- src/plugins/kibana_utils/kibana.json | 7 +- .../saved_objects_management/kibana.json | 3 +- src/plugins/telemetry/kibana.json | 3 + x-pack/plugins/alerts/kibana.json | 3 +- x-pack/plugins/apm/kibana.json | 3 + .../plugins/canvas/common/lib/autocomplete.ts | 2 +- x-pack/plugins/data_enhanced/kibana.json | 1 + x-pack/plugins/features/kibana.json | 3 +- x-pack/plugins/ingest_manager/kibana.json | 3 +- x-pack/plugins/lens/kibana.json | 3 +- x-pack/plugins/license_management/kibana.json | 3 +- x-pack/plugins/maps/kibana.json | 3 +- x-pack/plugins/spaces/kibana.json | 3 +- .../plugins/triggers_actions_ui/kibana.json | 3 +- .../dynamic_actions/dynamic_action_manager.ts | 5 +- 42 files changed, 546 insertions(+), 142 deletions(-) create mode 100644 packages/kbn-optimizer/src/common/bundle_refs.ts create mode 100644 packages/kbn-optimizer/src/worker/entry_point_creator.ts diff --git a/examples/demo_search/kibana.json b/examples/demo_search/kibana.json index cdf74121ea2db..f909ca47fcd55 100644 --- a/examples/demo_search/kibana.json +++ b/examples/demo_search/kibana.json @@ -5,5 +5,6 @@ "server": true, "ui": true, "requiredPlugins": ["data"], - "optionalPlugins": [] + "optionalPlugins": [], + "extraPublicDirs": ["common"] } diff --git a/examples/embeddable_examples/kibana.json b/examples/embeddable_examples/kibana.json index 489f768552b28..3b43916e7e2c4 100644 --- a/examples/embeddable_examples/kibana.json +++ b/examples/embeddable_examples/kibana.json @@ -5,5 +5,6 @@ "server": true, "ui": true, "requiredPlugins": ["embeddable"], - "optionalPlugins": [] + "optionalPlugins": [], + "extraPublicDirs": ["public/todo"] } diff --git a/examples/url_generators_examples/kibana.json b/examples/url_generators_examples/kibana.json index cdb2127fdd26f..9658f5c7300aa 100644 --- a/examples/url_generators_examples/kibana.json +++ b/examples/url_generators_examples/kibana.json @@ -5,5 +5,8 @@ "server": false, "ui": true, "requiredPlugins": ["share"], - "optionalPlugins": [] + "optionalPlugins": [], + "extraPublicDirs": [ + "public/url_generator" + ] } diff --git a/packages/kbn-optimizer/package.json b/packages/kbn-optimizer/package.json index a372b9e394b9a..c7bf1dd60985d 100644 --- a/packages/kbn-optimizer/package.json +++ b/packages/kbn-optimizer/package.json @@ -45,6 +45,7 @@ "terser-webpack-plugin": "^2.1.2", "tinymath": "1.2.1", "url-loader": "^2.2.0", + "val-loader": "^1.1.1", "watchpack": "^1.6.0", "webpack": "^4.41.5", "webpack-merge": "^4.2.2" diff --git a/packages/kbn-optimizer/src/common/bundle.test.ts b/packages/kbn-optimizer/src/common/bundle.test.ts index ec78a1bdf020e..60ea3f5b9c17c 100644 --- a/packages/kbn-optimizer/src/common/bundle.test.ts +++ b/packages/kbn-optimizer/src/common/bundle.test.ts @@ -23,7 +23,8 @@ jest.mock('fs'); const SPEC: BundleSpec = { contextDir: '/foo/bar', - entry: 'entry', + publicDirNames: ['public'], + dependencies: ['foo'], id: 'bar', outputDir: '/foo/bar/target', sourceRoot: '/foo', @@ -49,9 +50,14 @@ it('creates cache keys', () => { }, "spec": Object { "contextDir": "/foo/bar", - "entry": "entry", + "dependencies": Array [ + "foo", + ], "id": "bar", "outputDir": "/foo/bar/target", + "publicDirNames": Array [ + "public", + ], "sourceRoot": "/foo", "type": "plugin", }, @@ -82,9 +88,14 @@ it('parses bundles from JSON specs', () => { "state": undefined, }, "contextDir": "/foo/bar", - "entry": "entry", + "dependencies": Array [ + "foo", + ], "id": "bar", "outputDir": "/foo/bar/target", + "publicDirNames": Array [ + "public", + ], "sourceRoot": "/foo", "type": "plugin", }, diff --git a/packages/kbn-optimizer/src/common/bundle.ts b/packages/kbn-optimizer/src/common/bundle.ts index 9e2ad186ba40c..3f1a6e8c88cb2 100644 --- a/packages/kbn-optimizer/src/common/bundle.ts +++ b/packages/kbn-optimizer/src/common/bundle.ts @@ -20,6 +20,7 @@ import Path from 'path'; import { BundleCache } from './bundle_cache'; +import { BundleRefs } from './bundle_refs'; import { UnknownVals } from './ts_helpers'; import { includes, ascending, entriesToObject } from './array_helpers'; @@ -29,8 +30,10 @@ export interface BundleSpec { readonly type: typeof VALID_BUNDLE_TYPES[0]; /** Unique id for this bundle */ readonly id: string; - /** Webpack entry request for this plugin, relative to the contextDir */ - readonly entry: string; + /** directory names relative to the contextDir that can be imported from */ + readonly publicDirNames: string[]; + /** bundle ids which this bundle is allowed to depend on */ + readonly dependencies: string[]; /** Absolute path to the plugin source directory */ readonly contextDir: string; /** Absolute path to the root of the repository */ @@ -44,8 +47,10 @@ export class Bundle { public readonly type: BundleSpec['type']; /** Unique identifier for this bundle */ public readonly id: BundleSpec['id']; - /** Path, relative to `contextDir`, to the entry file for the Webpack bundle */ - public readonly entry: BundleSpec['entry']; + /** directory names relative to the contextDir that can be imported from */ + public readonly publicDirNames: BundleSpec['publicDirNames']; + /** bundle ids which this bundle is allowed to depend on */ + public readonly dependencies: BundleSpec['dependencies']; /** * Absolute path to the root of the bundle context (plugin directory) * where the entry is resolved relative to and the default output paths @@ -62,7 +67,8 @@ export class Bundle { constructor(spec: BundleSpec) { this.type = spec.type; this.id = spec.id; - this.entry = spec.entry; + this.publicDirNames = spec.publicDirNames; + this.dependencies = spec.dependencies; this.contextDir = spec.contextDir; this.sourceRoot = spec.sourceRoot; this.outputDir = spec.outputDir; @@ -73,8 +79,6 @@ export class Bundle { /** * Calculate the cache key for this bundle based from current * mtime values. - * - * @param mtimes pre-fetched mtimes (ms || undefined) for all referenced files */ createCacheKey(files: string[], mtimes: Map): unknown { return { @@ -94,7 +98,8 @@ export class Bundle { return { type: this.type, id: this.id, - entry: this.entry, + publicDirNames: this.publicDirNames, + dependencies: this.dependencies, contextDir: this.contextDir, sourceRoot: this.sourceRoot, outputDir: this.outputDir, @@ -134,9 +139,14 @@ export function parseBundles(json: string) { throw new Error('`bundles[]` must have a string `id` property'); } - const { entry } = spec; - if (!(typeof entry === 'string')) { - throw new Error('`bundles[]` must have a string `entry` property'); + const { publicDirNames } = spec; + if (!Array.isArray(publicDirNames) || !publicDirNames.every((d) => typeof d === 'string')) { + throw new Error('`bundles[]` must have an array of strings `publicDirNames` property'); + } + + const { dependencies } = spec; + if (!Array.isArray(dependencies) || !dependencies.every((d) => typeof d === 'string')) { + throw new Error('`bundles[]` must have an array of strings `dependencies` property'); } const { contextDir } = spec; @@ -157,7 +167,8 @@ export function parseBundles(json: string) { return new Bundle({ type, id, - entry, + publicDirNames, + dependencies, contextDir, sourceRoot, outputDir, diff --git a/packages/kbn-optimizer/src/common/bundle_cache.ts b/packages/kbn-optimizer/src/common/bundle_cache.ts index 1dbc7f1d1b6b0..5ae3e4c28a201 100644 --- a/packages/kbn-optimizer/src/common/bundle_cache.ts +++ b/packages/kbn-optimizer/src/common/bundle_cache.ts @@ -25,6 +25,7 @@ export interface State { cacheKey?: unknown; moduleCount?: number; files?: string[]; + bundleRefExportIds?: string[]; } const DEFAULT_STATE: State = {}; @@ -87,6 +88,10 @@ export class BundleCache { return this.get().files; } + public getBundleRefExportIds() { + return this.get().bundleRefExportIds; + } + public getCacheKey() { return this.get().cacheKey; } diff --git a/packages/kbn-optimizer/src/common/bundle_refs.ts b/packages/kbn-optimizer/src/common/bundle_refs.ts new file mode 100644 index 0000000000000..094b77e0f71ae --- /dev/null +++ b/packages/kbn-optimizer/src/common/bundle_refs.ts @@ -0,0 +1,233 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import Path from 'path'; + +import { Bundle } from './bundle'; +import { UnknownVals } from './ts_helpers'; + +interface BundleRef { + id: string; + contextDir: string; + contextPrefix: string; + entry: string; + exportId: string; +} + +interface BundleRefSpec { + id: string; + contextDir: string; + entry: string; + exportId: string; +} + +const RESOLVE_EXTENSIONS = ['.js', '.ts', '.tsx']; + +export class BundleRefs { + static getSpecFromBundles(bundles: Bundle[]) { + return bundles.reduce( + (acc: BundleRefSpec[], b) => [ + ...acc, + ...b.publicDirNames.map((name) => ({ + id: b.id, + contextDir: b.contextDir, + entry: name, + exportId: `${b.type}/${b.id}/${name}`, + })), + ], + [] + ); + } + + static parseSpec(json: unknown) { + if (typeof json !== 'string') { + throw new Error('expected `bundleRefs` spec to be a JSON string'); + } + + let spec; + try { + spec = JSON.parse(json); + } catch (error) { + throw new Error('`bundleRefs` spec must be valid JSON'); + } + + if (!Array.isArray(spec)) { + throw new Error('`bundleRefs` spec must be an array'); + } + + return new BundleRefs( + spec.map( + (zone: UnknownVals): BundleRef => { + if (typeof zone !== 'object' || !zone) { + throw new Error('`bundleRefs[*]` must be an object'); + } + + const { id } = zone; + if (typeof id !== 'string') { + throw new Error('`bundleRefs[*].id` must be a string'); + } + + const { contextDir } = zone; + if (typeof contextDir !== 'string' || !Path.isAbsolute(contextDir)) { + throw new Error('`bundleRefs[*].contextDir` must be an absolute directory'); + } + + const { entry } = zone; + if (typeof entry !== 'string') { + throw new Error('`bundleRefs[*].entry` must be a string'); + } + + const { exportId } = zone; + if (typeof exportId !== 'string') { + throw new Error('`bundleRefs[*].exportId` must be a string'); + } + + return { + id, + contextDir, + // Path.resolve converts separators and strips the final separator + contextPrefix: Path.resolve(contextDir) + Path.sep, + entry, + exportId, + }; + } + ) + ); + } + + private refsInBundle = new Map>(); + private resolvedRequestCache = new Map>(); + + constructor(private readonly refs: BundleRef[]) {} + + public filterByExportIds(exportIds: string[]) { + const refs: BundleRef[] = []; + for (const exportId of exportIds) { + const ref = this.refs.find((r) => r.exportId === exportId); + if (ref) { + refs.push(ref); + } + } + return refs; + } + + public filterByContextPrefix(bundle: Bundle, absolutePath: string) { + return this.refs.filter( + (ref) => ref.id !== bundle.id && absolutePath.startsWith(ref.contextPrefix) + ); + } + + private cachedResolveRequest(context: string, request: string) { + const absoluteRequest = Path.resolve(context, request); + const cached = this.resolvedRequestCache.get(absoluteRequest); + + if (cached) { + return cached; + } + + const promise = this.resolveRequest(absoluteRequest); + this.resolvedRequestCache.set(absoluteRequest, promise); + return promise; + } + + // TODO: Implement actual but fast resolver + private async resolveRequest(absoluteRequest: string) { + if (absoluteRequest.endsWith('index.ts')) { + return absoluteRequest; + } + + return Path.resolve(absoluteRequest, 'index.ts'); + } + + /** + * Determine externals statements for require/import statements by looking + * for requests resolving to the primary public export of the data, kibanaReact, + * amd kibanaUtils plugins. If this module is being imported then rewrite + * the import to access the global `__kbnBundles__` variables and access + * the relavent properties from that global object. + * + * @param bundle + * @param context the directory containing the module which made `request` + * @param request the request for a module from a commonjs require() call or import statement + */ + async checkForBundleRef(bundle: Bundle, context: string, request: string) { + // ignore imports that have loaders defined or are not relative seeming + if (request.includes('!') || !request.startsWith('.')) { + return; + } + + const requestExt = Path.extname(request); + if (requestExt && !RESOLVE_EXTENSIONS.includes(requestExt)) { + return; + } + + const resolved = await this.cachedResolveRequest(context, request); + + // the path was not resolved because it should be ignored, failed resolves throw + if (!resolved) { + return; + } + + const eligibleRefs = this.filterByContextPrefix(bundle, resolved); + if (!eligibleRefs.length) { + // import doesn't match a bundle context + return; + } + + let matchingRef: BundleRef | undefined; + for (const ref of eligibleRefs) { + const resolvedEntry = await this.cachedResolveRequest(ref.contextDir, ref.entry); + if (resolved === resolvedEntry) { + matchingRef = ref; + } + } + + if (!matchingRef) { + const bundleId = Array.from(new Set(eligibleRefs.map((r) => r.id))).join(', '); + const publicDir = eligibleRefs.map((r) => r.entry).join(', '); + throw new Error( + `import [${request}] references a non-public export of the [${bundleId}] bundle and must point to one of the public directories: [${publicDir}]` + ); + } + + if (!bundle.dependencies.includes(matchingRef.id)) { + throw new Error( + `import [${request}] references an export of the [${matchingRef.id}] bundle, but that plugin is not listed as a dependency in your kibana.json file.` + ); + } + + const refsInBundle = this.refsInBundle.get(bundle) || new Set(); + refsInBundle.add(matchingRef); + this.refsInBundle.set(bundle, refsInBundle); + + return `__kbnBundles__['${matchingRef.exportId}']`; + } + + getReferencedExportIds(bundle: Bundle) { + const refsInBundle = this.refsInBundle.get(bundle); + + if (!refsInBundle) { + return []; + } + + return Array.from(refsInBundle) + .map((ref) => ref.exportId) + .sort((a, b) => a.localeCompare(b)); + } +} diff --git a/packages/kbn-optimizer/src/common/index.ts b/packages/kbn-optimizer/src/common/index.ts index c51905be04565..7d021a5ee7847 100644 --- a/packages/kbn-optimizer/src/common/index.ts +++ b/packages/kbn-optimizer/src/common/index.ts @@ -19,6 +19,7 @@ export * from './bundle'; export * from './bundle_cache'; +export * from './bundle_refs'; export * from './worker_config'; export * from './worker_messages'; export * from './compiler_messages'; diff --git a/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.test.ts b/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.test.ts index 4671276797049..5b549de15802f 100644 --- a/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.test.ts +++ b/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.test.ts @@ -57,7 +57,8 @@ const assertReturnVal = (workers: Assignments[]) => { const testBundle = (id: string) => new Bundle({ contextDir: `/repo/plugin/${id}/public`, - entry: 'index.ts', + publicDirNames: ['public'], + dependencies: [], id, outputDir: `/repo/plugins/${id}/target/public`, sourceRoot: `/repo`, diff --git a/packages/kbn-optimizer/src/optimizer/bundle_cache.ts b/packages/kbn-optimizer/src/optimizer/bundle_cache.ts index 55e8e1d3fd084..028f27cf85dc9 100644 --- a/packages/kbn-optimizer/src/optimizer/bundle_cache.ts +++ b/packages/kbn-optimizer/src/optimizer/bundle_cache.ts @@ -20,7 +20,7 @@ import * as Rx from 'rxjs'; import { mergeAll } from 'rxjs/operators'; -import { Bundle } from '../common'; +import { Bundle, BundleRefs } from '../common'; import { OptimizerConfig } from './optimizer_config'; import { getMtimes } from './get_mtimes'; @@ -35,7 +35,9 @@ export interface BundleNotCachedEvent { | 'optimizer cache key mismatch' | 'missing cache key' | 'cache key mismatch' - | 'cache disabled'; + | 'cache disabled' + | 'bundle references missing' + | 'bundle references outdated'; diff?: string; bundle: Bundle; } @@ -52,6 +54,7 @@ export function getBundleCacheEvent$( return Rx.defer(async () => { const events: BundleCacheEvent[] = []; const eligibleBundles: Bundle[] = []; + const bundleRefs = BundleRefs.parseSpec(config.bundleRefsSpecJson); for (const bundle of config.bundles) { if (!config.cache) { @@ -93,6 +96,32 @@ export function getBundleCacheEvent$( continue; } + const bundleRefExportIds = bundle.cache.getBundleRefExportIds(); + if (!bundleRefExportIds) { + events.push({ + type: 'bundle not cached', + reason: 'bundle references missing', + bundle, + }); + continue; + } + + const refs = bundleRefs.filterByExportIds(bundleRefExportIds); + + const bundleRefsDiff = diffCacheKey( + refs.map((r) => r.exportId).sort((a, b) => a.localeCompare(b)), + bundleRefExportIds + ); + if (bundleRefsDiff) { + events.push({ + type: 'bundle not cached', + reason: 'bundle references outdated', + diff: bundleRefsDiff, + bundle, + }); + continue; + } + eligibleBundles.push(bundle); } diff --git a/packages/kbn-optimizer/src/optimizer/cache_keys.ts b/packages/kbn-optimizer/src/optimizer/cache_keys.ts index 2766f6d63702b..d0aaad979485d 100644 --- a/packages/kbn-optimizer/src/optimizer/cache_keys.ts +++ b/packages/kbn-optimizer/src/optimizer/cache_keys.ts @@ -37,16 +37,6 @@ import { OptimizerConfig } from './optimizer_config'; const OPTIMIZER_DIR = Path.dirname(require.resolve('../../package.json')); const RELATIVE_DIR = Path.relative(REPO_ROOT, OPTIMIZER_DIR); -function omit(obj: T, keys: K[]): Omit { - const result: any = {}; - for (const [key, value] of Object.entries(obj) as any) { - if (!keys.includes(key)) { - result[key] = value; - } - } - return result as Omit; -} - export function diffCacheKey(expected?: unknown, actual?: unknown) { const expectedJson = jsonStable(expected, { space: ' ', @@ -185,7 +175,7 @@ export async function getOptimizerCacheKey(config: OptimizerConfig) { bootstrap, deletedPaths, modifiedTimes: {} as Record, - workerConfig: omit(config.getWorkerConfig('♻'), ['watch', 'profileWebpack', 'cache']), + workerConfig: config.getCacheableWorkerConfig(), }; const mtimes = await getMtimes(modifiedPaths); diff --git a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts index b75a8a6edc264..e4644042fc2d7 100644 --- a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts +++ b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts @@ -31,7 +31,8 @@ export function getPluginBundles(plugins: KibanaPlatformPlugin[], repoRoot: stri new Bundle({ type: 'plugin', id: p.id, - entry: './public/index', + publicDirNames: ['public', ...p.extraPublicDirs], + dependencies: ['core', ...p.dependencies], sourceRoot: repoRoot, contextDir: p.directory, outputDir: Path.resolve(p.directory, 'target/public'), diff --git a/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.ts b/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.ts index 992feab6cd364..f72b1821f7f72 100644 --- a/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.ts +++ b/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.ts @@ -26,6 +26,8 @@ export interface KibanaPlatformPlugin { readonly directory: string; readonly id: string; readonly isUiPlugin: boolean; + readonly extraPublicDirs: string[]; + readonly dependencies: string[]; } /** @@ -64,9 +66,55 @@ function readKibanaPlatformPlugin(manifestPath: string): KibanaPlatformPlugin { throw new TypeError('expected new platform plugin manifest to have a string id'); } + let extraPublicDirs: string[] | undefined; + if (manifest.extraPublicDirs) { + if ( + !Array.isArray(manifest.extraPublicDirs) || + !manifest.extraPublicDirs.every((p) => typeof p === 'string') + ) { + throw new TypeError( + 'expected new platform plugin manifest to have an array of strings `extraPublicDirs` property' + ); + } + + extraPublicDirs = manifest.extraPublicDirs as string[]; + } + + const dependencies: string[] = []; + if (manifest.requiredPlugins) { + if ( + !Array.isArray(manifest.requiredPlugins) || + !manifest.requiredPlugins.every((p) => typeof p === 'string') + ) { + throw new TypeError( + 'expected new platform plugin manifest to have an array of strings `requiredPlugins` property' + ); + } + + for (const dep of manifest.requiredPlugins) { + dependencies.push(dep as string); + } + } + if (manifest.optionalPlugins) { + if ( + !Array.isArray(manifest.optionalPlugins) || + !manifest.optionalPlugins.every((p) => typeof p === 'string') + ) { + throw new TypeError( + 'expected new platform plugin manifest to have an array of strings `optionalPlugins` property' + ); + } + + for (const dep of manifest.optionalPlugins) { + dependencies.push(dep as string); + } + } + return { directory: Path.dirname(manifestPath), id: manifest.id, isUiPlugin: !!manifest.ui, + extraPublicDirs: extraPublicDirs || [], + dependencies, }; } diff --git a/packages/kbn-optimizer/src/optimizer/observe_worker.ts b/packages/kbn-optimizer/src/optimizer/observe_worker.ts index c929cf62d1bb0..45f5e311a9f02 100644 --- a/packages/kbn-optimizer/src/optimizer/observe_worker.ts +++ b/packages/kbn-optimizer/src/optimizer/observe_worker.ts @@ -24,7 +24,7 @@ import execa from 'execa'; import * as Rx from 'rxjs'; import { map, takeUntil } from 'rxjs/operators'; -import { isWorkerMsg, WorkerConfig, WorkerMsg, Bundle } from '../common'; +import { isWorkerMsg, WorkerConfig, WorkerMsg, Bundle, BundleRefs } from '../common'; import { OptimizerConfig } from './optimizer_config'; @@ -74,7 +74,11 @@ function usingWorkerProc( ) { return Rx.using( (): ProcResource => { - const args = [JSON.stringify(workerConfig), JSON.stringify(bundles.map((b) => b.toSpec()))]; + const args = [ + JSON.stringify(workerConfig), + JSON.stringify(bundles.map((b) => b.toSpec())), + config.bundleRefsSpecJson, + ]; const proc = execa.node(require.resolve('../worker/run_worker'), args, { nodeOptions: [ diff --git a/packages/kbn-optimizer/src/optimizer/optimizer_config.ts b/packages/kbn-optimizer/src/optimizer/optimizer_config.ts index 37d8a4f5eb8ae..e5d183333245d 100644 --- a/packages/kbn-optimizer/src/optimizer/optimizer_config.ts +++ b/packages/kbn-optimizer/src/optimizer/optimizer_config.ts @@ -20,7 +20,7 @@ import Path from 'path'; import Os from 'os'; -import { Bundle, WorkerConfig } from '../common'; +import { Bundle, BundleRefs, WorkerConfig, CacheableWorkerConfig } from '../common'; import { findKibanaPlatformPlugins, KibanaPlatformPlugin } from './kibana_platform_plugins'; import { getPluginBundles } from './get_plugin_bundles'; @@ -34,6 +34,16 @@ function pickMaxWorkerCount(dist: boolean) { return Math.max(maxWorkers, 2); } +function omit(obj: T, keys: K[]): Omit { + const result: any = {}; + for (const [key, value] of Object.entries(obj) as any) { + if (!keys.includes(key)) { + result[key] = value; + } + } + return result as Omit; +} + interface Options { /** absolute path to root of the repo/build */ repoRoot: string; @@ -152,7 +162,8 @@ export class OptimizerConfig { new Bundle({ type: 'entry', id: 'core', - entry: './public/index', + publicDirNames: ['public', 'public/utils'], + dependencies: [], sourceRoot: options.repoRoot, contextDir: Path.resolve(options.repoRoot, 'src/core'), outputDir: Path.resolve(options.repoRoot, 'src/core/target/public'), @@ -175,6 +186,8 @@ export class OptimizerConfig { ); } + public readonly bundleRefsSpecJson = JSON.stringify(BundleRefs.getSpecFromBundles(this.bundles)); + constructor( public readonly bundles: Bundle[], public readonly cache: boolean, @@ -198,4 +211,14 @@ export class OptimizerConfig { browserslistEnv: this.dist ? 'production' : process.env.BROWSERSLIST_ENV || 'dev', }; } + + getCacheableWorkerConfig(): CacheableWorkerConfig { + return omit(this.getWorkerConfig('♻'), [ + // these config options don't change the output of the bundles, so + // should not invalidate caches when they change + 'watch', + 'profileWebpack', + 'cache', + ]); + } } diff --git a/packages/kbn-optimizer/src/worker/entry_point_creator.ts b/packages/kbn-optimizer/src/worker/entry_point_creator.ts new file mode 100644 index 0000000000000..c81c3c9464fea --- /dev/null +++ b/packages/kbn-optimizer/src/worker/entry_point_creator.ts @@ -0,0 +1,28 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +module.exports = function ({ entries }: { entries: Array<{ importId: string; relPath: string }> }) { + const lines = entries.map( + ({ importId, relPath }) => `__kbnBundles__['${importId}'] = require('./${relPath}');` + ); + + return { + code: lines.join('\n'), + }; +}; diff --git a/packages/kbn-optimizer/src/worker/run_compilers.ts b/packages/kbn-optimizer/src/worker/run_compilers.ts index 4ab289d031d72..2fa454a81717e 100644 --- a/packages/kbn-optimizer/src/worker/run_compilers.ts +++ b/packages/kbn-optimizer/src/worker/run_compilers.ts @@ -35,6 +35,7 @@ import { WorkerConfig, ascending, parseFilePath, + BundleRefs, } from '../common'; import { getWebpackConfig } from './webpack.config'; import { isFailureStats, failedStatsToErrorMessage } from './webpack_helpers'; @@ -55,6 +56,7 @@ const PLUGIN_NAME = '@kbn/optimizer'; const observeCompiler = ( workerConfig: WorkerConfig, bundle: Bundle, + bundleRefs: BundleRefs, compiler: webpack.Compiler ): Rx.Observable => { const compilerMsgs = new CompilerMsgs(bundle.id); @@ -150,6 +152,7 @@ const observeCompiler = ( ); bundle.cache.set({ + bundleRefExportIds: bundleRefs.getReferencedExportIds(bundle), optimizerCacheKey: workerConfig.optimizerCacheKey, cacheKey: bundle.createCacheKey(files, mtimes), moduleCount: normalModules.length, @@ -185,8 +188,14 @@ const observeCompiler = ( /** * Run webpack compilers */ -export const runCompilers = (workerConfig: WorkerConfig, bundles: Bundle[]) => { - const multiCompiler = webpack(bundles.map((def) => getWebpackConfig(def, workerConfig))); +export const runCompilers = ( + workerConfig: WorkerConfig, + bundles: Bundle[], + bundleRefs: BundleRefs +) => { + const multiCompiler = webpack( + bundles.map((def) => getWebpackConfig(def, workerConfig, bundleRefs)) + ); return Rx.merge( /** @@ -199,7 +208,7 @@ export const runCompilers = (workerConfig: WorkerConfig, bundles: Bundle[]) => { Rx.from(multiCompiler.compilers.entries()).pipe( mergeMap(([compilerIndex, compiler]) => { const bundle = bundles[compilerIndex]; - return observeCompiler(workerConfig, bundle, compiler); + return observeCompiler(workerConfig, bundle, bundleRefs, compiler); }) ), diff --git a/packages/kbn-optimizer/src/worker/run_worker.ts b/packages/kbn-optimizer/src/worker/run_worker.ts index f83c69477f471..178637d39ab00 100644 --- a/packages/kbn-optimizer/src/worker/run_worker.ts +++ b/packages/kbn-optimizer/src/worker/run_worker.ts @@ -19,7 +19,14 @@ import * as Rx from 'rxjs'; -import { parseBundles, parseWorkerConfig, WorkerMsg, isWorkerMsg, WorkerMsgs } from '../common'; +import { + parseBundles, + parseWorkerConfig, + WorkerMsg, + isWorkerMsg, + WorkerMsgs, + BundleRefs, +} from '../common'; import { runCompilers } from './run_compilers'; @@ -76,11 +83,12 @@ setInterval(() => { Rx.defer(() => { const workerConfig = parseWorkerConfig(process.argv[2]); const bundles = parseBundles(process.argv[3]); + const bundleRefs = BundleRefs.parseSpec(process.argv[4]); // set BROWSERSLIST_ENV so that style/babel loaders see it before running compilers process.env.BROWSERSLIST_ENV = workerConfig.browserslistEnv; - return runCompilers(workerConfig, bundles); + return runCompilers(workerConfig, bundles, bundleRefs); }).subscribe( (msg) => { send(msg); diff --git a/packages/kbn-optimizer/src/worker/webpack.config.ts b/packages/kbn-optimizer/src/worker/webpack.config.ts index d31c098ca1f2e..5b8684bead034 100644 --- a/packages/kbn-optimizer/src/worker/webpack.config.ts +++ b/packages/kbn-optimizer/src/worker/webpack.config.ts @@ -17,10 +17,8 @@ * under the License. */ -import Fs from 'fs'; import Path from 'path'; -import normalizePath from 'normalize-path'; import { stringifyRequest } from 'loader-utils'; import webpack from 'webpack'; // @ts-ignore @@ -32,88 +30,21 @@ import { CleanWebpackPlugin } from 'clean-webpack-plugin'; import CompressionPlugin from 'compression-webpack-plugin'; import * as UiSharedDeps from '@kbn/ui-shared-deps'; -import { Bundle, WorkerConfig, parseDirPath, DisallowedSyntaxPlugin } from '../common'; +import { Bundle, BundleRefs, WorkerConfig, parseDirPath, DisallowedSyntaxPlugin } from '../common'; const IS_CODE_COVERAGE = !!process.env.CODE_COVERAGE; const ISTANBUL_PRESET_PATH = require.resolve('@kbn/babel-preset/istanbul_preset'); const BABEL_PRESET_PATH = require.resolve('@kbn/babel-preset/webpack_preset'); -const SHARED_BUNDLES = [ - { - type: 'entry', - id: 'core', - rootRelativeDir: 'src/core/public', - }, - { - type: 'plugin', - id: 'data', - rootRelativeDir: 'src/plugins/data/public', - }, - { - type: 'plugin', - id: 'kibanaReact', - rootRelativeDir: 'src/plugins/kibana_react/public', - }, - { - type: 'plugin', - id: 'kibanaUtils', - rootRelativeDir: 'src/plugins/kibana_utils/public', - }, - { - type: 'plugin', - id: 'esUiShared', - rootRelativeDir: 'src/plugins/es_ui_shared/public', - }, -]; - -/** - * Determine externals statements for require/import statements by looking - * for requests resolving to the primary public export of the data, kibanaReact, - * amd kibanaUtils plugins. If this module is being imported then rewrite - * the import to access the global `__kbnBundles__` variables and access - * the relavent properties from that global object. - * - * @param bundle - * @param context the directory containing the module which made `request` - * @param request the request for a module from a commonjs require() call or import statement - */ -function dynamicExternals(bundle: Bundle, context: string, request: string) { - // ignore imports that have loaders defined or are not relative seeming - if (request.includes('!') || !request.startsWith('.')) { - return; - } - - // determine the most acurate resolution string we can without running full resolution - const rootRelative = normalizePath( - Path.relative(bundle.sourceRoot, Path.resolve(context, request)) - ); - for (const sharedBundle of SHARED_BUNDLES) { - if ( - rootRelative !== sharedBundle.rootRelativeDir || - `${bundle.type}/${bundle.id}` === `${sharedBundle.type}/${sharedBundle.id}` - ) { - continue; - } - - return `__kbnBundles__['${sharedBundle.type}/${sharedBundle.id}']`; - } - - // import doesn't match a root public import - return undefined; -} - -export function getWebpackConfig(bundle: Bundle, worker: WorkerConfig) { - const extensions = ['.js', '.ts', '.tsx', '.json']; - const entryExtension = extensions.find((ext) => - Fs.existsSync(Path.resolve(bundle.contextDir, bundle.entry) + ext) - ); +export function getWebpackConfig(bundle: Bundle, worker: WorkerConfig, bundleRefs: BundleRefs) { + const ENTRY_CREATOR = require.resolve('./entry_point_creator'); const commonConfig: webpack.Configuration = { node: { fs: 'empty' }, context: bundle.contextDir, cache: true, entry: { - [bundle.id]: `${bundle.entry}${entryExtension}`, + [bundle.id]: ENTRY_CREATOR, }, devtool: worker.dist ? false : '#cheap-source-map', @@ -128,9 +59,6 @@ export function getWebpackConfig(bundle: Bundle, worker: WorkerConfig) { info.absoluteResourcePath )}${info.query}`, jsonpFunction: `${bundle.id}_bundle_jsonpfunction`, - // When the entry point is loaded, assign it's default export - // to a key on the global `__kbnBundles__` object. - library: ['__kbnBundles__', `${bundle.type}/${bundle.id}`], }, optimization: { @@ -141,7 +69,14 @@ export function getWebpackConfig(bundle: Bundle, worker: WorkerConfig) { UiSharedDeps.externals, function (context, request, cb) { try { - cb(undefined, dynamicExternals(bundle, context, request)); + bundleRefs.checkForBundleRef(bundle, context, request).then( + (resp) => { + cb(undefined, resp); + }, + (error) => { + cb(error, undefined); + } + ); } catch (error) { cb(error, undefined); } @@ -162,11 +97,28 @@ export function getWebpackConfig(bundle: Bundle, worker: WorkerConfig) { rules: [ { - include: [`${Path.resolve(bundle.contextDir, bundle.entry)}${entryExtension}`], - loader: UiSharedDeps.publicPathLoader, - options: { - key: bundle.id, - }, + include: [ENTRY_CREATOR], + use: [ + { + loader: UiSharedDeps.publicPathLoader, + options: { + key: bundle.id, + }, + }, + { + loader: require.resolve('val-loader'), + options: { + entries: bundle.publicDirNames.map((name) => { + const absolute = Path.resolve(bundle.contextDir, name); + const newContext = Path.dirname(ENTRY_CREATOR); + return { + importId: `${bundle.type}/${bundle.id}/${name}`, + relPath: Path.relative(newContext, absolute), + }; + }), + }, + }, + ], }, { test: /\.css$/, @@ -310,7 +262,7 @@ export function getWebpackConfig(bundle: Bundle, worker: WorkerConfig) { }, resolve: { - extensions, + extensions: ['.js', '.ts', '.tsx', 'json'], mainFields: ['browser', 'main'], alias: { tinymath: require.resolve('tinymath/lib/tinymath.es5.js'), diff --git a/src/core/server/plugins/discovery/plugin_manifest_parser.ts b/src/core/server/plugins/discovery/plugin_manifest_parser.ts index 27c3ca5a71e16..58a1abdf57fea 100644 --- a/src/core/server/plugins/discovery/plugin_manifest_parser.ts +++ b/src/core/server/plugins/discovery/plugin_manifest_parser.ts @@ -57,6 +57,7 @@ const KNOWN_MANIFEST_FIELDS = (() => { optionalPlugins: true, ui: true, server: true, + extraPublicDirs: true, }; return new Set(Object.keys(manifestFields)); @@ -175,6 +176,7 @@ export async function parseManifest(pluginPath: string, packageInfo: PackageInfo optionalPlugins: Array.isArray(manifest.optionalPlugins) ? manifest.optionalPlugins : [], ui: includesUiPlugin, server: includesServerPlugin, + extraPublicDirs: manifest.extraPublicDirs, }; } diff --git a/src/core/server/plugins/types.ts b/src/core/server/plugins/types.ts index 4fa4e1780e596..1200bb80d33a0 100644 --- a/src/core/server/plugins/types.ts +++ b/src/core/server/plugins/types.ts @@ -153,6 +153,12 @@ export interface PluginManifest { * Specifies whether plugin includes some server-side specific functionality. */ readonly server: boolean; + + /** + * Specifies directory names that can be imported by other plugins sharing + * the same instance of the @kbn/optimizer. + */ + readonly extraPublicDirs: string[]; } /** diff --git a/src/plugins/data/kibana.json b/src/plugins/data/kibana.json index f5df747f17e1e..3e5d96a4bc47b 100644 --- a/src/plugins/data/kibana.json +++ b/src/plugins/data/kibana.json @@ -7,5 +7,6 @@ "expressions", "uiActions" ], - "optionalPlugins": ["usageCollection"] + "optionalPlugins": ["usageCollection"], + "extraPublicDirs": ["common", "common/utils/abort_utils"] } diff --git a/src/plugins/es_ui_shared/kibana.json b/src/plugins/es_ui_shared/kibana.json index 5a3db3b344090..d5685b7d0a6a9 100644 --- a/src/plugins/es_ui_shared/kibana.json +++ b/src/plugins/es_ui_shared/kibana.json @@ -1,5 +1,13 @@ { "id": "esUiShared", "version": "kibana", - "ui": true + "ui": true, + "extraPublicDirs": [ + "static/ace_x_json/hooks", + "static/validators/string", + "static/forms/hook_form_lib", + "static/forms/helpers", + "static/forms/components", + "static/forms/helpers/field_validators/types" + ] } diff --git a/src/plugins/expressions/kibana.json b/src/plugins/expressions/kibana.json index 5d2112103e94d..4774c69cc29ff 100644 --- a/src/plugins/expressions/kibana.json +++ b/src/plugins/expressions/kibana.json @@ -5,5 +5,6 @@ "ui": true, "requiredPlugins": [ "bfetch" - ] + ], + "extraPublicDirs": ["common", "common/fonts"] } diff --git a/src/plugins/inspector/kibana.json b/src/plugins/inspector/kibana.json index 39d3ff65eed53..99a38d2928df6 100644 --- a/src/plugins/inspector/kibana.json +++ b/src/plugins/inspector/kibana.json @@ -2,5 +2,6 @@ "id": "inspector", "version": "kibana", "server": false, - "ui": true + "ui": true, + "extraPublicDirs": ["common", "common/adapters/request"] } diff --git a/src/plugins/kibana_legacy/kibana.json b/src/plugins/kibana_legacy/kibana.json index e96b4859a36d0..606acd8b88b05 100644 --- a/src/plugins/kibana_legacy/kibana.json +++ b/src/plugins/kibana_legacy/kibana.json @@ -2,5 +2,6 @@ "id": "kibanaLegacy", "version": "kibana", "server": true, - "ui": true + "ui": true, + "extraPublicDirs": ["common/kbn_base_url"] } diff --git a/src/plugins/kibana_utils/kibana.json b/src/plugins/kibana_utils/kibana.json index 6fa39d82d1021..7e2127c27548e 100644 --- a/src/plugins/kibana_utils/kibana.json +++ b/src/plugins/kibana_utils/kibana.json @@ -1,5 +1,10 @@ { "id": "kibanaUtils", "version": "kibana", - "ui": true + "ui": true, + "extraPublicDirs": [ + "common", + "demos/state_containers/todomvc", + "common/state_containers" + ] } diff --git a/src/plugins/saved_objects_management/kibana.json b/src/plugins/saved_objects_management/kibana.json index 22135ce4558ae..6184d890c415c 100644 --- a/src/plugins/saved_objects_management/kibana.json +++ b/src/plugins/saved_objects_management/kibana.json @@ -4,5 +4,6 @@ "server": true, "ui": true, "requiredPlugins": ["home", "management", "data"], - "optionalPlugins": ["dashboard", "visualizations", "discover"] + "optionalPlugins": ["dashboard", "visualizations", "discover"], + "extraPublicDirs": ["public/lib"] } diff --git a/src/plugins/telemetry/kibana.json b/src/plugins/telemetry/kibana.json index f623f4f2a565d..a497597762520 100644 --- a/src/plugins/telemetry/kibana.json +++ b/src/plugins/telemetry/kibana.json @@ -6,5 +6,8 @@ "requiredPlugins": [ "telemetryCollectionManager", "usageCollection" + ], + "extraPublicDirs": [ + "common/constants" ] } diff --git a/x-pack/plugins/alerts/kibana.json b/x-pack/plugins/alerts/kibana.json index 3509f79dbbe4d..eef61ff4b3d53 100644 --- a/x-pack/plugins/alerts/kibana.json +++ b/x-pack/plugins/alerts/kibana.json @@ -6,5 +6,6 @@ "kibanaVersion": "kibana", "configPath": ["xpack", "alerts"], "requiredPlugins": ["licensing", "taskManager", "encryptedSavedObjects", "actions", "eventLog"], - "optionalPlugins": ["usageCollection", "spaces", "security"] + "optionalPlugins": ["usageCollection", "spaces", "security"], + "extraPublicDirs": ["common", "common/parse_duration"] } diff --git a/x-pack/plugins/apm/kibana.json b/x-pack/plugins/apm/kibana.json index 1b8e7c4dc5431..56a9e226b6528 100644 --- a/x-pack/plugins/apm/kibana.json +++ b/x-pack/plugins/apm/kibana.json @@ -25,5 +25,8 @@ "configPath": [ "xpack", "apm" + ], + "extraPublicDirs": [ + "public/style/variables" ] } diff --git a/x-pack/plugins/canvas/common/lib/autocomplete.ts b/x-pack/plugins/canvas/common/lib/autocomplete.ts index 0ab549bd14e83..c97879de2137e 100644 --- a/x-pack/plugins/canvas/common/lib/autocomplete.ts +++ b/x-pack/plugins/canvas/common/lib/autocomplete.ts @@ -14,7 +14,7 @@ import { ExpressionFunction, ExpressionFunctionParameter, getByAlias, -} from '../../../../../src/plugins/expressions'; +} from '../../../../../src/plugins/expressions/common'; const MARKER = 'CANVAS_SUGGESTION_MARKER'; diff --git a/x-pack/plugins/data_enhanced/kibana.json b/x-pack/plugins/data_enhanced/kibana.json index 443bb63a27799..1be55d2b7a635 100644 --- a/x-pack/plugins/data_enhanced/kibana.json +++ b/x-pack/plugins/data_enhanced/kibana.json @@ -8,6 +8,7 @@ "requiredPlugins": [ "data" ], + "optionalPlugins": ["kibanaReact", "kibanaUtils"], "server": true, "ui": true } diff --git a/x-pack/plugins/features/kibana.json b/x-pack/plugins/features/kibana.json index 1cab1821b1bf5..92fdd08e93478 100644 --- a/x-pack/plugins/features/kibana.json +++ b/x-pack/plugins/features/kibana.json @@ -6,5 +6,6 @@ "optionalPlugins": ["visTypeTimelion"], "configPath": ["xpack", "features"], "server": true, - "ui": true + "ui": true, + "extraPublicDirs": ["common"] } diff --git a/x-pack/plugins/ingest_manager/kibana.json b/x-pack/plugins/ingest_manager/kibana.json index 382ea0444093d..35447139607a6 100644 --- a/x-pack/plugins/ingest_manager/kibana.json +++ b/x-pack/plugins/ingest_manager/kibana.json @@ -5,5 +5,6 @@ "ui": true, "configPath": ["xpack", "ingestManager"], "requiredPlugins": ["licensing", "data", "encryptedSavedObjects"], - "optionalPlugins": ["security", "features", "cloud"] + "optionalPlugins": ["security", "features", "cloud"], + "extraPublicDirs": ["common"] } diff --git a/x-pack/plugins/lens/kibana.json b/x-pack/plugins/lens/kibana.json index a8b22b3e22750..346a5a24c269f 100644 --- a/x-pack/plugins/lens/kibana.json +++ b/x-pack/plugins/lens/kibana.json @@ -13,5 +13,6 @@ "dashboard" ], "optionalPlugins": ["embeddable", "usageCollection", "taskManager", "uiActions"], - "configPath": ["xpack", "lens"] + "configPath": ["xpack", "lens"], + "extraPublicDirs": ["common/constants"] } diff --git a/x-pack/plugins/license_management/kibana.json b/x-pack/plugins/license_management/kibana.json index be28c8e978d8a..6da923c5cff5a 100644 --- a/x-pack/plugins/license_management/kibana.json +++ b/x-pack/plugins/license_management/kibana.json @@ -5,5 +5,6 @@ "ui": true, "requiredPlugins": ["home", "licensing", "management"], "optionalPlugins": ["telemetry"], - "configPath": ["xpack", "license_management"] + "configPath": ["xpack", "license_management"], + "extraPublicDirs": ["common/constants"] } diff --git a/x-pack/plugins/maps/kibana.json b/x-pack/plugins/maps/kibana.json index 67520321de761..f8a30b8d0337e 100644 --- a/x-pack/plugins/maps/kibana.json +++ b/x-pack/plugins/maps/kibana.json @@ -18,5 +18,6 @@ "usageCollection" ], "ui": true, - "server": true + "server": true, + "extraPublicDirs": ["common/constants"] } diff --git a/x-pack/plugins/spaces/kibana.json b/x-pack/plugins/spaces/kibana.json index 7e6d0425f2ae0..9483cb67392c4 100644 --- a/x-pack/plugins/spaces/kibana.json +++ b/x-pack/plugins/spaces/kibana.json @@ -13,5 +13,6 @@ "savedObjectsManagement" ], "server": true, - "ui": true + "ui": true, + "extraPublicDirs": ["common"] } diff --git a/x-pack/plugins/triggers_actions_ui/kibana.json b/x-pack/plugins/triggers_actions_ui/kibana.json index d8f5055368831..158cfa100d546 100644 --- a/x-pack/plugins/triggers_actions_ui/kibana.json +++ b/x-pack/plugins/triggers_actions_ui/kibana.json @@ -5,5 +5,6 @@ "ui": true, "optionalPlugins": ["alerts", "alertingBuiltins"], "requiredPlugins": ["management", "charts", "data"], - "configPath": ["xpack", "trigger_actions_ui"] + "configPath": ["xpack", "trigger_actions_ui"], + "extraPublicDirs": ["public/common", "public/common/constants"] } diff --git a/x-pack/plugins/ui_actions_enhanced/public/dynamic_actions/dynamic_action_manager.ts b/x-pack/plugins/ui_actions_enhanced/public/dynamic_actions/dynamic_action_manager.ts index 3e37bac0a17af..58344026079e7 100644 --- a/x-pack/plugins/ui_actions_enhanced/public/dynamic_actions/dynamic_action_manager.ts +++ b/x-pack/plugins/ui_actions_enhanced/public/dynamic_actions/dynamic_action_manager.ts @@ -12,7 +12,10 @@ import { UiActionsActionDefinition as ActionDefinition, } from '../../../../../src/plugins/ui_actions/public'; import { defaultState, transitions, selectors, State } from './dynamic_action_manager_state'; -import { StateContainer, createStateContainer } from '../../../../../src/plugins/kibana_utils'; +import { + StateContainer, + createStateContainer, +} from '../../../../../src/plugins/kibana_utils/common'; import { StartContract } from '../plugin'; import { SerializedAction, SerializedEvent } from './types'; From 4280135c0610c787c3b14e224f7cfeb761b0bcd1 Mon Sep 17 00:00:00 2001 From: spalger Date: Thu, 11 Jun 2020 22:48:44 -0700 Subject: [PATCH 02/31] add some debugging scripts --- scripts/find_cross_plugin_imports.ts | 91 ++++++++++++++++++++++++++++ scripts/find_plugin_dep_counts.ts | 91 ++++++++++++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 scripts/find_cross_plugin_imports.ts create mode 100644 scripts/find_plugin_dep_counts.ts diff --git a/scripts/find_cross_plugin_imports.ts b/scripts/find_cross_plugin_imports.ts new file mode 100644 index 0000000000000..a37ca000ed80d --- /dev/null +++ b/scripts/find_cross_plugin_imports.ts @@ -0,0 +1,91 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import Path from 'path'; +import Fs from 'fs'; + +import globby from 'globby'; +import { run, REPO_ROOT } from '@kbn/dev-utils'; + +run(async ({ log }) => { + const statFilePaths = await globby( + [ + '**/target/public/stats.json', + '!**/{examples,node_modules,fixtures,__fixtures__,sao_template}/**', + ], + { + cwd: REPO_ROOT, + } + ); + + // log.success(inspect(files, { maxArrayLength: Infinity })); + + const plugins = statFilePaths.map((statFilePath) => { + const root = Path.resolve(Path.dirname(statFilePath), '../..'); + + let id; + try { + const manifest = JSON.parse(Fs.readFileSync(Path.resolve(root, 'kibana.json'), 'utf8')); + id = manifest.id; + } catch (error) { + id = Path.basename(root); + } + + const publicDir = Path.resolve(root, 'public'); + return { + id, + publicDir, + statFilePath, + }; + }); + + for (const plugin of plugins) { + const stats = JSON.parse(Fs.readFileSync(plugin.statFilePath, 'utf8')); + const [id] = Object.keys(stats.entrypoints); + const mainChunk = stats.chunks.find((c) => c.id === id); + const moduleCount = stats.modules.length; + const mainAssetSize = stats.assets.find((a) => a.name === `${id}.entry.js` || `${id}.plugin.js`) + .size; + + const modules = (stats.modules as any[]) + .filter((m) => !m.identifier.startsWith('external')) + .filter((m) => m.chunks.includes(mainChunk.id)) + .map((m) => { + const idParts = m.identifier.split('!'); + const last = idParts.pop(); + const [modulePath] = last.split('?'); + return modulePath as string; + }) + .filter((m) => !m.includes('/node_modules/')); + + const importedPlugins = plugins + .map((p) => { + if (p.id === id) { + return { plugin: p, modules: [] }; + } + + return { plugin: p, modules: modules.filter((m) => m.startsWith(p.publicDir)) }; + }) + .filter((i) => i.modules.length); + + log.info(id, moduleCount, mainAssetSize, importedPlugins); + + await new Promise((resolve) => setTimeout(resolve, 10)); + } +}); diff --git a/scripts/find_plugin_dep_counts.ts b/scripts/find_plugin_dep_counts.ts new file mode 100644 index 0000000000000..d32c620e58761 --- /dev/null +++ b/scripts/find_plugin_dep_counts.ts @@ -0,0 +1,91 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import Fs from 'fs'; + +import globby from 'globby'; +import { run, REPO_ROOT } from '@kbn/dev-utils'; + +run(async ({ log }) => { + const files = await globby( + ['**/kibana.json', '!**/{node_modules,fixtures,__fixtures__,sao_template}/**'], + { + cwd: REPO_ROOT, + } + ); + + const plugins = files.map((path) => JSON.parse(Fs.readFileSync(path, 'utf8'))); + + const loadOrder = new Set(); + + function load(plugin: any, path: string[] = []) { + if (path.includes(plugin.id)) { + log.warning('circular dep', [...path.slice(path.indexOf(plugin.id)), plugin.id].join(' -> ')); + return; + } + path = [...path, plugin.id]; + + for (const depId of plugin.requiredPlugins || []) { + const dep = plugins.find((p) => p.id === depId); + if (dep.ui) { + load(dep, path); + } + } + + for (const depId of plugin.optionalPlugins || []) { + const dep = plugins.find((p) => p.id === depId); + if (dep && dep.ui) { + load(dep, path); + } + } + + if (!loadOrder.has(plugin.id)) { + loadOrder.add(plugin.id); + } + } + + for (const plugin of plugins) { + if (plugin.ui) { + load(plugin); + } + } + + const depCount = new Map(); + function incrDepCount(id: string) { + const count = depCount.get(id) || 0; + depCount.set(id, count + 1); + } + + for (const plugin of plugins) { + for (const depId of plugin.requiredPlugins || []) { + incrDepCount(depId); + } + for (const depId of plugin.optionalPlugins || []) { + incrDepCount(depId); + } + } + + const pluginUsageInLoadOrder = new Map(); + for (const id of loadOrder) { + const count = depCount.get(id) || 0; + pluginUsageInLoadOrder.set(id, count); + } + + log.success(pluginUsageInLoadOrder); +}); From 1929413789796d86c7c509e37ddf94e58b0adc0a Mon Sep 17 00:00:00 2001 From: spalger Date: Fri, 12 Jun 2020 11:45:39 -0700 Subject: [PATCH 03/31] mesh the bundles together so that module are loaded as needed --- packages/kbn-optimizer/src/common/bundle.ts | 13 ------- .../kbn-optimizer/src/common/bundle_refs.ts | 8 +---- .../src/optimizer/get_plugin_bundles.ts | 1 - .../src/optimizer/kibana_platform_plugins.ts | 32 ----------------- .../src/optimizer/optimizer_config.ts | 1 - .../src/worker/entry_point_creator.ts | 6 ++-- src/core/public/plugins/plugin_reader.ts | 10 ++++-- .../ui/ui_render/bootstrap/template.js.hbs | 35 +++++++++++++++---- 8 files changed, 40 insertions(+), 66 deletions(-) diff --git a/packages/kbn-optimizer/src/common/bundle.ts b/packages/kbn-optimizer/src/common/bundle.ts index 3f1a6e8c88cb2..80af94c30f8da 100644 --- a/packages/kbn-optimizer/src/common/bundle.ts +++ b/packages/kbn-optimizer/src/common/bundle.ts @@ -20,7 +20,6 @@ import Path from 'path'; import { BundleCache } from './bundle_cache'; -import { BundleRefs } from './bundle_refs'; import { UnknownVals } from './ts_helpers'; import { includes, ascending, entriesToObject } from './array_helpers'; @@ -32,8 +31,6 @@ export interface BundleSpec { readonly id: string; /** directory names relative to the contextDir that can be imported from */ readonly publicDirNames: string[]; - /** bundle ids which this bundle is allowed to depend on */ - readonly dependencies: string[]; /** Absolute path to the plugin source directory */ readonly contextDir: string; /** Absolute path to the root of the repository */ @@ -49,8 +46,6 @@ export class Bundle { public readonly id: BundleSpec['id']; /** directory names relative to the contextDir that can be imported from */ public readonly publicDirNames: BundleSpec['publicDirNames']; - /** bundle ids which this bundle is allowed to depend on */ - public readonly dependencies: BundleSpec['dependencies']; /** * Absolute path to the root of the bundle context (plugin directory) * where the entry is resolved relative to and the default output paths @@ -68,7 +63,6 @@ export class Bundle { this.type = spec.type; this.id = spec.id; this.publicDirNames = spec.publicDirNames; - this.dependencies = spec.dependencies; this.contextDir = spec.contextDir; this.sourceRoot = spec.sourceRoot; this.outputDir = spec.outputDir; @@ -99,7 +93,6 @@ export class Bundle { type: this.type, id: this.id, publicDirNames: this.publicDirNames, - dependencies: this.dependencies, contextDir: this.contextDir, sourceRoot: this.sourceRoot, outputDir: this.outputDir, @@ -144,11 +137,6 @@ export function parseBundles(json: string) { throw new Error('`bundles[]` must have an array of strings `publicDirNames` property'); } - const { dependencies } = spec; - if (!Array.isArray(dependencies) || !dependencies.every((d) => typeof d === 'string')) { - throw new Error('`bundles[]` must have an array of strings `dependencies` property'); - } - const { contextDir } = spec; if (!(typeof contextDir === 'string' && Path.isAbsolute(contextDir))) { throw new Error('`bundles[]` must have an absolute path `contextDir` property'); @@ -168,7 +156,6 @@ export function parseBundles(json: string) { type, id, publicDirNames, - dependencies, contextDir, sourceRoot, outputDir, diff --git a/packages/kbn-optimizer/src/common/bundle_refs.ts b/packages/kbn-optimizer/src/common/bundle_refs.ts index 094b77e0f71ae..fadccd7a050f6 100644 --- a/packages/kbn-optimizer/src/common/bundle_refs.ts +++ b/packages/kbn-optimizer/src/common/bundle_refs.ts @@ -206,17 +206,11 @@ export class BundleRefs { ); } - if (!bundle.dependencies.includes(matchingRef.id)) { - throw new Error( - `import [${request}] references an export of the [${matchingRef.id}] bundle, but that plugin is not listed as a dependency in your kibana.json file.` - ); - } - const refsInBundle = this.refsInBundle.get(bundle) || new Set(); refsInBundle.add(matchingRef); this.refsInBundle.set(bundle, refsInBundle); - return `__kbnBundles__['${matchingRef.exportId}']`; + return `__kbnBundles__.get('${matchingRef.exportId}')`; } getReferencedExportIds(bundle: Bundle) { diff --git a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts index e4644042fc2d7..2635289088725 100644 --- a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts +++ b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts @@ -32,7 +32,6 @@ export function getPluginBundles(plugins: KibanaPlatformPlugin[], repoRoot: stri type: 'plugin', id: p.id, publicDirNames: ['public', ...p.extraPublicDirs], - dependencies: ['core', ...p.dependencies], sourceRoot: repoRoot, contextDir: p.directory, outputDir: Path.resolve(p.directory, 'target/public'), diff --git a/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.ts b/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.ts index f72b1821f7f72..bfc60a29efa27 100644 --- a/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.ts +++ b/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.ts @@ -27,7 +27,6 @@ export interface KibanaPlatformPlugin { readonly id: string; readonly isUiPlugin: boolean; readonly extraPublicDirs: string[]; - readonly dependencies: string[]; } /** @@ -80,41 +79,10 @@ function readKibanaPlatformPlugin(manifestPath: string): KibanaPlatformPlugin { extraPublicDirs = manifest.extraPublicDirs as string[]; } - const dependencies: string[] = []; - if (manifest.requiredPlugins) { - if ( - !Array.isArray(manifest.requiredPlugins) || - !manifest.requiredPlugins.every((p) => typeof p === 'string') - ) { - throw new TypeError( - 'expected new platform plugin manifest to have an array of strings `requiredPlugins` property' - ); - } - - for (const dep of manifest.requiredPlugins) { - dependencies.push(dep as string); - } - } - if (manifest.optionalPlugins) { - if ( - !Array.isArray(manifest.optionalPlugins) || - !manifest.optionalPlugins.every((p) => typeof p === 'string') - ) { - throw new TypeError( - 'expected new platform plugin manifest to have an array of strings `optionalPlugins` property' - ); - } - - for (const dep of manifest.optionalPlugins) { - dependencies.push(dep as string); - } - } - return { directory: Path.dirname(manifestPath), id: manifest.id, isUiPlugin: !!manifest.ui, extraPublicDirs: extraPublicDirs || [], - dependencies, }; } diff --git a/packages/kbn-optimizer/src/optimizer/optimizer_config.ts b/packages/kbn-optimizer/src/optimizer/optimizer_config.ts index e5d183333245d..4ca4fb87ec185 100644 --- a/packages/kbn-optimizer/src/optimizer/optimizer_config.ts +++ b/packages/kbn-optimizer/src/optimizer/optimizer_config.ts @@ -163,7 +163,6 @@ export class OptimizerConfig { type: 'entry', id: 'core', publicDirNames: ['public', 'public/utils'], - dependencies: [], sourceRoot: options.repoRoot, contextDir: Path.resolve(options.repoRoot, 'src/core'), outputDir: Path.resolve(options.repoRoot, 'src/core/target/public'), diff --git a/packages/kbn-optimizer/src/worker/entry_point_creator.ts b/packages/kbn-optimizer/src/worker/entry_point_creator.ts index c81c3c9464fea..a613e3e8925a4 100644 --- a/packages/kbn-optimizer/src/worker/entry_point_creator.ts +++ b/packages/kbn-optimizer/src/worker/entry_point_creator.ts @@ -18,9 +18,9 @@ */ module.exports = function ({ entries }: { entries: Array<{ importId: string; relPath: string }> }) { - const lines = entries.map( - ({ importId, relPath }) => `__kbnBundles__['${importId}'] = require('./${relPath}');` - ); + const lines = entries.map(({ importId, relPath }) => [ + `__kbnBundles__.define('${importId}', __webpack_require__, require.resolve('./${relPath}'))`, + ]); return { code: lines.join('\n'), diff --git a/src/core/public/plugins/plugin_reader.ts b/src/core/public/plugins/plugin_reader.ts index 1907dfa6a3e99..2e1cd4b6aa6fa 100644 --- a/src/core/public/plugins/plugin_reader.ts +++ b/src/core/public/plugins/plugin_reader.ts @@ -31,7 +31,8 @@ export type UnknownPluginInitializer = PluginInitializer Date: Fri, 12 Jun 2020 13:48:05 -0700 Subject: [PATCH 04/31] split up BundleRefs and resolver logic --- .../mock_repo/plugins/bar/public/index.ts | 2 +- .../kbn-optimizer/src/common/bundle.test.ts | 7 - .../kbn-optimizer/src/common/bundle_refs.ts | 168 ++++-------------- .../basic_optimization.test.ts.snap | 17 +- .../basic_optimization.test.ts | 10 +- .../integration_tests/bundle_cache.test.ts | 10 +- .../watch_bundles_for_changes.test.ts | 4 +- .../assign_bundles_to_workers.test.ts | 1 - .../src/optimizer/bundle_cache.ts | 2 +- .../src/optimizer/get_plugin_bundles.test.ts | 11 +- .../optimizer/kibana_platform_plugins.test.ts | 4 + .../src/optimizer/observe_worker.ts | 2 +- .../src/optimizer/optimizer_config.ts | 4 +- .../src/worker/bundle_refs_resolver.ts | 123 +++++++++++++ .../kbn-optimizer/src/worker/run_compilers.ts | 12 +- .../kbn-optimizer/src/worker/run_worker.ts | 5 +- .../src/worker/webpack.config.ts | 11 +- 17 files changed, 217 insertions(+), 176 deletions(-) create mode 100644 packages/kbn-optimizer/src/worker/bundle_refs_resolver.ts diff --git a/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/index.ts b/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/index.ts index 7ddd10f4a388f..c881a15eac5b5 100644 --- a/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/index.ts +++ b/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/index.ts @@ -19,6 +19,6 @@ import './legacy/styles.scss'; import './index.scss'; -import { fooLibFn } from '../../foo/public/index'; +import { fooLibFn } from '../../foo/public'; export * from './lib'; export { fooLibFn }; diff --git a/packages/kbn-optimizer/src/common/bundle.test.ts b/packages/kbn-optimizer/src/common/bundle.test.ts index 60ea3f5b9c17c..b209bbca25ac4 100644 --- a/packages/kbn-optimizer/src/common/bundle.test.ts +++ b/packages/kbn-optimizer/src/common/bundle.test.ts @@ -24,7 +24,6 @@ jest.mock('fs'); const SPEC: BundleSpec = { contextDir: '/foo/bar', publicDirNames: ['public'], - dependencies: ['foo'], id: 'bar', outputDir: '/foo/bar/target', sourceRoot: '/foo', @@ -50,9 +49,6 @@ it('creates cache keys', () => { }, "spec": Object { "contextDir": "/foo/bar", - "dependencies": Array [ - "foo", - ], "id": "bar", "outputDir": "/foo/bar/target", "publicDirNames": Array [ @@ -88,9 +84,6 @@ it('parses bundles from JSON specs', () => { "state": undefined, }, "contextDir": "/foo/bar", - "dependencies": Array [ - "foo", - ], "id": "bar", "outputDir": "/foo/bar/target", "publicDirNames": Array [ diff --git a/packages/kbn-optimizer/src/common/bundle_refs.ts b/packages/kbn-optimizer/src/common/bundle_refs.ts index fadccd7a050f6..95af2ea7db387 100644 --- a/packages/kbn-optimizer/src/common/bundle_refs.ts +++ b/packages/kbn-optimizer/src/common/bundle_refs.ts @@ -22,7 +22,7 @@ import Path from 'path'; import { Bundle } from './bundle'; import { UnknownVals } from './ts_helpers'; -interface BundleRef { +export interface BundleRef { id: string; contextDir: string; contextPrefix: string; @@ -30,28 +30,25 @@ interface BundleRef { exportId: string; } -interface BundleRefSpec { - id: string; - contextDir: string; - entry: string; - exportId: string; -} - -const RESOLVE_EXTENSIONS = ['.js', '.ts', '.tsx']; - export class BundleRefs { - static getSpecFromBundles(bundles: Bundle[]) { - return bundles.reduce( - (acc: BundleRefSpec[], b) => [ - ...acc, - ...b.publicDirNames.map((name) => ({ - id: b.id, - contextDir: b.contextDir, - entry: name, - exportId: `${b.type}/${b.id}/${name}`, - })), - ], - [] + static fromBundles(bundles: Bundle[]) { + return new BundleRefs( + bundles.reduce( + (acc: BundleRef[], b) => [ + ...acc, + ...b.publicDirNames.map( + (name): BundleRef => ({ + id: b.id, + contextDir: b.contextDir, + // Path.resolve converts separators and strips the final separator + contextPrefix: Path.resolve(b.contextDir) + Path.sep, + entry: name, + exportId: `${b.type}/${b.id}/${name}`, + }) + ), + ], + [] + ) ); } @@ -73,47 +70,48 @@ export class BundleRefs { return new BundleRefs( spec.map( - (zone: UnknownVals): BundleRef => { - if (typeof zone !== 'object' || !zone) { - throw new Error('`bundleRefs[*]` must be an object'); + (refSpec: UnknownVals): BundleRef => { + if (typeof refSpec !== 'object' || !refSpec) { + throw new Error('`bundleRefs[]` must be an object'); } - const { id } = zone; + const { id } = refSpec; if (typeof id !== 'string') { - throw new Error('`bundleRefs[*].id` must be a string'); + throw new Error('`bundleRefs[].id` must be a string'); } - const { contextDir } = zone; + const { contextDir } = refSpec; if (typeof contextDir !== 'string' || !Path.isAbsolute(contextDir)) { - throw new Error('`bundleRefs[*].contextDir` must be an absolute directory'); + throw new Error('`bundleRefs[].contextDir` must be an absolute directory'); + } + + const { contextPrefix } = refSpec; + if (typeof contextPrefix !== 'string' || !Path.isAbsolute(contextPrefix)) { + throw new Error('`bundleRefs[].contextPrefix` must be an absolute directory'); } - const { entry } = zone; + const { entry } = refSpec; if (typeof entry !== 'string') { - throw new Error('`bundleRefs[*].entry` must be a string'); + throw new Error('`bundleRefs[].entry` must be a string'); } - const { exportId } = zone; + const { exportId } = refSpec; if (typeof exportId !== 'string') { - throw new Error('`bundleRefs[*].exportId` must be a string'); + throw new Error('`bundleRefs[].exportId` must be a string'); } return { - id, contextDir, - // Path.resolve converts separators and strips the final separator - contextPrefix: Path.resolve(contextDir) + Path.sep, + contextPrefix, entry, exportId, + id, }; } ) ); } - private refsInBundle = new Map>(); - private resolvedRequestCache = new Map>(); - constructor(private readonly refs: BundleRef[]) {} public filterByExportIds(exportIds: string[]) { @@ -133,95 +131,7 @@ export class BundleRefs { ); } - private cachedResolveRequest(context: string, request: string) { - const absoluteRequest = Path.resolve(context, request); - const cached = this.resolvedRequestCache.get(absoluteRequest); - - if (cached) { - return cached; - } - - const promise = this.resolveRequest(absoluteRequest); - this.resolvedRequestCache.set(absoluteRequest, promise); - return promise; - } - - // TODO: Implement actual but fast resolver - private async resolveRequest(absoluteRequest: string) { - if (absoluteRequest.endsWith('index.ts')) { - return absoluteRequest; - } - - return Path.resolve(absoluteRequest, 'index.ts'); - } - - /** - * Determine externals statements for require/import statements by looking - * for requests resolving to the primary public export of the data, kibanaReact, - * amd kibanaUtils plugins. If this module is being imported then rewrite - * the import to access the global `__kbnBundles__` variables and access - * the relavent properties from that global object. - * - * @param bundle - * @param context the directory containing the module which made `request` - * @param request the request for a module from a commonjs require() call or import statement - */ - async checkForBundleRef(bundle: Bundle, context: string, request: string) { - // ignore imports that have loaders defined or are not relative seeming - if (request.includes('!') || !request.startsWith('.')) { - return; - } - - const requestExt = Path.extname(request); - if (requestExt && !RESOLVE_EXTENSIONS.includes(requestExt)) { - return; - } - - const resolved = await this.cachedResolveRequest(context, request); - - // the path was not resolved because it should be ignored, failed resolves throw - if (!resolved) { - return; - } - - const eligibleRefs = this.filterByContextPrefix(bundle, resolved); - if (!eligibleRefs.length) { - // import doesn't match a bundle context - return; - } - - let matchingRef: BundleRef | undefined; - for (const ref of eligibleRefs) { - const resolvedEntry = await this.cachedResolveRequest(ref.contextDir, ref.entry); - if (resolved === resolvedEntry) { - matchingRef = ref; - } - } - - if (!matchingRef) { - const bundleId = Array.from(new Set(eligibleRefs.map((r) => r.id))).join(', '); - const publicDir = eligibleRefs.map((r) => r.entry).join(', '); - throw new Error( - `import [${request}] references a non-public export of the [${bundleId}] bundle and must point to one of the public directories: [${publicDir}]` - ); - } - - const refsInBundle = this.refsInBundle.get(bundle) || new Set(); - refsInBundle.add(matchingRef); - this.refsInBundle.set(bundle, refsInBundle); - - return `__kbnBundles__.get('${matchingRef.exportId}')`; - } - - getReferencedExportIds(bundle: Bundle) { - const refsInBundle = this.refsInBundle.get(bundle); - - if (!refsInBundle) { - return []; - } - - return Array.from(refsInBundle) - .map((ref) => ref.exportId) - .sort((a, b) => a.localeCompare(b)); + public toSpecJson() { + return JSON.stringify(this.refs); } } diff --git a/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap b/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap index 2814ab32017d2..350a93b88cca2 100644 --- a/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap +++ b/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap @@ -9,9 +9,11 @@ OptimizerConfig { "state": undefined, }, "contextDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar, - "entry": "./public/index", "id": "bar", "outputDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/target/public, + "publicDirNames": Array [ + "public", + ], "sourceRoot": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo, "type": "plugin", }, @@ -21,9 +23,11 @@ OptimizerConfig { "state": undefined, }, "contextDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo, - "entry": "./public/index", "id": "foo", "outputDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/target/public, + "publicDirNames": Array [ + "public", + ], "sourceRoot": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo, "type": "plugin", }, @@ -35,16 +39,19 @@ OptimizerConfig { "plugins": Array [ Object { "directory": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar, + "extraPublicDirs": Array [], "id": "bar", "isUiPlugin": true, }, Object { "directory": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/baz, + "extraPublicDirs": Array [], "id": "baz", "isUiPlugin": false, }, Object { "directory": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo, + "extraPublicDirs": Array [], "id": "foo", "isUiPlugin": true, }, @@ -55,8 +62,8 @@ OptimizerConfig { } `; -exports[`prepares assets for distribution: bar bundle 1`] = `"var __kbnBundles__=typeof __kbnBundles__===\\"object\\"?__kbnBundles__:{};__kbnBundles__[\\"plugin/bar\\"]=function(modules){function webpackJsonpCallback(data){var chunkIds=data[0];var moreModules=data[1];var moduleId,chunkId,i=0,resolves=[];for(;i { const foo = config.bundles.find((b) => b.id === 'foo')!; expect(foo).toBeTruthy(); foo.cache.refresh(); - expect(foo.cache.getModuleCount()).toBe(5); + expect(foo.cache.getModuleCount()).toBe(6); expect(foo.cache.getReferencedFiles()).toMatchInlineSnapshot(` Array [ /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/async_import.ts, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/ext.ts, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/index.ts, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/lib.ts, + /packages/kbn-optimizer/target/worker/entry_point_creator.js, /packages/kbn-ui-shared-deps/public_path_module_creator.js, ] `); @@ -148,7 +149,7 @@ it('builds expected bundles, saves bundle counts to metadata', async () => { bar.cache.refresh(); expect(bar.cache.getModuleCount()).toBe( // code + styles + style/css-loader runtimes + public path updater - 21 + 18 ); expect(bar.cache.getReferencedFiles()).toMatchInlineSnapshot(` @@ -159,11 +160,8 @@ it('builds expected bundles, saves bundle counts to metadata', async () => { /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/index.ts, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/legacy/styles.scss, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/lib.ts, - /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/async_import.ts, - /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/ext.ts, - /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/index.ts, - /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/lib.ts, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/src/legacy/ui/public/icon.svg, + /packages/kbn-optimizer/target/worker/entry_point_creator.js, /packages/kbn-ui-shared-deps/public_path_module_creator.js, ] `); diff --git a/packages/kbn-optimizer/src/integration_tests/bundle_cache.test.ts b/packages/kbn-optimizer/src/integration_tests/bundle_cache.test.ts index 39064c64062e8..d9fe04f66a799 100644 --- a/packages/kbn-optimizer/src/integration_tests/bundle_cache.test.ts +++ b/packages/kbn-optimizer/src/integration_tests/bundle_cache.test.ts @@ -85,7 +85,8 @@ it('emits "bundle cached" event when everything is updated', async () => { Array [ Object { "bundle": , - "type": "bundle cached", + "reason": "bundle references missing", + "type": "bundle not cached", }, ] `); @@ -288,12 +289,7 @@ it('emits "bundle not cached" event when cacheKey is outdated', async () => { Array [ Object { "bundle": , - "diff": "- Expected - + Received - - - \\"old\\" - + \\"new\\"", - "reason": "cache key mismatch", + "reason": "bundle references missing", "type": "bundle not cached", }, ] diff --git a/packages/kbn-optimizer/src/integration_tests/watch_bundles_for_changes.test.ts b/packages/kbn-optimizer/src/integration_tests/watch_bundles_for_changes.test.ts index 91d0f308e0ef6..176b17c979da9 100644 --- a/packages/kbn-optimizer/src/integration_tests/watch_bundles_for_changes.test.ts +++ b/packages/kbn-optimizer/src/integration_tests/watch_bundles_for_changes.test.ts @@ -29,14 +29,14 @@ jest.mock('fs'); jest.mock('watchpack'); const MockWatchPack: jest.MockedClass = jest.requireMock('watchpack'); -const bundleEntryPath = (bundle: Bundle) => `${bundle.contextDir}/${bundle.entry}`; +const bundleEntryPath = (bundle: Bundle) => `${bundle.contextDir}/public/index.ts`; const makeTestBundle = (id: string) => { const bundle = new Bundle({ type: 'plugin', id, contextDir: `/repo/plugins/${id}/public`, - entry: 'index.ts', + publicDirNames: ['public'], outputDir: `/repo/plugins/${id}/target/public`, sourceRoot: `/repo`, }); diff --git a/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.test.ts b/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.test.ts index 5b549de15802f..ca50a49e26913 100644 --- a/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.test.ts +++ b/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.test.ts @@ -58,7 +58,6 @@ const testBundle = (id: string) => new Bundle({ contextDir: `/repo/plugin/${id}/public`, publicDirNames: ['public'], - dependencies: [], id, outputDir: `/repo/plugins/${id}/target/public`, sourceRoot: `/repo`, diff --git a/packages/kbn-optimizer/src/optimizer/bundle_cache.ts b/packages/kbn-optimizer/src/optimizer/bundle_cache.ts index 028f27cf85dc9..83db8570bd408 100644 --- a/packages/kbn-optimizer/src/optimizer/bundle_cache.ts +++ b/packages/kbn-optimizer/src/optimizer/bundle_cache.ts @@ -54,7 +54,7 @@ export function getBundleCacheEvent$( return Rx.defer(async () => { const events: BundleCacheEvent[] = []; const eligibleBundles: Bundle[] = []; - const bundleRefs = BundleRefs.parseSpec(config.bundleRefsSpecJson); + const bundleRefs = BundleRefs.fromBundles(config.bundles); for (const bundle of config.bundles) { if (!config.cache) { diff --git a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts index 2174c488ad6cc..bbd3ddc11f448 100644 --- a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts +++ b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts @@ -31,16 +31,19 @@ it('returns a bundle for core and each plugin', () => { directory: '/repo/plugins/foo', id: 'foo', isUiPlugin: true, + extraPublicDirs: [], }, { directory: '/repo/plugins/bar', id: 'bar', isUiPlugin: false, + extraPublicDirs: [], }, { directory: '/outside/of/repo/plugins/baz', id: 'baz', isUiPlugin: true, + extraPublicDirs: [], }, ], '/repo' @@ -49,17 +52,21 @@ it('returns a bundle for core and each plugin', () => { Array [ Object { "contextDir": /plugins/foo, - "entry": "./public/index", "id": "foo", "outputDir": /plugins/foo/target/public, + "publicDirNames": Array [ + "public", + ], "sourceRoot": , "type": "plugin", }, Object { "contextDir": "/outside/of/repo/plugins/baz", - "entry": "./public/index", "id": "baz", "outputDir": "/outside/of/repo/plugins/baz/target/public", + "publicDirNames": Array [ + "public", + ], "sourceRoot": , "type": "plugin", }, diff --git a/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.test.ts b/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.test.ts index e047b6d1e44cf..0961881df461c 100644 --- a/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.test.ts +++ b/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.test.ts @@ -37,21 +37,25 @@ it('parses kibana.json files of plugins found in pluginDirs', () => { Array [ Object { "directory": /packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar, + "extraPublicDirs": Array [], "id": "bar", "isUiPlugin": true, }, Object { "directory": /packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/baz, + "extraPublicDirs": Array [], "id": "baz", "isUiPlugin": false, }, Object { "directory": /packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/foo, + "extraPublicDirs": Array [], "id": "foo", "isUiPlugin": true, }, Object { "directory": /packages/kbn-optimizer/src/__fixtures__/mock_repo/test_plugins/test_baz, + "extraPublicDirs": Array [], "id": "test_baz", "isUiPlugin": false, }, diff --git a/packages/kbn-optimizer/src/optimizer/observe_worker.ts b/packages/kbn-optimizer/src/optimizer/observe_worker.ts index 45f5e311a9f02..4527052fa821a 100644 --- a/packages/kbn-optimizer/src/optimizer/observe_worker.ts +++ b/packages/kbn-optimizer/src/optimizer/observe_worker.ts @@ -77,7 +77,7 @@ function usingWorkerProc( const args = [ JSON.stringify(workerConfig), JSON.stringify(bundles.map((b) => b.toSpec())), - config.bundleRefsSpecJson, + BundleRefs.fromBundles(config.bundles).toSpecJson(), ]; const proc = execa.node(require.resolve('../worker/run_worker'), args, { diff --git a/packages/kbn-optimizer/src/optimizer/optimizer_config.ts b/packages/kbn-optimizer/src/optimizer/optimizer_config.ts index 4ca4fb87ec185..c9e9b3ad01ccc 100644 --- a/packages/kbn-optimizer/src/optimizer/optimizer_config.ts +++ b/packages/kbn-optimizer/src/optimizer/optimizer_config.ts @@ -20,7 +20,7 @@ import Path from 'path'; import Os from 'os'; -import { Bundle, BundleRefs, WorkerConfig, CacheableWorkerConfig } from '../common'; +import { Bundle, WorkerConfig, CacheableWorkerConfig } from '../common'; import { findKibanaPlatformPlugins, KibanaPlatformPlugin } from './kibana_platform_plugins'; import { getPluginBundles } from './get_plugin_bundles'; @@ -185,8 +185,6 @@ export class OptimizerConfig { ); } - public readonly bundleRefsSpecJson = JSON.stringify(BundleRefs.getSpecFromBundles(this.bundles)); - constructor( public readonly bundles: Bundle[], public readonly cache: boolean, diff --git a/packages/kbn-optimizer/src/worker/bundle_refs_resolver.ts b/packages/kbn-optimizer/src/worker/bundle_refs_resolver.ts new file mode 100644 index 0000000000000..15f25f93b055c --- /dev/null +++ b/packages/kbn-optimizer/src/worker/bundle_refs_resolver.ts @@ -0,0 +1,123 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import Path from 'path'; + +import { Bundle, BundleRefs, BundleRef } from '../common'; + +const RESOLVE_EXTENSIONS = ['.js', '.ts', '.tsx']; + +export class BundleRefsResolver { + private refsInBundle = new Map>(); + private resolvedRequestCache = new Map>(); + + constructor(public readonly bundleRefs: BundleRefs) {} + + private cachedResolveRequest(context: string, request: string) { + const absoluteRequest = Path.resolve(context, request); + const cached = this.resolvedRequestCache.get(absoluteRequest); + + if (cached) { + return cached; + } + + const promise = this.resolveRequest(absoluteRequest); + this.resolvedRequestCache.set(absoluteRequest, promise); + return promise; + } + + // TODO: Implement actual but fast resolver + private async resolveRequest(absoluteRequest: string) { + if (absoluteRequest.endsWith('index.ts')) { + return absoluteRequest; + } + + return Path.resolve(absoluteRequest, 'index.ts'); + } + + /** + * Determine externals statements for require/import statements by looking + * for requests resolving to the primary public export of the data, kibanaReact, + * amd kibanaUtils plugins. If this module is being imported then rewrite + * the import to access the global `__kbnBundles__` variables and access + * the relavent properties from that global object. + * + * @param bundle + * @param context the directory containing the module which made `request` + * @param request the request for a module from a commonjs require() call or import statement + */ + async resolveExternal(bundle: Bundle, context: string, request: string) { + // ignore imports that have loaders defined or are not relative seeming + if (request.includes('!') || !request.startsWith('.')) { + return; + } + + const requestExt = Path.extname(request); + if (requestExt && !RESOLVE_EXTENSIONS.includes(requestExt)) { + return; + } + + const resolved = await this.cachedResolveRequest(context, request); + + // the path was not resolved because it should be ignored, failed resolves throw + if (!resolved) { + return; + } + + const eligibleRefs = this.bundleRefs.filterByContextPrefix(bundle, resolved); + if (!eligibleRefs.length) { + // import doesn't match a bundle context + return; + } + + let matchingRef: BundleRef | undefined; + for (const ref of eligibleRefs) { + const resolvedEntry = await this.cachedResolveRequest(ref.contextDir, ref.entry); + if (resolved === resolvedEntry) { + matchingRef = ref; + } + } + + if (!matchingRef) { + const bundleId = Array.from(new Set(eligibleRefs.map((r) => r.id))).join(', '); + const publicDir = eligibleRefs.map((r) => r.entry).join(', '); + throw new Error( + `import [${request}] references a non-public export of the [${bundleId}] bundle and must point to one of the public directories: [${publicDir}]` + ); + } + + const refsInBundle = this.refsInBundle.get(bundle) || new Set(); + refsInBundle.add(matchingRef); + this.refsInBundle.set(bundle, refsInBundle); + + return `__kbnBundles__.get('${matchingRef.exportId}')`; + } + + getReferencedExportIds(bundle: Bundle) { + const refsInBundle = this.refsInBundle.get(bundle); + + if (!refsInBundle) { + return []; + } + + return Array.from(refsInBundle) + .map((ref) => ref.exportId) + .sort((a, b) => a.localeCompare(b)); + } +} diff --git a/packages/kbn-optimizer/src/worker/run_compilers.ts b/packages/kbn-optimizer/src/worker/run_compilers.ts index 2fa454a81717e..d787a5943af4f 100644 --- a/packages/kbn-optimizer/src/worker/run_compilers.ts +++ b/packages/kbn-optimizer/src/worker/run_compilers.ts @@ -35,8 +35,8 @@ import { WorkerConfig, ascending, parseFilePath, - BundleRefs, } from '../common'; +import { BundleRefsResolver } from './bundle_refs_resolver'; import { getWebpackConfig } from './webpack.config'; import { isFailureStats, failedStatsToErrorMessage } from './webpack_helpers'; import { @@ -56,7 +56,7 @@ const PLUGIN_NAME = '@kbn/optimizer'; const observeCompiler = ( workerConfig: WorkerConfig, bundle: Bundle, - bundleRefs: BundleRefs, + refResolver: BundleRefsResolver, compiler: webpack.Compiler ): Rx.Observable => { const compilerMsgs = new CompilerMsgs(bundle.id); @@ -152,7 +152,7 @@ const observeCompiler = ( ); bundle.cache.set({ - bundleRefExportIds: bundleRefs.getReferencedExportIds(bundle), + bundleRefExportIds: refResolver.getReferencedExportIds(bundle), optimizerCacheKey: workerConfig.optimizerCacheKey, cacheKey: bundle.createCacheKey(files, mtimes), moduleCount: normalModules.length, @@ -191,10 +191,10 @@ const observeCompiler = ( export const runCompilers = ( workerConfig: WorkerConfig, bundles: Bundle[], - bundleRefs: BundleRefs + refResolver: BundleRefsResolver ) => { const multiCompiler = webpack( - bundles.map((def) => getWebpackConfig(def, workerConfig, bundleRefs)) + bundles.map((def) => getWebpackConfig(def, workerConfig, refResolver)) ); return Rx.merge( @@ -208,7 +208,7 @@ export const runCompilers = ( Rx.from(multiCompiler.compilers.entries()).pipe( mergeMap(([compilerIndex, compiler]) => { const bundle = bundles[compilerIndex]; - return observeCompiler(workerConfig, bundle, bundleRefs, compiler); + return observeCompiler(workerConfig, bundle, refResolver, compiler); }) ), diff --git a/packages/kbn-optimizer/src/worker/run_worker.ts b/packages/kbn-optimizer/src/worker/run_worker.ts index 178637d39ab00..3037ca75b5053 100644 --- a/packages/kbn-optimizer/src/worker/run_worker.ts +++ b/packages/kbn-optimizer/src/worker/run_worker.ts @@ -28,6 +28,7 @@ import { BundleRefs, } from '../common'; +import { BundleRefsResolver } from './bundle_refs_resolver'; import { runCompilers } from './run_compilers'; /** @@ -83,12 +84,12 @@ setInterval(() => { Rx.defer(() => { const workerConfig = parseWorkerConfig(process.argv[2]); const bundles = parseBundles(process.argv[3]); - const bundleRefs = BundleRefs.parseSpec(process.argv[4]); + const refResolver = new BundleRefsResolver(BundleRefs.parseSpec(process.argv[4])); // set BROWSERSLIST_ENV so that style/babel loaders see it before running compilers process.env.BROWSERSLIST_ENV = workerConfig.browserslistEnv; - return runCompilers(workerConfig, bundles, bundleRefs); + return runCompilers(workerConfig, bundles, refResolver); }).subscribe( (msg) => { send(msg); diff --git a/packages/kbn-optimizer/src/worker/webpack.config.ts b/packages/kbn-optimizer/src/worker/webpack.config.ts index 5b8684bead034..772dcce7b2a97 100644 --- a/packages/kbn-optimizer/src/worker/webpack.config.ts +++ b/packages/kbn-optimizer/src/worker/webpack.config.ts @@ -30,13 +30,18 @@ import { CleanWebpackPlugin } from 'clean-webpack-plugin'; import CompressionPlugin from 'compression-webpack-plugin'; import * as UiSharedDeps from '@kbn/ui-shared-deps'; -import { Bundle, BundleRefs, WorkerConfig, parseDirPath, DisallowedSyntaxPlugin } from '../common'; +import { Bundle, WorkerConfig, parseDirPath, DisallowedSyntaxPlugin } from '../common'; +import { BundleRefsResolver } from './bundle_refs_resolver'; const IS_CODE_COVERAGE = !!process.env.CODE_COVERAGE; const ISTANBUL_PRESET_PATH = require.resolve('@kbn/babel-preset/istanbul_preset'); const BABEL_PRESET_PATH = require.resolve('@kbn/babel-preset/webpack_preset'); -export function getWebpackConfig(bundle: Bundle, worker: WorkerConfig, bundleRefs: BundleRefs) { +export function getWebpackConfig( + bundle: Bundle, + worker: WorkerConfig, + refResolver: BundleRefsResolver +) { const ENTRY_CREATOR = require.resolve('./entry_point_creator'); const commonConfig: webpack.Configuration = { @@ -69,7 +74,7 @@ export function getWebpackConfig(bundle: Bundle, worker: WorkerConfig, bundleRef UiSharedDeps.externals, function (context, request, cb) { try { - bundleRefs.checkForBundleRef(bundle, context, request).then( + refResolver.resolveExternal(bundle, context, request).then( (resp) => { cb(undefined, resp); }, From 5d1e4533f57749d47bf041a5017bcba4765ae914 Mon Sep 17 00:00:00 2001 From: spalger Date: Fri, 12 Jun 2020 16:42:37 -0700 Subject: [PATCH 05/31] add references used by test plugins --- src/plugins/embeddable/kibana.json | 3 +++ src/plugins/ui_actions/kibana.json | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/plugins/embeddable/kibana.json b/src/plugins/embeddable/kibana.json index 535527b4d09db..06b0e88da334f 100644 --- a/src/plugins/embeddable/kibana.json +++ b/src/plugins/embeddable/kibana.json @@ -6,5 +6,8 @@ "requiredPlugins": [ "inspector", "uiActions" + ], + "extraPublicDirs": [ + "public/lib/test_samples" ] } diff --git a/src/plugins/ui_actions/kibana.json b/src/plugins/ui_actions/kibana.json index 44ecbbfa68408..907cbabbdf9c9 100644 --- a/src/plugins/ui_actions/kibana.json +++ b/src/plugins/ui_actions/kibana.json @@ -2,5 +2,8 @@ "id": "uiActions", "version": "kibana", "server": false, - "ui": true + "ui": true, + "extraPublicDirs": [ + "public/tests/test_samples" + ] } From aefcd45a0bc5e0604ab0bf155cebcdac65b632d1 Mon Sep 17 00:00:00 2001 From: spalger Date: Fri, 12 Jun 2020 16:43:35 -0700 Subject: [PATCH 06/31] import from public dir --- .../plugins/kbn_tp_run_pipeline/public/app/components/main.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/interpreter_functional/plugins/kbn_tp_run_pipeline/public/app/components/main.tsx b/test/interpreter_functional/plugins/kbn_tp_run_pipeline/public/app/components/main.tsx index ace2af2b4f0cf..b4f9634b23d29 100644 --- a/test/interpreter_functional/plugins/kbn_tp_run_pipeline/public/app/components/main.tsx +++ b/test/interpreter_functional/plugins/kbn_tp_run_pipeline/public/app/components/main.tsx @@ -21,7 +21,7 @@ import React from 'react'; import { EuiPage, EuiPageBody, EuiPageContent, EuiPageContentHeader } from '@elastic/eui'; import { first } from 'rxjs/operators'; import { IInterpreterRenderHandlers, ExpressionValue } from 'src/plugins/expressions'; -import { RequestAdapter, DataAdapter } from '../../../../../../../src/plugins/inspector'; +import { RequestAdapter, DataAdapter } from '../../../../../../../src/plugins/inspector/public'; import { Adapters, ExpressionRenderHandler } from '../../types'; import { getExpressions } from '../../services'; From c7ad86f92157ae7af7fc40759eaec1093185aa32 Mon Sep 17 00:00:00 2001 From: spalger Date: Fri, 12 Jun 2020 22:18:41 -0700 Subject: [PATCH 07/31] use custom module type so we can scan for bundleRef modules and avoid commonjs conversion --- .../src/worker/bundle_ref_module.ts | 82 +++++++++++++++++++ ...refs_resolver.ts => bundle_refs_plugin.ts} | 72 +++++++++++----- .../kbn-optimizer/src/worker/run_compilers.ts | 70 ++++++++-------- .../kbn-optimizer/src/worker/run_worker.ts | 5 +- .../src/worker/webpack.config.ts | 34 ++------ 5 files changed, 179 insertions(+), 84 deletions(-) create mode 100644 packages/kbn-optimizer/src/worker/bundle_ref_module.ts rename packages/kbn-optimizer/src/worker/{bundle_refs_resolver.ts => bundle_refs_plugin.ts} (67%) diff --git a/packages/kbn-optimizer/src/worker/bundle_ref_module.ts b/packages/kbn-optimizer/src/worker/bundle_ref_module.ts new file mode 100644 index 0000000000000..2120be3c071f9 --- /dev/null +++ b/packages/kbn-optimizer/src/worker/bundle_ref_module.ts @@ -0,0 +1,82 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// @ts-ignore not typed by @types/webpack +import Module from 'webpack/lib/Module'; + +export class BundleRefModule extends Module { + public built = false; + public buildMeta?: any; + public buildInfo?: any; + public exportsArgument = '__webpack_exports__'; + + constructor(public readonly exportId: string) { + super('kbn/bundleRef', null); + } + + libIdent() { + return this.exportId; + } + + chunkCondition(chunk: any) { + return chunk.hasEntryModule(); + } + + identifier() { + return '@kbn/bundleRef ' + JSON.stringify(this.exportId); + } + + readableIdentifier() { + return this.identifier(); + } + + needRebuild() { + return false; + } + + build(_: any, __: any, ___: any, ____: any, callback: () => void) { + this.built = true; + this.buildMeta = {}; + this.buildInfo = {}; + callback(); + } + + source() { + return ` + __webpack_require__.r(__webpack_exports__); + var ns = __kbnBundles__.get('${this.exportId}'); + var descriptor = Object.getOwnPropertyDescriptors(ns); + Object.keys(descriptor).forEach(function (key) { + var desc = descriptor[key] + if (desc.get) { + __webpack_require__.d(__webpack_exports__, key, desc.get) + } + }) + `; + } + + size() { + return 42; + } + + updateHash(hash: any) { + hash.update(this.identifier()); + super.updateHash(hash); + } +} diff --git a/packages/kbn-optimizer/src/worker/bundle_refs_resolver.ts b/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts similarity index 67% rename from packages/kbn-optimizer/src/worker/bundle_refs_resolver.ts rename to packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts index 15f25f93b055c..36befccc3366f 100644 --- a/packages/kbn-optimizer/src/worker/bundle_refs_resolver.ts +++ b/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts @@ -18,16 +18,60 @@ */ import Path from 'path'; +import webpack from 'webpack'; import { Bundle, BundleRefs, BundleRef } from '../common'; +import { BundleRefModule } from './bundle_ref_module'; const RESOLVE_EXTENSIONS = ['.js', '.ts', '.tsx']; -export class BundleRefsResolver { - private refsInBundle = new Map>(); +interface RequestData { + context: string; + dependencies: Array<{ request: string }>; +} + +type Callback = (error?: any, result?: T) => void; +type ModuleFactory = (data: RequestData, callback: Callback) => void; + +function compileHook( + compiler: webpack.Compiler, + handler: (context: string, request: string) => Promise +) { + compiler.hooks.compile.tap('BundleRefsPlugin', (args: any) => { + args.normalModuleFactory.hooks.factory.tap( + 'BundleRefsPlugin/normalModuleFactory/factory', + (wrappedFactory: ModuleFactory): ModuleFactory => (data, callback) => { + const context = data.context; + const dep = data.dependencies[0]; + + handler(context, dep.request).then( + (result) => { + if (!result) { + wrappedFactory(data, callback); + } else { + callback(undefined, result); + } + }, + (error) => callback(error) + ); + } + ); + }); +} + +export class BundleRefsPlugin { private resolvedRequestCache = new Map>(); - constructor(public readonly bundleRefs: BundleRefs) {} + constructor(private readonly bundle: Bundle, public readonly bundleRefs: BundleRefs) {} + + apply(compiler: webpack.Compiler) { + compileHook(compiler, async (context, request) => { + const exportId = await this.resolveExternal(context, request); + if (exportId) { + return new BundleRefModule(exportId); + } + }); + } private cachedResolveRequest(context: string, request: string) { const absoluteRequest = Path.resolve(context, request); @@ -62,7 +106,7 @@ export class BundleRefsResolver { * @param context the directory containing the module which made `request` * @param request the request for a module from a commonjs require() call or import statement */ - async resolveExternal(bundle: Bundle, context: string, request: string) { + async resolveExternal(context: string, request: string) { // ignore imports that have loaders defined or are not relative seeming if (request.includes('!') || !request.startsWith('.')) { return; @@ -80,7 +124,7 @@ export class BundleRefsResolver { return; } - const eligibleRefs = this.bundleRefs.filterByContextPrefix(bundle, resolved); + const eligibleRefs = this.bundleRefs.filterByContextPrefix(this.bundle, resolved); if (!eligibleRefs.length) { // import doesn't match a bundle context return; @@ -102,22 +146,6 @@ export class BundleRefsResolver { ); } - const refsInBundle = this.refsInBundle.get(bundle) || new Set(); - refsInBundle.add(matchingRef); - this.refsInBundle.set(bundle, refsInBundle); - - return `__kbnBundles__.get('${matchingRef.exportId}')`; - } - - getReferencedExportIds(bundle: Bundle) { - const refsInBundle = this.refsInBundle.get(bundle); - - if (!refsInBundle) { - return []; - } - - return Array.from(refsInBundle) - .map((ref) => ref.exportId) - .sort((a, b) => a.localeCompare(b)); + return matchingRef.exportId; } } diff --git a/packages/kbn-optimizer/src/worker/run_compilers.ts b/packages/kbn-optimizer/src/worker/run_compilers.ts index d787a5943af4f..de5e9372e9e7a 100644 --- a/packages/kbn-optimizer/src/worker/run_compilers.ts +++ b/packages/kbn-optimizer/src/worker/run_compilers.ts @@ -35,8 +35,9 @@ import { WorkerConfig, ascending, parseFilePath, + BundleRefs, } from '../common'; -import { BundleRefsResolver } from './bundle_refs_resolver'; +import { BundleRefModule } from './bundle_ref_module'; import { getWebpackConfig } from './webpack.config'; import { isFailureStats, failedStatsToErrorMessage } from './webpack_helpers'; import { @@ -44,7 +45,6 @@ import { isNormalModule, isIgnoredModule, isConcatenatedModule, - WebpackNormalModule, getModulePath, } from './webpack_helpers'; @@ -56,7 +56,6 @@ const PLUGIN_NAME = '@kbn/optimizer'; const observeCompiler = ( workerConfig: WorkerConfig, bundle: Bundle, - refResolver: BundleRefsResolver, compiler: webpack.Compiler ): Rx.Observable => { const compilerMsgs = new CompilerMsgs(bundle.id); @@ -100,40 +99,43 @@ const observeCompiler = ( }); } - const normalModules = stats.compilation.modules.filter( - (module): module is WebpackNormalModule => { - if (isNormalModule(module)) { - return true; - } + const bundleRefExportIds: string[] = []; + const referencedFiles = new Set(); + let normalModuleCount = 0; + + for (const module of stats.compilation.modules) { + if (isNormalModule(module)) { + normalModuleCount += 1; + const path = getModulePath(module); + const parsedPath = parseFilePath(path); - if (isExternalModule(module) || isIgnoredModule(module) || isConcatenatedModule(module)) { - return false; + if (!parsedPath.dirs.includes('node_modules')) { + referencedFiles.add(path); + continue; } - throw new Error(`Unexpected module type: ${inspect(module)}`); + const nmIndex = parsedPath.dirs.lastIndexOf('node_modules'); + const isScoped = parsedPath.dirs[nmIndex + 1].startsWith('@'); + referencedFiles.add( + Path.join( + parsedPath.root, + ...parsedPath.dirs.slice(0, nmIndex + 1 + (isScoped ? 2 : 1)), + 'package.json' + ) + ); + continue; } - ); - const referencedFiles = new Set(); - - for (const module of normalModules) { - const path = getModulePath(module); - const parsedPath = parseFilePath(path); + if (module instanceof BundleRefModule) { + bundleRefExportIds.push(module.exportId); + continue; + } - if (!parsedPath.dirs.includes('node_modules')) { - referencedFiles.add(path); + if (isExternalModule(module) || isIgnoredModule(module) || isConcatenatedModule(module)) { continue; } - const nmIndex = parsedPath.dirs.lastIndexOf('node_modules'); - const isScoped = parsedPath.dirs[nmIndex + 1].startsWith('@'); - referencedFiles.add( - Path.join( - parsedPath.root, - ...parsedPath.dirs.slice(0, nmIndex + 1 + (isScoped ? 2 : 1)), - 'package.json' - ) - ); + throw new Error(`Unexpected module type: ${inspect(module)}`); } const files = Array.from(referencedFiles).sort(ascending((p) => p)); @@ -152,15 +154,15 @@ const observeCompiler = ( ); bundle.cache.set({ - bundleRefExportIds: refResolver.getReferencedExportIds(bundle), + bundleRefExportIds, optimizerCacheKey: workerConfig.optimizerCacheKey, cacheKey: bundle.createCacheKey(files, mtimes), - moduleCount: normalModules.length, + moduleCount: normalModuleCount, files, }); return compilerMsgs.compilerSuccess({ - moduleCount: normalModules.length, + moduleCount: normalModuleCount, }); }) ); @@ -191,10 +193,10 @@ const observeCompiler = ( export const runCompilers = ( workerConfig: WorkerConfig, bundles: Bundle[], - refResolver: BundleRefsResolver + bundleRefs: BundleRefs ) => { const multiCompiler = webpack( - bundles.map((def) => getWebpackConfig(def, workerConfig, refResolver)) + bundles.map((def) => getWebpackConfig(def, bundleRefs, workerConfig)) ); return Rx.merge( @@ -208,7 +210,7 @@ export const runCompilers = ( Rx.from(multiCompiler.compilers.entries()).pipe( mergeMap(([compilerIndex, compiler]) => { const bundle = bundles[compilerIndex]; - return observeCompiler(workerConfig, bundle, refResolver, compiler); + return observeCompiler(workerConfig, bundle, compiler); }) ), diff --git a/packages/kbn-optimizer/src/worker/run_worker.ts b/packages/kbn-optimizer/src/worker/run_worker.ts index 3037ca75b5053..178637d39ab00 100644 --- a/packages/kbn-optimizer/src/worker/run_worker.ts +++ b/packages/kbn-optimizer/src/worker/run_worker.ts @@ -28,7 +28,6 @@ import { BundleRefs, } from '../common'; -import { BundleRefsResolver } from './bundle_refs_resolver'; import { runCompilers } from './run_compilers'; /** @@ -84,12 +83,12 @@ setInterval(() => { Rx.defer(() => { const workerConfig = parseWorkerConfig(process.argv[2]); const bundles = parseBundles(process.argv[3]); - const refResolver = new BundleRefsResolver(BundleRefs.parseSpec(process.argv[4])); + const bundleRefs = BundleRefs.parseSpec(process.argv[4]); // set BROWSERSLIST_ENV so that style/babel loaders see it before running compilers process.env.BROWSERSLIST_ENV = workerConfig.browserslistEnv; - return runCompilers(workerConfig, bundles, refResolver); + return runCompilers(workerConfig, bundles, bundleRefs); }).subscribe( (msg) => { send(msg); diff --git a/packages/kbn-optimizer/src/worker/webpack.config.ts b/packages/kbn-optimizer/src/worker/webpack.config.ts index 772dcce7b2a97..3daf21cdc38cc 100644 --- a/packages/kbn-optimizer/src/worker/webpack.config.ts +++ b/packages/kbn-optimizer/src/worker/webpack.config.ts @@ -30,18 +30,14 @@ import { CleanWebpackPlugin } from 'clean-webpack-plugin'; import CompressionPlugin from 'compression-webpack-plugin'; import * as UiSharedDeps from '@kbn/ui-shared-deps'; -import { Bundle, WorkerConfig, parseDirPath, DisallowedSyntaxPlugin } from '../common'; -import { BundleRefsResolver } from './bundle_refs_resolver'; +import { Bundle, BundleRefs, WorkerConfig, parseDirPath, DisallowedSyntaxPlugin } from '../common'; +import { BundleRefsPlugin } from './bundle_refs_plugin'; const IS_CODE_COVERAGE = !!process.env.CODE_COVERAGE; const ISTANBUL_PRESET_PATH = require.resolve('@kbn/babel-preset/istanbul_preset'); const BABEL_PRESET_PATH = require.resolve('@kbn/babel-preset/webpack_preset'); -export function getWebpackConfig( - bundle: Bundle, - worker: WorkerConfig, - refResolver: BundleRefsResolver -) { +export function getWebpackConfig(bundle: Bundle, bundleRefs: BundleRefs, worker: WorkerConfig) { const ENTRY_CREATOR = require.resolve('./entry_point_creator'); const commonConfig: webpack.Configuration = { @@ -70,25 +66,13 @@ export function getWebpackConfig( noEmitOnErrors: true, }, - externals: [ - UiSharedDeps.externals, - function (context, request, cb) { - try { - refResolver.resolveExternal(bundle, context, request).then( - (resp) => { - cb(undefined, resp); - }, - (error) => { - cb(error, undefined); - } - ); - } catch (error) { - cb(error, undefined); - } - }, - ], + externals: [UiSharedDeps.externals], - plugins: [new CleanWebpackPlugin(), new DisallowedSyntaxPlugin()], + plugins: [ + new CleanWebpackPlugin(), + new DisallowedSyntaxPlugin(), + new BundleRefsPlugin(bundle, bundleRefs), + ], module: { // no parse rules for a few known large packages which have no require() statements From 4029278c31cace19a9c0e156949161453a78993c Mon Sep 17 00:00:00 2001 From: spalger Date: Fri, 12 Jun 2020 23:01:51 -0700 Subject: [PATCH 08/31] implement actual resolution --- .../src/worker/bundle_refs_plugin.ts | 46 +++++++++++++++---- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts b/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts index 36befccc3366f..b63b0f7b0433c 100644 --- a/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts +++ b/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts @@ -18,6 +18,9 @@ */ import Path from 'path'; +import Fs from 'fs'; +import { promisify } from 'util'; + import webpack from 'webpack'; import { Bundle, BundleRefs, BundleRef } from '../common'; @@ -25,6 +28,20 @@ import { BundleRefModule } from './bundle_ref_module'; const RESOLVE_EXTENSIONS = ['.js', '.ts', '.tsx']; +function safeStat(path: string): Promise { + return new Promise((resolve, reject) => { + Fs.stat(path, (error, stat) => { + if (error?.code === 'ENOENT') { + resolve(undefined); + } else if (error) { + reject(error); + } else { + resolve(stat); + } + }); + }); +} + interface RequestData { context: string; dependencies: Array<{ request: string }>; @@ -33,12 +50,15 @@ interface RequestData { type Callback = (error?: any, result?: T) => void; type ModuleFactory = (data: RequestData, callback: Callback) => void; -function compileHook( +/** + * Isolate the weired type juggling we have to do to add a hook to the webpack compiler + */ +function hookIntoCompiler( compiler: webpack.Compiler, handler: (context: string, request: string) => Promise ) { - compiler.hooks.compile.tap('BundleRefsPlugin', (args: any) => { - args.normalModuleFactory.hooks.factory.tap( + compiler.hooks.compile.tap('BundleRefsPlugin', (compilationParams: any) => { + compilationParams.normalModuleFactory.hooks.factory.tap( 'BundleRefsPlugin/normalModuleFactory/factory', (wrappedFactory: ModuleFactory): ModuleFactory => (data, callback) => { const context = data.context; @@ -65,7 +85,7 @@ export class BundleRefsPlugin { constructor(private readonly bundle: Bundle, public readonly bundleRefs: BundleRefs) {} apply(compiler: webpack.Compiler) { - compileHook(compiler, async (context, request) => { + hookIntoCompiler(compiler, async (context, request) => { const exportId = await this.resolveExternal(context, request); if (exportId) { return new BundleRefModule(exportId); @@ -86,13 +106,23 @@ export class BundleRefsPlugin { return promise; } - // TODO: Implement actual but fast resolver private async resolveRequest(absoluteRequest: string) { - if (absoluteRequest.endsWith('index.ts')) { + const stats = await safeStat(absoluteRequest); + if (stats && stats.isFile()) { return absoluteRequest; } - return Path.resolve(absoluteRequest, 'index.ts'); + if (stats?.isDirectory()) { + for (const ext of RESOLVE_EXTENSIONS) { + const indexPath = Path.resolve(absoluteRequest, `index${ext}`); + const indexStats = await safeStat(indexPath); + if (indexStats?.isFile()) { + return indexPath; + } + } + } + + return; } /** @@ -118,8 +148,6 @@ export class BundleRefsPlugin { } const resolved = await this.cachedResolveRequest(context, request); - - // the path was not resolved because it should be ignored, failed resolves throw if (!resolved) { return; } From d92de6d7f771334c06a9dcb31390510b2f5b3279 Mon Sep 17 00:00:00 2001 From: spalger Date: Fri, 12 Jun 2020 23:18:09 -0700 Subject: [PATCH 09/31] remove specific load order --- src/legacy/ui/ui_render/ui_render_mixin.js | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/legacy/ui/ui_render/ui_render_mixin.js b/src/legacy/ui/ui_render/ui_render_mixin.js index b09d4861b343b..673e19155879a 100644 --- a/src/legacy/ui/ui_render/ui_render_mixin.js +++ b/src/legacy/ui/ui_render/ui_render_mixin.js @@ -26,8 +26,6 @@ import { AppBootstrap } from './bootstrap'; import { getApmConfig } from '../apm'; import { DllCompiler } from '../../../optimize/dynamic_dll_plugin'; -const uniq = (...items) => Array.from(new Set(items)); - /** * @typedef {import('../../server/kbn_server').default} KbnServer * @typedef {import('../../server/kbn_server').ResponseToolkit} ResponseToolkit @@ -150,15 +148,7 @@ export function uiRenderMixin(kbnServer, server, config) { ]), ]; - const kpPluginIds = uniq( - // load these plugins first, they are "shared" and other bundles access their - // public/index exports without considering topographic sorting by plugin deps (for now) - 'kibanaUtils', - 'kibanaReact', - 'data', - 'esUiShared', - ...kbnServer.newPlatform.__internals.uiPlugins.public.keys() - ); + const kpPluginIds = Array.from(kbnServer.newPlatform.__internals.uiPlugins.public.keys()); const jsDependencyPaths = [ ...UiSharedDeps.jsDepFilenames.map( From e1f4d083c54fb09cdc723c0e5bf823af1fb11c0c Mon Sep 17 00:00:00 2001 From: spalger Date: Fri, 12 Jun 2020 23:38:36 -0700 Subject: [PATCH 10/31] copy all properties from exported bundle into local bundle --- packages/kbn-optimizer/src/worker/bundle_ref_module.ts | 8 +------- src/legacy/ui/ui_render/bootstrap/template.js.hbs | 4 ++-- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/packages/kbn-optimizer/src/worker/bundle_ref_module.ts b/packages/kbn-optimizer/src/worker/bundle_ref_module.ts index 2120be3c071f9..2bc21e5c2971d 100644 --- a/packages/kbn-optimizer/src/worker/bundle_ref_module.ts +++ b/packages/kbn-optimizer/src/worker/bundle_ref_module.ts @@ -61,13 +61,7 @@ export class BundleRefModule extends Module { return ` __webpack_require__.r(__webpack_exports__); var ns = __kbnBundles__.get('${this.exportId}'); - var descriptor = Object.getOwnPropertyDescriptors(ns); - Object.keys(descriptor).forEach(function (key) { - var desc = descriptor[key] - if (desc.get) { - __webpack_require__.d(__webpack_exports__, key, desc.get) - } - }) + Object.defineProperties(__webpack_exports__, Object.getOwnPropertyDescriptors(ns)) `; } diff --git a/src/legacy/ui/ui_render/bootstrap/template.js.hbs b/src/legacy/ui/ui_render/bootstrap/template.js.hbs index dfba316c8faa7..c15e296aceded 100644 --- a/src/legacy/ui/ui_render/bootstrap/template.js.hbs +++ b/src/legacy/ui/ui_render/bootstrap/template.js.hbs @@ -7,7 +7,7 @@ window.__kbnBundles__ = (function () { var modules = {}; function has(prop) { - return Object.prototype.hasOwnProperty.call(modules, prop) + return Object.prototype.hasOwnProperty.call(modules, prop); } function define(key, bundleRequire, bundleModuleKey) { @@ -26,7 +26,7 @@ window.__kbnBundles__ = (function () { throw new Error('__kbnBundles__ does not have a module defined for "' + key + '"'); } - return modules[key].bundleRequire(modules[key].bundleModuleKey) + return modules[key].bundleRequire(modules[key].bundleModuleKey); } return { has: has, define: define, get: get }; From 78fd12a8103cea6a2ab30b066bc99cd3719251e1 Mon Sep 17 00:00:00 2001 From: spalger Date: Fri, 12 Jun 2020 23:39:09 -0700 Subject: [PATCH 11/31] update snapshot --- .../__snapshots__/basic_optimization.test.ts.snap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap b/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap index 350a93b88cca2..2265bad9f6afa 100644 --- a/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap +++ b/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap @@ -62,7 +62,7 @@ OptimizerConfig { } `; -exports[`prepares assets for distribution: bar bundle 1`] = `"(function(modules){var installedModules={};function __webpack_require__(moduleId){if(installedModules[moduleId]){return installedModules[moduleId].exports}var module=installedModules[moduleId]={i:moduleId,l:false,exports:{}};modules[moduleId].call(module.exports,module,module.exports,__webpack_require__);module.l=true;return module.exports}__webpack_require__.m=modules;__webpack_require__.c=installedModules;__webpack_require__.d=function(exports,name,getter){if(!__webpack_require__.o(exports,name)){Object.defineProperty(exports,name,{enumerable:true,get:getter})}};__webpack_require__.r=function(exports){if(typeof Symbol!==\\"undefined\\"&&Symbol.toStringTag){Object.defineProperty(exports,Symbol.toStringTag,{value:\\"Module\\"})}Object.defineProperty(exports,\\"__esModule\\",{value:true})};__webpack_require__.t=function(value,mode){if(mode&1)value=__webpack_require__(value);if(mode&8)return value;if(mode&4&&typeof value===\\"object\\"&&value&&value.__esModule)return value;var ns=Object.create(null);__webpack_require__.r(ns);Object.defineProperty(ns,\\"default\\",{enumerable:true,value:value});if(mode&2&&typeof value!=\\"string\\")for(var key in value)__webpack_require__.d(ns,key,function(key){return value[key]}.bind(null,key));return ns};__webpack_require__.n=function(module){var getter=module&&module.__esModule?function getDefault(){return module[\\"default\\"]}:function getModuleExports(){return module};__webpack_require__.d(getter,\\"a\\",getter);return getter};__webpack_require__.o=function(object,property){return Object.prototype.hasOwnProperty.call(object,property)};__webpack_require__.p=\\"\\";return __webpack_require__(__webpack_require__.s=5)})([function(module,exports,__webpack_require__){\\"use strict\\";var isOldIE=function isOldIE(){var memo;return function memorize(){if(typeof memo===\\"undefined\\"){memo=Boolean(window&&document&&document.all&&!window.atob)}return memo}}();var getTarget=function getTarget(){var memo={};return function memorize(target){if(typeof memo[target]===\\"undefined\\"){var styleTarget=document.querySelector(target);if(window.HTMLIFrameElement&&styleTarget instanceof window.HTMLIFrameElement){try{styleTarget=styleTarget.contentDocument.head}catch(e){styleTarget=null}}memo[target]=styleTarget}return memo[target]}}();var stylesInDom=[];function getIndexByIdentifier(identifier){var result=-1;for(var i=0;i Date: Fri, 12 Jun 2020 23:56:03 -0700 Subject: [PATCH 12/31] build examples separate from test plugins --- test/scripts/jenkins_build_kibana.sh | 9 +++++++-- test/scripts/jenkins_xpack_build_kibana.sh | 8 +++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/test/scripts/jenkins_build_kibana.sh b/test/scripts/jenkins_build_kibana.sh index e3f46e7a6ada4..5f1676594b3dd 100755 --- a/test/scripts/jenkins_build_kibana.sh +++ b/test/scripts/jenkins_build_kibana.sh @@ -2,9 +2,14 @@ source src/dev/ci_setup/setup_env.sh -echo " -> building kibana platform plugins" +echo " -> building examples separate from test plugins" node scripts/build_kibana_platform_plugins \ - --oss \ + --examples \ + --verbose; + +echo " -> building test plugins" +node scripts/build_kibana_platform_plugins \ + --no-examples \ --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \ --scan-dir "$KIBANA_DIR/test/interpreter_functional/plugins" \ --verbose; diff --git a/test/scripts/jenkins_xpack_build_kibana.sh b/test/scripts/jenkins_xpack_build_kibana.sh index c962b962b1e5e..58ef6a42d3fe4 100755 --- a/test/scripts/jenkins_xpack_build_kibana.sh +++ b/test/scripts/jenkins_xpack_build_kibana.sh @@ -3,8 +3,14 @@ cd "$KIBANA_DIR" source src/dev/ci_setup/setup_env.sh -echo " -> building kibana platform plugins" +echo " -> building examples separate from test plugins" node scripts/build_kibana_platform_plugins \ + --examples \ + --verbose; + +echo " -> building test plugins" +node scripts/build_kibana_platform_plugins \ + --no-examples \ --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \ --scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \ --scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \ From 55351f6c697afd95a787b66dc27d963fa07569c4 Mon Sep 17 00:00:00 2001 From: spalger Date: Sat, 13 Jun 2020 00:37:04 -0700 Subject: [PATCH 13/31] restore --oss flags for --oss example builds --- test/scripts/jenkins_build_kibana.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/scripts/jenkins_build_kibana.sh b/test/scripts/jenkins_build_kibana.sh index 5f1676594b3dd..3e49edc8e6ae5 100755 --- a/test/scripts/jenkins_build_kibana.sh +++ b/test/scripts/jenkins_build_kibana.sh @@ -4,11 +4,13 @@ source src/dev/ci_setup/setup_env.sh echo " -> building examples separate from test plugins" node scripts/build_kibana_platform_plugins \ + --oss \ --examples \ --verbose; echo " -> building test plugins" node scripts/build_kibana_platform_plugins \ + --oss \ --no-examples \ --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \ --scan-dir "$KIBANA_DIR/test/interpreter_functional/plugins" \ From 59a3af0ff643bf64295cb48eb9c87fe67c64fb59 Mon Sep 17 00:00:00 2001 From: spalger Date: Sat, 13 Jun 2020 01:11:03 -0700 Subject: [PATCH 14/31] fix mapsLegacy leaflet wrapper, it used to rely on multiple instances of the module being loaded --- src/plugins/maps_legacy/public/leaflet.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/plugins/maps_legacy/public/leaflet.js b/src/plugins/maps_legacy/public/leaflet.js index e36da2c52b8c5..929c866a2b30b 100644 --- a/src/plugins/maps_legacy/public/leaflet.js +++ b/src/plugins/maps_legacy/public/leaflet.js @@ -21,9 +21,9 @@ export let L; if (!window.hasOwnProperty('L')) { require('leaflet/dist/leaflet.css'); - window.L = require('leaflet/dist/leaflet.js'); - window.L.Browser.touch = false; - window.L.Browser.pointer = false; + L = window.L = require('leaflet/dist/leaflet.js'); + L.Browser.touch = false; + L.Browser.pointer = false; require('leaflet-vega'); require('leaflet.heat/dist/leaflet-heat.js'); From eeb8c1b0a63e64894ee703abd1deaab7f103faf5 Mon Sep 17 00:00:00 2001 From: spalger Date: Sat, 13 Jun 2020 01:14:06 -0700 Subject: [PATCH 15/31] avoid mutable export --- src/plugins/maps_legacy/public/leaflet.js | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/plugins/maps_legacy/public/leaflet.js b/src/plugins/maps_legacy/public/leaflet.js index 929c866a2b30b..bee75021c76ad 100644 --- a/src/plugins/maps_legacy/public/leaflet.js +++ b/src/plugins/maps_legacy/public/leaflet.js @@ -17,13 +17,11 @@ * under the License. */ -export let L; - if (!window.hasOwnProperty('L')) { require('leaflet/dist/leaflet.css'); - L = window.L = require('leaflet/dist/leaflet.js'); - L.Browser.touch = false; - L.Browser.pointer = false; + window.L = require('leaflet/dist/leaflet.js'); + window.L.Browser.touch = false; + window.L.Browser.pointer = false; require('leaflet-vega'); require('leaflet.heat/dist/leaflet-heat.js'); @@ -31,6 +29,6 @@ if (!window.hasOwnProperty('L')) { require('leaflet-draw/dist/leaflet.draw.js'); require('leaflet-responsive-popup/leaflet.responsive.popup.css'); require('leaflet-responsive-popup/leaflet.responsive.popup.js'); -} else { - L = window.L; } + +export const L = window.L; From 977e0dbead3174007eb4017ba9396ae93604ebfb Mon Sep 17 00:00:00 2001 From: spalger Date: Sat, 13 Jun 2020 03:01:47 -0700 Subject: [PATCH 16/31] remove test scripts --- scripts/find_cross_plugin_imports.ts | 91 ---------------------------- scripts/find_plugin_dep_counts.ts | 91 ---------------------------- 2 files changed, 182 deletions(-) delete mode 100644 scripts/find_cross_plugin_imports.ts delete mode 100644 scripts/find_plugin_dep_counts.ts diff --git a/scripts/find_cross_plugin_imports.ts b/scripts/find_cross_plugin_imports.ts deleted file mode 100644 index a37ca000ed80d..0000000000000 --- a/scripts/find_cross_plugin_imports.ts +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import Path from 'path'; -import Fs from 'fs'; - -import globby from 'globby'; -import { run, REPO_ROOT } from '@kbn/dev-utils'; - -run(async ({ log }) => { - const statFilePaths = await globby( - [ - '**/target/public/stats.json', - '!**/{examples,node_modules,fixtures,__fixtures__,sao_template}/**', - ], - { - cwd: REPO_ROOT, - } - ); - - // log.success(inspect(files, { maxArrayLength: Infinity })); - - const plugins = statFilePaths.map((statFilePath) => { - const root = Path.resolve(Path.dirname(statFilePath), '../..'); - - let id; - try { - const manifest = JSON.parse(Fs.readFileSync(Path.resolve(root, 'kibana.json'), 'utf8')); - id = manifest.id; - } catch (error) { - id = Path.basename(root); - } - - const publicDir = Path.resolve(root, 'public'); - return { - id, - publicDir, - statFilePath, - }; - }); - - for (const plugin of plugins) { - const stats = JSON.parse(Fs.readFileSync(plugin.statFilePath, 'utf8')); - const [id] = Object.keys(stats.entrypoints); - const mainChunk = stats.chunks.find((c) => c.id === id); - const moduleCount = stats.modules.length; - const mainAssetSize = stats.assets.find((a) => a.name === `${id}.entry.js` || `${id}.plugin.js`) - .size; - - const modules = (stats.modules as any[]) - .filter((m) => !m.identifier.startsWith('external')) - .filter((m) => m.chunks.includes(mainChunk.id)) - .map((m) => { - const idParts = m.identifier.split('!'); - const last = idParts.pop(); - const [modulePath] = last.split('?'); - return modulePath as string; - }) - .filter((m) => !m.includes('/node_modules/')); - - const importedPlugins = plugins - .map((p) => { - if (p.id === id) { - return { plugin: p, modules: [] }; - } - - return { plugin: p, modules: modules.filter((m) => m.startsWith(p.publicDir)) }; - }) - .filter((i) => i.modules.length); - - log.info(id, moduleCount, mainAssetSize, importedPlugins); - - await new Promise((resolve) => setTimeout(resolve, 10)); - } -}); diff --git a/scripts/find_plugin_dep_counts.ts b/scripts/find_plugin_dep_counts.ts deleted file mode 100644 index d32c620e58761..0000000000000 --- a/scripts/find_plugin_dep_counts.ts +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import Fs from 'fs'; - -import globby from 'globby'; -import { run, REPO_ROOT } from '@kbn/dev-utils'; - -run(async ({ log }) => { - const files = await globby( - ['**/kibana.json', '!**/{node_modules,fixtures,__fixtures__,sao_template}/**'], - { - cwd: REPO_ROOT, - } - ); - - const plugins = files.map((path) => JSON.parse(Fs.readFileSync(path, 'utf8'))); - - const loadOrder = new Set(); - - function load(plugin: any, path: string[] = []) { - if (path.includes(plugin.id)) { - log.warning('circular dep', [...path.slice(path.indexOf(plugin.id)), plugin.id].join(' -> ')); - return; - } - path = [...path, plugin.id]; - - for (const depId of plugin.requiredPlugins || []) { - const dep = plugins.find((p) => p.id === depId); - if (dep.ui) { - load(dep, path); - } - } - - for (const depId of plugin.optionalPlugins || []) { - const dep = plugins.find((p) => p.id === depId); - if (dep && dep.ui) { - load(dep, path); - } - } - - if (!loadOrder.has(plugin.id)) { - loadOrder.add(plugin.id); - } - } - - for (const plugin of plugins) { - if (plugin.ui) { - load(plugin); - } - } - - const depCount = new Map(); - function incrDepCount(id: string) { - const count = depCount.get(id) || 0; - depCount.set(id, count + 1); - } - - for (const plugin of plugins) { - for (const depId of plugin.requiredPlugins || []) { - incrDepCount(depId); - } - for (const depId of plugin.optionalPlugins || []) { - incrDepCount(depId); - } - } - - const pluginUsageInLoadOrder = new Map(); - for (const id of loadOrder) { - const count = depCount.get(id) || 0; - pluginUsageInLoadOrder.set(id, count); - } - - log.success(pluginUsageInLoadOrder); -}); From fdd8d3668271c966156e94780323b461f933abc3 Mon Sep 17 00:00:00 2001 From: spalger Date: Sat, 13 Jun 2020 11:43:47 -0700 Subject: [PATCH 17/31] remove unused import --- packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts b/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts index b63b0f7b0433c..1e197a4333e51 100644 --- a/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts +++ b/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts @@ -19,7 +19,6 @@ import Path from 'path'; import Fs from 'fs'; -import { promisify } from 'util'; import webpack from 'webpack'; From d6703227fe21cb2e4f14509ead67f432da8c79c4 Mon Sep 17 00:00:00 2001 From: spalger Date: Sat, 13 Jun 2020 11:44:21 -0700 Subject: [PATCH 18/31] fix __kbnBundles__ stub --- src/core/public/plugins/plugin_reader.test.ts | 33 ++++++++++++------- src/core/public/plugins/plugin_reader.ts | 10 +++--- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/src/core/public/plugins/plugin_reader.test.ts b/src/core/public/plugins/plugin_reader.test.ts index d4324f81de8e6..b3bc84f30daac 100644 --- a/src/core/public/plugins/plugin_reader.test.ts +++ b/src/core/public/plugins/plugin_reader.test.ts @@ -17,25 +17,37 @@ * under the License. */ -import { CoreWindow, read, UnknownPluginInitializer } from './plugin_reader'; +import { CoreWindow, read } from './plugin_reader'; + +const coreWindow: CoreWindow & { + __kbnBundles__: { stub(key: string, value: any): void }; +} = window as any; -const coreWindow: CoreWindow = window as any; beforeEach(() => { - coreWindow.__kbnBundles__ = {}; + const stubs = new Map(); + coreWindow.__kbnBundles__ = { + get(key) { + return stubs.get(key); + }, + has(key) { + return stubs.has(key); + }, + stub(key, value) { + stubs.set(key, value); + }, + }; }); it('handles undefined plugin exports', () => { - coreWindow.__kbnBundles__['plugin/foo'] = undefined; - expect(() => { read('foo'); }).toThrowError(`Definition of plugin "foo" not found and may have failed to load.`); }); it('handles plugin exports with a "plugin" export that is not a function', () => { - coreWindow.__kbnBundles__['plugin/foo'] = { + coreWindow.__kbnBundles__.stub('plugin/foo/public', { plugin: 1234, - } as any; + }); expect(() => { read('foo'); @@ -43,11 +55,8 @@ it('handles plugin exports with a "plugin" export that is not a function', () => }); it('returns the plugin initializer when the "plugin" named export is a function', () => { - const plugin: UnknownPluginInitializer = () => { - return undefined as any; - }; - - coreWindow.__kbnBundles__['plugin/foo'] = { plugin }; + const plugin = () => {}; + coreWindow.__kbnBundles__.stub('plugin/foo/public', { plugin }); expect(read('foo')).toBe(plugin); }); diff --git a/src/core/public/plugins/plugin_reader.ts b/src/core/public/plugins/plugin_reader.ts index 2e1cd4b6aa6fa..d80bda7483775 100644 --- a/src/core/public/plugins/plugin_reader.ts +++ b/src/core/public/plugins/plugin_reader.ts @@ -42,13 +42,13 @@ export interface CoreWindow { export function read(name: string) { const coreWindow = (window as unknown) as CoreWindow; const exportId = `plugin/${name}/public`; - const pluginExport = coreWindow.__kbnBundles__.has(exportId) - ? coreWindow.__kbnBundles__.get(exportId) - : undefined; - if (!pluginExport) { + if (!coreWindow.__kbnBundles__.has(exportId)) { throw new Error(`Definition of plugin "${name}" not found and may have failed to load.`); - } else if (typeof pluginExport.plugin !== 'function') { + } + + const pluginExport = coreWindow.__kbnBundles__.get(exportId); + if (typeof pluginExport?.plugin !== 'function') { throw new Error(`Definition of plugin "${name}" should be a function.`); } else { return pluginExport.plugin; From 95c65bd83d1d39a9d9f3ffb629f36139bb13d142 Mon Sep 17 00:00:00 2001 From: spalger Date: Sat, 13 Jun 2020 11:45:07 -0700 Subject: [PATCH 19/31] fix manifest parsing --- .../discovery/plugin_manifest_parser.ts | 19 ++++++++++++++++++- src/core/server/plugins/types.ts | 2 +- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/core/server/plugins/discovery/plugin_manifest_parser.ts b/src/core/server/plugins/discovery/plugin_manifest_parser.ts index 58a1abdf57fea..f2c3a29eca0ac 100644 --- a/src/core/server/plugins/discovery/plugin_manifest_parser.ts +++ b/src/core/server/plugins/discovery/plugin_manifest_parser.ts @@ -71,7 +71,11 @@ const KNOWN_MANIFEST_FIELDS = (() => { * @param packageInfo Kibana package info. * @internal */ -export async function parseManifest(pluginPath: string, packageInfo: PackageInfo, log: Logger) { +export async function parseManifest( + pluginPath: string, + packageInfo: PackageInfo, + log: Logger +): Promise { const manifestPath = resolve(pluginPath, MANIFEST_FILE_NAME); let manifestContent; @@ -131,6 +135,19 @@ export async function parseManifest(pluginPath: string, packageInfo: PackageInfo ); } + if ( + manifest.extraPublicDirs && + (!Array.isArray(manifest.extraPublicDirs) || + !manifest.extraPublicDirs.every((dir) => typeof dir === 'string')) + ) { + throw PluginDiscoveryError.invalidManifest( + manifestPath, + new Error( + `The "extraPublicDirs" in plugin manifest for "${manifest.id}" should be an array of strings.` + ) + ); + } + const expectedKibanaVersion = typeof manifest.kibanaVersion === 'string' && manifest.kibanaVersion ? manifest.kibanaVersion diff --git a/src/core/server/plugins/types.ts b/src/core/server/plugins/types.ts index 1200bb80d33a0..370f05c9afc10 100644 --- a/src/core/server/plugins/types.ts +++ b/src/core/server/plugins/types.ts @@ -158,7 +158,7 @@ export interface PluginManifest { * Specifies directory names that can be imported by other plugins sharing * the same instance of the @kbn/optimizer. */ - readonly extraPublicDirs: string[]; + readonly extraPublicDirs?: string[]; } /** From a73e37f2c98b65299434e41de1732efda9fbedf5 Mon Sep 17 00:00:00 2001 From: spalger Date: Sat, 13 Jun 2020 12:46:50 -0700 Subject: [PATCH 20/31] update server api docs --- src/core/server/server.api.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index 78cc02d39e6c4..b71ac418b6fb3 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -1520,6 +1520,7 @@ export interface PluginInitializerContext { // @public export interface PluginManifest { readonly configPath: ConfigPath; + readonly extraPublicDirs?: string[]; readonly id: PluginName; readonly kibanaVersion: string; readonly optionalPlugins: readonly PluginName[]; @@ -2549,8 +2550,8 @@ export const validBodyOutput: readonly ["data", "stream"]; // src/core/server/legacy/types.ts:165:3 - (ae-forgotten-export) The symbol "LegacyNavLinkSpec" needs to be exported by the entry point index.d.ts // src/core/server/legacy/types.ts:166:3 - (ae-forgotten-export) The symbol "LegacyAppSpec" needs to be exported by the entry point index.d.ts // src/core/server/legacy/types.ts:167:16 - (ae-forgotten-export) The symbol "LegacyPluginSpec" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:230:3 - (ae-forgotten-export) The symbol "KibanaConfigType" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:230:3 - (ae-forgotten-export) The symbol "SharedGlobalConfigKeys" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:232:3 - (ae-forgotten-export) The symbol "PathConfigType" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:236:3 - (ae-forgotten-export) The symbol "KibanaConfigType" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:236:3 - (ae-forgotten-export) The symbol "SharedGlobalConfigKeys" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:238:3 - (ae-forgotten-export) The symbol "PathConfigType" needs to be exported by the entry point index.d.ts ``` From af3bbc212450d253dd66199c8ce803923a58830b Mon Sep 17 00:00:00 2001 From: spalger Date: Mon, 15 Jun 2020 09:59:45 -0700 Subject: [PATCH 21/31] add a chunk to the readme about bundle refs --- packages/kbn-optimizer/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/kbn-optimizer/README.md b/packages/kbn-optimizer/README.md index c7f50c6af8dfd..cba4c5acf98b7 100644 --- a/packages/kbn-optimizer/README.md +++ b/packages/kbn-optimizer/README.md @@ -30,6 +30,16 @@ Bundles built by the the optimizer include a cache file which describes the info When a bundle is determined to be up-to-date a worker is not started for the bundle. If running the optimizer with the `--dev/--watch` flag, then all the files referenced by cached bundles are watched for changes. Once a change is detected in any of the files referenced by the built bundle a worker is started. If a file is changed that is referenced by several bundles then workers will be started for each bundle, combining workers together to respect the worker limit. +## Bundle Refs + +In order to dramatically reduce the size of our bundles, and the time it takes to build them, this PR makes the relationships between plugins/bundles explicit using "bundle refs". When the optimizer starts a list of "refs" that could be had is compiled from the list of bundles being built and sent to each worker. That list is used to determine which import statements in a bundle should be replaced with a runtime reference to the output of another bundle. + +At runtime the bundles now share a set of entry points. By default a plugin shares `public` so that other code can use relative imports to access that directory. To expose additional directories they must be listed in the plugin's kibana.json "extraPublicDirs" field. The directories listed there will **also** be exported from the plugins bundle so that any other plugin can import that directory. "common" is commonly in the list of "extraPublicDirs". + +When a directory is listed in the "extraPublicDirs" it will always be included in the bundle so that other plugins have access to it. The worker building the bundle has no way of knowing whether another plugin is using the directory, so be careful of adding test code or unnecessary directories to that list. + +Any import in a bundle which resolves into another bundles "context" directory, ie `src/plugins/*`, must map explicitly to a "public dir" exported by that plugin. If the resolved import is not in the list of public dirs an error will be thrown and the optimizer will fail to build the bundles until the error is fixed. + ## API To run the optimizer from code, you can import the [`OptimizerConfig`][OptimizerConfig] class and [`runOptimizer`][Optimizer] function. Create an [`OptimizerConfig`][OptimizerConfig] instance by calling it's static `create()` method with some options, then pass it to the [`runOptimizer`][Optimizer] function. `runOptimizer()` returns an observable of update objects, which are summaries of the optimizer state plus an optional `event` property which describes the internal events occuring and may be of use. You can use the [`logOptimizerState()`][LogOptimizerState] helper to write the relevant bits of state to a tooling log or checkout it's implementation to see how the internal events like [`WorkerStdio`][ObserveWorker] and [`WorkerStarted`][ObserveWorker] are used. From ce04f667e538d2c3eef089ebe9af4b50bde77f4b Mon Sep 17 00:00:00 2001 From: spalger Date: Mon, 15 Jun 2020 10:55:26 -0700 Subject: [PATCH 22/31] rename ref.id to ref.bundleId --- packages/kbn-optimizer/src/common/bundle_refs.ts | 14 +++++++------- .../kbn-optimizer/src/worker/bundle_refs_plugin.ts | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/kbn-optimizer/src/common/bundle_refs.ts b/packages/kbn-optimizer/src/common/bundle_refs.ts index 95af2ea7db387..c9ca152ed076e 100644 --- a/packages/kbn-optimizer/src/common/bundle_refs.ts +++ b/packages/kbn-optimizer/src/common/bundle_refs.ts @@ -23,7 +23,7 @@ import { Bundle } from './bundle'; import { UnknownVals } from './ts_helpers'; export interface BundleRef { - id: string; + bundleId: string; contextDir: string; contextPrefix: string; entry: string; @@ -38,7 +38,7 @@ export class BundleRefs { ...acc, ...b.publicDirNames.map( (name): BundleRef => ({ - id: b.id, + bundleId: b.id, contextDir: b.contextDir, // Path.resolve converts separators and strips the final separator contextPrefix: Path.resolve(b.contextDir) + Path.sep, @@ -75,9 +75,9 @@ export class BundleRefs { throw new Error('`bundleRefs[]` must be an object'); } - const { id } = refSpec; - if (typeof id !== 'string') { - throw new Error('`bundleRefs[].id` must be a string'); + const { bundleId } = refSpec; + if (typeof bundleId !== 'string') { + throw new Error('`bundleRefs[].bundleId` must be a string'); } const { contextDir } = refSpec; @@ -101,11 +101,11 @@ export class BundleRefs { } return { + bundleId, contextDir, contextPrefix, entry, exportId, - id, }; } ) @@ -127,7 +127,7 @@ export class BundleRefs { public filterByContextPrefix(bundle: Bundle, absolutePath: string) { return this.refs.filter( - (ref) => ref.id !== bundle.id && absolutePath.startsWith(ref.contextPrefix) + (ref) => ref.bundleId !== bundle.id && absolutePath.startsWith(ref.contextPrefix) ); } diff --git a/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts b/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts index 1e197a4333e51..25d9eb89e6b14 100644 --- a/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts +++ b/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts @@ -166,7 +166,7 @@ export class BundleRefsPlugin { } if (!matchingRef) { - const bundleId = Array.from(new Set(eligibleRefs.map((r) => r.id))).join(', '); + const bundleId = Array.from(new Set(eligibleRefs.map((r) => r.bundleId))).join(', '); const publicDir = eligibleRefs.map((r) => r.entry).join(', '); throw new Error( `import [${request}] references a non-public export of the [${bundleId}] bundle and must point to one of the public directories: [${publicDir}]` From a9886f8e2490b8309ea2b01dd879ff358f2e7d59 Mon Sep 17 00:00:00 2001 From: spalger Date: Mon, 15 Jun 2020 10:55:47 -0700 Subject: [PATCH 23/31] stop looking for a matching ref after the first match --- packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts b/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts index 25d9eb89e6b14..2c054dbd585a2 100644 --- a/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts +++ b/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts @@ -162,6 +162,7 @@ export class BundleRefsPlugin { const resolvedEntry = await this.cachedResolveRequest(ref.contextDir, ref.entry); if (resolved === resolvedEntry) { matchingRef = ref; + break; } } From 2973469f936bfdf51efe9e218aa2ffbfddb74e7e Mon Sep 17 00:00:00 2001 From: spalger Date: Mon, 15 Jun 2020 11:00:41 -0700 Subject: [PATCH 24/31] simplify BundleRefs#filterByExportIds() --- packages/kbn-optimizer/src/common/bundle_refs.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/packages/kbn-optimizer/src/common/bundle_refs.ts b/packages/kbn-optimizer/src/common/bundle_refs.ts index c9ca152ed076e..a5c60f2031c0b 100644 --- a/packages/kbn-optimizer/src/common/bundle_refs.ts +++ b/packages/kbn-optimizer/src/common/bundle_refs.ts @@ -115,14 +115,7 @@ export class BundleRefs { constructor(private readonly refs: BundleRef[]) {} public filterByExportIds(exportIds: string[]) { - const refs: BundleRef[] = []; - for (const exportId of exportIds) { - const ref = this.refs.find((r) => r.exportId === exportId); - if (ref) { - refs.push(ref); - } - } - return refs; + return this.refs.filter((r) => exportIds.includes(r.exportId)); } public filterByContextPrefix(bundle: Bundle, absolutePath: string) { From 8fb33308f604d16dc3b3e2b59a6afe0a0e2746f4 Mon Sep 17 00:00:00 2001 From: spalger Date: Mon, 15 Jun 2020 11:07:47 -0700 Subject: [PATCH 25/31] fix bundle_cache tests by defining bundleRefExportIds --- .../integration_tests/bundle_cache.test.ts | 63 ++++++++++++++++++- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/packages/kbn-optimizer/src/integration_tests/bundle_cache.test.ts b/packages/kbn-optimizer/src/integration_tests/bundle_cache.test.ts index d9fe04f66a799..48cab508954a0 100644 --- a/packages/kbn-optimizer/src/integration_tests/bundle_cache.test.ts +++ b/packages/kbn-optimizer/src/integration_tests/bundle_cache.test.ts @@ -75,6 +75,7 @@ it('emits "bundle cached" event when everything is updated', async () => { optimizerCacheKey, files, moduleCount: files.length, + bundleRefExportIds: [], }); const cacheEvents = await getBundleCacheEvent$(config, optimizerCacheKey) @@ -85,8 +86,7 @@ it('emits "bundle cached" event when everything is updated', async () => { Array [ Object { "bundle": , - "reason": "bundle references missing", - "type": "bundle not cached", + "type": "bundle cached", }, ] `); @@ -116,6 +116,7 @@ it('emits "bundle not cached" event when cacheKey is up to date but caching is d optimizerCacheKey, files, moduleCount: files.length, + bundleRefExportIds: [], }); const cacheEvents = await getBundleCacheEvent$(config, optimizerCacheKey) @@ -156,6 +157,7 @@ it('emits "bundle not cached" event when optimizerCacheKey is missing', async () optimizerCacheKey: undefined, files, moduleCount: files.length, + bundleRefExportIds: [], }); const cacheEvents = await getBundleCacheEvent$(config, optimizerCacheKey) @@ -196,6 +198,7 @@ it('emits "bundle not cached" event when optimizerCacheKey is outdated, includes optimizerCacheKey: 'old', files, moduleCount: files.length, + bundleRefExportIds: [], }); const cacheEvents = await getBundleCacheEvent$(config, optimizerCacheKey) @@ -218,6 +221,53 @@ it('emits "bundle not cached" event when optimizerCacheKey is outdated, includes `); }); +it('emits "bundle not cached" event when bundleRefExportIds is outdated, includes diff', async () => { + const config = OptimizerConfig.create({ + repoRoot: MOCK_REPO_DIR, + pluginScanDirs: [], + pluginPaths: [Path.resolve(MOCK_REPO_DIR, 'plugins/foo')], + maxWorkerCount: 1, + }); + const [bundle] = config.bundles; + + const optimizerCacheKey = 'optimizerCacheKey'; + const files = [ + Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/ext.ts'), + Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/index.ts'), + Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/lib.ts'), + ]; + const mtimes = await getMtimes(files); + const cacheKey = bundle.createCacheKey(files, mtimes); + + bundle.cache.set({ + cacheKey, + optimizerCacheKey, + files, + moduleCount: files.length, + bundleRefExportIds: ['plugin/bar/public'], + }); + + const cacheEvents = await getBundleCacheEvent$(config, optimizerCacheKey) + .pipe(toArray()) + .toPromise(); + + expect(cacheEvents).toMatchInlineSnapshot(` + Array [ + Object { + "bundle": , + "diff": "- Expected + + Received + +  [ + + \\"plugin/bar/public\\" +  ]", + "reason": "bundle references outdated", + "type": "bundle not cached", + }, + ] + `); +}); + it('emits "bundle not cached" event when cacheKey is missing', async () => { const config = OptimizerConfig.create({ repoRoot: MOCK_REPO_DIR, @@ -239,6 +289,7 @@ it('emits "bundle not cached" event when cacheKey is missing', async () => { optimizerCacheKey, files, moduleCount: files.length, + bundleRefExportIds: [], }); const cacheEvents = await getBundleCacheEvent$(config, optimizerCacheKey) @@ -277,6 +328,7 @@ it('emits "bundle not cached" event when cacheKey is outdated', async () => { optimizerCacheKey, files, moduleCount: files.length, + bundleRefExportIds: [], }); jest.spyOn(bundle, 'createCacheKey').mockImplementation(() => 'new'); @@ -289,7 +341,12 @@ it('emits "bundle not cached" event when cacheKey is outdated', async () => { Array [ Object { "bundle": , - "reason": "bundle references missing", + "diff": "- Expected + + Received + + - \\"old\\" + + \\"new\\"", + "reason": "cache key mismatch", "type": "bundle not cached", }, ] From c67020c4baae61e619bb98305df9f1a7939dee12 Mon Sep 17 00:00:00 2001 From: spalger Date: Mon, 15 Jun 2020 11:35:21 -0700 Subject: [PATCH 26/31] add new public dir for embeddable_examples plugin --- examples/embeddable_examples/kibana.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/embeddable_examples/kibana.json b/examples/embeddable_examples/kibana.json index 3b43916e7e2c4..b3ee0de096989 100644 --- a/examples/embeddable_examples/kibana.json +++ b/examples/embeddable_examples/kibana.json @@ -6,5 +6,5 @@ "ui": true, "requiredPlugins": ["embeddable"], "optionalPlugins": [], - "extraPublicDirs": ["public/todo"] + "extraPublicDirs": ["public/todo", "public/hello_world"] } From 01abf178fe9915f868b66d7e869a0188495410a8 Mon Sep 17 00:00:00 2001 From: spalger Date: Mon, 15 Jun 2020 13:07:53 -0700 Subject: [PATCH 27/31] fix comment and rename internal method for clarity --- .../src/worker/bundle_refs_plugin.ts | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts b/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts index 2c054dbd585a2..4c7ae0b9df667 100644 --- a/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts +++ b/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts @@ -85,9 +85,9 @@ export class BundleRefsPlugin { apply(compiler: webpack.Compiler) { hookIntoCompiler(compiler, async (context, request) => { - const exportId = await this.resolveExternal(context, request); - if (exportId) { - return new BundleRefModule(exportId); + const ref = await this.resolveRef(context, request); + if (ref) { + return new BundleRefModule(ref.exportId); } }); } @@ -125,17 +125,12 @@ export class BundleRefsPlugin { } /** - * Determine externals statements for require/import statements by looking - * for requests resolving to the primary public export of the data, kibanaReact, - * amd kibanaUtils plugins. If this module is being imported then rewrite - * the import to access the global `__kbnBundles__` variables and access - * the relavent properties from that global object. - * - * @param bundle - * @param context the directory containing the module which made `request` - * @param request the request for a module from a commonjs require() call or import statement + * Determine if an import request resolves to a bundleRef export id. If the + * request resolves to a bundle ref context but none of the exported directories + * then an error is thrown. If the request does not resolve to a bundleRef then + * undefined is returned. Otherwise it returns the referenced bundleRef. */ - async resolveExternal(context: string, request: string) { + private async resolveRef(context: string, request: string) { // ignore imports that have loaders defined or are not relative seeming if (request.includes('!') || !request.startsWith('.')) { return; @@ -174,6 +169,6 @@ export class BundleRefsPlugin { ); } - return matchingRef.exportId; + return matchingRef; } } From 6f16b8729027c47a90950d6d1251555bdfe53bae Mon Sep 17 00:00:00 2001 From: spalger Date: Mon, 15 Jun 2020 13:21:14 -0700 Subject: [PATCH 28/31] add notice comments to the modules inspired by webpack's externals plugin --- NOTICE.txt | 5 ++++ .../src/worker/bundle_ref_module.ts | 23 ++++++------------- .../src/worker/bundle_refs_plugin.ts | 23 ++++++------------- 3 files changed, 19 insertions(+), 32 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 33c1d535d7df3..946b328b8766c 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -21,6 +21,11 @@ used. Logarithmic ticks are places at powers of ten and at half those values if there are not to many ticks already (e.g. [1, 5, 10, 50, 100]). For details, see https://github.com/flot/flot/pull/1328 +--- +This module was heavily inspired by the externals plugin that ships with webpack@97d58d31 +MIT License http://www.opensource.org/licenses/mit-license.php +Author Tobias Koppers @sokra + --- This product has relied on ASTExplorer that is licensed under MIT. diff --git a/packages/kbn-optimizer/src/worker/bundle_ref_module.ts b/packages/kbn-optimizer/src/worker/bundle_ref_module.ts index 2bc21e5c2971d..cde25564cf528 100644 --- a/packages/kbn-optimizer/src/worker/bundle_ref_module.ts +++ b/packages/kbn-optimizer/src/worker/bundle_ref_module.ts @@ -1,20 +1,11 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 +/* eslint-disable @kbn/eslint/require-license-header */ + +/** + * @notice * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This module was heavily inspired by the externals plugin that ships with webpack@97d58d31 + * MIT License http://www.opensource.org/licenses/mit-license.php + * Author Tobias Koppers @sokra */ // @ts-ignore not typed by @types/webpack diff --git a/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts b/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts index 4c7ae0b9df667..6defcaa787b7d 100644 --- a/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts +++ b/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts @@ -1,20 +1,11 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 +/* eslint-disable @kbn/eslint/require-license-header */ + +/** + * @notice * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This module was heavily inspired by the externals plugin that ships with webpack@97d58d31 + * MIT License http://www.opensource.org/licenses/mit-license.php + * Author Tobias Koppers @sokra */ import Path from 'path'; From 7e9ceaad31d588a021b821a7ac89b0b8c6808130 Mon Sep 17 00:00:00 2001 From: spalger Date: Mon, 15 Jun 2020 13:48:18 -0700 Subject: [PATCH 29/31] improve comment for extraPublicDirs and immediately deprecate, regen docs --- ...ore-server.pluginmanifest.extrapublicdirs.md | 17 +++++++++++++++++ .../kibana-plugin-core-server.pluginmanifest.md | 1 + src/core/server/plugins/types.ts | 6 ++++-- src/core/server/server.api.md | 7 ++++--- 4 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 docs/development/core/server/kibana-plugin-core-server.pluginmanifest.extrapublicdirs.md diff --git a/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.extrapublicdirs.md b/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.extrapublicdirs.md new file mode 100644 index 0000000000000..c46e60f2ecf6d --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.extrapublicdirs.md @@ -0,0 +1,17 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [PluginManifest](./kibana-plugin-core-server.pluginmanifest.md) > [extraPublicDirs](./kibana-plugin-core-server.pluginmanifest.extrapublicdirs.md) + +## PluginManifest.extraPublicDirs property + +> Warning: This API is now obsolete. +> +> + +Specifies directory names that can be imported by other ui-plugins built using the same instance of the @kbn/optimizer. A temporary measure we plan to replace with better mechanisms for sharing static code between plugins + +Signature: + +```typescript +readonly extraPublicDirs?: string[]; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md b/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md index fe0ca476bbcb2..5edee51d6c523 100644 --- a/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md +++ b/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md @@ -21,6 +21,7 @@ Should never be used in code outside of Core but is exported for documentation p | Property | Type | Description | | --- | --- | --- | | [configPath](./kibana-plugin-core-server.pluginmanifest.configpath.md) | ConfigPath | Root [configuration path](./kibana-plugin-core-server.configpath.md) used by the plugin, defaults to "id" in snake\_case format. | +| [extraPublicDirs](./kibana-plugin-core-server.pluginmanifest.extrapublicdirs.md) | string[] | Specifies directory names that can be imported by other ui-plugins built using the same instance of the @kbn/optimizer. A temporary measure we plan to replace with better mechanisms for sharing static code between plugins | | [id](./kibana-plugin-core-server.pluginmanifest.id.md) | PluginName | Identifier of the plugin. Must be a string in camelCase. Part of a plugin public contract. Other plugins leverage it to access plugin API, navigate to the plugin, etc. | | [kibanaVersion](./kibana-plugin-core-server.pluginmanifest.kibanaversion.md) | string | The version of Kibana the plugin is compatible with, defaults to "version". | | [optionalPlugins](./kibana-plugin-core-server.pluginmanifest.optionalplugins.md) | readonly PluginName[] | An optional list of the other plugins that if installed and enabled \*\*may be\*\* leveraged by this plugin for some additional functionality but otherwise are not required for this plugin to work properly. | diff --git a/src/core/server/plugins/types.ts b/src/core/server/plugins/types.ts index 370f05c9afc10..2ca5c9f6ed3c5 100644 --- a/src/core/server/plugins/types.ts +++ b/src/core/server/plugins/types.ts @@ -155,8 +155,10 @@ export interface PluginManifest { readonly server: boolean; /** - * Specifies directory names that can be imported by other plugins sharing - * the same instance of the @kbn/optimizer. + * Specifies directory names that can be imported by other ui-plugins built + * using the same instance of the @kbn/optimizer. A temporary measure we plan + * to replace with better mechanisms for sharing static code between plugins + * @deprecated */ readonly extraPublicDirs?: string[]; } diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index b71ac418b6fb3..ecfa09fbd37f3 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -1520,6 +1520,7 @@ export interface PluginInitializerContext { // @public export interface PluginManifest { readonly configPath: ConfigPath; + // @deprecated readonly extraPublicDirs?: string[]; readonly id: PluginName; readonly kibanaVersion: string; @@ -2550,8 +2551,8 @@ export const validBodyOutput: readonly ["data", "stream"]; // src/core/server/legacy/types.ts:165:3 - (ae-forgotten-export) The symbol "LegacyNavLinkSpec" needs to be exported by the entry point index.d.ts // src/core/server/legacy/types.ts:166:3 - (ae-forgotten-export) The symbol "LegacyAppSpec" needs to be exported by the entry point index.d.ts // src/core/server/legacy/types.ts:167:16 - (ae-forgotten-export) The symbol "LegacyPluginSpec" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:236:3 - (ae-forgotten-export) The symbol "KibanaConfigType" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:236:3 - (ae-forgotten-export) The symbol "SharedGlobalConfigKeys" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:238:3 - (ae-forgotten-export) The symbol "PathConfigType" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:238:3 - (ae-forgotten-export) The symbol "KibanaConfigType" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:238:3 - (ae-forgotten-export) The symbol "SharedGlobalConfigKeys" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:240:3 - (ae-forgotten-export) The symbol "PathConfigType" needs to be exported by the entry point index.d.ts ``` From 1cb966b0566dab90952fd9950f684dae6df95080 Mon Sep 17 00:00:00 2001 From: spalger Date: Mon, 15 Jun 2020 13:55:47 -0700 Subject: [PATCH 30/31] tweak the readme text since it's not just for this PR --- packages/kbn-optimizer/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/kbn-optimizer/README.md b/packages/kbn-optimizer/README.md index cba4c5acf98b7..9ff0f56344274 100644 --- a/packages/kbn-optimizer/README.md +++ b/packages/kbn-optimizer/README.md @@ -32,13 +32,15 @@ When a bundle is determined to be up-to-date a worker is not started for the bun ## Bundle Refs -In order to dramatically reduce the size of our bundles, and the time it takes to build them, this PR makes the relationships between plugins/bundles explicit using "bundle refs". When the optimizer starts a list of "refs" that could be had is compiled from the list of bundles being built and sent to each worker. That list is used to determine which import statements in a bundle should be replaced with a runtime reference to the output of another bundle. +In order to dramatically reduce the size of our bundles, and the time it takes to build them, bundles will "ref" other bundles being built at the same time. When the optimizer starts it creates a list of "refs" that could be had from the list of bundles being built. Each worker uses that list to determine which import statements in a bundle should be replaced with a runtime reference to the output of another bundle. -At runtime the bundles now share a set of entry points. By default a plugin shares `public` so that other code can use relative imports to access that directory. To expose additional directories they must be listed in the plugin's kibana.json "extraPublicDirs" field. The directories listed there will **also** be exported from the plugins bundle so that any other plugin can import that directory. "common" is commonly in the list of "extraPublicDirs". +At runtime the bundles share a set of entry points via the `__kbnBundles__` global. By default a plugin shares `public` so that other code can use relative imports to access that directory. To expose additional directories they must be listed in the plugin's kibana.json "extraPublicDirs" field. The directories listed there will **also** be exported from the plugins bundle so that any other plugin can import that directory. "common" is commonly in the list of "extraPublicDirs". + +> NOTE: We plan to replace the `extraPublicDirs` functionality soon with better mechanisms for statically sharing code between bundles. When a directory is listed in the "extraPublicDirs" it will always be included in the bundle so that other plugins have access to it. The worker building the bundle has no way of knowing whether another plugin is using the directory, so be careful of adding test code or unnecessary directories to that list. -Any import in a bundle which resolves into another bundles "context" directory, ie `src/plugins/*`, must map explicitly to a "public dir" exported by that plugin. If the resolved import is not in the list of public dirs an error will be thrown and the optimizer will fail to build the bundles until the error is fixed. +Any import in a bundle which resolves into another bundles "context" directory, ie `src/plugins/*`, must map explicitly to a "public dir" exported by that plugin. If the resolved import is not in the list of public dirs an error will be thrown and the optimizer will fail to build that bundle until the error is fixed. ## API From 9ec9881e34f8cec3d1e0da8840cbc28f0746169d Mon Sep 17 00:00:00 2001 From: spalger Date: Mon, 15 Jun 2020 14:10:32 -0700 Subject: [PATCH 31/31] store __kbnBundles__ source in a separate module for legibility --- .eslintrc.js | 1 + .../ui/ui_render/bootstrap/app_bootstrap.js | 4 +- .../bootstrap/kbn_bundles_loader_source.js | 51 +++++++++++++++++++ .../ui/ui_render/bootstrap/template.js.hbs | 29 +---------- 4 files changed, 56 insertions(+), 29 deletions(-) create mode 100644 src/legacy/ui/ui_render/bootstrap/kbn_bundles_loader_source.js diff --git a/.eslintrc.js b/.eslintrc.js index 9657719f0f526..8d5b4525d51ba 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -472,6 +472,7 @@ module.exports = { { files: [ 'test/functional/services/lib/web_element_wrapper/scroll_into_view_if_necessary.js', + 'src/legacy/ui/ui_render/bootstrap/kbn_bundles_loader_source.js', '**/browser_exec_scripts/**/*.js', ], rules: { diff --git a/src/legacy/ui/ui_render/bootstrap/app_bootstrap.js b/src/legacy/ui/ui_render/bootstrap/app_bootstrap.js index 0e6936dd64a15..19f75317883d7 100644 --- a/src/legacy/ui/ui_render/bootstrap/app_bootstrap.js +++ b/src/legacy/ui/ui_render/bootstrap/app_bootstrap.js @@ -22,9 +22,11 @@ import { createHash } from 'crypto'; import { readFile } from 'fs'; import { resolve } from 'path'; +import { kbnBundlesLoaderSource } from './kbn_bundles_loader_source'; + export class AppBootstrap { constructor({ templateData }) { - this.templateData = templateData; + this.templateData = { ...templateData, kbnBundlesLoaderSource }; this._rawTemplate = undefined; } diff --git a/src/legacy/ui/ui_render/bootstrap/kbn_bundles_loader_source.js b/src/legacy/ui/ui_render/bootstrap/kbn_bundles_loader_source.js new file mode 100644 index 0000000000000..cb5488118cc77 --- /dev/null +++ b/src/legacy/ui/ui_render/bootstrap/kbn_bundles_loader_source.js @@ -0,0 +1,51 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +module.exports = { + kbnBundlesLoaderSource: `(${kbnBundlesLoader.toString()})();`, +}; + +function kbnBundlesLoader() { + var modules = {}; + + function has(prop) { + return Object.prototype.hasOwnProperty.call(modules, prop); + } + + function define(key, bundleRequire, bundleModuleKey) { + if (has(key)) { + throw new Error('__kbnBundles__ already has a module defined for "' + key + '"'); + } + + modules[key] = { + bundleRequire, + bundleModuleKey, + }; + } + + function get(key) { + if (!has(key)) { + throw new Error('__kbnBundles__ does not have a module defined for "' + key + '"'); + } + + return modules[key].bundleRequire(modules[key].bundleModuleKey); + } + + return { has: has, define: define, get: get }; +} diff --git a/src/legacy/ui/ui_render/bootstrap/template.js.hbs b/src/legacy/ui/ui_render/bootstrap/template.js.hbs index c15e296aceded..ca2e944489a73 100644 --- a/src/legacy/ui/ui_render/bootstrap/template.js.hbs +++ b/src/legacy/ui/ui_render/bootstrap/template.js.hbs @@ -3,34 +3,7 @@ window.__kbnStrictCsp__ = kbnCsp.strictCsp; window.__kbnDarkMode__ = {{darkMode}}; window.__kbnThemeVersion__ = "{{themeVersion}}"; window.__kbnPublicPath__ = {{publicPathMap}}; -window.__kbnBundles__ = (function () { - var modules = {}; - - function has(prop) { - return Object.prototype.hasOwnProperty.call(modules, prop); - } - - function define(key, bundleRequire, bundleModuleKey) { - if (has(key)) { - throw new Error('__kbnBundles__ already has a module defined for "' + key + '"'); - } - - modules[key] = { - bundleRequire, - bundleModuleKey - }; - } - - function get(key) { - if (!has(key)) { - throw new Error('__kbnBundles__ does not have a module defined for "' + key + '"'); - } - - return modules[key].bundleRequire(modules[key].bundleModuleKey); - } - - return { has: has, define: define, get: get }; -}()); +window.__kbnBundles__ = {{kbnBundlesLoaderSource}} if (window.__kbnStrictCsp__ && window.__kbnCspNotEnforced__) { var legacyBrowserError = document.getElementById('kbn_legacy_browser_error');