[kbn/optimizer] share all plugin bundles

examples/demo_search/kibana.json

@@ -5,5 +5,6 @@
   "server": true,
   "ui": true,
   "requiredPlugins": ["data"],
-  "optionalPlugins": []
+  "optionalPlugins": [],
+  "extraPublicDirs": ["common"]
 }

examples/embeddable_examples/kibana.json

@@ -5,5 +5,6 @@
   "server": true,
   "ui": true,
   "requiredPlugins": ["embeddable"],
-  "optionalPlugins": []
+  "optionalPlugins": [],
+  "extraPublicDirs": ["public/todo"]
 }

examples/url_generators_examples/kibana.json

@@ -5,5 +5,8 @@
   "server": false,
   "ui": true,
   "requiredPlugins": ["share"],
-  "optionalPlugins": []
+  "optionalPlugins": [],
+  "extraPublicDirs": [
+    "public/url_generator"
+  ]
 }

packages/kbn-optimizer/README.md

@@ -30,6 +30,16 @@ Bundles built by the the optimizer include a cache file which describes the info
 
 When a bundle is determined to be up-to-date a worker is not started for the bundle. If running the optimizer with the `--dev/--watch` flag, then all the files referenced by cached bundles are watched for changes. Once a change is detected in any of the files referenced by the built bundle a worker is started. If a file is changed that is referenced by several bundles then workers will be started for each bundle, combining workers together to respect the worker limit.
 
+## Bundle Refs
+
+In order to dramatically reduce the size of our bundles, and the time it takes to build them, this PR makes the relationships between plugins/bundles explicit using "bundle refs". When the optimizer starts a list of "refs" that could be had is compiled from the list of bundles being built and sent to each worker. That list is used to determine which import statements in a bundle should be replaced with a runtime reference to the output of another bundle.
+
+At runtime the bundles now share a set of entry points. By default a plugin shares `public` so that other code can use relative imports to access that directory. To expose additional directories they must be listed in the plugin's kibana.json "extraPublicDirs" field. The directories listed there will **also** be exported from the plugins bundle so that any other plugin can import that directory. "common" is commonly in the list of "extraPublicDirs". 
+
+When a directory is listed in the "extraPublicDirs" it will always be included in the bundle so that other plugins have access to it. The worker building the bundle has no way of knowing whether another plugin is using the directory, so be careful of adding test code or unnecessary directories to that list.
+
+Any import in a bundle which resolves into another bundles "context" directory, ie `src/plugins/*`, must map explicitly to a "public dir" exported by that plugin. If the resolved import is not in the list of public dirs an error will be thrown and the optimizer will fail to build the bundles until the error is fixed.
+
 ## API
 
 To run the optimizer from code, you can import the [`OptimizerConfig`][OptimizerConfig] class and [`runOptimizer`][Optimizer] function. Create an [`OptimizerConfig`][OptimizerConfig] instance by calling it's static `create()` method with some options, then pass it to the [`runOptimizer`][Optimizer] function. `runOptimizer()` returns an observable of update objects, which are summaries of the optimizer state plus an optional `event` property which describes the internal events occuring and may be of use. You can use the [`logOptimizerState()`][LogOptimizerState] helper to write the relevant bits of state to a tooling log or checkout it's implementation to see how the internal events like [`WorkerStdio`][ObserveWorker] and [`WorkerStarted`][ObserveWorker] are used.

packages/kbn-optimizer/package.json

@@ -45,6 +45,7 @@
     "terser-webpack-plugin": "^2.1.2",
     "tinymath": "1.2.1",
     "url-loader": "^2.2.0",
+    "val-loader": "^1.1.1",
     "watchpack": "^1.6.0",
     "webpack": "^4.41.5",
     "webpack-merge": "^4.2.2"

packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/index.ts

@@ -19,6 +19,6 @@
 
 import './legacy/styles.scss';
 import './index.scss';
-import { fooLibFn } from '../../foo/public/index';
+import { fooLibFn } from '../../foo/public';
 export * from './lib';
 export { fooLibFn };

packages/kbn-optimizer/src/common/bundle.test.ts

@@ -23,7 +23,7 @@ jest.mock('fs');
 
 const SPEC: BundleSpec = {
   contextDir: '/foo/bar',
-  entry: 'entry',
+  publicDirNames: ['public'],
   id: 'bar',
   outputDir: '/foo/bar/target',
   sourceRoot: '/foo',
@@ -49,9 +49,11 @@ it('creates cache keys', () => {
       },
       "spec": Object {
         "contextDir": "/foo/bar",
-        "entry": "entry",
         "id": "bar",
         "outputDir": "/foo/bar/target",
+        "publicDirNames": Array [
+          "public",
+        ],
         "sourceRoot": "/foo",
         "type": "plugin",
       },
@@ -82,9 +84,11 @@ it('parses bundles from JSON specs', () => {
           "state": undefined,
         },
         "contextDir": "/foo/bar",
-        "entry": "entry",
         "id": "bar",
         "outputDir": "/foo/bar/target",
+        "publicDirNames": Array [
+          "public",
+        ],
         "sourceRoot": "/foo",
         "type": "plugin",
       },

packages/kbn-optimizer/src/common/bundle.ts

@@ -29,8 +29,8 @@ export interface BundleSpec {
   readonly type: typeof VALID_BUNDLE_TYPES[0];
   /** Unique id for this bundle */
   readonly id: string;
-  /** Webpack entry request for this plugin, relative to the contextDir */
-  readonly entry: string;
+  /** directory names relative to the contextDir that can be imported from */
+  readonly publicDirNames: string[];
   /** Absolute path to the plugin source directory */
   readonly contextDir: string;
   /** Absolute path to the root of the repository */
@@ -44,8 +44,8 @@ export class Bundle {
   public readonly type: BundleSpec['type'];
   /** Unique identifier for this bundle */
   public readonly id: BundleSpec['id'];
-  /** Path, relative to `contextDir`, to the entry file for the Webpack bundle */
-  public readonly entry: BundleSpec['entry'];
+  /** directory names relative to the contextDir that can be imported from */
+  public readonly publicDirNames: BundleSpec['publicDirNames'];
   /**
    * Absolute path to the root of the bundle context (plugin directory)
    * where the entry is resolved relative to and the default output paths
@@ -62,7 +62,7 @@ export class Bundle {
   constructor(spec: BundleSpec) {
     this.type = spec.type;
     this.id = spec.id;
-    this.entry = spec.entry;
+    this.publicDirNames = spec.publicDirNames;
     this.contextDir = spec.contextDir;
     this.sourceRoot = spec.sourceRoot;
     this.outputDir = spec.outputDir;
@@ -73,8 +73,6 @@ export class Bundle {
   /**
    * Calculate the cache key for this bundle based from current
    * mtime values.
-   *
-   * @param mtimes pre-fetched mtimes (ms || undefined) for all referenced files
    */
   createCacheKey(files: string[], mtimes: Map<string, number | undefined>): unknown {
     return {
@@ -94,7 +92,7 @@ export class Bundle {
     return {
       type: this.type,
       id: this.id,
-      entry: this.entry,
+      publicDirNames: this.publicDirNames,
       contextDir: this.contextDir,
       sourceRoot: this.sourceRoot,
       outputDir: this.outputDir,
@@ -134,9 +132,9 @@ export function parseBundles(json: string) {
           throw new Error('`bundles[]` must have a string `id` property');
         }
 
-        const { entry } = spec;
-        if (!(typeof entry === 'string')) {
-          throw new Error('`bundles[]` must have a string `entry` property');
+        const { publicDirNames } = spec;
+        if (!Array.isArray(publicDirNames) || !publicDirNames.every((d) => typeof d === 'string')) {
+          throw new Error('`bundles[]` must have an array of strings `publicDirNames` property');
         }
 
         const { contextDir } = spec;
@@ -157,7 +155,7 @@ export function parseBundles(json: string) {
         return new Bundle({
           type,
           id,
-          entry,
+          publicDirNames,
           contextDir,
           sourceRoot,
           outputDir,

packages/kbn-optimizer/src/common/bundle_cache.ts

@@ -25,6 +25,7 @@ export interface State {
   cacheKey?: unknown;
   moduleCount?: number;
   files?: string[];
+  bundleRefExportIds?: string[];
 }
 
 const DEFAULT_STATE: State = {};
@@ -87,6 +88,10 @@ export class BundleCache {
     return this.get().files;
   }
 
+  public getBundleRefExportIds() {
+    return this.get().bundleRefExportIds;
+  }
+
   public getCacheKey() {
     return this.get().cacheKey;
   }

packages/kbn-optimizer/src/common/bundle_refs.ts

@@ -0,0 +1,137 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Path from 'path';
+
+import { Bundle } from './bundle';
+import { UnknownVals } from './ts_helpers';
+
+export interface BundleRef {
+  id: string;
+  contextDir: string;
+  contextPrefix: string;
+  entry: string;
+  exportId: string;
+}
+
+export class BundleRefs {
+  static fromBundles(bundles: Bundle[]) {
+    return new BundleRefs(
+      bundles.reduce(
+        (acc: BundleRef[], b) => [
+          ...acc,
+          ...b.publicDirNames.map(
+            (name): BundleRef => ({
+              id: b.id,
+              contextDir: b.contextDir,
+              // Path.resolve converts separators and strips the final separator
+              contextPrefix: Path.resolve(b.contextDir) + Path.sep,
+              entry: name,
+              exportId: `${b.type}/${b.id}/${name}`,
+            })
+          ),
+        ],
+        []
+      )
+    );
+  }
+
+  static parseSpec(json: unknown) {
+    if (typeof json !== 'string') {
+      throw new Error('expected `bundleRefs` spec to be a JSON string');
+    }
+
+    let spec;
+    try {
+      spec = JSON.parse(json);
+    } catch (error) {
+      throw new Error('`bundleRefs` spec must be valid JSON');
+    }
+
+    if (!Array.isArray(spec)) {
+      throw new Error('`bundleRefs` spec must be an array');
+    }
+
+    return new BundleRefs(
+      spec.map(
+        (refSpec: UnknownVals<BundleRef>): BundleRef => {
+          if (typeof refSpec !== 'object' || !refSpec) {
+            throw new Error('`bundleRefs[]` must be an object');
+          }
+
+          const { id } = refSpec;
+          if (typeof id !== 'string') {
+            throw new Error('`bundleRefs[].id` must be a string');
+          }
+
+          const { contextDir } = refSpec;
+          if (typeof contextDir !== 'string' || !Path.isAbsolute(contextDir)) {
+            throw new Error('`bundleRefs[].contextDir` must be an absolute directory');
+          }
+
+          const { contextPrefix } = refSpec;
+          if (typeof contextPrefix !== 'string' || !Path.isAbsolute(contextPrefix)) {
+            throw new Error('`bundleRefs[].contextPrefix` must be an absolute directory');
+          }
+
+          const { entry } = refSpec;
+          if (typeof entry !== 'string') {
+            throw new Error('`bundleRefs[].entry` must be a string');
+          }
+
+          const { exportId } = refSpec;
+          if (typeof exportId !== 'string') {
+            throw new Error('`bundleRefs[].exportId` must be a string');
+          }
+
+          return {
+            contextDir,
+            contextPrefix,
+            entry,
+            exportId,
+            id,
+          };
+        }
+      )
+    );
+  }
+
+  constructor(private readonly refs: BundleRef[]) {}
+
+  public filterByExportIds(exportIds: string[]) {
+    const refs: BundleRef[] = [];
+    for (const exportId of exportIds) {
+      const ref = this.refs.find((r) => r.exportId === exportId);
+      if (ref) {
+        refs.push(ref);
+      }
+    }
+    return refs;
+  }
+
+  public filterByContextPrefix(bundle: Bundle, absolutePath: string) {
+    return this.refs.filter(
+      (ref) => ref.id !== bundle.id && absolutePath.startsWith(ref.contextPrefix)
+    );
+  }
+
+  public toSpecJson() {
+    return JSON.stringify(this.refs);
+  }
+}

packages/kbn-optimizer/src/common/index.ts

@@ -19,6 +19,7 @@
 
 export * from './bundle';
 export * from './bundle_cache';
+export * from './bundle_refs';
 export * from './worker_config';
 export * from './worker_messages';
 export * from './compiler_messages';

packages/kbn-optimizer/src/integration_tests/basic_optimization.test.ts

@@ -132,13 +132,14 @@ it('builds expected bundles, saves bundle counts to metadata', async () => {
   const foo = config.bundles.find((b) => b.id === 'foo')!;
   expect(foo).toBeTruthy();
   foo.cache.refresh();
-  expect(foo.cache.getModuleCount()).toBe(5);
+  expect(foo.cache.getModuleCount()).toBe(6);
   expect(foo.cache.getReferencedFiles()).toMatchInlineSnapshot(`
     Array [
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/async_import.ts,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/ext.ts,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/index.ts,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/lib.ts,
+      <absolute path>/packages/kbn-optimizer/target/worker/entry_point_creator.js,
       <absolute path>/packages/kbn-ui-shared-deps/public_path_module_creator.js,
     ]
   `);
@@ -148,7 +149,7 @@ it('builds expected bundles, saves bundle counts to metadata', async () => {
   bar.cache.refresh();
   expect(bar.cache.getModuleCount()).toBe(
     // code + styles + style/css-loader runtimes + public path updater
-    21
+    18
   );
 
   expect(bar.cache.getReferencedFiles()).toMatchInlineSnapshot(`
@@ -159,11 +160,8 @@ it('builds expected bundles, saves bundle counts to metadata', async () => {
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/index.ts,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/legacy/styles.scss,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/lib.ts,
-      <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/async_import.ts,
-      <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/ext.ts,
-      <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/index.ts,
-      <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/lib.ts,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/src/legacy/ui/public/icon.svg,
+      <absolute path>/packages/kbn-optimizer/target/worker/entry_point_creator.js,
       <absolute path>/packages/kbn-ui-shared-deps/public_path_module_creator.js,
     ]
   `);

packages/kbn-optimizer/src/integration_tests/bundle_cache.test.ts

@@ -85,7 +85,8 @@ it('emits "bundle cached" event when everything is updated', async () => {
     Array [
       Object {
         "bundle": <Bundle>,
-        "type": "bundle cached",
+        "reason": "bundle references missing",
+        "type": "bundle not cached",
       },
     ]
   `);
@@ -288,12 +289,7 @@ it('emits "bundle not cached" event when cacheKey is outdated', async () => {
     Array [
       Object {
         "bundle": <Bundle>,
-        "diff": "- Expected
-    + Received
-
-    - \\"old\\"
-    + \\"new\\"",
-        "reason": "cache key mismatch",
+        "reason": "bundle references missing",
         "type": "bundle not cached",
       },
     ]