Merge remote-tracking branch 'upstream/master' into ingest/flatten-ag…

…ent-config

.ci/end2end.groovy

@@ -76,7 +76,7 @@ pipeline {
         }
       }
       steps{
-        notifyStatus('Running smoke tests', 'PENDING')
+        notifyTestStatus('Running smoke tests', 'PENDING')
         dir("${BASE_DIR}"){
           sh "${E2E_DIR}/ci/run-e2e.sh"
         }
@@ -95,10 +95,10 @@ pipeline {
           }
         }
         unsuccessful {
-          notifyStatus('Test failures', 'FAILURE')
+          notifyTestStatus('Test failures', 'FAILURE')
         }
         success {
-          notifyStatus('Tests passed', 'SUCCESS')
+          notifyTestStatus('Tests passed', 'SUCCESS')
         }
       }
     }
@@ -113,5 +113,9 @@ pipeline {
 }
 
 def notifyStatus(String description, String status) {
-  withGithubNotify.notify('end2end-for-apm-ui', description, status, getBlueoceanDisplayURL())
+  withGithubNotify.notify('end2end-for-apm-ui', description, status, getBlueoceanTabURL('pipeline'))
+}
+
+def notifyTestStatus(String description, String status) {
+  withGithubNotify.notify('end2end-for-apm-ui', description, status, getBlueoceanTabURL('tests'))
 }

.eslintrc.js

@@ -472,6 +472,7 @@ module.exports = {
     {
       files: [
         'test/functional/services/lib/web_element_wrapper/scroll_into_view_if_necessary.js',
+        'src/legacy/ui/ui_render/bootstrap/kbn_bundles_loader_source.js',
         '**/browser_exec_scripts/**/*.js',
       ],
       rules: {

.github/CODEOWNERS

@@ -66,11 +66,10 @@
 
 # APM
 /x-pack/plugins/apm/  @elastic/apm-ui
-/x-pack/plugins/apm/  @elastic/apm-ui
 /x-pack/test/functional/apps/apm/  @elastic/apm-ui
 /src/legacy/core_plugins/apm_oss/  @elastic/apm-ui
 /src/plugins/apm_oss/  @elastic/apm-ui
-/src/apm.js @watson
+/src/apm.js @watson @vigneshshanmugam
 
 # Beats
 /x-pack/legacy/plugins/beats_management/  @elastic/beats

NOTICE.txt

@@ -21,6 +21,11 @@ used. Logarithmic ticks are places at powers of ten and at half those
 values if there are not to many ticks already (e.g. [1, 5, 10, 50, 100]).
 For details,  see https://github.com/flot/flot/pull/1328
 
+---
+This module was heavily inspired by the externals plugin that ships with webpack@97d58d31
+MIT License http://www.opensource.org/licenses/mit-license.php
+Author Tobias Koppers @sokra
+
 ---
 This product has relied on ASTExplorer that is licensed under MIT.
 

docs/development/core/server/kibana-plugin-core-server.pluginmanifest.extrapublicdirs.md

@@ -0,0 +1,17 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [PluginManifest](./kibana-plugin-core-server.pluginmanifest.md) &gt; [extraPublicDirs](./kibana-plugin-core-server.pluginmanifest.extrapublicdirs.md)
+
+## PluginManifest.extraPublicDirs property
+
+> Warning: This API is now obsolete.
+> 
+> 
+
+Specifies directory names that can be imported by other ui-plugins built using the same instance of the @<!-- -->kbn/optimizer. A temporary measure we plan to replace with better mechanisms for sharing static code between plugins
+
+<b>Signature:</b>
+
+```typescript
+readonly extraPublicDirs?: string[];
+```

docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md

@@ -21,6 +21,7 @@ Should never be used in code outside of Core but is exported for documentation p
 |  Property | Type | Description |
 |  --- | --- | --- |
 |  [configPath](./kibana-plugin-core-server.pluginmanifest.configpath.md) | <code>ConfigPath</code> | Root [configuration path](./kibana-plugin-core-server.configpath.md) used by the plugin, defaults to "id" in snake\_case format. |
+|  [extraPublicDirs](./kibana-plugin-core-server.pluginmanifest.extrapublicdirs.md) | <code>string[]</code> | Specifies directory names that can be imported by other ui-plugins built using the same instance of the @<!-- -->kbn/optimizer. A temporary measure we plan to replace with better mechanisms for sharing static code between plugins |
 |  [id](./kibana-plugin-core-server.pluginmanifest.id.md) | <code>PluginName</code> | Identifier of the plugin. Must be a string in camelCase. Part of a plugin public contract. Other plugins leverage it to access plugin API, navigate to the plugin, etc. |
 |  [kibanaVersion](./kibana-plugin-core-server.pluginmanifest.kibanaversion.md) | <code>string</code> | The version of Kibana the plugin is compatible with, defaults to "version". |
 |  [optionalPlugins](./kibana-plugin-core-server.pluginmanifest.optionalplugins.md) | <code>readonly PluginName[]</code> | An optional list of the other plugins that if installed and enabled \*\*may be\*\* leveraged by this plugin for some additional functionality but otherwise are not required for this plugin to work properly. |

docs/user/ml/index.asciidoc

@@ -87,12 +87,14 @@ and {ml-docs}/xpack-ml.html[{ml-cap} {anomaly-detect}].
 [[xpack-ml-dfanalytics]]
 == {dfanalytics-cap}
 
+experimental[]
+
 The Elastic {ml} {dfanalytics} feature enables you to analyze your data using
-{oldetection} and {regression} algorithms and generate new indices that contain
-the results alongside your source data. 
+{classification}, {oldetection}, and {regression} algorithms and generate new
+indices that contain the results alongside your source data. 
 
 If you have a license that includes the {ml-features}, you can create
-{oldetection} {dfanalytics-jobs} and view their results on the *Analytics* page
+{dfanalytics-jobs} and view their results on the *Analytics* page
 in {kib}. For example:
 
 [role="screenshot"]

docs/visualize/vega.asciidoc

@@ -32,7 +32,7 @@ image::images/vega_lite_default.png[]
 The default visualization uses Vega-Lite version 2. To use Vega version 4, edit
 the `schema`.
 
-Go to `$schema`, enter `https://vega.github.io/schema/vega/v4.json`, then click
+Go to `$schema`, enter `https://vega.github.io/schema/vega/v5.json`, then click
 *Update*.
 
 [float]

examples/demo_search/kibana.json

@@ -5,5 +5,6 @@
   "server": true,
   "ui": true,
   "requiredPlugins": ["data"],
-  "optionalPlugins": []
+  "optionalPlugins": [],
+  "extraPublicDirs": ["common"]
 }

examples/embeddable_examples/kibana.json

@@ -5,5 +5,6 @@
   "server": true,
   "ui": true,
   "requiredPlugins": ["embeddable"],
-  "optionalPlugins": []
+  "optionalPlugins": [],
+  "extraPublicDirs": ["public/todo", "public/hello_world"]
 }

examples/url_generators_examples/kibana.json

@@ -5,5 +5,8 @@
   "server": false,
   "ui": true,
   "requiredPlugins": ["share"],
-  "optionalPlugins": []
+  "optionalPlugins": [],
+  "extraPublicDirs": [
+    "public/url_generator"
+  ]
 }

package.json

@@ -122,7 +122,7 @@
     "@babel/core": "^7.10.2",
     "@babel/plugin-transform-modules-commonjs": "^7.10.1",
     "@babel/register": "^7.10.1",
-    "@elastic/apm-rum": "^5.1.1",
+    "@elastic/apm-rum": "^5.2.0",
     "@elastic/charts": "19.2.0",
     "@elastic/datemath": "5.0.3",
     "@elastic/ems-client": "7.9.3",
@@ -243,7 +243,7 @@
     "react-input-range": "^1.3.0",
     "react-markdown": "^4.3.1",
     "react-monaco-editor": "~0.27.0",
-    "react-redux": "^7.1.3",
+    "react-redux": "^7.2.0",
     "react-resize-detector": "^4.2.0",
     "react-router": "^5.1.2",
     "react-router-dom": "^5.1.2",
@@ -280,9 +280,9 @@
     "uuid": "3.3.2",
     "val-loader": "^1.1.1",
     "validate-npm-package-name": "2.2.2",
-    "vega-lib": "4.3.0",
-    "vega-lite": "^2.6.0",
-    "vega-schema-url-parser": "1.0.0",
+    "vega": "^5.13.0",
+    "vega-lite": "^4.13.1",
+    "vega-schema-url-parser": "^1.1.0",
     "vega-tooltip": "^0.12.0",
     "vision": "^5.3.3",
     "webpack": "^4.41.5",
@@ -375,7 +375,7 @@
     "@types/react": "^16.9.36",
     "@types/react-dom": "^16.9.8",
     "@types/react-grid-layout": "^0.16.7",
-    "@types/react-redux": "^7.1.7",
+    "@types/react-redux": "^7.1.9",
     "@types/react-resize-detector": "^4.0.1",
     "@types/react-router": "^5.1.3",
     "@types/react-router-dom": "^5.1.3",

packages/kbn-es/src/utils/native_realm.js

@@ -37,13 +37,8 @@ exports.NativeRealm = class NativeRealm {
     this._log = log;
   }
 
-  async setPassword(username, password = this._elasticPassword, { attempt = 1 } = {}) {
-    await this._autoRetry(async () => {
-      this._log.info(
-        (attempt > 1 ? `attempt ${attempt}: ` : '') +
-          `setting ${chalk.bold(username)} password to ${chalk.bold(password)}`
-      );
-
+  async setPassword(username, password = this._elasticPassword, retryOpts = {}) {
+    await this._autoRetry(retryOpts, async () => {
       try {
         await this._client.security.changePassword({
           username,
@@ -83,8 +78,8 @@ exports.NativeRealm = class NativeRealm {
     );
   }
 
-  async getReservedUsers() {
-    return await this._autoRetry(async () => {
+  async getReservedUsers(retryOpts = {}) {
+    return await this._autoRetry(retryOpts, async () => {
       const resp = await this._client.security.getUser();
       const usernames = Object.keys(resp.body).filter(
         (user) => resp.body[user].metadata._reserved === true
@@ -98,9 +93,9 @@ exports.NativeRealm = class NativeRealm {
     });
   }
 
-  async isSecurityEnabled() {
+  async isSecurityEnabled(retryOpts = {}) {
     try {
-      return await this._autoRetry(async () => {
+      return await this._autoRetry(retryOpts, async () => {
         const {
           body: { features },
         } = await this._client.xpack.info({ categories: 'features' });
@@ -115,18 +110,25 @@ exports.NativeRealm = class NativeRealm {
     }
   }
 
-  async _autoRetry(fn, attempt = 1) {
+  async _autoRetry(opts, fn) {
+    const { attempt = 1, maxAttempts = 3 } = opts;
+
     try {
       return await fn(attempt);
     } catch (error) {
-      if (attempt >= 3) {
+      if (attempt >= maxAttempts) {
         throw error;
       }
 
       const sec = 1.5 * attempt;
       this._log.warning(`assuming ES isn't initialized completely, trying again in ${sec} seconds`);
       await new Promise((resolve) => setTimeout(resolve, sec * 1000));
-      return await this._autoRetry(fn, attempt + 1);
+
+      const nextOpts = {
+        ...opts,
+        attempt: attempt + 1,
+      };
+      return await this._autoRetry(nextOpts, fn);
     }
   }
 };

packages/kbn-es/src/utils/native_realm.test.js

@@ -85,7 +85,7 @@ describe('isSecurityEnabled', () => {
       throw error;
     });
 
-    expect(await nativeRealm.isSecurityEnabled()).toBe(false);
+    expect(await nativeRealm.isSecurityEnabled({ maxAttempts: 1 })).toBe(false);
   });
 
   test('rejects if unexpected error is thrown', async () => {
@@ -97,9 +97,9 @@ describe('isSecurityEnabled', () => {
       throw error;
     });
 
-    await expect(nativeRealm.isSecurityEnabled()).rejects.toThrowErrorMatchingInlineSnapshot(
-      `"ResponseError"`
-    );
+    await expect(
+      nativeRealm.isSecurityEnabled({ maxAttempts: 1 })
+    ).rejects.toThrowErrorMatchingInlineSnapshot(`"ResponseError"`);
   });
 });
 
@@ -226,7 +226,7 @@ describe('setPassword', () => {
     });
 
     await expect(
-      nativeRealm.setPassword('kibana_system', 'foo')
+      nativeRealm.setPassword('kibana_system', 'foo', { maxAttempts: 1 })
     ).rejects.toThrowErrorMatchingInlineSnapshot(`"SomeError"`);
   });
 });

packages/kbn-optimizer/README.md

@@ -30,6 +30,18 @@ Bundles built by the the optimizer include a cache file which describes the info
 
 When a bundle is determined to be up-to-date a worker is not started for the bundle. If running the optimizer with the `--dev/--watch` flag, then all the files referenced by cached bundles are watched for changes. Once a change is detected in any of the files referenced by the built bundle a worker is started. If a file is changed that is referenced by several bundles then workers will be started for each bundle, combining workers together to respect the worker limit.
 
+## Bundle Refs
+
+In order to dramatically reduce the size of our bundles, and the time it takes to build them, bundles will "ref" other bundles being built at the same time. When the optimizer starts it creates a list of "refs" that could be had from the list of bundles being built. Each worker uses that list to determine which import statements in a bundle should be replaced with a runtime reference to the output of another bundle.
+
+At runtime the bundles share a set of entry points via the `__kbnBundles__` global. By default a plugin shares `public` so that other code can use relative imports to access that directory. To expose additional directories they must be listed in the plugin's kibana.json "extraPublicDirs" field. The directories listed there will **also** be exported from the plugins bundle so that any other plugin can import that directory. "common" is commonly in the list of "extraPublicDirs".
+
+> NOTE: We plan to replace the `extraPublicDirs` functionality soon with better mechanisms for statically sharing code between bundles.
+
+When a directory is listed in the "extraPublicDirs" it will always be included in the bundle so that other plugins have access to it. The worker building the bundle has no way of knowing whether another plugin is using the directory, so be careful of adding test code or unnecessary directories to that list.
+
+Any import in a bundle which resolves into another bundles "context" directory, ie `src/plugins/*`, must map explicitly to a "public dir" exported by that plugin. If the resolved import is not in the list of public dirs an error will be thrown and the optimizer will fail to build that bundle until the error is fixed.
+
 ## API
 
 To run the optimizer from code, you can import the [`OptimizerConfig`][OptimizerConfig] class and [`runOptimizer`][Optimizer] function. Create an [`OptimizerConfig`][OptimizerConfig] instance by calling it's static `create()` method with some options, then pass it to the [`runOptimizer`][Optimizer] function. `runOptimizer()` returns an observable of update objects, which are summaries of the optimizer state plus an optional `event` property which describes the internal events occuring and may be of use. You can use the [`logOptimizerState()`][LogOptimizerState] helper to write the relevant bits of state to a tooling log or checkout it's implementation to see how the internal events like [`WorkerStdio`][ObserveWorker] and [`WorkerStarted`][ObserveWorker] are used.

packages/kbn-optimizer/package.json

@@ -45,6 +45,7 @@
     "terser-webpack-plugin": "^2.1.2",
     "tinymath": "1.2.1",
     "url-loader": "^2.2.0",
+    "val-loader": "^1.1.1",
     "watchpack": "^1.6.0",
     "webpack": "^4.41.5",
     "webpack-merge": "^4.2.2"

packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/index.ts

@@ -19,6 +19,6 @@
 
 import './legacy/styles.scss';
 import './index.scss';
-import { fooLibFn } from '../../foo/public/index';
+import { fooLibFn } from '../../foo/public';
 export * from './lib';
 export { fooLibFn };

packages/kbn-optimizer/src/common/bundle.test.ts

@@ -23,7 +23,7 @@ jest.mock('fs');
 
 const SPEC: BundleSpec = {
   contextDir: '/foo/bar',
-  entry: 'entry',
+  publicDirNames: ['public'],
   id: 'bar',
   outputDir: '/foo/bar/target',
   sourceRoot: '/foo',
@@ -49,9 +49,11 @@ it('creates cache keys', () => {
       },
       "spec": Object {
         "contextDir": "/foo/bar",
-        "entry": "entry",
         "id": "bar",
         "outputDir": "/foo/bar/target",
+        "publicDirNames": Array [
+          "public",
+        ],
         "sourceRoot": "/foo",
         "type": "plugin",
       },
@@ -82,9 +84,11 @@ it('parses bundles from JSON specs', () => {
           "state": undefined,
         },
         "contextDir": "/foo/bar",
-        "entry": "entry",
         "id": "bar",
         "outputDir": "/foo/bar/target",
+        "publicDirNames": Array [
+          "public",
+        ],
         "sourceRoot": "/foo",
         "type": "plugin",
       },