Merge branch 'app/painless' of github.com:elastic/kibana into app/pai…

…nless

* 'app/painless' of github.com:elastic/kibana: (64 commits)
  Fix filter scope in bool query (#60488)
  change index pattern id to be the same as index pattern title (#60436)
  [Endpoint] resolver v1 events (#59233)
  Branding fixes for dashboard, loader and space selector (#60073)
  skip flaky suite (#60535)
  [SIEM][Detection Engine] Fixes bug with timeline templates not working
  Fixed errors which are happening if switch between alert types (#60453)
  [EPM] Add mapping field types to index template generation v2 (#60266)
  [NP] Cutover ensureDefaultIndexPattern to kibana_utils (#59895)
  Closes #60265. Adds Beta badge to service map (#60482)
  [Visualize] Duplicated query filters in es request (#60106)
  [ML] Disable functional transform tests
  Fixes to service map single node banner (#60072)
  [Uptime] replace fetch with kibana http (#59881)
  Upgrade @types/node to match Node.js runtime (#60368)
  [License Management] NP migration (#60250)
  Fix create alert button from not showing in alerts list (#60444)
  [SIEM][Case] Update connector through flyout (#60307)
  add data-test-subj where possible on SO management table (#60226)
  Enforce `required` presence for value/key validation of `recordOf` and `mapOf`. (#60406)
  ...

.github/CODEOWNERS

@@ -29,33 +29,32 @@
 /src/plugins/dashboard/ @elastic/kibana-app
 
 # App Architecture
+/examples/url_generators_examples/ @elastic/kibana-app-arch
+/examples/url_generators_explorer/ @elastic/kibana-app-arch
 /packages/kbn-interpreter/ @elastic/kibana-app-arch
-/src/legacy/core_plugins/data/ @elastic/kibana-app-arch
-/src/legacy/core_plugins/elasticsearch/lib/create_proxy.js @elastic/kibana-app-arch
 /src/legacy/core_plugins/embeddable_api/ @elastic/kibana-app-arch
 /src/legacy/core_plugins/interpreter/ @elastic/kibana-app-arch
 /src/legacy/core_plugins/kibana_react/ @elastic/kibana-app-arch
 /src/legacy/core_plugins/kibana/public/management/ @elastic/kibana-app-arch
-/src/legacy/core_plugins/kibana/server/field_formats/ @elastic/kibana-app-arch
 /src/legacy/core_plugins/kibana/server/routes/api/management/ @elastic/kibana-app-arch
-/src/legacy/core_plugins/kibana/server/routes/api/suggestions/ @elastic/kibana-app-arch
 /src/legacy/core_plugins/visualizations/ @elastic/kibana-app-arch
 /src/legacy/server/index_patterns/ @elastic/kibana-app-arch
+/src/plugins/advanced_settings/ @elastic/kibana-app-arch
 /src/plugins/bfetch/ @elastic/kibana-app-arch
 /src/plugins/data/ @elastic/kibana-app-arch
 /src/plugins/embeddable/ @elastic/kibana-app-arch
 /src/plugins/expressions/ @elastic/kibana-app-arch
 /src/plugins/inspector/ @elastic/kibana-app-arch
 /src/plugins/kibana_react/ @elastic/kibana-app-arch
+/src/plugins/kibana_react/public/code_editor @elastic/kibana-canvas
 /src/plugins/kibana_utils/ @elastic/kibana-app-arch
 /src/plugins/management/ @elastic/kibana-app-arch
 /src/plugins/navigation/ @elastic/kibana-app-arch
+/src/plugins/share/ @elastic/kibana-app-arch
 /src/plugins/ui_actions/ @elastic/kibana-app-arch
 /src/plugins/visualizations/ @elastic/kibana-app-arch
-/src/plugins/share/ @elastic/kibana-app-arch
-/examples/url_generators_examples/ @elastic/kibana-app-arch
-/examples/url_generators_explorer/ @elastic/kibana-app-arch
 /x-pack/plugins/advanced_ui_actions/ @elastic/kibana-app-arch
+/x-pack/plugins/data_enhanced/ @elastic/kibana-app-arch
 /x-pack/plugins/drilldowns/ @elastic/kibana-app-arch
 
 # APM
@@ -75,9 +74,9 @@
 # Observability UIs
 /x-pack/legacy/plugins/infra/ @elastic/logs-metrics-ui
 /x-pack/plugins/infra/ @elastic/logs-metrics-ui
-/x-pack/plugins/ingest_manager/ @elastic/ingest
-/x-pack/legacy/plugins/ingest_manager/ @elastic/ingest
-/x-pack/plugins/observability/ @elastic/logs-metrics-ui @elastic/apm-ui @elastic/uptime @elastic/ingest
+/x-pack/plugins/ingest_manager/ @elastic/ingest-management
+/x-pack/legacy/plugins/ingest_manager/ @elastic/ingest-management
+/x-pack/plugins/observability/ @elastic/logs-metrics-ui @elastic/apm-ui @elastic/uptime @elastic/ingest-management
 /x-pack/legacy/plugins/monitoring/ @elastic/stack-monitoring-ui
 
 # Machine Learning

docs/developer/core/development-functional-tests.asciidoc

@@ -178,6 +178,25 @@ To run tests on Firefox locally, use `config.firefox.js`:
 node scripts/functional_test_runner --config test/functional/config.firefox.js
 -----------
 
+[float]
+===== Using the test_user service
+
+Tests should run at the positive security boundry condition, meaning that they should be run with the mimimum privileges required (and documented) and not as the superuser.
+ This prevents the type of regression where additional privleges accidentally become required to perform the same action. 
+
+The functional UI tests now default to logging in with a user named `test_user` and the roles of this user can be changed dynamically without logging in and out.  
+
+In order to achieve this a new service was introduced called `createTestUserService` (see `test/common/services/security/test_user.ts`). The purpose of this test user service is to create roles defined in the test config files and setRoles() or restoreDefaults().
+
+An example of how to set the role like how its defined below:
+
+`await security.testUser.setRoles(['kibana_user', 'kibana_date_nanos']);`
+
+Here we are setting the `test_user` to have the `kibana_user` role and also role access to a specific data index (`kibana_date_nanos`).
+
+Tests should normally setRoles() in the before() and restoreDefaults() in the after().
+
+
 [float]
 ===== Anatomy of a test file
 

docs/development/core/public/kibana-plugin-core-public.chromestart.getiscollapsed_.md → docs/development/core/public/kibana-plugin-core-public.chromestart.getisnavdrawerlocked_.md

@@ -1,15 +1,15 @@
 <!-- Do not edit this file. It is automatically generated by API Documenter. -->
 
-[Home](./index.md) &gt; [kibana-plugin-core-public](./kibana-plugin-core-public.md) &gt; [ChromeStart](./kibana-plugin-core-public.chromestart.md) &gt; [getIsCollapsed$](./kibana-plugin-core-public.chromestart.getiscollapsed_.md)
+[Home](./index.md) &gt; [kibana-plugin-core-public](./kibana-plugin-core-public.md) &gt; [ChromeStart](./kibana-plugin-core-public.chromestart.md) &gt; [getIsNavDrawerLocked$](./kibana-plugin-core-public.chromestart.getisnavdrawerlocked_.md)
 
-## ChromeStart.getIsCollapsed$() method
+## ChromeStart.getIsNavDrawerLocked$() method
 
-Get an observable of the current collapsed state of the chrome.
+Get an observable of the current locked state of the nav drawer.
 
 <b>Signature:</b>
 
 ```typescript
-getIsCollapsed$(): Observable<boolean>;
+getIsNavDrawerLocked$(): Observable<boolean>;
 ```
 <b>Returns:</b>
 

docs/development/core/public/kibana-plugin-core-public.chromestart.md

@@ -56,7 +56,7 @@ core.chrome.setHelpExtension(elem => {
 |  [getBrand$()](./kibana-plugin-core-public.chromestart.getbrand_.md) | Get an observable of the current brand information. |
 |  [getBreadcrumbs$()](./kibana-plugin-core-public.chromestart.getbreadcrumbs_.md) | Get an observable of the current list of breadcrumbs |
 |  [getHelpExtension$()](./kibana-plugin-core-public.chromestart.gethelpextension_.md) | Get an observable of the current custom help conttent |
-|  [getIsCollapsed$()](./kibana-plugin-core-public.chromestart.getiscollapsed_.md) | Get an observable of the current collapsed state of the chrome. |
+|  [getIsNavDrawerLocked$()](./kibana-plugin-core-public.chromestart.getisnavdrawerlocked_.md) | Get an observable of the current locked state of the nav drawer. |
 |  [getIsVisible$()](./kibana-plugin-core-public.chromestart.getisvisible_.md) | Get an observable of the current visibility state of the chrome. |
 |  [removeApplicationClass(className)](./kibana-plugin-core-public.chromestart.removeapplicationclass.md) | Remove a className added with <code>addApplicationClass()</code>. If className is unknown it is ignored. |
 |  [setAppTitle(appTitle)](./kibana-plugin-core-public.chromestart.setapptitle.md) | Sets the current app's title |
@@ -65,6 +65,5 @@ core.chrome.setHelpExtension(elem => {
 |  [setBreadcrumbs(newBreadcrumbs)](./kibana-plugin-core-public.chromestart.setbreadcrumbs.md) | Override the current set of breadcrumbs |
 |  [setHelpExtension(helpExtension)](./kibana-plugin-core-public.chromestart.sethelpextension.md) | Override the current set of custom help content |
 |  [setHelpSupportUrl(url)](./kibana-plugin-core-public.chromestart.sethelpsupporturl.md) | Override the default support URL shown in the help menu |
-|  [setIsCollapsed(isCollapsed)](./kibana-plugin-core-public.chromestart.setiscollapsed.md) | Set the collapsed state of the chrome navigation. |
 |  [setIsVisible(isVisible)](./kibana-plugin-core-public.chromestart.setisvisible.md) | Set the temporary visibility for the chrome. This does nothing if the chrome is hidden by default and should be used to hide the chrome for things like full-screen modes with an exit button. |
 

docs/development/core/server/kibana-plugin-core-server.routeconfig.validate.md

@@ -14,7 +14,7 @@ validate: RouteValidatorFullConfig<P, Q, B> | false;
 
 ## Remarks
 
-You \*must\* specify a validation schema to be able to read: - url path segments - request query - request body To opt out of validating the request, specify `validate: false`<!-- -->. In this case request params, query, and body will be \*\*empty\*\* objects and have no access to raw values. In some cases you may want to use another validation library. To do this, you need to instruct the `@kbn/config-schema` library to output \*\*non-validated values\*\* with setting schema as `schema.object({}, { allowUnknowns: true })`<!-- -->;
+You \*must\* specify a validation schema to be able to read: - url path segments - request query - request body To opt out of validating the request, specify `validate: false`<!-- -->. In this case request params, query, and body will be \*\*empty\*\* objects and have no access to raw values. In some cases you may want to use another validation library. To do this, you need to instruct the `@kbn/config-schema` library to output \*\*non-validated values\*\* with setting schema as `schema.object({}, { unknowns: 'allow' })`<!-- -->;
 
 ## Example
 
@@ -49,7 +49,7 @@ router.get({
   path: 'path/{id}',
   validate: {
     // handler has access to raw non-validated params in runtime
-    params: schema.object({}, { allowUnknowns: true })
+    params: schema.object({}, { unknowns: 'allow' })
   },
 },
 (context, req, res,) {

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.esfilters.md

@@ -44,8 +44,8 @@ esFilters: {
     getPhraseFilterField: (filter: import("../common").PhraseFilter) => string;
     getPhraseFilterValue: (filter: import("../common").PhraseFilter) => string | number | boolean;
     getDisplayValueFromFilter: typeof getDisplayValueFromFilter;
-    compareFilters: (first: import("../common").Filter | import("../common").Filter[], second: import("../common").Filter | import("../common").Filter[], comparatorOptions?: import("./query/filter_manager/lib/compare_filters").FilterCompareOptions) => boolean;
-    COMPARE_ALL_OPTIONS: import("./query/filter_manager/lib/compare_filters").FilterCompareOptions;
+    compareFilters: (first: import("../common").Filter | import("../common").Filter[], second: import("../common").Filter | import("../common").Filter[], comparatorOptions?: import("../common").FilterCompareOptions) => boolean;
+    COMPARE_ALL_OPTIONS: import("../common").FilterCompareOptions;
     generateFilters: typeof generateFilters;
     onlyDisabledFiltersChanged: (newFilters?: import("../common").Filter[] | undefined, oldFilters?: import("../common").Filter[] | undefined) => boolean;
     changeTimeFilter: typeof changeTimeFilter;

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchcancel.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) &gt; [ISearchCancel](./kibana-plugin-plugins-data-server.isearchcancel.md)
+
+## ISearchCancel type
+
+<b>Signature:</b>
+
+```typescript
+export declare type ISearchCancel<T extends TStrategyTypes> = (id: string) => Promise<void>;
+```

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.md

@@ -67,9 +67,9 @@
 |  Type Alias | Description |
 |  --- | --- |
 |  [FieldFormatsGetConfigFn](./kibana-plugin-plugins-data-server.fieldformatsgetconfigfn.md) |  |
-|  [ICancel](./kibana-plugin-plugins-data-server.icancel.md) |  |
 |  [IFieldFormatsRegistry](./kibana-plugin-plugins-data-server.ifieldformatsregistry.md) |  |
 |  [ISearch](./kibana-plugin-plugins-data-server.isearch.md) |  |
+|  [ISearchCancel](./kibana-plugin-plugins-data-server.isearchcancel.md) |  |
 |  [ParsedInterval](./kibana-plugin-plugins-data-server.parsedinterval.md) |  |
 |  [TSearchStrategyProvider](./kibana-plugin-plugins-data-server.tsearchstrategyprovider.md) | Search strategy provider creates an instance of a search strategy with the request handler context bound to it. This way every search strategy can use whatever information they require from the request context. |
 

docs/settings/monitoring-settings.asciidoc

@@ -20,9 +20,7 @@ which support the same values as <<settings,{kib} configuration settings>>.
 To control how data is collected from your {es} nodes, you configure
 {ref}/monitoring-settings.html[`xpack.monitoring.collection`
 settings] in `elasticsearch.yml`. To control how monitoring data is collected
-from Logstash, you configure
-{logstash-ref}/monitoring-internal-collection.html#monitoring-settings[`xpack.monitoring` settings]
-in `logstash.yml`.
+from Logstash, configure monitoring settings in `logstash.yml`.
 
 For more information, see
 {ref}/monitor-elasticsearch-cluster.html[Monitor a cluster].

package.json

@@ -77,7 +77,7 @@
     "url": "https://github.com/elastic/kibana.git"
   },
   "resolutions": {
-    "**/@types/node": "10.12.27",
+    "**/@types/node": ">=10.17.17 <10.20.0",
     "**/@types/react": "^16.9.19",
     "**/@types/react-router": "^5.1.3",
     "**/@types/hapi": "^17.0.18",
@@ -119,7 +119,7 @@
     "@elastic/apm-rum": "^4.6.0",
     "@elastic/charts": "^17.1.1",
     "@elastic/datemath": "5.0.2",
-    "@elastic/ems-client": "7.6.0",
+    "@elastic/ems-client": "7.7.0",
     "@elastic/eui": "20.0.2",
     "@elastic/filesaver": "1.1.2",
     "@elastic/good": "8.1.1-kibana2",
@@ -239,7 +239,7 @@
     "react-resize-detector": "^4.2.0",
     "react-router-dom": "^5.1.2",
     "react-sizeme": "^2.3.6",
-    "react-use": "^13.13.0",
+    "react-use": "^13.27.0",
     "reactcss": "1.2.3",
     "redux": "^4.0.5",
     "redux-actions": "^2.6.5",
@@ -314,6 +314,7 @@
     "@types/cheerio": "^0.22.10",
     "@types/chromedriver": "^2.38.0",
     "@types/classnames": "^2.2.9",
+    "@types/color": "^3.0.0",
     "@types/d3": "^3.5.43",
     "@types/dedent": "^0.7.0",
     "@types/deep-freeze-strict": "^1.1.0",
@@ -349,7 +350,7 @@
     "@types/mocha": "^5.2.7",
     "@types/moment-timezone": "^0.5.12",
     "@types/mustache": "^0.8.31",
-    "@types/node": "^10.12.27",
+    "@types/node": ">=10.17.17 <10.20.0",
     "@types/node-forge": "^0.9.0",
     "@types/normalize-path": "^3.0.0",
     "@types/numeral": "^0.0.26",

packages/kbn-config-schema/README.md

@@ -239,7 +239,7 @@ __Output type:__ `{ [K in keyof TProps]: TypeOf<TProps[K]> } as TObject`
 __Options:__
   * `defaultValue: TObject | Reference<TObject> | (() => TObject)` - defines a default value, see [Default values](#default-values) section for more details.
   * `validate: (value: TObject) => string | void` - defines a custom validator function, see [Custom validation](#custom-validation) section for more details.
-  * `allowUnknowns: boolean` - indicates whether unknown object properties should be allowed. It's `false` by default.
+  * `unknowns: 'allow' | 'ignore' | 'forbid'` - indicates whether unknown object properties should be allowed, ignored, or forbidden. It's `forbid` by default.
 
 __Usage:__
 ```typescript
@@ -250,7 +250,7 @@ const valueSchema = schema.object({
 ```
 
 __Notes:__
-* Using `allowUnknowns` is discouraged and should only be used in exceptional circumstances. Consider using `schema.recordOf()` instead.
+* Using `unknowns: 'allow'` is discouraged and should only be used in exceptional circumstances. Consider using `schema.recordOf()` instead.
 * Currently `schema.object()` always has a default value of `{}`, but this may change in the near future. Try to not rely on this behaviour and specify default value explicitly or use `schema.maybe()` if the value is optional.
 * `schema.object()` also supports a json string as input if it can be safely parsed using `JSON.parse` and if the resulting value is a plain object.
 

packages/kbn-config-schema/src/internals/index.ts

@@ -314,7 +314,8 @@ export const internals = Joi.extend([
           for (const [entryKey, entryValue] of value) {
             const { value: validatedEntryKey, error: keyError } = Joi.validate(
               entryKey,
-              params.key
+              params.key,
+              { presence: 'required' }
             );
 
             if (keyError) {
@@ -323,7 +324,8 @@ export const internals = Joi.extend([
 
             const { value: validatedEntryValue, error: valueError } = Joi.validate(
               entryValue,
-              params.value
+              params.value,
+              { presence: 'required' }
             );
 
             if (valueError) {
@@ -374,7 +376,8 @@ export const internals = Joi.extend([
           for (const [entryKey, entryValue] of Object.entries(value)) {
             const { value: validatedEntryKey, error: keyError } = Joi.validate(
               entryKey,
-              params.key
+              params.key,
+              { presence: 'required' }
             );
 
             if (keyError) {
@@ -383,7 +386,8 @@ export const internals = Joi.extend([
 
             const { value: validatedEntryValue, error: valueError } = Joi.validate(
               entryValue,
-              params.value
+              params.value,
+              { presence: 'required' }
             );
 
             if (valueError) {

packages/kbn-config-schema/src/types/map_of_type.test.ts

@@ -159,6 +159,24 @@ test('object within mapOf', () => {
   expect(type.validate(value)).toEqual(expected);
 });
 
+test('enforces required object fields within mapOf', () => {
+  const type = schema.mapOf(
+    schema.string(),
+    schema.object({
+      bar: schema.object({
+        baz: schema.number(),
+      }),
+    })
+  );
+  const value = {
+    foo: {},
+  };
+
+  expect(() => type.validate(value)).toThrowErrorMatchingInlineSnapshot(
+    `"[foo.bar.baz]: expected value of type [number] but got [undefined]"`
+  );
+});
+
 test('error preserves full path', () => {
   const type = schema.object({
     grandParentKey: schema.object({

packages/kbn-config-schema/src/types/map_type.ts

@@ -57,7 +57,10 @@ export class MapOfType<K, V> extends Type<Map<K, V>> {
           path.length,
           0,
           // If `key` validation failed, let's stress that to make error more obvious.
-          type === 'map.key' ? `key("${entryKey}")` : entryKey.toString()
+          type === 'map.key' ? `key("${entryKey}")` : entryKey.toString(),
+          // Error could have happened deep inside value/key schema and error message should
+          // include full path.
+          ...(reason instanceof SchemaTypeError ? reason.path : [])
         );
 
         return reason instanceof SchemaTypesError