Merge branch 'master' into alerting/consumer-based-rbac

* master:
  [Encrypted Saved Objects] Adds support for migrations in ESO (elastic#69513)
  [SIEM] Replace WithSource with useWithSource hook (elastic#68722)
  [Endpoint]EMT-451: add ability to filter endpoint metadata based on presence of unenrolled events (elastic#69708)
  rename old siem kibana config to securitySolution (elastic#69874)
  Remove unused Resolver code (elastic#69914)
  [Observability] Fixing dynamic return type based on the appName (elastic#69894)
  [SECURITY SOLUTION][INGEST] Task/endpoint list tests (elastic#69419)
  Fixes special clicks and 3rd party icon sizes in nav (elastic#69767)
  [APM] Catch annotations index permission error and log warning (elastic#69881)
  [Endpoint][Ingest Manager] minor code cleanup (elastic#69844)
  [Logs UI] Logs ui context menu (elastic#69915)
  Index pattern serialize and de-serialize (elastic#68844)

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.ifieldtype.md

@@ -28,6 +28,7 @@ export interface IFieldType
 |  [searchable](./kibana-plugin-plugins-data-public.ifieldtype.searchable.md) | <code>boolean</code> |  |
 |  [sortable](./kibana-plugin-plugins-data-public.ifieldtype.sortable.md) | <code>boolean</code> |  |
 |  [subType](./kibana-plugin-plugins-data-public.ifieldtype.subtype.md) | <code>IFieldSubType</code> |  |
+|  [toSpec](./kibana-plugin-plugins-data-public.ifieldtype.tospec.md) | <code>() =&gt; FieldSpec</code> |  |
 |  [type](./kibana-plugin-plugins-data-public.ifieldtype.type.md) | <code>string</code> |  |
 |  [visualizable](./kibana-plugin-plugins-data-public.ifieldtype.visualizable.md) | <code>boolean</code> |  |
 

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.ifieldtype.tospec.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [IFieldType](./kibana-plugin-plugins-data-public.ifieldtype.md) &gt; [toSpec](./kibana-plugin-plugins-data-public.ifieldtype.tospec.md)
+
+## IFieldType.toSpec property
+
+<b>Signature:</b>
+
+```typescript
+toSpec?: () => FieldSpec;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.fields.md

@@ -7,5 +7,7 @@
 <b>Signature:</b>
 
 ```typescript
-fields: IIndexPatternFieldList;
+fields: IIndexPatternFieldList & {
+        toSpec: () => FieldSpec[];
+    };
 ```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.initfromspec.md

@@ -0,0 +1,22 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [IndexPattern](./kibana-plugin-plugins-data-public.indexpattern.md) &gt; [initFromSpec](./kibana-plugin-plugins-data-public.indexpattern.initfromspec.md)
+
+## IndexPattern.initFromSpec() method
+
+<b>Signature:</b>
+
+```typescript
+initFromSpec(spec: IndexPatternSpec): this;
+```
+
+## Parameters
+
+|  Parameter | Type | Description |
+|  --- | --- | --- |
+|  spec | <code>IndexPatternSpec</code> |  |
+
+<b>Returns:</b>
+
+`this`
+

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.md

@@ -21,7 +21,7 @@ export declare class IndexPattern implements IIndexPattern
 |  Property | Modifiers | Type | Description |
 |  --- | --- | --- | --- |
 |  [fieldFormatMap](./kibana-plugin-plugins-data-public.indexpattern.fieldformatmap.md) |  | <code>any</code> |  |
-|  [fields](./kibana-plugin-plugins-data-public.indexpattern.fields.md) |  | <code>IIndexPatternFieldList</code> |  |
+|  [fields](./kibana-plugin-plugins-data-public.indexpattern.fields.md) |  | <code>IIndexPatternFieldList &amp; {</code><br/><code>        toSpec: () =&gt; FieldSpec[];</code><br/><code>    }</code> |  |
 |  [fieldsFetcher](./kibana-plugin-plugins-data-public.indexpattern.fieldsfetcher.md) |  | <code>any</code> |  |
 |  [flattenHit](./kibana-plugin-plugins-data-public.indexpattern.flattenhit.md) |  | <code>any</code> |  |
 |  [formatField](./kibana-plugin-plugins-data-public.indexpattern.formatfield.md) |  | <code>any</code> |  |
@@ -30,7 +30,6 @@ export declare class IndexPattern implements IIndexPattern
 |  [metaFields](./kibana-plugin-plugins-data-public.indexpattern.metafields.md) |  | <code>string[]</code> |  |
 |  [timeFieldName](./kibana-plugin-plugins-data-public.indexpattern.timefieldname.md) |  | <code>string &#124; undefined</code> |  |
 |  [title](./kibana-plugin-plugins-data-public.indexpattern.title.md) |  | <code>string</code> |  |
-|  [type](./kibana-plugin-plugins-data-public.indexpattern.type.md) |  | <code>string</code> |  |
 |  [typeMeta](./kibana-plugin-plugins-data-public.indexpattern.typemeta.md) |  | <code>TypeMeta</code> |  |
 
 ## Methods
@@ -49,6 +48,7 @@ export declare class IndexPattern implements IIndexPattern
 |  [getSourceFiltering()](./kibana-plugin-plugins-data-public.indexpattern.getsourcefiltering.md) |  |  |
 |  [getTimeField()](./kibana-plugin-plugins-data-public.indexpattern.gettimefield.md) |  |  |
 |  [init(forceFieldRefresh)](./kibana-plugin-plugins-data-public.indexpattern.init.md) |  |  |
+|  [initFromSpec(spec)](./kibana-plugin-plugins-data-public.indexpattern.initfromspec.md) |  |  |
 |  [isTimeBased()](./kibana-plugin-plugins-data-public.indexpattern.istimebased.md) |  |  |
 |  [isTimeBasedWildcard()](./kibana-plugin-plugins-data-public.indexpattern.istimebasedwildcard.md) |  |  |
 |  [isTimeNanosBased()](./kibana-plugin-plugins-data-public.indexpattern.istimenanosbased.md) |  |  |
@@ -59,5 +59,6 @@ export declare class IndexPattern implements IIndexPattern
 |  [removeScriptedField(field)](./kibana-plugin-plugins-data-public.indexpattern.removescriptedfield.md) |  |  |
 |  [save(saveAttempts)](./kibana-plugin-plugins-data-public.indexpattern.save.md) |  |  |
 |  [toJSON()](./kibana-plugin-plugins-data-public.indexpattern.tojson.md) |  |  |
+|  [toSpec()](./kibana-plugin-plugins-data-public.indexpattern.tospec.md) |  |  |
 |  [toString()](./kibana-plugin-plugins-data-public.indexpattern.tostring.md) |  |  |
 

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.type.md → docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.tospec.md

@@ -1,11 +1,15 @@
 <!-- Do not edit this file. It is automatically generated by API Documenter. -->
 
-[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [IndexPattern](./kibana-plugin-plugins-data-public.indexpattern.md) &gt; [type](./kibana-plugin-plugins-data-public.indexpattern.type.md)
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [IndexPattern](./kibana-plugin-plugins-data-public.indexpattern.md) &gt; [toSpec](./kibana-plugin-plugins-data-public.indexpattern.tospec.md)
 
-## IndexPattern.type property
+## IndexPattern.toSpec() method
 
 <b>Signature:</b>
 
 ```typescript
-type?: string;
+toSpec(): IndexPatternSpec;
 ```
+<b>Returns:</b>
+
+`IndexPatternSpec`
+

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternfield._constructor_.md

@@ -9,15 +9,15 @@ Constructs a new instance of the `Field` class
 <b>Signature:</b>
 
 ```typescript
-constructor(indexPattern: IIndexPattern, spec: FieldSpec | Field, shortDotsEnable: boolean, { fieldFormats, onNotification }: FieldDependencies);
+constructor(indexPattern: IIndexPattern, spec: FieldSpecExportFmt | FieldSpec | Field, shortDotsEnable: boolean, { fieldFormats, onNotification }: FieldDependencies);
 ```
 
 ## Parameters
 
 |  Parameter | Type | Description |
 |  --- | --- | --- |
 |  indexPattern | <code>IIndexPattern</code> |  |
-|  spec | <code>FieldSpec &#124; Field</code> |  |
+|  spec | <code>FieldSpecExportFmt &#124; FieldSpec &#124; Field</code> |  |
 |  shortDotsEnable | <code>boolean</code> |  |
 |  { fieldFormats, onNotification } | <code>FieldDependencies</code> |  |
 

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternfield.conflictdescriptions.md

@@ -7,5 +7,5 @@
 <b>Signature:</b>
 
 ```typescript
-conflictDescriptions?: Record<string, string[]>;
+conflictDescriptions?: FieldSpecConflictDescriptions;
 ```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternfield.md

@@ -22,7 +22,7 @@ export declare class Field implements IFieldType
 |  --- | --- | --- | --- |
 |  [$$spec](./kibana-plugin-plugins-data-public.indexpatternfield.__spec.md) |  | <code>FieldSpec</code> |  |
 |  [aggregatable](./kibana-plugin-plugins-data-public.indexpatternfield.aggregatable.md) |  | <code>boolean</code> |  |
-|  [conflictDescriptions](./kibana-plugin-plugins-data-public.indexpatternfield.conflictdescriptions.md) |  | <code>Record&lt;string, string[]&gt;</code> |  |
+|  [conflictDescriptions](./kibana-plugin-plugins-data-public.indexpatternfield.conflictdescriptions.md) |  | <code>FieldSpecConflictDescriptions</code> |  |
 |  [count](./kibana-plugin-plugins-data-public.indexpatternfield.count.md) |  | <code>number</code> |  |
 |  [displayName](./kibana-plugin-plugins-data-public.indexpatternfield.displayname.md) |  | <code>string</code> |  |
 |  [esTypes](./kibana-plugin-plugins-data-public.indexpatternfield.estypes.md) |  | <code>string[]</code> |  |
@@ -37,6 +37,7 @@ export declare class Field implements IFieldType
 |  [searchable](./kibana-plugin-plugins-data-public.indexpatternfield.searchable.md) |  | <code>boolean</code> |  |
 |  [sortable](./kibana-plugin-plugins-data-public.indexpatternfield.sortable.md) |  | <code>boolean</code> |  |
 |  [subType](./kibana-plugin-plugins-data-public.indexpatternfield.subtype.md) |  | <code>IFieldSubType</code> |  |
+|  [toSpec](./kibana-plugin-plugins-data-public.indexpatternfield.tospec.md) |  | <code>() =&gt; FieldSpecExportFmt</code> |  |
 |  [type](./kibana-plugin-plugins-data-public.indexpatternfield.type.md) |  | <code>string</code> |  |
 |  [visualizable](./kibana-plugin-plugins-data-public.indexpatternfield.visualizable.md) |  | <code>boolean</code> |  |
 

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternfield.tospec.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [IndexPatternField](./kibana-plugin-plugins-data-public.indexpatternfield.md) &gt; [toSpec](./kibana-plugin-plugins-data-public.indexpatternfield.tospec.md)
+
+## IndexPatternField.toSpec property
+
+<b>Signature:</b>
+
+```typescript
+toSpec: () => FieldSpecExportFmt;
+```

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.ifieldtype.md

@@ -28,6 +28,7 @@ export interface IFieldType
 |  [searchable](./kibana-plugin-plugins-data-server.ifieldtype.searchable.md) | <code>boolean</code> |  |
 |  [sortable](./kibana-plugin-plugins-data-server.ifieldtype.sortable.md) | <code>boolean</code> |  |
 |  [subType](./kibana-plugin-plugins-data-server.ifieldtype.subtype.md) | <code>IFieldSubType</code> |  |
+|  [toSpec](./kibana-plugin-plugins-data-server.ifieldtype.tospec.md) | <code>() =&gt; FieldSpec</code> |  |
 |  [type](./kibana-plugin-plugins-data-server.ifieldtype.type.md) | <code>string</code> |  |
 |  [visualizable](./kibana-plugin-plugins-data-server.ifieldtype.visualizable.md) | <code>boolean</code> |  |
 

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.ifieldtype.tospec.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) &gt; [IFieldType](./kibana-plugin-plugins-data-server.ifieldtype.md) &gt; [toSpec](./kibana-plugin-plugins-data-server.ifieldtype.tospec.md)
+
+## IFieldType.toSpec property
+
+<b>Signature:</b>
+
+```typescript
+toSpec?: () => FieldSpec;
+```

src/core/public/chrome/ui/header/collapsible_nav.tsx

@@ -38,7 +38,7 @@ import { AppCategory } from '../../../../types';
 import { InternalApplicationStart } from '../../../application/types';
 import { HttpStart } from '../../../http';
 import { OnIsLockedUpdate } from './';
-import { createEuiListItem, createRecentNavLink } from './nav_link';
+import { createEuiListItem, createRecentNavLink, isModifiedOrPrevented } from './nav_link';
 
 function getAllCategories(allCategorizedLinks: Record<string, ChromeNavLink[]>) {
   const allCategories = {} as Record<string, AppCategory | undefined>;
@@ -184,17 +184,13 @@ export function CollapsibleNav({
                 label: 'Home',
                 iconType: 'home',
                 href: homeHref,
-                onClick: (event: React.MouseEvent<HTMLButtonElement, MouseEvent>) => {
-                  closeNav();
-                  if (
-                    event.isDefaultPrevented() ||
-                    event.altKey ||
-                    event.metaKey ||
-                    event.ctrlKey
-                  ) {
+                onClick: (event) => {
+                  if (isModifiedOrPrevented(event)) {
                     return;
                   }
+
                   event.preventDefault();
+                  closeNav();
                   navigateToApp('home');
                 },
               },
@@ -230,7 +226,13 @@ export function CollapsibleNav({
               return {
                 ...hydratedLink,
                 'data-test-subj': 'collapsibleNavAppLink--recent',
-                onClick: closeNav,
+                onClick: (event) => {
+                  if (isModifiedOrPrevented(event)) {
+                    return;
+                  }
+
+                  closeNav();
+                },
               };
             })}
             maxWidth="none"

src/core/public/chrome/ui/header/nav_link.tsx

@@ -17,20 +17,15 @@
  * under the License.
  */
 
-import { EuiImage } from '@elastic/eui';
+import { EuiIcon } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import React from 'react';
 import { ChromeNavLink, ChromeRecentlyAccessedHistoryItem, CoreStart } from '../../..';
 import { HttpStart } from '../../../http';
 import { relativeToAbsolute } from '../../nav_links/to_nav_link';
 
-function isModifiedEvent(event: React.MouseEvent<HTMLButtonElement, MouseEvent>) {
-  return !!(event.metaKey || event.altKey || event.ctrlKey || event.shiftKey);
-}
-
-function LinkIcon({ url }: { url: string }) {
-  return <EuiImage size="s" alt="" aria-hidden={true} url={url} />;
-}
+export const isModifiedOrPrevented = (event: React.MouseEvent<HTMLButtonElement, MouseEvent>) =>
+  event.metaKey || event.altKey || event.ctrlKey || event.shiftKey || event.defaultPrevented;
 
 interface Props {
   link: ChromeNavLink;
@@ -69,14 +64,16 @@ export function createEuiListItem({
     href,
     /* Use href and onClick to support "open in new tab" and SPA navigation in the same link */
     onClick(event: React.MouseEvent<HTMLButtonElement, MouseEvent>) {
-      onClick();
+      if (!isModifiedOrPrevented(event)) {
+        onClick();
+      }
+
       if (
         !externalLink && // ignore external links
         !legacyMode && // ignore when in legacy mode
         !legacy && // ignore links to legacy apps
-        !event.defaultPrevented && // onClick prevented default
         event.button === 0 && // ignore everything but left clicks
-        !isModifiedEvent(event) // ignore clicks with modifier keys
+        !isModifiedOrPrevented(event)
       ) {
         event.preventDefault();
         navigateToApp(id);
@@ -88,7 +85,8 @@ export function createEuiListItem({
     'data-test-subj': dataTestSubj,
     ...(basePath && {
       iconType: euiIconType,
-      icon: !euiIconType && icon ? <LinkIcon url={basePath.prepend(`/${icon}`)} /> : undefined,
+      icon:
+        !euiIconType && icon ? <EuiIcon type={basePath.prepend(`/${icon}`)} size="m" /> : undefined,
     }),
   };
 }

src/core/server/mocks.ts

@@ -46,6 +46,7 @@ export { httpServiceMock } from './http/http_service.mock';
 export { loggingSystemMock } from './logging/logging_system.mock';
 export { savedObjectsRepositoryMock } from './saved_objects/service/lib/repository.mock';
 export { savedObjectsServiceMock } from './saved_objects/saved_objects_service.mock';
+export { migrationMocks } from './saved_objects/migrations/mocks';
 export { typeRegistryMock as savedObjectsTypeRegistryMock } from './saved_objects/saved_objects_type_registry.mock';
 export { uiSettingsServiceMock } from './ui_settings/ui_settings_service.mock';
 export { metricsServiceMock } from './metrics/metrics_service.mock';

src/core/server/saved_objects/migrations/core/index_migrator.ts

@@ -195,7 +195,7 @@ async function migrateSourceToDest(context: Context) {
     await Index.write(
       callCluster,
       dest.indexName,
-      migrateRawDocs(serializer, documentMigrator.migrate, docs, log)
+      await migrateRawDocs(serializer, documentMigrator.migrate, docs, log)
     );
   }
 }

src/core/server/saved_objects/migrations/core/migrate_raw_docs.test.ts

@@ -26,7 +26,7 @@ import { createSavedObjectsMigrationLoggerMock } from '../../migrations/mocks';
 describe('migrateRawDocs', () => {
   test('converts raw docs to saved objects', async () => {
     const transform = jest.fn<any, any>((doc: any) => _.set(doc, 'attributes.name', 'HOI!'));
-    const result = migrateRawDocs(
+    const result = await migrateRawDocs(
       new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
       transform,
       [
@@ -55,7 +55,7 @@ describe('migrateRawDocs', () => {
     const transform = jest.fn<any, any>((doc: any) =>
       _.set(_.cloneDeep(doc), 'attributes.name', 'TADA')
     );
-    const result = migrateRawDocs(
+    const result = await migrateRawDocs(
       new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
       transform,
       [

src/core/server/saved_objects/migrations/core/migrate_raw_docs.ts

@@ -21,7 +21,11 @@
  * This file provides logic for migrating raw documents.
  */
 
-import { SavedObjectsRawDoc, SavedObjectsSerializer } from '../../serialization';
+import {
+  SavedObjectsRawDoc,
+  SavedObjectsSerializer,
+  SavedObjectUnsanitizedDoc,
+} from '../../serialization';
 import { TransformFn } from './document_migrator';
 import { SavedObjectsMigrationLogger } from '.';
 
@@ -33,26 +37,51 @@ import { SavedObjectsMigrationLogger } from '.';
  * @param {SavedObjectsRawDoc[]} rawDocs
  * @returns {SavedObjectsRawDoc[]}
  */
-export function migrateRawDocs(
+export async function migrateRawDocs(
   serializer: SavedObjectsSerializer,
   migrateDoc: TransformFn,
   rawDocs: SavedObjectsRawDoc[],
   log: SavedObjectsMigrationLogger
-): SavedObjectsRawDoc[] {
-  return rawDocs.map((raw) => {
+): Promise<SavedObjectsRawDoc[]> {
+  const migrateDocWithoutBlocking = transformNonBlocking(migrateDoc);
+  const processedDocs = [];
+  for (const raw of rawDocs) {
     if (serializer.isRawSavedObject(raw)) {
       const savedObject = serializer.rawToSavedObject(raw);
       savedObject.migrationVersion = savedObject.migrationVersion || {};
-      return serializer.savedObjectToRaw({
-        references: [],
-        ...migrateDoc(savedObject),
-      });
+      processedDocs.push(
+        serializer.savedObjectToRaw({
+          references: [],
+          ...(await migrateDocWithoutBlocking(savedObject)),
+        })
+      );
+    } else {
+      log.error(
+        `Error: Unable to migrate the corrupt Saved Object document ${raw._id}. To prevent Kibana from performing a migration on every restart, please delete or fix this document by ensuring that the namespace and type in the document's id matches the values in the namespace and type fields.`,
+        { rawDocument: raw }
+      );
+      processedDocs.push(raw);
     }
+  }
+  return processedDocs;
+}
 
-    log.error(
-      `Error: Unable to migrate the corrupt Saved Object document ${raw._id}. To prevent Kibana from performing a migration on every restart, please delete or fix this document by ensuring that the namespace and type in the document's id matches the values in the namespace and type fields.`,
-      { rawDocument: raw }
-    );
-    return raw;
-  });
+/**
+ * Migration transform functions are potentially CPU heavy e.g. doing decryption/encryption
+ * or (de)/serializing large JSON payloads.
+ * Executing all transforms for a batch in a synchronous loop can block the event-loop for a long time.
+ * To prevent this we use setImmediate to ensure that the event-loop can process other parallel
+ * work in between each transform.
+ */
+function transformNonBlocking(
+  transform: TransformFn
+): (doc: SavedObjectUnsanitizedDoc) => Promise<SavedObjectUnsanitizedDoc> {
+  // promises aren't enough to unblock the event loop
+  return (doc: SavedObjectUnsanitizedDoc) =>
+    new Promise((resolve) => {
+      // set immediate is though
+      setImmediate(() => {
+        resolve(transform(doc));
+      });
+    });
 }

src/fixtures/stubbed_saved_object_index_pattern.js

@@ -27,6 +27,7 @@ export function stubbedSavedObjectIndexPattern(id) {
     id,
     type: 'index-pattern',
     attributes: {
+      timeFieldName: 'timestamp',
       customFormats: '{}',
       fields: mockLogstashFields,
     },