Merge branch 'master' of github.com:elastic/kibana into ilm/update-co…

…ndition-for-hiding-recommded-allocation

* 'master' of github.com:elastic/kibana:
  [Discover] Fix toggling multi fields from doc view table (elastic#91121)
  [ML] Data Frame Analytics: ROC Curve Chart (elastic#89991)
  skip flaky suite (elastic#86948)
  skip flaky suite (elastic#91191)
  Fix date histogram time zone for rollup index (elastic#90632)
  [Search Source] Fix retrieval of unmapped fields; Add field filters (elastic#89837)
  [Logs UI] Use useMlHref hook for ML links (elastic#90935)
  Fix values of `products.min_price` field in Kibana sample ecommerce data set (elastic#90428)
  [APM] Darker shade for Error group details labels (elastic#91349)
  [Lens] Adjust new copy for 7.12 (elastic#90413)
  [ML] Unskip test. Fix modelMemoryLimit value. (elastic#91280)
  [Lens] Fix empty display name issue in XY chart (elastic#91132)
  [Lens] Improves error messages when in Dashboard (elastic#90668)
  [Lens] Keyboard-selected items follow user traversal of drop zones (elastic#90546)
  [Lens] Improves ranking feature in Top values (elastic#90749)
  [ILM] Rollover min age tooltip and copy fixes (elastic#91110)

# Conflicts:
#	x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.test.ts

src/plugins/data/common/search/aggs/buckets/date_histogram.ts

@@ -11,6 +11,7 @@ import moment from 'moment-timezone';
 import { i18n } from '@kbn/i18n';
 
 import { KBN_FIELD_TYPES, TimeRange, TimeRangeBounds, UI_SETTINGS } from '../../../../common';
+import { IFieldType } from '../../../index_patterns';
 
 import { intervalOptions, autoInterval, isAutoInterval } from './_interval_options';
 import { createFilterDateHistogram } from './create_filter/date_histogram';
@@ -58,7 +59,7 @@ export function isDateHistogramBucketAggConfig(agg: any): agg is IBucketDateHist
 }
 
 export interface AggParamsDateHistogram extends BaseAggParams {
-  field?: string;
+  field?: IFieldType | string;
   timeRange?: TimeRange;
   useNormalizedEsInterval?: boolean;
   scaleMetricValues?: boolean;

src/plugins/data/common/search/aggs/utils/infer_time_zone.test.ts

@@ -18,7 +18,7 @@ jest.mock('moment', () => {
   return moment;
 });
 
-import { IndexPattern } from '../../../index_patterns';
+import { IndexPattern, IndexPatternField } from '../../../index_patterns';
 import { AggParamsDateHistogram } from '../buckets';
 import { inferTimeZone } from './infer_time_zone';
 
@@ -51,6 +51,31 @@ describe('inferTimeZone', () => {
     ).toEqual('UTC');
   });
 
+  it('reads time zone from index pattern type meta if available when the field is not a string', () => {
+    expect(
+      inferTimeZone(
+        {
+          field: {
+            name: 'mydatefield',
+          } as IndexPatternField,
+        },
+        ({
+          typeMeta: {
+            aggs: {
+              date_histogram: {
+                mydatefield: {
+                  time_zone: 'UTC',
+                },
+              },
+            },
+          },
+        } as unknown) as IndexPattern,
+        () => false,
+        jest.fn()
+      )
+    ).toEqual('UTC');
+  });
+
   it('reads time zone from moment if set to default', () => {
     expect(inferTimeZone({}, {} as IndexPattern, () => true, jest.fn())).toEqual('CET');
   });

src/plugins/data/common/search/aggs/utils/infer_time_zone.ts

@@ -20,7 +20,8 @@ export function inferTimeZone(
   if (!tz && params.field) {
     // If a field has been configured check the index pattern's typeMeta if a date_histogram on that
     // field requires a specific time_zone
-    tz = indexPattern.typeMeta?.aggs?.date_histogram?.[params.field]?.time_zone;
+    const fieldName = typeof params.field === 'string' ? params.field : params.field.name;
+    tz = indexPattern.typeMeta?.aggs?.date_histogram?.[fieldName]?.time_zone;
   }
   if (!tz) {
     // If the index pattern typeMeta data, didn't had a time zone assigned for the selected field use the configured tz

src/plugins/data/common/search/search_source/search_source.test.ts

@@ -28,6 +28,7 @@ const mockSource2 = { excludes: ['bar-*'] };
 
 const indexPattern = ({
   title: 'foo',
+  fields: [{ name: 'foo-bar' }, { name: 'field1' }, { name: 'field2' }],
   getComputedFields,
   getSourceFiltering: () => mockSource,
 } as unknown) as IndexPattern;
@@ -51,6 +52,11 @@ describe('SearchSource', () => {
   let searchSource: SearchSource;
 
   beforeEach(() => {
+    const getConfigMock = jest
+      .fn()
+      .mockImplementation((param) => param === 'metaFields' && ['_type', '_source'])
+      .mockName('getConfig');
+
     mockSearchMethod = jest
       .fn()
       .mockReturnValue(
@@ -61,7 +67,7 @@ describe('SearchSource', () => {
       );
 
     searchSourceDependencies = {
-      getConfig: jest.fn(),
+      getConfig: getConfigMock,
       search: mockSearchMethod,
       onResponse: (req, res) => res,
       legacy: {
@@ -518,6 +524,54 @@ describe('SearchSource', () => {
         expect(request.script_fields).toEqual({ hello: {} });
       });
 
+      test('request all fields except the ones specified with source filters', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: [],
+            docvalueFields: [],
+          }),
+        } as unknown) as IndexPattern);
+        searchSource.setField('fields', ['hello', 'foo']);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request.fields).toEqual(['hello']);
+      });
+
+      test('request all fields from index pattern except the ones specified with source filters', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: [],
+            docvalueFields: [],
+          }),
+        } as unknown) as IndexPattern);
+        searchSource.setField('fields', ['*']);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request.fields).toEqual([{ field: 'field1' }, { field: 'field2' }]);
+      });
+
+      test('request all fields from index pattern except the ones specified with source filters with unmapped_fields option', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: [],
+            docvalueFields: [],
+          }),
+        } as unknown) as IndexPattern);
+        searchSource.setField('fields', [{ field: '*', include_unmapped: 'true' }]);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request.fields).toEqual([
+          { field: 'field1', include_unmapped: 'true' },
+          { field: 'field2', include_unmapped: 'true' },
+        ]);
+      });
+
       test('returns all scripted fields when one fields entry is *', async () => {
         searchSource.setField('index', ({
           ...indexPattern,
@@ -836,5 +890,25 @@ describe('SearchSource', () => {
       expect(references[1].type).toEqual('index-pattern');
       expect(JSON.parse(searchSourceJSON).filter[0].meta.indexRefName).toEqual(references[1].name);
     });
+
+    test('mvt geoshape layer test', async () => {
+      // @ts-expect-error TS won't like using this field name, but technically it's possible.
+      searchSource.setField('docvalue_fields', ['prop1']);
+      searchSource.setField('source', ['geometry']);
+      searchSource.setField('fieldsFromSource', ['geometry', 'prop1']);
+      searchSource.setField('index', ({
+        ...indexPattern,
+        getSourceFiltering: () => ({ excludes: [] }),
+        getComputedFields: () => ({
+          storedFields: ['*'],
+          scriptFields: {},
+          docvalueFields: [],
+        }),
+      } as unknown) as IndexPattern);
+      const request = await searchSource.getSearchRequestBody();
+      expect(request.stored_fields).toEqual(['geometry', 'prop1']);
+      expect(request.docvalue_fields).toEqual(['prop1']);
+      expect(request._source).toEqual(['geometry']);
+    });
   });
 });

src/plugins/data/common/search/search_source/search_source.ts

@@ -59,12 +59,13 @@
  */
 
 import { setWith } from '@elastic/safer-lodash-set';
-import { uniqueId, keyBy, pick, difference, omit, isObject, isFunction } from 'lodash';
+import { uniqueId, keyBy, pick, difference, omit, isFunction, isEqual } from 'lodash';
 import { map, switchMap, tap } from 'rxjs/operators';
 import { defer, from } from 'rxjs';
+import { isObject } from 'rxjs/internal-compatibility';
 import { normalizeSortRequest } from './normalize_sort_request';
 import { fieldWildcardFilter } from '../../../../kibana_utils/common';
-import { IIndexPattern } from '../../index_patterns';
+import { IIndexPattern, IndexPattern, IndexPatternField } from '../../index_patterns';
 import { ISearchGeneric, ISearchOptions } from '../..';
 import type {
   ISearchSource,
@@ -500,10 +501,53 @@ export class SearchSource {
     }
   }
 
+  private readonly getFieldName = (fld: string | Record<string, any>): string =>
+    typeof fld === 'string' ? fld : fld.field;
+
+  private getFieldsWithoutSourceFilters(
+    index: IndexPattern | undefined,
+    bodyFields: SearchFieldValue[]
+  ) {
+    if (!index) {
+      return bodyFields;
+    }
+    const { fields } = index;
+    const sourceFilters = index.getSourceFiltering();
+    if (!sourceFilters || sourceFilters.excludes?.length === 0 || bodyFields.length === 0) {
+      return bodyFields;
+    }
+    const metaFields = this.dependencies.getConfig(UI_SETTINGS.META_FIELDS);
+    const sourceFiltersValues = sourceFilters.excludes;
+    const wildcardField = bodyFields.find(
+      (el: SearchFieldValue) => el === '*' || (el as Record<string, string>).field === '*'
+    );
+    const filterSourceFields = (fieldName: string) => {
+      return (
+        fieldName &&
+        !sourceFiltersValues.some((sourceFilter) => fieldName.match(sourceFilter)) &&
+        !metaFields.includes(fieldName)
+      );
+    };
+    if (!wildcardField) {
+      // we already have an explicit list of fields, so we just remove source filters from that list
+      return bodyFields.filter((fld: SearchFieldValue) =>
+        filterSourceFields(this.getFieldName(fld))
+      );
+    }
+    // we need to get the list of fields from an index pattern
+    return fields
+      .filter((fld: IndexPatternField) => filterSourceFields(fld.name))
+      .map((fld: IndexPatternField) => ({
+        field: fld.name,
+        ...((wildcardField as Record<string, string>)?.include_unmapped && {
+          include_unmapped: (wildcardField as Record<string, string>).include_unmapped,
+        }),
+      }));
+  }
+
   private flatten() {
     const { getConfig } = this.dependencies;
     const searchRequest = this.mergeProps();
-
     searchRequest.body = searchRequest.body || {};
     const { body, index, query, filters, highlightAll } = searchRequest;
     searchRequest.indexType = this.getIndexType(index);
@@ -517,10 +561,7 @@ export class SearchSource {
           storedFields: ['*'],
           runtimeFields: {},
         };
-
     const fieldListProvided = !!body.fields;
-    const getFieldName = (fld: string | Record<string, any>): string =>
-      typeof fld === 'string' ? fld : fld.field;
 
     // set defaults
     let fieldsFromSource = searchRequest.fieldsFromSource || [];
@@ -539,26 +580,22 @@ export class SearchSource {
       if (!body.hasOwnProperty('_source')) {
         body._source = sourceFilters;
       }
-      if (body._source.excludes) {
-        const filter = fieldWildcardFilter(
-          body._source.excludes,
-          getConfig(UI_SETTINGS.META_FIELDS)
-        );
-        // also apply filters to provided fields & default docvalueFields
-        body.fields = body.fields.filter((fld: SearchFieldValue) => filter(getFieldName(fld)));
-        fieldsFromSource = fieldsFromSource.filter((fld: SearchFieldValue) =>
-          filter(getFieldName(fld))
-        );
-        filteredDocvalueFields = filteredDocvalueFields.filter((fld: SearchFieldValue) =>
-          filter(getFieldName(fld))
-        );
-      }
+
+      const filter = fieldWildcardFilter(body._source.excludes, getConfig(UI_SETTINGS.META_FIELDS));
+      // also apply filters to provided fields & default docvalueFields
+      body.fields = body.fields.filter((fld: SearchFieldValue) => filter(this.getFieldName(fld)));
+      fieldsFromSource = fieldsFromSource.filter((fld: SearchFieldValue) =>
+        filter(this.getFieldName(fld))
+      );
+      filteredDocvalueFields = filteredDocvalueFields.filter((fld: SearchFieldValue) =>
+        filter(this.getFieldName(fld))
+      );
     }
 
     // specific fields were provided, so we need to exclude any others
     if (fieldListProvided || fieldsFromSource.length) {
       const bodyFieldNames = body.fields.map((field: string | Record<string, any>) =>
-        getFieldName(field)
+        this.getFieldName(field)
       );
       const uniqFieldNames = [...new Set([...bodyFieldNames, ...fieldsFromSource])];
 
@@ -579,28 +616,31 @@ export class SearchSource {
       const remainingFields = difference(uniqFieldNames, [
         ...Object.keys(body.script_fields),
         ...Object.keys(body.runtime_mappings),
-      ]).filter(Boolean);
+      ]).filter((remainingField) => {
+        if (!remainingField) return false;
+        if (!body._source || !body._source.excludes) return true;
+        return !body._source.excludes.includes(remainingField);
+      });
 
-      // only include unique values
       body.stored_fields = [...new Set(remainingFields)];
-
+      // only include unique values
       if (fieldsFromSource.length) {
-        // include remaining fields in _source
-        setWith(body, '_source.includes', remainingFields, (nsValue) =>
-          isObject(nsValue) ? {} : nsValue
-        );
-
+        if (!isEqual(remainingFields, fieldsFromSource)) {
+          setWith(body, '_source.includes', remainingFields, (nsValue) =>
+            isObject(nsValue) ? {} : nsValue
+          );
+        }
         // if items that are in the docvalueFields are provided, we should
         // make sure those are added to the fields API unless they are
         // already set in docvalue_fields
         body.fields = [
           ...body.fields,
           ...filteredDocvalueFields.filter((fld: SearchFieldValue) => {
             return (
-              fieldsFromSource.includes(getFieldName(fld)) &&
+              fieldsFromSource.includes(this.getFieldName(fld)) &&
               !(body.docvalue_fields || [])
-                .map((d: string | Record<string, any>) => getFieldName(d))
-                .includes(getFieldName(fld))
+                .map((d: string | Record<string, any>) => this.getFieldName(d))
+                .includes(this.getFieldName(fld))
             );
           }),
         ];
@@ -614,20 +654,22 @@ export class SearchSource {
         // if items that are in the docvalueFields are provided, we should
         // inject the format from the computed fields if one isn't given
         const docvaluesIndex = keyBy(filteredDocvalueFields, 'field');
-        body.fields = body.fields.map((fld: SearchFieldValue) => {
-          const fieldName = getFieldName(fld);
-          if (Object.keys(docvaluesIndex).includes(fieldName)) {
-            // either provide the field object from computed docvalues,
-            // or merge the user-provided field with the one in docvalues
-            return typeof fld === 'string'
-              ? docvaluesIndex[fld]
-              : {
-                  ...docvaluesIndex[fieldName],
-                  ...fld,
-                };
+        body.fields = this.getFieldsWithoutSourceFilters(index, body.fields).map(
+          (fld: SearchFieldValue) => {
+            const fieldName = this.getFieldName(fld);
+            if (Object.keys(docvaluesIndex).includes(fieldName)) {
+              // either provide the field object from computed docvalues,
+              // or merge the user-provided field with the one in docvalues
+              return typeof fld === 'string'
+                ? docvaluesIndex[fld]
+                : {
+                    ...docvaluesIndex[fieldName],
+                    ...fld,
+                  };
+            }
+            return fld;
           }
-          return fld;
-        });
+        );
       }
     } else {
       body.fields = filteredDocvalueFields;