[Search Source] Fix retrieval of unmapped fields; Add field filters

src/plugins/data/common/search/search_source/search_source.test.ts

@@ -28,6 +28,7 @@ const mockSource2 = { excludes: ['bar-*'] };
 
 const indexPattern = ({
   title: 'foo',
+  fields: [{ name: 'foo-bar' }, { name: 'field1' }, { name: 'field2' }],
   getComputedFields,
   getSourceFiltering: () => mockSource,
 } as unknown) as IndexPattern;
@@ -51,6 +52,11 @@ describe('SearchSource', () => {
   let searchSource: SearchSource;
 
   beforeEach(() => {
+    const getConfigMock = jest
+      .fn()
+      .mockImplementation((param) => param === 'metaFields' && ['_type', '_source'])
+      .mockName('getConfig');
+
     mockSearchMethod = jest
       .fn()
       .mockReturnValue(
@@ -61,7 +67,7 @@ describe('SearchSource', () => {
       );
 
     searchSourceDependencies = {
-      getConfig: jest.fn(),
+      getConfig: getConfigMock,
       search: mockSearchMethod,
       onResponse: (req, res) => res,
       legacy: {
@@ -518,6 +524,54 @@ describe('SearchSource', () => {
         expect(request.script_fields).toEqual({ hello: {} });
       });
 
+      test('request all fields except the ones specified with source filters', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: [],
+            docvalueFields: [],
+          }),
+        } as unknown) as IndexPattern);
+        searchSource.setField('fields', ['hello', 'foo']);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request.fields).toEqual(['hello']);
+      });
+
+      test('request all fields from index pattern except the ones specified with source filters', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: [],
+            docvalueFields: [],
+          }),
+        } as unknown) as IndexPattern);
+        searchSource.setField('fields', ['*']);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request.fields).toEqual([{ field: 'field1' }, { field: 'field2' }]);
+      });
+
+      test('request all fields from index pattern except the ones specified with source filters with unmapped_fields option', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: [],
+            docvalueFields: [],
+          }),
+        } as unknown) as IndexPattern);
+        searchSource.setField('fields', [{ field: '*', include_unmapped: 'true' }]);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request.fields).toEqual([
+          { field: 'field1', include_unmapped: 'true' },
+          { field: 'field2', include_unmapped: 'true' },
+        ]);
+      });
+
       test('returns all scripted fields when one fields entry is *', async () => {
         searchSource.setField('index', ({
           ...indexPattern,
@@ -836,5 +890,25 @@ describe('SearchSource', () => {
       expect(references[1].type).toEqual('index-pattern');
       expect(JSON.parse(searchSourceJSON).filter[0].meta.indexRefName).toEqual(references[1].name);
     });
+
+    test('mvt geoshape layer test', async () => {
+      // @ts-expect-error TS won't like using this field name, but technically it's possible.
+      searchSource.setField('docvalue_fields', ['prop1']);
+      searchSource.setField('source', ['geometry']);
+      searchSource.setField('fieldsFromSource', ['geometry', 'prop1']);
+      searchSource.setField('index', ({
+        ...indexPattern,
+        getSourceFiltering: () => ({ excludes: [] }),
+        getComputedFields: () => ({
+          storedFields: ['*'],
+          scriptFields: {},
+          docvalueFields: [],
+        }),
+      } as unknown) as IndexPattern);
+      const request = await searchSource.getSearchRequestBody();
+      expect(request.stored_fields).toEqual(['geometry', 'prop1']);
+      expect(request.docvalue_fields).toEqual(['prop1']);
+      expect(request._source).toEqual(['geometry']);
+    });
   });
 });

src/plugins/data/common/search/search_source/search_source.ts

@@ -59,12 +59,13 @@
  */
 
 import { setWith } from '@elastic/safer-lodash-set';
-import { uniqueId, keyBy, pick, difference, omit, isObject, isFunction } from 'lodash';
+import { uniqueId, keyBy, pick, difference, omit, isFunction, isEqual } from 'lodash';
 import { map, switchMap, tap } from 'rxjs/operators';
 import { defer, from } from 'rxjs';
+import { isObject } from 'rxjs/internal-compatibility';
 import { normalizeSortRequest } from './normalize_sort_request';
 import { fieldWildcardFilter } from '../../../../kibana_utils/common';
-import { IIndexPattern } from '../../index_patterns';
+import { IIndexPattern, IndexPattern, IndexPatternField } from '../../index_patterns';
 import { ISearchGeneric, ISearchOptions } from '../..';
 import type {
   ISearchSource,
@@ -500,10 +501,53 @@ export class SearchSource {
     }
   }
 
+  private readonly getFieldName = (fld: string | Record<string, any>): string =>
+    typeof fld === 'string' ? fld : fld.field;
+
+  private getFieldsWithoutSourceFilters(
+    index: IndexPattern | undefined,
+    bodyFields: SearchFieldValue[]
+  ) {
+    if (!index) {
+      return bodyFields;
+    }
+    const { fields } = index;
+    const sourceFilters = index.getSourceFiltering();
+    if (!sourceFilters || sourceFilters.excludes?.length === 0 || bodyFields.length === 0) {
+      return bodyFields;
+    }
+    const metaFields = this.dependencies.getConfig(UI_SETTINGS.META_FIELDS);
+    const sourceFiltersValues = sourceFilters.excludes;
+    const wildcardField = bodyFields.find(
+      (el: SearchFieldValue) => el === '*' || (el as Record<string, string>).field === '*'
+    );
+    const filterSourceFields = (fieldName: string) => {
+      return (
+        fieldName &&
+        !sourceFiltersValues.some((sourceFilter) => fieldName.match(sourceFilter)) &&
+        !metaFields.includes(fieldName)
+      );
+    };
+    if (!wildcardField) {
+      // we already have an explicit list of fields, so we just remove source filters from that list
+      return bodyFields.filter((fld: SearchFieldValue) =>
+        filterSourceFields(this.getFieldName(fld))
+      );
+    }
+    // we need to get the list of fields from an index pattern
+    return fields
+      .filter((fld: IndexPatternField) => filterSourceFields(fld.name))
+      .map((fld: IndexPatternField) => ({
+        field: fld.name,
+        ...((wildcardField as Record<string, string>)?.include_unmapped && {
+          include_unmapped: (wildcardField as Record<string, string>).include_unmapped,
+        }),
+      }));
+  }
+
   private flatten() {
     const { getConfig } = this.dependencies;
     const searchRequest = this.mergeProps();
-
     searchRequest.body = searchRequest.body || {};
     const { body, index, query, filters, highlightAll } = searchRequest;
     searchRequest.indexType = this.getIndexType(index);
@@ -517,10 +561,7 @@ export class SearchSource {
           storedFields: ['*'],
           runtimeFields: {},
         };
-
     const fieldListProvided = !!body.fields;
-    const getFieldName = (fld: string | Record<string, any>): string =>
-      typeof fld === 'string' ? fld : fld.field;
 
     // set defaults
     let fieldsFromSource = searchRequest.fieldsFromSource || [];
@@ -539,26 +580,22 @@ export class SearchSource {
       if (!body.hasOwnProperty('_source')) {
         body._source = sourceFilters;
       }
-      if (body._source.excludes) {
-        const filter = fieldWildcardFilter(
-          body._source.excludes,
-          getConfig(UI_SETTINGS.META_FIELDS)
-        );
-        // also apply filters to provided fields & default docvalueFields
-        body.fields = body.fields.filter((fld: SearchFieldValue) => filter(getFieldName(fld)));
-        fieldsFromSource = fieldsFromSource.filter((fld: SearchFieldValue) =>
-          filter(getFieldName(fld))
-        );
-        filteredDocvalueFields = filteredDocvalueFields.filter((fld: SearchFieldValue) =>
-          filter(getFieldName(fld))
-        );
-      }
+
+      const filter = fieldWildcardFilter(body._source.excludes, getConfig(UI_SETTINGS.META_FIELDS));
+      // also apply filters to provided fields & default docvalueFields
+      body.fields = body.fields.filter((fld: SearchFieldValue) => filter(this.getFieldName(fld)));
+      fieldsFromSource = fieldsFromSource.filter((fld: SearchFieldValue) =>
+        filter(this.getFieldName(fld))
+      );
+      filteredDocvalueFields = filteredDocvalueFields.filter((fld: SearchFieldValue) =>
+        filter(this.getFieldName(fld))
+      );
     }
 
     // specific fields were provided, so we need to exclude any others
     if (fieldListProvided || fieldsFromSource.length) {
       const bodyFieldNames = body.fields.map((field: string | Record<string, any>) =>
-        getFieldName(field)
+        this.getFieldName(field)
       );
       const uniqFieldNames = [...new Set([...bodyFieldNames, ...fieldsFromSource])];
 
@@ -579,28 +616,31 @@ export class SearchSource {
       const remainingFields = difference(uniqFieldNames, [
         ...Object.keys(body.script_fields),
         ...Object.keys(body.runtime_mappings),
-      ]).filter(Boolean);
+      ]).filter((remainingField) => {
+        if (!remainingField) return false;
+        if (!body._source || !body._source.excludes) return true;
+        return !body._source.excludes.includes(remainingField);
+      });
 
-      // only include unique values
       body.stored_fields = [...new Set(remainingFields)];
-
+      // only include unique values
       if (fieldsFromSource.length) {
-        // include remaining fields in _source
-        setWith(body, '_source.includes', remainingFields, (nsValue) =>
-          isObject(nsValue) ? {} : nsValue
-        );
-
+        if (!isEqual(remainingFields, fieldsFromSource)) {
+          setWith(body, '_source.includes', remainingFields, (nsValue) =>
+            isObject(nsValue) ? {} : nsValue
+          );
+        }
         // if items that are in the docvalueFields are provided, we should
         // make sure those are added to the fields API unless they are
         // already set in docvalue_fields
         body.fields = [
           ...body.fields,
           ...filteredDocvalueFields.filter((fld: SearchFieldValue) => {
             return (
-              fieldsFromSource.includes(getFieldName(fld)) &&
+              fieldsFromSource.includes(this.getFieldName(fld)) &&
               !(body.docvalue_fields || [])
-                .map((d: string | Record<string, any>) => getFieldName(d))
-                .includes(getFieldName(fld))
+                .map((d: string | Record<string, any>) => this.getFieldName(d))
+                .includes(this.getFieldName(fld))
             );
           }),
         ];
@@ -614,20 +654,22 @@ export class SearchSource {
         // if items that are in the docvalueFields are provided, we should
         // inject the format from the computed fields if one isn't given
         const docvaluesIndex = keyBy(filteredDocvalueFields, 'field');
-        body.fields = body.fields.map((fld: SearchFieldValue) => {
-          const fieldName = getFieldName(fld);
-          if (Object.keys(docvaluesIndex).includes(fieldName)) {
-            // either provide the field object from computed docvalues,
-            // or merge the user-provided field with the one in docvalues
-            return typeof fld === 'string'
-              ? docvaluesIndex[fld]
-              : {
-                  ...docvaluesIndex[fieldName],
-                  ...fld,
-                };
+        body.fields = this.getFieldsWithoutSourceFilters(index, body.fields).map(
+          (fld: SearchFieldValue) => {
+            const fieldName = this.getFieldName(fld);
+            if (Object.keys(docvaluesIndex).includes(fieldName)) {
+              // either provide the field object from computed docvalues,
+              // or merge the user-provided field with the one in docvalues
+              return typeof fld === 'string'
+                ? docvaluesIndex[fld]
+                : {
+                    ...docvaluesIndex[fieldName],
+                    ...fld,
+                  };
+            }
+            return fld;
           }
-          return fld;
-        });
+        );
       }
     } else {
       body.fields = filteredDocvalueFields;

x-pack/test/functional/apps/maps/mvt_scaling.js

@@ -30,7 +30,7 @@ export default function ({ getPageObjects, getService }) {
 
       //Source should be correct
       expect(mapboxStyle.sources[VECTOR_SOURCE_ID].tiles[0]).to.equal(
-        '/api/maps/mvt/getTile?x={x}&y={y}&z={z}&geometryFieldName=geometry&index=geo_shapes*&requestBody=(_source:(includes:!(geometry,prop1)),docvalue_fields:!(prop1),query:(bool:(filter:!((match_all:())),must:!(),must_not:!(),should:!())),runtime_mappings:(),script_fields:(),size:10000,stored_fields:!(geometry,prop1))&geoFieldType=geo_shape'
+        '/api/maps/mvt/getTile?x={x}&y={y}&z={z}&geometryFieldName=geometry&index=geo_shapes*&requestBody=(_source:!(geometry),docvalue_fields:!(prop1),query:(bool:(filter:!((match_all:())),must:!(),must_not:!(),should:!())),runtime_mappings:(),script_fields:(),size:10000,stored_fields:!(geometry,prop1))&geoFieldType=geo_shape'
       );
 
       //Should correctly load meta for style-rule (sigma is set to 1, opacity to 1)