Move logic for requesting a field list inside search_source

src/plugins/data/common/search/search_source/search_source.test.ts

@@ -28,6 +28,7 @@ const mockSource2 = { excludes: ['bar-*'] };
 
 const indexPattern = ({
   title: 'foo',
+  fields: [{ name: 'foo-bar' }, { name: 'field1' }, { name: 'field2' }],
   getComputedFields,
   getSourceFiltering: () => mockSource,
 } as unknown) as IndexPattern;
@@ -518,6 +519,54 @@ describe('SearchSource', () => {
         expect(request.script_fields).toEqual({ hello: {} });
       });
 
+      test('request all fields except the ones specified with source filters', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: [],
+            docvalueFields: [],
+          }),
+        } as unknown) as IndexPattern);
+        searchSource.setField('fields', ['hello', 'foo']);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request.fields).toEqual(['hello']);
+      });
+
+      test('request all fields from index pattern except the ones specified with source filters', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: [],
+            docvalueFields: [],
+          }),
+        } as unknown) as IndexPattern);
+        searchSource.setField('fields', ['*']);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request.fields).toEqual([{ field: 'field1' }, { field: 'field2' }]);
+      });
+
+      test('request all fields from index pattern except the ones specified with source filters with unmapped_fields option', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: [],
+            docvalueFields: [],
+          }),
+        } as unknown) as IndexPattern);
+        searchSource.setField('fields', [{ field: '*', include_unmapped: 'true' }]);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request.fields).toEqual([
+          { field: 'field1', include_unmapped: 'true' },
+          { field: 'field2', include_unmapped: 'true' },
+        ]);
+      });
+
       test('returns all scripted fields when one fields entry is *', async () => {
         searchSource.setField('index', ({
           ...indexPattern,

src/plugins/data/common/search/search_source/search_source.ts

@@ -65,7 +65,7 @@ import { defer, from } from 'rxjs';
 import { isObject } from 'rxjs/internal-compatibility';
 import { normalizeSortRequest } from './normalize_sort_request';
 import { fieldWildcardFilter } from '../../../../kibana_utils/common';
-import { IIndexPattern } from '../../index_patterns';
+import { IIndexPattern, IndexPattern, IndexPatternField } from '../../index_patterns';
 import { ISearchGeneric, ISearchOptions } from '../..';
 import type {
   ISearchSource,
@@ -100,6 +100,8 @@ export interface SearchSourceDependencies extends FetchHandlers {
   search: ISearchGeneric;
 }
 
+const META_FIELDS = ['_type', '_source'];
+
 /** @public **/
 export class SearchSource {
   private id: string = uniqueId('data_source');
@@ -501,6 +503,60 @@ export class SearchSource {
     }
   }
 
+  private checkMatch(sourceFilters: string[], field: string) {
+    return sourceFilters.some((sourceFilter) => field.match(sourceFilter));
+  }
+
+  private getFieldsWithoutSourceFilters(
+    index: IndexPattern | undefined,
+    bodyFields: SearchFieldValue[]
+  ) {
+    if (!index) {
+      return bodyFields;
+    }
+    const { fields } = index;
+    const sourceFilters = index.getSourceFiltering();
+    if (!sourceFilters || sourceFilters.excludes?.length === 0 || bodyFields.length === 0) {
+      return bodyFields;
+    }
+    const sourceFiltersValues = sourceFilters.excludes;
+    const wildcardField = bodyFields.find(
+      (el: SearchFieldValue) => el === '*' || (el as Record<string, string>).field === '*'
+    );
+    if (!wildcardField) {
+      // we already have an explicit list of fields, so we just remove source filters from that list
+      return bodyFields.filter((fld: SearchFieldValue) => {
+        if (typeof fld === 'string') {
+          return !this.checkMatch(sourceFiltersValues, fld) && !META_FIELDS.includes(fld);
+        }
+        if (!fld.hasOwnProperty('field')) {
+          return false;
+        }
+        const searchFieldValue = fld as Record<string, string>;
+        return (
+          !this.checkMatch(sourceFiltersValues, searchFieldValue.field) &&
+          !META_FIELDS.includes(searchFieldValue.field)
+        );
+      });
+    }
+    // we need to get the list of fields from an index pattern
+    const fieldsToInclude = fields.filter((field: IndexPatternField) => {
+      return !this.checkMatch(sourceFiltersValues, field.name) && !META_FIELDS.includes(field.name);
+    });
+    return fieldsToInclude.map((fld: IndexPatternField) => {
+      const fieldToInclude: Record<string, string> = {
+        field: fld.name,
+      };
+      if (wildcardField && wildcardField.hasOwnProperty('include_unmapped')) {
+        fieldToInclude.include_unmapped = (wildcardField as Record<
+          string,
+          string
+        >).include_unmapped;
+      }
+      return fieldToInclude;
+    });
+  }
+
   private flatten() {
     const { getConfig } = this.dependencies;
     const searchRequest = this.mergeProps();
@@ -617,6 +673,7 @@ export class SearchSource {
         // if items that are in the docvalueFields are provided, we should
         // inject the format from the computed fields if one isn't given
         const docvaluesIndex = keyBy(filteredDocvalueFields, 'field');
+        body.fields = this.getFieldsWithoutSourceFilters(index, body.fields);
         body.fields = body.fields.map((fld: SearchFieldValue) => {
           const fieldName = getFieldName(fld);
           if (Object.keys(docvaluesIndex).includes(fieldName)) {

src/plugins/discover/public/application/helpers/update_search_source.ts

@@ -11,7 +11,6 @@ import { SAMPLE_SIZE_SETTING, SORT_DEFAULT_ORDER_SETTING } from '../../../common
 import { IndexPattern, ISearchSource } from '../../../../data/common/';
 import { SortOrder } from '../../saved_searches/types';
 import { DiscoverServices } from '../../build_services';
-import { getFieldListFromIndexPattern } from './get_field_list';
 
 /**
  * Helper function to update the given searchSource before fetching/sharing/persisting
@@ -49,8 +48,11 @@ export function updateSearchSource(
     .setField('filter', data.query.filterManager.getFilters());
   if (useNewFieldsApi) {
     searchSource.removeField('fieldsFromSource');
-    const fields = getFieldListFromIndexPattern(indexPattern, showUnmappedFields);
-    searchSource.setField('fields', fields);
+    const fields: Record<string, string> = { field: '*' };
+    if (showUnmappedFields) {
+      fields.include_unmapped = 'true';
+    }
+    searchSource.setField('fields', [fields]);
   } else {
     searchSource.removeField('fields');
     const fieldNames = indexPattern.fields.map((field) => field.name);