[Search Source] Fix retrieval of unmapped fields; Add field filters

src/plugins/data/common/search/search_source/search_source.ts

@@ -58,10 +58,10 @@
  *    `appSearchSource`.
  */
 
-import { setWith } from '@elastic/safer-lodash-set';
-import { uniqueId, keyBy, pick, difference, omit, isObject, isFunction } from 'lodash';
+import { uniqueId, keyBy, pick, difference, omit, isFunction, setWith, isEqual } from 'lodash';
 import { map, switchMap, tap } from 'rxjs/operators';
 import { defer, from } from 'rxjs';
+import { isObject } from 'rxjs/internal-compatibility';
 import { normalizeSortRequest } from './normalize_sort_request';
 import { fieldWildcardFilter } from '../../../../kibana_utils/common';
 import { IIndexPattern } from '../../index_patterns';
@@ -503,7 +503,6 @@ export class SearchSource {
   private flatten() {
     const { getConfig } = this.dependencies;
     const searchRequest = this.mergeProps();
-
     searchRequest.body = searchRequest.body || {};
     const { body, index, query, filters, highlightAll } = searchRequest;
     searchRequest.indexType = this.getIndexType(index);
@@ -539,7 +538,7 @@ export class SearchSource {
       if (!body.hasOwnProperty('_source')) {
         body._source = sourceFilters;
       }
-      if (body._source.excludes) {
+      if (body._source && body._source.excludes) {
         const filter = fieldWildcardFilter(
           body._source.excludes,
           getConfig(UI_SETTINGS.META_FIELDS)
@@ -576,20 +575,26 @@ export class SearchSource {
 
       // request the remaining fields from stored_fields just in case, since the
       // fields API does not handle stored fields
-      const remainingFields = difference(uniqFieldNames, [
+      let remainingFields = difference(uniqFieldNames, [
         ...Object.keys(body.script_fields),
         ...Object.keys(body.runtime_mappings),
       ]).filter(Boolean);
 
-      // only include unique values
-      body.stored_fields = [...new Set(remainingFields)];
+      if (body._source && body._source.excludes) {
+        // TODO: optimize complexity
+        remainingFields = remainingFields.filter((remainingField) => {
+          return !body._source.excludes.includes(remainingField);
+        });
+      }
 
+      body.stored_fields = [...new Set(remainingFields)];
+      // only include unique values
       if (fieldsFromSource.length) {
-        // include remaining fields in _source
-        setWith(body, '_source.includes', remainingFields, (nsValue) =>
-          isObject(nsValue) ? {} : nsValue
-        );
-
+        if (!isEqual(remainingFields, fieldsFromSource)) {
+          setWith(body, '_source.includes', remainingFields, (nsValue) =>
+            isObject(nsValue) ? {} : nsValue
+          );
+        }
         // if items that are in the docvalueFields are provided, we should
         // make sure those are added to the fields API unless they are
         // already set in docvalue_fields

src/plugins/discover/public/application/helpers/get_field_list.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * and the Server Side Public License, v 1; you may not use this file except in
+ * compliance with, at your election, the Elastic License or the Server Side
+ * Public License, v 1.
+ */
+import { IndexPattern } from '../../../../data/common/index_patterns/index_patterns';
+
+const META_FIELDS = ['_type', '_source'];
+
+export function getFieldListFromIndexPattern(indexPattern: IndexPattern) {
+  const { sourceFilters, fields } = indexPattern;
+  if (!sourceFilters || sourceFilters.length === 0) {
+    return ['*'];
+  }
+  const sourceFiltersValues = sourceFilters.map((sourceFilter) => sourceFilter.value);
+  const fieldsToInclude = fields.filter((field) => {
+    return !sourceFiltersValues.includes(field.name) && !META_FIELDS.includes(field.name);
+  });
+  return fieldsToInclude.map((field) => field.name);
+}

src/plugins/discover/public/application/helpers/update_search_source.ts

@@ -11,6 +11,7 @@ import { SAMPLE_SIZE_SETTING, SORT_DEFAULT_ORDER_SETTING } from '../../../common
 import { IndexPattern, ISearchSource } from '../../../../data/common/';
 import { SortOrder } from '../../saved_searches/types';
 import { DiscoverServices } from '../../build_services';
+import { getFieldListFromIndexPattern } from './get_field_list';
 
 /**
  * Helper function to update the given searchSource before fetching/sharing/persisting
@@ -46,7 +47,8 @@ export function updateSearchSource(
     .setField('filter', data.query.filterManager.getFilters());
   if (useNewFieldsApi) {
     searchSource.removeField('fieldsFromSource');
-    searchSource.setField('fields', ['*']);
+    const fields = getFieldListFromIndexPattern(indexPattern);
+    searchSource.setField('fields', fields);
   } else {
     searchSource.removeField('fields');
     const fieldNames = indexPattern.fields.map((field) => field.name);