diff --git a/src/plugins/data/common/search/search_source/search_source.test.ts b/src/plugins/data/common/search/search_source/search_source.test.ts index 4c64a117f8cfe..9e889e85734ee 100644 --- a/src/plugins/data/common/search/search_source/search_source.test.ts +++ b/src/plugins/data/common/search/search_source/search_source.test.ts @@ -28,6 +28,7 @@ const mockSource2 = { excludes: ['bar-*'] }; const indexPattern = ({ title: 'foo', + fields: [{ name: 'foo-bar' }, { name: 'field1' }, { name: 'field2' }], getComputedFields, getSourceFiltering: () => mockSource, } as unknown) as IndexPattern; @@ -51,6 +52,11 @@ describe('SearchSource', () => { let searchSource: SearchSource; beforeEach(() => { + const getConfigMock = jest + .fn() + .mockImplementation((param) => param === 'metaFields' && ['_type', '_source']) + .mockName('getConfig'); + mockSearchMethod = jest .fn() .mockReturnValue( @@ -61,7 +67,7 @@ describe('SearchSource', () => { ); searchSourceDependencies = { - getConfig: jest.fn(), + getConfig: getConfigMock, search: mockSearchMethod, onResponse: (req, res) => res, legacy: { @@ -518,6 +524,54 @@ describe('SearchSource', () => { expect(request.script_fields).toEqual({ hello: {} }); }); + test('request all fields except the ones specified with source filters', async () => { + searchSource.setField('index', ({ + ...indexPattern, + getComputedFields: () => ({ + storedFields: [], + scriptFields: [], + docvalueFields: [], + }), + } as unknown) as IndexPattern); + searchSource.setField('fields', ['hello', 'foo']); + + const request = await searchSource.getSearchRequestBody(); + expect(request.fields).toEqual(['hello']); + }); + + test('request all fields from index pattern except the ones specified with source filters', async () => { + searchSource.setField('index', ({ + ...indexPattern, + getComputedFields: () => ({ + storedFields: [], + scriptFields: [], + docvalueFields: [], + }), + } as unknown) as IndexPattern); + searchSource.setField('fields', ['*']); + + const request = await searchSource.getSearchRequestBody(); + expect(request.fields).toEqual([{ field: 'field1' }, { field: 'field2' }]); + }); + + test('request all fields from index pattern except the ones specified with source filters with unmapped_fields option', async () => { + searchSource.setField('index', ({ + ...indexPattern, + getComputedFields: () => ({ + storedFields: [], + scriptFields: [], + docvalueFields: [], + }), + } as unknown) as IndexPattern); + searchSource.setField('fields', [{ field: '*', include_unmapped: 'true' }]); + + const request = await searchSource.getSearchRequestBody(); + expect(request.fields).toEqual([ + { field: 'field1', include_unmapped: 'true' }, + { field: 'field2', include_unmapped: 'true' }, + ]); + }); + test('returns all scripted fields when one fields entry is *', async () => { searchSource.setField('index', ({ ...indexPattern, @@ -836,5 +890,25 @@ describe('SearchSource', () => { expect(references[1].type).toEqual('index-pattern'); expect(JSON.parse(searchSourceJSON).filter[0].meta.indexRefName).toEqual(references[1].name); }); + + test('mvt geoshape layer test', async () => { + // @ts-expect-error TS won't like using this field name, but technically it's possible. + searchSource.setField('docvalue_fields', ['prop1']); + searchSource.setField('source', ['geometry']); + searchSource.setField('fieldsFromSource', ['geometry', 'prop1']); + searchSource.setField('index', ({ + ...indexPattern, + getSourceFiltering: () => ({ excludes: [] }), + getComputedFields: () => ({ + storedFields: ['*'], + scriptFields: {}, + docvalueFields: [], + }), + } as unknown) as IndexPattern); + const request = await searchSource.getSearchRequestBody(); + expect(request.stored_fields).toEqual(['geometry', 'prop1']); + expect(request.docvalue_fields).toEqual(['prop1']); + expect(request._source).toEqual(['geometry']); + }); }); }); diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index 1c1360414cb2e..8580fb7910735 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -59,12 +59,13 @@ */ import { setWith } from '@elastic/safer-lodash-set'; -import { uniqueId, keyBy, pick, difference, omit, isObject, isFunction } from 'lodash'; +import { uniqueId, keyBy, pick, difference, omit, isFunction, isEqual } from 'lodash'; import { map, switchMap, tap } from 'rxjs/operators'; import { defer, from } from 'rxjs'; +import { isObject } from 'rxjs/internal-compatibility'; import { normalizeSortRequest } from './normalize_sort_request'; import { fieldWildcardFilter } from '../../../../kibana_utils/common'; -import { IIndexPattern } from '../../index_patterns'; +import { IIndexPattern, IndexPattern, IndexPatternField } from '../../index_patterns'; import { ISearchGeneric, ISearchOptions } from '../..'; import type { ISearchSource, @@ -500,10 +501,53 @@ export class SearchSource { } } + private readonly getFieldName = (fld: string | Record): string => + typeof fld === 'string' ? fld : fld.field; + + private getFieldsWithoutSourceFilters( + index: IndexPattern | undefined, + bodyFields: SearchFieldValue[] + ) { + if (!index) { + return bodyFields; + } + const { fields } = index; + const sourceFilters = index.getSourceFiltering(); + if (!sourceFilters || sourceFilters.excludes?.length === 0 || bodyFields.length === 0) { + return bodyFields; + } + const metaFields = this.dependencies.getConfig(UI_SETTINGS.META_FIELDS); + const sourceFiltersValues = sourceFilters.excludes; + const wildcardField = bodyFields.find( + (el: SearchFieldValue) => el === '*' || (el as Record).field === '*' + ); + const filterSourceFields = (fieldName: string) => { + return ( + fieldName && + !sourceFiltersValues.some((sourceFilter) => fieldName.match(sourceFilter)) && + !metaFields.includes(fieldName) + ); + }; + if (!wildcardField) { + // we already have an explicit list of fields, so we just remove source filters from that list + return bodyFields.filter((fld: SearchFieldValue) => + filterSourceFields(this.getFieldName(fld)) + ); + } + // we need to get the list of fields from an index pattern + return fields + .filter((fld: IndexPatternField) => filterSourceFields(fld.name)) + .map((fld: IndexPatternField) => ({ + field: fld.name, + ...((wildcardField as Record)?.include_unmapped && { + include_unmapped: (wildcardField as Record).include_unmapped, + }), + })); + } + private flatten() { const { getConfig } = this.dependencies; const searchRequest = this.mergeProps(); - searchRequest.body = searchRequest.body || {}; const { body, index, query, filters, highlightAll } = searchRequest; searchRequest.indexType = this.getIndexType(index); @@ -517,10 +561,7 @@ export class SearchSource { storedFields: ['*'], runtimeFields: {}, }; - const fieldListProvided = !!body.fields; - const getFieldName = (fld: string | Record): string => - typeof fld === 'string' ? fld : fld.field; // set defaults let fieldsFromSource = searchRequest.fieldsFromSource || []; @@ -539,26 +580,22 @@ export class SearchSource { if (!body.hasOwnProperty('_source')) { body._source = sourceFilters; } - if (body._source.excludes) { - const filter = fieldWildcardFilter( - body._source.excludes, - getConfig(UI_SETTINGS.META_FIELDS) - ); - // also apply filters to provided fields & default docvalueFields - body.fields = body.fields.filter((fld: SearchFieldValue) => filter(getFieldName(fld))); - fieldsFromSource = fieldsFromSource.filter((fld: SearchFieldValue) => - filter(getFieldName(fld)) - ); - filteredDocvalueFields = filteredDocvalueFields.filter((fld: SearchFieldValue) => - filter(getFieldName(fld)) - ); - } + + const filter = fieldWildcardFilter(body._source.excludes, getConfig(UI_SETTINGS.META_FIELDS)); + // also apply filters to provided fields & default docvalueFields + body.fields = body.fields.filter((fld: SearchFieldValue) => filter(this.getFieldName(fld))); + fieldsFromSource = fieldsFromSource.filter((fld: SearchFieldValue) => + filter(this.getFieldName(fld)) + ); + filteredDocvalueFields = filteredDocvalueFields.filter((fld: SearchFieldValue) => + filter(this.getFieldName(fld)) + ); } // specific fields were provided, so we need to exclude any others if (fieldListProvided || fieldsFromSource.length) { const bodyFieldNames = body.fields.map((field: string | Record) => - getFieldName(field) + this.getFieldName(field) ); const uniqFieldNames = [...new Set([...bodyFieldNames, ...fieldsFromSource])]; @@ -579,17 +616,20 @@ export class SearchSource { const remainingFields = difference(uniqFieldNames, [ ...Object.keys(body.script_fields), ...Object.keys(body.runtime_mappings), - ]).filter(Boolean); + ]).filter((remainingField) => { + if (!remainingField) return false; + if (!body._source || !body._source.excludes) return true; + return !body._source.excludes.includes(remainingField); + }); - // only include unique values body.stored_fields = [...new Set(remainingFields)]; - + // only include unique values if (fieldsFromSource.length) { - // include remaining fields in _source - setWith(body, '_source.includes', remainingFields, (nsValue) => - isObject(nsValue) ? {} : nsValue - ); - + if (!isEqual(remainingFields, fieldsFromSource)) { + setWith(body, '_source.includes', remainingFields, (nsValue) => + isObject(nsValue) ? {} : nsValue + ); + } // if items that are in the docvalueFields are provided, we should // make sure those are added to the fields API unless they are // already set in docvalue_fields @@ -597,10 +637,10 @@ export class SearchSource { ...body.fields, ...filteredDocvalueFields.filter((fld: SearchFieldValue) => { return ( - fieldsFromSource.includes(getFieldName(fld)) && + fieldsFromSource.includes(this.getFieldName(fld)) && !(body.docvalue_fields || []) - .map((d: string | Record) => getFieldName(d)) - .includes(getFieldName(fld)) + .map((d: string | Record) => this.getFieldName(d)) + .includes(this.getFieldName(fld)) ); }), ]; @@ -614,20 +654,22 @@ export class SearchSource { // if items that are in the docvalueFields are provided, we should // inject the format from the computed fields if one isn't given const docvaluesIndex = keyBy(filteredDocvalueFields, 'field'); - body.fields = body.fields.map((fld: SearchFieldValue) => { - const fieldName = getFieldName(fld); - if (Object.keys(docvaluesIndex).includes(fieldName)) { - // either provide the field object from computed docvalues, - // or merge the user-provided field with the one in docvalues - return typeof fld === 'string' - ? docvaluesIndex[fld] - : { - ...docvaluesIndex[fieldName], - ...fld, - }; + body.fields = this.getFieldsWithoutSourceFilters(index, body.fields).map( + (fld: SearchFieldValue) => { + const fieldName = this.getFieldName(fld); + if (Object.keys(docvaluesIndex).includes(fieldName)) { + // either provide the field object from computed docvalues, + // or merge the user-provided field with the one in docvalues + return typeof fld === 'string' + ? docvaluesIndex[fld] + : { + ...docvaluesIndex[fieldName], + ...fld, + }; + } + return fld; } - return fld; - }); + ); } } else { body.fields = filteredDocvalueFields; diff --git a/x-pack/test/functional/apps/maps/mvt_scaling.js b/x-pack/test/functional/apps/maps/mvt_scaling.js index c7d711e9fbe3c..ba3cdf33ae24e 100644 --- a/x-pack/test/functional/apps/maps/mvt_scaling.js +++ b/x-pack/test/functional/apps/maps/mvt_scaling.js @@ -30,7 +30,7 @@ export default function ({ getPageObjects, getService }) { //Source should be correct expect(mapboxStyle.sources[VECTOR_SOURCE_ID].tiles[0]).to.equal( - '/api/maps/mvt/getTile?x={x}&y={y}&z={z}&geometryFieldName=geometry&index=geo_shapes*&requestBody=(_source:(includes:!(geometry,prop1)),docvalue_fields:!(prop1),query:(bool:(filter:!((match_all:())),must:!(),must_not:!(),should:!())),runtime_mappings:(),script_fields:(),size:10000,stored_fields:!(geometry,prop1))&geoFieldType=geo_shape' + '/api/maps/mvt/getTile?x={x}&y={y}&z={z}&geometryFieldName=geometry&index=geo_shapes*&requestBody=(_source:!(geometry),docvalue_fields:!(prop1),query:(bool:(filter:!((match_all:())),must:!(),must_not:!(),should:!())),runtime_mappings:(),script_fields:(),size:10000,stored_fields:!(geometry,prop1))&geoFieldType=geo_shape' ); //Should correctly load meta for style-rule (sigma is set to 1, opacity to 1)