Skip to content

Commit

Permalink
[Search Source] Fix retrieval of unmapped fields; Add field filters (#…
Browse files Browse the repository at this point in the history 
…89837) (#91396)

* [Search Source] Remove includes when retrieving fields from source

* Removing unusded imports

* Use exclusion filters to retrieve a list of fields

* Exclude _source from fields list

* Fix small check in getting the field list

* Fixing faulty import

* Filter logic

* Adding a unit test for maps use case

* Updating maps unit & functional test

* Add unit test

* Move logic for requesting a field list inside search_source

* Remove unnecessary mock

* Code cleanup as per PR comments

Co-authored-by: Kibana Machine <[email protected]>

Co-authored-by: Kibana Machine <[email protected]>
  • Loading branch information
@kibanamachine
Maja Grubic and kibanamachine authored Feb 15, 2021
1 parent 5ace3cb commit 60a4895
Show file tree
Hide file tree
Showing 3 changed files with 163 additions and 47 deletions.
76 changes: 75 additions & 1 deletion src/plugins/data/common/search/search_source/search_source.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ const mockSource2 = { excludes: ['bar-*'] };

const indexPattern = ({
title: 'foo',
fields: [{ name: 'foo-bar' }, { name: 'field1' }, { name: 'field2' }],
getComputedFields,
getSourceFiltering: () => mockSource,
} as unknown) as IndexPattern;
Expand All @@ -51,6 +52,11 @@ describe('SearchSource', () => {
let searchSource: SearchSource;

beforeEach(() => {
const getConfigMock = jest
.fn()
.mockImplementation((param) => param === 'metaFields' && ['_type', '_source'])
.mockName('getConfig');

mockSearchMethod = jest
.fn()
.mockReturnValue(
Expand All @@ -61,7 +67,7 @@ describe('SearchSource', () => {
);

searchSourceDependencies = {
getConfig: jest.fn(),
getConfig: getConfigMock,
search: mockSearchMethod,
onResponse: (req, res) => res,
legacy: {
Expand Down Expand Up @@ -518,6 +524,54 @@ describe('SearchSource', () => {
expect(request.script_fields).toEqual({ hello: {} });
});

test('request all fields except the ones specified with source filters', async () => {
searchSource.setField('index', ({
...indexPattern,
getComputedFields: () => ({
storedFields: [],
scriptFields: [],
docvalueFields: [],
}),
} as unknown) as IndexPattern);
searchSource.setField('fields', ['hello', 'foo']);

const request = await searchSource.getSearchRequestBody();
expect(request.fields).toEqual(['hello']);
});

test('request all fields from index pattern except the ones specified with source filters', async () => {
searchSource.setField('index', ({
...indexPattern,
getComputedFields: () => ({
storedFields: [],
scriptFields: [],
docvalueFields: [],
}),
} as unknown) as IndexPattern);
searchSource.setField('fields', ['*']);

const request = await searchSource.getSearchRequestBody();
expect(request.fields).toEqual([{ field: 'field1' }, { field: 'field2' }]);
});

test('request all fields from index pattern except the ones specified with source filters with unmapped_fields option', async () => {
searchSource.setField('index', ({
...indexPattern,
getComputedFields: () => ({
storedFields: [],
scriptFields: [],
docvalueFields: [],
}),
} as unknown) as IndexPattern);
searchSource.setField('fields', [{ field: '*', include_unmapped: 'true' }]);

const request = await searchSource.getSearchRequestBody();
expect(request.fields).toEqual([
{ field: 'field1', include_unmapped: 'true' },
{ field: 'field2', include_unmapped: 'true' },
]);
});

test('returns all scripted fields when one fields entry is *', async () => {
searchSource.setField('index', ({
...indexPattern,
Expand Down Expand Up @@ -836,5 +890,25 @@ describe('SearchSource', () => {
expect(references[1].type).toEqual('index-pattern');
expect(JSON.parse(searchSourceJSON).filter[0].meta.indexRefName).toEqual(references[1].name);
});

test('mvt geoshape layer test', async () => {
// @ts-expect-error TS won't like using this field name, but technically it's possible.
searchSource.setField('docvalue_fields', ['prop1']);
searchSource.setField('source', ['geometry']);
searchSource.setField('fieldsFromSource', ['geometry', 'prop1']);
searchSource.setField('index', ({
...indexPattern,
getSourceFiltering: () => ({ excludes: [] }),
getComputedFields: () => ({
storedFields: ['*'],
scriptFields: {},
docvalueFields: [],
}),
} as unknown) as IndexPattern);
const request = await searchSource.getSearchRequestBody();
expect(request.stored_fields).toEqual(['geometry', 'prop1']);
expect(request.docvalue_fields).toEqual(['prop1']);
expect(request._source).toEqual(['geometry']);
});
});
});
132 changes: 87 additions & 45 deletions src/plugins/data/common/search/search_source/search_source.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,12 +59,13 @@
*/

import { setWith } from '@elastic/safer-lodash-set';
import { uniqueId, keyBy, pick, difference, omit, isObject, isFunction } from 'lodash';
import { uniqueId, keyBy, pick, difference, omit, isFunction, isEqual } from 'lodash';
import { map, switchMap, tap } from 'rxjs/operators';
import { defer, from } from 'rxjs';
import { isObject } from 'rxjs/internal-compatibility';
import { normalizeSortRequest } from './normalize_sort_request';
import { fieldWildcardFilter } from '../../../../kibana_utils/common';
import { IIndexPattern } from '../../index_patterns';
import { IIndexPattern, IndexPattern, IndexPatternField } from '../../index_patterns';
import { ISearchGeneric, ISearchOptions } from '../..';
import type {
ISearchSource,
Expand Down Expand Up @@ -500,10 +501,53 @@ export class SearchSource {
}
}

private readonly getFieldName = (fld: string | Record<string, any>): string =>
typeof fld === 'string' ? fld : fld.field;

private getFieldsWithoutSourceFilters(
index: IndexPattern | undefined,
bodyFields: SearchFieldValue[]
) {
if (!index) {
return bodyFields;
}
const { fields } = index;
const sourceFilters = index.getSourceFiltering();
if (!sourceFilters || sourceFilters.excludes?.length === 0 || bodyFields.length === 0) {
return bodyFields;
}
const metaFields = this.dependencies.getConfig(UI_SETTINGS.META_FIELDS);
const sourceFiltersValues = sourceFilters.excludes;
const wildcardField = bodyFields.find(
(el: SearchFieldValue) => el === '*' || (el as Record<string, string>).field === '*'
);
const filterSourceFields = (fieldName: string) => {
return (
fieldName &&
!sourceFiltersValues.some((sourceFilter) => fieldName.match(sourceFilter)) &&
!metaFields.includes(fieldName)
);
};
if (!wildcardField) {
// we already have an explicit list of fields, so we just remove source filters from that list
return bodyFields.filter((fld: SearchFieldValue) =>
filterSourceFields(this.getFieldName(fld))
);
}
// we need to get the list of fields from an index pattern
return fields
.filter((fld: IndexPatternField) => filterSourceFields(fld.name))
.map((fld: IndexPatternField) => ({
field: fld.name,
...((wildcardField as Record<string, string>)?.include_unmapped && {
include_unmapped: (wildcardField as Record<string, string>).include_unmapped,
}),
}));
}

private flatten() {
const { getConfig } = this.dependencies;
const searchRequest = this.mergeProps();

searchRequest.body = searchRequest.body || {};
const { body, index, query, filters, highlightAll } = searchRequest;
searchRequest.indexType = this.getIndexType(index);
Expand All @@ -517,10 +561,7 @@ export class SearchSource {
storedFields: ['*'],
runtimeFields: {},
};

const fieldListProvided = !!body.fields;
const getFieldName = (fld: string | Record<string, any>): string =>
typeof fld === 'string' ? fld : fld.field;

// set defaults
let fieldsFromSource = searchRequest.fieldsFromSource || [];
Expand All @@ -539,26 +580,22 @@ export class SearchSource {
if (!body.hasOwnProperty('_source')) {
body._source = sourceFilters;
}
if (body._source.excludes) {
const filter = fieldWildcardFilter(
body._source.excludes,
getConfig(UI_SETTINGS.META_FIELDS)
);
// also apply filters to provided fields & default docvalueFields
body.fields = body.fields.filter((fld: SearchFieldValue) => filter(getFieldName(fld)));
fieldsFromSource = fieldsFromSource.filter((fld: SearchFieldValue) =>
filter(getFieldName(fld))
);
filteredDocvalueFields = filteredDocvalueFields.filter((fld: SearchFieldValue) =>
filter(getFieldName(fld))
);
}

const filter = fieldWildcardFilter(body._source.excludes, getConfig(UI_SETTINGS.META_FIELDS));
// also apply filters to provided fields & default docvalueFields
body.fields = body.fields.filter((fld: SearchFieldValue) => filter(this.getFieldName(fld)));
fieldsFromSource = fieldsFromSource.filter((fld: SearchFieldValue) =>
filter(this.getFieldName(fld))
);
filteredDocvalueFields = filteredDocvalueFields.filter((fld: SearchFieldValue) =>
filter(this.getFieldName(fld))
);
}

// specific fields were provided, so we need to exclude any others
if (fieldListProvided || fieldsFromSource.length) {
const bodyFieldNames = body.fields.map((field: string | Record<string, any>) =>
getFieldName(field)
this.getFieldName(field)
);
const uniqFieldNames = [...new Set([...bodyFieldNames, ...fieldsFromSource])];

Expand All @@ -579,28 +616,31 @@ export class SearchSource {
const remainingFields = difference(uniqFieldNames, [
...Object.keys(body.script_fields),
...Object.keys(body.runtime_mappings),
]).filter(Boolean);
]).filter((remainingField) => {
if (!remainingField) return false;
if (!body._source || !body._source.excludes) return true;
return !body._source.excludes.includes(remainingField);
});

// only include unique values
body.stored_fields = [...new Set(remainingFields)];

// only include unique values
if (fieldsFromSource.length) {
// include remaining fields in _source
setWith(body, '_source.includes', remainingFields, (nsValue) =>
isObject(nsValue) ? {} : nsValue
);

if (!isEqual(remainingFields, fieldsFromSource)) {
setWith(body, '_source.includes', remainingFields, (nsValue) =>
isObject(nsValue) ? {} : nsValue
);
}
// if items that are in the docvalueFields are provided, we should
// make sure those are added to the fields API unless they are
// already set in docvalue_fields
body.fields = [
...body.fields,
...filteredDocvalueFields.filter((fld: SearchFieldValue) => {
return (
fieldsFromSource.includes(getFieldName(fld)) &&
fieldsFromSource.includes(this.getFieldName(fld)) &&
!(body.docvalue_fields || [])
.map((d: string | Record<string, any>) => getFieldName(d))
.includes(getFieldName(fld))
.map((d: string | Record<string, any>) => this.getFieldName(d))
.includes(this.getFieldName(fld))
);
}),
];
Expand All @@ -614,20 +654,22 @@ export class SearchSource {
// if items that are in the docvalueFields are provided, we should
// inject the format from the computed fields if one isn't given
const docvaluesIndex = keyBy(filteredDocvalueFields, 'field');
body.fields = body.fields.map((fld: SearchFieldValue) => {
const fieldName = getFieldName(fld);
if (Object.keys(docvaluesIndex).includes(fieldName)) {
// either provide the field object from computed docvalues,
// or merge the user-provided field with the one in docvalues
return typeof fld === 'string'
? docvaluesIndex[fld]
: {
...docvaluesIndex[fieldName],
...fld,
};
body.fields = this.getFieldsWithoutSourceFilters(index, body.fields).map(
(fld: SearchFieldValue) => {
const fieldName = this.getFieldName(fld);
if (Object.keys(docvaluesIndex).includes(fieldName)) {
// either provide the field object from computed docvalues,
// or merge the user-provided field with the one in docvalues
return typeof fld === 'string'
? docvaluesIndex[fld]
: {
...docvaluesIndex[fieldName],
...fld,
};
}
return fld;
}
return fld;
});
);
}
} else {
body.fields = filteredDocvalueFields;
Expand Down
2 changes: 1 addition & 1 deletion x-pack/test/functional/apps/maps/mvt_scaling.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ export default function ({ getPageObjects, getService }) {

//Source should be correct
expect(mapboxStyle.sources[VECTOR_SOURCE_ID].tiles[0]).to.equal(
'/api/maps/mvt/getTile?x={x}&y={y}&z={z}&geometryFieldName=geometry&index=geo_shapes*&requestBody=(_source:(includes:!(geometry,prop1)),docvalue_fields:!(prop1),query:(bool:(filter:!((match_all:())),must:!(),must_not:!(),should:!())),runtime_mappings:(),script_fields:(),size:10000,stored_fields:!(geometry,prop1))&geoFieldType=geo_shape'
'/api/maps/mvt/getTile?x={x}&y={y}&z={z}&geometryFieldName=geometry&index=geo_shapes*&requestBody=(_source:!(geometry),docvalue_fields:!(prop1),query:(bool:(filter:!((match_all:())),must:!(),must_not:!(),should:!())),runtime_mappings:(),script_fields:(),size:10000,stored_fields:!(geometry,prop1))&geoFieldType=geo_shape'
);

//Should correctly load meta for style-rule (sigma is set to 1, opacity to 1)
Expand Down

0 comments on commit 60a4895

Please sign in to comment.