Skip to content

Commit

Permalink
[Maps] get max_result_window and max_inner_result_window from index s…
Browse files Browse the repository at this point in the history
…ettings (elastic#53500)

* [Maps] pull ES_SIZE_LIMIT and top hits limit from index settings

* get fetch working

* get min values from indicies response

* use indexSettings.maxResultWindow in documents request size

* use max_inner_result_window to define top hits max

* update jest test

* update docs

* more docs changes for top hits

* fix line spacing

* Update docs/maps/maps-aggregations.asciidoc

Co-Authored-By: gchaps <[email protected]>

* Update docs/maps/vector-layer.asciidoc

Co-Authored-By: gchaps <[email protected]>

* add api integration test for indexSettings route

* eslint fixes

* review feedback

* display toast on first index settings fetch failure

* clean up

Co-authored-by: gchaps <[email protected]>
Co-authored-by: Elastic Machine <[email protected]>
  • Loading branch information
3 people committed Jan 2, 2020
1 parent cfec717 commit c5e9455
Show file tree
Hide file tree
Showing 16 changed files with 298 additions and 22 deletions.
1 change: 1 addition & 0 deletions docs/maps/maps-aggregations.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ To enable top hits:
. Set *Entity* to the field that identifies entities in your documents.
This field will be used in the terms aggregation to group your documents into entity buckets.
. Set *Documents per entity* to configure the maximum number of documents accumulated per entity.
This setting is limited to the `index.max_inner_result_window` index setting, which defaults to 100.

[role="screenshot"]
image::maps/images/top_hits.png[]
Expand Down
2 changes: 1 addition & 1 deletion docs/maps/vector-layer.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ See map.regionmap.* in <<settings>> for details.
*Documents*:: Vector data from a Kibana index pattern.
The index must contain at least one field mapped as {ref}/geo-point.html[geo_point] or {ref}/geo-shape.html[geo_shape].

NOTE: Document results are limited to the first 10000 matching documents.
NOTE: Document results are limited to the `index.max_result_window` index setting, which defaults to 10000.
Use <<maps-aggregations, aggregations>> to plot large data sets.

*Grid aggregation*:: Geospatial data grouped in grids with metrics for each gridded cell.
Expand Down
5 changes: 4 additions & 1 deletion x-pack/legacy/plugins/maps/common/constants.js
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ export const APP_ICON = 'gisApp';

export const MAP_APP_PATH = `app/${APP_ID}`;
export const GIS_API_PATH = `api/${APP_ID}`;
export const INDEX_SETTINGS_API_PATH = `${GIS_API_PATH}/indexSettings`;

export const MAP_BASE_URL = `/${MAP_APP_PATH}#/${MAP_SAVED_OBJECT_TYPE}`;

Expand Down Expand Up @@ -69,7 +70,9 @@ export const MAX_ZOOM = 24;

export const DECIMAL_DEGREES_PRECISION = 5; // meters precision
export const ZOOM_PRECISION = 2;
export const ES_SIZE_LIMIT = 10000;
export const DEFAULT_MAX_RESULT_WINDOW = 10000;
export const DEFAULT_MAX_INNER_RESULT_WINDOW = 100;
export const DEFAULT_MAX_BUCKETS_LIMIT = 10000;

export const FEATURE_ID_PROPERTY_NAME = '__kbn__feature_id__';
export const FEATURE_VISIBLE_PROPERTY_NAME = '__kbn_isvisibleduetojoin__';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,11 @@

import { AbstractVectorSource } from '../vector_source';
import React from 'react';
import { ES_GEO_FIELD_TYPE, GEOJSON_FILE, ES_SIZE_LIMIT } from '../../../../common/constants';
import {
ES_GEO_FIELD_TYPE,
GEOJSON_FILE,
DEFAULT_MAX_RESULT_WINDOW,
} from '../../../../common/constants';
import { ClientFileCreateSourceEditor } from './create_client_file_source_editor';
import { ESSearchSource } from '../es_search_source';
import uuid from 'uuid/v4';
Expand Down Expand Up @@ -82,7 +86,7 @@ export class GeojsonFileSource extends AbstractVectorSource {
addAndViewSource(null);
} else {
// Only turn on bounds filter for large doc counts
const filterByMapBounds = indexDataResp.docCount > ES_SIZE_LIMIT;
const filterByMapBounds = indexDataResp.docCount > DEFAULT_MAX_RESULT_WINDOW;
const source = new ESSearchSource(
{
id: uuid(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,11 @@ import { NoIndexPatternCallout } from '../../../components/no_index_pattern_call
import { FormattedMessage } from '@kbn/i18n/react';
import { i18n } from '@kbn/i18n';
import { kfetch } from 'ui/kfetch';
import { ES_GEO_FIELD_TYPE, GIS_API_PATH, ES_SIZE_LIMIT } from '../../../../common/constants';
import {
ES_GEO_FIELD_TYPE,
GIS_API_PATH,
DEFAULT_MAX_RESULT_WINDOW,
} from '../../../../common/constants';
import { DEFAULT_FILTER_BY_MAP_BOUNDS } from './constants';

import { npStart } from 'ui/new_platform';
Expand Down Expand Up @@ -96,7 +100,7 @@ export class CreateSourceEditor extends Component {
let indexHasSmallDocCount = false;
try {
const indexDocCount = await this.loadIndexDocCount(indexPattern.title);
indexHasSmallDocCount = indexDocCount <= ES_SIZE_LIMIT;
indexHasSmallDocCount = indexDocCount <= DEFAULT_MAX_RESULT_WINDOW;
} catch (error) {
// retrieving index count is a nice to have and is not essential
// do not interrupt user flow if unable to retrieve count
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,13 @@ import { UpdateSourceEditor } from './update_source_editor';
import {
ES_SEARCH,
ES_GEO_FIELD_TYPE,
ES_SIZE_LIMIT,
DEFAULT_MAX_BUCKETS_LIMIT,
SORT_ORDER,
} from '../../../../common/constants';
import { i18n } from '@kbn/i18n';
import { getDataSourceLabel } from '../../../../common/i18n_getters';
import { getSourceFields } from '../../../index_pattern_util';
import { loadIndexSettings } from './load_index_settings';

import { DEFAULT_FILTER_BY_MAP_BOUNDS } from './constants';
import { ESDocField } from '../../fields/es_doc_field';
Expand Down Expand Up @@ -267,8 +268,8 @@ export class ESSearchSource extends AbstractESSource {
entitySplit: {
terms: {
field: topHitsSplitField,
size: ES_SIZE_LIMIT,
shard_size: ES_SIZE_LIMIT,
size: DEFAULT_MAX_BUCKETS_LIMIT,
shard_size: DEFAULT_MAX_BUCKETS_LIMIT,
},
aggs: {
entityHits: {
Expand All @@ -290,7 +291,7 @@ export class ESSearchSource extends AbstractESSource {
const entityBuckets = _.get(resp, 'aggregations.entitySplit.buckets', []);
const totalEntities = _.get(resp, 'aggregations.totalEntities.value', 0);
// can not compare entityBuckets.length to totalEntities because totalEntities is an approximate
const areEntitiesTrimmed = entityBuckets.length >= ES_SIZE_LIMIT;
const areEntitiesTrimmed = entityBuckets.length >= DEFAULT_MAX_BUCKETS_LIMIT;
let areTopHitsTrimmed = false;
entityBuckets.forEach(entityBucket => {
const total = _.get(entityBucket, 'entityHits.hits.total', 0);
Expand All @@ -315,7 +316,7 @@ export class ESSearchSource extends AbstractESSource {

// searchFilters.fieldNames contains geo field and any fields needed for styling features
// Performs Elasticsearch search request being careful to pull back only required fields to minimize response size
async _getSearchHits(layerName, searchFilters, registerCancelCallback) {
async _getSearchHits(layerName, searchFilters, maxResultWindow, registerCancelCallback) {
const initialSearchContext = {
docvalue_fields: await this._getDateDocvalueFields(searchFilters.fieldNames),
};
Expand All @@ -331,7 +332,7 @@ export class ESSearchSource extends AbstractESSource {
);
searchSource = await this._makeSearchSource(
searchFilters,
ES_SIZE_LIMIT,
maxResultWindow,
initialSearchContext
);
searchSource.setField('source', false); // do not need anything from _source
Expand All @@ -340,7 +341,7 @@ export class ESSearchSource extends AbstractESSource {
// geo_shape fields do not support docvalue_fields yet, so still have to be pulled from _source
searchSource = await this._makeSearchSource(
searchFilters,
ES_SIZE_LIMIT,
maxResultWindow,
initialSearchContext
);
// Setting "fields" instead of "source: { includes: []}"
Expand Down Expand Up @@ -382,11 +383,19 @@ export class ESSearchSource extends AbstractESSource {
}

async getGeoJsonWithMeta(layerName, searchFilters, registerCancelCallback) {
const indexPattern = await this.getIndexPattern();

const indexSettings = await loadIndexSettings(indexPattern.title);

const { hits, meta } = this._isTopHits()
? await this._getTopHits(layerName, searchFilters, registerCancelCallback)
: await this._getSearchHits(layerName, searchFilters, registerCancelCallback);
: await this._getSearchHits(
layerName,
searchFilters,
indexSettings.maxResultWindow,
registerCancelCallback
);

const indexPattern = await this.getIndexPattern();
const unusedMetaFields = indexPattern.metaFields.filter(metaField => {
return !['_id', '_index'].includes(metaField);
});
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import {
DEFAULT_MAX_RESULT_WINDOW,
DEFAULT_MAX_INNER_RESULT_WINDOW,
INDEX_SETTINGS_API_PATH,
} from '../../../../common/constants';
import { kfetch } from 'ui/kfetch';
import { toastNotifications } from 'ui/notify';
import { i18n } from '@kbn/i18n';

let toastDisplayed = false;
const indexSettings = new Map();

export async function loadIndexSettings(indexPatternTitle) {
if (indexSettings.has(indexPatternTitle)) {
return indexSettings.get(indexPatternTitle);
}

const fetchPromise = fetchIndexSettings(indexPatternTitle);
indexSettings.set(indexPatternTitle, fetchPromise);
return fetchPromise;
}

async function fetchIndexSettings(indexPatternTitle) {
try {
const indexSettings = await kfetch({
pathname: `../${INDEX_SETTINGS_API_PATH}`,
query: {
indexPatternTitle,
},
});
return indexSettings;
} catch (err) {
const warningMsg = i18n.translate('xpack.maps.indexSettings.fetchErrorMsg', {
defaultMessage: `Unable to fetch index settings for index pattern '{indexPatternTitle}'.
Ensure you have '{viewIndexMetaRole}' role.`,
values: {
indexPatternTitle,
viewIndexMetaRole: 'view_index_metadata',
},
});
if (!toastDisplayed) {
// Only show toast for first failure to avoid flooding user with warnings
toastDisplayed = true;
toastNotifications.addWarning(warningMsg);
}
console.warn(warningMsg);
return {
maxResultWindow: DEFAULT_MAX_RESULT_WINDOW,
maxInnerResultWindow: DEFAULT_MAX_INNER_RESULT_WINDOW,
};
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,10 @@ import { indexPatternService } from '../../../kibana_services';
import { i18n } from '@kbn/i18n';
import { getTermsFields, getSourceFields } from '../../../index_pattern_util';
import { ValidatedRange } from '../../../components/validated_range';
import { SORT_ORDER } from '../../../../common/constants';
import { DEFAULT_MAX_INNER_RESULT_WINDOW, SORT_ORDER } from '../../../../common/constants';
import { ESDocField } from '../../fields/es_doc_field';
import { FormattedMessage } from '@kbn/i18n/react';
import { loadIndexSettings } from './load_index_settings';

export class UpdateSourceEditor extends Component {
static propTypes = {
Expand All @@ -43,17 +44,31 @@ export class UpdateSourceEditor extends Component {
sourceFields: null,
termFields: null,
sortFields: null,
maxInnerResultWindow: DEFAULT_MAX_INNER_RESULT_WINDOW,
};

componentDidMount() {
this._isMounted = true;
this.loadFields();
this.loadIndexSettings();
}

componentWillUnmount() {
this._isMounted = false;
}

async loadIndexSettings() {
try {
const indexPattern = await indexPatternService.get(this.props.indexPatternId);
const { maxInnerResultWindow } = await loadIndexSettings(indexPattern.title);
if (this._isMounted) {
this.setState({ maxInnerResultWindow });
}
} catch (err) {
return;
}
}

async loadFields() {
let indexPattern;
try {
Expand Down Expand Up @@ -149,7 +164,7 @@ export class UpdateSourceEditor extends Component {
>
<ValidatedRange
min={1}
max={100}
max={this.state.maxInnerResultWindow}
step={1}
value={this.props.topHitsSize}
onChange={this.onTopHitsSizeChange}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@

jest.mock('../../../kibana_services', () => ({}));

jest.mock('./load_index_settings', () => ({
loadIndexSettings: async () => {
return { maxInnerResultWindow: 100 };
},
}));

import React from 'react';
import { shallow } from 'enzyme';

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import _ from 'lodash';
import { Schemas } from 'ui/vis/editors/default/schemas';
import { AggConfigs } from 'ui/agg_types';
import { i18n } from '@kbn/i18n';
import { ES_SIZE_LIMIT, FIELD_ORIGIN, METRIC_TYPE } from '../../../common/constants';
import { DEFAULT_MAX_BUCKETS_LIMIT, FIELD_ORIGIN, METRIC_TYPE } from '../../../common/constants';
import { ESDocField } from '../fields/es_doc_field';
import { AbstractESAggSource } from './es_agg_source';

Expand Down Expand Up @@ -170,7 +170,7 @@ export class ESTermSource extends AbstractESAggSource {
schema: 'segment',
params: {
field: this._termField.getName(),
size: ES_SIZE_LIMIT,
size: DEFAULT_MAX_BUCKETS_LIMIT,
},
},
];
Expand Down
1 change: 0 additions & 1 deletion x-pack/legacy/plugins/maps/public/layers/vector_layer.js
Original file line number Diff line number Diff line change
Expand Up @@ -373,7 +373,6 @@ export class VectorLayer extends AbstractLayer {
const requestToken = Symbol(`layer-${this.getId()}-${SOURCE_DATA_ID_ORIGIN}`);
const searchFilters = this._getSearchFilters(dataFilters);
const prevDataRequest = this.getSourceDataRequest();

const canSkipFetch = await canSkipSourceUpdate({
source: this._source,
prevDataRequest,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import _ from 'lodash';
import { DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_INNER_RESULT_WINDOW } from '../../common/constants';

export function getIndexPatternSettings(indicesSettingsResp) {
let maxResultWindow = Infinity;
let maxInnerResultWindow = Infinity;
Object.values(indicesSettingsResp).forEach(indexSettings => {
const indexMaxResultWindow = _.get(
indexSettings,
'settings.index.max_result_window',
DEFAULT_MAX_RESULT_WINDOW
);
maxResultWindow = Math.min(maxResultWindow, indexMaxResultWindow);

const indexMaxInnerResultWindow = _.get(
indexSettings,
'settings.index.max_inner_result_window',
DEFAULT_MAX_INNER_RESULT_WINDOW
);
maxInnerResultWindow = Math.min(indexMaxInnerResultWindow, indexMaxResultWindow);
});

return { maxResultWindow, maxInnerResultWindow };
}
Loading

0 comments on commit c5e9455

Please sign in to comment.