From 8499dac8061af65eb49e9f06365c2e025426823c Mon Sep 17 00:00:00 2001 From: Devon Thomson Date: Mon, 23 Mar 2020 11:53:51 -0400 Subject: [PATCH] Support Histogram Data Type (#59387) Added the histogram field type to Kibana, to be used in the percentiles, percentiles ranks, and median aggregations. --- ...ugin-plugins-data-public.es_field_types.md | 1 + ...gin-plugins-data-public.kbn_field_types.md | 1 + ...ugin-plugins-data-server.es_field_types.md | 1 + ...gin-plugins-data-server.kbn_field_types.md | 1 + .../__snapshots__/field_editor.test.js.snap | 4 + .../kbn_field_types/kbn_field_types.test.ts | 1 + .../kbn_field_types_factory.ts | 5 + .../data/common/kbn_field_types/types.ts | 3 + src/plugins/data/public/public.api.md | 4 + .../public/search/aggs/metrics/cardinality.ts | 3 + .../data/public/search/aggs/metrics/median.ts | 2 +- .../search/aggs/metrics/percentile_ranks.ts | 2 +- .../public/search/aggs/metrics/percentiles.ts | 2 +- .../public/search/aggs/metrics/top_hit.ts | 4 +- src/plugins/data/server/server.api.md | 4 + .../test/functional/apps/visualize/index.ts | 1 + .../apps/visualize/precalculated_histogram.ts | 60 ++++++ .../pre_calculated_histogram/data.json | 197 ++++++++++++++++++ .../pre_calculated_histogram/mappings.json | 29 +++ 19 files changed, 321 insertions(+), 4 deletions(-) create mode 100644 x-pack/test/functional/apps/visualize/precalculated_histogram.ts create mode 100644 x-pack/test/functional/es_archives/pre_calculated_histogram/data.json create mode 100644 x-pack/test/functional/es_archives/pre_calculated_histogram/mappings.json diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.es_field_types.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.es_field_types.md index e7341caf7b3cd..c5e01715534d1 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.es_field_types.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.es_field_types.md @@ -30,6 +30,7 @@ export declare enum ES_FIELD_TYPES | GEO\_POINT | "geo_point" | | | GEO\_SHAPE | "geo_shape" | | | HALF\_FLOAT | "half_float" | | +| HISTOGRAM | "histogram" | | | INTEGER | "integer" | | | IP | "ip" | | | KEYWORD | "keyword" | | diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.kbn_field_types.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.kbn_field_types.md index e5ae8ffbd2877..30c3aa946c1ce 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.kbn_field_types.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.kbn_field_types.md @@ -23,6 +23,7 @@ export declare enum KBN_FIELD_TYPES | DATE | "date" | | | GEO\_POINT | "geo_point" | | | GEO\_SHAPE | "geo_shape" | | +| HISTOGRAM | "histogram" | | | IP | "ip" | | | MURMUR3 | "murmur3" | | | NESTED | "nested" | | diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.es_field_types.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.es_field_types.md index 81a7cbca77c48..d071955f4f522 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.es_field_types.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.es_field_types.md @@ -30,6 +30,7 @@ export declare enum ES_FIELD_TYPES | GEO\_POINT | "geo_point" | | | GEO\_SHAPE | "geo_shape" | | | HALF\_FLOAT | "half_float" | | +| HISTOGRAM | "histogram" | | | INTEGER | "integer" | | | IP | "ip" | | | KEYWORD | "keyword" | | diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.kbn_field_types.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.kbn_field_types.md index 40b81d2f6ac4d..a0a64190497c8 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.kbn_field_types.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.kbn_field_types.md @@ -23,6 +23,7 @@ export declare enum KBN_FIELD_TYPES | DATE | "date" | | | GEO\_POINT | "geo_point" | | | GEO\_SHAPE | "geo_shape" | | +| HISTOGRAM | "histogram" | | | IP | "ip" | | | MURMUR3 | "murmur3" | | | NESTED | "nested" | | diff --git a/src/legacy/ui/public/field_editor/__snapshots__/field_editor.test.js.snap b/src/legacy/ui/public/field_editor/__snapshots__/field_editor.test.js.snap index 6c454370f59f5..19d12f4bbbd4c 100644 --- a/src/legacy/ui/public/field_editor/__snapshots__/field_editor.test.js.snap +++ b/src/legacy/ui/public/field_editor/__snapshots__/field_editor.test.js.snap @@ -945,6 +945,10 @@ exports[`FieldEditor should show deprecated lang warning 1`] = ` "text": "_source", "value": "_source", }, + Object { + "text": "histogram", + "value": "histogram", + }, Object { "text": "conflict", "value": "conflict", diff --git a/src/plugins/data/common/kbn_field_types/kbn_field_types.test.ts b/src/plugins/data/common/kbn_field_types/kbn_field_types.test.ts index 09fc4555992a8..a3fe19fa9b2fc 100644 --- a/src/plugins/data/common/kbn_field_types/kbn_field_types.test.ts +++ b/src/plugins/data/common/kbn_field_types/kbn_field_types.test.ts @@ -87,6 +87,7 @@ describe('utils/kbn_field_types', () => { KBN_FIELD_TYPES.DATE, KBN_FIELD_TYPES.GEO_POINT, KBN_FIELD_TYPES.GEO_SHAPE, + KBN_FIELD_TYPES.HISTOGRAM, KBN_FIELD_TYPES.IP, KBN_FIELD_TYPES.MURMUR3, KBN_FIELD_TYPES.NESTED, diff --git a/src/plugins/data/common/kbn_field_types/kbn_field_types_factory.ts b/src/plugins/data/common/kbn_field_types/kbn_field_types_factory.ts index 192e8bc4f3727..cb9357eb9865e 100644 --- a/src/plugins/data/common/kbn_field_types/kbn_field_types_factory.ts +++ b/src/plugins/data/common/kbn_field_types/kbn_field_types_factory.ts @@ -95,6 +95,11 @@ export const createKbnFieldTypes = (): KbnFieldType[] => [ name: KBN_FIELD_TYPES._SOURCE, esTypes: [ES_FIELD_TYPES._SOURCE], }), + new KbnFieldType({ + name: KBN_FIELD_TYPES.HISTOGRAM, + filterable: true, + esTypes: [ES_FIELD_TYPES.HISTOGRAM], + }), new KbnFieldType({ name: KBN_FIELD_TYPES.CONFLICT, }), diff --git a/src/plugins/data/common/kbn_field_types/types.ts b/src/plugins/data/common/kbn_field_types/types.ts index 11c62e8f86dce..acd7a36b01fb3 100644 --- a/src/plugins/data/common/kbn_field_types/types.ts +++ b/src/plugins/data/common/kbn_field_types/types.ts @@ -59,6 +59,8 @@ export enum ES_FIELD_TYPES { ATTACHMENT = 'attachment', TOKEN_COUNT = 'token_count', MURMUR3 = 'murmur3', + + HISTOGRAM = 'histogram', } /** @public **/ @@ -77,4 +79,5 @@ export enum KBN_FIELD_TYPES { CONFLICT = 'conflict', OBJECT = 'object', NESTED = 'nested', + HISTOGRAM = 'histogram', } diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 848044217767e..8c4387e6a13cd 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -284,6 +284,8 @@ export enum ES_FIELD_TYPES { // (undocumented) HALF_FLOAT = "half_float", // (undocumented) + HISTOGRAM = "histogram", + // (undocumented) _ID = "_id", // (undocumented) _INDEX = "_index", @@ -1136,6 +1138,8 @@ export enum KBN_FIELD_TYPES { // (undocumented) GEO_SHAPE = "geo_shape", // (undocumented) + HISTOGRAM = "histogram", + // (undocumented) IP = "ip", // (undocumented) MURMUR3 = "murmur3", diff --git a/src/plugins/data/public/search/aggs/metrics/cardinality.ts b/src/plugins/data/public/search/aggs/metrics/cardinality.ts index aa41307b2a052..88cdf3175665e 100644 --- a/src/plugins/data/public/search/aggs/metrics/cardinality.ts +++ b/src/plugins/data/public/search/aggs/metrics/cardinality.ts @@ -45,6 +45,9 @@ export const cardinalityMetricAgg = new MetricAggType({ { name: 'field', type: 'field', + filterFieldTypes: Object.values(KBN_FIELD_TYPES).filter( + type => type !== KBN_FIELD_TYPES.HISTOGRAM + ), }, ], }); diff --git a/src/plugins/data/public/search/aggs/metrics/median.ts b/src/plugins/data/public/search/aggs/metrics/median.ts index f2636d52e3484..faa0694cd5312 100644 --- a/src/plugins/data/public/search/aggs/metrics/median.ts +++ b/src/plugins/data/public/search/aggs/metrics/median.ts @@ -40,7 +40,7 @@ export const medianMetricAgg = new MetricAggType({ { name: 'field', type: 'field', - filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.DATE], + filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.DATE, KBN_FIELD_TYPES.HISTOGRAM], write(agg, output) { output.params.field = agg.getParam('field').name; output.params.percents = [50]; diff --git a/src/plugins/data/public/search/aggs/metrics/percentile_ranks.ts b/src/plugins/data/public/search/aggs/metrics/percentile_ranks.ts index 71b1c1415d98e..7dc0f70ea7b80 100644 --- a/src/plugins/data/public/search/aggs/metrics/percentile_ranks.ts +++ b/src/plugins/data/public/search/aggs/metrics/percentile_ranks.ts @@ -59,7 +59,7 @@ export const percentileRanksMetricAgg = new MetricAggType({ { name: 'field', type: 'field', - filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.DATE], + filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.DATE, KBN_FIELD_TYPES.HISTOGRAM], }, { name: 'percents', diff --git a/src/plugins/data/public/search/aggs/metrics/top_hit.ts b/src/plugins/data/public/search/aggs/metrics/top_hit.ts index 738de6b62bccb..d0c668c577e62 100644 --- a/src/plugins/data/public/search/aggs/metrics/top_hit.ts +++ b/src/plugins/data/public/search/aggs/metrics/top_hit.ts @@ -60,7 +60,9 @@ export const topHitMetricAgg = new MetricAggType({ name: 'field', type: 'field', onlyAggregatable: false, - filterFieldTypes: '*', + filterFieldTypes: Object.values(KBN_FIELD_TYPES).filter( + type => type !== KBN_FIELD_TYPES.HISTOGRAM + ), write(agg, output) { const field = agg.getParam('field'); output.params = {}; diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index 627c7d9849b71..788ab90856405 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -176,6 +176,8 @@ export enum ES_FIELD_TYPES { // (undocumented) HALF_FLOAT = "half_float", // (undocumented) + HISTOGRAM = "histogram", + // (undocumented) _ID = "_id", // (undocumented) _INDEX = "_index", @@ -549,6 +551,8 @@ export enum KBN_FIELD_TYPES { // (undocumented) GEO_SHAPE = "geo_shape", // (undocumented) + HISTOGRAM = "histogram", + // (undocumented) IP = "ip", // (undocumented) MURMUR3 = "murmur3", diff --git a/x-pack/test/functional/apps/visualize/index.ts b/x-pack/test/functional/apps/visualize/index.ts index 264e2e807f0fb..60b5a0af52928 100644 --- a/x-pack/test/functional/apps/visualize/index.ts +++ b/x-pack/test/functional/apps/visualize/index.ts @@ -13,5 +13,6 @@ export default function visualize({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./feature_controls/visualize_security')); loadTestFile(require.resolve('./feature_controls/visualize_spaces')); loadTestFile(require.resolve('./hybrid_visualization')); + loadTestFile(require.resolve('./precalculated_histogram')); }); } diff --git a/x-pack/test/functional/apps/visualize/precalculated_histogram.ts b/x-pack/test/functional/apps/visualize/precalculated_histogram.ts new file mode 100644 index 0000000000000..5d362d29b640c --- /dev/null +++ b/x-pack/test/functional/apps/visualize/precalculated_histogram.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function({ getService, getPageObjects }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const PageObjects = getPageObjects(['common', 'visualize', 'discover', 'visChart', 'visEditor']); + const kibanaServer = getService('kibanaServer'); + const log = getService('log'); + + describe('pre_calculated_histogram', function() { + before(async function() { + log.debug('Starting pre_calculated_histogram before method'); + await esArchiver.load('pre_calculated_histogram'); + await kibanaServer.uiSettings.replace({ defaultIndex: 'test-histogram' }); + }); + + after(function() { + return esArchiver.unload('pre_calculated_histogram'); + }); + + const initHistogramBarChart = async () => { + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVerticalBarChart(); + await PageObjects.visualize.clickNewSearch('histogram-test'); + await PageObjects.visChart.waitForVisualization(); + }; + + const getFieldOptionsForAggregation = async (aggregation: string): Promise => { + await PageObjects.visEditor.clickBucket('Y-axis', 'metrics'); + await PageObjects.visEditor.selectAggregation(aggregation, 'metrics'); + const fieldValues = await PageObjects.visEditor.getField(); + return fieldValues; + }; + + it('appears correctly in discover', async function() { + await PageObjects.common.navigateToApp('discover'); + const rowData = await PageObjects.discover.getDocTableIndex(1); + expect(rowData.includes('"values": [ 0.3, 1, 3, 4.2, 4.8 ]')).to.be.ok(); + }); + + it('appears in the field options of a Percentiles aggregation', async function() { + await initHistogramBarChart(); + const fieldValues: string[] = await getFieldOptionsForAggregation('Percentiles'); + log.debug('Percentiles Fields = ' + fieldValues); + expect(fieldValues[0]).to.be('histogram-content'); + }); + + it('appears in the field options of a Percentile Ranks aggregation', async function() { + const fieldValues: string[] = await getFieldOptionsForAggregation('Percentile Ranks'); + log.debug('Percentile Ranks Fields = ' + fieldValues); + expect(fieldValues[0]).to.be('histogram-content'); + }); + }); +} diff --git a/x-pack/test/functional/es_archives/pre_calculated_histogram/data.json b/x-pack/test/functional/es_archives/pre_calculated_histogram/data.json new file mode 100644 index 0000000000000..cab1dbdf84483 --- /dev/null +++ b/x-pack/test/functional/es_archives/pre_calculated_histogram/data.json @@ -0,0 +1,197 @@ +{ + "type": "doc", + "value": { + "id": "index-pattern:histogram-test", + "index": ".kibana", + "source": { + "index-pattern": { + "title": "histogram-test", + "fields": "[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"histogram-content\",\"type\":\"histogram\",\"esTypes\":[\"histogram\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"histogram-title\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]" + }, + "type": "index-pattern" + } + } +} + +{ + "type": "doc", + "value": { + "id": "5e69404d93193e4074f0ec1a", + "index": "histogram-test", + "source": { + "histogram-title": "incididunt reprehenderit mollit", + "histogram-content": { + "values": [ + 0.3, + 1, + 3, + 4.2, + 4.8 + ], + "counts": [ + 237, + 170, + 33, + 149, + 241 + ] + } + } + } +} + +{ + "type": "doc", + "value": { + "id": "5e69408f2fc61f57fd5bc762", + "index": "histogram-test", + "source": { + "histogram-title": "culpa cillum ullamco", + "histogram-content": { + "values": [ + 0.5, + 1, + 1.2, + 1.3, + 2.8, + 3.9, + 4.3 + ], + "counts": [ + 113, + 197, + 20, + 66, + 20, + 39, + 178 + ] + } + } + } +} + +{ + "type": "doc", + "value": { + "id": "5e6940b979b57ad343114cc3", + "index": "histogram-test", + "source": { + "histogram-title": "enim veniam et", + "histogram-content": { + "values": [ + 3.7, + 4.2 + ], + "counts": [ + 227, + 141 + ] + } + } + } +} + +{ + "type": "doc", + "value": { + "id": "5e6940d3e95de786eeb7586d", + "index": "histogram-test", + "source": { + "histogram-title": "est incididunt sunt", + "histogram-content": { + "values": [ + 1.8, + 2.4, + 2.6, + 4.9 + ], + "counts": [ + 92, + 101, + 122, + 244 + ] + } + } + } +} + +{ + "type": "doc", + "value": { + "id": "5e694119fb2f956a822b93b9", + "index": "histogram-test", + "source": { + "histogram-title": "qui qui tempor", + "histogram-content": { + "values": [ + 0.5, + 2.1, + 2.7, + 3, + 3.2, + 3.5, + 4.2, + 5 + ], + "counts": [ + 210, + 168, + 182, + 181, + 97, + 164, + 77, + 2 + ] + } + } + } +} + +{ + "type": "doc", + "value": { + "id": "5e694145ad3c741aa12d6e8e", + "index": "histogram-test", + "source": { + "histogram-title": "ullamco nisi sunt", + "histogram-content": { + "values": [ + 1.7, + 4.5, + 4.8 + ], + "counts": [ + 74, + 146, + 141 + ] + } + } + } +} + +{ + "type": "doc", + "value": { + "id": "5e694159d909d9d99b5e12d1", + "index": "histogram-test", + "source": { + "histogram-title": "magna eu incididunt", + "histogram-content": { + "values": [ + 1, + 3.4, + 4.8 + ], + "counts": [ + 103, + 205, + 11 + ] + } + } + } +} diff --git a/x-pack/test/functional/es_archives/pre_calculated_histogram/mappings.json b/x-pack/test/functional/es_archives/pre_calculated_histogram/mappings.json new file mode 100644 index 0000000000000..f616daf9d5ccb --- /dev/null +++ b/x-pack/test/functional/es_archives/pre_calculated_histogram/mappings.json @@ -0,0 +1,29 @@ +{ + "type": "index", + "value": { + "aliases": {}, + "index": "histogram-test", + "mappings": { + "properties": { + "histogram-title": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "histogram-content": { + "type": "histogram" + } + } + }, + "settings": { + "index": { + "number_of_replicas": "0", + "number_of_shards": "1" + } + } + } +}