Support Histogram Data Type (elastic#59387)

Added the histogram field type to Kibana, to be used in the percentiles, percentiles ranks, and median aggregations.

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.es_field_types.md

@@ -30,6 +30,7 @@ export declare enum ES_FIELD_TYPES
 |  GEO\_POINT | <code>&quot;geo_point&quot;</code> |  |
 |  GEO\_SHAPE | <code>&quot;geo_shape&quot;</code> |  |
 |  HALF\_FLOAT | <code>&quot;half_float&quot;</code> |  |
+|  HISTOGRAM | <code>&quot;histogram&quot;</code> |  |
 |  INTEGER | <code>&quot;integer&quot;</code> |  |
 |  IP | <code>&quot;ip&quot;</code> |  |
 |  KEYWORD | <code>&quot;keyword&quot;</code> |  |

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.kbn_field_types.md

@@ -23,6 +23,7 @@ export declare enum KBN_FIELD_TYPES
 |  DATE | <code>&quot;date&quot;</code> |  |
 |  GEO\_POINT | <code>&quot;geo_point&quot;</code> |  |
 |  GEO\_SHAPE | <code>&quot;geo_shape&quot;</code> |  |
+|  HISTOGRAM | <code>&quot;histogram&quot;</code> |  |
 |  IP | <code>&quot;ip&quot;</code> |  |
 |  MURMUR3 | <code>&quot;murmur3&quot;</code> |  |
 |  NESTED | <code>&quot;nested&quot;</code> |  |

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.es_field_types.md

@@ -30,6 +30,7 @@ export declare enum ES_FIELD_TYPES
 |  GEO\_POINT | <code>&quot;geo_point&quot;</code> |  |
 |  GEO\_SHAPE | <code>&quot;geo_shape&quot;</code> |  |
 |  HALF\_FLOAT | <code>&quot;half_float&quot;</code> |  |
+|  HISTOGRAM | <code>&quot;histogram&quot;</code> |  |
 |  INTEGER | <code>&quot;integer&quot;</code> |  |
 |  IP | <code>&quot;ip&quot;</code> |  |
 |  KEYWORD | <code>&quot;keyword&quot;</code> |  |

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.kbn_field_types.md

@@ -23,6 +23,7 @@ export declare enum KBN_FIELD_TYPES
 |  DATE | <code>&quot;date&quot;</code> |  |
 |  GEO\_POINT | <code>&quot;geo_point&quot;</code> |  |
 |  GEO\_SHAPE | <code>&quot;geo_shape&quot;</code> |  |
+|  HISTOGRAM | <code>&quot;histogram&quot;</code> |  |
 |  IP | <code>&quot;ip&quot;</code> |  |
 |  MURMUR3 | <code>&quot;murmur3&quot;</code> |  |
 |  NESTED | <code>&quot;nested&quot;</code> |  |

src/plugins/data/common/kbn_field_types/kbn_field_types.test.ts

@@ -87,6 +87,7 @@ describe('utils/kbn_field_types', () => {
         KBN_FIELD_TYPES.DATE,
         KBN_FIELD_TYPES.GEO_POINT,
         KBN_FIELD_TYPES.GEO_SHAPE,
+        KBN_FIELD_TYPES.HISTOGRAM,
         KBN_FIELD_TYPES.IP,
         KBN_FIELD_TYPES.MURMUR3,
         KBN_FIELD_TYPES.NESTED,

src/plugins/data/common/kbn_field_types/kbn_field_types_factory.ts

@@ -95,6 +95,11 @@ export const createKbnFieldTypes = (): KbnFieldType[] => [
     name: KBN_FIELD_TYPES._SOURCE,
     esTypes: [ES_FIELD_TYPES._SOURCE],
   }),
+  new KbnFieldType({
+    name: KBN_FIELD_TYPES.HISTOGRAM,
+    filterable: true,
+    esTypes: [ES_FIELD_TYPES.HISTOGRAM],
+  }),
   new KbnFieldType({
     name: KBN_FIELD_TYPES.CONFLICT,
   }),

src/plugins/data/common/kbn_field_types/types.ts

@@ -59,6 +59,8 @@ export enum ES_FIELD_TYPES {
   ATTACHMENT = 'attachment',
   TOKEN_COUNT = 'token_count',
   MURMUR3 = 'murmur3',
+
+  HISTOGRAM = 'histogram',
 }
 
 /** @public **/
@@ -77,4 +79,5 @@ export enum KBN_FIELD_TYPES {
   CONFLICT = 'conflict',
   OBJECT = 'object',
   NESTED = 'nested',
+  HISTOGRAM = 'histogram',
 }

src/plugins/data/public/public.api.md

@@ -284,6 +284,8 @@ export enum ES_FIELD_TYPES {
     // (undocumented)
     HALF_FLOAT = "half_float",
     // (undocumented)
+    HISTOGRAM = "histogram",
+    // (undocumented)
     _ID = "_id",
     // (undocumented)
     _INDEX = "_index",
@@ -1136,6 +1138,8 @@ export enum KBN_FIELD_TYPES {
     // (undocumented)
     GEO_SHAPE = "geo_shape",
     // (undocumented)
+    HISTOGRAM = "histogram",
+    // (undocumented)
     IP = "ip",
     // (undocumented)
     MURMUR3 = "murmur3",

src/plugins/data/public/search/aggs/metrics/cardinality.ts

@@ -45,6 +45,9 @@ export const cardinalityMetricAgg = new MetricAggType({
     {
       name: 'field',
       type: 'field',
+      filterFieldTypes: Object.values(KBN_FIELD_TYPES).filter(
+        type => type !== KBN_FIELD_TYPES.HISTOGRAM
+      ),
     },
   ],
 });

src/plugins/data/public/search/aggs/metrics/median.ts

@@ -40,7 +40,7 @@ export const medianMetricAgg = new MetricAggType({
     {
       name: 'field',
       type: 'field',
-      filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.DATE],
+      filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.DATE, KBN_FIELD_TYPES.HISTOGRAM],
       write(agg, output) {
         output.params.field = agg.getParam('field').name;
         output.params.percents = [50];

src/plugins/data/public/search/aggs/metrics/percentile_ranks.ts

@@ -59,7 +59,7 @@ export const percentileRanksMetricAgg = new MetricAggType<IPercentileRanksAggCon
     {
       name: 'field',
       type: 'field',
-      filterFieldTypes: KBN_FIELD_TYPES.NUMBER,
+      filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.HISTOGRAM],
     },
     {
       name: 'values',

src/plugins/data/public/search/aggs/metrics/percentiles.ts

@@ -54,7 +54,7 @@ export const percentilesMetricAgg = new MetricAggType<IPercentileAggConfig>({
     {
       name: 'field',
       type: 'field',
-      filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.DATE],
+      filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.DATE, KBN_FIELD_TYPES.HISTOGRAM],
     },
     {
       name: 'percents',

src/plugins/data/public/search/aggs/metrics/top_hit.ts

@@ -60,7 +60,9 @@ export const topHitMetricAgg = new MetricAggType({
       name: 'field',
       type: 'field',
       onlyAggregatable: false,
-      filterFieldTypes: '*',
+      filterFieldTypes: Object.values(KBN_FIELD_TYPES).filter(
+        type => type !== KBN_FIELD_TYPES.HISTOGRAM
+      ),
       write(agg, output) {
         const field = agg.getParam('field');
         output.params = {};

src/plugins/data/server/server.api.md

@@ -176,6 +176,8 @@ export enum ES_FIELD_TYPES {
     // (undocumented)
     HALF_FLOAT = "half_float",
     // (undocumented)
+    HISTOGRAM = "histogram",
+    // (undocumented)
     _ID = "_id",
     // (undocumented)
     _INDEX = "_index",
@@ -549,6 +551,8 @@ export enum KBN_FIELD_TYPES {
     // (undocumented)
     GEO_SHAPE = "geo_shape",
     // (undocumented)
+    HISTOGRAM = "histogram",
+    // (undocumented)
     IP = "ip",
     // (undocumented)
     MURMUR3 = "murmur3",

x-pack/test/functional/apps/visualize/index.ts

@@ -13,5 +13,6 @@ export default function visualize({ loadTestFile }: FtrProviderContext) {
     loadTestFile(require.resolve('./feature_controls/visualize_security'));
     loadTestFile(require.resolve('./feature_controls/visualize_spaces'));
     loadTestFile(require.resolve('./hybrid_visualization'));
+    loadTestFile(require.resolve('./precalculated_histogram'));
   });
 }

x-pack/test/functional/apps/visualize/precalculated_histogram.ts

@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function({ getService, getPageObjects }: FtrProviderContext) {
+  const esArchiver = getService('esArchiver');
+  const PageObjects = getPageObjects(['common', 'visualize', 'discover', 'visChart', 'visEditor']);
+  const kibanaServer = getService('kibanaServer');
+  const log = getService('log');
+
+  describe('pre_calculated_histogram', function() {
+    before(async function() {
+      log.debug('Starting pre_calculated_histogram before method');
+      await esArchiver.load('pre_calculated_histogram');
+      await kibanaServer.uiSettings.replace({ defaultIndex: 'test-histogram' });
+    });
+
+    after(function() {
+      return esArchiver.unload('pre_calculated_histogram');
+    });
+
+    const initHistogramBarChart = async () => {
+      await PageObjects.visualize.navigateToNewVisualization();
+      await PageObjects.visualize.clickVerticalBarChart();
+      await PageObjects.visualize.clickNewSearch('histogram-test');
+      await PageObjects.visChart.waitForVisualization();
+    };
+
+    const getFieldOptionsForAggregation = async (aggregation: string): Promise<string[]> => {
+      await PageObjects.visEditor.clickBucket('Y-axis', 'metrics');
+      await PageObjects.visEditor.selectAggregation(aggregation, 'metrics');
+      const fieldValues = await PageObjects.visEditor.getField();
+      return fieldValues;
+    };
+
+    it('appears correctly in discover', async function() {
+      await PageObjects.common.navigateToApp('discover');
+      const rowData = await PageObjects.discover.getDocTableIndex(1);
+      expect(rowData.includes('"values": [ 0.3, 1, 3, 4.2, 4.8 ]')).to.be.ok();
+    });
+
+    it('appears in the field options of a Percentiles aggregation', async function() {
+      await initHistogramBarChart();
+      const fieldValues: string[] = await getFieldOptionsForAggregation('Percentiles');
+      log.debug('Percentiles Fields = ' + fieldValues);
+      expect(fieldValues[0]).to.be('histogram-content');
+    });
+
+    it('appears in the field options of a Percentile Ranks aggregation', async function() {
+      const fieldValues: string[] = await getFieldOptionsForAggregation('Percentile Ranks');
+      log.debug('Percentile Ranks Fields = ' + fieldValues);
+      expect(fieldValues[0]).to.be('histogram-content');
+    });
+  });
+}