Implement nested support for histograms aggregations

Signed-off-by: Merwane Hamadi <[email protected]>

src/plugins/data/common/index_patterns/field.stub.ts

@@ -89,4 +89,30 @@ export const stubFields: IFieldType[] = [
     searchable: true,
     filterable: true,
   },
+  {
+    name: 'nested.field.example',
+    type: 'number',
+    esTypes: ['float'],
+    aggregatable: true,
+    searchable: true,
+    filterable: true,
+    subType: {
+      nested: {
+        path: 'nested.field.example',
+      },
+    },
+    // This spec is necessary to build the nested dsl.
+    spec: {
+      name: 'nested.field.example',
+      type: 'number',
+      esTypes: ['float'],
+      aggregatable: true,
+      searchable: true,
+      subType: {
+        nested: {
+          path: 'nested.field.example',
+        },
+      },
+    },
+  },
 ];

src/plugins/data/common/index_patterns/fields/types.ts

@@ -51,5 +51,6 @@ export interface IFieldType {
   subType?: IFieldSubType;
   displayName?: string;
   format?: any;
+  spec?: FieldSpec;
   toSpec?: (options?: { getFormatterForField?: IndexPattern['getFormatterForField'] }) => FieldSpec;
 }

src/plugins/data/common/search/aggs/agg_config.ts

@@ -252,8 +252,32 @@ export class AggConfig {
    * @return {void|Object} - if the config has a dsl representation, it is
    *                         returned, else undefined is returned
    */
-  toDsl(aggConfigs?: IAggConfigs) {
+  toDsl(aggConfigs?: IAggConfigs, prevNestedPath?: string) {
     if (this.type.hasNoDsl) return;
+    if (this.params.orderAgg) {
+      let reverseNested = false;
+      let nestedPath = this.params.orderAgg.params?.field;
+
+      if (prevNestedPath !== undefined) {
+        if (
+          nestedPath === undefined ||
+          (nestedPath !== prevNestedPath && prevNestedPath.startsWith(nestedPath))
+        ) {
+          reverseNested = true;
+        }
+      }
+
+      if (nestedPath !== undefined) {
+        if (nestedPath === prevNestedPath) {
+          nestedPath = undefined;
+        } else {
+          prevNestedPath = nestedPath;
+        }
+      }
+      this.params.orderAgg.nestedPath = nestedPath;
+      this.params.orderAgg.reverseNested = reverseNested;
+    }
+
     const output = this.write(aggConfigs) as any;
 
     const configDsl = {} as any;
@@ -278,6 +302,62 @@ export class AggConfig {
     return configDsl;
   }
 
+  toDslNested(
+    aggConfigs: IAggConfigs,
+    destination: Record<string, any>,
+    nestedPath: string,
+    reverseNested: boolean
+  ) {
+    let id = this.id;
+    let dsl = this.toDsl(aggConfigs, nestedPath);
+    const result = dsl; // save the original dsl to return later
+
+    if (this.params.countByParent) {
+      const countId = 'count_' + this.id;
+      const aggsKey = 'aggs';
+      let countAgg: { [index: string]: any } = {};
+
+      if (dsl.aggs) {
+        countAgg = dsl.aggs;
+      }
+      countAgg[countId] = {
+        reverse_nested: {},
+      };
+      dsl[aggsKey] = countAgg;
+    }
+
+    if (nestedPath || reverseNested) {
+      // save the current dsl as a sub-agg of the nested agg
+      const aggs: { [index: string]: any } = {};
+
+      aggs[id] = dsl;
+      if (reverseNested) {
+        // let reverseNestedDsl: IReverseNestedDsl;
+        const reverseNestedDsl: { [index: string]: any } = {};
+        // when reverse nesting, the path is optional
+        if (nestedPath) {
+          reverseNestedDsl.path = nestedPath;
+        }
+        id = 'nested_' + this.id;
+        dsl = {
+          reverse_nested: reverseNestedDsl,
+          aggs,
+        };
+      } else if (nestedPath) {
+        id = 'nested_' + this.id;
+        dsl = {
+          nested: {
+            path: nestedPath,
+          },
+          aggs,
+        };
+      }
+    }
+    // apply the change to the destination
+    destination[id] = dsl;
+    return result;
+  }
+
   /**
    * @returns Returns a serialized representation of an AggConfig.
    */

src/plugins/data/common/search/aggs/agg_configs.test.ts

@@ -6,6 +6,8 @@
  * compatible open source license.
  */
 
+import { stubIndexPatternWithNestedFields } from './../../index_patterns/index_pattern.stub';
+
 /*
  * Licensed to Elasticsearch B.V. under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -428,5 +430,35 @@ describe('AggConfigs', () => {
         '1-bucket>_count'
       );
     });
+    it('supports nested aggregations on histograms', () => {
+      const configStates = [
+        {
+          enabled: true,
+          id: '1',
+          type: 'count',
+          schema: 'metric',
+          params: {},
+        },
+        {
+          enabled: true,
+          id: '2',
+          type: 'histogram',
+          schema: 'segment',
+          params: {
+            field: 'nested.field.example',
+            extended_bound: {
+              max: '',
+              min: '',
+            },
+            has_extended_bounds: false,
+            interval: 'auto',
+            min_doc_count: false,
+          },
+        },
+      ];
+      const ac = new AggConfigs(indexPattern, configStates, { typesRegistry });
+      const topLevelDsl = ac.toDsl(true);
+      expect(topLevelDsl.nested_2.aggs[2].histogram.field).toContain('nested.field.example');
+    });
   });
 });

src/plugins/data/common/search/aggs/agg_configs.ts

@@ -175,6 +175,7 @@ export class AggConfigs {
     const dslTopLvl = {};
     let dslLvlCursor: Record<string, any>;
     let nestedMetrics: Array<{ config: AggConfig; dsl: Record<string, any> }> | [];
+    let prevNestedPath: string | undefined;
 
     if (hierarchical) {
       // collect all metrics, and filter out the ones that we won't be copying
@@ -192,20 +193,42 @@ export class AggConfigs {
     this.getRequestAggs()
       .filter((config: AggConfig) => !config.type.hasNoDsl)
       .forEach((config: AggConfig, i: number, list) => {
+        let nestedPath = config.params.field
+          ? config.params.field.spec?.subType?.nested?.path
+          : undefined;
+
         if (!dslLvlCursor) {
           // start at the top level
           dslLvlCursor = dslTopLvl;
-        } else {
-          const prevConfig: AggConfig = list[i - 1];
-          const prevDsl = dslLvlCursor[prevConfig.id];
+        }
 
-          // advance the cursor and nest under the previous agg, or
-          // put it on the same level if the previous agg doesn't accept
-          // sub aggs
-          dslLvlCursor = prevDsl.aggs || dslLvlCursor;
+        // nested field support
+        let reverseNested = false;
+
+        if (config.params.filters) {
+          config.params.filters.forEach(function findNestedPath(filter: { [index: string]: any }) {
+            nestedPath = filter.input.query.nested?.path;
+          });
         }
 
-        const dsl = (dslLvlCursor[config.id] = config.toDsl(this));
+        if (prevNestedPath !== undefined) {
+          if (
+            nestedPath === undefined ||
+            (nestedPath !== prevNestedPath && prevNestedPath.startsWith(nestedPath))
+          ) {
+            reverseNested = true;
+          }
+        }
+
+        if (nestedPath !== undefined) {
+          if (nestedPath === prevNestedPath) {
+            nestedPath = undefined;
+          } else {
+            prevNestedPath = nestedPath;
+          }
+        }
+
+        const dsl = config.toDslNested(config.aggConfigs, dslLvlCursor, nestedPath, reverseNested);
         let subAggs: any;
 
         parseParentAggs(dslLvlCursor, dsl);
@@ -216,7 +239,7 @@ export class AggConfigs {
         }
 
         if (subAggs && nestedMetrics) {
-          nestedMetrics.forEach((agg: any) => {
+          nestedMetrics.forEach((agg) => {
             subAggs[agg.config.id] = agg.dsl;
             // if a nested metric agg has parent aggs, we have to add them to every level of the tree
             // to make sure "bucket_path" references in the nested metric agg itself are still working
@@ -226,7 +249,17 @@ export class AggConfigs {
               });
             }
           });
+        } else {
+          if (dsl.aggs === undefined && !(config.type.type === 'metrics')) {
+            prevNestedPath = undefined;
+          }
         }
+
+        // advance the cursor and nest under the previous agg, or
+        // put it on the same level if the previous agg doesn't accept
+        // sub aggs
+        dslLvlCursor =
+          dsl.aggs || (nestedPath ? dslLvlCursor['nested_' + config.id].aggs : dslLvlCursor);
       });
 
     removeParentAggs(dslTopLvl);

src/plugins/data/common/search/aggs/param_types/field.ts

@@ -47,6 +47,7 @@ export type IFieldParamType = FieldParamType;
 export class FieldParamType extends BaseParamType {
   required = true;
   scriptable = true;
+  nestedAggregationsSupported = ['histogram'];
   filterFieldTypes: FieldTypes;
   onlyAggregatable: boolean;
 
@@ -123,10 +124,17 @@ export class FieldParamType extends BaseParamType {
   getAvailableFields = (aggConfig: IAggConfig) => {
     const fields = aggConfig.getIndexPattern().fields;
     const filteredFields = fields.filter((field: IndexPatternField) => {
-      const { onlyAggregatable, scriptable, filterFieldTypes } = this;
+      const { onlyAggregatable, scriptable, nestedAggregationsSupported, filterFieldTypes } = this;
+
+      let nestedFieldsAllowed = false;
+
+      if (nestedAggregationsSupported.includes(aggConfig.type?.name)) {
+        nestedFieldsAllowed = true;
+      }
 
       if (
-        (onlyAggregatable && (!field.aggregatable || isNestedField(field))) ||
+        (onlyAggregatable &&
+          (!field.aggregatable || (isNestedField(field) && !nestedFieldsAllowed))) ||
         (!scriptable && field.scripted)
       ) {
         return false;

src/plugins/data/common/search/tabify/tabify.ts

@@ -62,11 +62,19 @@ export function tabifyAggResponse(
     if (column) {
       const agg = column.aggConfig;
       const aggInfo = agg.write(aggs);
+      const nestedKey = 'nested_' + agg.id;
+
       aggScale *= aggInfo.metricScale || 1;
 
       switch (agg.type.type) {
         case AggGroupNames.Buckets:
-          const aggBucket = get(bucket, agg.id);
+          let prefix = '';
+
+          if (nestedKey in bucket) {
+            prefix = nestedKey + '.';
+          }
+
+          const aggBucket = get(bucket, prefix + agg.id);
           const tabifyBuckets = new TabifyBuckets(aggBucket, agg.params, respOpts?.timeRange);
 
           if (tabifyBuckets.length) {

src/plugins/vis_type_vislib/public/vislib/helpers/hierarchical/build_hierarchical_data.ts

@@ -90,7 +90,6 @@ export const buildHierarchicalData = (table: Table, { metric, buckets = [] }: Di
         const name = bucketFormatter.convert(row[bucketColumn.id]);
         const size = row[bucketValueColumn.id] as number;
         names[name] = name;
-
         let slice = dataLevel.find((dataLevelSlice) => dataLevelSlice.name === name);
         if (!slice) {
           slice = {