Skip to content

Commit

Permalink
[ML] Transforms: Filter aggregation support (elastic#67591)
Browse files Browse the repository at this point in the history 
* [ML] WIP filter support

* [ML] value selector

* [ML] only supported filter aggs as options

* [ML] WIP apply config

* [ML] fix form persistence

* [ML] refactor

* [ML] support clone

* [ML] validation, get es config

* [ML] support "exists", fixes for the term form, validation

* [ML] fix ts issues

* [ML] don't perform request on adding incomplete agg

* [ML] basic range number support

* [ML] filter bool agg support

* [ML] functional tests

* [ML] getAggConfigFromEsAgg tests

* [ML] fix unit tests

* [ML] agg name update on config change, add unit tests

* [ML] update snapshot

* [ML] range selector enhancements

* [ML] improve types

* [ML] update step for range selector to support float numbers

* [ML] range validation

* [ML] term selector improvements

* [ML] fix switch between advanced editor

* [ML] prefix test ids

* [ML] support helper text for aggs item
  • Loading branch information
@darnautov
darnautov committed Jun 4, 2020
1 parent 7c06af6 commit 9a818de
Show file tree
Hide file tree
Showing 28 changed files with 1,401 additions and 141 deletions.
2 changes: 2 additions & 0 deletions x-pack/plugins/ml/common/util/validators.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ export function requiredValidator() {
};
}

export type ValidationResult = object | null;

export function memoryInputValidator(allowedUnits = ALLOWED_DATA_UNITS) {
return (value: any) => {
if (typeof value !== 'string' || value === '') {
Expand Down
47 changes: 47 additions & 0 deletions x-pack/plugins/transform/public/app/common/pivot_aggs.test.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import { getAggConfigFromEsAgg } from './pivot_aggs';
import {
FilterAggForm,
FilterTermForm,
} from '../sections/create_transform/components/step_define/common/filter_agg/components';

describe('getAggConfigFromEsAgg', () => {
test('should throw an error for unsupported agg', () => {
expect(() => getAggConfigFromEsAgg({ terms: {} }, 'test')).toThrowError();
});

test('should return a common config if the agg does not have a custom config defined', () => {
expect(getAggConfigFromEsAgg({ avg: { field: 'region' } }, 'test_1')).toEqual({
agg: 'avg',
aggName: 'test_1',
dropDownName: 'test_1',
field: 'region',
});
});

test('should return a custom config for recognized aggregation type', () => {
expect(
getAggConfigFromEsAgg({ filter: { term: { region: 'sa-west-1' } } }, 'test_2')
).toMatchObject({
agg: 'filter',
aggName: 'test_2',
dropDownName: 'test_2',
field: 'region',
AggFormComponent: FilterAggForm,
aggConfig: {
filterAgg: 'term',
aggTypeConfig: {
FilterAggFormComponent: FilterTermForm,
filterAggConfig: {
value: 'sa-west-1',
},
},
},
});
});
});
158 changes: 130 additions & 28 deletions x-pack/plugins/transform/public/app/common/pivot_aggs.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,36 +4,51 @@
* you may not use this file except in compliance with the Elastic License.
*/

import { FC } from 'react';
import { Dictionary } from '../../../common/types/common';
import { KBN_FIELD_TYPES } from '../../../../../../src/plugins/data/common';

import { AggName } from './aggregations';
import { EsFieldName } from './fields';
import { getAggFormConfig } from '../sections/create_transform/components/step_define/common/get_agg_form_config';
import { PivotAggsConfigFilter } from '../sections/create_transform/components/step_define/common/filter_agg/types';

export enum PIVOT_SUPPORTED_AGGS {
AVG = 'avg',
CARDINALITY = 'cardinality',
MAX = 'max',
MIN = 'min',
PERCENTILES = 'percentiles',
SUM = 'sum',
VALUE_COUNT = 'value_count',
export type PivotSupportedAggs = typeof PIVOT_SUPPORTED_AGGS[keyof typeof PIVOT_SUPPORTED_AGGS];

export function isPivotSupportedAggs(arg: any): arg is PivotSupportedAggs {
return Object.values(PIVOT_SUPPORTED_AGGS).includes(arg);
}

export const PIVOT_SUPPORTED_AGGS = {
AVG: 'avg',
CARDINALITY: 'cardinality',
MAX: 'max',
MIN: 'min',
PERCENTILES: 'percentiles',
SUM: 'sum',
VALUE_COUNT: 'value_count',
FILTER: 'filter',
} as const;

export const PERCENTILES_AGG_DEFAULT_PERCENTS = [1, 5, 25, 50, 75, 95, 99];

export const pivotAggsFieldSupport = {
[KBN_FIELD_TYPES.ATTACHMENT]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
[KBN_FIELD_TYPES.BOOLEAN]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
[KBN_FIELD_TYPES.ATTACHMENT]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
[KBN_FIELD_TYPES.BOOLEAN]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
[KBN_FIELD_TYPES.DATE]: [
PIVOT_SUPPORTED_AGGS.MAX,
PIVOT_SUPPORTED_AGGS.MIN,
PIVOT_SUPPORTED_AGGS.VALUE_COUNT,
PIVOT_SUPPORTED_AGGS.FILTER,
],
[KBN_FIELD_TYPES.GEO_POINT]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
[KBN_FIELD_TYPES.GEO_SHAPE]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
[KBN_FIELD_TYPES.IP]: [
PIVOT_SUPPORTED_AGGS.CARDINALITY,
PIVOT_SUPPORTED_AGGS.VALUE_COUNT,
PIVOT_SUPPORTED_AGGS.FILTER,
],
[KBN_FIELD_TYPES.GEO_POINT]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
[KBN_FIELD_TYPES.GEO_SHAPE]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
[KBN_FIELD_TYPES.IP]: [PIVOT_SUPPORTED_AGGS.CARDINALITY, PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
[KBN_FIELD_TYPES.MURMUR3]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
[KBN_FIELD_TYPES.MURMUR3]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
[KBN_FIELD_TYPES.NUMBER]: [
PIVOT_SUPPORTED_AGGS.AVG,
PIVOT_SUPPORTED_AGGS.CARDINALITY,
Expand All @@ -42,49 +57,122 @@ export const pivotAggsFieldSupport = {
PIVOT_SUPPORTED_AGGS.PERCENTILES,
PIVOT_SUPPORTED_AGGS.SUM,
PIVOT_SUPPORTED_AGGS.VALUE_COUNT,
PIVOT_SUPPORTED_AGGS.FILTER,
],
[KBN_FIELD_TYPES.STRING]: [
PIVOT_SUPPORTED_AGGS.CARDINALITY,
PIVOT_SUPPORTED_AGGS.VALUE_COUNT,
PIVOT_SUPPORTED_AGGS.FILTER,
],
[KBN_FIELD_TYPES.STRING]: [PIVOT_SUPPORTED_AGGS.CARDINALITY, PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
[KBN_FIELD_TYPES._SOURCE]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
[KBN_FIELD_TYPES.UNKNOWN]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
[KBN_FIELD_TYPES.CONFLICT]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
[KBN_FIELD_TYPES._SOURCE]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
[KBN_FIELD_TYPES.UNKNOWN]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
[KBN_FIELD_TYPES.CONFLICT]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
};

export type PivotAgg = {
[key in PIVOT_SUPPORTED_AGGS]?: {
[key in PivotSupportedAggs]?: {
field: EsFieldName;
};
};

export type PivotAggDict = { [key in AggName]: PivotAgg };
export type PivotAggDict = {
[key in AggName]: PivotAgg;
};

// The internal representation of an aggregation definition.
export interface PivotAggsConfigBase {
agg: PIVOT_SUPPORTED_AGGS;
agg: PivotSupportedAggs;
aggName: AggName;
dropDownName: string;
}

interface PivotAggsConfigWithUiBase extends PivotAggsConfigBase {
/**
* Resolves agg UI config from provided ES agg definition
*/
export function getAggConfigFromEsAgg(esAggDefinition: Record<string, any>, aggName: string) {
const aggKeys = Object.keys(esAggDefinition);

// Find the main aggregation key
const agg = aggKeys.find((aggKey) => aggKey !== 'aggs');

if (!isPivotSupportedAggs(agg)) {
throw new Error(`Aggregation "${agg}" is not supported`);
}

const commonConfig: PivotAggsConfigBase = {
...esAggDefinition[agg],
agg,
aggName,
dropDownName: aggName,
};

const config = getAggFormConfig(agg, commonConfig);

if (isPivotAggsWithExtendedForm(config)) {
config.setUiConfigFromEs(esAggDefinition[agg]);
}

if (aggKeys.includes('aggs')) {
// TODO process sub-aggregation
}

return config;
}

export interface PivotAggsConfigWithUiBase extends PivotAggsConfigBase {
field: EsFieldName;
}

export interface PivotAggsConfigWithExtra<T> extends PivotAggsConfigWithUiBase {
/** Form component */
AggFormComponent: FC<{
aggConfig: Partial<T>;
onChange: (arg: Partial<T>) => void;
selectedField: string;
}>;
/** Aggregation specific configuration */
aggConfig: Partial<T>;
/** Set UI configuration from ES aggregation definition */
setUiConfigFromEs: (arg: { [key: string]: any }) => void;
/** Converts UI agg config form to ES agg request object */
getEsAggConfig: () => { [key: string]: any } | null;
/** Indicates if the configuration is valid */
isValid: () => boolean;
/** Provides aggregation name generated based on the configuration */
getAggName?: () => string | undefined;
/** Helper text for the aggregation reflecting some configuration info */
helperText?: () => string | undefined;
}

interface PivotAggsConfigPercentiles extends PivotAggsConfigWithUiBase {
agg: PIVOT_SUPPORTED_AGGS.PERCENTILES;
agg: typeof PIVOT_SUPPORTED_AGGS.PERCENTILES;
percents: number[];
}

export type PivotAggsConfigWithUiSupport = PivotAggsConfigWithUiBase | PivotAggsConfigPercentiles;
export type PivotAggsConfigWithUiSupport =
| PivotAggsConfigWithUiBase
| PivotAggsConfigPercentiles
| PivotAggsConfigWithExtendedForm;

export function isPivotAggsConfigWithUiSupport(arg: any): arg is PivotAggsConfigWithUiSupport {
return (
arg.hasOwnProperty('agg') &&
arg.hasOwnProperty('aggName') &&
arg.hasOwnProperty('dropDownName') &&
arg.hasOwnProperty('field') &&
Object.values(PIVOT_SUPPORTED_AGGS).includes(arg.agg)
isPivotSupportedAggs(arg.agg)
);
}

/**
* Union type for agg configs with extended forms
*/
type PivotAggsConfigWithExtendedForm = PivotAggsConfigFilter;

export function isPivotAggsWithExtendedForm(arg: any): arg is PivotAggsConfigWithExtendedForm {
return arg.hasOwnProperty('AggFormComponent');
}

export function isPivotAggsConfigPercentiles(arg: any): arg is PivotAggsConfigPercentiles {
return (
arg.hasOwnProperty('agg') &&
Expand All @@ -99,14 +187,28 @@ export type PivotAggsConfig = PivotAggsConfigBase | PivotAggsConfigWithUiSupport
export type PivotAggsConfigWithUiSupportDict = Dictionary<PivotAggsConfigWithUiSupport>;
export type PivotAggsConfigDict = Dictionary<PivotAggsConfig>;

export function getEsAggFromAggConfig(groupByConfig: PivotAggsConfigBase): PivotAgg {
const esAgg = { ...groupByConfig };
/**
* Extracts Elasticsearch-ready aggregation configuration
* from the UI config
*/
export function getEsAggFromAggConfig(
pivotAggsConfig: PivotAggsConfigBase | PivotAggsConfigWithExtendedForm
): PivotAgg | null {
let esAgg: { [key: string]: any } | null = { ...pivotAggsConfig };

delete esAgg.agg;
delete esAgg.aggName;
delete esAgg.dropDownName;

if (isPivotAggsWithExtendedForm(pivotAggsConfig)) {
esAgg = pivotAggsConfig.getEsAggConfig();

if (esAgg === null) {
return null;
}
}

return {
[groupByConfig.agg]: esAgg,
[pivotAggsConfig.agg]: esAgg,
};
}
6 changes: 5 additions & 1 deletion x-pack/plugins/transform/public/app/common/request.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,11 @@ export function getPreviewRequestBody(
});

aggs.forEach((agg) => {
request.pivot.aggregations[agg.aggName] = getEsAggFromAggConfig(agg);
const result = getEsAggFromAggConfig(agg);
if (result === null) {
return;
}
request.pivot.aggregations[agg.aggName] = result;
});

return request;
Expand Down
Loading

0 comments on commit 9a818de

Please sign in to comment.