diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.cumulative_sum.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.cumulative_sum.md new file mode 100644 index 0000000000000..ad1de0cc5f45b --- /dev/null +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.cumulative_sum.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [ExpressionFunctionDefinitions](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.md) > [cumulative\_sum](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.cumulative_sum.md) + +## ExpressionFunctionDefinitions.cumulative\_sum property + +Signature: + +```typescript +cumulative_sum: ExpressionFunctionCumulativeSum; +``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.md index 914c5d6ebe2f6..d1703a1e019e6 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.md @@ -17,6 +17,7 @@ export interface ExpressionFunctionDefinitions | Property | Type | Description | | --- | --- | --- | | [clog](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.clog.md) | ExpressionFunctionClog | | +| [cumulative\_sum](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.cumulative_sum.md) | ExpressionFunctionCumulativeSum | | | [font](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.font.md) | ExpressionFunctionFont | | | [kibana\_context](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.kibana_context.md) | ExpressionFunctionKibanaContext | | | [kibana](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.kibana.md) | ExpressionFunctionKibana | | diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.cumulative_sum.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.cumulative_sum.md new file mode 100644 index 0000000000000..2fb8cde92e877 --- /dev/null +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.cumulative_sum.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-expressions-server](./kibana-plugin-plugins-expressions-server.md) > [ExpressionFunctionDefinitions](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.md) > [cumulative\_sum](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.cumulative_sum.md) + +## ExpressionFunctionDefinitions.cumulative\_sum property + +Signature: + +```typescript +cumulative_sum: ExpressionFunctionCumulativeSum; +``` diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.md index 71cd0b98a68c2..05b4ddce4ccde 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.md @@ -17,6 +17,7 @@ export interface ExpressionFunctionDefinitions | Property | Type | Description | | --- | --- | --- | | [clog](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.clog.md) | ExpressionFunctionClog | | +| [cumulative\_sum](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.cumulative_sum.md) | ExpressionFunctionCumulativeSum | | | [font](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.font.md) | ExpressionFunctionFont | | | [kibana\_context](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.kibana_context.md) | ExpressionFunctionKibanaContext | | | [kibana](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.kibana.md) | ExpressionFunctionKibana | | diff --git a/src/plugins/expressions/common/expression_functions/specs/cumulative_sum.ts b/src/plugins/expressions/common/expression_functions/specs/cumulative_sum.ts new file mode 100644 index 0000000000000..970015638794f --- /dev/null +++ b/src/plugins/expressions/common/expression_functions/specs/cumulative_sum.ts @@ -0,0 +1,170 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { i18n } from '@kbn/i18n'; +import { ExpressionFunctionDefinition } from '../types'; +import { Datatable, DatatableRow } from '../../expression_types'; + +export interface CumulativeSumArgs { + by?: string[]; + inputColumnId: string; + outputColumnId: string; + outputColumnName?: string; +} + +export type ExpressionFunctionCumulativeSum = ExpressionFunctionDefinition< + 'cumulative_sum', + Datatable, + CumulativeSumArgs, + Datatable +>; + +/** + * Returns a string identifying the group of a row by a list of columns to group by + */ +function getBucketIdentifier(row: DatatableRow, groupColumns?: string[]) { + return (groupColumns || []) + .map((groupColumnId) => (row[groupColumnId] == null ? '' : String(row[groupColumnId]))) + .join('|'); +} + +/** + * Calculates the cumulative sum of a specified column in the data table. + * + * Also supports multiple series in a single data table - use the `by` argument + * to specify the columns to split the calculation by. + * For each unique combination of all `by` columns a separate cumulative sum will be calculated. + * The order of rows won't be changed - this function is not modifying any existing columns, it's only + * adding the specified `outputColumnId` column to every row of the table without adding or removing rows. + * + * Behavior: + * * Will write the cumulative sum of `inputColumnId` into `outputColumnId` + * * If provided will use `outputColumnName` as name for the newly created column. Otherwise falls back to `outputColumnId` + * * Cumulative sums always start with 0, a cell will contain its own value plus the values of + * all cells of the same series further up in the table. + * + * Edge cases: + * * Will return the input table if `inputColumnId` does not exist + * * Will throw an error if `outputColumnId` exists already in provided data table + * * If the row value contains `null` or `undefined`, it will be ignored and overwritten with the cumulative sum of + * all cells of the same series further up in the table. + * * For all values besides `null` and `undefined`, the value will be cast to a number before it's added to the + * cumulative sum of the current series - if this results in `NaN` (like in case of objects), all cells of the + * current series will be set to `NaN`. + * * To determine separate series defined by the `by` columns, the values of these columns will be cast to strings + * before comparison. If the values are objects, the return value of their `toString` method will be used for comparison. + * Missing values (`null` and `undefined`) will be treated as empty strings. + */ +export const cumulativeSum: ExpressionFunctionCumulativeSum = { + name: 'cumulative_sum', + type: 'datatable', + + inputTypes: ['datatable'], + + help: i18n.translate('expressions.functions.cumulativeSum.help', { + defaultMessage: 'Calculates the cumulative sum of a column in a data table', + }), + + args: { + by: { + help: i18n.translate('expressions.functions.cumulativeSum.args.byHelpText', { + defaultMessage: 'Column to split the cumulative sum calculation by', + }), + multi: true, + types: ['string'], + required: false, + }, + inputColumnId: { + help: i18n.translate('expressions.functions.cumulativeSum.args.inputColumnIdHelpText', { + defaultMessage: 'Column to calculate the cumulative sum of', + }), + types: ['string'], + required: true, + }, + outputColumnId: { + help: i18n.translate('expressions.functions.cumulativeSum.args.outputColumnIdHelpText', { + defaultMessage: 'Column to store the resulting cumulative sum in', + }), + types: ['string'], + required: true, + }, + outputColumnName: { + help: i18n.translate('expressions.functions.cumulativeSum.args.outputColumnNameHelpText', { + defaultMessage: 'Name of the column to store the resulting cumulative sum in', + }), + types: ['string'], + required: false, + }, + }, + + fn(input, { by, inputColumnId, outputColumnId, outputColumnName }) { + if (input.columns.some((column) => column.id === outputColumnId)) { + throw new Error( + i18n.translate('expressions.functions.cumulativeSum.columnConflictMessage', { + defaultMessage: + 'Specified outputColumnId {columnId} already exists. Please pick another column id.', + values: { + columnId: outputColumnId, + }, + }) + ); + } + + const inputColumnDefinition = input.columns.find((column) => column.id === inputColumnId); + + if (!inputColumnDefinition) { + return input; + } + + const outputColumnDefinition = { + ...inputColumnDefinition, + id: outputColumnId, + name: outputColumnName || outputColumnId, + }; + + const resultColumns = [...input.columns]; + // add output column after input column in the table + resultColumns.splice( + resultColumns.indexOf(inputColumnDefinition) + 1, + 0, + outputColumnDefinition + ); + + const accumulators: Partial> = {}; + return { + ...input, + columns: resultColumns, + rows: input.rows.map((row) => { + const newRow = { ...row }; + + const bucketIdentifier = getBucketIdentifier(row, by); + const accumulatorValue = accumulators[bucketIdentifier] ?? 0; + const currentValue = newRow[inputColumnId]; + if (currentValue != null) { + newRow[outputColumnId] = Number(currentValue) + accumulatorValue; + accumulators[bucketIdentifier] = newRow[outputColumnId]; + } else { + newRow[outputColumnId] = accumulatorValue; + } + + return newRow; + }), + }; + }, +}; diff --git a/src/plugins/expressions/common/expression_functions/specs/index.ts b/src/plugins/expressions/common/expression_functions/specs/index.ts index 5b9562dae5f2e..aadea5882b9c0 100644 --- a/src/plugins/expressions/common/expression_functions/specs/index.ts +++ b/src/plugins/expressions/common/expression_functions/specs/index.ts @@ -25,6 +25,7 @@ import { variableSet } from './var_set'; import { variable } from './var'; import { AnyExpressionFunctionDefinition } from '../types'; import { theme } from './theme'; +import { cumulativeSum } from './cumulative_sum'; export const functionSpecs: AnyExpressionFunctionDefinition[] = [ clog, @@ -34,6 +35,7 @@ export const functionSpecs: AnyExpressionFunctionDefinition[] = [ variableSet, variable, theme, + cumulativeSum, ]; export * from './clog'; @@ -43,3 +45,4 @@ export * from './kibana_context'; export * from './var_set'; export * from './var'; export * from './theme'; +export * from './cumulative_sum'; diff --git a/src/plugins/expressions/common/expression_functions/specs/tests/cumulative_sum.test.ts b/src/plugins/expressions/common/expression_functions/specs/tests/cumulative_sum.test.ts new file mode 100644 index 0000000000000..037b3ddc25f89 --- /dev/null +++ b/src/plugins/expressions/common/expression_functions/specs/tests/cumulative_sum.test.ts @@ -0,0 +1,361 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { functionWrapper } from './utils'; +import { cumulativeSum, CumulativeSumArgs } from '../cumulative_sum'; +import { ExecutionContext } from '../../../execution/types'; +import { Datatable } from '../../../expression_types/specs/datatable'; + +describe('interpreter/functions#cumulative_sum', () => { + const fn = functionWrapper(cumulativeSum); + const runFn = (input: Datatable, args: CumulativeSumArgs) => + fn(input, args, {} as ExecutionContext) as Datatable; + + it('calculates cumulative sum', () => { + const result = runFn( + { + type: 'datatable', + columns: [{ id: 'val', name: 'val', meta: { type: 'number' } }], + rows: [{ val: 5 }, { val: 7 }, { val: 3 }, { val: 2 }], + }, + { inputColumnId: 'val', outputColumnId: 'output' } + ); + expect(result.columns).toContainEqual({ + id: 'output', + name: 'output', + meta: { type: 'number' }, + }); + expect(result.rows.map((row) => row.output)).toEqual([5, 12, 15, 17]); + }); + + it('replaces null or undefined data with zeroes until there is real data', () => { + const result = runFn( + { + type: 'datatable', + columns: [{ id: 'val', name: 'val', meta: { type: 'number' } }], + rows: [{}, { val: null }, { val: undefined }, { val: 1 }], + }, + { inputColumnId: 'val', outputColumnId: 'output' } + ); + expect(result.columns).toContainEqual({ + id: 'output', + name: 'output', + meta: { type: 'number' }, + }); + expect(result.rows.map((row) => row.output)).toEqual([0, 0, 0, 1]); + }); + + it('calculates cumulative sum for multiple series', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { id: 'val', name: 'val', meta: { type: 'number' } }, + { id: 'split', name: 'split', meta: { type: 'string' } }, + ], + rows: [ + { val: 1, split: 'A' }, + { val: 2, split: 'B' }, + { val: 3, split: 'B' }, + { val: 4, split: 'A' }, + { val: 5, split: 'A' }, + { val: 6, split: 'A' }, + { val: 7, split: 'B' }, + { val: 8, split: 'B' }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output', by: ['split'] } + ); + + expect(result.rows.map((row) => row.output)).toEqual([ + 1, + 2, + 2 + 3, + 1 + 4, + 1 + 4 + 5, + 1 + 4 + 5 + 6, + 2 + 3 + 7, + 2 + 3 + 7 + 8, + ]); + }); + + it('treats missing split column as separate series', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { id: 'val', name: 'val', meta: { type: 'number' } }, + { id: 'split', name: 'split', meta: { type: 'string' } }, + ], + rows: [ + { val: 1, split: 'A' }, + { val: 2, split: 'B' }, + { val: 3 }, + { val: 4, split: 'A' }, + { val: 5 }, + { val: 6, split: 'A' }, + { val: 7, split: 'B' }, + { val: 8, split: 'B' }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output', by: ['split'] } + ); + expect(result.rows.map((row) => row.output)).toEqual([ + 1, + 2, + 3, + 1 + 4, + 3 + 5, + 1 + 4 + 6, + 2 + 7, + 2 + 7 + 8, + ]); + }); + + it('treats null like undefined and empty string for split columns', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { id: 'val', name: 'val', meta: { type: 'number' } }, + { id: 'split', name: 'split', meta: { type: 'string' } }, + ], + rows: [ + { val: 1, split: 'A' }, + { val: 2, split: 'B' }, + { val: 3 }, + { val: 4, split: 'A' }, + { val: 5 }, + { val: 6, split: 'A' }, + { val: 7, split: null }, + { val: 8, split: 'B' }, + { val: 9, split: '' }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output', by: ['split'] } + ); + expect(result.rows.map((row) => row.output)).toEqual([ + 1, + 2, + 3, + 1 + 4, + 3 + 5, + 1 + 4 + 6, + 3 + 5 + 7, + 2 + 8, + 3 + 5 + 7 + 9, + ]); + }); + + it('calculates cumulative sum for multiple series by multiple split columns', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { id: 'val', name: 'val', meta: { type: 'number' } }, + { id: 'split', name: 'split', meta: { type: 'string' } }, + { id: 'split2', name: 'split2', meta: { type: 'string' } }, + ], + rows: [ + { val: 1, split: 'A', split2: 'C' }, + { val: 2, split: 'B', split2: 'C' }, + { val: 3, split2: 'C' }, + { val: 4, split: 'A', split2: 'C' }, + { val: 5 }, + { val: 6, split: 'A', split2: 'D' }, + { val: 7, split: 'B', split2: 'D' }, + { val: 8, split: 'B', split2: 'D' }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output', by: ['split', 'split2'] } + ); + expect(result.rows.map((row) => row.output)).toEqual([1, 2, 3, 1 + 4, 5, 6, 7, 7 + 8]); + }); + + it('splits separate series by the string representation of the cell values', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { id: 'val', name: 'val', meta: { type: 'number' } }, + { id: 'split', name: 'split', meta: { type: 'string' } }, + ], + rows: [ + { val: 1, split: { anObj: 3 } }, + { val: 2, split: { anotherObj: 5 } }, + { val: 10, split: 5 }, + { val: 11, split: '5' }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output', by: ['split'] } + ); + + expect(result.rows.map((row) => row.output)).toEqual([1, 1 + 2, 10, 21]); + }); + + it('casts values to number before calculating cumulative sum', () => { + const result = runFn( + { + type: 'datatable', + columns: [{ id: 'val', name: 'val', meta: { type: 'number' } }], + rows: [{ val: 5 }, { val: '7' }, { val: '3' }, { val: 2 }], + }, + { inputColumnId: 'val', outputColumnId: 'output' } + ); + expect(result.rows.map((row) => row.output)).toEqual([5, 12, 15, 17]); + }); + + it('casts values to number before calculating cumulative sum for NaN like values', () => { + const result = runFn( + { + type: 'datatable', + columns: [{ id: 'val', name: 'val', meta: { type: 'number' } }], + rows: [{ val: 5 }, { val: '7' }, { val: {} }, { val: 2 }], + }, + { inputColumnId: 'val', outputColumnId: 'output' } + ); + expect(result.rows.map((row) => row.output)).toEqual([5, 12, NaN, NaN]); + }); + + it('skips undefined and null values', () => { + const result = runFn( + { + type: 'datatable', + columns: [{ id: 'val', name: 'val', meta: { type: 'number' } }], + rows: [ + { val: null }, + { val: 7 }, + { val: undefined }, + { val: undefined }, + { val: undefined }, + { val: undefined }, + { val: '3' }, + { val: 2 }, + { val: null }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output' } + ); + expect(result.rows.map((row) => row.output)).toEqual([0, 7, 7, 7, 7, 7, 10, 12, 12]); + }); + + it('copies over meta information from the source column', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { + id: 'val', + name: 'val', + meta: { + type: 'number', + + field: 'afield', + index: 'anindex', + params: { id: 'number', params: { pattern: '000' } }, + source: 'synthetic', + sourceParams: { + some: 'params', + }, + }, + }, + ], + rows: [{ val: 5 }], + }, + { inputColumnId: 'val', outputColumnId: 'output' } + ); + expect(result.columns).toContainEqual({ + id: 'output', + name: 'output', + meta: { + type: 'number', + + field: 'afield', + index: 'anindex', + params: { id: 'number', params: { pattern: '000' } }, + source: 'synthetic', + sourceParams: { + some: 'params', + }, + }, + }); + }); + + it('sets output name on output column if specified', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { + id: 'val', + name: 'val', + meta: { + type: 'number', + }, + }, + ], + rows: [{ val: 5 }], + }, + { inputColumnId: 'val', outputColumnId: 'output', outputColumnName: 'Output name' } + ); + expect(result.columns).toContainEqual({ + id: 'output', + name: 'Output name', + meta: { type: 'number' }, + }); + }); + + it('returns source table if input column does not exist', () => { + const input: Datatable = { + type: 'datatable', + columns: [ + { + id: 'val', + name: 'val', + meta: { + type: 'number', + }, + }, + ], + rows: [{ val: 5 }], + }; + expect(runFn(input, { inputColumnId: 'nonexisting', outputColumnId: 'output' })).toBe(input); + }); + + it('throws an error if output column exists already', () => { + expect(() => + runFn( + { + type: 'datatable', + columns: [ + { + id: 'val', + name: 'val', + meta: { + type: 'number', + }, + }, + ], + rows: [{ val: 5 }], + }, + { inputColumnId: 'val', outputColumnId: 'val' } + ) + ).toThrow(); + }); +}); diff --git a/src/plugins/expressions/common/expression_functions/types.ts b/src/plugins/expressions/common/expression_functions/types.ts index caaef541aefd5..fb1823e85b391 100644 --- a/src/plugins/expressions/common/expression_functions/types.ts +++ b/src/plugins/expressions/common/expression_functions/types.ts @@ -29,6 +29,7 @@ import { ExpressionFunctionVarSet, ExpressionFunctionVar, ExpressionFunctionTheme, + ExpressionFunctionCumulativeSum, } from './specs'; import { ExpressionAstFunction } from '../ast'; import { PersistableStateDefinition } from '../../../kibana_utils/common'; @@ -131,4 +132,5 @@ export interface ExpressionFunctionDefinitions { var_set: ExpressionFunctionVarSet; var: ExpressionFunctionVar; theme: ExpressionFunctionTheme; + cumulative_sum: ExpressionFunctionCumulativeSum; } diff --git a/src/plugins/expressions/public/public.api.md b/src/plugins/expressions/public/public.api.md index 95ee651d433ac..4739b9434bdaa 100644 --- a/src/plugins/expressions/public/public.api.md +++ b/src/plugins/expressions/public/public.api.md @@ -377,6 +377,10 @@ export interface ExpressionFunctionDefinitions { // // (undocumented) clog: ExpressionFunctionClog; + // Warning: (ae-forgotten-export) The symbol "ExpressionFunctionCumulativeSum" needs to be exported by the entry point index.d.ts + // + // (undocumented) + cumulative_sum: ExpressionFunctionCumulativeSum; // Warning: (ae-forgotten-export) The symbol "ExpressionFunctionFont" needs to be exported by the entry point index.d.ts // // (undocumented) diff --git a/src/plugins/expressions/server/server.api.md b/src/plugins/expressions/server/server.api.md index d5da60af8f8e5..fcdfd5ef3246c 100644 --- a/src/plugins/expressions/server/server.api.md +++ b/src/plugins/expressions/server/server.api.md @@ -349,6 +349,10 @@ export interface ExpressionFunctionDefinitions { // // (undocumented) clog: ExpressionFunctionClog; + // Warning: (ae-forgotten-export) The symbol "ExpressionFunctionCumulativeSum" needs to be exported by the entry point index.d.ts + // + // (undocumented) + cumulative_sum: ExpressionFunctionCumulativeSum; // Warning: (ae-forgotten-export) The symbol "ExpressionFunctionFont" needs to be exported by the entry point index.d.ts // // (undocumented) diff --git a/x-pack/plugins/apm/public/components/app/Correlations/index.tsx b/x-pack/plugins/apm/public/components/app/Correlations/index.tsx new file mode 100644 index 0000000000000..afee2b9f5e881 --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/Correlations/index.tsx @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import url from 'url'; +import { useParams } from 'react-router-dom'; +import { useLocation } from 'react-router-dom'; +import { EuiTitle, EuiListGroup } from '@elastic/eui'; +import { useUrlParams } from '../../../hooks/useUrlParams'; +import { useApmPluginContext } from '../../../hooks/useApmPluginContext'; + +const SESSION_STORAGE_KEY = 'apm.debug.show_correlations'; + +export function Correlations() { + const location = useLocation(); + const { serviceName } = useParams<{ serviceName?: string }>(); + const { urlParams, uiFilters } = useUrlParams(); + const { core } = useApmPluginContext(); + const { transactionName, transactionType, start, end } = urlParams; + + if ( + !location.search.includes('&_show_correlations') && + sessionStorage.getItem(SESSION_STORAGE_KEY) !== 'true' + ) { + return null; + } + + sessionStorage.setItem(SESSION_STORAGE_KEY, 'true'); + + const query = { + serviceName, + transactionName, + transactionType, + start, + end, + uiFilters: JSON.stringify(uiFilters), + fieldNames: + 'user.username,user.id,host.ip,user_agent.name,kubernetes.pod.uuid,url.domain,container.id,service.node.name', + }; + + const listItems = [ + { + label: 'Show correlations between two ranges', + href: url.format({ + query: { + ...query, + gap: 24, + }, + pathname: core.http.basePath.prepend(`/api/apm/correlations/ranges`), + }), + isDisabled: false, + iconType: 'tokenRange', + size: 's' as const, + }, + + { + label: 'Show correlations for slow transactions', + href: url.format({ + query: { + ...query, + durationPercentile: 95, + }, + pathname: core.http.basePath.prepend( + `/api/apm/correlations/slow_durations` + ), + }), + isDisabled: false, + iconType: 'clock', + size: 's' as const, + }, + ]; + + return ( + <> + +

Correlations

+
+ + + + ); +} diff --git a/x-pack/plugins/apm/public/components/app/ServiceOverview/index.tsx b/x-pack/plugins/apm/public/components/app/ServiceOverview/index.tsx index bc4a3212bafe2..fba5df5b16a4a 100644 --- a/x-pack/plugins/apm/public/components/app/ServiceOverview/index.tsx +++ b/x-pack/plugins/apm/public/components/app/ServiceOverview/index.tsx @@ -21,6 +21,7 @@ import { useApmPluginContext } from '../../../hooks/useApmPluginContext'; import { MLCallout } from './ServiceList/MLCallout'; import { useLocalStorage } from '../../../hooks/useLocalStorage'; import { useAnomalyDetectionJobs } from '../../../hooks/useAnomalyDetectionJobs'; +import { Correlations } from '../Correlations'; const initialData = { items: [], @@ -117,6 +118,9 @@ export function ServiceOverview() { return ( <> + + + diff --git a/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx b/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx index e31a2b24f1d15..b79186a90cd1d 100644 --- a/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx +++ b/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx @@ -31,6 +31,7 @@ import { fromQuery, toQuery } from '../../shared/Links/url_helpers'; import { useUrlParams } from '../../../hooks/useUrlParams'; import { LocalUIFilters } from '../../shared/LocalUIFilters'; import { HeightRetainer } from '../../shared/HeightRetainer'; +import { Correlations } from '../Correlations'; interface Sample { traceId: string; @@ -111,6 +112,8 @@ export function TransactionDetails({ + + diff --git a/x-pack/plugins/apm/public/components/app/TransactionOverview/index.tsx b/x-pack/plugins/apm/public/components/app/TransactionOverview/index.tsx index 8c7d088d36eb2..003df632d11b3 100644 --- a/x-pack/plugins/apm/public/components/app/TransactionOverview/index.tsx +++ b/x-pack/plugins/apm/public/components/app/TransactionOverview/index.tsx @@ -37,6 +37,7 @@ import { TransactionList } from './TransactionList'; import { useRedirect } from './useRedirect'; import { TRANSACTION_PAGE_LOAD } from '../../../../common/transaction_types'; import { UserExperienceCallout } from './user_experience_callout'; +import { Correlations } from '../Correlations'; function getRedirectLocation({ urlParams, @@ -117,6 +118,7 @@ export function TransactionOverview({ serviceName }: TransactionOverviewProps) { return ( <> + diff --git a/x-pack/plugins/apm/server/lib/helpers/setup_request.ts b/x-pack/plugins/apm/server/lib/helpers/setup_request.ts index 26896a050dd88..a8a128937fb1c 100644 --- a/x-pack/plugins/apm/server/lib/helpers/setup_request.ts +++ b/x-pack/plugins/apm/server/lib/helpers/setup_request.ts @@ -49,7 +49,15 @@ export interface SetupTimeRange { interface SetupRequestParams { query?: { _debug?: boolean; + + /** + * Timestamp in ms since epoch + */ start?: string; + + /** + * Timestamp in ms since epoch + */ end?: string; uiFilters?: string; processorEvent?: ProcessorEvent; diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_correlations_for_ranges.ts b/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_correlations_for_ranges.ts new file mode 100644 index 0000000000000..3cf0271baa1c6 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_correlations_for_ranges.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { rangeFilter } from '../../../../common/utils/range_filter'; +import { + SERVICE_NAME, + TRANSACTION_NAME, + TRANSACTION_TYPE, +} from '../../../../common/elasticsearch_fieldnames'; +import { ProcessorEvent } from '../../../../common/processor_event'; +import { Setup, SetupTimeRange } from '../../helpers/setup_request'; +import { + getSignificantTermsAgg, + formatAggregationResponse, +} from './get_significant_terms_agg'; +import { SignificantTermsScoring } from './scoring_rt'; + +export async function getCorrelationsForRanges({ + serviceName, + transactionType, + transactionName, + scoring, + gapBetweenRanges, + fieldNames, + setup, +}: { + serviceName: string | undefined; + transactionType: string | undefined; + transactionName: string | undefined; + scoring: SignificantTermsScoring; + gapBetweenRanges: number; + fieldNames: string[]; + setup: Setup & SetupTimeRange; +}) { + const { start, end, esFilter, apmEventClient } = setup; + + const baseFilters = [...esFilter]; + + if (serviceName) { + baseFilters.push({ term: { [SERVICE_NAME]: serviceName } }); + } + + if (transactionType) { + baseFilters.push({ term: { [TRANSACTION_TYPE]: transactionType } }); + } + + if (transactionName) { + baseFilters.push({ term: { [TRANSACTION_NAME]: transactionName } }); + } + + const diff = end - start + gapBetweenRanges; + const baseRangeStart = start - diff; + const baseRangeEnd = end - diff; + const backgroundFilters = [ + ...baseFilters, + { range: rangeFilter(baseRangeStart, baseRangeEnd) }, + ]; + + const params = { + apm: { events: [ProcessorEvent.transaction] }, + body: { + size: 0, + query: { + bool: { filter: [...baseFilters, { range: rangeFilter(start, end) }] }, + }, + aggs: getSignificantTermsAgg({ + fieldNames, + backgroundFilters, + backgroundIsSuperset: false, + scoring, + }), + }, + }; + + const response = await apmEventClient.search(params); + + return { + message: `Showing significant fields between the ranges`, + firstRange: `${new Date(baseRangeStart).toISOString()} - ${new Date( + baseRangeEnd + ).toISOString()}`, + lastRange: `${new Date(start).toISOString()} - ${new Date( + end + ).toISOString()}`, + response: formatAggregationResponse(response.aggregations), + }; +} diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_correlations_for_slow_transactions.ts b/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_correlations_for_slow_transactions.ts new file mode 100644 index 0000000000000..3efc65afdfd28 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_correlations_for_slow_transactions.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { asDuration } from '../../../../common/utils/formatters'; +import { ESFilter } from '../../../../typings/elasticsearch'; +import { rangeFilter } from '../../../../common/utils/range_filter'; +import { + SERVICE_NAME, + TRANSACTION_DURATION, + TRANSACTION_NAME, + TRANSACTION_TYPE, +} from '../../../../common/elasticsearch_fieldnames'; +import { ProcessorEvent } from '../../../../common/processor_event'; +import { Setup, SetupTimeRange } from '../../helpers/setup_request'; +import { getDurationForPercentile } from './get_duration_for_percentile'; +import { + formatAggregationResponse, + getSignificantTermsAgg, +} from './get_significant_terms_agg'; +import { SignificantTermsScoring } from './scoring_rt'; + +export async function getCorrelationsForSlowTransactions({ + serviceName, + transactionType, + transactionName, + durationPercentile, + fieldNames, + scoring, + setup, +}: { + serviceName: string | undefined; + transactionType: string | undefined; + transactionName: string | undefined; + scoring: SignificantTermsScoring; + durationPercentile: number; + fieldNames: string[]; + setup: Setup & SetupTimeRange; +}) { + const { start, end, esFilter, apmEventClient } = setup; + + const backgroundFilters: ESFilter[] = [ + ...esFilter, + { range: rangeFilter(start, end) }, + ]; + + if (serviceName) { + backgroundFilters.push({ term: { [SERVICE_NAME]: serviceName } }); + } + + if (transactionType) { + backgroundFilters.push({ term: { [TRANSACTION_TYPE]: transactionType } }); + } + + if (transactionName) { + backgroundFilters.push({ term: { [TRANSACTION_NAME]: transactionName } }); + } + + const durationForPercentile = await getDurationForPercentile({ + durationPercentile, + backgroundFilters, + setup, + }); + + const params = { + apm: { events: [ProcessorEvent.transaction] }, + body: { + size: 0, + query: { + bool: { + // foreground filters + filter: [ + ...backgroundFilters, + { + range: { [TRANSACTION_DURATION]: { gte: durationForPercentile } }, + }, + ], + }, + }, + aggs: getSignificantTermsAgg({ fieldNames, backgroundFilters, scoring }), + }, + }; + + const response = await apmEventClient.search(params); + + return { + message: `Showing significant fields for transactions slower than ${durationPercentile}th percentile (${asDuration( + durationForPercentile + )})`, + response: formatAggregationResponse(response.aggregations), + }; +} diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_duration_for_percentile.ts b/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_duration_for_percentile.ts new file mode 100644 index 0000000000000..37ee19ff40f62 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_duration_for_percentile.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ESFilter } from '../../../../typings/elasticsearch'; +import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; +import { ProcessorEvent } from '../../../../common/processor_event'; +import { Setup, SetupTimeRange } from '../../helpers/setup_request'; + +export async function getDurationForPercentile({ + durationPercentile, + backgroundFilters, + setup, +}: { + durationPercentile: number; + backgroundFilters: ESFilter[]; + setup: Setup & SetupTimeRange; +}) { + const { apmEventClient } = setup; + const res = await apmEventClient.search({ + apm: { + events: [ProcessorEvent.transaction], + }, + body: { + size: 0, + query: { + bool: { filter: backgroundFilters }, + }, + aggs: { + percentile: { + percentiles: { + field: TRANSACTION_DURATION, + percents: [durationPercentile], + }, + }, + }, + }, + }); + + return Object.values(res.aggregations?.percentile.values || {})[0]; +} diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_significant_terms_agg.ts b/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_significant_terms_agg.ts new file mode 100644 index 0000000000000..1cf0787c1d970 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_significant_terms_agg.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ESFilter } from '../../../../typings/elasticsearch'; +import { SignificantTermsScoring } from './scoring_rt'; + +export function getSignificantTermsAgg({ + fieldNames, + backgroundFilters, + backgroundIsSuperset = true, + scoring = 'percentage', +}: { + fieldNames: string[]; + backgroundFilters: ESFilter[]; + backgroundIsSuperset?: boolean; + scoring: SignificantTermsScoring; +}) { + return fieldNames.reduce((acc, fieldName) => { + return { + ...acc, + [fieldName]: { + significant_terms: { + size: 10, + field: fieldName, + background_filter: { bool: { filter: backgroundFilters } }, + + // indicate whether background is a superset of the foreground + mutual_information: { background_is_superset: backgroundIsSuperset }, + + // different scorings https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-significantterms-aggregation.html#significantterms-aggregation-parameters + [scoring]: {}, + min_doc_count: 5, + shard_min_doc_count: 5, + }, + }, + [`cardinality-${fieldName}`]: { + cardinality: { field: fieldName }, + }, + }; + }, {} as Record); +} + +export function formatAggregationResponse(aggs?: Record) { + if (!aggs) { + return; + } + + return Object.entries(aggs).reduce((acc, [key, value]) => { + if (key.startsWith('cardinality-')) { + if (value.value > 0) { + const fieldName = key.slice(12); + acc[fieldName] = { + ...acc[fieldName], + cardinality: value.value, + }; + } + } else if (value.buckets.length > 0) { + acc[key] = { + ...acc[key], + value, + }; + } + return acc; + }, {} as Record); +} diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/correlations/scoring_rt.ts b/x-pack/plugins/apm/server/lib/transaction_groups/correlations/scoring_rt.ts new file mode 100644 index 0000000000000..cb94b6251eb07 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/transaction_groups/correlations/scoring_rt.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; + +export const scoringRt = t.union([ + t.literal('jlh'), + t.literal('chi_square'), + t.literal('gnd'), + t.literal('percentage'), +]); + +export type SignificantTermsScoring = t.TypeOf; diff --git a/x-pack/plugins/apm/server/routes/correlations.ts b/x-pack/plugins/apm/server/routes/correlations.ts new file mode 100644 index 0000000000000..5f8d2afd544f3 --- /dev/null +++ b/x-pack/plugins/apm/server/routes/correlations.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { rangeRt } from './default_api_types'; +import { getCorrelationsForSlowTransactions } from '../lib/transaction_groups/correlations/get_correlations_for_slow_transactions'; +import { getCorrelationsForRanges } from '../lib/transaction_groups/correlations/get_correlations_for_ranges'; +import { scoringRt } from '../lib/transaction_groups/correlations/scoring_rt'; +import { createRoute } from './create_route'; +import { setupRequest } from '../lib/helpers/setup_request'; + +export const correlationsForSlowTransactionsRoute = createRoute(() => ({ + path: '/api/apm/correlations/slow_durations', + params: { + query: t.intersection([ + t.partial({ + serviceName: t.string, + transactionName: t.string, + transactionType: t.string, + scoring: scoringRt, + }), + t.type({ + durationPercentile: t.string, + fieldNames: t.string, + }), + t.partial({ uiFilters: t.string }), + rangeRt, + ]), + }, + handler: async ({ context, request }) => { + const setup = await setupRequest(context, request); + const { + serviceName, + transactionType, + transactionName, + durationPercentile, + fieldNames, + scoring = 'percentage', + } = context.params.query; + + return getCorrelationsForSlowTransactions({ + serviceName, + transactionType, + transactionName, + durationPercentile: parseInt(durationPercentile, 10), + fieldNames: fieldNames.split(','), + scoring, + setup, + }); + }, +})); + +export const correlationsForRangesRoute = createRoute(() => ({ + path: '/api/apm/correlations/ranges', + params: { + query: t.intersection([ + t.partial({ + serviceName: t.string, + transactionName: t.string, + transactionType: t.string, + scoring: scoringRt, + gap: t.string, + }), + t.type({ + fieldNames: t.string, + }), + t.partial({ uiFilters: t.string }), + rangeRt, + ]), + }, + handler: async ({ context, request }) => { + const setup = await setupRequest(context, request); + + const { + serviceName, + transactionType, + transactionName, + scoring = 'percentage', + gap, + fieldNames, + } = context.params.query; + + const gapBetweenRanges = parseInt(gap || '0', 10) * 3600 * 1000; + if (gapBetweenRanges < 0) { + throw new Error('gap must be 0 or positive'); + } + + return getCorrelationsForRanges({ + serviceName, + transactionType, + transactionName, + scoring, + gapBetweenRanges, + fieldNames: fieldNames.split(','), + setup, + }); + }, +})); diff --git a/x-pack/plugins/apm/server/routes/create_apm_api.ts b/x-pack/plugins/apm/server/routes/create_apm_api.ts index c1f13ee646e49..2fbe404a70d82 100644 --- a/x-pack/plugins/apm/server/routes/create_apm_api.ts +++ b/x-pack/plugins/apm/server/routes/create_apm_api.ts @@ -41,6 +41,10 @@ import { metricsChartsRoute } from './metrics'; import { serviceNodesRoute } from './service_nodes'; import { tracesRoute, tracesByIdRoute } from './traces'; import { transactionByTraceIdRoute } from './transaction'; +import { + correlationsForRangesRoute, + correlationsForSlowTransactionsRoute, +} from './correlations'; import { transactionGroupsBreakdownRoute, transactionGroupsChartsRoute, @@ -122,6 +126,10 @@ const createApmApi = () => { .add(listAgentConfigurationServicesRoute) .add(createOrUpdateAgentConfigurationRoute) + // Correlations + .add(correlationsForSlowTransactionsRoute) + .add(correlationsForRangesRoute) + // APM indices .add(apmIndexSettingsRoute) .add(apmIndicesRoute) diff --git a/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts b/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts index 534321201938d..8b3163e44915a 100644 --- a/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts +++ b/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts @@ -158,6 +158,11 @@ export interface AggregationOptionsByType { from?: number; size?: number; }; + significant_terms: { + size?: number; + field?: string; + background_filter?: Record; + } & AggregationSourceOptions; } type AggregationType = keyof AggregationOptionsByType; @@ -334,6 +339,17 @@ interface AggregationResponsePart< ? Array<{ key: number; value: number }> : Record; }; + significant_terms: { + doc_count: number; + bg_count: number; + buckets: Array< + { + bg_count: number; + doc_count: number; + key: string | number; + } & SubAggregationResponseOf + >; + }; bucket_sort: undefined; } diff --git a/x-pack/test/apm_api_integration/basic/tests/correlations/ranges.ts b/x-pack/test/apm_api_integration/basic/tests/correlations/ranges.ts new file mode 100644 index 0000000000000..f013520fa163b --- /dev/null +++ b/x-pack/test/apm_api_integration/basic/tests/correlations/ranges.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import { format } from 'url'; +import { expectSnapshot } from '../../../common/match_snapshot'; +import { PromiseReturnType } from '../../../../../plugins/apm/typings/common'; +import { FtrProviderContext } from '../../../common/ftr_provider_context'; +import archives_metadata from '../../../common/archives_metadata'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const archiveName = 'apm_8.0.0'; + const range = archives_metadata[archiveName]; + + // url parameters + const start = '2020-09-29T14:45:00.000Z'; + const end = range.end; + const fieldNames = + 'user.username,user.id,host.ip,user_agent.name,kubernetes.pod.uuid,url.domain,container.id,service.node.name'; + + describe('Ranges', () => { + const url = format({ + pathname: `/api/apm/correlations/ranges`, + query: { start, end, fieldNames }, + }); + + describe('when data is not loaded ', () => { + it('handles the empty state', async () => { + const response = await supertest.get(url); + expect(response.status).to.be(200); + expect(response.body.response).to.be(undefined); + }); + }); + + describe('when data is loaded', () => { + let response: PromiseReturnType; + before(async () => { + await esArchiver.load(archiveName); + response = await supertest.get(url); + }); + + after(() => esArchiver.unload(archiveName)); + + it('returns successfully', () => { + expect(response.status).to.eql(200); + }); + + it('returns fields in response', () => { + expectSnapshot(Object.keys(response.body.response)).toMatchInline(` + Array [ + "service.node.name", + "host.ip", + "user.id", + "user_agent.name", + "container.id", + "url.domain", + ] + `); + }); + + it('returns cardinality for each field', () => { + const cardinalitys = Object.values(response.body.response).map( + (field: any) => field.cardinality + ); + + expectSnapshot(cardinalitys).toMatchInline(` + Array [ + 5, + 6, + 20, + 6, + 5, + 4, + ] + `); + }); + + it('returns buckets', () => { + const { buckets } = response.body.response['user.id'].value; + expectSnapshot(buckets[0]).toMatchInline(` + Object { + "bg_count": 2, + "doc_count": 7, + "key": "20", + "score": 3.5, + } + `); + }); + }); + }); +} diff --git a/x-pack/test/apm_api_integration/basic/tests/correlations/slow_durations.ts b/x-pack/test/apm_api_integration/basic/tests/correlations/slow_durations.ts new file mode 100644 index 0000000000000..78dca5100dece --- /dev/null +++ b/x-pack/test/apm_api_integration/basic/tests/correlations/slow_durations.ts @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import { format } from 'url'; +import { PromiseReturnType } from '../../../../../plugins/apm/typings/common'; +import { FtrProviderContext } from '../../../common/ftr_provider_context'; +import { expectSnapshot } from '../../../common/match_snapshot'; +import archives_metadata from '../../../common/archives_metadata'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const archiveName = 'apm_8.0.0'; + const range = archives_metadata[archiveName]; + + // url parameters + const start = range.start; + const end = range.end; + const durationPercentile = 95; + const fieldNames = + 'user.username,user.id,host.ip,user_agent.name,kubernetes.pod.uuid,url.domain,container.id,service.node.name'; + + describe('Slow durations', () => { + const url = format({ + pathname: `/api/apm/correlations/slow_durations`, + query: { start, end, durationPercentile, fieldNames }, + }); + + describe('when data is not loaded ', () => { + it('handles the empty state', async () => { + const response = await supertest.get(url); + + expect(response.status).to.be(200); + expect(response.body.response).to.be(undefined); + }); + }); + + describe('with default scoring', () => { + let response: PromiseReturnType; + before(async () => { + await esArchiver.load(archiveName); + response = await supertest.get(url); + }); + + after(() => esArchiver.unload(archiveName)); + + it('returns successfully', () => { + expect(response.status).to.eql(200); + }); + + it('returns fields in response', () => { + expectSnapshot(Object.keys(response.body.response)).toMatchInline(` + Array [ + "service.node.name", + "host.ip", + "user.id", + "user_agent.name", + "container.id", + "url.domain", + ] + `); + }); + + it('returns cardinality for each field', () => { + const cardinalitys = Object.values(response.body.response).map( + (field: any) => field.cardinality + ); + + expectSnapshot(cardinalitys).toMatchInline(` + Array [ + 5, + 6, + 3, + 5, + 5, + 4, + ] + `); + }); + + it('returns buckets', () => { + const { buckets } = response.body.response['user.id'].value; + expectSnapshot(buckets[0]).toMatchInline(` + Object { + "bg_count": 32, + "doc_count": 6, + "key": "2", + "score": 0.1875, + } + `); + }); + }); + + describe('with different scoring', () => { + before(async () => esArchiver.load(archiveName)); + after(() => esArchiver.unload(archiveName)); + + it(`returns buckets for each score`, async () => { + const promises = ['percentage', 'jlh', 'chi_square', 'gnd'].map(async (scoring) => { + const response = await supertest.get( + format({ + pathname: `/api/apm/correlations/slow_durations`, + query: { start, end, durationPercentile, fieldNames, scoring }, + }) + ); + + return { name: scoring, value: response.body.response['user.id'].value.buckets[0].score }; + }); + + const res = await Promise.all(promises); + expectSnapshot(res).toMatchInline(` + Array [ + Object { + "name": "percentage", + "value": 0.1875, + }, + Object { + "name": "jlh", + "value": 3.33506905769659, + }, + Object { + "name": "chi_square", + "value": 219.192006524483, + }, + Object { + "name": "gnd", + "value": 0.671406580688819, + }, + ] + `); + }); + }); + }); +} diff --git a/x-pack/test/apm_api_integration/basic/tests/index.ts b/x-pack/test/apm_api_integration/basic/tests/index.ts index 19dd82d617bd9..df3e60d79aca5 100644 --- a/x-pack/test/apm_api_integration/basic/tests/index.ts +++ b/x-pack/test/apm_api_integration/basic/tests/index.ts @@ -56,5 +56,10 @@ export default function apmApiIntegrationTests({ loadTestFile }: FtrProviderCont describe('Metrics', function () { loadTestFile(require.resolve('./metrics_charts/metrics_charts')); }); + + describe('Correlations', function () { + loadTestFile(require.resolve('./correlations/slow_durations')); + loadTestFile(require.resolve('./correlations/ranges')); + }); }); }