From ca42b955e87923d9d5cea03e66f352145804e5cf Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Mon, 24 May 2021 15:18:00 -0700 Subject: [PATCH 01/35] add comments --- .../pages/DetectorResults/containers/AnomalyHistory.tsx | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/public/pages/DetectorResults/containers/AnomalyHistory.tsx b/public/pages/DetectorResults/containers/AnomalyHistory.tsx index f2ef5741..a7dba6c1 100644 --- a/public/pages/DetectorResults/containers/AnomalyHistory.tsx +++ b/public/pages/DetectorResults/containers/AnomalyHistory.tsx @@ -307,6 +307,9 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { const fetchHCAnomalySummaries = async () => { setIsLoadingAnomalyResults(true); + + // This query may work fine. It is doing a terms aggregation based on "entity.value". Combining + // these into unique buckets for each combination may not work as expected though const query = getTopAnomalousEntitiesQuery( dateRange.startDate, dateRange.endDate, @@ -317,12 +320,18 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { taskId.current ); const result = await dispatch(searchResults(query)); + + // This should work as expected. For each bucket, it's parsing to get the summary for each const topEnityAnomalySummaries = parseTopEntityAnomalySummaryResults( result ); const entities = topEnityAnomalySummaries.map((summary) => summary.entity); const promises = entities.map(async (entity: Entity) => { + // This is getting the anomaly summary per entity + // Currently runs a term query to make sure the categorical field (ex: "host") and field value (ex: "i-5xysgt") exist + // One soln may be adding additional terms queries for a second categorical field & value. Or, possibly removing + // categorical field altogether? Seems the field value may be all that's needed here const entityResultQuery = getEntityAnomalySummariesQuery( dateRange.startDate, dateRange.endDate, From 0c2a49565402d1ad3bb8911b18769fa69e189e42 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Mon, 24 May 2021 17:18:49 -0700 Subject: [PATCH 02/35] Add multi category fields in combo box --- .../CategoryField/CategoryField.tsx | 193 ++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 public/pages/EditFeatures/components/CategoryField/CategoryField.tsx diff --git a/public/pages/EditFeatures/components/CategoryField/CategoryField.tsx b/public/pages/EditFeatures/components/CategoryField/CategoryField.tsx new file mode 100644 index 00000000..0bbd608e --- /dev/null +++ b/public/pages/EditFeatures/components/CategoryField/CategoryField.tsx @@ -0,0 +1,193 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { + EuiFlexItem, + EuiFlexGroup, + EuiText, + EuiLink, + EuiIcon, + EuiFormRow, + EuiPage, + EuiPageBody, + EuiComboBox, + EuiCheckbox, + EuiTitle, + EuiCallOut, + EuiSpacer, +} from '@elastic/eui'; +import { Field, FieldProps } from 'formik'; +import { get, isEmpty } from 'lodash'; +import { + MULTI_ENTITY_SHINGLE_SIZE, + BASE_DOCS_LINK, +} from '../../../../utils/constants'; +import React, { useState, useEffect } from 'react'; +import ContentPanel from '../../../../components/ContentPanel/ContentPanel'; +import { + isInvalid, + getError, + validateCategoryField, +} from '../../../../utils/utils'; + +interface CategoryFieldProps { + isHCDetector: boolean; + categoryFieldOptions: string[]; + setIsHCDetector(isHCDetector: boolean): void; + isLoading: boolean; + originalShingleSize: number; +} + +export function CategoryField(props: CategoryFieldProps) { + const [enabled, setEnabled] = useState(props.isHCDetector); + const noCategoryFields = isEmpty(props.categoryFieldOptions); + const convertedOptions = props.categoryFieldOptions.map((option: string) => { + return { + label: option, + }; + }); + + useEffect(() => { + setEnabled(props.isHCDetector); + }, [props.isHCDetector]); + + return ( + + + +

Category field

+ + } + subTitle={ + + Categorize anomalies based on unique partitions. For example, with + clickstream data you can categorize anomalies into a given day, + week, or month.{' '} + + Learn more + + + } + > + {noCategoryFields && !props.isLoading ? ( + + ) : null} + {noCategoryFields ? : null} + + {({ field, form }: FieldProps) => ( + + + { + if (!enabled) { + props.setIsHCDetector(true); + } + if (enabled) { + props.setIsHCDetector(false); + form.setFieldValue('categoryField', []); + form.setFieldValue( + 'shingleSize', + props.originalShingleSize + ); + } + setEnabled(!enabled); + }} + /> + + {enabled && !noCategoryFields ? ( + + + { + form.setFieldTouched('categoryField', true); + }} + onChange={(options) => { + const selection = options.map( + (option) => option.label + ); + if (!isEmpty(selection)) { + if (selection.length <= 2) { + form.setFieldValue('categoryField', selection); + form.setFieldValue( + 'shingleSize', + MULTI_ENTITY_SHINGLE_SIZE + ); + } + } else { + form.setFieldValue('categoryField', []); + + form.setFieldValue( + 'shingleSize', + props.originalShingleSize + ); + } + }} + selectedOptions={ + field.value + ? field.value.map((value: any) => { + return { + label: value, + }; + }) + : [] + } + singleSelection={false} + isClearable={true} + /> + + + ) : null} + + )} + + + + + ); +} From 296df97f9f33f923335d334701208cdda54c1e87 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 25 May 2021 15:05:57 -0700 Subject: [PATCH 03/35] Set character limit on heatmap chart y axis --- .../AnomalyCharts/containers/AnomalyHeatmapChart.tsx | 10 ++++++++++ public/pages/AnomalyCharts/utils/constants.ts | 2 ++ 2 files changed, 12 insertions(+) diff --git a/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx b/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx index 4310b37a..11955ee0 100644 --- a/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx +++ b/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx @@ -56,6 +56,7 @@ import { } from '../utils/anomalyChartUtils'; import { MIN_IN_MILLI_SECS } from '../../../../server/utils/constants'; import { EntityAnomalySummaries } from '../../../../server/models/interfaces'; +import { HEATMAP_CHART_Y_AXIS_WIDTH } from '../utils/constants'; interface AnomalyHeatmapChartProps { title: string; @@ -542,6 +543,15 @@ export const AnomalyHeatmapChart = React.memo( showline: true, showgrid: false, fixedrange: true, + automargin: true, + tickmode: 'array', + tickvals: heatmapData[0].y, + ticktext: heatmapData[0].y.map((label: string) => + label.length <= HEATMAP_CHART_Y_AXIS_WIDTH + ? label + : label.substring(0, HEATMAP_CHART_Y_AXIS_WIDTH - 3) + + '...' + ), }, margin: { l: 100, diff --git a/public/pages/AnomalyCharts/utils/constants.ts b/public/pages/AnomalyCharts/utils/constants.ts index cd58f5dd..50350b5c 100644 --- a/public/pages/AnomalyCharts/utils/constants.ts +++ b/public/pages/AnomalyCharts/utils/constants.ts @@ -106,3 +106,5 @@ export const DEFAULT_ANOMALY_SUMMARY = { maxConfidence: 0, lastAnomalyOccurrence: '-', }; + +export const HEATMAP_CHART_Y_AXIS_WIDTH = 30; From 83244aecbc047f37c54f01b117307b96db389cec Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 26 May 2021 15:13:01 -0700 Subject: [PATCH 04/35] Support multi category for bucket agg query and result parsing --- .../containers/AnomalyHistory.tsx | 8 +- public/pages/utils/anomalyResultUtils.ts | 209 +++++++++++++----- server/utils/constants.ts | 2 + 3 files changed, 158 insertions(+), 61 deletions(-) diff --git a/public/pages/DetectorResults/containers/AnomalyHistory.tsx b/public/pages/DetectorResults/containers/AnomalyHistory.tsx index a7dba6c1..b612ec63 100644 --- a/public/pages/DetectorResults/containers/AnomalyHistory.tsx +++ b/public/pages/DetectorResults/containers/AnomalyHistory.tsx @@ -165,6 +165,7 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { const detectorCategoryField = get(props.detector, 'categoryField', []); const isHCDetector = !isEmpty(detectorCategoryField); + const isMultiCategory = detectorCategoryField.length > 1; const backgroundColor = darkModeEnabled() ? '#29017' : '#F7F7F7'; // We load at most 10k AD result data points for one call. If user choose @@ -308,22 +309,21 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { const fetchHCAnomalySummaries = async () => { setIsLoadingAnomalyResults(true); - // This query may work fine. It is doing a terms aggregation based on "entity.value". Combining - // these into unique buckets for each combination may not work as expected though const query = getTopAnomalousEntitiesQuery( dateRange.startDate, dateRange.endDate, props.detector.id, heatmapDisplayOption.entityOption.value, heatmapDisplayOption.sortType, + isMultiCategory, props.isHistorical, taskId.current ); const result = await dispatch(searchResults(query)); - // This should work as expected. For each bucket, it's parsing to get the summary for each const topEnityAnomalySummaries = parseTopEntityAnomalySummaryResults( - result + result, + isMultiCategory ); const entities = topEnityAnomalySummaries.map((summary) => summary.entity); diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index 63125ad1..2acedce1 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -44,6 +44,8 @@ import { DAY_IN_MILLI_SECS, SORT_DIRECTION, WEEK_IN_MILLI_SECS, + MODEL_ID_FIELD, + ENTITY_LIST_FIELD, } from '../../../server/utils/constants'; import { toFixedNumberForAnomaly } from '../../../server/utils/helpers'; import { @@ -992,64 +994,67 @@ export const getTopAnomalousEntitiesQuery = ( detectorId: string, size: number, sortType: AnomalyHeatmapSortType, + isMultiCategory: boolean, isHistorical?: boolean, taskId?: string ) => { const termField = isHistorical && taskId ? { task_id: taskId } : { detector_id: detectorId }; - const requestBody = { - size: 0, - query: { - bool: { - filter: [ - { - range: { - [AD_DOC_FIELDS.ANOMALY_GRADE]: { - gt: 0, + // To handle BWC, we will return 2 possible queries based on the # of categorical fields: + // (1) legacy way (1 category field): bucket aggregate over the single, nested, 'entity.value' field + // (2) new way (>= 2 category fields): bucket aggregate over the new 'model_id' field + const requestBody = isMultiCategory + ? { + size: 0, + query: { + bool: { + filter: [ + { + range: { + [AD_DOC_FIELDS.ANOMALY_GRADE]: { + gt: 0, + }, + }, }, - }, - }, - { - range: { - data_end_time: { - gte: startTime, - lte: endTime, + { + range: { + data_end_time: { + gte: startTime, + lte: endTime, + }, + }, }, - }, - }, - { - term: termField, + { + term: termField, + }, + ], }, - ], - }, - }, - aggs: { - [TOP_ENTITIES_FIELD]: { - nested: { - path: ENTITY_FIELD, }, aggs: { [TOP_ENTITY_AGGS]: { terms: { - field: ENTITY_VALUE_PATH_FIELD, + field: MODEL_ID_FIELD, size: size, ...(sortType === AnomalyHeatmapSortType.SEVERITY ? { order: { - [TOP_ANOMALY_GRADE_SORT_AGGS]: SORT_DIRECTION.DESC, + [MAX_ANOMALY_AGGS]: SORT_DIRECTION.DESC, }, } : {}), }, aggs: { - [TOP_ANOMALY_GRADE_SORT_AGGS]: { - reverse_nested: {}, - aggs: { - [MAX_ANOMALY_AGGS]: { - max: { - field: AD_DOC_FIELDS.ANOMALY_GRADE, - }, + [MAX_ANOMALY_AGGS]: { + max: { + field: AD_DOC_FIELDS.ANOMALY_GRADE, + }, + }, + [ENTITY_LIST_FIELD]: { + top_hits: { + size: 1, + _source: { + include: [ENTITY_FIELD], }, }, }, @@ -1059,7 +1064,7 @@ export const getTopAnomalousEntitiesQuery = ( bucket_sort: { sort: [ { - [`${TOP_ANOMALY_GRADE_SORT_AGGS}.${MAX_ANOMALY_AGGS}`]: { + [`${MAX_ANOMALY_AGGS}`]: { order: SORT_DIRECTION.DESC, }, }, @@ -1071,9 +1076,85 @@ export const getTopAnomalousEntitiesQuery = ( }, }, }, - }, - }, - }; + } + : { + size: 0, + query: { + bool: { + filter: [ + { + range: { + [AD_DOC_FIELDS.ANOMALY_GRADE]: { + gt: 0, + }, + }, + }, + { + range: { + data_end_time: { + gte: startTime, + lte: endTime, + }, + }, + }, + { + term: { + detector_id: detectorId, + }, + }, + ], + }, + }, + aggs: { + [TOP_ENTITIES_FIELD]: { + nested: { + path: ENTITY_FIELD, + }, + aggs: { + [TOP_ENTITY_AGGS]: { + terms: { + field: ENTITY_VALUE_PATH_FIELD, + size: size, + ...(sortType === AnomalyHeatmapSortType.SEVERITY + ? { + order: { + [TOP_ANOMALY_GRADE_SORT_AGGS]: SORT_DIRECTION.DESC, + }, + } + : {}), + }, + aggs: { + [TOP_ANOMALY_GRADE_SORT_AGGS]: { + reverse_nested: {}, + aggs: { + [MAX_ANOMALY_AGGS]: { + max: { + field: AD_DOC_FIELDS.ANOMALY_GRADE, + }, + }, + }, + }, + ...(sortType === AnomalyHeatmapSortType.SEVERITY + ? { + [MAX_ANOMALY_SORT_AGGS]: { + bucket_sort: { + sort: [ + { + [`${TOP_ANOMALY_GRADE_SORT_AGGS}.${MAX_ANOMALY_AGGS}`]: { + order: SORT_DIRECTION.DESC, + }, + }, + ], + }, + }, + } + : {}), + }, + }, + }, + }, + }, + }; if (!isHistorical) { requestBody.query.bool = { @@ -1092,25 +1173,30 @@ export const getTopAnomalousEntitiesQuery = ( }; export const parseTopEntityAnomalySummaryResults = ( - result: any + result: any, + isMultiCategory: boolean ): EntityAnomalySummaries[] => { - const rawEntityAnomalySummaries = get( - result, - `response.aggregations.${TOP_ENTITIES_FIELD}.${TOP_ENTITY_AGGS}.buckets`, - [] - ) as any[]; + const rawEntityAnomalySummaries = isMultiCategory + ? get(result, `response.aggregations.${TOP_ENTITY_AGGS}.buckets`, []) + : (get( + result, + `response.aggregations.${TOP_ENTITIES_FIELD}.${TOP_ENTITY_AGGS}.buckets`, + [] + ) as any[]); let topEntityAnomalySummaries = [] as EntityAnomalySummaries[]; - rawEntityAnomalySummaries.forEach((item) => { + rawEntityAnomalySummaries.forEach((item: any) => { const anomalyCount = get(item, DOC_COUNT_FIELD, 0); - const entityValue = get(item, KEY_FIELD, 0); + const entityValue = isMultiCategory + ? convertToEntityString( + get(item, `${ENTITY_LIST_FIELD}.hits.hits.0._source.entity`, []) + ) + : get(item, KEY_FIELD, 0); const entity = { value: entityValue, } as Entity; - const maxAnomalyGrade = get( - item, - [TOP_ANOMALY_GRADE_SORT_AGGS, MAX_ANOMALY_AGGS].join('.'), - 0 - ); + const maxAnomalyGrade = isMultiCategory + ? get(item, MAX_ANOMALY_AGGS, 0) + : get(item, [TOP_ANOMALY_GRADE_SORT_AGGS, MAX_ANOMALY_AGGS].join('.'), 0); const enityAnomalySummary = { maxAnomaly: maxAnomalyGrade, anomalyCount: anomalyCount, @@ -1251,18 +1337,27 @@ export const parseEntityAnomalySummaryResults = ( const anomalyCount = get(item, `${COUNT_ANOMALY_AGGS}.value`, 0); const startTime = get(item, 'key', 0); const maxAnomalyGrade = get(item, `${MAX_ANOMALY_AGGS}.value`, 0); - const enityAnomalySummary = { + const entityAnomalySummary = { startTime: startTime, maxAnomaly: maxAnomalyGrade, anomalyCount: anomalyCount, } as EntityAnomalySummary; - anomalySummaries.push(enityAnomalySummary); + anomalySummaries.push(entityAnomalySummary); }); - const enityAnomalySummaries = { + const entityAnomalySummaries = { entity: entity, anomalySummaries: anomalySummaries, } as EntityAnomalySummaries; - return enityAnomalySummaries; + return entityAnomalySummaries; +}; + +const convertToEntityString = (entityArray: any[]) => { + let entityString = ''; + entityArray.forEach((entity: any) => { + entityString += entity.value; + entityString += '/'; + }); + return entityString.slice(0, -1); }; export const getAnomalyDataRangeQuery = ( diff --git a/server/utils/constants.ts b/server/utils/constants.ts index 1aa14e2f..7434467e 100644 --- a/server/utils/constants.ts +++ b/server/utils/constants.ts @@ -101,9 +101,11 @@ export enum SAMPLE_TYPE { export const ENTITY_FIELD = 'entity'; export const ENTITY_VALUE_PATH_FIELD = 'entity.value'; export const ENTITY_NAME_PATH_FIELD = 'entity.name'; +export const MODEL_ID_FIELD = 'model_id'; export const DOC_COUNT_FIELD = 'doc_count'; export const KEY_FIELD = 'key'; +export const ENTITY_LIST_FIELD = 'entity_list'; export const STACK_TRACE_PATTERN = '.java:'; export const OPENSEARCH_EXCEPTION_PREFIX = From 64749f19e758d00564a781b44a74f84609867ea6 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 26 May 2021 17:38:30 -0700 Subject: [PATCH 05/35] Persist modelId to support multi category in parseTopEntityAnomalySummaryResults --- .../containers/AnomalyHistory.tsx | 45 ++++++------ public/pages/utils/anomalyResultUtils.ts | 71 +++++++++++-------- server/models/interfaces.ts | 1 + 3 files changed, 69 insertions(+), 48 deletions(-) diff --git a/public/pages/DetectorResults/containers/AnomalyHistory.tsx b/public/pages/DetectorResults/containers/AnomalyHistory.tsx index b612ec63..5a9b23d8 100644 --- a/public/pages/DetectorResults/containers/AnomalyHistory.tsx +++ b/public/pages/DetectorResults/containers/AnomalyHistory.tsx @@ -321,29 +321,28 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { ); const result = await dispatch(searchResults(query)); - const topEnityAnomalySummaries = parseTopEntityAnomalySummaryResults( + const topEntityAnomalySummaries = parseTopEntityAnomalySummaryResults( result, isMultiCategory ); - const entities = topEnityAnomalySummaries.map((summary) => summary.entity); - - const promises = entities.map(async (entity: Entity) => { - // This is getting the anomaly summary per entity - // Currently runs a term query to make sure the categorical field (ex: "host") and field value (ex: "i-5xysgt") exist - // One soln may be adding additional terms queries for a second categorical field & value. Or, possibly removing - // categorical field altogether? Seems the field value may be all that's needed here - const entityResultQuery = getEntityAnomalySummariesQuery( - dateRange.startDate, - dateRange.endDate, - props.detector.id, - NUM_CELLS, - get(props.detector, 'categoryField[0]', ''), - entity.value, - props.isHistorical, - taskId.current - ); - return dispatch(searchResults(entityResultQuery)); - }); + + const promises = topEntityAnomalySummaries.map( + async (summary: EntityAnomalySummaries) => { + const entityResultQuery = getEntityAnomalySummariesQuery( + dateRange.startDate, + dateRange.endDate, + props.detector.id, + NUM_CELLS, + get(props.detector, 'categoryField[0]', ''), + summary.entity.value, + props.isHistorical, + taskId.current, + isMultiCategory, + summary.modelId + ); + return dispatch(searchResults(entityResultQuery)); + } + ); const allEntityAnomalySummaries = await Promise.all(promises).catch( (error) => { @@ -352,9 +351,15 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { core.notifications.toasts.addDanger(prettifyErrorMessage(errorMessage)); } ); + + console.log('all entity anomaly summaries: ', allEntityAnomalySummaries); + const entitiesAnomalySummaries = [] as EntityAnomalySummaries[]; if (!isEmpty(allEntityAnomalySummaries)) { + const entities = topEntityAnomalySummaries.map( + (summary) => summary.entity + ); //@ts-ignore allEntityAnomalySummaries.forEach((entityResponse, i) => { const entityAnomalySummariesResult = parseEntityAnomalySummaryResults( diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index 2acedce1..62d36072 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -1197,15 +1197,18 @@ export const parseTopEntityAnomalySummaryResults = ( const maxAnomalyGrade = isMultiCategory ? get(item, MAX_ANOMALY_AGGS, 0) : get(item, [TOP_ANOMALY_GRADE_SORT_AGGS, MAX_ANOMALY_AGGS].join('.'), 0); - const enityAnomalySummary = { + const entityAnomalySummary = { maxAnomaly: maxAnomalyGrade, anomalyCount: anomalyCount, } as EntityAnomalySummary; - const enityAnomaliSummaries = { + const entityAnomalySummaries = { entity: entity, - anomalySummaries: [enityAnomalySummary], + anomalySummaries: [entityAnomalySummary], } as EntityAnomalySummaries; - topEntityAnomalySummaries.push(enityAnomaliSummaries); + if (isMultiCategory) { + entityAnomalySummaries.modelId = get(item, KEY_FIELD, ''); + } + topEntityAnomalySummaries.push(entityAnomalySummaries); }); return topEntityAnomalySummaries; }; @@ -1218,7 +1221,9 @@ export const getEntityAnomalySummariesQuery = ( categoryField: string, entityValue: string, isHistorical?: boolean, - taskId?: string + taskId?: string, + isMultiCategory: boolean, + modelId: string | undefined ) => { const termField = isHistorical && taskId ? { task_id: taskId } : { detector_id: detectorId }; @@ -1232,6 +1237,34 @@ export const getEntityAnomalySummariesQuery = ( // if startTime is not divisible by fixedInterval, there will be remainder, // this can be offset for bucket_key const offsetInMillisec = startTime % (fixedInterval * MIN_IN_MILLI_SECS); + const entityValueFilter = { + nested: { + path: ENTITY_FIELD, + query: { + term: { + [ENTITY_VALUE_PATH_FIELD]: { + value: entityValue, + }, + }, + }, + }, + }; + const categoryFieldFilter = { + nested: { + path: ENTITY_FIELD, + query: { + term: { + [ENTITY_NAME_PATH_FIELD]: { value: categoryField }, + }, + }, + }, + }; + const modelIdFilter = { + term: { + model_id: modelId, + }, + }; + const requestBody = { size: 0, query: { @@ -1255,30 +1288,12 @@ export const getEntityAnomalySummariesQuery = ( { term: termField, }, - { - nested: { - path: ENTITY_FIELD, - query: { - term: { - [ENTITY_VALUE_PATH_FIELD]: { - value: entityValue, - }, - }, - }, - }, - }, - { - nested: { - path: ENTITY_FIELD, - query: { - term: { - [ENTITY_NAME_PATH_FIELD]: { - value: categoryField, - }, - }, + isMultiCategory + ? modelIdFilter + : { + ...entityValueFilter, + ...categoryFieldFilter, }, - }, - }, ], }, }, diff --git a/server/models/interfaces.ts b/server/models/interfaces.ts index 7b427812..8c87dfaa 100644 --- a/server/models/interfaces.ts +++ b/server/models/interfaces.ts @@ -92,4 +92,5 @@ export interface EntityAnomalySummary { export interface EntityAnomalySummaries { entity: Entity; anomalySummaries: EntityAnomalySummary[]; + modelId?: string; } From 7afabf4467b740f9c6bf59d10e84285042004b6e Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Thu, 27 May 2021 09:25:23 -0700 Subject: [PATCH 06/35] Clean up filtering for all HC result queries --- .../containers/AnomalyHeatmapChart.tsx | 1 + .../containers/AnomalyHistory.tsx | 20 ++- public/pages/utils/anomalyResultUtils.ts | 142 ++++++------------ 3 files changed, 61 insertions(+), 102 deletions(-) diff --git a/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx b/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx index 11955ee0..849bf21e 100644 --- a/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx +++ b/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx @@ -78,6 +78,7 @@ interface AnomalyHeatmapChartProps { export interface HeatmapCell { dateRange: DateRange; entityValue: string; + modelId?: string; } export interface HeatmapDisplayOption { diff --git a/public/pages/DetectorResults/containers/AnomalyHistory.tsx b/public/pages/DetectorResults/containers/AnomalyHistory.tsx index 5a9b23d8..73962b23 100644 --- a/public/pages/DetectorResults/containers/AnomalyHistory.tsx +++ b/public/pages/DetectorResults/containers/AnomalyHistory.tsx @@ -173,7 +173,8 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { // aggregation to load data points in whole time range with larger interval. // If entity is specified, we only query AD result data points for this entity. async function getBucketizedAnomalyResults( - entity: Entity | undefined = undefined + entity: Entity | undefined = undefined, + modelId?: string ) { try { setIsLoadingAnomalyResults(true); @@ -185,7 +186,9 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { props.detector.id, entity, props.isHistorical, - taskId.current + taskId.current, + isMultiCategory, + modelId ) ) ); @@ -201,7 +204,9 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { props.detector.id, entity, props.isHistorical, - taskId.current + taskId.current, + isMultiCategory, + modelId ) ) ); @@ -333,12 +338,11 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { dateRange.endDate, props.detector.id, NUM_CELLS, - get(props.detector, 'categoryField[0]', ''), summary.entity.value, - props.isHistorical, - taskId.current, isMultiCategory, - summary.modelId + summary.modelId, + props.isHistorical, + taskId.current ); return dispatch(searchResults(entityResultQuery)); } @@ -383,8 +387,10 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { MAX_ANOMALIES ) ) { + // TODO: inject model id in heatmapCell to propagate fetchBucketizedEntityAnomalyData(heatmapCell); } else { + // TODO: inject model id in heatmapCell to propagate fetchAllEntityAnomalyData(heatmapCell); setBucketizedAnomalyResults(undefined); } diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index 62d36072..f22577ce 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -312,7 +312,9 @@ export const getAnomalySummaryQuery = ( detectorId: string, entity: Entity | undefined = undefined, isHistorical?: boolean, - taskId?: string + taskId?: string, + isMultiCategory?: boolean, + modelId?: string ) => { const termField = isHistorical && taskId ? { task_id: taskId } : { detector_id: detectorId }; @@ -339,34 +341,11 @@ export const getAnomalySummaryQuery = ( { term: termField, }, - ...(entity - ? [ - { - nested: { - path: ENTITY_FIELD, - query: { - term: { - [ENTITY_VALUE_PATH_FIELD]: { - value: entity.value, - }, - }, - }, - }, - }, - { - nested: { - path: ENTITY_FIELD, - query: { - term: { - [ENTITY_NAME_PATH_FIELD]: { - value: entity.name, - }, - }, - }, - }, - }, - ] - : []), + getResultFilters( + isMultiCategory, + modelId, + entity ? entity.value : undefined + ), ], }, }, @@ -434,7 +413,9 @@ export const getBucketizedAnomalyResultsQuery = ( detectorId: string, entity: Entity | undefined = undefined, isHistorical?: boolean, - taskId?: string + taskId?: string, + isMultiCategory?: boolean, + modelId?: string ) => { const termField = isHistorical && taskId ? { task_id: taskId } : { detector_id: detectorId }; @@ -457,34 +438,11 @@ export const getBucketizedAnomalyResultsQuery = ( { term: termField, }, - ...(entity - ? [ - { - nested: { - path: ENTITY_FIELD, - query: { - term: { - [ENTITY_VALUE_PATH_FIELD]: { - value: entity.value, - }, - }, - }, - }, - }, - { - nested: { - path: ENTITY_FIELD, - query: { - term: { - [ENTITY_NAME_PATH_FIELD]: { - value: entity.name, - }, - }, - }, - }, - }, - ] - : []), + getResultFilters( + isMultiCategory, + modelId, + entity ? entity.value : undefined + ), ], }, }, @@ -1218,12 +1176,11 @@ export const getEntityAnomalySummariesQuery = ( endTime: number, detectorId: string, size: number, - categoryField: string, entityValue: string, - isHistorical?: boolean, - taskId?: string, isMultiCategory: boolean, - modelId: string | undefined + modelId: string | undefined, + isHistorical?: boolean, + taskId?: string ) => { const termField = isHistorical && taskId ? { task_id: taskId } : { detector_id: detectorId }; @@ -1237,33 +1194,6 @@ export const getEntityAnomalySummariesQuery = ( // if startTime is not divisible by fixedInterval, there will be remainder, // this can be offset for bucket_key const offsetInMillisec = startTime % (fixedInterval * MIN_IN_MILLI_SECS); - const entityValueFilter = { - nested: { - path: ENTITY_FIELD, - query: { - term: { - [ENTITY_VALUE_PATH_FIELD]: { - value: entityValue, - }, - }, - }, - }, - }; - const categoryFieldFilter = { - nested: { - path: ENTITY_FIELD, - query: { - term: { - [ENTITY_NAME_PATH_FIELD]: { value: categoryField }, - }, - }, - }, - }; - const modelIdFilter = { - term: { - model_id: modelId, - }, - }; const requestBody = { size: 0, @@ -1288,12 +1218,7 @@ export const getEntityAnomalySummariesQuery = ( { term: termField, }, - isMultiCategory - ? modelIdFilter - : { - ...entityValueFilter, - ...categoryFieldFilter, - }, + getResultFilters(isMultiCategory, modelId, entityValue), ], }, }, @@ -1535,3 +1460,30 @@ export const parseHistoricalAggregatedAnomalies = ( return anomalies; }; +// Helper fn to get the correct filters based on how many categorical fields there are. +// If there are multiple (entity list > 1), filter results by model id. +// If there is only one (entity list = 1), filter by entity value. +const getResultFilters = ( + isMultiCategory: boolean | undefined, + modelId: string | undefined, + entityValue: string | undefined +) => { + return isMultiCategory === true + ? { + term: { + model_id: modelId, + }, + } + : { + nested: { + path: ENTITY_FIELD, + query: { + term: { + [ENTITY_VALUE_PATH_FIELD]: { + value: entityValue, + }, + }, + }, + }, + }; +}; From 46876d75160b5b01c401def62098e7b410b77d3d Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 1 Jun 2021 09:13:28 -0700 Subject: [PATCH 07/35] Handle entity string edge case --- public/pages/utils/anomalyResultUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index f22577ce..32dcb1ca 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -1297,7 +1297,7 @@ const convertToEntityString = (entityArray: any[]) => { entityString += entity.value; entityString += '/'; }); - return entityString.slice(0, -1); + return entityString === '' ? entityString : entityString.slice(0, -1); }; export const getAnomalyDataRangeQuery = ( From 767110bf6cc3f521606c81a6361dab7f9252484d Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 2 Jun 2021 15:03:13 -0700 Subject: [PATCH 08/35] Change data models to support entity list instead of single entity --- .../containers/AnomalyHeatmapChart.tsx | 2 + .../AnomalyCharts/utils/anomalyChartUtils.ts | 60 ++++++++++++++----- .../containers/AnomalyHistory.tsx | 30 +++++----- public/pages/utils/anomalyResultUtils.ts | 57 +++++++++--------- server/models/interfaces.ts | 2 +- 5 files changed, 90 insertions(+), 61 deletions(-) diff --git a/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx b/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx index 849bf21e..56d435b9 100644 --- a/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx +++ b/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx @@ -77,6 +77,8 @@ interface AnomalyHeatmapChartProps { export interface HeatmapCell { dateRange: DateRange; + // TODO: change this to entityList. Will use entity list as the key to pass + // to helper fn to build the query to fetch the results entityValue: string; modelId?: string; } diff --git a/public/pages/AnomalyCharts/utils/anomalyChartUtils.ts b/public/pages/AnomalyCharts/utils/anomalyChartUtils.ts index 6dbad7a1..b8aeaf3b 100644 --- a/public/pages/AnomalyCharts/utils/anomalyChartUtils.ts +++ b/public/pages/AnomalyCharts/utils/anomalyChartUtils.ts @@ -37,14 +37,17 @@ import { RectAnnotationDatum } from '@elastic/charts'; import { DEFAULT_ANOMALY_SUMMARY } from './constants'; import { Datum, PlotData } from 'plotly.js'; import moment from 'moment'; -import { calculateTimeWindowsWithMaxDataPoints } from '../../utils/anomalyResultUtils'; +import { + calculateTimeWindowsWithMaxDataPoints, + convertToEntityString, +} from '../../utils/anomalyResultUtils'; import { HeatmapCell } from '../containers/AnomalyHeatmapChart'; import { EntityAnomalySummaries, EntityAnomalySummary, } from '../../../../server/models/interfaces'; import { toFixedNumberForAnomaly } from '../../../../server/utils/helpers'; -import { ENTITY_VALUE_PATH_FIELD } from '../../../../server/utils/constants'; +import { Entity } from '../../../../server/models/interfaces'; export const convertAlerts = (response: any): MonitorAlert[] => { const alerts = get(response, 'response.alerts', []); @@ -223,6 +226,7 @@ const buildBlankStringWithLength = (length: number) => { return result; }; +// TODO: fix this + its helper fns. Related to preview results export const getAnomaliesHeatmapData = ( anomalies: any[] | undefined, dateRange: DateRange, @@ -291,6 +295,8 @@ export const getAnomaliesHeatmapData = ( entityValues, maxAnomalyGrades, numAnomalyGrades, + // TODO: change to something like entityListsExpanded in getEntitytAnomaliesHeatmapData + [], cellTimeInterval ); const resultPlotData = sortHeatmapPlotData(plotData, sortType, displayTopNum); @@ -301,7 +307,8 @@ const buildHeatmapPlotData = ( x: any[], y: any[], z: any[], - text: any[], + anomalyOccurrences: any[], + entityLists: any[], cellTimeInterval: number ): PlotData => { //@ts-ignore @@ -317,11 +324,13 @@ const buildHeatmapPlotData = ( xgap: 2, ygap: 2, opacity: 1, - text: text, + text: anomalyOccurrences, + customdata: entityLists, hovertemplate: 'Time: %{x}
' + 'Max anomaly grade: %{z}
' + - 'Anomaly occurrences: %{text}' + + 'Anomaly occurrences: %{text}
' + + 'Entities: %{customdata}' + '', cellTimeInterval: cellTimeInterval, } as PlotData; @@ -332,7 +341,8 @@ export const getEntitytAnomaliesHeatmapData = ( entitiesAnomalySummaryResult: EntityAnomalySummaries[], displayTopNum: number ) => { - const entityValues = [] as string[]; + const entityStrings = [] as string[]; + const entityLists = [] as any[]; const maxAnomalyGrades = [] as any[]; const numAnomalyGrades = [] as any[]; @@ -350,9 +360,11 @@ export const getEntitytAnomaliesHeatmapData = ( // only 1 whitesapce for all entities, to avoid heatmap with single row const blankStrValue = buildBlankStringWithLength(i); entitiesAnomalySummaries.push({ - entity: { - value: blankStrValue, - }, + entityList: [ + { + value: blankStrValue, + }, + ], } as EntityAnomalySummaries); } } else { @@ -363,17 +375,23 @@ export const getEntitytAnomaliesHeatmapData = ( const maxAnomalyGradesForEntity = [] as number[]; const numAnomalyGradesForEntity = [] as number[]; - const entityValue = get( - entityAnomalySummaries, - ENTITY_VALUE_PATH_FIELD, - '' + const entityString = convertToEntityString( + get(entityAnomalySummaries, 'entityList', []), + ' / ' ) as string; const anomaliesSummary = get( entityAnomalySummaries, 'anomalySummaries', [] ) as EntityAnomalySummary[]; - entityValues.push(entityValue); + entityStrings.push(entityString); + + const entityList = get( + entityAnomalySummaries, + 'entityList', + [] + ) as Entity[]; + entityLists.push(entityList); timeWindows.forEach((timeWindow) => { const anomalySummaryInTimeRange = anomaliesSummary.filter( @@ -413,11 +431,23 @@ export const getEntitytAnomaliesHeatmapData = ( const timeStamps = plotTimes.map((timestamp) => moment(timestamp).format(HEATMAP_X_AXIS_DATE_FORMAT) ); + let entityListsExpanded = [] as any[]; + entityLists.forEach((entityList: Entity[]) => { + const listAsString = convertToEntityString(entityList, ', '); + let row = []; + var i; + for (i = 0; i < NUM_CELLS; i++) { + row.push(listAsString); + } + entityListsExpanded.push(row); + }); + const plotData = buildHeatmapPlotData( timeStamps, - entityValues.reverse(), + entityStrings.reverse(), maxAnomalyGrades.reverse(), numAnomalyGrades.reverse(), + entityListsExpanded.reverse(), timeWindows[0].endDate - timeWindows[0].startDate ); return [plotData]; diff --git a/public/pages/DetectorResults/containers/AnomalyHistory.tsx b/public/pages/DetectorResults/containers/AnomalyHistory.tsx index 73962b23..5b234e07 100644 --- a/public/pages/DetectorResults/containers/AnomalyHistory.tsx +++ b/public/pages/DetectorResults/containers/AnomalyHistory.tsx @@ -173,7 +173,7 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { // aggregation to load data points in whole time range with larger interval. // If entity is specified, we only query AD result data points for this entity. async function getBucketizedAnomalyResults( - entity: Entity | undefined = undefined, + entityList: Entity[] | undefined = undefined, modelId?: string ) { try { @@ -184,7 +184,7 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { dateRange.startDate, dateRange.endDate, props.detector.id, - entity, + entityList, props.isHistorical, taskId.current, isMultiCategory, @@ -202,7 +202,7 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { dateRange.startDate, dateRange.endDate, props.detector.id, - entity, + entityList, props.isHistorical, taskId.current, isMultiCategory, @@ -338,7 +338,7 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { dateRange.endDate, props.detector.id, NUM_CELLS, - summary.entity.value, + summary.entityList, isMultiCategory, summary.modelId, props.isHistorical, @@ -355,24 +355,22 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { core.notifications.toasts.addDanger(prettifyErrorMessage(errorMessage)); } ); - - console.log('all entity anomaly summaries: ', allEntityAnomalySummaries); - const entitiesAnomalySummaries = [] as EntityAnomalySummaries[]; if (!isEmpty(allEntityAnomalySummaries)) { - const entities = topEntityAnomalySummaries.map( - (summary) => summary.entity + const entityLists = topEntityAnomalySummaries.map( + (summary) => summary.entityList ); //@ts-ignore allEntityAnomalySummaries.forEach((entityResponse, i) => { const entityAnomalySummariesResult = parseEntityAnomalySummaryResults( entityResponse, - entities[i] + entityLists[i] ); entitiesAnomalySummaries.push(entityAnomalySummariesResult); }); } + setEntityAnomalySummaries(entitiesAnomalySummaries); setIsLoadingAnomalyResults(false); }; @@ -438,11 +436,13 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { }; const fetchBucketizedEntityAnomalyData = async (heatmapCell: HeatmapCell) => { - getBucketizedAnomalyResults({ - //@ts-ignore - name: props.detector.categoryField[0], - value: heatmapCell.entityValue, - }); + getBucketizedAnomalyResults([ + { + //@ts-ignore + name: props.detector.categoryField[0], + value: heatmapCell.entityValue, + }, + ]); }; const [atomicAnomalyResults, setAtomicAnomalyResults] = useState(); const [rawAnomalyResults, setRawAnomalyResults] = useState(); diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index 32dcb1ca..efa93e61 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -310,7 +310,7 @@ export const getAnomalySummaryQuery = ( startTime: number, endTime: number, detectorId: string, - entity: Entity | undefined = undefined, + entityList: Entity[] | undefined = undefined, isHistorical?: boolean, taskId?: string, isMultiCategory?: boolean, @@ -341,11 +341,7 @@ export const getAnomalySummaryQuery = ( { term: termField, }, - getResultFilters( - isMultiCategory, - modelId, - entity ? entity.value : undefined - ), + getResultFilters(isMultiCategory, modelId, entityList), ], }, }, @@ -411,7 +407,7 @@ export const getBucketizedAnomalyResultsQuery = ( startTime: number, endTime: number, detectorId: string, - entity: Entity | undefined = undefined, + entityList: Entity[] | undefined = undefined, isHistorical?: boolean, taskId?: string, isMultiCategory?: boolean, @@ -438,11 +434,7 @@ export const getBucketizedAnomalyResultsQuery = ( { term: termField, }, - getResultFilters( - isMultiCategory, - modelId, - entity ? entity.value : undefined - ), + getResultFilters(isMultiCategory, modelId, entityList), ], }, }, @@ -1144,14 +1136,14 @@ export const parseTopEntityAnomalySummaryResults = ( let topEntityAnomalySummaries = [] as EntityAnomalySummaries[]; rawEntityAnomalySummaries.forEach((item: any) => { const anomalyCount = get(item, DOC_COUNT_FIELD, 0); - const entityValue = isMultiCategory - ? convertToEntityString( - get(item, `${ENTITY_LIST_FIELD}.hits.hits.0._source.entity`, []) - ) - : get(item, KEY_FIELD, 0); - const entity = { - value: entityValue, - } as Entity; + const entityList = isMultiCategory + ? get(item, `${ENTITY_LIST_FIELD}.hits.hits.0._source.entity`, []) + : ([ + { + value: get(item, KEY_FIELD, 0), + }, + ] as Entity[]); + const maxAnomalyGrade = isMultiCategory ? get(item, MAX_ANOMALY_AGGS, 0) : get(item, [TOP_ANOMALY_GRADE_SORT_AGGS, MAX_ANOMALY_AGGS].join('.'), 0); @@ -1160,7 +1152,7 @@ export const parseTopEntityAnomalySummaryResults = ( anomalyCount: anomalyCount, } as EntityAnomalySummary; const entityAnomalySummaries = { - entity: entity, + entityList: entityList, anomalySummaries: [entityAnomalySummary], } as EntityAnomalySummaries; if (isMultiCategory) { @@ -1176,7 +1168,7 @@ export const getEntityAnomalySummariesQuery = ( endTime: number, detectorId: string, size: number, - entityValue: string, + entityList: Entity[], isMultiCategory: boolean, modelId: string | undefined, isHistorical?: boolean, @@ -1218,7 +1210,7 @@ export const getEntityAnomalySummariesQuery = ( { term: termField, }, - getResultFilters(isMultiCategory, modelId, entityValue), + getResultFilters(isMultiCategory, modelId, entityList), ], }, }, @@ -1265,7 +1257,7 @@ export const getEntityAnomalySummariesQuery = ( export const parseEntityAnomalySummaryResults = ( result: any, - entity: Entity + entityList: Entity[] ): EntityAnomalySummaries => { const rawEntityAnomalySummaries = get( result, @@ -1285,19 +1277,24 @@ export const parseEntityAnomalySummaryResults = ( anomalySummaries.push(entityAnomalySummary); }); const entityAnomalySummaries = { - entity: entity, + entityList: entityList, anomalySummaries: anomalySummaries, } as EntityAnomalySummaries; return entityAnomalySummaries; }; -const convertToEntityString = (entityArray: any[]) => { +export const convertToEntityString = ( + entityArray: any[], + delimiter: string +) => { let entityString = ''; entityArray.forEach((entity: any) => { entityString += entity.value; - entityString += '/'; + entityString += delimiter; }); - return entityString === '' ? entityString : entityString.slice(0, -1); + return entityString === '' + ? entityString + : entityString.slice(0, -delimiter.length); }; export const getAnomalyDataRangeQuery = ( @@ -1466,7 +1463,7 @@ export const parseHistoricalAggregatedAnomalies = ( const getResultFilters = ( isMultiCategory: boolean | undefined, modelId: string | undefined, - entityValue: string | undefined + entityList: Entity[] | undefined ) => { return isMultiCategory === true ? { @@ -1480,7 +1477,7 @@ const getResultFilters = ( query: { term: { [ENTITY_VALUE_PATH_FIELD]: { - value: entityValue, + value: get(entityList, '0.value', ''), }, }, }, diff --git a/server/models/interfaces.ts b/server/models/interfaces.ts index 8c87dfaa..881eb6b3 100644 --- a/server/models/interfaces.ts +++ b/server/models/interfaces.ts @@ -90,7 +90,7 @@ export interface EntityAnomalySummary { } export interface EntityAnomalySummaries { - entity: Entity; + entityList: Entity[]; anomalySummaries: EntityAnomalySummary[]; modelId?: string; } From 7ace4568bd3c4ccf08b3d9e2be595396a76d3dfe Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 2 Jun 2021 17:01:55 -0700 Subject: [PATCH 09/35] Store all entity list data in heatmap cell; add ability to see results in charts when cell selected --- .../containers/AnomaliesChart.tsx | 17 ++- .../containers/AnomalyHeatmapChart.tsx | 26 ++-- .../containers/FeatureBreakDown.tsx | 28 +++- .../containers/AnomalyHistory.tsx | 136 ++++++++++++------ public/pages/utils/__tests__/constants.ts | 2 +- public/pages/utils/anomalyResultUtils.ts | 61 ++++++-- server/routes/ad.ts | 3 + 7 files changed, 194 insertions(+), 79 deletions(-) diff --git a/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx b/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx index f6d1c1bd..649dfe76 100644 --- a/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx +++ b/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx @@ -45,7 +45,10 @@ import { Detector, Monitor, } from '../../../models/interfaces'; -import { generateAnomalyAnnotations } from '../../utils/anomalyResultUtils'; +import { + generateAnomalyAnnotations, + convertToEntityString, +} from '../../utils/anomalyResultUtils'; import { AlertsButton } from '../components/AlertsButton/AlertsButton'; import { AnomalyDetailsChart } from '../containers/AnomalyDetailsChart'; import { @@ -266,6 +269,7 @@ export const AnomaliesChart = React.memo((props: AnomaliesChartProps) => { onDisplayOptionChanged={props.onDisplayOptionChanged} heatmapDisplayOption={props.heatmapDisplayOption} isNotSample={props.isNotSample} + categoryField={get(props.detector, 'categoryField', [])} />, props.isNotSample !== true ? [ @@ -289,10 +293,13 @@ export const AnomaliesChart = React.memo((props: AnomaliesChartProps) => { 'detectorCategoryField.0' )} `} - { - props.selectedHeatmapCell - ?.entityValue - } + {props.selectedHeatmapCell + ? convertToEntityString( + props.selectedHeatmapCell + .entityList, + ' / ' + ) + : '-'} diff --git a/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx b/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx index 56d435b9..d1a7eeb9 100644 --- a/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx +++ b/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx @@ -55,8 +55,12 @@ import { getEntitytAnomaliesHeatmapData, } from '../utils/anomalyChartUtils'; import { MIN_IN_MILLI_SECS } from '../../../../server/utils/constants'; -import { EntityAnomalySummaries } from '../../../../server/models/interfaces'; +import { + EntityAnomalySummaries, + Entity, +} from '../../../../server/models/interfaces'; import { HEATMAP_CHART_Y_AXIS_WIDTH } from '../utils/constants'; +import { convertToEntityList } from '../../utils/anomalyResultUtils'; interface AnomalyHeatmapChartProps { title: string; @@ -73,13 +77,12 @@ interface AnomalyHeatmapChartProps { heatmapDisplayOption?: HeatmapDisplayOption; entityAnomalySummaries?: EntityAnomalySummaries[]; isNotSample?: boolean; + categoryField?: string[]; } export interface HeatmapCell { dateRange: DateRange; - // TODO: change this to entityList. Will use entity list as the key to pass - // to helper fn to build the query to fetch the results - entityValue: string; + entityList: Entity[]; modelId?: string; } @@ -127,14 +130,15 @@ export const AnomalyHeatmapChart = React.memo( //@ts-ignore individualEntities = inputHeatmapData[0].y.filter( //@ts-ignore - (entityValue) => entityValue && entityValue.trim().length > 0 + (entityListAsString) => + entityListAsString && entityListAsString.trim().length > 0 ); } const individualEntityOptions = [] as any[]; //@ts-ignore - individualEntities.forEach((entityValue) => { + individualEntities.forEach((entityListAsString) => { individualEntityOptions.push({ - label: entityValue, + label: entityListAsString, }); }); @@ -221,7 +225,7 @@ export const AnomalyHeatmapChart = React.memo( const handleHeatmapClick = (event: Plotly.PlotMouseEvent) => { const selectedCellIndices = get(event, 'points[0].pointIndex', []); - const selectedEntity = get(event, 'points[0].y', ''); + const selectedEntityString = get(event, 'points[0].y', ''); if (!isEmpty(selectedCellIndices)) { let anomalyCount = get(event, 'points[0].text', 0); if ( @@ -266,7 +270,11 @@ export const AnomalyHeatmapChart = React.memo( startDate: selectedStartDate, endDate: selectedEndDate, }, - entityValue: selectedEntity, + entityList: convertToEntityList( + selectedEntityString, + get(props, 'categoryField', []), + ' / ' + ), } as HeatmapCell); } } diff --git a/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx b/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx index 4686ecad..71c6a87b 100644 --- a/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx +++ b/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx @@ -40,14 +40,18 @@ import { Anomalies, DateRange, FEATURE_TYPE, - EntityData, } from '../../../models/interfaces'; import { NoFeaturePrompt } from '../components/FeatureChart/NoFeaturePrompt'; import { focusOnFeatureAccordion } from '../../ConfigureModel/utils/helpers'; import moment from 'moment'; import { HeatmapCell } from './AnomalyHeatmapChart'; -import { filterWithHeatmapFilter } from '../../utils/anomalyResultUtils'; +import { + filterWithHeatmapFilter, + entityListsMatch, + convertToEntityString, +} from '../../utils/anomalyResultUtils'; import { getDateRangeWithSelectedHeatmapCell } from '../utils/anomalyChartUtils'; +import { Entity } from '../../../../server/models/interfaces'; interface FeatureBreakDownProps { title?: string; @@ -80,10 +84,19 @@ export const FeatureBreakDown = React.memo((props: FeatureBreakDownProps) => { const filteredFeatureData = []; for (let i = 0; i < anomaliesFound.length; i++) { const currentAnomalyData = anomaliesResult.anomalies[i]; + const dataEntityList = get( + currentAnomalyData, + 'entity', + [] + ) as Entity[]; + const cellEntityList = get( + props, + 'selectedHeatmapCell.entityList', + [] + ) as Entity[]; if ( - !isEmpty(get(currentAnomalyData, 'entity', [] as EntityData[])) && - get(currentAnomalyData, 'entity', [] as EntityData[])[0].value === - props.selectedHeatmapCell.entityValue && + !isEmpty(dataEntityList) && + entityListsMatch(dataEntityList, cellEntityList) && get(currentAnomalyData, 'plotTime', 0) >= props.selectedHeatmapCell.dateRange.startDate && get(currentAnomalyData, 'plotTime', 0) <= @@ -203,7 +216,10 @@ export const FeatureBreakDown = React.memo((props: FeatureBreakDownProps) => { titlePrefix={ props.selectedHeatmapCell && props.title !== 'Sample feature breakdown' - ? props.selectedHeatmapCell.entityValue + ? convertToEntityString( + props.selectedHeatmapCell.entityList, + ' / ' + ) : undefined } /> diff --git a/public/pages/DetectorResults/containers/AnomalyHistory.tsx b/public/pages/DetectorResults/containers/AnomalyHistory.tsx index 5b234e07..e809843f 100644 --- a/public/pages/DetectorResults/containers/AnomalyHistory.tsx +++ b/public/pages/DetectorResults/containers/AnomalyHistory.tsx @@ -68,6 +68,7 @@ import { parseTopEntityAnomalySummaryResults, getEntityAnomalySummariesQuery, parseEntityAnomalySummaryResults, + convertToEntityString, } from '../../utils/anomalyResultUtils'; import { AnomalyResultsTable } from './AnomalyResultsTable'; import { AnomaliesChart } from '../../AnomalyCharts/containers/AnomaliesChart'; @@ -394,7 +395,10 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { } } catch (err) { console.error( - `Failed to get anomaly results for entity ${heatmapCell.entityValue}`, + `Failed to get anomaly results for the following entities: ${convertToEntityString( + heatmapCell.entityList, + ', ' + )}`, err ); } finally { @@ -407,11 +411,7 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { heatmapCell.dateRange.startDate, heatmapCell.dateRange.endDate, false, - { - //@ts-ignore - name: props.detector.categoryField[0], - value: heatmapCell.entityValue, - } + heatmapCell.entityList ); const entityAnomalyResultResponse = await dispatch( @@ -436,13 +436,7 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { }; const fetchBucketizedEntityAnomalyData = async (heatmapCell: HeatmapCell) => { - getBucketizedAnomalyResults([ - { - //@ts-ignore - name: props.detector.categoryField[0], - value: heatmapCell.entityValue, - }, - ]); + getBucketizedAnomalyResults(heatmapCell.entityList); }; const [atomicAnomalyResults, setAtomicAnomalyResults] = useState(); const [rawAnomalyResults, setRawAnomalyResults] = useState(); @@ -560,39 +554,89 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { heatmapDisplayOption={heatmapDisplayOption} entityAnomalySummaries={entityAnomalySummaries} > -
- {/* - TODO: update title and occurrence chart to support multi category field support - */} - {isHCDetector - ? [ - -
- {`${get(props, 'detector.categoryField.0')}`}{' '} - {selectedHeatmapCell?.entityValue} - -
- - ) : ( - '-' - ) - } - dateRange={dateRange} - onDateRangeChange={handleDateRangeChange} - onZoomRangeChange={handleZoomChange} - anomalies={anomalyResults ? anomalyResults.anomalies : []} - bucketizedAnomalies={bucketizedAnomalyResults !== undefined} - anomalySummary={bucketizedAnomalySummary} - isLoading={isLoading || isLoadingAnomalyResults} - anomalyGradeSeriesName="Anomaly grade" - confidenceSeriesName="Confidence" - showAlerts={true} - isNotSample={true} - detector={props.detector} - monitor={props.monitor} + {renderTabs()} + + {isLoading || isLoadingAnomalyResults ? ( + + + + + + ) : ( +
+ {selectedTabId === ANOMALY_HISTORY_TABS.FEATURE_BREAKDOWN ? ( + + ) : ( + [ + isHCDetector + ? [ + , + , + ] + : null, + { return { from: 0, @@ -126,8 +125,9 @@ export const buildParamsForGetAnomalyResultsWithDateRange = ( endTime: endTime, fieldName: AD_DOC_FIELDS.DATA_START_TIME, anomalyThreshold: anomalyOnly ? 0 : -1, - entityName: entity?.name, - entityValue: entity?.value, + // TODO: change to include all of the entity names & values + entityName: get(entityList, '0.name', ''), + entityValue: get(entityList, '0.value', ''), }; }; @@ -931,9 +931,11 @@ export const filterWithHeatmapFilter = ( if (isFilteringWithEntity) { data = data .filter((anomaly) => !isEmpty(get(anomaly, 'entity', []))) - .filter( - (anomaly) => get(anomaly, 'entity')[0].value === heatmapCell.entityValue - ); + .filter((anomaly) => { + const dataEntityList = get(anomaly, 'entity'); + const cellEntityList = get(heatmapCell, 'entityList'); + return entityListsMatch(dataEntityList, cellEntityList); + }); } return filterWithDateRange(data, heatmapCell.dateRange, timeField); }; @@ -1284,14 +1286,16 @@ export const parseEntityAnomalySummaryResults = ( }; export const convertToEntityString = ( - entityArray: any[], + entityList: Entity[], delimiter: string ) => { let entityString = ''; - entityArray.forEach((entity: any) => { - entityString += entity.value; - entityString += delimiter; - }); + if (!isEmpty(entityList)) { + entityList.forEach((entity: any) => { + entityString += entity.value; + entityString += delimiter; + }); + } return entityString === '' ? entityString : entityString.slice(0, -delimiter.length); @@ -1457,6 +1461,39 @@ export const parseHistoricalAggregatedAnomalies = ( return anomalies; }; +export const convertToEntityList = ( + entityListAsString: string, + categoryFields: string[], + delimiter: string +) => { + let entityList = [] as Entity[]; + const valueArr = entityListAsString.split(delimiter); + var i; + for (i = 0; i < valueArr.length; i++) { + entityList.push({ + name: categoryFields[i], + value: valueArr[i], + }); + } + return entityList; +}; + +export const entityListsMatch = ( + entityListA: Entity[], + entityListB: Entity[] +) => { + if (get(entityListA, 'length') !== get(entityListB, 'length')) { + return false; + } + var i; + for (i = 0; i < entityListA.length; i++) { + if (entityListA[i].value !== entityListB[i].value) { + return false; + } + } + return true; +}; + // Helper fn to get the correct filters based on how many categorical fields there are. // If there are multiple (entity list > 1), filter results by model id. // If there is only one (entity list = 1), filter by entity value. diff --git a/server/routes/ad.ts b/server/routes/ad.ts index d3aab502..fe8c9540 100644 --- a/server/routes/ad.ts +++ b/server/routes/ad.ts @@ -752,6 +752,9 @@ export default class AdService { }, }, }, + // TODO: here is where we need a helper fn to dynamically create this + // filter array based on a list of entity name/value pairs, rather than just one + // can make something in adHelpers.tsx to handle this ...(entityName && entityValue ? [ { From c595caf2cea7aa777f42ac8107b7adc22f893c24 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Thu, 3 Jun 2021 09:32:09 -0700 Subject: [PATCH 10/35] Add filters for each entity name/value pair --- public/pages/utils/anomalyResultUtils.ts | 4 +- server/routes/ad.ts | 49 +++++------------------- server/routes/utils/adHelpers.ts | 36 ++++++++++++++++- 3 files changed, 46 insertions(+), 43 deletions(-) diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index cdb42c44..9b664062 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -125,9 +125,7 @@ export const buildParamsForGetAnomalyResultsWithDateRange = ( endTime: endTime, fieldName: AD_DOC_FIELDS.DATA_START_TIME, anomalyThreshold: anomalyOnly ? 0 : -1, - // TODO: change to include all of the entity names & values - entityName: get(entityList, '0.name', ''), - entityValue: get(entityList, '0.value', ''), + entityList: JSON.stringify(entityList), }; }; diff --git a/server/routes/ad.ts b/server/routes/ad.ts index fe8c9540..c66e6601 100644 --- a/server/routes/ad.ts +++ b/server/routes/ad.ts @@ -24,7 +24,6 @@ * permissions and limitations under the License. */ -//@ts-ignore import { get, orderBy, pullAll, isEmpty } from 'lodash'; import { AnomalyResults, SearchResponse } from '../models/interfaces'; import { @@ -39,9 +38,6 @@ import { Router } from '../router'; import { SORT_DIRECTION, AD_DOC_FIELDS, - ENTITY_FIELD, - ENTITY_NAME_PATH_FIELD, - ENTITY_VALUE_PATH_FIELD, DETECTOR_STATE, } from '../utils/constants'; import { @@ -69,6 +65,7 @@ import { processTaskError, getLatestDetectorTasksQuery, isRealTimeTask, + getFiltersFromEntityList, } from './utils/adHelpers'; import { isNumber, set } from 'lodash'; import { @@ -701,8 +698,7 @@ export default class AdService { endTime = 0, fieldName = '', anomalyThreshold = -1, - entityName = undefined, - entityValue = undefined, + entityList = '', } = request.query as { from: number; size: number; @@ -712,10 +708,15 @@ export default class AdService { endTime: number; fieldName: string; anomalyThreshold: number; - entityName: string; - entityValue: string; + entityList: string; }; + const entityListAsObj = + entityList.length === 0 ? {} : JSON.parse(entityList); + const entityFilters = isEmpty(entityListAsObj) + ? [] + : getFiltersFromEntityList(entityListAsObj); + //Allowed sorting columns const sortQueryMap = { anomalyGrade: { anomaly_grade: sortDirection }, @@ -752,37 +753,7 @@ export default class AdService { }, }, }, - // TODO: here is where we need a helper fn to dynamically create this - // filter array based on a list of entity name/value pairs, rather than just one - // can make something in adHelpers.tsx to handle this - ...(entityName && entityValue - ? [ - { - nested: { - path: ENTITY_FIELD, - query: { - term: { - [ENTITY_NAME_PATH_FIELD]: { - value: entityName, - }, - }, - }, - }, - }, - { - nested: { - path: ENTITY_FIELD, - query: { - term: { - [ENTITY_VALUE_PATH_FIELD]: { - value: entityValue, - }, - }, - }, - }, - }, - ] - : []), + ...entityFilters, ], }, }, diff --git a/server/routes/utils/adHelpers.ts b/server/routes/utils/adHelpers.ts index 44ff3e68..fdb51641 100644 --- a/server/routes/utils/adHelpers.ts +++ b/server/routes/utils/adHelpers.ts @@ -25,7 +25,7 @@ */ import { get, omit, cloneDeep, isEmpty } from 'lodash'; -import { AnomalyResults } from '../../models/interfaces'; +import { AnomalyResults, Entity } from '../../models/interfaces'; import { GetDetectorsQueryParams, Detector } from '../../models/types'; import { mapKeysDeep, toCamel, toSnake } from '../../utils/helpers'; import { @@ -33,6 +33,9 @@ import { STACK_TRACE_PATTERN, OPENSEARCH_EXCEPTION_PREFIX, REALTIME_TASK_TYPE_PREFIX, + ENTITY_FIELD, + ENTITY_NAME_PATH_FIELD, + ENTITY_VALUE_PATH_FIELD, } from '../../utils/constants'; import { InitProgress } from '../../models/interfaces'; import { MAX_DETECTORS } from '../../utils/constants'; @@ -436,3 +439,34 @@ export const getLatestDetectorTasksQuery = () => { export const isRealTimeTask = (task: any) => { return get(task, 'task_type', '').includes(REALTIME_TASK_TYPE_PREFIX); }; + +export const getFiltersFromEntityList = (entityListAsObj: object) => { + let filters = [] as any[]; + Object.values(entityListAsObj).forEach((entity: Entity) => { + filters.push({ + nested: { + path: ENTITY_FIELD, + query: { + term: { + [ENTITY_NAME_PATH_FIELD]: { + value: entity.name, + }, + }, + }, + }, + }); + filters.push({ + nested: { + path: ENTITY_FIELD, + query: { + term: { + [ENTITY_VALUE_PATH_FIELD]: { + value: entity.value, + }, + }, + }, + }, + }); + }); + return filters; +}; From 9b23f162d54bb57e1495d9e0f853a594e9d33c6c Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Thu, 3 Jun 2021 11:46:20 -0700 Subject: [PATCH 11/35] Support multi category in preview heatmap --- .../AnomalyCharts/utils/anomalyChartUtils.ts | 45 +++++++++---------- public/pages/utils/anomalyResultUtils.ts | 19 +++++++- 2 files changed, 40 insertions(+), 24 deletions(-) diff --git a/public/pages/AnomalyCharts/utils/anomalyChartUtils.ts b/public/pages/AnomalyCharts/utils/anomalyChartUtils.ts index b8aeaf3b..e5d297d2 100644 --- a/public/pages/AnomalyCharts/utils/anomalyChartUtils.ts +++ b/public/pages/AnomalyCharts/utils/anomalyChartUtils.ts @@ -40,6 +40,7 @@ import moment from 'moment'; import { calculateTimeWindowsWithMaxDataPoints, convertToEntityString, + transformEntityListsForHeatmap, } from '../../utils/anomalyResultUtils'; import { HeatmapCell } from '../containers/AnomalyHeatmapChart'; import { @@ -247,7 +248,8 @@ export const getAnomaliesHeatmapData = ( } } - const entityValues = [] as string[]; + const entityStrings = [] as string[]; + const entityLists = [] as any[]; const maxAnomalyGrades = [] as any[]; const numAnomalyGrades = [] as any[]; @@ -256,11 +258,12 @@ export const getAnomaliesHeatmapData = ( dateRange ); - entityAnomaliesMap.forEach((entityAnomalies, entity) => { + entityAnomaliesMap.forEach((entityAnomalies, entityListAsString) => { const maxAnomalyGradesForEntity = [] as number[]; const numAnomalyGradesForEntity = [] as number[]; - entityValues.push(entity); + entityStrings.push(entityListAsString); + entityLists.push(get(entityAnomalies, '0.entity', {})); timeWindows.forEach((timeWindow) => { const anomaliesInWindow = entityAnomalies.filter( (anomaly) => @@ -290,13 +293,13 @@ export const getAnomaliesHeatmapData = ( moment(timestamp).format(HEATMAP_X_AXIS_DATE_FORMAT) ); const cellTimeInterval = timeWindows[0].endDate - timeWindows[0].startDate; + const entityListsTransformed = transformEntityListsForHeatmap(entityLists); const plotData = buildHeatmapPlotData( plotTimesInString, - entityValues, + entityStrings, maxAnomalyGrades, numAnomalyGrades, - // TODO: change to something like entityListsExpanded in getEntitytAnomaliesHeatmapData - [], + entityListsTransformed, cellTimeInterval ); const resultPlotData = sortHeatmapPlotData(plotData, sortType, displayTopNum); @@ -431,23 +434,14 @@ export const getEntitytAnomaliesHeatmapData = ( const timeStamps = plotTimes.map((timestamp) => moment(timestamp).format(HEATMAP_X_AXIS_DATE_FORMAT) ); - let entityListsExpanded = [] as any[]; - entityLists.forEach((entityList: Entity[]) => { - const listAsString = convertToEntityString(entityList, ', '); - let row = []; - var i; - for (i = 0; i < NUM_CELLS; i++) { - row.push(listAsString); - } - entityListsExpanded.push(row); - }); + const entityListsTransformed = transformEntityListsForHeatmap(entityLists); const plotData = buildHeatmapPlotData( timeStamps, entityStrings.reverse(), maxAnomalyGrades.reverse(), numAnomalyGrades.reverse(), - entityListsExpanded.reverse(), + entityListsTransformed.reverse(), timeWindows[0].endDate - timeWindows[0].startDate ); return [plotData]; @@ -461,18 +455,18 @@ const getEntityAnomaliesMap = ( return entityAnomaliesMap; } anomalies.forEach((anomaly) => { - const entity = get(anomaly, 'entity', [] as EntityData[]); - if (isEmpty(entity)) { + const entityList = get(anomaly, 'entity', [] as EntityData[]); + if (isEmpty(entityList)) { return; } - const entityValue = entity[0].value; + const entityListAsString = convertToEntityString(entityList, ' / '); let singleEntityAnomalies = []; - if (entityAnomaliesMap.has(entityValue)) { + if (entityAnomaliesMap.has(entityListAsString)) { //@ts-ignore - singleEntityAnomalies = entityAnomaliesMap.get(entityValue); + singleEntityAnomalies = entityAnomaliesMap.get(entityListAsString); } singleEntityAnomalies.push(anomaly); - entityAnomaliesMap.set(entityValue, singleEntityAnomalies); + entityAnomaliesMap.set(entityListAsString, singleEntityAnomalies); }); return entityAnomaliesMap; }; @@ -534,6 +528,8 @@ export const sortHeatmapPlotData = ( const originalYs = cloneDeep(heatmapData.y); const originalZs = cloneDeep(heatmapData.z); const originalTexts = cloneDeep(heatmapData.text); + const originalEntityLists = cloneDeep(heatmapData.customdata); + const originalValuesToSort = sortType === AnomalyHeatmapSortType.SEVERITY ? cloneDeep(originalZs) @@ -561,17 +557,20 @@ export const sortHeatmapPlotData = ( const resultYs = [] as any[]; const resultZs = [] as any[]; const resultTexts = [] as any[]; + const resultEntityLists = [] as any[]; for (let i = sortedYIndices.length - 1; i >= 0; i--) { const index = get(sortedYIndices[i], 'index', 0); resultYs.push(originalYs[index]); resultZs.push(originalZs[index]); resultTexts.push(originalTexts[index]); + resultEntityLists.push(originalEntityLists[index]); } return { ...cloneDeep(heatmapData), y: resultYs, z: resultZs, text: resultTexts, + customdata: resultEntityLists, } as PlotData; }; diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index 9b664062..6a3213eb 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -62,7 +62,10 @@ import { MISSING_FEATURE_DATA_SEVERITY, } from '../../utils/constants'; import { HeatmapCell } from '../AnomalyCharts/containers/AnomalyHeatmapChart'; -import { AnomalyHeatmapSortType } from '../AnomalyCharts/utils/anomalyChartUtils'; +import { + AnomalyHeatmapSortType, + NUM_CELLS, +} from '../AnomalyCharts/utils/anomalyChartUtils'; import { DETECTOR_INIT_FAILURES } from '../DetectorDetail/utils/constants'; import { COUNT_ANOMALY_AGGS, @@ -1519,3 +1522,17 @@ const getResultFilters = ( }, }; }; + +export const transformEntityListsForHeatmap = (entityLists: any[]) => { + let transformedEntityLists = [] as any[]; + entityLists.forEach((entityList: Entity[]) => { + const listAsString = convertToEntityString(entityList, ', '); + let row = []; + var i; + for (i = 0; i < NUM_CELLS; i++) { + row.push(listAsString); + } + transformedEntityLists.push(row); + }); + return transformedEntityLists; +}; From a4a2455b98754d844c94d449bcca3c911608778d Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Thu, 3 Jun 2021 14:15:25 -0700 Subject: [PATCH 12/35] Support multi category in results table --- .../containers/AnomalyResultsTable.tsx | 11 +++++++---- public/pages/DetectorResults/utils/tableUtils.tsx | 10 +++++----- public/pages/utils/anomalyResultUtils.ts | 12 ++++++++++++ 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/public/pages/DetectorResults/containers/AnomalyResultsTable.tsx b/public/pages/DetectorResults/containers/AnomalyResultsTable.tsx index dd4413c4..3d4c2c1c 100644 --- a/public/pages/DetectorResults/containers/AnomalyResultsTable.tsx +++ b/public/pages/DetectorResults/containers/AnomalyResultsTable.tsx @@ -35,13 +35,14 @@ import React, { useEffect, useState } from 'react'; import { SORT_DIRECTION } from '../../../../server/utils/constants'; import ContentPanel from '../../../components/ContentPanel/ContentPanel'; import { - entityValueColumn, - ENTITY_VALUE_FIELD, + categoryFieldsColumn, staticColumn, + CATEGORY_FIELDS, } from '../utils/tableUtils'; import { DetectorResultsQueryParams } from 'server/models/types'; import { AnomalyData } from '../../../models/interfaces'; import { getTitleWithCount } from '../../../utils/utils'; +import { convertToCategoryFieldAndEntityString } from '../../utils/anomalyResultUtils'; interface AnomalyResultsTableProps { anomalies: AnomalyData[]; @@ -89,7 +90,9 @@ export function AnomalyResultsTable(props: AnomalyResultsTableProps) { anomalies = anomalies.map((anomaly) => { return { ...anomaly, - [ENTITY_VALUE_FIELD]: get(anomaly, 'entity[0].value'), + [CATEGORY_FIELDS]: convertToCategoryFieldAndEntityString( + get(anomaly, 'entity', []) + ), }; }); } @@ -155,7 +158,7 @@ export function AnomalyResultsTable(props: AnomalyResultsTableProps) { : props.isHCDetector ? [ ...staticColumn.slice(0, 2), - entityValueColumn, + categoryFieldsColumn, ...staticColumn.slice(2), ] : props.isHistorical diff --git a/public/pages/DetectorResults/utils/tableUtils.tsx b/public/pages/DetectorResults/utils/tableUtils.tsx index 6e464726..6521090e 100644 --- a/public/pages/DetectorResults/utils/tableUtils.tsx +++ b/public/pages/DetectorResults/utils/tableUtils.tsx @@ -38,7 +38,7 @@ const renderTime = (time: number) => { return DEFAULT_EMPTY_DATA; }; -export const ENTITY_VALUE_FIELD = 'entityValue'; +export const CATEGORY_FIELDS = 'categoryFields'; export const staticColumn = [ { @@ -91,10 +91,10 @@ export const staticColumn = [ }, ] as EuiBasicTableColumn[]; -export const entityValueColumn = { - field: ENTITY_VALUE_FIELD, - name: 'Entity', +export const categoryFieldsColumn = { + field: CATEGORY_FIELDS, + name: 'Category fields', sortable: true, truncateText: false, - dataType: 'number', + dataType: 'string', } as EuiBasicTableColumn; diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index 6a3213eb..fc9b3651 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -1462,6 +1462,18 @@ export const parseHistoricalAggregatedAnomalies = ( return anomalies; }; +export const convertToCategoryFieldAndEntityString = (entityList: Entity[]) => { + let entityString = ''; + if (!isEmpty(entityList)) { + entityList.forEach((entity: any) => { + entityString += entity.name + ': ' + entity.value; + entityString += '\n'; + }); + entityString.slice(0, -1); + } + return entityString; +}; + export const convertToEntityList = ( entityListAsString: string, categoryFields: string[], From 3a44ec970b430ebc02b56d09038f872147497ad9 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Thu, 3 Jun 2021 15:16:11 -0700 Subject: [PATCH 13/35] Minor wording changes; add list of category fields in heatmap title --- .../containers/AnomaliesChart.tsx | 2 - .../containers/AnomalyHeatmapChart.tsx | 77 +++++++++++-------- .../AnomalyCharts/utils/anomalyChartUtils.ts | 4 +- public/pages/utils/anomalyResultUtils.ts | 16 ++++ 4 files changed, 64 insertions(+), 35 deletions(-) diff --git a/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx b/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx index 649dfe76..d009fcb0 100644 --- a/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx +++ b/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx @@ -249,8 +249,6 @@ export const AnomaliesChart = React.memo((props: AnomaliesChartProps) => { detectorId={get(props.detector, 'id', '')} detectorName={get(props.detector, 'name', '')} dateRange={props.dateRange} - //@ts-ignore - title={props.detectorCategoryField[0]} anomalies={anomalies} isLoading={props.isLoading} showAlerts={props.showAlerts} diff --git a/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx b/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx index d1a7eeb9..c17b6973 100644 --- a/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx +++ b/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx @@ -60,10 +60,12 @@ import { Entity, } from '../../../../server/models/interfaces'; import { HEATMAP_CHART_Y_AXIS_WIDTH } from '../utils/constants'; -import { convertToEntityList } from '../../utils/anomalyResultUtils'; +import { + convertToEntityList, + convertToCategoryFieldString, +} from '../../utils/anomalyResultUtils'; interface AnomalyHeatmapChartProps { - title: string; detectorId: string; detectorName: string; anomalies?: any[]; @@ -419,40 +421,53 @@ export const AnomalyHeatmapChart = React.memo( - - - -

{props.title}

- - - - - - - - handleViewEntityOptionsChange(selectedOptions) - } - /> - - - handleSortByFieldChange(value)} - hasDividers - /> - - - + + + +

+ View by:  + + {convertToCategoryFieldString( + get(props, 'categoryField', []) as string[], + ', ' + )} + +

+ + + + + + + + handleViewEntityOptionsChange(selectedOptions) + } + /> + + + handleSortByFieldChange(value)} + hasDividers + /> + + + + diff --git a/public/pages/AnomalyCharts/utils/anomalyChartUtils.ts b/public/pages/AnomalyCharts/utils/anomalyChartUtils.ts index e5d297d2..24f298fe 100644 --- a/public/pages/AnomalyCharts/utils/anomalyChartUtils.ts +++ b/public/pages/AnomalyCharts/utils/anomalyChartUtils.ts @@ -330,10 +330,10 @@ const buildHeatmapPlotData = ( text: anomalyOccurrences, customdata: entityLists, hovertemplate: + 'Entities: %{customdata}
' + 'Time: %{x}
' + 'Max anomaly grade: %{z}
' + - 'Anomaly occurrences: %{text}
' + - 'Entities: %{customdata}' + + 'Anomaly occurrences: %{text}' + '', cellTimeInterval: cellTimeInterval, } as PlotData; diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index fc9b3651..37d0a983 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -1462,6 +1462,22 @@ export const parseHistoricalAggregatedAnomalies = ( return anomalies; }; +export const convertToCategoryFieldString = ( + categoryFields: string[], + delimiter: string +) => { + let categoryFieldString = ''; + if (!isEmpty(categoryFields)) { + categoryFields.forEach((categoryField: any) => { + categoryFieldString += categoryField; + categoryFieldString += delimiter; + }); + } + return categoryFieldString === '' + ? categoryFieldString + : categoryFieldString.slice(0, -delimiter.length); +}; + export const convertToCategoryFieldAndEntityString = (entityList: Entity[]) => { let entityString = ''; if (!isEmpty(entityList)) { From 530693a6d9a341c3ab08cecb705d440df3c49002 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Thu, 3 Jun 2021 17:08:50 -0700 Subject: [PATCH 14/35] Tune anomaly occurence chart title --- .../containers/AnomaliesChart.tsx | 6 +++--- ...alyChartUtils.ts => anomalyChartUtils.tsx} | 21 +++++++++++++++++++ .../containers/AnomalyHistory.tsx | 6 ++---- 3 files changed, 26 insertions(+), 7 deletions(-) rename public/pages/AnomalyCharts/utils/{anomalyChartUtils.ts => anomalyChartUtils.tsx} (97%) diff --git a/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx b/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx index d009fcb0..18928500 100644 --- a/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx +++ b/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx @@ -61,6 +61,7 @@ import { getConfidenceWording, getFeatureBreakdownWording, getFeatureDataWording, + getHCTitle, } from '../utils/anomalyChartUtils'; import { DATE_PICKER_QUICK_OPTIONS, @@ -292,10 +293,9 @@ export const AnomaliesChart = React.memo((props: AnomaliesChartProps) => { )} `} {props.selectedHeatmapCell - ? convertToEntityString( + ? getHCTitle( props.selectedHeatmapCell - .entityList, - ' / ' + .entityList ) : '-'} diff --git a/public/pages/AnomalyCharts/utils/anomalyChartUtils.ts b/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx similarity index 97% rename from public/pages/AnomalyCharts/utils/anomalyChartUtils.ts rename to public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx index 24f298fe..5f1ccd90 100644 --- a/public/pages/AnomalyCharts/utils/anomalyChartUtils.ts +++ b/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx @@ -25,6 +25,8 @@ */ import { cloneDeep, defaultTo, get, isEmpty, orderBy } from 'lodash'; +import React from 'react'; +import { EuiTitle, EuiSpacer } from '@elastic/eui'; import { DateRange, Detector, @@ -672,3 +674,22 @@ export const getDateRangeWithSelectedHeatmapCell = ( } return originalDateRange; }; + +export const getHCTitle = (entityList: Entity[]) => { + return ( +
+ +

+ {entityList.map((entity: Entity) => { + return ( +
+ {entity.name}: {entity.value}{' '} +
+ ); + })} +

+ + +
+ ); +}; diff --git a/public/pages/DetectorResults/containers/AnomalyHistory.tsx b/public/pages/DetectorResults/containers/AnomalyHistory.tsx index e809843f..97a323e9 100644 --- a/public/pages/DetectorResults/containers/AnomalyHistory.tsx +++ b/public/pages/DetectorResults/containers/AnomalyHistory.tsx @@ -91,6 +91,7 @@ import { import { getAnomalyHistoryWording, NUM_CELLS, + getHCTitle, } from '../../AnomalyCharts/utils/anomalyChartUtils'; import { darkModeEnabled } from '../../../utils/opensearchDashboardsUtils'; import { @@ -593,10 +594,7 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { Date: Thu, 3 Jun 2021 17:29:24 -0700 Subject: [PATCH 15/35] Make multi entity delimiters consistent; remove feature entity prefix --- .../components/FeatureChart/FeatureChart.tsx | 6 ++---- .../AnomalyCharts/containers/AnomalyHeatmapChart.tsx | 2 +- .../pages/AnomalyCharts/containers/FeatureBreakDown.tsx | 9 --------- public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx | 4 ++-- 4 files changed, 5 insertions(+), 16 deletions(-) diff --git a/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx b/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx index 3b6619b1..c9bbac07 100644 --- a/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx +++ b/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx @@ -74,7 +74,6 @@ interface FeatureChartProps { showFeatureMissingDataPointAnnotation?: boolean; detectorEnabledTime?: number; rawFeatureData: FeatureAggregationData[]; - titlePrefix?: string; } export const FeatureChart = (props: FeatureChartProps) => { @@ -165,10 +164,9 @@ export const FeatureChart = (props: FeatureChartProps) => { return ( { props.showFeatureMissingDataPointAnnotation } detectorEnabledTime={props.detector.enabledTime} - titlePrefix={ - props.selectedHeatmapCell && - props.title !== 'Sample feature breakdown' - ? convertToEntityString( - props.selectedHeatmapCell.entityList, - ' / ' - ) - : undefined - } /> {index + 1 === get(props, 'detector.featureAttributes', []).length ? null : ( diff --git a/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx b/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx index 5f1ccd90..dca433c1 100644 --- a/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx +++ b/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx @@ -382,7 +382,7 @@ export const getEntitytAnomaliesHeatmapData = ( const entityString = convertToEntityString( get(entityAnomalySummaries, 'entityList', []), - ' / ' + ', ' ) as string; const anomaliesSummary = get( entityAnomalySummaries, @@ -461,7 +461,7 @@ const getEntityAnomaliesMap = ( if (isEmpty(entityList)) { return; } - const entityListAsString = convertToEntityString(entityList, ' / '); + const entityListAsString = convertToEntityString(entityList, ', '); let singleEntityAnomalies = []; if (entityAnomaliesMap.has(entityListAsString)) { //@ts-ignore From c21002d72a09bf5172ca72e835d8fbc7d0561df0 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Fri, 4 Jun 2021 16:31:09 -0700 Subject: [PATCH 16/35] Sort category field list on the fly; add max category field num constant --- .../AnomalyCharts/containers/AnomaliesChart.tsx | 14 ++++++++------ .../AnomalyCharts/utils/anomalyChartUtils.tsx | 1 - .../components/CategoryField/CategoryField.tsx | 3 ++- public/utils/constants.ts | 2 ++ 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx b/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx index 18928500..c5ebc7eb 100644 --- a/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx +++ b/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx @@ -33,7 +33,7 @@ import { EuiSuperDatePicker, EuiTitle, } from '@elastic/eui'; -import { get } from 'lodash'; +import { get, orderBy } from 'lodash'; import moment, { DurationInputArg2 } from 'moment'; import React, { useState } from 'react'; import { EntityAnomalySummaries } from '../../../../server/models/interfaces'; @@ -45,10 +45,7 @@ import { Detector, Monitor, } from '../../../models/interfaces'; -import { - generateAnomalyAnnotations, - convertToEntityString, -} from '../../utils/anomalyResultUtils'; +import { generateAnomalyAnnotations } from '../../utils/anomalyResultUtils'; import { AlertsButton } from '../components/AlertsButton/AlertsButton'; import { AnomalyDetailsChart } from '../containers/AnomalyDetailsChart'; import { @@ -268,7 +265,12 @@ export const AnomaliesChart = React.memo((props: AnomaliesChartProps) => { onDisplayOptionChanged={props.onDisplayOptionChanged} heatmapDisplayOption={props.heatmapDisplayOption} isNotSample={props.isNotSample} - categoryField={get(props.detector, 'categoryField', [])} + // Category fields in HC results are always sorted alphabetically. To make all chart + // wording consistent with the returned results, we sort the given category + // fields in alphabetical order as well. + categoryField={orderBy(props.detectorCategoryField, [ + (categoryField) => categoryField.toLowerCase(), + ])} />, props.isNotSample !== true ? [ diff --git a/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx b/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx index dca433c1..b8aba78c 100644 --- a/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx +++ b/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx @@ -229,7 +229,6 @@ const buildBlankStringWithLength = (length: number) => { return result; }; -// TODO: fix this + its helper fns. Related to preview results export const getAnomaliesHeatmapData = ( anomalies: any[] | undefined, dateRange: DateRange, diff --git a/public/pages/EditFeatures/components/CategoryField/CategoryField.tsx b/public/pages/EditFeatures/components/CategoryField/CategoryField.tsx index 0bbd608e..23eb0eeb 100644 --- a/public/pages/EditFeatures/components/CategoryField/CategoryField.tsx +++ b/public/pages/EditFeatures/components/CategoryField/CategoryField.tsx @@ -52,6 +52,7 @@ import { getError, validateCategoryField, } from '../../../../utils/utils'; +import { MAX_CATEGORY_FIELD_NUM } from '../../../../utils/constants'; interface CategoryFieldProps { isHCDetector: boolean; @@ -152,7 +153,7 @@ export function CategoryField(props: CategoryFieldProps) { (option) => option.label ); if (!isEmpty(selection)) { - if (selection.length <= 2) { + if (selection.length <= MAX_CATEGORY_FIELD_NUM) { form.setFieldValue('categoryField', selection); form.setFieldValue( 'shingleSize', diff --git a/public/utils/constants.ts b/public/utils/constants.ts index d8dab0df..efcccdfe 100644 --- a/public/utils/constants.ts +++ b/public/utils/constants.ts @@ -79,6 +79,8 @@ export const MAX_FEATURE_NUM = 5; export const MAX_FEATURE_NAME_SIZE = 64; +export const MAX_CATEGORY_FIELD_NUM = 2; + export const NAME_REGEX = RegExp('^[a-zA-Z0-9._-]+$'); //https://github.com/opensearch-project/anomaly-detection/blob/main/src/main/java/com/amazon/opendistroforelasticsearch/ad/settings/AnomalyDetectorSettings.java From a63c1eacd43bccc5386a4e41466403b8c826dc9b Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Mon, 7 Jun 2021 15:12:22 -0700 Subject: [PATCH 17/35] Support entity list in config page; tune wording on category field --- .../AdditionalSettings/AdditionalSettings.tsx | 4 ++-- .../components/CategoryField/CategoryField.tsx | 13 +++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx b/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx index 60e576fa..26ae142a 100644 --- a/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx +++ b/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx @@ -25,9 +25,9 @@ */ import React from 'react'; -import { get } from 'lodash'; import { EuiBasicTable } from '@elastic/eui'; import ContentPanel from '../../../../components/ContentPanel/ContentPanel'; +import { convertToCategoryFieldString } from '../../../utils/anomalyResultUtils'; interface AdditionalSettingsProps { shingleSize: number; @@ -37,7 +37,7 @@ interface AdditionalSettingsProps { export function AdditionalSettings(props: AdditionalSettingsProps) { const tableItems = [ { - categoryField: get(props.categoryField, 0, '-'), + categoryField: convertToCategoryFieldString(props.categoryField, ', '), windowSize: props.shingleSize, }, ]; diff --git a/public/pages/EditFeatures/components/CategoryField/CategoryField.tsx b/public/pages/EditFeatures/components/CategoryField/CategoryField.tsx index 23eb0eeb..1aec8e69 100644 --- a/public/pages/EditFeatures/components/CategoryField/CategoryField.tsx +++ b/public/pages/EditFeatures/components/CategoryField/CategoryField.tsx @@ -30,7 +30,6 @@ import { EuiText, EuiLink, EuiIcon, - EuiFormRow, EuiPage, EuiPageBody, EuiComboBox, @@ -40,7 +39,7 @@ import { EuiSpacer, } from '@elastic/eui'; import { Field, FieldProps } from 'formik'; -import { get, isEmpty } from 'lodash'; +import { isEmpty } from 'lodash'; import { MULTI_ENTITY_SHINGLE_SIZE, BASE_DOCS_LINK, @@ -53,6 +52,7 @@ import { validateCategoryField, } from '../../../../utils/utils'; import { MAX_CATEGORY_FIELD_NUM } from '../../../../utils/constants'; +import { FormattedFormRow } from '../../../createDetector/components/FormattedFormRow/FormattedFormRow'; interface CategoryFieldProps { isHCDetector: boolean; @@ -81,7 +81,7 @@ export function CategoryField(props: CategoryFieldProps) { -

Category field

+

Categorical fields

} subTitle={ @@ -134,8 +134,9 @@ export function CategoryField(props: CategoryFieldProps) { {enabled && !noCategoryFields ? ( - - + ) : null} From 40c0aff1b9ef14dd38e389665c7549079a14c4ae Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Mon, 7 Jun 2021 15:19:50 -0700 Subject: [PATCH 18/35] Clean up detector config changes --- .../components/AdditionalSettings/AdditionalSettings.tsx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx b/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx index 26ae142a..96b42e5a 100644 --- a/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx +++ b/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx @@ -25,6 +25,7 @@ */ import React from 'react'; +import { get, isEmpty } from 'lodash'; import { EuiBasicTable } from '@elastic/eui'; import ContentPanel from '../../../../components/ContentPanel/ContentPanel'; import { convertToCategoryFieldString } from '../../../utils/anomalyResultUtils'; @@ -37,12 +38,14 @@ interface AdditionalSettingsProps { export function AdditionalSettings(props: AdditionalSettingsProps) { const tableItems = [ { - categoryField: convertToCategoryFieldString(props.categoryField, ', '), + categoryField: isEmpty(get(props, 'categoryField', [])) + ? '-' + : convertToCategoryFieldString(props.categoryField, ', '), windowSize: props.shingleSize, }, ]; const tableColumns = [ - { name: 'Category field', field: 'categoryField' }, + { name: 'Category fields', field: 'categoryField' }, { name: 'Window size', field: 'windowSize' }, ]; return ( From 0119573062036dd4f0c68efc67045ab82a7705ba Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Mon, 7 Jun 2021 16:03:55 -0700 Subject: [PATCH 19/35] Update and add UT --- .../__tests__/AnomalyHeatmapChart.test.tsx | 46 +- .../AnomalyHeatmapChart.test.tsx.snap | 1464 +++++++++++++---- .../__tests__/CategoryField.test.tsx | 59 +- .../__snapshots__/CategoryField.test.tsx.snap | 637 +++++++ 4 files changed, 1892 insertions(+), 314 deletions(-) create mode 100644 public/pages/EditFeatures/components/CategoryField/__tests__/__snapshots__/CategoryField.test.tsx.snap diff --git a/public/pages/AnomalyCharts/containers/__tests__/AnomalyHeatmapChart.test.tsx b/public/pages/AnomalyCharts/containers/__tests__/AnomalyHeatmapChart.test.tsx index 1523a6e6..7267ca33 100644 --- a/public/pages/AnomalyCharts/containers/__tests__/AnomalyHeatmapChart.test.tsx +++ b/public/pages/AnomalyCharts/containers/__tests__/AnomalyHeatmapChart.test.tsx @@ -24,7 +24,7 @@ * permissions and limitations under the License. */ -import { render } from '@testing-library/react'; +import { render, getByText } from '@testing-library/react'; import React from 'react'; import { AnomalyHeatmapChart, @@ -79,4 +79,48 @@ describe(' spec', () => { ); expect(container).toMatchSnapshot(); }); + test('AnomalyHeatmapChart with one category field', () => { + const { container, getByText } = render( + + ); + expect(container).toMatchSnapshot(); + getByText('View by:'); + getByText('test-field'); + }); + test('AnomalyHeatmapChart with multiple category fields', () => { + const { container, getByText } = render( + + ); + expect(container).toMatchSnapshot(); + getByText('View by:'); + getByText('test-field-1, test-field-2'); + }); }); diff --git a/public/pages/AnomalyCharts/containers/__tests__/__snapshots__/AnomalyHeatmapChart.test.tsx.snap b/public/pages/AnomalyCharts/containers/__tests__/__snapshots__/AnomalyHeatmapChart.test.tsx.snap index 3d1758dd..0c9d8163 100644 --- a/public/pages/AnomalyCharts/containers/__tests__/__snapshots__/AnomalyHeatmapChart.test.tsx.snap +++ b/public/pages/AnomalyCharts/containers/__tests__/__snapshots__/AnomalyHeatmapChart.test.tsx.snap @@ -38,26 +38,6 @@ exports[` spec AnomalyHeatmapChart with Sample anomaly da class="euiFlexGroup euiFlexGroup--gutterLarge euiFlexGroup--directionRow euiFlexGroup--responsive" style="padding: 0px;" > -
-
-
-
-

- test-tile -

-
-
-
-
spec AnomalyHeatmapChart with Sample anomaly da style="padding: 0px;" >
+

+ View by:  + +

+
+
+
+
-
-
-
+
- - Select an option: By severity, is selected - -
- + Select an option: By severity, is selected + +
+ + + +
@@ -442,26 +439,6 @@ exports[` spec AnomalyHeatmapChart with anomaly summaries class="euiFlexGroup euiFlexGroup--gutterLarge euiFlexGroup--directionRow euiFlexGroup--responsive" style="padding: 0px;" > -
-
-
-
-

- test-tile -

-
-
-
-
spec AnomalyHeatmapChart with anomaly summaries style="padding: 0px;" >
+

+ View by:  + +

+
+
+
+
-
-
-
- -
-
- - Select an option: By severity, is selected - - +
+
- + + + +
+
+
+
+
+
+
+
+ +
+
+ + Select an option: By severity, is selected + +
+ + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Anomaly grade + + + + + + +
+
+
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+
+
+
+
+ + 0.0 + + (None) +
+
+
+
+ (Critical) + + 1.0 + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+`; + +exports[` spec AnomalyHeatmapChart with multiple category fields 1`] = ` +
+
+
+
+
+ + + No anomalies found in the specified date range. + +
+
+
+
+
+
+
+
+
+
+

+ View by:  + + test-field-1, test-field-2 + +

+
+
+
+
+
+ +
+
+
+ +
+
+ + Select an option: By severity, is selected + + +
+ + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Anomaly grade + + + + + + +
+
+
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+
+
+
+
+ + 0.0 + + (None) +
+
+
+
+ (Critical) + + 1.0 + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+`; + +exports[` spec AnomalyHeatmapChart with one category field 1`] = ` +
+
+
+
+
+ + + No anomalies found in the specified date range. + +
+
+
+
+
+
+
+
+
+
+

+ View by:  + + test-field + +

+
+
+
+
+
+ +
+
+
+ +
+
+ + Select an option: By severity, is selected + + +
+ + + +
diff --git a/public/pages/ConfigureModel/components/CategoryField/__tests__/CategoryField.test.tsx b/public/pages/ConfigureModel/components/CategoryField/__tests__/CategoryField.test.tsx index 1239088b..2706c7cb 100644 --- a/public/pages/ConfigureModel/components/CategoryField/__tests__/CategoryField.test.tsx +++ b/public/pages/ConfigureModel/components/CategoryField/__tests__/CategoryField.test.tsx @@ -25,7 +25,7 @@ */ import React, { Fragment } from 'react'; -import { render } from '@testing-library/react'; +import { render, fireEvent, getByRole } from '@testing-library/react'; import { Form, Formik } from 'formik'; import { CategoryField } from '../CategoryField'; @@ -66,7 +66,13 @@ describe(' spec', () => { expect(queryByText('Enable categorical field')).not.toBeNull(); }); test('renders the component when enabled', () => { - const { container, queryByText, queryByTestId } = render( + const { + container, + queryByText, + queryByTestId, + getByTestId, + getByText, + } = render( spec', () => {
{ return; }} @@ -99,6 +105,9 @@ describe(' spec', () => { expect(queryByTestId('noCategoryFieldsCallout')).toBeNull(); expect(queryByTestId('categoryFieldComboBox')).not.toBeNull(); expect(queryByText('Enable categorical field')).not.toBeNull(); + fireEvent.click(getByTestId('comboBoxToggleListButton')); + getByText('a'); + getByText('b'); }); test('shows callout when there are no available category fields', () => { const { container, queryByText, queryByTestId } = render( @@ -169,4 +178,48 @@ describe(' spec', () => { expect(queryByTestId('noCategoryFieldsCallout')).toBeNull(); expect(queryByText('Enable categorical field')).not.toBeNull(); }); + test(`limits selection to a maximum of 2 entities`, () => { + const { getAllByRole, getByTestId, queryByText } = render( + + {}} + > + + + { + return; + }} + isLoading={false} + originalShingleSize={1} + /> + + + + + ); + // open combo box + fireEvent.click(getByTestId('comboBoxToggleListButton')); + expect(queryByText('a')).not.toBeNull(); + expect(queryByText('b')).not.toBeNull(); + expect(queryByText('c')).not.toBeNull(); + + // select top 3 options (a,b,c) + fireEvent.click(getAllByRole('option')[0]); + fireEvent.click(getAllByRole('option')[0]); + fireEvent.click(getAllByRole('option')[0]); + + // close combo box + fireEvent.click(getByTestId('comboBoxToggleListButton')); + + // the last selection (c) is still not selected + expect(queryByText('a')).not.toBeNull(); + expect(queryByText('b')).not.toBeNull(); + expect(queryByText('c')).toBeNull(); + }); }); diff --git a/public/pages/EditFeatures/components/CategoryField/__tests__/__snapshots__/CategoryField.test.tsx.snap b/public/pages/EditFeatures/components/CategoryField/__tests__/__snapshots__/CategoryField.test.tsx.snap new file mode 100644 index 00000000..54e6c969 --- /dev/null +++ b/public/pages/EditFeatures/components/CategoryField/__tests__/__snapshots__/CategoryField.test.tsx.snap @@ -0,0 +1,637 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[` spec hides callout if component is loading 1`] = ` +
+
+
+
+
+
+
+
+
+

+ Categorical fields +

+
+
+
+
+
+ Categorize anomalies based on unique partitions. For example, with clickstream data you can categorize anomalies into a given day, week, or month. + + + Learn more + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+ +
+
+
+
+
+
+
+
+
+`; + +exports[` spec renders the component when disabled 1`] = ` +
+
+
+
+
+
+
+
+
+

+ Categorical fields +

+
+
+
+
+
+ Categorize anomalies based on unique partitions. For example, with clickstream data you can categorize anomalies into a given day, week, or month. + + + Learn more + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+ +
+
+
+
+
+
+
+
+
+`; + +exports[` spec renders the component when enabled 1`] = ` +
+
+
+
+
+
+
+
+
+

+ Categorical fields +

+
+
+
+
+
+ Categorize anomalies based on unique partitions. For example, with clickstream data you can categorize anomalies into a given day, week, or month. + + + Learn more + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+ +
+
+
+
+
+ +
+
+ +
+
+
+
+
+
+
+
+
+`; + +exports[` spec shows callout when there are no available category fields 1`] = ` +
+
+
+
+
+
+
+
+
+

+ Categorical fields +

+
+
+
+
+
+ Categorize anomalies based on unique partitions. For example, with clickstream data you can categorize anomalies into a given day, week, or month. + + + Learn more + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+ + + There are no available category fields for the selected index + +
+
+
+
+
+
+ +
+ +
+
+
+
+
+
+
+
+
+`; From bf37d5e5a4decbdce70be60106ae8b923cd1f5a2 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 9 Jun 2021 12:28:39 -0700 Subject: [PATCH 20/35] Change y axis delim to \n --- .../AnomalyCharts/containers/AnomalyHeatmapChart.tsx | 11 +++++++---- .../AnomalyCharts/containers/FeatureBreakDown.tsx | 1 - .../pages/AnomalyCharts/utils/anomalyChartUtils.tsx | 6 +++--- public/pages/utils/anomalyResultUtils.ts | 8 +++++--- server/utils/constants.ts | 1 + 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx b/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx index 43abda97..64c9246a 100644 --- a/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx +++ b/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx @@ -54,7 +54,10 @@ import { filterHeatmapPlotDataByY, getEntitytAnomaliesHeatmapData, } from '../utils/anomalyChartUtils'; -import { MIN_IN_MILLI_SECS } from '../../../../server/utils/constants'; +import { + MIN_IN_MILLI_SECS, + ENTITY_LIST_DELIMITER, +} from '../../../../server/utils/constants'; import { EntityAnomalySummaries, Entity, @@ -138,9 +141,9 @@ export const AnomalyHeatmapChart = React.memo( } const individualEntityOptions = [] as any[]; //@ts-ignore - individualEntities.forEach((entityListAsString) => { + individualEntities.forEach((entityListAsString: string) => { individualEntityOptions.push({ - label: entityListAsString, + label: entityListAsString.replace(ENTITY_LIST_DELIMITER, ', '), }); }); @@ -275,7 +278,7 @@ export const AnomalyHeatmapChart = React.memo( entityList: convertToEntityList( selectedEntityString, get(props, 'categoryField', []), - ', ' + ENTITY_LIST_DELIMITER ), } as HeatmapCell); } diff --git a/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx b/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx index db714ba6..15de9939 100644 --- a/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx +++ b/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx @@ -48,7 +48,6 @@ import { HeatmapCell } from './AnomalyHeatmapChart'; import { filterWithHeatmapFilter, entityListsMatch, - convertToEntityString, } from '../../utils/anomalyResultUtils'; import { getDateRangeWithSelectedHeatmapCell } from '../utils/anomalyChartUtils'; import { Entity } from '../../../../server/models/interfaces'; diff --git a/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx b/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx index b8aba78c..f64c7710 100644 --- a/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx +++ b/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx @@ -380,9 +380,9 @@ export const getEntitytAnomaliesHeatmapData = ( const numAnomalyGradesForEntity = [] as number[]; const entityString = convertToEntityString( - get(entityAnomalySummaries, 'entityList', []), - ', ' + get(entityAnomalySummaries, 'entityList', []) ) as string; + const anomaliesSummary = get( entityAnomalySummaries, 'anomalySummaries', @@ -460,7 +460,7 @@ const getEntityAnomaliesMap = ( if (isEmpty(entityList)) { return; } - const entityListAsString = convertToEntityString(entityList, ', '); + const entityListAsString = convertToEntityString(entityList); let singleEntityAnomalies = []; if (entityAnomaliesMap.has(entityListAsString)) { //@ts-ignore diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index 37d0a983..b257105c 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -45,6 +45,7 @@ import { WEEK_IN_MILLI_SECS, MODEL_ID_FIELD, ENTITY_LIST_FIELD, + ENTITY_LIST_DELIMITER, } from '../../../server/utils/constants'; import { toFixedNumberForAnomaly } from '../../../server/utils/helpers'; import { @@ -1288,18 +1289,19 @@ export const parseEntityAnomalySummaryResults = ( export const convertToEntityString = ( entityList: Entity[], - delimiter: string + delimiter?: string ) => { let entityString = ''; + const delimiterToUse = delimiter ? delimiter : ENTITY_LIST_DELIMITER; if (!isEmpty(entityList)) { entityList.forEach((entity: any) => { entityString += entity.value; - entityString += delimiter; + entityString += delimiterToUse; }); } return entityString === '' ? entityString - : entityString.slice(0, -delimiter.length); + : entityString.slice(0, -delimiterToUse.length); }; export const getAnomalyDataRangeQuery = ( diff --git a/server/utils/constants.ts b/server/utils/constants.ts index 7434467e..df46f837 100644 --- a/server/utils/constants.ts +++ b/server/utils/constants.ts @@ -106,6 +106,7 @@ export const MODEL_ID_FIELD = 'model_id'; export const DOC_COUNT_FIELD = 'doc_count'; export const KEY_FIELD = 'key'; export const ENTITY_LIST_FIELD = 'entity_list'; +export const ENTITY_LIST_DELIMITER = '
'; export const STACK_TRACE_PATTERN = '.java:'; export const OPENSEARCH_EXCEPTION_PREFIX = From 62cb8246b1839d9c1ca18f7e473a1e1a6004bd3e Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 9 Jun 2021 13:42:40 -0700 Subject: [PATCH 21/35] Fix bug of selecting indiv entity; clean up helpers --- .../containers/AnomalyHeatmapChart.tsx | 2 +- public/pages/utils/anomalyResultUtils.ts | 27 ++++++++++--------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx b/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx index 64c9246a..6368efa3 100644 --- a/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx +++ b/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx @@ -362,7 +362,7 @@ export const AnomalyHeatmapChart = React.memo( setNumEntities(nonCombinedOptions.length); const selectedYs = nonCombinedOptions.map((option) => - get(option, 'label', '') + get(option, 'label', '').replace(', ', ENTITY_LIST_DELIMITER) ); let selectedHeatmapData = filterHeatmapPlotDataByY( diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index b257105c..54379446 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -1294,14 +1294,14 @@ export const convertToEntityString = ( let entityString = ''; const delimiterToUse = delimiter ? delimiter : ENTITY_LIST_DELIMITER; if (!isEmpty(entityList)) { - entityList.forEach((entity: any) => { + entityList.forEach((entity: any, index) => { + if (index > 0) { + entityString += delimiterToUse; + } entityString += entity.value; - entityString += delimiterToUse; }); } - return entityString === '' - ? entityString - : entityString.slice(0, -delimiterToUse.length); + return entityString; }; export const getAnomalyDataRangeQuery = ( @@ -1470,24 +1470,25 @@ export const convertToCategoryFieldString = ( ) => { let categoryFieldString = ''; if (!isEmpty(categoryFields)) { - categoryFields.forEach((categoryField: any) => { + categoryFields.forEach((categoryField: any, index) => { + if (index > 0) { + categoryFieldString += delimiter; + } categoryFieldString += categoryField; - categoryFieldString += delimiter; }); } - return categoryFieldString === '' - ? categoryFieldString - : categoryFieldString.slice(0, -delimiter.length); + return categoryFieldString; }; export const convertToCategoryFieldAndEntityString = (entityList: Entity[]) => { let entityString = ''; if (!isEmpty(entityList)) { - entityList.forEach((entity: any) => { + entityList.forEach((entity: any, index) => { + if (index > 0) { + entityString += '\n'; + } entityString += entity.name + ': ' + entity.value; - entityString += '\n'; }); - entityString.slice(0, -1); } return entityString; }; From cd5986fd996640232aff9179f7766c64be25f8b9 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 28 Jul 2021 09:53:31 -0700 Subject: [PATCH 22/35] Clean up anomaly history rebase changes --- .../containers/AnomalyHistory.tsx | 69 +------------------ 1 file changed, 3 insertions(+), 66 deletions(-) diff --git a/public/pages/DetectorResults/containers/AnomalyHistory.tsx b/public/pages/DetectorResults/containers/AnomalyHistory.tsx index 97a323e9..b7ad7e74 100644 --- a/public/pages/DetectorResults/containers/AnomalyHistory.tsx +++ b/public/pages/DetectorResults/containers/AnomalyHistory.tsx @@ -637,74 +637,11 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { )} isHCDetector={isHCDetector} isHistorical={props.isHistorical} - selectedHeatmapCell={selectedHeatmapCell} />, - , ] - : null} - {renderTabs()} - - {isLoading || isLoadingAnomalyResults ? ( - - - - - - ) : ( -
- - {selectedTabId === ANOMALY_HISTORY_TABS.FEATURE_BREAKDOWN ? ( - - ) : ( - - )} - -
- )} -
+ )} +
+ )} ); From 7e619b0f831e86557723cc7b10eb50b4d67fd3a1 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 28 Jul 2021 11:12:32 -0700 Subject: [PATCH 23/35] Add multi category field selection in create flow; clean up formatting in HC preview component --- .../containers/AnomaliesChart.tsx | 4 -- .../CategoryField/CategoryField.tsx | 38 +++++++++++-------- .../containers/AnomalyResultsTable.tsx | 8 ++-- .../DetectorResults/utils/tableUtils.tsx | 10 ++--- 4 files changed, 32 insertions(+), 28 deletions(-) diff --git a/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx b/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx index c5ebc7eb..a9aef6f4 100644 --- a/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx +++ b/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx @@ -289,10 +289,6 @@ export const AnomaliesChart = React.memo((props: AnomaliesChartProps) => {

- {`${get( - props, - 'detectorCategoryField.0' - )} `} {props.selectedHeatmapCell ? getHCTitle( diff --git a/public/pages/ConfigureModel/components/CategoryField/CategoryField.tsx b/public/pages/ConfigureModel/components/CategoryField/CategoryField.tsx index 4517e77f..56dac70b 100644 --- a/public/pages/ConfigureModel/components/CategoryField/CategoryField.tsx +++ b/public/pages/ConfigureModel/components/CategoryField/CategoryField.tsx @@ -84,7 +84,7 @@ export function CategoryField(props: CategoryFieldProps) { -

Categorical field

+

Categorical fields

} subTitle={ @@ -92,8 +92,8 @@ export function CategoryField(props: CategoryFieldProps) { className="content-panel-subTitle" style={{ lineHeight: 'normal' }} > - Split a single time series into multiple time series based on a - categorical field.{' '} + Split a single time series into multiple time series based on + categorical fields.{' '} Learn more @@ -119,7 +119,7 @@ export function CategoryField(props: CategoryFieldProps) { { @@ -144,24 +144,26 @@ export function CategoryField(props: CategoryFieldProps) { label="Field" isInvalid={isInvalid(field.name, form)} error={getError(field.name, form)} - helpText={`You can only apply the category field to the 'ip' and 'keyword' OpenSearch data types.`} + helpText={`You can only apply the categorical fields to the 'ip' and 'keyword' OpenSearch data types.`} > { form.setFieldTouched('categoryField', true); }} onChange={(options) => { - const selection = get(options, '0.label'); - if (selection) { - form.setFieldValue('categoryField', [selection]); - form.setFieldValue( - 'shingleSize', - MULTI_ENTITY_SHINGLE_SIZE - ); + const selection = options.map((option) => option.label); + if (!isEmpty(selection)) { + if (selection.length <= 2) { + form.setFieldValue('categoryField', selection); + form.setFieldValue( + 'shingleSize', + MULTI_ENTITY_SHINGLE_SIZE + ); + } } else { form.setFieldValue('categoryField', []); @@ -172,9 +174,15 @@ export function CategoryField(props: CategoryFieldProps) { } }} selectedOptions={ - (field.value[0] && [{ label: field.value[0] }]) || [] + field.value + ? field.value.map((value: any) => { + return { + label: value, + }; + }) + : [] } - singleSelection={{ asPlainText: true }} + singleSelection={false} isClearable={true} /> diff --git a/public/pages/DetectorResults/containers/AnomalyResultsTable.tsx b/public/pages/DetectorResults/containers/AnomalyResultsTable.tsx index 3d4c2c1c..9043bc9c 100644 --- a/public/pages/DetectorResults/containers/AnomalyResultsTable.tsx +++ b/public/pages/DetectorResults/containers/AnomalyResultsTable.tsx @@ -35,9 +35,9 @@ import React, { useEffect, useState } from 'react'; import { SORT_DIRECTION } from '../../../../server/utils/constants'; import ContentPanel from '../../../components/ContentPanel/ContentPanel'; import { - categoryFieldsColumn, + entityValueColumn, staticColumn, - CATEGORY_FIELDS, + ENTITY_VALUE_FIELD, } from '../utils/tableUtils'; import { DetectorResultsQueryParams } from 'server/models/types'; import { AnomalyData } from '../../../models/interfaces'; @@ -90,7 +90,7 @@ export function AnomalyResultsTable(props: AnomalyResultsTableProps) { anomalies = anomalies.map((anomaly) => { return { ...anomaly, - [CATEGORY_FIELDS]: convertToCategoryFieldAndEntityString( + [ENTITY_VALUE_FIELD]: convertToCategoryFieldAndEntityString( get(anomaly, 'entity', []) ), }; @@ -158,7 +158,7 @@ export function AnomalyResultsTable(props: AnomalyResultsTableProps) { : props.isHCDetector ? [ ...staticColumn.slice(0, 2), - categoryFieldsColumn, + entityValueColumn, ...staticColumn.slice(2), ] : props.isHistorical diff --git a/public/pages/DetectorResults/utils/tableUtils.tsx b/public/pages/DetectorResults/utils/tableUtils.tsx index 6521090e..6e464726 100644 --- a/public/pages/DetectorResults/utils/tableUtils.tsx +++ b/public/pages/DetectorResults/utils/tableUtils.tsx @@ -38,7 +38,7 @@ const renderTime = (time: number) => { return DEFAULT_EMPTY_DATA; }; -export const CATEGORY_FIELDS = 'categoryFields'; +export const ENTITY_VALUE_FIELD = 'entityValue'; export const staticColumn = [ { @@ -91,10 +91,10 @@ export const staticColumn = [ }, ] as EuiBasicTableColumn[]; -export const categoryFieldsColumn = { - field: CATEGORY_FIELDS, - name: 'Category fields', +export const entityValueColumn = { + field: ENTITY_VALUE_FIELD, + name: 'Entity', sortable: true, truncateText: false, - dataType: 'string', + dataType: 'number', } as EuiBasicTableColumn; From d56aeceff54a4ed3e15bc00fbaff87efb63e61dd Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 28 Jul 2021 12:11:00 -0700 Subject: [PATCH 24/35] Refactor AnomalyHistory to show anomaly + feature charts simultaneously --- .../AdditionalSettings/AdditionalSettings.tsx | 2 +- .../containers/AnomalyHistory.tsx | 169 +++++++++--------- .../DetectorResults/utils/tableUtils.tsx | 4 +- 3 files changed, 88 insertions(+), 87 deletions(-) diff --git a/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx b/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx index 96b42e5a..ad277994 100644 --- a/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx +++ b/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx @@ -45,7 +45,7 @@ export function AdditionalSettings(props: AdditionalSettingsProps) { }, ]; const tableColumns = [ - { name: 'Category fields', field: 'categoryField' }, + { name: 'Categorical fields', field: 'categoryField' }, { name: 'Window size', field: 'windowSize' }, ]; return ( diff --git a/public/pages/DetectorResults/containers/AnomalyHistory.tsx b/public/pages/DetectorResults/containers/AnomalyHistory.tsx index b7ad7e74..def02ddd 100644 --- a/public/pages/DetectorResults/containers/AnomalyHistory.tsx +++ b/public/pages/DetectorResults/containers/AnomalyHistory.tsx @@ -555,93 +555,94 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { heatmapDisplayOption={heatmapDisplayOption} entityAnomalySummaries={entityAnomalySummaries} > - {renderTabs()} - - {isLoading || isLoadingAnomalyResults ? ( - - - - - - ) : ( -
- {selectedTabId === ANOMALY_HISTORY_TABS.FEATURE_BREAKDOWN ? ( - - ) : ( - [ - isHCDetector - ? [ - , - , - ] - : null, - + {isHCDetector + ? [ + , + , ] - )} -
- )} + : null} + {renderTabs()} + + {isLoading || isLoadingAnomalyResults ? ( + + + + + + ) : ( +
+ + {selectedTabId === ANOMALY_HISTORY_TABS.FEATURE_BREAKDOWN ? ( + + ) : ( + + )} + +
+ )} +

); diff --git a/public/pages/DetectorResults/utils/tableUtils.tsx b/public/pages/DetectorResults/utils/tableUtils.tsx index 6e464726..8bd2864e 100644 --- a/public/pages/DetectorResults/utils/tableUtils.tsx +++ b/public/pages/DetectorResults/utils/tableUtils.tsx @@ -93,8 +93,8 @@ export const staticColumn = [ export const entityValueColumn = { field: ENTITY_VALUE_FIELD, - name: 'Entity', + name: 'Entities', sortable: true, truncateText: false, - dataType: 'number', + dataType: 'string', } as EuiBasicTableColumn; From df081c45c3ddbf118c705fe8b0d7bce22c46c567 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 28 Jul 2021 15:04:29 -0700 Subject: [PATCH 25/35] Fix bug of showing RT HC non-multi-category results on historical heatmap; Fix bug of entity details wrong when filtering by indiv entity --- public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx | 4 ++++ public/pages/utils/anomalyResultUtils.ts | 4 +--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx b/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx index f64c7710..0525804c 100644 --- a/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx +++ b/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx @@ -497,15 +497,18 @@ export const filterHeatmapPlotDataByY = ( const originalYs = cloneDeep(heatmapData.y); const originalZs = cloneDeep(heatmapData.z); const originalTexts = cloneDeep(heatmapData.text); + const originalEntityLists = cloneDeep(heatmapData.customdata); const resultYs = []; const resultZs = []; const resultTexts = []; + const resultEntityLists = []; for (let i = 0; i < originalYs.length; i++) { //@ts-ignore if (selectedYs.includes(originalYs[i])) { resultYs.push(originalYs[i]); resultZs.push(originalZs[i]); resultTexts.push(originalTexts[i]); + resultEntityLists.push(originalEntityLists[i]); } } const updateHeatmapPlotData = { @@ -513,6 +516,7 @@ export const filterHeatmapPlotDataByY = ( y: resultYs, z: resultZs, text: resultTexts, + customdata: resultEntityLists, } as PlotData; return sortHeatmapPlotData( updateHeatmapPlotData, diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index 54379446..4618c510 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -1052,9 +1052,7 @@ export const getTopAnomalousEntitiesQuery = ( }, }, { - term: { - detector_id: detectorId, - }, + term: termField, }, ], }, From f2bd2cae5eda814bcb2e83f50e07c6c4a89931d9 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 28 Jul 2021 15:16:51 -0700 Subject: [PATCH 26/35] Clean up tests --- .../__tests__/CategoryField.test.tsx | 18 +- .../__snapshots__/CategoryField.test.tsx.snap | 30 +- .../ConfigureModel.test.tsx.snap | 12 +- .../__snapshots__/CategoryField.test.tsx.snap | 637 ------------------ 4 files changed, 35 insertions(+), 662 deletions(-) delete mode 100644 public/pages/EditFeatures/components/CategoryField/__tests__/__snapshots__/CategoryField.test.tsx.snap diff --git a/public/pages/ConfigureModel/components/CategoryField/__tests__/CategoryField.test.tsx b/public/pages/ConfigureModel/components/CategoryField/__tests__/CategoryField.test.tsx index 2706c7cb..8082928a 100644 --- a/public/pages/ConfigureModel/components/CategoryField/__tests__/CategoryField.test.tsx +++ b/public/pages/ConfigureModel/components/CategoryField/__tests__/CategoryField.test.tsx @@ -42,6 +42,7 @@ describe(' spec', () => {
{ @@ -63,7 +64,7 @@ describe(' spec', () => { expect(container).toMatchSnapshot(); expect(queryByTestId('noCategoryFieldsCallout')).toBeNull(); expect(queryByTestId('categoryFieldComboBox')).toBeNull(); - expect(queryByText('Enable categorical field')).not.toBeNull(); + expect(queryByText('Enable categorical fields')).not.toBeNull(); }); test('renders the component when enabled', () => { const { @@ -83,6 +84,7 @@ describe(' spec', () => { { @@ -104,7 +106,7 @@ describe(' spec', () => { expect(container).toMatchSnapshot(); expect(queryByTestId('noCategoryFieldsCallout')).toBeNull(); expect(queryByTestId('categoryFieldComboBox')).not.toBeNull(); - expect(queryByText('Enable categorical field')).not.toBeNull(); + expect(queryByText('Enable categorical fields')).not.toBeNull(); fireEvent.click(getByTestId('comboBoxToggleListButton')); getByText('a'); getByText('b'); @@ -121,6 +123,7 @@ describe(' spec', () => { { @@ -142,7 +145,7 @@ describe(' spec', () => { expect(container).toMatchSnapshot(); expect(queryByTestId('noCategoryFieldsCallout')).not.toBeNull(); expect(queryByTestId('categoryFieldComboBox')).toBeNull(); - expect(queryByText('Enable categorical field')).not.toBeNull(); + expect(queryByText('Enable categorical fields')).not.toBeNull(); }); test('hides callout if component is loading', () => { const { container, queryByText, queryByTestId } = render( @@ -156,6 +159,7 @@ describe(' spec', () => { { @@ -176,7 +180,7 @@ describe(' spec', () => { ); expect(container).toMatchSnapshot(); expect(queryByTestId('noCategoryFieldsCallout')).toBeNull(); - expect(queryByText('Enable categorical field')).not.toBeNull(); + expect(queryByText('Enable categorical fields')).not.toBeNull(); }); test(`limits selection to a maximum of 2 entities`, () => { const { getAllByRole, getByTestId, queryByText } = render( @@ -190,6 +194,7 @@ describe(' spec', () => { { @@ -197,6 +202,11 @@ describe(' spec', () => { }} isLoading={false} originalShingleSize={1} + formikProps={{ + values: { + categoryFieldEnabled: true, + }, + }} /> diff --git a/public/pages/ConfigureModel/components/CategoryField/__tests__/__snapshots__/CategoryField.test.tsx.snap b/public/pages/ConfigureModel/components/CategoryField/__tests__/__snapshots__/CategoryField.test.tsx.snap index 5d093848..ab2389a0 100644 --- a/public/pages/ConfigureModel/components/CategoryField/__tests__/__snapshots__/CategoryField.test.tsx.snap +++ b/public/pages/ConfigureModel/components/CategoryField/__tests__/__snapshots__/CategoryField.test.tsx.snap @@ -26,7 +26,7 @@ exports[` spec hides callout if component is loading 1`] = ` class="euiTitle euiTitle--small" id="categoryFieldTitle" > - Categorical field + Categorical fields
@@ -41,7 +41,7 @@ exports[` spec hides callout if component is loading 1`] = ` class="euiText euiText--medium content-panel-subTitle" style="line-height: normal;" > - Split a single time series into multiple time series based on a categorical field. + Split a single time series into multiple time series based on categorical fields. spec hides callout if component is loading 1`] = ` class="euiCheckbox__label" for="categoryFieldCheckbox" > - Enable categorical field + Enable categorical fields
@@ -152,7 +152,7 @@ exports[` spec renders the component when disabled 1`] = ` class="euiTitle euiTitle--small" id="categoryFieldTitle" > - Categorical field + Categorical fields
@@ -167,7 +167,7 @@ exports[` spec renders the component when disabled 1`] = ` class="euiText euiText--medium content-panel-subTitle" style="line-height: normal;" > - Split a single time series into multiple time series based on a categorical field. + Split a single time series into multiple time series based on categorical fields. spec renders the component when disabled 1`] = ` class="euiCheckbox__label" for="categoryFieldCheckbox" > - Enable categorical field + Enable categorical fields
@@ -269,7 +269,7 @@ exports[` spec renders the component when enabled 1`] = ` class="euiTitle euiTitle--small" id="categoryFieldTitle" > - Categorical field + Categorical fields
@@ -284,7 +284,7 @@ exports[` spec renders the component when enabled 1`] = ` class="euiText euiText--medium content-panel-subTitle" style="line-height: normal;" > - Split a single time series into multiple time series based on a categorical field. + Split a single time series into multiple time series based on categorical fields. spec renders the component when enabled 1`] = ` class="euiCheckbox__label" for="categoryFieldCheckbox" > - Enable categorical field + Enable categorical fields
@@ -393,14 +393,14 @@ exports[` spec renders the component when enabled 1`] = ` class="euiFormControlLayout__childrenWrapper" >

- Select your category field + Select your categorical fields

spec renders the component when enabled 1`] = ` class="euiFormHelpText euiFormRow__text" id="random_html_id-help" > - You can only apply the category field to the 'ip' and 'keyword' OpenSearch data types. + You can only apply the categorical fields to the 'ip' and 'keyword' OpenSearch data types.
@@ -486,7 +486,7 @@ exports[` spec shows callout when there are no available catego class="euiTitle euiTitle--small" id="categoryFieldTitle" > - Categorical field + Categorical fields
@@ -501,7 +501,7 @@ exports[` spec shows callout when there are no available catego class="euiText euiText--medium content-panel-subTitle" style="line-height: normal;" > - Split a single time series into multiple time series based on a categorical field. + Split a single time series into multiple time series based on categorical fields. spec shows callout when there are no available catego class="euiCheckbox__label" for="categoryFieldCheckbox" > - Enable categorical field + Enable categorical fields diff --git a/public/pages/ConfigureModel/containers/__tests__/__snapshots__/ConfigureModel.test.tsx.snap b/public/pages/ConfigureModel/containers/__tests__/__snapshots__/ConfigureModel.test.tsx.snap index ba2643e5..fb89578a 100644 --- a/public/pages/ConfigureModel/containers/__tests__/__snapshots__/ConfigureModel.test.tsx.snap +++ b/public/pages/ConfigureModel/containers/__tests__/__snapshots__/ConfigureModel.test.tsx.snap @@ -584,7 +584,7 @@ exports[` spec creating model configuration renders the compon class="euiTitle euiTitle--small" id="categoryFieldTitle" > - Categorical field + Categorical fields @@ -599,7 +599,7 @@ exports[` spec creating model configuration renders the compon class="euiText euiText--medium content-panel-subTitle" style="line-height: normal;" > - Split a single time series into multiple time series based on a categorical field. + Split a single time series into multiple time series based on categorical fields. spec creating model configuration renders the compon class="euiCheckbox__label" for="categoryFieldCheckbox" > - Enable categorical field + Enable categorical fields @@ -1515,7 +1515,7 @@ exports[` spec editing model configuration renders the compone class="euiTitle euiTitle--small" id="categoryFieldTitle" > - Categorical field + Categorical fields @@ -1530,7 +1530,7 @@ exports[` spec editing model configuration renders the compone class="euiText euiText--medium content-panel-subTitle" style="line-height: normal;" > - Split a single time series into multiple time series based on a categorical field. + Split a single time series into multiple time series based on categorical fields. spec editing model configuration renders the compone class="euiCheckbox__label" for="categoryFieldCheckbox" > - Enable categorical field + Enable categorical fields diff --git a/public/pages/EditFeatures/components/CategoryField/__tests__/__snapshots__/CategoryField.test.tsx.snap b/public/pages/EditFeatures/components/CategoryField/__tests__/__snapshots__/CategoryField.test.tsx.snap deleted file mode 100644 index 54e6c969..00000000 --- a/public/pages/EditFeatures/components/CategoryField/__tests__/__snapshots__/CategoryField.test.tsx.snap +++ /dev/null @@ -1,637 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[` spec hides callout if component is loading 1`] = ` - -`; - -exports[` spec renders the component when disabled 1`] = ` - -`; - -exports[` spec renders the component when enabled 1`] = ` - -`; - -exports[` spec shows callout when there are no available category fields 1`] = ` - -`; From 8fc9c5a3c10c088566075c3aaa87e3824c2d2ac9 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Thu, 29 Jul 2021 09:43:09 -0700 Subject: [PATCH 27/35] Add UT for AnomaliesChart --- .../containers/AnomaliesChart.tsx | 2 +- .../containers/AnomalyHeatmapChart.tsx | 4 +- .../__tests__/AnomaliesChart.test.tsx | 153 ++++++++++++++++-- 3 files changed, 139 insertions(+), 20 deletions(-) diff --git a/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx b/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx index a9aef6f4..68eb2846 100644 --- a/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx +++ b/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx @@ -68,7 +68,7 @@ import { AnomalyOccurrenceChart } from './AnomalyOccurrenceChart'; import { FeatureBreakDown } from './FeatureBreakDown'; import { convertTimestampToString } from '../../../utils/utils'; -interface AnomaliesChartProps { +export interface AnomaliesChartProps { onDateRangeChange( startDate: number, endDate: number, diff --git a/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx b/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx index 6368efa3..80e3b886 100644 --- a/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx +++ b/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx @@ -178,7 +178,7 @@ export const AnomalyHeatmapChart = React.memo( getEntitytAnomaliesHeatmapData( props.dateRange, props.entityAnomalySummaries, - props.heatmapDisplayOption.entityOption.value + props.heatmapDisplayOption?.entityOption.value ) : // use anomalies data in case of sample result getAnomaliesHeatmapData( @@ -197,7 +197,7 @@ export const AnomalyHeatmapChart = React.memo( AnomalyHeatmapSortType >( props.isNotSample - ? props.heatmapDisplayOption.sortType + ? props.heatmapDisplayOption?.sortType : SORT_BY_FIELD_OPTIONS[0].value ); diff --git a/public/pages/AnomalyCharts/containers/__tests__/AnomaliesChart.test.tsx b/public/pages/AnomalyCharts/containers/__tests__/AnomaliesChart.test.tsx index f7875697..7645f4e2 100644 --- a/public/pages/AnomalyCharts/containers/__tests__/AnomaliesChart.test.tsx +++ b/public/pages/AnomalyCharts/containers/__tests__/AnomaliesChart.test.tsx @@ -26,8 +26,8 @@ import React from 'react'; import { render } from '@testing-library/react'; -import { AnomaliesChart } from '../AnomaliesChart'; -import { initialState, mockedStore } from '../../../../redux/utils/testUtils'; +import { AnomaliesChart, AnomaliesChartProps } from '../AnomaliesChart'; +import { mockedStore } from '../../../../redux/utils/testUtils'; import { Provider } from 'react-redux'; import { INITIAL_ANOMALY_SUMMARY } from '../../utils/constants'; import { getRandomDetector } from '../../../../redux/reducers/__tests__/utils'; @@ -38,30 +38,149 @@ import { FAKE_DATE_RANGE, } from '../../../../pages/utils/__tests__/constants'; -const renderDataFilter = () => ({ +const DEFAULT_PROPS = { + onDateRangeChange: jest.fn(), + onZoomRangeChange: jest.fn(), + title: 'Test title', + bucketizedAnomalies: true, + anomalySummary: INITIAL_ANOMALY_SUMMARY, + dateRange: FAKE_DATE_RANGE, + isLoading: false, + showAlerts: false, + isNotSample: true, + detector: getRandomDetector(true), + children: [], + isHCDetector: false, + isHistorical: false, + detectorCategoryField: [], + onHeatmapCellSelected: jest.fn(), + onDisplayOptionChanged: jest.fn(), + selectedHeatmapCell: undefined, + newDetector: undefined, + zoomRange: undefined, + anomaliesResult: FAKE_ANOMALIES_RESULT, + entityAnomalySummaries: [], +} as AnomaliesChartProps; + +const renderDataFilter = (chartProps: AnomaliesChartProps) => ({ ...render( - + ), }); describe(' spec', () => { - test('renders the component', () => { + test('renders the component for sample / preview', () => { + console.error = jest.fn(); + const { getByText, getAllByText } = renderDataFilter({ + ...DEFAULT_PROPS, + isHistorical: false, + isHCDetector: false, + isNotSample: false, + }); + expect(getByText('Test title')).not.toBeNull(); + expect(getAllByText('Sample anomaly occurrences').length >= 1); + expect(getAllByText('Sample anomaly grade').length >= 1); + expect(getAllByText('Sample confidence').length >= 1); + }); + test('renders the component for RT, non-HC detector', () => { + console.error = jest.fn(); + const { getByText, getAllByText } = renderDataFilter({ + ...DEFAULT_PROPS, + isHistorical: false, + isHCDetector: false, + }); + expect(getByText('Test title')).not.toBeNull(); + expect(getAllByText('Anomaly occurrences').length >= 1); + expect(getAllByText('Anomaly grade').length >= 1); + expect(getAllByText('Confidence').length >= 1); + expect(getAllByText('Last anomaly occurrence').length >= 1); + }); + test('renders the component for RT, HC detector', () => { + console.error = jest.fn(); + const { getByText } = renderDataFilter({ + ...DEFAULT_PROPS, + isHistorical: false, + isHCDetector: true, + detectorCategoryField: ['category-1'], + }); + getByText('Test title'); + getByText('Top 10'); + getByText('category-1'); + }); + test('renders the component for RT, multi-category-HC detector', () => { + console.error = jest.fn(); + const { getByText } = renderDataFilter({ + ...DEFAULT_PROPS, + isHistorical: false, + isHCDetector: true, + detectorCategoryField: ['category-1, category-2'], + }); + getByText('Test title'); + getByText('Top 10'); + getByText('category-1, category-2'); + }); + test('renders the component for historical, non-HC detector', () => { + console.error = jest.fn(); + const { getAllByText, queryByText } = renderDataFilter({ + ...DEFAULT_PROPS, + isHistorical: true, + isHCDetector: false, + }); + expect(queryByText('Test title')).not.toBeNull(); + expect(getAllByText('Anomaly occurrences').length >= 1); + expect(getAllByText('Average anomaly grade').length >= 1); + expect(queryByText('Confidence')).toBeNull(); + }); + test('renders the component for historical, HC detector', () => { + console.error = jest.fn(); + const { getByText } = renderDataFilter({ + ...DEFAULT_PROPS, + isHistorical: true, + isHCDetector: true, + detectorCategoryField: ['category-1'], + }); + getByText('Test title'); + getByText('Top 10'); + getByText('category-1'); + }); + test('renders the component for historical, multi-category-HC detector', () => { + console.error = jest.fn(); + const { getByText } = renderDataFilter({ + ...DEFAULT_PROPS, + isHistorical: true, + isHCDetector: true, + detectorCategoryField: ['category-1', 'category-2'], + }); + getByText('Test title'); + getByText('Top 10'); + getByText('category-1, category-2'); + }); + test('renders multiple category fields if stored in alphabetical order', () => { + console.error = jest.fn(); + const { getByText } = renderDataFilter({ + ...DEFAULT_PROPS, + isHistorical: false, + isHCDetector: true, + detectorCategoryField: ['a', 'b'], + }); + getByText('Test title'); + getByText('Top 10'); + getByText('a, b'); + }); + test('renders multiple category fields if stored in non-alphabetical order', () => { console.error = jest.fn(); - const { getByText } = renderDataFilter(); - expect(getByText('Sample anomaly grade')).not.toBeNull(); + const { getByText } = renderDataFilter({ + ...DEFAULT_PROPS, + isHistorical: false, + isHCDetector: true, + detectorCategoryField: ['b', 'a'], + }); + getByText('Test title'); + getByText('Top 10'); + getByText('a, b'); }); }); From 0771e7a5f02a0ee40d7d9f9470776540ff54947a Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Thu, 29 Jul 2021 09:47:29 -0700 Subject: [PATCH 28/35] Remove leftover EditFeatures --- .../CategoryField/CategoryField.tsx | 195 ------------------ .../components/ConfirmModal/index.ts | 27 --- public/pages/EditFeatures/utils/constants.ts | 72 ------- 3 files changed, 294 deletions(-) delete mode 100644 public/pages/EditFeatures/components/CategoryField/CategoryField.tsx delete mode 100644 public/pages/EditFeatures/components/ConfirmModal/index.ts delete mode 100644 public/pages/EditFeatures/utils/constants.ts diff --git a/public/pages/EditFeatures/components/CategoryField/CategoryField.tsx b/public/pages/EditFeatures/components/CategoryField/CategoryField.tsx deleted file mode 100644 index 1aec8e69..00000000 --- a/public/pages/EditFeatures/components/CategoryField/CategoryField.tsx +++ /dev/null @@ -1,195 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -import { - EuiFlexItem, - EuiFlexGroup, - EuiText, - EuiLink, - EuiIcon, - EuiPage, - EuiPageBody, - EuiComboBox, - EuiCheckbox, - EuiTitle, - EuiCallOut, - EuiSpacer, -} from '@elastic/eui'; -import { Field, FieldProps } from 'formik'; -import { isEmpty } from 'lodash'; -import { - MULTI_ENTITY_SHINGLE_SIZE, - BASE_DOCS_LINK, -} from '../../../../utils/constants'; -import React, { useState, useEffect } from 'react'; -import ContentPanel from '../../../../components/ContentPanel/ContentPanel'; -import { - isInvalid, - getError, - validateCategoryField, -} from '../../../../utils/utils'; -import { MAX_CATEGORY_FIELD_NUM } from '../../../../utils/constants'; -import { FormattedFormRow } from '../../../createDetector/components/FormattedFormRow/FormattedFormRow'; - -interface CategoryFieldProps { - isHCDetector: boolean; - categoryFieldOptions: string[]; - setIsHCDetector(isHCDetector: boolean): void; - isLoading: boolean; - originalShingleSize: number; -} - -export function CategoryField(props: CategoryFieldProps) { - const [enabled, setEnabled] = useState(props.isHCDetector); - const noCategoryFields = isEmpty(props.categoryFieldOptions); - const convertedOptions = props.categoryFieldOptions.map((option: string) => { - return { - label: option, - }; - }); - - useEffect(() => { - setEnabled(props.isHCDetector); - }, [props.isHCDetector]); - - return ( - - - -

Categorical fields

- - } - subTitle={ - - Categorize anomalies based on unique partitions. For example, with - clickstream data you can categorize anomalies into a given day, - week, or month.{' '} - - Learn more - - - } - > - {noCategoryFields && !props.isLoading ? ( - - ) : null} - {noCategoryFields ? : null} - - {({ field, form }: FieldProps) => ( - - - { - if (!enabled) { - props.setIsHCDetector(true); - } - if (enabled) { - props.setIsHCDetector(false); - form.setFieldValue('categoryField', []); - form.setFieldValue( - 'shingleSize', - props.originalShingleSize - ); - } - setEnabled(!enabled); - }} - /> - - {enabled && !noCategoryFields ? ( - - - { - form.setFieldTouched('categoryField', true); - }} - onChange={(options) => { - const selection = options.map( - (option) => option.label - ); - if (!isEmpty(selection)) { - if (selection.length <= MAX_CATEGORY_FIELD_NUM) { - form.setFieldValue('categoryField', selection); - form.setFieldValue( - 'shingleSize', - MULTI_ENTITY_SHINGLE_SIZE - ); - } - } else { - form.setFieldValue('categoryField', []); - - form.setFieldValue( - 'shingleSize', - props.originalShingleSize - ); - } - }} - selectedOptions={ - field.value - ? field.value.map((value: any) => { - return { - label: value, - }; - }) - : [] - } - singleSelection={false} - isClearable={true} - /> - - - ) : null} - - )} - - - - - ); -} diff --git a/public/pages/EditFeatures/components/ConfirmModal/index.ts b/public/pages/EditFeatures/components/ConfirmModal/index.ts deleted file mode 100644 index 0d9786eb..00000000 --- a/public/pages/EditFeatures/components/ConfirmModal/index.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -export { SaveFeaturesConfirmModal } from './SaveFeaturesConfirmModal'; diff --git a/public/pages/EditFeatures/utils/constants.ts b/public/pages/EditFeatures/utils/constants.ts deleted file mode 100644 index 18c74710..00000000 --- a/public/pages/EditFeatures/utils/constants.ts +++ /dev/null @@ -1,72 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -import { FEATURE_TYPE } from '../../../models/interfaces'; -import { FeaturesFormikValues } from '../containers/utils/formikToFeatures'; - -export const FEATURE_TYPES = [ - { text: 'Custom Aggregation', value: FEATURE_TYPE.CUSTOM }, - { text: 'Defined Aggregation', value: FEATURE_TYPE.SIMPLE }, -]; - -export const FEATURE_TYPE_OPTIONS = [ - { text: 'Field value', value: FEATURE_TYPE.SIMPLE }, - { text: 'Custom expression', value: FEATURE_TYPE.CUSTOM }, -]; - -export enum SAVE_FEATURE_OPTIONS { - START_AD_JOB = 'start_ad_job', - KEEP_AD_JOB_STOPPED = 'keep_ad_job_stopped', -} - -export const AGGREGATION_TYPES = [ - { value: 'avg', text: 'average()' }, - { value: 'value_count', text: 'count()' }, - { value: 'sum', text: 'sum()' }, - { value: 'min', text: 'min()' }, - { value: 'max', text: 'max()' }, -]; - -export const FEATURE_FIELDS = [ - 'featureName', - 'aggregationOf', - 'aggregationBy', - 'aggregationQuery', -]; - -export const INITIAL_VALUES: FeaturesFormikValues = { - featureId: '', - featureName: '', - featureEnabled: true, - featureType: FEATURE_TYPE.SIMPLE, - aggregationQuery: JSON.stringify( - { aggregation_name: { sum: { field: 'field_name' } } }, - null, - 4 - ), - aggregationBy: '', - aggregationOf: [], -}; From 3c5f100dd1c0c0369f360991453651369c44ca00 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Mon, 9 Aug 2021 16:11:47 -0700 Subject: [PATCH 29/35] Show partial results in historical HC case --- .../containers/AnomaliesChart.tsx | 6 +++++ .../containers/AnomalyHeatmapChart.tsx | 22 +++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx b/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx index 68eb2846..63bb9863 100644 --- a/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx +++ b/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx @@ -246,6 +246,12 @@ export const AnomaliesChart = React.memo((props: AnomaliesChartProps) => { { + if (props.isHistorical) { + console.log('updating partial HC results'); + const updateHeatmapPlotData = getAnomaliesHeatmapData( + props.anomalies, + props.dateRange, + sortByFieldValue, + get(COMBINED_OPTIONS.options[0], 'value') + ); + setOriginalHeatmapData(updateHeatmapPlotData); + setHeatmapData(updateHeatmapPlotData); + setNumEntities(updateHeatmapPlotData[0].y.length); + setEntityViewOptions(getViewEntityOptions(updateHeatmapPlotData)); + } + }, [props.detectorTaskProgress]); + const handleHeatmapClick = (event: Plotly.PlotMouseEvent) => { const selectedCellIndices = get(event, 'points[0].pointIndex', []); const selectedEntityString = get(event, 'points[0].y', ''); From f66b09a87e5777ae18a6f3a97bcb68513eeb6b21 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Mon, 9 Aug 2021 16:13:54 -0700 Subject: [PATCH 30/35] Remove log statement --- public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx b/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx index 35d22729..5afac85f 100644 --- a/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx +++ b/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx @@ -232,7 +232,6 @@ export const AnomalyHeatmapChart = React.memo( // Custom hook to refresh all of the heatmap data when running a historical task useEffect(() => { if (props.isHistorical) { - console.log('updating partial HC results'); const updateHeatmapPlotData = getAnomaliesHeatmapData( props.anomalies, props.dateRange, From 8704639c81cc1f3301448dcdf4bbbf0a9f9ea05c Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 10 Aug 2021 11:27:32 -0700 Subject: [PATCH 31/35] Fix bug of single-entity bucketized results not showing --- .../containers/AnomalyHistory.tsx | 3 -- public/pages/utils/anomalyResultUtils.ts | 37 +++++++++++++------ 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/public/pages/DetectorResults/containers/AnomalyHistory.tsx b/public/pages/DetectorResults/containers/AnomalyHistory.tsx index def02ddd..c3467ed1 100644 --- a/public/pages/DetectorResults/containers/AnomalyHistory.tsx +++ b/public/pages/DetectorResults/containers/AnomalyHistory.tsx @@ -189,7 +189,6 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { entityList, props.isHistorical, taskId.current, - isMultiCategory, modelId ) ) @@ -207,7 +206,6 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { entityList, props.isHistorical, taskId.current, - isMultiCategory, modelId ) ) @@ -341,7 +339,6 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { props.detector.id, NUM_CELLS, summary.entityList, - isMultiCategory, summary.modelId, props.isHistorical, taskId.current diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index 4618c510..f255c48f 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -315,7 +315,6 @@ export const getAnomalySummaryQuery = ( entityList: Entity[] | undefined = undefined, isHistorical?: boolean, taskId?: string, - isMultiCategory?: boolean, modelId?: string ) => { const termField = @@ -343,7 +342,6 @@ export const getAnomalySummaryQuery = ( { term: termField, }, - getResultFilters(isMultiCategory, modelId, entityList), ], }, }, @@ -389,6 +387,8 @@ export const getAnomalySummaryQuery = ( }, }; + // If querying RT results: remove any results that include a task_id, as this indicates + // a historical result from a historical task. if (!isHistorical) { requestBody.query.bool = { ...requestBody.query.bool, @@ -402,6 +402,12 @@ export const getAnomalySummaryQuery = ( }; } + // Add entity filters if this is a HC detector + if (entityList !== undefined && entityList.length > 0) { + //@ts-ignore + requestBody.query.bool.filter.push(getEntityFilters(modelId, entityList)); + } + return requestBody; }; @@ -412,7 +418,6 @@ export const getBucketizedAnomalyResultsQuery = ( entityList: Entity[] | undefined = undefined, isHistorical?: boolean, taskId?: string, - isMultiCategory?: boolean, modelId?: string ) => { const termField = @@ -436,7 +441,6 @@ export const getBucketizedAnomalyResultsQuery = ( { term: termField, }, - getResultFilters(isMultiCategory, modelId, entityList), ], }, }, @@ -467,6 +471,8 @@ export const getBucketizedAnomalyResultsQuery = ( }, }; + // If querying RT results: remove any results that include a task_id, as this indicates + // a historical result from a historical task. if (!isHistorical) { requestBody.query.bool = { ...requestBody.query.bool, @@ -480,6 +486,12 @@ export const getBucketizedAnomalyResultsQuery = ( }; } + // Add entity filters if this is a HC detector + if (entityList !== undefined && entityList.length > 0) { + //@ts-ignore + requestBody.query.bool.filter.push(getEntityFilters(modelId, entityList)); + } + return requestBody; }; @@ -1171,7 +1183,6 @@ export const getEntityAnomalySummariesQuery = ( detectorId: string, size: number, entityList: Entity[], - isMultiCategory: boolean, modelId: string | undefined, isHistorical?: boolean, taskId?: string @@ -1212,7 +1223,6 @@ export const getEntityAnomalySummariesQuery = ( { term: termField, }, - getResultFilters(isMultiCategory, modelId, entityList), ], }, }, @@ -1254,6 +1264,12 @@ export const getEntityAnomalySummariesQuery = ( }; } + // Add entity filters if this is a HC detector + if (entityList !== undefined && entityList.length > 0) { + //@ts-ignore + requestBody.query.bool.filter.push(getEntityFilters(modelId, entityList)); + } + return requestBody; }; @@ -1525,14 +1541,11 @@ export const entityListsMatch = ( }; // Helper fn to get the correct filters based on how many categorical fields there are. -// If there are multiple (entity list > 1), filter results by model id. -// If there is only one (entity list = 1), filter by entity value. -const getResultFilters = ( - isMultiCategory: boolean | undefined, +const getEntityFilters = ( modelId: string | undefined, - entityList: Entity[] | undefined + entityList: Entity[] ) => { - return isMultiCategory === true + return entityList.length > 1 ? { term: { model_id: modelId, From 35c6a27200cb7630daf49c4d2373462e99becc4e Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 10 Aug 2021 15:26:44 -0700 Subject: [PATCH 32/35] Render newline on results table correctly --- public/pages/DetectorResults/utils/tableUtils.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/public/pages/DetectorResults/utils/tableUtils.tsx b/public/pages/DetectorResults/utils/tableUtils.tsx index 8bd2864e..5f39baaa 100644 --- a/public/pages/DetectorResults/utils/tableUtils.tsx +++ b/public/pages/DetectorResults/utils/tableUtils.tsx @@ -97,4 +97,6 @@ export const entityValueColumn = { sortable: true, truncateText: false, dataType: 'string', + // To render newline character correctly + style: { whiteSpace: 'pre-wrap' }, } as EuiBasicTableColumn; From c03ca08ddfa832b80536aa2337ed004aea2799e2 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 11 Aug 2021 15:34:39 -0700 Subject: [PATCH 33/35] Remove unused historical detector code; show RT + historical detectors in detector list --- public/models/interfaces.ts | 10 -- public/pages/main/Main.tsx | 9 +- public/pages/utils/helpers.ts | 37 +----- public/redux/reducers/ad.ts | 45 +------ server/routes/ad.ts | 212 +------------------------------ server/routes/utils/adHelpers.ts | 14 -- 6 files changed, 9 insertions(+), 318 deletions(-) diff --git a/public/models/interfaces.ts b/public/models/interfaces.ts index 7ce788ec..cc506e38 100644 --- a/public/models/interfaces.ts +++ b/public/models/interfaces.ts @@ -143,16 +143,6 @@ export type DetectorListItem = { enabledTime?: number; }; -export type HistoricalDetectorListItem = { - id: string; - name: string; - curState: DETECTOR_STATE; - indices: string[]; - totalAnomalies: number; - dataStartTime: number; - dataEndTime: number; -}; - export type EntityData = { name: string; value: string; diff --git a/public/pages/main/Main.tsx b/public/pages/main/Main.tsx index 2743404d..f1d07042 100644 --- a/public/pages/main/Main.tsx +++ b/public/pages/main/Main.tsx @@ -28,7 +28,6 @@ import { Switch, Route, RouteComponentProps } from 'react-router-dom'; import React from 'react'; import { AppState } from '../../redux/reducers'; import { DetectorList } from '../DetectorsList'; -import { SampleData } from '../SampleData'; import { ListRouterParams } from '../DetectorsList/containers/List/List'; import { CreateDetectorSteps } from '../CreateDetectorSteps'; import { EuiSideNav, EuiPage, EuiPageBody, EuiPageSideBar } from '@elastic/eui'; @@ -44,12 +43,8 @@ import { AnomalyDetectionOverview } from '../Overview'; enum Navigation { AnomalyDetection = 'Anomaly detection', - Realtime = 'Real-time', Dashboard = 'Dashboard', Detectors = 'Detectors', - HistoricalDetectors = 'Historical detectors', - SampleDetectors = 'Sample detectors', - CreateDetectorSteps = 'Create detector steps', } interface MainProps extends RouteComponentProps {} @@ -60,7 +55,7 @@ export function Main(props: MainProps) { ); const adState = useSelector((state: AppState) => state.ad); - const totalRealtimeDetectors = adState.totalDetectors; + const totalDetectors = adState.totalDetectors; const errorGettingDetectors = adState.errorMessage; const isLoadingDetectors = adState.requesting; const sideNav = [ @@ -149,7 +144,7 @@ export function Main(props: MainProps) { )} /> - {totalRealtimeDetectors > 0 ? ( + {totalDetectors > 0 ? ( // ) : ( diff --git a/public/pages/utils/helpers.ts b/public/pages/utils/helpers.ts index bbde810b..ecb317be 100644 --- a/public/pages/utils/helpers.ts +++ b/public/pages/utils/helpers.ts @@ -26,10 +26,7 @@ import { CatIndex, IndexAlias } from '../../../server/models/types'; import sortBy from 'lodash/sortBy'; -import { - DetectorListItem, - HistoricalDetectorListItem, -} from '../../models/interfaces'; +import { DetectorListItem } from '../../models/interfaces'; import { SORT_DIRECTION } from '../../../server/utils/constants'; import { ALL_INDICES, ALL_DETECTOR_STATES } from './constants'; import { DETECTOR_STATE } from '../../../server/utils/constants'; @@ -130,35 +127,3 @@ export const formatNumber = (data: any) => { return ''; } }; - -export const filterAndSortHistoricalDetectors = ( - detectors: HistoricalDetectorListItem[], - search: string, - selectedDetectorStates: DETECTOR_STATE[], - sortField: string, - sortDirection: string -) => { - let filteredBySearch = - search == '' - ? detectors - : detectors.filter((detector) => detector.name.includes(search)); - let filteredBySearchAndState = - selectedDetectorStates == ALL_DETECTOR_STATES - ? filteredBySearch - : filteredBySearch.filter((detector) => - selectedDetectorStates.includes(detector.curState) - ); - let sorted = sortBy(filteredBySearchAndState, sortField); - if (sortDirection == SORT_DIRECTION.DESC) { - sorted = sorted.reverse(); - } - return sorted; -}; - -export const getHistoricalDetectorsToDisplay = ( - detectors: HistoricalDetectorListItem[], - page: number, - size: number -) => { - return detectors.slice(size * page, page * size + size); -}; diff --git a/public/redux/reducers/ad.ts b/public/redux/reducers/ad.ts index 65e3dc84..b2bc8d3c 100644 --- a/public/redux/reducers/ad.ts +++ b/public/redux/reducers/ad.ts @@ -31,11 +31,7 @@ import { APIErrorAction, } from '../middleware/types'; import handleActions from '../utils/handleActions'; -import { - Detector, - DetectorListItem, - HistoricalDetectorListItem, -} from '../../models/interfaces'; +import { Detector, DetectorListItem } from '../../models/interfaces'; import { AD_NODE_API } from '../../../utils/constants'; import { GetDetectorsQueryParams } from '../../../server/models/types'; import { cloneDeep, get } from 'lodash'; @@ -55,13 +51,11 @@ const STOP_HISTORICAL_DETECTOR = 'ad/STOP_HISTORICAL_DETECTOR'; const GET_DETECTOR_PROFILE = 'ad/GET_DETECTOR_PROFILE'; const MATCH_DETECTOR = 'ad/MATCH_DETECTOR'; const GET_DETECTOR_COUNT = 'ad/GET_DETECTOR_COUNT'; -const GET_HISTORICAL_DETECTOR_LIST = 'ad/GET_HISTORICAL_DETECTOR_LIST'; export interface Detectors { requesting: boolean; detectors: { [key: string]: Detector }; detectorList: { [key: string]: DetectorListItem }; - historicalDetectorList: { [key: string]: HistoricalDetectorListItem }; totalDetectors: number; errorMessage: string; } @@ -69,7 +63,6 @@ export const initialDetectorsState: Detectors = { requesting: false, detectors: {}, detectorList: {}, - historicalDetectorList: {}, errorMessage: '', totalDetectors: 0, }; @@ -369,34 +362,6 @@ const reducer = handleActions( errorMessage: action.error, }), }, - [GET_HISTORICAL_DETECTOR_LIST]: { - REQUEST: (state: Detectors): Detectors => ({ - ...state, - requesting: true, - errorMessage: '', - }), - SUCCESS: (state: Detectors, action: APIResponseAction): Detectors => ({ - ...state, - requesting: false, - historicalDetectorList: action.result.response.detectorList.reduce( - (acc: any, detector: Detector) => ({ - ...acc, - [detector.id]: { - ...detector, - dataStartTime: get(detector, 'detectionDateRange.startTime', 0), - dataEndTime: get(detector, 'detectionDateRange.endTime', 0), - }, - }), - {} - ), - totalDetectors: action.result.response.totalDetectors, - }), - FAILURE: (state: Detectors, action: APIErrorAction): Detectors => ({ - ...state, - requesting: false, - errorMessage: action.error, - }), - }, }, initialDetectorsState ); @@ -509,12 +474,4 @@ export const getDetectorCount = (): APIAction => ({ client.get(`..${AD_NODE_API.DETECTOR}/_count`, {}), }); -export const getHistoricalDetectorList = ( - queryParams: GetDetectorsQueryParams -): APIAction => ({ - type: GET_HISTORICAL_DETECTOR_LIST, - request: (client: HttpSetup) => - client.get(`..${AD_NODE_API.DETECTOR}/historical`, { query: queryParams }), -}); - export default reducer; diff --git a/server/routes/ad.ts b/server/routes/ad.ts index c66e6601..6f918dd3 100644 --- a/server/routes/ad.ts +++ b/server/routes/ad.ts @@ -56,8 +56,6 @@ import { getTaskInitProgress, isIndexNotFoundError, getErrorMessage, - getRealtimeDetectors, - getHistoricalDetectors, getDetectorTasks, appendTaskInfo, getDetectorResults, @@ -106,7 +104,6 @@ export function registerADRoutes(apiRouter: Router, adService: AdService) { ); apiRouter.get('/detectors/{detectorName}/_match', adService.matchDetector); apiRouter.get('/detectors/_count', adService.getDetectorCount); - apiRouter.get('/detectors/historical', adService.getHistoricalDetectors); } export default class AdService { @@ -541,15 +538,13 @@ export default class AdService { {} ); - const realtimeDetectors = getRealtimeDetectors( - Object.values(allDetectors) - ).reduce( + const allDetectorsMap = Object.values(allDetectors).reduce( (acc: any, detector: any) => ({ ...acc, [detector.id]: detector }), {} - ); + ) as { [key: string]: Detector }; //Given each detector from previous result, get aggregation to power list - const allDetectorIds = Object.keys(realtimeDetectors); + const allDetectorIds = Object.keys(allDetectorsMap); const aggregationResult = await this.client .asScoped(request) .callAsCurrentUser('ad.searchResults', { @@ -570,7 +565,7 @@ export default class AdService { return { ...acc, [agg.key]: { - ...realtimeDetectors[agg.key], + ...allDetectorsMap[agg.key], totalAnomalies: agg.total_anomalies_in_24hr.doc_count, lastActiveAnomaly: agg.latest_anomaly_time.value, }, @@ -585,7 +580,7 @@ export default class AdService { return { ...acc, [unusedDetector]: { - ...realtimeDetectors[unusedDetector], + ...allDetectorsMap[unusedDetector], totalAnomalies: 0, lastActiveAnomaly: 0, }, @@ -941,201 +936,4 @@ export default class AdService { }); return featureResult; }; - - getHistoricalDetectors = async ( - context: RequestHandlerContext, - request: OpenSearchDashboardsRequest, - opensearchDashboardsResponse: OpenSearchDashboardsResponseFactory - ): Promise> => { - try { - const { - from = 0, - size = 20, - search = '', - indices = '', - sortDirection = SORT_DIRECTION.DESC, - sortField = 'name', - } = request.query as GetDetectorsQueryParams; - const mustQueries = []; - if (search.trim()) { - mustQueries.push({ - query_string: { - fields: ['name', 'description'], - default_operator: 'AND', - query: `*${search.trim().split('-').join('* *')}*`, - }, - }); - } - if (indices.trim()) { - mustQueries.push({ - query_string: { - fields: ['indices'], - default_operator: 'AND', - query: `*${indices.trim().split('-').join('* *')}*`, - }, - }); - } - // Allowed sorting columns - const sortQueryMap = { - name: { 'name.keyword': sortDirection }, - indices: { 'indices.keyword': sortDirection }, - lastUpdateTime: { last_update_time: sortDirection }, - } as { [key: string]: object }; - let sort = {}; - const sortQuery = sortQueryMap[sortField]; - if (sortQuery) { - sort = sortQuery; - } - // Preparing search request - const requestBody = { - sort, - size, - from, - query: { - bool: { - must: mustQueries, - }, - }, - }; - const response: any = await this.client - .asScoped(request) - .callAsCurrentUser('ad.searchDetector', { body: requestBody }); - - // Get all detectors from search detector API - const allDetectors = get(response, 'hits.hits', []).reduce( - (acc: any, detector: any) => ({ - ...acc, - [detector._id]: { - id: detector._id, - description: get(detector, '_source.description', ''), - indices: get(detector, '_source.indices', []), - lastUpdateTime: get(detector, '_source.last_update_time', 0), - ...convertDetectorKeysToCamelCase(get(detector, '_source', {})), - }, - }), - {} - ); - - // Filter out to just include historical detectors - const allHistoricalDetectors = getHistoricalDetectors( - Object.values(allDetectors) - ).reduce( - (acc, detector: any) => ({ ...acc, [detector.id]: detector }), - {} - ) as { [key: string]: Detector }; - - // Get related info for each historical detector (detector state, task info, etc.) - const allIds = Object.values(allHistoricalDetectors).map( - (detector) => detector.id - ) as string[]; - - const detectorDetailPromises = allIds.map(async (id: string) => { - try { - const detectorDetailResp = await this.client - .asScoped(request) - .callAsCurrentUser('ad.getDetector', { - detectorId: id, - }); - return detectorDetailResp; - } catch (err) { - console.log('Error getting historical detector ', err); - return Promise.reject( - new Error( - 'Error retrieving all historical detectors: ' + - getErrorMessage(err) - ) - ); - } - }); - - const detectorDetailResponses = await Promise.all( - detectorDetailPromises - ).catch((err) => { - throw err; - }); - - // Get the mapping from detector to task - const detectorTasks = getDetectorTasks(detectorDetailResponses); - - // Get results for each task - const detectorResultPromises = Object.values(detectorTasks).map( - async (task) => { - const taskId = get(task, 'task_id', ''); - try { - const reqBody = { - query: { - bool: { - must: [ - { range: { anomaly_grade: { gt: 0 } } }, - { - term: { - task_id: { - value: taskId, - }, - }, - }, - ], - }, - }, - }; - - const detectorResultResp = await this.client - .asScoped(request) - .callAsCurrentUser('ad.searchResults', { - body: reqBody, - }); - return detectorResultResp; - } catch (err) { - console.log('Error getting historical detector results ', err); - return Promise.reject( - new Error( - 'Error retrieving all historical detector results: ' + - getErrorMessage(err) - ) - ); - } - } - ); - - const detectorResultResponses = await Promise.all( - detectorResultPromises - ).catch((err) => { - throw err; - }); - - // Get the mapping from detector to anomaly results - const detectorResults = getDetectorResults(detectorResultResponses); - - // Append the task-related info for each detector. - // If no task: set state to DISABLED and total anomalies to 0 - const detectorsWithTaskInfo = appendTaskInfo( - allHistoricalDetectors, - detectorTasks, - detectorResults - ); - - return opensearchDashboardsResponse.ok({ - body: { - ok: true, - response: { - totalDetectors: Object.values(detectorsWithTaskInfo).length, - detectorList: Object.values(detectorsWithTaskInfo), - }, - }, - }); - } catch (err) { - console.log('Anomaly detector - Unable to search detectors', err); - if (isIndexNotFoundError(err)) { - return opensearchDashboardsResponse.ok({ - body: { ok: true, response: { totalDetectors: 0, detectorList: [] } }, - }); - } - return opensearchDashboardsResponse.ok({ - body: { - ok: false, - error: getErrorMessage(err), - }, - }); - } - }; } diff --git a/server/routes/utils/adHelpers.ts b/server/routes/utils/adHelpers.ts index fdb51641..5847c828 100644 --- a/server/routes/utils/adHelpers.ts +++ b/server/routes/utils/adHelpers.ts @@ -296,20 +296,6 @@ export const getErrorMessage = (err: any) => { : get(err, 'message'); }; -// Currently: detector w/ detection date range is considered a 'historical' detector -export const getHistoricalDetectors = (detectors: Detector[]) => { - return detectors.filter( - (detector) => detector.detectionDateRange !== undefined - ); -}; - -// Currently: detector w/ no detection date range is considered a 'realtime' detector -export const getRealtimeDetectors = (detectors: Detector[]) => { - return detectors.filter( - (detector) => detector.detectionDateRange === undefined - ); -}; - export const getDetectorTasks = (detectorTaskResponses: any[]) => { const detectorToTaskMap = {} as { [key: string]: any }; detectorTaskResponses.forEach((response) => { From de4d6a9f3b827f9badd5cbfc8ce31b2be463ab94 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Thu, 12 Aug 2021 09:53:43 -0700 Subject: [PATCH 34/35] Tune wording to specify 2 category fields are supported --- .../components/CategoryField/CategoryField.tsx | 2 +- .../__tests__/__snapshots__/CategoryField.test.tsx.snap | 8 ++++---- .../__tests__/__snapshots__/ConfigureModel.test.tsx.snap | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/public/pages/ConfigureModel/components/CategoryField/CategoryField.tsx b/public/pages/ConfigureModel/components/CategoryField/CategoryField.tsx index 56dac70b..d2d65f80 100644 --- a/public/pages/ConfigureModel/components/CategoryField/CategoryField.tsx +++ b/public/pages/ConfigureModel/components/CategoryField/CategoryField.tsx @@ -93,7 +93,7 @@ export function CategoryField(props: CategoryFieldProps) { style={{ lineHeight: 'normal' }} > Split a single time series into multiple time series based on - categorical fields.{' '} + categorical fields. You can select up to 2.{' '} Learn more diff --git a/public/pages/ConfigureModel/components/CategoryField/__tests__/__snapshots__/CategoryField.test.tsx.snap b/public/pages/ConfigureModel/components/CategoryField/__tests__/__snapshots__/CategoryField.test.tsx.snap index ab2389a0..b984eb01 100644 --- a/public/pages/ConfigureModel/components/CategoryField/__tests__/__snapshots__/CategoryField.test.tsx.snap +++ b/public/pages/ConfigureModel/components/CategoryField/__tests__/__snapshots__/CategoryField.test.tsx.snap @@ -41,7 +41,7 @@ exports[` spec hides callout if component is loading 1`] = ` class="euiText euiText--medium content-panel-subTitle" style="line-height: normal;" > - Split a single time series into multiple time series based on categorical fields. + Split a single time series into multiple time series based on categorical fields. You can select up to 2. spec renders the component when disabled 1`] = ` class="euiText euiText--medium content-panel-subTitle" style="line-height: normal;" > - Split a single time series into multiple time series based on categorical fields. + Split a single time series into multiple time series based on categorical fields. You can select up to 2. spec renders the component when enabled 1`] = ` class="euiText euiText--medium content-panel-subTitle" style="line-height: normal;" > - Split a single time series into multiple time series based on categorical fields. + Split a single time series into multiple time series based on categorical fields. You can select up to 2. spec shows callout when there are no available catego class="euiText euiText--medium content-panel-subTitle" style="line-height: normal;" > - Split a single time series into multiple time series based on categorical fields. + Split a single time series into multiple time series based on categorical fields. You can select up to 2. spec creating model configuration renders the compon class="euiText euiText--medium content-panel-subTitle" style="line-height: normal;" > - Split a single time series into multiple time series based on categorical fields. + Split a single time series into multiple time series based on categorical fields. You can select up to 2. spec editing model configuration renders the compone class="euiText euiText--medium content-panel-subTitle" style="line-height: normal;" > - Split a single time series into multiple time series based on categorical fields. + Split a single time series into multiple time series based on categorical fields. You can select up to 2. Date: Thu, 19 Aug 2021 20:43:16 -0700 Subject: [PATCH 35/35] Change default refresh rate to 30s for historical HC --- public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx | 5 +++++ .../containers/HistoricalDetectorResults.tsx | 5 ++++- public/pages/HistoricalDetectorResults/utils/constants.tsx | 3 +++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx b/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx index 0525804c..a3cb77de 100644 --- a/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx +++ b/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx @@ -249,6 +249,11 @@ export const getAnomaliesHeatmapData = ( } } + // entityStrings are the string representations used as y-axis labels for the heatmap, + // and only contain the entity values. + // entityLists are the entity objects (containing name and value) which are + // populated in each of the heatmap cells, and used to fetch model-specific results when + // a user clicks on a particular cell const entityStrings = [] as string[]; const entityLists = [] as any[]; const maxAnomalyGrades = [] as any[]; diff --git a/public/pages/HistoricalDetectorResults/containers/HistoricalDetectorResults.tsx b/public/pages/HistoricalDetectorResults/containers/HistoricalDetectorResults.tsx index 0377622e..d5844c47 100644 --- a/public/pages/HistoricalDetectorResults/containers/HistoricalDetectorResults.tsx +++ b/public/pages/HistoricalDetectorResults/containers/HistoricalDetectorResults.tsx @@ -52,6 +52,7 @@ import { getDetectorStateDetails } from '../../DetectorDetail/utils/helpers'; import { HistoricalRangeModal } from '../components/HistoricalRangeModal'; import { HISTORICAL_DETECTOR_RESULT_REFRESH_RATE, + HISTORICAL_HC_DETECTOR_RESULT_REFRESH_RATE, HISTORICAL_DETECTOR_STOP_THRESHOLD, } from '../utils/constants'; import { CoreStart } from '../../../../../../src/core/public'; @@ -115,7 +116,9 @@ export function HistoricalDetectorResults( ) { const intervalId = setInterval( fetchDetector, - HISTORICAL_DETECTOR_RESULT_REFRESH_RATE + isHCDetector + ? HISTORICAL_HC_DETECTOR_RESULT_REFRESH_RATE + : HISTORICAL_DETECTOR_RESULT_REFRESH_RATE ); return () => { clearInterval(intervalId); diff --git a/public/pages/HistoricalDetectorResults/utils/constants.tsx b/public/pages/HistoricalDetectorResults/utils/constants.tsx index c0a80210..c83da670 100644 --- a/public/pages/HistoricalDetectorResults/utils/constants.tsx +++ b/public/pages/HistoricalDetectorResults/utils/constants.tsx @@ -26,7 +26,10 @@ // Current backend implementation: limited to running model on 1000 intervals every 5s. // Frontend should refresh at some rate > than this, to auto-refresh and show partial results. +// For historical non-high-cardinality detectors: refresh every 10s +// For historical high-cardinality detectors: refresh every 30s export const HISTORICAL_DETECTOR_RESULT_REFRESH_RATE = 10000; +export const HISTORICAL_HC_DETECTOR_RESULT_REFRESH_RATE = 30000; // Current backend implementation will handle stopping a historical detector task asynchronously. It is assumed // that if the task is not in a stopped state after 5s, then there was a problem stopping.