diff --git a/public/models/interfaces.ts b/public/models/interfaces.ts index 7ce788ec..cc506e38 100644 --- a/public/models/interfaces.ts +++ b/public/models/interfaces.ts @@ -143,16 +143,6 @@ export type DetectorListItem = { enabledTime?: number; }; -export type HistoricalDetectorListItem = { - id: string; - name: string; - curState: DETECTOR_STATE; - indices: string[]; - totalAnomalies: number; - dataStartTime: number; - dataEndTime: number; -}; - export type EntityData = { name: string; value: string; diff --git a/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx b/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx index 3b6619b1..c9bbac07 100644 --- a/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx +++ b/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx @@ -74,7 +74,6 @@ interface FeatureChartProps { showFeatureMissingDataPointAnnotation?: boolean; detectorEnabledTime?: number; rawFeatureData: FeatureAggregationData[]; - titlePrefix?: string; } export const FeatureChart = (props: FeatureChartProps) => { @@ -165,10 +164,9 @@ export const FeatureChart = (props: FeatureChartProps) => { return ( { { onDisplayOptionChanged={props.onDisplayOptionChanged} heatmapDisplayOption={props.heatmapDisplayOption} isNotSample={props.isNotSample} + // Category fields in HC results are always sorted alphabetically. To make all chart + // wording consistent with the returned results, we sort the given category + // fields in alphabetical order as well. + categoryField={orderBy(props.detectorCategoryField, [ + (categoryField) => categoryField.toLowerCase(), + ])} />, props.isNotSample !== true ? [ @@ -284,15 +295,13 @@ export const AnomaliesChart = React.memo((props: AnomaliesChartProps) => {

- {`${get( - props, - 'detectorCategoryField.0' - )} `} - { - props.selectedHeatmapCell - ?.entityValue - } + {props.selectedHeatmapCell + ? getHCTitle( + props.selectedHeatmapCell + .entityList + ) + : '-'}

diff --git a/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx b/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx index 4310b37a..5afac85f 100644 --- a/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx +++ b/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx @@ -24,8 +24,7 @@ * permissions and limitations under the License. */ -import React, { useState } from 'react'; - +import React, { useState, useEffect } from 'react'; import moment from 'moment'; import Plotly, { PlotData } from 'plotly.js-dist'; import plotComponentFactory from 'react-plotly.js/factory'; @@ -54,13 +53,25 @@ import { filterHeatmapPlotDataByY, getEntitytAnomaliesHeatmapData, } from '../utils/anomalyChartUtils'; -import { MIN_IN_MILLI_SECS } from '../../../../server/utils/constants'; -import { EntityAnomalySummaries } from '../../../../server/models/interfaces'; +import { + MIN_IN_MILLI_SECS, + ENTITY_LIST_DELIMITER, +} from '../../../../server/utils/constants'; +import { + EntityAnomalySummaries, + Entity, +} from '../../../../server/models/interfaces'; +import { HEATMAP_CHART_Y_AXIS_WIDTH } from '../utils/constants'; +import { + convertToEntityList, + convertToCategoryFieldString, +} from '../../utils/anomalyResultUtils'; interface AnomalyHeatmapChartProps { - title: string; detectorId: string; detectorName: string; + detectorTaskProgress?: number; + isHistorical?: boolean; anomalies?: any[]; dateRange: DateRange; isLoading: boolean; @@ -72,11 +83,13 @@ interface AnomalyHeatmapChartProps { heatmapDisplayOption?: HeatmapDisplayOption; entityAnomalySummaries?: EntityAnomalySummaries[]; isNotSample?: boolean; + categoryField?: string[]; } export interface HeatmapCell { dateRange: DateRange; - entityValue: string; + entityList: Entity[]; + modelId?: string; } export interface HeatmapDisplayOption { @@ -123,14 +136,15 @@ export const AnomalyHeatmapChart = React.memo( //@ts-ignore individualEntities = inputHeatmapData[0].y.filter( //@ts-ignore - (entityValue) => entityValue && entityValue.trim().length > 0 + (entityListAsString) => + entityListAsString && entityListAsString.trim().length > 0 ); } const individualEntityOptions = [] as any[]; //@ts-ignore - individualEntities.forEach((entityValue) => { + individualEntities.forEach((entityListAsString: string) => { individualEntityOptions.push({ - label: entityValue, + label: entityListAsString.replace(ENTITY_LIST_DELIMITER, ', '), }); }); @@ -165,7 +179,7 @@ export const AnomalyHeatmapChart = React.memo( getEntitytAnomaliesHeatmapData( props.dateRange, props.entityAnomalySummaries, - props.heatmapDisplayOption.entityOption.value + props.heatmapDisplayOption?.entityOption.value ) : // use anomalies data in case of sample result getAnomaliesHeatmapData( @@ -184,7 +198,7 @@ export const AnomalyHeatmapChart = React.memo( AnomalyHeatmapSortType >( props.isNotSample - ? props.heatmapDisplayOption.sortType + ? props.heatmapDisplayOption?.sortType : SORT_BY_FIELD_OPTIONS[0].value ); @@ -215,9 +229,25 @@ export const AnomalyHeatmapChart = React.memo( return false; }; + // Custom hook to refresh all of the heatmap data when running a historical task + useEffect(() => { + if (props.isHistorical) { + const updateHeatmapPlotData = getAnomaliesHeatmapData( + props.anomalies, + props.dateRange, + sortByFieldValue, + get(COMBINED_OPTIONS.options[0], 'value') + ); + setOriginalHeatmapData(updateHeatmapPlotData); + setHeatmapData(updateHeatmapPlotData); + setNumEntities(updateHeatmapPlotData[0].y.length); + setEntityViewOptions(getViewEntityOptions(updateHeatmapPlotData)); + } + }, [props.detectorTaskProgress]); + const handleHeatmapClick = (event: Plotly.PlotMouseEvent) => { const selectedCellIndices = get(event, 'points[0].pointIndex', []); - const selectedEntity = get(event, 'points[0].y', ''); + const selectedEntityString = get(event, 'points[0].y', ''); if (!isEmpty(selectedCellIndices)) { let anomalyCount = get(event, 'points[0].text', 0); if ( @@ -262,7 +292,11 @@ export const AnomalyHeatmapChart = React.memo( startDate: selectedStartDate, endDate: selectedEndDate, }, - entityValue: selectedEntity, + entityList: convertToEntityList( + selectedEntityString, + get(props, 'categoryField', []), + ENTITY_LIST_DELIMITER + ), } as HeatmapCell); } } @@ -345,7 +379,7 @@ export const AnomalyHeatmapChart = React.memo( setNumEntities(nonCombinedOptions.length); const selectedYs = nonCombinedOptions.map((option) => - get(option, 'label', '') + get(option, 'label', '').replace(', ', ENTITY_LIST_DELIMITER) ); let selectedHeatmapData = filterHeatmapPlotDataByY( @@ -407,40 +441,53 @@ export const AnomalyHeatmapChart = React.memo( - - - -

{props.title}

- - - - - - - - handleViewEntityOptionsChange(selectedOptions) - } - /> - - - handleSortByFieldChange(value)} - hasDividers - /> - - - + + + +

+ View by:  + + {convertToCategoryFieldString( + get(props, 'categoryField', []) as string[], + ', ' + )} + +

+ + + + + + + + handleViewEntityOptionsChange(selectedOptions) + } + /> + + + handleSortByFieldChange(value)} + hasDividers + /> + + + + @@ -542,6 +589,15 @@ export const AnomalyHeatmapChart = React.memo( showline: true, showgrid: false, fixedrange: true, + automargin: true, + tickmode: 'array', + tickvals: heatmapData[0].y, + ticktext: heatmapData[0].y.map((label: string) => + label.length <= HEATMAP_CHART_Y_AXIS_WIDTH + ? label + : label.substring(0, HEATMAP_CHART_Y_AXIS_WIDTH - 3) + + '...' + ), }, margin: { l: 100, diff --git a/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx b/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx index 4686ecad..15de9939 100644 --- a/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx +++ b/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx @@ -40,14 +40,17 @@ import { Anomalies, DateRange, FEATURE_TYPE, - EntityData, } from '../../../models/interfaces'; import { NoFeaturePrompt } from '../components/FeatureChart/NoFeaturePrompt'; import { focusOnFeatureAccordion } from '../../ConfigureModel/utils/helpers'; import moment from 'moment'; import { HeatmapCell } from './AnomalyHeatmapChart'; -import { filterWithHeatmapFilter } from '../../utils/anomalyResultUtils'; +import { + filterWithHeatmapFilter, + entityListsMatch, +} from '../../utils/anomalyResultUtils'; import { getDateRangeWithSelectedHeatmapCell } from '../utils/anomalyChartUtils'; +import { Entity } from '../../../../server/models/interfaces'; interface FeatureBreakDownProps { title?: string; @@ -80,10 +83,19 @@ export const FeatureBreakDown = React.memo((props: FeatureBreakDownProps) => { const filteredFeatureData = []; for (let i = 0; i < anomaliesFound.length; i++) { const currentAnomalyData = anomaliesResult.anomalies[i]; + const dataEntityList = get( + currentAnomalyData, + 'entity', + [] + ) as Entity[]; + const cellEntityList = get( + props, + 'selectedHeatmapCell.entityList', + [] + ) as Entity[]; if ( - !isEmpty(get(currentAnomalyData, 'entity', [] as EntityData[])) && - get(currentAnomalyData, 'entity', [] as EntityData[])[0].value === - props.selectedHeatmapCell.entityValue && + !isEmpty(dataEntityList) && + entityListsMatch(dataEntityList, cellEntityList) && get(currentAnomalyData, 'plotTime', 0) >= props.selectedHeatmapCell.dateRange.startDate && get(currentAnomalyData, 'plotTime', 0) <= @@ -200,12 +212,6 @@ export const FeatureBreakDown = React.memo((props: FeatureBreakDownProps) => { props.showFeatureMissingDataPointAnnotation } detectorEnabledTime={props.detector.enabledTime} - titlePrefix={ - props.selectedHeatmapCell && - props.title !== 'Sample feature breakdown' - ? props.selectedHeatmapCell.entityValue - : undefined - } /> {index + 1 === get(props, 'detector.featureAttributes', []).length ? null : ( diff --git a/public/pages/AnomalyCharts/containers/__tests__/AnomaliesChart.test.tsx b/public/pages/AnomalyCharts/containers/__tests__/AnomaliesChart.test.tsx index f7875697..7645f4e2 100644 --- a/public/pages/AnomalyCharts/containers/__tests__/AnomaliesChart.test.tsx +++ b/public/pages/AnomalyCharts/containers/__tests__/AnomaliesChart.test.tsx @@ -26,8 +26,8 @@ import React from 'react'; import { render } from '@testing-library/react'; -import { AnomaliesChart } from '../AnomaliesChart'; -import { initialState, mockedStore } from '../../../../redux/utils/testUtils'; +import { AnomaliesChart, AnomaliesChartProps } from '../AnomaliesChart'; +import { mockedStore } from '../../../../redux/utils/testUtils'; import { Provider } from 'react-redux'; import { INITIAL_ANOMALY_SUMMARY } from '../../utils/constants'; import { getRandomDetector } from '../../../../redux/reducers/__tests__/utils'; @@ -38,30 +38,149 @@ import { FAKE_DATE_RANGE, } from '../../../../pages/utils/__tests__/constants'; -const renderDataFilter = () => ({ +const DEFAULT_PROPS = { + onDateRangeChange: jest.fn(), + onZoomRangeChange: jest.fn(), + title: 'Test title', + bucketizedAnomalies: true, + anomalySummary: INITIAL_ANOMALY_SUMMARY, + dateRange: FAKE_DATE_RANGE, + isLoading: false, + showAlerts: false, + isNotSample: true, + detector: getRandomDetector(true), + children: [], + isHCDetector: false, + isHistorical: false, + detectorCategoryField: [], + onHeatmapCellSelected: jest.fn(), + onDisplayOptionChanged: jest.fn(), + selectedHeatmapCell: undefined, + newDetector: undefined, + zoomRange: undefined, + anomaliesResult: FAKE_ANOMALIES_RESULT, + entityAnomalySummaries: [], +} as AnomaliesChartProps; + +const renderDataFilter = (chartProps: AnomaliesChartProps) => ({ ...render( - + ), }); describe(' spec', () => { - test('renders the component', () => { + test('renders the component for sample / preview', () => { + console.error = jest.fn(); + const { getByText, getAllByText } = renderDataFilter({ + ...DEFAULT_PROPS, + isHistorical: false, + isHCDetector: false, + isNotSample: false, + }); + expect(getByText('Test title')).not.toBeNull(); + expect(getAllByText('Sample anomaly occurrences').length >= 1); + expect(getAllByText('Sample anomaly grade').length >= 1); + expect(getAllByText('Sample confidence').length >= 1); + }); + test('renders the component for RT, non-HC detector', () => { + console.error = jest.fn(); + const { getByText, getAllByText } = renderDataFilter({ + ...DEFAULT_PROPS, + isHistorical: false, + isHCDetector: false, + }); + expect(getByText('Test title')).not.toBeNull(); + expect(getAllByText('Anomaly occurrences').length >= 1); + expect(getAllByText('Anomaly grade').length >= 1); + expect(getAllByText('Confidence').length >= 1); + expect(getAllByText('Last anomaly occurrence').length >= 1); + }); + test('renders the component for RT, HC detector', () => { + console.error = jest.fn(); + const { getByText } = renderDataFilter({ + ...DEFAULT_PROPS, + isHistorical: false, + isHCDetector: true, + detectorCategoryField: ['category-1'], + }); + getByText('Test title'); + getByText('Top 10'); + getByText('category-1'); + }); + test('renders the component for RT, multi-category-HC detector', () => { + console.error = jest.fn(); + const { getByText } = renderDataFilter({ + ...DEFAULT_PROPS, + isHistorical: false, + isHCDetector: true, + detectorCategoryField: ['category-1, category-2'], + }); + getByText('Test title'); + getByText('Top 10'); + getByText('category-1, category-2'); + }); + test('renders the component for historical, non-HC detector', () => { + console.error = jest.fn(); + const { getAllByText, queryByText } = renderDataFilter({ + ...DEFAULT_PROPS, + isHistorical: true, + isHCDetector: false, + }); + expect(queryByText('Test title')).not.toBeNull(); + expect(getAllByText('Anomaly occurrences').length >= 1); + expect(getAllByText('Average anomaly grade').length >= 1); + expect(queryByText('Confidence')).toBeNull(); + }); + test('renders the component for historical, HC detector', () => { + console.error = jest.fn(); + const { getByText } = renderDataFilter({ + ...DEFAULT_PROPS, + isHistorical: true, + isHCDetector: true, + detectorCategoryField: ['category-1'], + }); + getByText('Test title'); + getByText('Top 10'); + getByText('category-1'); + }); + test('renders the component for historical, multi-category-HC detector', () => { + console.error = jest.fn(); + const { getByText } = renderDataFilter({ + ...DEFAULT_PROPS, + isHistorical: true, + isHCDetector: true, + detectorCategoryField: ['category-1', 'category-2'], + }); + getByText('Test title'); + getByText('Top 10'); + getByText('category-1, category-2'); + }); + test('renders multiple category fields if stored in alphabetical order', () => { + console.error = jest.fn(); + const { getByText } = renderDataFilter({ + ...DEFAULT_PROPS, + isHistorical: false, + isHCDetector: true, + detectorCategoryField: ['a', 'b'], + }); + getByText('Test title'); + getByText('Top 10'); + getByText('a, b'); + }); + test('renders multiple category fields if stored in non-alphabetical order', () => { console.error = jest.fn(); - const { getByText } = renderDataFilter(); - expect(getByText('Sample anomaly grade')).not.toBeNull(); + const { getByText } = renderDataFilter({ + ...DEFAULT_PROPS, + isHistorical: false, + isHCDetector: true, + detectorCategoryField: ['b', 'a'], + }); + getByText('Test title'); + getByText('Top 10'); + getByText('a, b'); }); }); diff --git a/public/pages/AnomalyCharts/containers/__tests__/AnomalyHeatmapChart.test.tsx b/public/pages/AnomalyCharts/containers/__tests__/AnomalyHeatmapChart.test.tsx index 1523a6e6..7267ca33 100644 --- a/public/pages/AnomalyCharts/containers/__tests__/AnomalyHeatmapChart.test.tsx +++ b/public/pages/AnomalyCharts/containers/__tests__/AnomalyHeatmapChart.test.tsx @@ -24,7 +24,7 @@ * permissions and limitations under the License. */ -import { render } from '@testing-library/react'; +import { render, getByText } from '@testing-library/react'; import React from 'react'; import { AnomalyHeatmapChart, @@ -79,4 +79,48 @@ describe(' spec', () => { ); expect(container).toMatchSnapshot(); }); + test('AnomalyHeatmapChart with one category field', () => { + const { container, getByText } = render( + + ); + expect(container).toMatchSnapshot(); + getByText('View by:'); + getByText('test-field'); + }); + test('AnomalyHeatmapChart with multiple category fields', () => { + const { container, getByText } = render( + + ); + expect(container).toMatchSnapshot(); + getByText('View by:'); + getByText('test-field-1, test-field-2'); + }); }); diff --git a/public/pages/AnomalyCharts/containers/__tests__/__snapshots__/AnomalyHeatmapChart.test.tsx.snap b/public/pages/AnomalyCharts/containers/__tests__/__snapshots__/AnomalyHeatmapChart.test.tsx.snap index 3d1758dd..0c9d8163 100644 --- a/public/pages/AnomalyCharts/containers/__tests__/__snapshots__/AnomalyHeatmapChart.test.tsx.snap +++ b/public/pages/AnomalyCharts/containers/__tests__/__snapshots__/AnomalyHeatmapChart.test.tsx.snap @@ -38,26 +38,6 @@ exports[` spec AnomalyHeatmapChart with Sample anomaly da class="euiFlexGroup euiFlexGroup--gutterLarge euiFlexGroup--directionRow euiFlexGroup--responsive" style="padding: 0px;" > -
-
-
-
-

- test-tile -

-
-
-
-
spec AnomalyHeatmapChart with Sample anomaly da style="padding: 0px;" >
+

+ View by:  + +

+
+
+
+
-
-
-
+
- - Select an option: By severity, is selected - -
- + Select an option: By severity, is selected + +
+ + + +
@@ -442,26 +439,6 @@ exports[` spec AnomalyHeatmapChart with anomaly summaries class="euiFlexGroup euiFlexGroup--gutterLarge euiFlexGroup--directionRow euiFlexGroup--responsive" style="padding: 0px;" > -
-
-
-
-

- test-tile -

-
-
-
-
spec AnomalyHeatmapChart with anomaly summaries style="padding: 0px;" >
+

+ View by:  + +

+
+
+
+
-
-
-
- -
-
- - Select an option: By severity, is selected - - +
+
- + + + +
+
+
+
+
+
+
+
+ +
+
+ + Select an option: By severity, is selected + +
+ + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Anomaly grade + + + + + + +
+
+
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+
+
+
+
+ + 0.0 + + (None) +
+
+
+
+ (Critical) + + 1.0 + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+`; + +exports[` spec AnomalyHeatmapChart with multiple category fields 1`] = ` +
+
+
+
+
+ + + No anomalies found in the specified date range. + +
+
+
+
+
+
+
+
+
+
+

+ View by:  + + test-field-1, test-field-2 + +

+
+
+
+
+
+ +
+
+
+ +
+
+ + Select an option: By severity, is selected + + +
+ + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Anomaly grade + + + + + + +
+
+
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+
+
+
+
+ + 0.0 + + (None) +
+
+
+
+ (Critical) + + 1.0 + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+`; + +exports[` spec AnomalyHeatmapChart with one category field 1`] = ` +
+
+
+
+
+ + + No anomalies found in the specified date range. + +
+
+
+
+
+
+
+
+
+
+

+ View by:  + + test-field + +

+
+
+
+
+
+ +
+
+
+ +
+
+ + Select an option: By severity, is selected + + +
+ + + +
diff --git a/public/pages/AnomalyCharts/utils/anomalyChartUtils.ts b/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx similarity index 86% rename from public/pages/AnomalyCharts/utils/anomalyChartUtils.ts rename to public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx index 6dbad7a1..a3cb77de 100644 --- a/public/pages/AnomalyCharts/utils/anomalyChartUtils.ts +++ b/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx @@ -25,6 +25,8 @@ */ import { cloneDeep, defaultTo, get, isEmpty, orderBy } from 'lodash'; +import React from 'react'; +import { EuiTitle, EuiSpacer } from '@elastic/eui'; import { DateRange, Detector, @@ -37,14 +39,18 @@ import { RectAnnotationDatum } from '@elastic/charts'; import { DEFAULT_ANOMALY_SUMMARY } from './constants'; import { Datum, PlotData } from 'plotly.js'; import moment from 'moment'; -import { calculateTimeWindowsWithMaxDataPoints } from '../../utils/anomalyResultUtils'; +import { + calculateTimeWindowsWithMaxDataPoints, + convertToEntityString, + transformEntityListsForHeatmap, +} from '../../utils/anomalyResultUtils'; import { HeatmapCell } from '../containers/AnomalyHeatmapChart'; import { EntityAnomalySummaries, EntityAnomalySummary, } from '../../../../server/models/interfaces'; import { toFixedNumberForAnomaly } from '../../../../server/utils/helpers'; -import { ENTITY_VALUE_PATH_FIELD } from '../../../../server/utils/constants'; +import { Entity } from '../../../../server/models/interfaces'; export const convertAlerts = (response: any): MonitorAlert[] => { const alerts = get(response, 'response.alerts', []); @@ -243,7 +249,13 @@ export const getAnomaliesHeatmapData = ( } } - const entityValues = [] as string[]; + // entityStrings are the string representations used as y-axis labels for the heatmap, + // and only contain the entity values. + // entityLists are the entity objects (containing name and value) which are + // populated in each of the heatmap cells, and used to fetch model-specific results when + // a user clicks on a particular cell + const entityStrings = [] as string[]; + const entityLists = [] as any[]; const maxAnomalyGrades = [] as any[]; const numAnomalyGrades = [] as any[]; @@ -252,11 +264,12 @@ export const getAnomaliesHeatmapData = ( dateRange ); - entityAnomaliesMap.forEach((entityAnomalies, entity) => { + entityAnomaliesMap.forEach((entityAnomalies, entityListAsString) => { const maxAnomalyGradesForEntity = [] as number[]; const numAnomalyGradesForEntity = [] as number[]; - entityValues.push(entity); + entityStrings.push(entityListAsString); + entityLists.push(get(entityAnomalies, '0.entity', {})); timeWindows.forEach((timeWindow) => { const anomaliesInWindow = entityAnomalies.filter( (anomaly) => @@ -286,11 +299,13 @@ export const getAnomaliesHeatmapData = ( moment(timestamp).format(HEATMAP_X_AXIS_DATE_FORMAT) ); const cellTimeInterval = timeWindows[0].endDate - timeWindows[0].startDate; + const entityListsTransformed = transformEntityListsForHeatmap(entityLists); const plotData = buildHeatmapPlotData( plotTimesInString, - entityValues, + entityStrings, maxAnomalyGrades, numAnomalyGrades, + entityListsTransformed, cellTimeInterval ); const resultPlotData = sortHeatmapPlotData(plotData, sortType, displayTopNum); @@ -301,7 +316,8 @@ const buildHeatmapPlotData = ( x: any[], y: any[], z: any[], - text: any[], + anomalyOccurrences: any[], + entityLists: any[], cellTimeInterval: number ): PlotData => { //@ts-ignore @@ -317,8 +333,10 @@ const buildHeatmapPlotData = ( xgap: 2, ygap: 2, opacity: 1, - text: text, + text: anomalyOccurrences, + customdata: entityLists, hovertemplate: + 'Entities: %{customdata}
' + 'Time: %{x}
' + 'Max anomaly grade: %{z}
' + 'Anomaly occurrences: %{text}' + @@ -332,7 +350,8 @@ export const getEntitytAnomaliesHeatmapData = ( entitiesAnomalySummaryResult: EntityAnomalySummaries[], displayTopNum: number ) => { - const entityValues = [] as string[]; + const entityStrings = [] as string[]; + const entityLists = [] as any[]; const maxAnomalyGrades = [] as any[]; const numAnomalyGrades = [] as any[]; @@ -350,9 +369,11 @@ export const getEntitytAnomaliesHeatmapData = ( // only 1 whitesapce for all entities, to avoid heatmap with single row const blankStrValue = buildBlankStringWithLength(i); entitiesAnomalySummaries.push({ - entity: { - value: blankStrValue, - }, + entityList: [ + { + value: blankStrValue, + }, + ], } as EntityAnomalySummaries); } } else { @@ -363,17 +384,23 @@ export const getEntitytAnomaliesHeatmapData = ( const maxAnomalyGradesForEntity = [] as number[]; const numAnomalyGradesForEntity = [] as number[]; - const entityValue = get( - entityAnomalySummaries, - ENTITY_VALUE_PATH_FIELD, - '' + const entityString = convertToEntityString( + get(entityAnomalySummaries, 'entityList', []) ) as string; + const anomaliesSummary = get( entityAnomalySummaries, 'anomalySummaries', [] ) as EntityAnomalySummary[]; - entityValues.push(entityValue); + entityStrings.push(entityString); + + const entityList = get( + entityAnomalySummaries, + 'entityList', + [] + ) as Entity[]; + entityLists.push(entityList); timeWindows.forEach((timeWindow) => { const anomalySummaryInTimeRange = anomaliesSummary.filter( @@ -413,11 +440,14 @@ export const getEntitytAnomaliesHeatmapData = ( const timeStamps = plotTimes.map((timestamp) => moment(timestamp).format(HEATMAP_X_AXIS_DATE_FORMAT) ); + const entityListsTransformed = transformEntityListsForHeatmap(entityLists); + const plotData = buildHeatmapPlotData( timeStamps, - entityValues.reverse(), + entityStrings.reverse(), maxAnomalyGrades.reverse(), numAnomalyGrades.reverse(), + entityListsTransformed.reverse(), timeWindows[0].endDate - timeWindows[0].startDate ); return [plotData]; @@ -431,18 +461,18 @@ const getEntityAnomaliesMap = ( return entityAnomaliesMap; } anomalies.forEach((anomaly) => { - const entity = get(anomaly, 'entity', [] as EntityData[]); - if (isEmpty(entity)) { + const entityList = get(anomaly, 'entity', [] as EntityData[]); + if (isEmpty(entityList)) { return; } - const entityValue = entity[0].value; + const entityListAsString = convertToEntityString(entityList); let singleEntityAnomalies = []; - if (entityAnomaliesMap.has(entityValue)) { + if (entityAnomaliesMap.has(entityListAsString)) { //@ts-ignore - singleEntityAnomalies = entityAnomaliesMap.get(entityValue); + singleEntityAnomalies = entityAnomaliesMap.get(entityListAsString); } singleEntityAnomalies.push(anomaly); - entityAnomaliesMap.set(entityValue, singleEntityAnomalies); + entityAnomaliesMap.set(entityListAsString, singleEntityAnomalies); }); return entityAnomaliesMap; }; @@ -472,15 +502,18 @@ export const filterHeatmapPlotDataByY = ( const originalYs = cloneDeep(heatmapData.y); const originalZs = cloneDeep(heatmapData.z); const originalTexts = cloneDeep(heatmapData.text); + const originalEntityLists = cloneDeep(heatmapData.customdata); const resultYs = []; const resultZs = []; const resultTexts = []; + const resultEntityLists = []; for (let i = 0; i < originalYs.length; i++) { //@ts-ignore if (selectedYs.includes(originalYs[i])) { resultYs.push(originalYs[i]); resultZs.push(originalZs[i]); resultTexts.push(originalTexts[i]); + resultEntityLists.push(originalEntityLists[i]); } } const updateHeatmapPlotData = { @@ -488,6 +521,7 @@ export const filterHeatmapPlotDataByY = ( y: resultYs, z: resultZs, text: resultTexts, + customdata: resultEntityLists, } as PlotData; return sortHeatmapPlotData( updateHeatmapPlotData, @@ -504,6 +538,8 @@ export const sortHeatmapPlotData = ( const originalYs = cloneDeep(heatmapData.y); const originalZs = cloneDeep(heatmapData.z); const originalTexts = cloneDeep(heatmapData.text); + const originalEntityLists = cloneDeep(heatmapData.customdata); + const originalValuesToSort = sortType === AnomalyHeatmapSortType.SEVERITY ? cloneDeep(originalZs) @@ -531,17 +567,20 @@ export const sortHeatmapPlotData = ( const resultYs = [] as any[]; const resultZs = [] as any[]; const resultTexts = [] as any[]; + const resultEntityLists = [] as any[]; for (let i = sortedYIndices.length - 1; i >= 0; i--) { const index = get(sortedYIndices[i], 'index', 0); resultYs.push(originalYs[index]); resultZs.push(originalZs[index]); resultTexts.push(originalTexts[index]); + resultEntityLists.push(originalEntityLists[index]); } return { ...cloneDeep(heatmapData), y: resultYs, z: resultZs, text: resultTexts, + customdata: resultEntityLists, } as PlotData; }; @@ -643,3 +682,22 @@ export const getDateRangeWithSelectedHeatmapCell = ( } return originalDateRange; }; + +export const getHCTitle = (entityList: Entity[]) => { + return ( +
+ +

+ {entityList.map((entity: Entity) => { + return ( +
+ {entity.name}: {entity.value}{' '} +
+ ); + })} +

+ + +
+ ); +}; diff --git a/public/pages/AnomalyCharts/utils/constants.ts b/public/pages/AnomalyCharts/utils/constants.ts index cd58f5dd..50350b5c 100644 --- a/public/pages/AnomalyCharts/utils/constants.ts +++ b/public/pages/AnomalyCharts/utils/constants.ts @@ -106,3 +106,5 @@ export const DEFAULT_ANOMALY_SUMMARY = { maxConfidence: 0, lastAnomalyOccurrence: '-', }; + +export const HEATMAP_CHART_Y_AXIS_WIDTH = 30; diff --git a/public/pages/ConfigureModel/components/CategoryField/CategoryField.tsx b/public/pages/ConfigureModel/components/CategoryField/CategoryField.tsx index 4517e77f..d2d65f80 100644 --- a/public/pages/ConfigureModel/components/CategoryField/CategoryField.tsx +++ b/public/pages/ConfigureModel/components/CategoryField/CategoryField.tsx @@ -84,7 +84,7 @@ export function CategoryField(props: CategoryFieldProps) { -

Categorical field

+

Categorical fields

} subTitle={ @@ -92,8 +92,8 @@ export function CategoryField(props: CategoryFieldProps) { className="content-panel-subTitle" style={{ lineHeight: 'normal' }} > - Split a single time series into multiple time series based on a - categorical field.{' '} + Split a single time series into multiple time series based on + categorical fields. You can select up to 2.{' '} Learn more @@ -119,7 +119,7 @@ export function CategoryField(props: CategoryFieldProps) { { @@ -144,24 +144,26 @@ export function CategoryField(props: CategoryFieldProps) { label="Field" isInvalid={isInvalid(field.name, form)} error={getError(field.name, form)} - helpText={`You can only apply the category field to the 'ip' and 'keyword' OpenSearch data types.`} + helpText={`You can only apply the categorical fields to the 'ip' and 'keyword' OpenSearch data types.`} > { form.setFieldTouched('categoryField', true); }} onChange={(options) => { - const selection = get(options, '0.label'); - if (selection) { - form.setFieldValue('categoryField', [selection]); - form.setFieldValue( - 'shingleSize', - MULTI_ENTITY_SHINGLE_SIZE - ); + const selection = options.map((option) => option.label); + if (!isEmpty(selection)) { + if (selection.length <= 2) { + form.setFieldValue('categoryField', selection); + form.setFieldValue( + 'shingleSize', + MULTI_ENTITY_SHINGLE_SIZE + ); + } } else { form.setFieldValue('categoryField', []); @@ -172,9 +174,15 @@ export function CategoryField(props: CategoryFieldProps) { } }} selectedOptions={ - (field.value[0] && [{ label: field.value[0] }]) || [] + field.value + ? field.value.map((value: any) => { + return { + label: value, + }; + }) + : [] } - singleSelection={{ asPlainText: true }} + singleSelection={false} isClearable={true} /> diff --git a/public/pages/ConfigureModel/components/CategoryField/__tests__/CategoryField.test.tsx b/public/pages/ConfigureModel/components/CategoryField/__tests__/CategoryField.test.tsx index 1239088b..8082928a 100644 --- a/public/pages/ConfigureModel/components/CategoryField/__tests__/CategoryField.test.tsx +++ b/public/pages/ConfigureModel/components/CategoryField/__tests__/CategoryField.test.tsx @@ -25,7 +25,7 @@ */ import React, { Fragment } from 'react'; -import { render } from '@testing-library/react'; +import { render, fireEvent, getByRole } from '@testing-library/react'; import { Form, Formik } from 'formik'; import { CategoryField } from '../CategoryField'; @@ -42,6 +42,7 @@ describe(' spec', () => {
{ @@ -63,10 +64,16 @@ describe(' spec', () => { expect(container).toMatchSnapshot(); expect(queryByTestId('noCategoryFieldsCallout')).toBeNull(); expect(queryByTestId('categoryFieldComboBox')).toBeNull(); - expect(queryByText('Enable categorical field')).not.toBeNull(); + expect(queryByText('Enable categorical fields')).not.toBeNull(); }); test('renders the component when enabled', () => { - const { container, queryByText, queryByTestId } = render( + const { + container, + queryByText, + queryByTestId, + getByTestId, + getByText, + } = render( spec', () => { { return; }} @@ -98,7 +106,10 @@ describe(' spec', () => { expect(container).toMatchSnapshot(); expect(queryByTestId('noCategoryFieldsCallout')).toBeNull(); expect(queryByTestId('categoryFieldComboBox')).not.toBeNull(); - expect(queryByText('Enable categorical field')).not.toBeNull(); + expect(queryByText('Enable categorical fields')).not.toBeNull(); + fireEvent.click(getByTestId('comboBoxToggleListButton')); + getByText('a'); + getByText('b'); }); test('shows callout when there are no available category fields', () => { const { container, queryByText, queryByTestId } = render( @@ -112,6 +123,7 @@ describe(' spec', () => { { @@ -133,7 +145,7 @@ describe(' spec', () => { expect(container).toMatchSnapshot(); expect(queryByTestId('noCategoryFieldsCallout')).not.toBeNull(); expect(queryByTestId('categoryFieldComboBox')).toBeNull(); - expect(queryByText('Enable categorical field')).not.toBeNull(); + expect(queryByText('Enable categorical fields')).not.toBeNull(); }); test('hides callout if component is loading', () => { const { container, queryByText, queryByTestId } = render( @@ -147,6 +159,7 @@ describe(' spec', () => { { @@ -167,6 +180,56 @@ describe(' spec', () => { ); expect(container).toMatchSnapshot(); expect(queryByTestId('noCategoryFieldsCallout')).toBeNull(); - expect(queryByText('Enable categorical field')).not.toBeNull(); + expect(queryByText('Enable categorical fields')).not.toBeNull(); + }); + test(`limits selection to a maximum of 2 entities`, () => { + const { getAllByRole, getByTestId, queryByText } = render( + + {}} + > + + + { + return; + }} + isLoading={false} + originalShingleSize={1} + formikProps={{ + values: { + categoryFieldEnabled: true, + }, + }} + /> + + + + + ); + // open combo box + fireEvent.click(getByTestId('comboBoxToggleListButton')); + expect(queryByText('a')).not.toBeNull(); + expect(queryByText('b')).not.toBeNull(); + expect(queryByText('c')).not.toBeNull(); + + // select top 3 options (a,b,c) + fireEvent.click(getAllByRole('option')[0]); + fireEvent.click(getAllByRole('option')[0]); + fireEvent.click(getAllByRole('option')[0]); + + // close combo box + fireEvent.click(getByTestId('comboBoxToggleListButton')); + + // the last selection (c) is still not selected + expect(queryByText('a')).not.toBeNull(); + expect(queryByText('b')).not.toBeNull(); + expect(queryByText('c')).toBeNull(); }); }); diff --git a/public/pages/ConfigureModel/components/CategoryField/__tests__/__snapshots__/CategoryField.test.tsx.snap b/public/pages/ConfigureModel/components/CategoryField/__tests__/__snapshots__/CategoryField.test.tsx.snap index 5d093848..b984eb01 100644 --- a/public/pages/ConfigureModel/components/CategoryField/__tests__/__snapshots__/CategoryField.test.tsx.snap +++ b/public/pages/ConfigureModel/components/CategoryField/__tests__/__snapshots__/CategoryField.test.tsx.snap @@ -26,7 +26,7 @@ exports[` spec hides callout if component is loading 1`] = ` class="euiTitle euiTitle--small" id="categoryFieldTitle" > - Categorical field + Categorical fields
@@ -41,7 +41,7 @@ exports[` spec hides callout if component is loading 1`] = ` class="euiText euiText--medium content-panel-subTitle" style="line-height: normal;" > - Split a single time series into multiple time series based on a categorical field. + Split a single time series into multiple time series based on categorical fields. You can select up to 2. spec hides callout if component is loading 1`] = ` class="euiCheckbox__label" for="categoryFieldCheckbox" > - Enable categorical field + Enable categorical fields
@@ -152,7 +152,7 @@ exports[` spec renders the component when disabled 1`] = ` class="euiTitle euiTitle--small" id="categoryFieldTitle" > - Categorical field + Categorical fields
@@ -167,7 +167,7 @@ exports[` spec renders the component when disabled 1`] = ` class="euiText euiText--medium content-panel-subTitle" style="line-height: normal;" > - Split a single time series into multiple time series based on a categorical field. + Split a single time series into multiple time series based on categorical fields. You can select up to 2. spec renders the component when disabled 1`] = ` class="euiCheckbox__label" for="categoryFieldCheckbox" > - Enable categorical field + Enable categorical fields
@@ -269,7 +269,7 @@ exports[` spec renders the component when enabled 1`] = ` class="euiTitle euiTitle--small" id="categoryFieldTitle" > - Categorical field + Categorical fields
@@ -284,7 +284,7 @@ exports[` spec renders the component when enabled 1`] = ` class="euiText euiText--medium content-panel-subTitle" style="line-height: normal;" > - Split a single time series into multiple time series based on a categorical field. + Split a single time series into multiple time series based on categorical fields. You can select up to 2. spec renders the component when enabled 1`] = ` class="euiCheckbox__label" for="categoryFieldCheckbox" > - Enable categorical field + Enable categorical fields
@@ -393,14 +393,14 @@ exports[` spec renders the component when enabled 1`] = ` class="euiFormControlLayout__childrenWrapper" >

- Select your category field + Select your categorical fields

spec renders the component when enabled 1`] = ` class="euiFormHelpText euiFormRow__text" id="random_html_id-help" > - You can only apply the category field to the 'ip' and 'keyword' OpenSearch data types. + You can only apply the categorical fields to the 'ip' and 'keyword' OpenSearch data types.
@@ -486,7 +486,7 @@ exports[` spec shows callout when there are no available catego class="euiTitle euiTitle--small" id="categoryFieldTitle" > - Categorical field + Categorical fields
@@ -501,7 +501,7 @@ exports[` spec shows callout when there are no available catego class="euiText euiText--medium content-panel-subTitle" style="line-height: normal;" > - Split a single time series into multiple time series based on a categorical field. + Split a single time series into multiple time series based on categorical fields. You can select up to 2. spec shows callout when there are no available catego class="euiCheckbox__label" for="categoryFieldCheckbox" > - Enable categorical field + Enable categorical fields diff --git a/public/pages/ConfigureModel/containers/__tests__/__snapshots__/ConfigureModel.test.tsx.snap b/public/pages/ConfigureModel/containers/__tests__/__snapshots__/ConfigureModel.test.tsx.snap index ba2643e5..da72b033 100644 --- a/public/pages/ConfigureModel/containers/__tests__/__snapshots__/ConfigureModel.test.tsx.snap +++ b/public/pages/ConfigureModel/containers/__tests__/__snapshots__/ConfigureModel.test.tsx.snap @@ -584,7 +584,7 @@ exports[` spec creating model configuration renders the compon class="euiTitle euiTitle--small" id="categoryFieldTitle" > - Categorical field + Categorical fields @@ -599,7 +599,7 @@ exports[` spec creating model configuration renders the compon class="euiText euiText--medium content-panel-subTitle" style="line-height: normal;" > - Split a single time series into multiple time series based on a categorical field. + Split a single time series into multiple time series based on categorical fields. You can select up to 2. spec creating model configuration renders the compon class="euiCheckbox__label" for="categoryFieldCheckbox" > - Enable categorical field + Enable categorical fields @@ -1515,7 +1515,7 @@ exports[` spec editing model configuration renders the compone class="euiTitle euiTitle--small" id="categoryFieldTitle" > - Categorical field + Categorical fields @@ -1530,7 +1530,7 @@ exports[` spec editing model configuration renders the compone class="euiText euiText--medium content-panel-subTitle" style="line-height: normal;" > - Split a single time series into multiple time series based on a categorical field. + Split a single time series into multiple time series based on categorical fields. You can select up to 2. spec editing model configuration renders the compone class="euiCheckbox__label" for="categoryFieldCheckbox" > - Enable categorical field + Enable categorical fields diff --git a/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx b/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx index 60e576fa..ad277994 100644 --- a/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx +++ b/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx @@ -25,9 +25,10 @@ */ import React from 'react'; -import { get } from 'lodash'; +import { get, isEmpty } from 'lodash'; import { EuiBasicTable } from '@elastic/eui'; import ContentPanel from '../../../../components/ContentPanel/ContentPanel'; +import { convertToCategoryFieldString } from '../../../utils/anomalyResultUtils'; interface AdditionalSettingsProps { shingleSize: number; @@ -37,12 +38,14 @@ interface AdditionalSettingsProps { export function AdditionalSettings(props: AdditionalSettingsProps) { const tableItems = [ { - categoryField: get(props.categoryField, 0, '-'), + categoryField: isEmpty(get(props, 'categoryField', [])) + ? '-' + : convertToCategoryFieldString(props.categoryField, ', '), windowSize: props.shingleSize, }, ]; const tableColumns = [ - { name: 'Category field', field: 'categoryField' }, + { name: 'Categorical fields', field: 'categoryField' }, { name: 'Window size', field: 'windowSize' }, ]; return ( diff --git a/public/pages/DetectorResults/containers/AnomalyHistory.tsx b/public/pages/DetectorResults/containers/AnomalyHistory.tsx index f2ef5741..c3467ed1 100644 --- a/public/pages/DetectorResults/containers/AnomalyHistory.tsx +++ b/public/pages/DetectorResults/containers/AnomalyHistory.tsx @@ -68,6 +68,7 @@ import { parseTopEntityAnomalySummaryResults, getEntityAnomalySummariesQuery, parseEntityAnomalySummaryResults, + convertToEntityString, } from '../../utils/anomalyResultUtils'; import { AnomalyResultsTable } from './AnomalyResultsTable'; import { AnomaliesChart } from '../../AnomalyCharts/containers/AnomaliesChart'; @@ -90,6 +91,7 @@ import { import { getAnomalyHistoryWording, NUM_CELLS, + getHCTitle, } from '../../AnomalyCharts/utils/anomalyChartUtils'; import { darkModeEnabled } from '../../../utils/opensearchDashboardsUtils'; import { @@ -165,6 +167,7 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { const detectorCategoryField = get(props.detector, 'categoryField', []); const isHCDetector = !isEmpty(detectorCategoryField); + const isMultiCategory = detectorCategoryField.length > 1; const backgroundColor = darkModeEnabled() ? '#29017' : '#F7F7F7'; // We load at most 10k AD result data points for one call. If user choose @@ -172,7 +175,8 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { // aggregation to load data points in whole time range with larger interval. // If entity is specified, we only query AD result data points for this entity. async function getBucketizedAnomalyResults( - entity: Entity | undefined = undefined + entityList: Entity[] | undefined = undefined, + modelId?: string ) { try { setIsLoadingAnomalyResults(true); @@ -182,9 +186,10 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { dateRange.startDate, dateRange.endDate, props.detector.id, - entity, + entityList, props.isHistorical, - taskId.current + taskId.current, + modelId ) ) ); @@ -198,9 +203,10 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { dateRange.startDate, dateRange.endDate, props.detector.id, - entity, + entityList, props.isHistorical, - taskId.current + taskId.current, + modelId ) ) ); @@ -307,34 +313,39 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { const fetchHCAnomalySummaries = async () => { setIsLoadingAnomalyResults(true); + const query = getTopAnomalousEntitiesQuery( dateRange.startDate, dateRange.endDate, props.detector.id, heatmapDisplayOption.entityOption.value, heatmapDisplayOption.sortType, + isMultiCategory, props.isHistorical, taskId.current ); const result = await dispatch(searchResults(query)); - const topEnityAnomalySummaries = parseTopEntityAnomalySummaryResults( - result + + const topEntityAnomalySummaries = parseTopEntityAnomalySummaryResults( + result, + isMultiCategory ); - const entities = topEnityAnomalySummaries.map((summary) => summary.entity); - const promises = entities.map(async (entity: Entity) => { - const entityResultQuery = getEntityAnomalySummariesQuery( - dateRange.startDate, - dateRange.endDate, - props.detector.id, - NUM_CELLS, - get(props.detector, 'categoryField[0]', ''), - entity.value, - props.isHistorical, - taskId.current - ); - return dispatch(searchResults(entityResultQuery)); - }); + const promises = topEntityAnomalySummaries.map( + async (summary: EntityAnomalySummaries) => { + const entityResultQuery = getEntityAnomalySummariesQuery( + dateRange.startDate, + dateRange.endDate, + props.detector.id, + NUM_CELLS, + summary.entityList, + summary.modelId, + props.isHistorical, + taskId.current + ); + return dispatch(searchResults(entityResultQuery)); + } + ); const allEntityAnomalySummaries = await Promise.all(promises).catch( (error) => { @@ -346,15 +357,19 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { const entitiesAnomalySummaries = [] as EntityAnomalySummaries[]; if (!isEmpty(allEntityAnomalySummaries)) { + const entityLists = topEntityAnomalySummaries.map( + (summary) => summary.entityList + ); //@ts-ignore allEntityAnomalySummaries.forEach((entityResponse, i) => { const entityAnomalySummariesResult = parseEntityAnomalySummaryResults( entityResponse, - entities[i] + entityLists[i] ); entitiesAnomalySummaries.push(entityAnomalySummariesResult); }); } + setEntityAnomalySummaries(entitiesAnomalySummaries); setIsLoadingAnomalyResults(false); }; @@ -369,14 +384,19 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { MAX_ANOMALIES ) ) { + // TODO: inject model id in heatmapCell to propagate fetchBucketizedEntityAnomalyData(heatmapCell); } else { + // TODO: inject model id in heatmapCell to propagate fetchAllEntityAnomalyData(heatmapCell); setBucketizedAnomalyResults(undefined); } } catch (err) { console.error( - `Failed to get anomaly results for entity ${heatmapCell.entityValue}`, + `Failed to get anomaly results for the following entities: ${convertToEntityString( + heatmapCell.entityList, + ', ' + )}`, err ); } finally { @@ -389,11 +409,7 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { heatmapCell.dateRange.startDate, heatmapCell.dateRange.endDate, false, - { - //@ts-ignore - name: props.detector.categoryField[0], - value: heatmapCell.entityValue, - } + heatmapCell.entityList ); const entityAnomalyResultResponse = await dispatch( @@ -418,11 +434,7 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { }; const fetchBucketizedEntityAnomalyData = async (heatmapCell: HeatmapCell) => { - getBucketizedAnomalyResults({ - //@ts-ignore - name: props.detector.categoryField[0], - value: heatmapCell.entityValue, - }); + getBucketizedAnomalyResults(heatmapCell.entityList); }; const [atomicAnomalyResults, setAtomicAnomalyResults] = useState(); const [rawAnomalyResults, setRawAnomalyResults] = useState(); @@ -541,24 +553,13 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { entityAnomalySummaries={entityAnomalySummaries} >
- {/* - TODO: update title and occurrence chart to support multi category field support - */} {isHCDetector ? [ -
- {`${get(props, 'detector.categoryField.0')}`}{' '} - {selectedHeatmapCell?.entityValue} - -
- - ) : ( - '-' - ) + selectedHeatmapCell + ? getHCTitle(selectedHeatmapCell.entityList) + : '-' } dateRange={dateRange} onDateRangeChange={handleDateRangeChange} @@ -574,7 +575,6 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { detector={props.detector} monitor={props.monitor} isHCDetector={isHCDetector} - isHistorical={props.isHistorical} selectedHeatmapCell={selectedHeatmapCell} />, , @@ -608,12 +608,9 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { dateRange={zoomRange} featureDataSeriesName="Feature output" showFeatureMissingDataPointAnnotation={ - // disable showing missing feature alert when it is HC or historical - props.isHistorical - ? false - : props.detector.enabled && !isHCDetector - ? true - : false + props.detector.enabled && + // disable showing missing feature alert when it is HC Detector + !isHCDetector } isFeatureDataMissing={props.isFeatureDataMissing} isHCDetector={isHCDetector} diff --git a/public/pages/DetectorResults/containers/AnomalyResultsTable.tsx b/public/pages/DetectorResults/containers/AnomalyResultsTable.tsx index dd4413c4..9043bc9c 100644 --- a/public/pages/DetectorResults/containers/AnomalyResultsTable.tsx +++ b/public/pages/DetectorResults/containers/AnomalyResultsTable.tsx @@ -36,12 +36,13 @@ import { SORT_DIRECTION } from '../../../../server/utils/constants'; import ContentPanel from '../../../components/ContentPanel/ContentPanel'; import { entityValueColumn, - ENTITY_VALUE_FIELD, staticColumn, + ENTITY_VALUE_FIELD, } from '../utils/tableUtils'; import { DetectorResultsQueryParams } from 'server/models/types'; import { AnomalyData } from '../../../models/interfaces'; import { getTitleWithCount } from '../../../utils/utils'; +import { convertToCategoryFieldAndEntityString } from '../../utils/anomalyResultUtils'; interface AnomalyResultsTableProps { anomalies: AnomalyData[]; @@ -89,7 +90,9 @@ export function AnomalyResultsTable(props: AnomalyResultsTableProps) { anomalies = anomalies.map((anomaly) => { return { ...anomaly, - [ENTITY_VALUE_FIELD]: get(anomaly, 'entity[0].value'), + [ENTITY_VALUE_FIELD]: convertToCategoryFieldAndEntityString( + get(anomaly, 'entity', []) + ), }; }); } diff --git a/public/pages/DetectorResults/utils/tableUtils.tsx b/public/pages/DetectorResults/utils/tableUtils.tsx index 6e464726..5f39baaa 100644 --- a/public/pages/DetectorResults/utils/tableUtils.tsx +++ b/public/pages/DetectorResults/utils/tableUtils.tsx @@ -93,8 +93,10 @@ export const staticColumn = [ export const entityValueColumn = { field: ENTITY_VALUE_FIELD, - name: 'Entity', + name: 'Entities', sortable: true, truncateText: false, - dataType: 'number', + dataType: 'string', + // To render newline character correctly + style: { whiteSpace: 'pre-wrap' }, } as EuiBasicTableColumn; diff --git a/public/pages/HistoricalDetectorResults/containers/HistoricalDetectorResults.tsx b/public/pages/HistoricalDetectorResults/containers/HistoricalDetectorResults.tsx index 0377622e..d5844c47 100644 --- a/public/pages/HistoricalDetectorResults/containers/HistoricalDetectorResults.tsx +++ b/public/pages/HistoricalDetectorResults/containers/HistoricalDetectorResults.tsx @@ -52,6 +52,7 @@ import { getDetectorStateDetails } from '../../DetectorDetail/utils/helpers'; import { HistoricalRangeModal } from '../components/HistoricalRangeModal'; import { HISTORICAL_DETECTOR_RESULT_REFRESH_RATE, + HISTORICAL_HC_DETECTOR_RESULT_REFRESH_RATE, HISTORICAL_DETECTOR_STOP_THRESHOLD, } from '../utils/constants'; import { CoreStart } from '../../../../../../src/core/public'; @@ -115,7 +116,9 @@ export function HistoricalDetectorResults( ) { const intervalId = setInterval( fetchDetector, - HISTORICAL_DETECTOR_RESULT_REFRESH_RATE + isHCDetector + ? HISTORICAL_HC_DETECTOR_RESULT_REFRESH_RATE + : HISTORICAL_DETECTOR_RESULT_REFRESH_RATE ); return () => { clearInterval(intervalId); diff --git a/public/pages/HistoricalDetectorResults/utils/constants.tsx b/public/pages/HistoricalDetectorResults/utils/constants.tsx index c0a80210..c83da670 100644 --- a/public/pages/HistoricalDetectorResults/utils/constants.tsx +++ b/public/pages/HistoricalDetectorResults/utils/constants.tsx @@ -26,7 +26,10 @@ // Current backend implementation: limited to running model on 1000 intervals every 5s. // Frontend should refresh at some rate > than this, to auto-refresh and show partial results. +// For historical non-high-cardinality detectors: refresh every 10s +// For historical high-cardinality detectors: refresh every 30s export const HISTORICAL_DETECTOR_RESULT_REFRESH_RATE = 10000; +export const HISTORICAL_HC_DETECTOR_RESULT_REFRESH_RATE = 30000; // Current backend implementation will handle stopping a historical detector task asynchronously. It is assumed // that if the task is not in a stopped state after 5s, then there was a problem stopping. diff --git a/public/pages/main/Main.tsx b/public/pages/main/Main.tsx index 2743404d..f1d07042 100644 --- a/public/pages/main/Main.tsx +++ b/public/pages/main/Main.tsx @@ -28,7 +28,6 @@ import { Switch, Route, RouteComponentProps } from 'react-router-dom'; import React from 'react'; import { AppState } from '../../redux/reducers'; import { DetectorList } from '../DetectorsList'; -import { SampleData } from '../SampleData'; import { ListRouterParams } from '../DetectorsList/containers/List/List'; import { CreateDetectorSteps } from '../CreateDetectorSteps'; import { EuiSideNav, EuiPage, EuiPageBody, EuiPageSideBar } from '@elastic/eui'; @@ -44,12 +43,8 @@ import { AnomalyDetectionOverview } from '../Overview'; enum Navigation { AnomalyDetection = 'Anomaly detection', - Realtime = 'Real-time', Dashboard = 'Dashboard', Detectors = 'Detectors', - HistoricalDetectors = 'Historical detectors', - SampleDetectors = 'Sample detectors', - CreateDetectorSteps = 'Create detector steps', } interface MainProps extends RouteComponentProps {} @@ -60,7 +55,7 @@ export function Main(props: MainProps) { ); const adState = useSelector((state: AppState) => state.ad); - const totalRealtimeDetectors = adState.totalDetectors; + const totalDetectors = adState.totalDetectors; const errorGettingDetectors = adState.errorMessage; const isLoadingDetectors = adState.requesting; const sideNav = [ @@ -149,7 +144,7 @@ export function Main(props: MainProps) { )} /> - {totalRealtimeDetectors > 0 ? ( + {totalDetectors > 0 ? ( //
) : ( diff --git a/public/pages/utils/__tests__/constants.ts b/public/pages/utils/__tests__/constants.ts index a6654da8..4b630629 100644 --- a/public/pages/utils/__tests__/constants.ts +++ b/public/pages/utils/__tests__/constants.ts @@ -85,6 +85,6 @@ export const FAKE_ENTITY_ANOMALY_SUMMARY = { } as EntityAnomalySummary; export const FAKE_ENTITY_ANOMALY_SUMMARIES = { - entity: FAKE_ENTITY, + entityList: [FAKE_ENTITY], anomalySummaries: [FAKE_ENTITY_ANOMALY_SUMMARY], } as EntityAnomalySummaries; diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index 63125ad1..f255c48f 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -36,7 +36,6 @@ import { AD_DOC_FIELDS, DOC_COUNT_FIELD, ENTITY_FIELD, - ENTITY_NAME_PATH_FIELD, ENTITY_VALUE_PATH_FIELD, KEY_FIELD, MIN_IN_MILLI_SECS, @@ -44,6 +43,9 @@ import { DAY_IN_MILLI_SECS, SORT_DIRECTION, WEEK_IN_MILLI_SECS, + MODEL_ID_FIELD, + ENTITY_LIST_FIELD, + ENTITY_LIST_DELIMITER, } from '../../../server/utils/constants'; import { toFixedNumberForAnomaly } from '../../../server/utils/helpers'; import { @@ -61,7 +63,10 @@ import { MISSING_FEATURE_DATA_SEVERITY, } from '../../utils/constants'; import { HeatmapCell } from '../AnomalyCharts/containers/AnomalyHeatmapChart'; -import { AnomalyHeatmapSortType } from '../AnomalyCharts/utils/anomalyChartUtils'; +import { + AnomalyHeatmapSortType, + NUM_CELLS, +} from '../AnomalyCharts/utils/anomalyChartUtils'; import { DETECTOR_INIT_FAILURES } from '../DetectorDetail/utils/constants'; import { COUNT_ANOMALY_AGGS, @@ -113,7 +118,7 @@ export const buildParamsForGetAnomalyResultsWithDateRange = ( startTime: number, endTime: number, anomalyOnly: boolean = false, - entity: Entity | undefined = undefined + entityList: Entity[] | undefined = undefined ) => { return { from: 0, @@ -124,8 +129,7 @@ export const buildParamsForGetAnomalyResultsWithDateRange = ( endTime: endTime, fieldName: AD_DOC_FIELDS.DATA_START_TIME, anomalyThreshold: anomalyOnly ? 0 : -1, - entityName: entity?.name, - entityValue: entity?.value, + entityList: JSON.stringify(entityList), }; }; @@ -308,9 +312,10 @@ export const getAnomalySummaryQuery = ( startTime: number, endTime: number, detectorId: string, - entity: Entity | undefined = undefined, + entityList: Entity[] | undefined = undefined, isHistorical?: boolean, - taskId?: string + taskId?: string, + modelId?: string ) => { const termField = isHistorical && taskId ? { task_id: taskId } : { detector_id: detectorId }; @@ -337,34 +342,6 @@ export const getAnomalySummaryQuery = ( { term: termField, }, - ...(entity - ? [ - { - nested: { - path: ENTITY_FIELD, - query: { - term: { - [ENTITY_VALUE_PATH_FIELD]: { - value: entity.value, - }, - }, - }, - }, - }, - { - nested: { - path: ENTITY_FIELD, - query: { - term: { - [ENTITY_NAME_PATH_FIELD]: { - value: entity.name, - }, - }, - }, - }, - }, - ] - : []), ], }, }, @@ -410,6 +387,8 @@ export const getAnomalySummaryQuery = ( }, }; + // If querying RT results: remove any results that include a task_id, as this indicates + // a historical result from a historical task. if (!isHistorical) { requestBody.query.bool = { ...requestBody.query.bool, @@ -423,6 +402,12 @@ export const getAnomalySummaryQuery = ( }; } + // Add entity filters if this is a HC detector + if (entityList !== undefined && entityList.length > 0) { + //@ts-ignore + requestBody.query.bool.filter.push(getEntityFilters(modelId, entityList)); + } + return requestBody; }; @@ -430,9 +415,10 @@ export const getBucketizedAnomalyResultsQuery = ( startTime: number, endTime: number, detectorId: string, - entity: Entity | undefined = undefined, + entityList: Entity[] | undefined = undefined, isHistorical?: boolean, - taskId?: string + taskId?: string, + modelId?: string ) => { const termField = isHistorical && taskId ? { task_id: taskId } : { detector_id: detectorId }; @@ -455,34 +441,6 @@ export const getBucketizedAnomalyResultsQuery = ( { term: termField, }, - ...(entity - ? [ - { - nested: { - path: ENTITY_FIELD, - query: { - term: { - [ENTITY_VALUE_PATH_FIELD]: { - value: entity.value, - }, - }, - }, - }, - }, - { - nested: { - path: ENTITY_FIELD, - query: { - term: { - [ENTITY_NAME_PATH_FIELD]: { - value: entity.name, - }, - }, - }, - }, - }, - ] - : []), ], }, }, @@ -513,6 +471,8 @@ export const getBucketizedAnomalyResultsQuery = ( }, }; + // If querying RT results: remove any results that include a task_id, as this indicates + // a historical result from a historical task. if (!isHistorical) { requestBody.query.bool = { ...requestBody.query.bool, @@ -526,6 +486,12 @@ export const getBucketizedAnomalyResultsQuery = ( }; } + // Add entity filters if this is a HC detector + if (entityList !== undefined && entityList.length > 0) { + //@ts-ignore + requestBody.query.bool.filter.push(getEntityFilters(modelId, entityList)); + } + return requestBody; }; @@ -979,9 +945,11 @@ export const filterWithHeatmapFilter = ( if (isFilteringWithEntity) { data = data .filter((anomaly) => !isEmpty(get(anomaly, 'entity', []))) - .filter( - (anomaly) => get(anomaly, 'entity')[0].value === heatmapCell.entityValue - ); + .filter((anomaly) => { + const dataEntityList = get(anomaly, 'entity'); + const cellEntityList = get(heatmapCell, 'entityList'); + return entityListsMatch(dataEntityList, cellEntityList); + }); } return filterWithDateRange(data, heatmapCell.dateRange, timeField); }; @@ -992,64 +960,67 @@ export const getTopAnomalousEntitiesQuery = ( detectorId: string, size: number, sortType: AnomalyHeatmapSortType, + isMultiCategory: boolean, isHistorical?: boolean, taskId?: string ) => { const termField = isHistorical && taskId ? { task_id: taskId } : { detector_id: detectorId }; - const requestBody = { - size: 0, - query: { - bool: { - filter: [ - { - range: { - [AD_DOC_FIELDS.ANOMALY_GRADE]: { - gt: 0, + // To handle BWC, we will return 2 possible queries based on the # of categorical fields: + // (1) legacy way (1 category field): bucket aggregate over the single, nested, 'entity.value' field + // (2) new way (>= 2 category fields): bucket aggregate over the new 'model_id' field + const requestBody = isMultiCategory + ? { + size: 0, + query: { + bool: { + filter: [ + { + range: { + [AD_DOC_FIELDS.ANOMALY_GRADE]: { + gt: 0, + }, + }, }, - }, - }, - { - range: { - data_end_time: { - gte: startTime, - lte: endTime, + { + range: { + data_end_time: { + gte: startTime, + lte: endTime, + }, + }, }, - }, - }, - { - term: termField, + { + term: termField, + }, + ], }, - ], - }, - }, - aggs: { - [TOP_ENTITIES_FIELD]: { - nested: { - path: ENTITY_FIELD, }, aggs: { [TOP_ENTITY_AGGS]: { terms: { - field: ENTITY_VALUE_PATH_FIELD, + field: MODEL_ID_FIELD, size: size, ...(sortType === AnomalyHeatmapSortType.SEVERITY ? { order: { - [TOP_ANOMALY_GRADE_SORT_AGGS]: SORT_DIRECTION.DESC, + [MAX_ANOMALY_AGGS]: SORT_DIRECTION.DESC, }, } : {}), }, aggs: { - [TOP_ANOMALY_GRADE_SORT_AGGS]: { - reverse_nested: {}, - aggs: { - [MAX_ANOMALY_AGGS]: { - max: { - field: AD_DOC_FIELDS.ANOMALY_GRADE, - }, + [MAX_ANOMALY_AGGS]: { + max: { + field: AD_DOC_FIELDS.ANOMALY_GRADE, + }, + }, + [ENTITY_LIST_FIELD]: { + top_hits: { + size: 1, + _source: { + include: [ENTITY_FIELD], }, }, }, @@ -1059,7 +1030,7 @@ export const getTopAnomalousEntitiesQuery = ( bucket_sort: { sort: [ { - [`${TOP_ANOMALY_GRADE_SORT_AGGS}.${MAX_ANOMALY_AGGS}`]: { + [`${MAX_ANOMALY_AGGS}`]: { order: SORT_DIRECTION.DESC, }, }, @@ -1071,9 +1042,83 @@ export const getTopAnomalousEntitiesQuery = ( }, }, }, - }, - }, - }; + } + : { + size: 0, + query: { + bool: { + filter: [ + { + range: { + [AD_DOC_FIELDS.ANOMALY_GRADE]: { + gt: 0, + }, + }, + }, + { + range: { + data_end_time: { + gte: startTime, + lte: endTime, + }, + }, + }, + { + term: termField, + }, + ], + }, + }, + aggs: { + [TOP_ENTITIES_FIELD]: { + nested: { + path: ENTITY_FIELD, + }, + aggs: { + [TOP_ENTITY_AGGS]: { + terms: { + field: ENTITY_VALUE_PATH_FIELD, + size: size, + ...(sortType === AnomalyHeatmapSortType.SEVERITY + ? { + order: { + [TOP_ANOMALY_GRADE_SORT_AGGS]: SORT_DIRECTION.DESC, + }, + } + : {}), + }, + aggs: { + [TOP_ANOMALY_GRADE_SORT_AGGS]: { + reverse_nested: {}, + aggs: { + [MAX_ANOMALY_AGGS]: { + max: { + field: AD_DOC_FIELDS.ANOMALY_GRADE, + }, + }, + }, + }, + ...(sortType === AnomalyHeatmapSortType.SEVERITY + ? { + [MAX_ANOMALY_SORT_AGGS]: { + bucket_sort: { + sort: [ + { + [`${TOP_ANOMALY_GRADE_SORT_AGGS}.${MAX_ANOMALY_AGGS}`]: { + order: SORT_DIRECTION.DESC, + }, + }, + ], + }, + }, + } + : {}), + }, + }, + }, + }, + }, + }; if (!isHistorical) { requestBody.query.bool = { @@ -1092,34 +1137,42 @@ export const getTopAnomalousEntitiesQuery = ( }; export const parseTopEntityAnomalySummaryResults = ( - result: any + result: any, + isMultiCategory: boolean ): EntityAnomalySummaries[] => { - const rawEntityAnomalySummaries = get( - result, - `response.aggregations.${TOP_ENTITIES_FIELD}.${TOP_ENTITY_AGGS}.buckets`, - [] - ) as any[]; + const rawEntityAnomalySummaries = isMultiCategory + ? get(result, `response.aggregations.${TOP_ENTITY_AGGS}.buckets`, []) + : (get( + result, + `response.aggregations.${TOP_ENTITIES_FIELD}.${TOP_ENTITY_AGGS}.buckets`, + [] + ) as any[]); let topEntityAnomalySummaries = [] as EntityAnomalySummaries[]; - rawEntityAnomalySummaries.forEach((item) => { + rawEntityAnomalySummaries.forEach((item: any) => { const anomalyCount = get(item, DOC_COUNT_FIELD, 0); - const entityValue = get(item, KEY_FIELD, 0); - const entity = { - value: entityValue, - } as Entity; - const maxAnomalyGrade = get( - item, - [TOP_ANOMALY_GRADE_SORT_AGGS, MAX_ANOMALY_AGGS].join('.'), - 0 - ); - const enityAnomalySummary = { + const entityList = isMultiCategory + ? get(item, `${ENTITY_LIST_FIELD}.hits.hits.0._source.entity`, []) + : ([ + { + value: get(item, KEY_FIELD, 0), + }, + ] as Entity[]); + + const maxAnomalyGrade = isMultiCategory + ? get(item, MAX_ANOMALY_AGGS, 0) + : get(item, [TOP_ANOMALY_GRADE_SORT_AGGS, MAX_ANOMALY_AGGS].join('.'), 0); + const entityAnomalySummary = { maxAnomaly: maxAnomalyGrade, anomalyCount: anomalyCount, } as EntityAnomalySummary; - const enityAnomaliSummaries = { - entity: entity, - anomalySummaries: [enityAnomalySummary], + const entityAnomalySummaries = { + entityList: entityList, + anomalySummaries: [entityAnomalySummary], } as EntityAnomalySummaries; - topEntityAnomalySummaries.push(enityAnomaliSummaries); + if (isMultiCategory) { + entityAnomalySummaries.modelId = get(item, KEY_FIELD, ''); + } + topEntityAnomalySummaries.push(entityAnomalySummaries); }); return topEntityAnomalySummaries; }; @@ -1129,8 +1182,8 @@ export const getEntityAnomalySummariesQuery = ( endTime: number, detectorId: string, size: number, - categoryField: string, - entityValue: string, + entityList: Entity[], + modelId: string | undefined, isHistorical?: boolean, taskId?: string ) => { @@ -1146,6 +1199,7 @@ export const getEntityAnomalySummariesQuery = ( // if startTime is not divisible by fixedInterval, there will be remainder, // this can be offset for bucket_key const offsetInMillisec = startTime % (fixedInterval * MIN_IN_MILLI_SECS); + const requestBody = { size: 0, query: { @@ -1169,30 +1223,6 @@ export const getEntityAnomalySummariesQuery = ( { term: termField, }, - { - nested: { - path: ENTITY_FIELD, - query: { - term: { - [ENTITY_VALUE_PATH_FIELD]: { - value: entityValue, - }, - }, - }, - }, - }, - { - nested: { - path: ENTITY_FIELD, - query: { - term: { - [ENTITY_NAME_PATH_FIELD]: { - value: categoryField, - }, - }, - }, - }, - }, ], }, }, @@ -1234,12 +1264,18 @@ export const getEntityAnomalySummariesQuery = ( }; } + // Add entity filters if this is a HC detector + if (entityList !== undefined && entityList.length > 0) { + //@ts-ignore + requestBody.query.bool.filter.push(getEntityFilters(modelId, entityList)); + } + return requestBody; }; export const parseEntityAnomalySummaryResults = ( result: any, - entity: Entity + entityList: Entity[] ): EntityAnomalySummaries => { const rawEntityAnomalySummaries = get( result, @@ -1251,18 +1287,35 @@ export const parseEntityAnomalySummaryResults = ( const anomalyCount = get(item, `${COUNT_ANOMALY_AGGS}.value`, 0); const startTime = get(item, 'key', 0); const maxAnomalyGrade = get(item, `${MAX_ANOMALY_AGGS}.value`, 0); - const enityAnomalySummary = { + const entityAnomalySummary = { startTime: startTime, maxAnomaly: maxAnomalyGrade, anomalyCount: anomalyCount, } as EntityAnomalySummary; - anomalySummaries.push(enityAnomalySummary); + anomalySummaries.push(entityAnomalySummary); }); - const enityAnomalySummaries = { - entity: entity, + const entityAnomalySummaries = { + entityList: entityList, anomalySummaries: anomalySummaries, } as EntityAnomalySummaries; - return enityAnomalySummaries; + return entityAnomalySummaries; +}; + +export const convertToEntityString = ( + entityList: Entity[], + delimiter?: string +) => { + let entityString = ''; + const delimiterToUse = delimiter ? delimiter : ENTITY_LIST_DELIMITER; + if (!isEmpty(entityList)) { + entityList.forEach((entity: any, index) => { + if (index > 0) { + entityString += delimiterToUse; + } + entityString += entity.value; + }); + } + return entityString; }; export const getAnomalyDataRangeQuery = ( @@ -1425,3 +1478,103 @@ export const parseHistoricalAggregatedAnomalies = ( return anomalies; }; +export const convertToCategoryFieldString = ( + categoryFields: string[], + delimiter: string +) => { + let categoryFieldString = ''; + if (!isEmpty(categoryFields)) { + categoryFields.forEach((categoryField: any, index) => { + if (index > 0) { + categoryFieldString += delimiter; + } + categoryFieldString += categoryField; + }); + } + return categoryFieldString; +}; + +export const convertToCategoryFieldAndEntityString = (entityList: Entity[]) => { + let entityString = ''; + if (!isEmpty(entityList)) { + entityList.forEach((entity: any, index) => { + if (index > 0) { + entityString += '\n'; + } + entityString += entity.name + ': ' + entity.value; + }); + } + return entityString; +}; + +export const convertToEntityList = ( + entityListAsString: string, + categoryFields: string[], + delimiter: string +) => { + let entityList = [] as Entity[]; + const valueArr = entityListAsString.split(delimiter); + var i; + for (i = 0; i < valueArr.length; i++) { + entityList.push({ + name: categoryFields[i], + value: valueArr[i], + }); + } + return entityList; +}; + +export const entityListsMatch = ( + entityListA: Entity[], + entityListB: Entity[] +) => { + if (get(entityListA, 'length') !== get(entityListB, 'length')) { + return false; + } + var i; + for (i = 0; i < entityListA.length; i++) { + if (entityListA[i].value !== entityListB[i].value) { + return false; + } + } + return true; +}; + +// Helper fn to get the correct filters based on how many categorical fields there are. +const getEntityFilters = ( + modelId: string | undefined, + entityList: Entity[] +) => { + return entityList.length > 1 + ? { + term: { + model_id: modelId, + }, + } + : { + nested: { + path: ENTITY_FIELD, + query: { + term: { + [ENTITY_VALUE_PATH_FIELD]: { + value: get(entityList, '0.value', ''), + }, + }, + }, + }, + }; +}; + +export const transformEntityListsForHeatmap = (entityLists: any[]) => { + let transformedEntityLists = [] as any[]; + entityLists.forEach((entityList: Entity[]) => { + const listAsString = convertToEntityString(entityList, ', '); + let row = []; + var i; + for (i = 0; i < NUM_CELLS; i++) { + row.push(listAsString); + } + transformedEntityLists.push(row); + }); + return transformedEntityLists; +}; diff --git a/public/pages/utils/helpers.ts b/public/pages/utils/helpers.ts index bbde810b..ecb317be 100644 --- a/public/pages/utils/helpers.ts +++ b/public/pages/utils/helpers.ts @@ -26,10 +26,7 @@ import { CatIndex, IndexAlias } from '../../../server/models/types'; import sortBy from 'lodash/sortBy'; -import { - DetectorListItem, - HistoricalDetectorListItem, -} from '../../models/interfaces'; +import { DetectorListItem } from '../../models/interfaces'; import { SORT_DIRECTION } from '../../../server/utils/constants'; import { ALL_INDICES, ALL_DETECTOR_STATES } from './constants'; import { DETECTOR_STATE } from '../../../server/utils/constants'; @@ -130,35 +127,3 @@ export const formatNumber = (data: any) => { return ''; } }; - -export const filterAndSortHistoricalDetectors = ( - detectors: HistoricalDetectorListItem[], - search: string, - selectedDetectorStates: DETECTOR_STATE[], - sortField: string, - sortDirection: string -) => { - let filteredBySearch = - search == '' - ? detectors - : detectors.filter((detector) => detector.name.includes(search)); - let filteredBySearchAndState = - selectedDetectorStates == ALL_DETECTOR_STATES - ? filteredBySearch - : filteredBySearch.filter((detector) => - selectedDetectorStates.includes(detector.curState) - ); - let sorted = sortBy(filteredBySearchAndState, sortField); - if (sortDirection == SORT_DIRECTION.DESC) { - sorted = sorted.reverse(); - } - return sorted; -}; - -export const getHistoricalDetectorsToDisplay = ( - detectors: HistoricalDetectorListItem[], - page: number, - size: number -) => { - return detectors.slice(size * page, page * size + size); -}; diff --git a/public/redux/reducers/ad.ts b/public/redux/reducers/ad.ts index 65e3dc84..b2bc8d3c 100644 --- a/public/redux/reducers/ad.ts +++ b/public/redux/reducers/ad.ts @@ -31,11 +31,7 @@ import { APIErrorAction, } from '../middleware/types'; import handleActions from '../utils/handleActions'; -import { - Detector, - DetectorListItem, - HistoricalDetectorListItem, -} from '../../models/interfaces'; +import { Detector, DetectorListItem } from '../../models/interfaces'; import { AD_NODE_API } from '../../../utils/constants'; import { GetDetectorsQueryParams } from '../../../server/models/types'; import { cloneDeep, get } from 'lodash'; @@ -55,13 +51,11 @@ const STOP_HISTORICAL_DETECTOR = 'ad/STOP_HISTORICAL_DETECTOR'; const GET_DETECTOR_PROFILE = 'ad/GET_DETECTOR_PROFILE'; const MATCH_DETECTOR = 'ad/MATCH_DETECTOR'; const GET_DETECTOR_COUNT = 'ad/GET_DETECTOR_COUNT'; -const GET_HISTORICAL_DETECTOR_LIST = 'ad/GET_HISTORICAL_DETECTOR_LIST'; export interface Detectors { requesting: boolean; detectors: { [key: string]: Detector }; detectorList: { [key: string]: DetectorListItem }; - historicalDetectorList: { [key: string]: HistoricalDetectorListItem }; totalDetectors: number; errorMessage: string; } @@ -69,7 +63,6 @@ export const initialDetectorsState: Detectors = { requesting: false, detectors: {}, detectorList: {}, - historicalDetectorList: {}, errorMessage: '', totalDetectors: 0, }; @@ -369,34 +362,6 @@ const reducer = handleActions( errorMessage: action.error, }), }, - [GET_HISTORICAL_DETECTOR_LIST]: { - REQUEST: (state: Detectors): Detectors => ({ - ...state, - requesting: true, - errorMessage: '', - }), - SUCCESS: (state: Detectors, action: APIResponseAction): Detectors => ({ - ...state, - requesting: false, - historicalDetectorList: action.result.response.detectorList.reduce( - (acc: any, detector: Detector) => ({ - ...acc, - [detector.id]: { - ...detector, - dataStartTime: get(detector, 'detectionDateRange.startTime', 0), - dataEndTime: get(detector, 'detectionDateRange.endTime', 0), - }, - }), - {} - ), - totalDetectors: action.result.response.totalDetectors, - }), - FAILURE: (state: Detectors, action: APIErrorAction): Detectors => ({ - ...state, - requesting: false, - errorMessage: action.error, - }), - }, }, initialDetectorsState ); @@ -509,12 +474,4 @@ export const getDetectorCount = (): APIAction => ({ client.get(`..${AD_NODE_API.DETECTOR}/_count`, {}), }); -export const getHistoricalDetectorList = ( - queryParams: GetDetectorsQueryParams -): APIAction => ({ - type: GET_HISTORICAL_DETECTOR_LIST, - request: (client: HttpSetup) => - client.get(`..${AD_NODE_API.DETECTOR}/historical`, { query: queryParams }), -}); - export default reducer; diff --git a/public/utils/constants.ts b/public/utils/constants.ts index d8dab0df..efcccdfe 100644 --- a/public/utils/constants.ts +++ b/public/utils/constants.ts @@ -79,6 +79,8 @@ export const MAX_FEATURE_NUM = 5; export const MAX_FEATURE_NAME_SIZE = 64; +export const MAX_CATEGORY_FIELD_NUM = 2; + export const NAME_REGEX = RegExp('^[a-zA-Z0-9._-]+$'); //https://github.com/opensearch-project/anomaly-detection/blob/main/src/main/java/com/amazon/opendistroforelasticsearch/ad/settings/AnomalyDetectorSettings.java diff --git a/server/models/interfaces.ts b/server/models/interfaces.ts index 7b427812..881eb6b3 100644 --- a/server/models/interfaces.ts +++ b/server/models/interfaces.ts @@ -90,6 +90,7 @@ export interface EntityAnomalySummary { } export interface EntityAnomalySummaries { - entity: Entity; + entityList: Entity[]; anomalySummaries: EntityAnomalySummary[]; + modelId?: string; } diff --git a/server/routes/ad.ts b/server/routes/ad.ts index d3aab502..6f918dd3 100644 --- a/server/routes/ad.ts +++ b/server/routes/ad.ts @@ -24,7 +24,6 @@ * permissions and limitations under the License. */ -//@ts-ignore import { get, orderBy, pullAll, isEmpty } from 'lodash'; import { AnomalyResults, SearchResponse } from '../models/interfaces'; import { @@ -39,9 +38,6 @@ import { Router } from '../router'; import { SORT_DIRECTION, AD_DOC_FIELDS, - ENTITY_FIELD, - ENTITY_NAME_PATH_FIELD, - ENTITY_VALUE_PATH_FIELD, DETECTOR_STATE, } from '../utils/constants'; import { @@ -60,8 +56,6 @@ import { getTaskInitProgress, isIndexNotFoundError, getErrorMessage, - getRealtimeDetectors, - getHistoricalDetectors, getDetectorTasks, appendTaskInfo, getDetectorResults, @@ -69,6 +63,7 @@ import { processTaskError, getLatestDetectorTasksQuery, isRealTimeTask, + getFiltersFromEntityList, } from './utils/adHelpers'; import { isNumber, set } from 'lodash'; import { @@ -109,7 +104,6 @@ export function registerADRoutes(apiRouter: Router, adService: AdService) { ); apiRouter.get('/detectors/{detectorName}/_match', adService.matchDetector); apiRouter.get('/detectors/_count', adService.getDetectorCount); - apiRouter.get('/detectors/historical', adService.getHistoricalDetectors); } export default class AdService { @@ -544,15 +538,13 @@ export default class AdService { {} ); - const realtimeDetectors = getRealtimeDetectors( - Object.values(allDetectors) - ).reduce( + const allDetectorsMap = Object.values(allDetectors).reduce( (acc: any, detector: any) => ({ ...acc, [detector.id]: detector }), {} - ); + ) as { [key: string]: Detector }; //Given each detector from previous result, get aggregation to power list - const allDetectorIds = Object.keys(realtimeDetectors); + const allDetectorIds = Object.keys(allDetectorsMap); const aggregationResult = await this.client .asScoped(request) .callAsCurrentUser('ad.searchResults', { @@ -573,7 +565,7 @@ export default class AdService { return { ...acc, [agg.key]: { - ...realtimeDetectors[agg.key], + ...allDetectorsMap[agg.key], totalAnomalies: agg.total_anomalies_in_24hr.doc_count, lastActiveAnomaly: agg.latest_anomaly_time.value, }, @@ -588,7 +580,7 @@ export default class AdService { return { ...acc, [unusedDetector]: { - ...realtimeDetectors[unusedDetector], + ...allDetectorsMap[unusedDetector], totalAnomalies: 0, lastActiveAnomaly: 0, }, @@ -701,8 +693,7 @@ export default class AdService { endTime = 0, fieldName = '', anomalyThreshold = -1, - entityName = undefined, - entityValue = undefined, + entityList = '', } = request.query as { from: number; size: number; @@ -712,10 +703,15 @@ export default class AdService { endTime: number; fieldName: string; anomalyThreshold: number; - entityName: string; - entityValue: string; + entityList: string; }; + const entityListAsObj = + entityList.length === 0 ? {} : JSON.parse(entityList); + const entityFilters = isEmpty(entityListAsObj) + ? [] + : getFiltersFromEntityList(entityListAsObj); + //Allowed sorting columns const sortQueryMap = { anomalyGrade: { anomaly_grade: sortDirection }, @@ -752,34 +748,7 @@ export default class AdService { }, }, }, - ...(entityName && entityValue - ? [ - { - nested: { - path: ENTITY_FIELD, - query: { - term: { - [ENTITY_NAME_PATH_FIELD]: { - value: entityName, - }, - }, - }, - }, - }, - { - nested: { - path: ENTITY_FIELD, - query: { - term: { - [ENTITY_VALUE_PATH_FIELD]: { - value: entityValue, - }, - }, - }, - }, - }, - ] - : []), + ...entityFilters, ], }, }, @@ -967,201 +936,4 @@ export default class AdService { }); return featureResult; }; - - getHistoricalDetectors = async ( - context: RequestHandlerContext, - request: OpenSearchDashboardsRequest, - opensearchDashboardsResponse: OpenSearchDashboardsResponseFactory - ): Promise> => { - try { - const { - from = 0, - size = 20, - search = '', - indices = '', - sortDirection = SORT_DIRECTION.DESC, - sortField = 'name', - } = request.query as GetDetectorsQueryParams; - const mustQueries = []; - if (search.trim()) { - mustQueries.push({ - query_string: { - fields: ['name', 'description'], - default_operator: 'AND', - query: `*${search.trim().split('-').join('* *')}*`, - }, - }); - } - if (indices.trim()) { - mustQueries.push({ - query_string: { - fields: ['indices'], - default_operator: 'AND', - query: `*${indices.trim().split('-').join('* *')}*`, - }, - }); - } - // Allowed sorting columns - const sortQueryMap = { - name: { 'name.keyword': sortDirection }, - indices: { 'indices.keyword': sortDirection }, - lastUpdateTime: { last_update_time: sortDirection }, - } as { [key: string]: object }; - let sort = {}; - const sortQuery = sortQueryMap[sortField]; - if (sortQuery) { - sort = sortQuery; - } - // Preparing search request - const requestBody = { - sort, - size, - from, - query: { - bool: { - must: mustQueries, - }, - }, - }; - const response: any = await this.client - .asScoped(request) - .callAsCurrentUser('ad.searchDetector', { body: requestBody }); - - // Get all detectors from search detector API - const allDetectors = get(response, 'hits.hits', []).reduce( - (acc: any, detector: any) => ({ - ...acc, - [detector._id]: { - id: detector._id, - description: get(detector, '_source.description', ''), - indices: get(detector, '_source.indices', []), - lastUpdateTime: get(detector, '_source.last_update_time', 0), - ...convertDetectorKeysToCamelCase(get(detector, '_source', {})), - }, - }), - {} - ); - - // Filter out to just include historical detectors - const allHistoricalDetectors = getHistoricalDetectors( - Object.values(allDetectors) - ).reduce( - (acc, detector: any) => ({ ...acc, [detector.id]: detector }), - {} - ) as { [key: string]: Detector }; - - // Get related info for each historical detector (detector state, task info, etc.) - const allIds = Object.values(allHistoricalDetectors).map( - (detector) => detector.id - ) as string[]; - - const detectorDetailPromises = allIds.map(async (id: string) => { - try { - const detectorDetailResp = await this.client - .asScoped(request) - .callAsCurrentUser('ad.getDetector', { - detectorId: id, - }); - return detectorDetailResp; - } catch (err) { - console.log('Error getting historical detector ', err); - return Promise.reject( - new Error( - 'Error retrieving all historical detectors: ' + - getErrorMessage(err) - ) - ); - } - }); - - const detectorDetailResponses = await Promise.all( - detectorDetailPromises - ).catch((err) => { - throw err; - }); - - // Get the mapping from detector to task - const detectorTasks = getDetectorTasks(detectorDetailResponses); - - // Get results for each task - const detectorResultPromises = Object.values(detectorTasks).map( - async (task) => { - const taskId = get(task, 'task_id', ''); - try { - const reqBody = { - query: { - bool: { - must: [ - { range: { anomaly_grade: { gt: 0 } } }, - { - term: { - task_id: { - value: taskId, - }, - }, - }, - ], - }, - }, - }; - - const detectorResultResp = await this.client - .asScoped(request) - .callAsCurrentUser('ad.searchResults', { - body: reqBody, - }); - return detectorResultResp; - } catch (err) { - console.log('Error getting historical detector results ', err); - return Promise.reject( - new Error( - 'Error retrieving all historical detector results: ' + - getErrorMessage(err) - ) - ); - } - } - ); - - const detectorResultResponses = await Promise.all( - detectorResultPromises - ).catch((err) => { - throw err; - }); - - // Get the mapping from detector to anomaly results - const detectorResults = getDetectorResults(detectorResultResponses); - - // Append the task-related info for each detector. - // If no task: set state to DISABLED and total anomalies to 0 - const detectorsWithTaskInfo = appendTaskInfo( - allHistoricalDetectors, - detectorTasks, - detectorResults - ); - - return opensearchDashboardsResponse.ok({ - body: { - ok: true, - response: { - totalDetectors: Object.values(detectorsWithTaskInfo).length, - detectorList: Object.values(detectorsWithTaskInfo), - }, - }, - }); - } catch (err) { - console.log('Anomaly detector - Unable to search detectors', err); - if (isIndexNotFoundError(err)) { - return opensearchDashboardsResponse.ok({ - body: { ok: true, response: { totalDetectors: 0, detectorList: [] } }, - }); - } - return opensearchDashboardsResponse.ok({ - body: { - ok: false, - error: getErrorMessage(err), - }, - }); - } - }; } diff --git a/server/routes/utils/adHelpers.ts b/server/routes/utils/adHelpers.ts index 44ff3e68..5847c828 100644 --- a/server/routes/utils/adHelpers.ts +++ b/server/routes/utils/adHelpers.ts @@ -25,7 +25,7 @@ */ import { get, omit, cloneDeep, isEmpty } from 'lodash'; -import { AnomalyResults } from '../../models/interfaces'; +import { AnomalyResults, Entity } from '../../models/interfaces'; import { GetDetectorsQueryParams, Detector } from '../../models/types'; import { mapKeysDeep, toCamel, toSnake } from '../../utils/helpers'; import { @@ -33,6 +33,9 @@ import { STACK_TRACE_PATTERN, OPENSEARCH_EXCEPTION_PREFIX, REALTIME_TASK_TYPE_PREFIX, + ENTITY_FIELD, + ENTITY_NAME_PATH_FIELD, + ENTITY_VALUE_PATH_FIELD, } from '../../utils/constants'; import { InitProgress } from '../../models/interfaces'; import { MAX_DETECTORS } from '../../utils/constants'; @@ -293,20 +296,6 @@ export const getErrorMessage = (err: any) => { : get(err, 'message'); }; -// Currently: detector w/ detection date range is considered a 'historical' detector -export const getHistoricalDetectors = (detectors: Detector[]) => { - return detectors.filter( - (detector) => detector.detectionDateRange !== undefined - ); -}; - -// Currently: detector w/ no detection date range is considered a 'realtime' detector -export const getRealtimeDetectors = (detectors: Detector[]) => { - return detectors.filter( - (detector) => detector.detectionDateRange === undefined - ); -}; - export const getDetectorTasks = (detectorTaskResponses: any[]) => { const detectorToTaskMap = {} as { [key: string]: any }; detectorTaskResponses.forEach((response) => { @@ -436,3 +425,34 @@ export const getLatestDetectorTasksQuery = () => { export const isRealTimeTask = (task: any) => { return get(task, 'task_type', '').includes(REALTIME_TASK_TYPE_PREFIX); }; + +export const getFiltersFromEntityList = (entityListAsObj: object) => { + let filters = [] as any[]; + Object.values(entityListAsObj).forEach((entity: Entity) => { + filters.push({ + nested: { + path: ENTITY_FIELD, + query: { + term: { + [ENTITY_NAME_PATH_FIELD]: { + value: entity.name, + }, + }, + }, + }, + }); + filters.push({ + nested: { + path: ENTITY_FIELD, + query: { + term: { + [ENTITY_VALUE_PATH_FIELD]: { + value: entity.value, + }, + }, + }, + }, + }); + }); + return filters; +}; diff --git a/server/utils/constants.ts b/server/utils/constants.ts index 1aa14e2f..df46f837 100644 --- a/server/utils/constants.ts +++ b/server/utils/constants.ts @@ -101,9 +101,12 @@ export enum SAMPLE_TYPE { export const ENTITY_FIELD = 'entity'; export const ENTITY_VALUE_PATH_FIELD = 'entity.value'; export const ENTITY_NAME_PATH_FIELD = 'entity.name'; +export const MODEL_ID_FIELD = 'model_id'; export const DOC_COUNT_FIELD = 'doc_count'; export const KEY_FIELD = 'key'; +export const ENTITY_LIST_FIELD = 'entity_list'; +export const ENTITY_LIST_DELIMITER = '
'; export const STACK_TRACE_PATTERN = '.java:'; export const OPENSEARCH_EXCEPTION_PREFIX =