From 20588fc7777c3c5bcbe162e20f10b31bfe367ba6 Mon Sep 17 00:00:00 2001 From: Yizhe Liu Date: Thu, 18 Jun 2020 21:07:46 -0700 Subject: [PATCH 01/10] Add missing feature alert if recent feature data is missing --- .../components/FeatureChart/FeatureChart.tsx | 179 +++++---- .../containers/FeatureBreakDown.tsx | 16 +- .../containers/AnomalyHistory.tsx | 87 +++-- .../containers/AnomalyResults.tsx | 270 ++++++++++--- public/pages/utils/anomalyResultUtils.ts | 369 ++++++++++++++++-- public/utils/constants.ts | 11 + 6 files changed, 749 insertions(+), 183 deletions(-) diff --git a/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx b/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx index 9e3fb131..2a5e9a4f 100644 --- a/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx +++ b/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx @@ -22,8 +22,10 @@ import { Position, Settings, ScaleType, + LineAnnotation, + AnnotationDomainTypes, } from '@elastic/charts'; -import { EuiEmptyPrompt, EuiText, EuiLink, EuiButton } from '@elastic/eui'; +import { EuiText, EuiLink, EuiButton, EuiIcon } from '@elastic/eui'; import React, { useState, Fragment } from 'react'; import ContentPanel from '../../../../components/ContentPanel/ContentPanel'; import { useDelayedLoader } from '../../../../hooks/useDelayedLoader'; @@ -32,14 +34,15 @@ import { FeatureAttributes, DateRange, FEATURE_TYPE, + Schedule, } from '../../../../models/interfaces'; import { darkModeEnabled } from '../../../../utils/kibanaUtils'; -import { prepareDataForChart } from '../../../utils/anomalyResultUtils'; -import { CodeModal } from '../../../DetectorConfig/components/CodeModal/CodeModal'; import { - CHART_FIELDS, - FEATURE_CHART_THEME, -} from '../../utils/constants'; + prepareDataForChart, + getFeatureMissingDataAnnotations, +} from '../../../utils/anomalyResultUtils'; +import { CodeModal } from '../../../DetectorConfig/components/CodeModal/CodeModal'; +import { CHART_FIELDS, FEATURE_CHART_THEME } from '../../utils/constants'; interface FeatureChartProps { feature: FeatureAttributes; @@ -54,6 +57,10 @@ interface FeatureChartProps { featureDataSeriesName: string; edit?: boolean; onEdit?(): void; + detectorInterval: Schedule; + showFeatureMissingDataPointAnnotation?: boolean; + detectorEnabledTime?: number; + rawFeatureData: FeatureAggregationData[]; } const getDisabledChartBackground = () => darkModeEnabled() ? '#25262E' : '#F0F0F0'; @@ -108,6 +115,24 @@ export const FeatureChart = (props: FeatureChartProps) => { ); const featureData = prepareDataForChart(props.featureData, props.dateRange); + + // return undefined if featureMissingDataPointAnnotationStartDate is missing + const getFeatureMissingAnnotationDateRange = ( + dateRange: DateRange, + featureMissingDataPointAnnotationStartDate?: number + ) => { + if (!featureMissingDataPointAnnotationStartDate) { + return undefined; + } + return { + startDate: Math.max( + dateRange.startDate, + featureMissingDataPointAnnotationStartDate + ), + endDate: dateRange.endDate, + }; + }; + return ( { props.edit ? Edit : null } > - {props.featureData.length > 0 ? ( -
- - - {props.feature.featureEnabled ? ( - - ) : null} - - - - - {showCustomExpression ? ( - true} - closeModal={() => setShowCustomExpression(false)} +
+ + + {props.feature.featureEnabled ? ( + ) : null} -
- ) : ( - -

{`There is no data to display for feature ${props.feature.featureName}`}

- - } - /> - )} + {props.feature.featureEnabled && + props.showFeatureMissingDataPointAnnotation && + props.detectorEnabledTime + ? [ + } + style={{ + line: { stroke: 'red', strokeWidth: 1, opacity: 0.8 }, + }} + />, + ] + : null} + + + + + {showCustomExpression ? ( + true} + closeModal={() => setShowCustomExpression(false)} + /> + ) : null} +
); }; diff --git a/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx b/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx index a755e2c7..b5c21132 100644 --- a/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx +++ b/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx @@ -35,6 +35,8 @@ interface FeatureBreakDownProps { isLoading: boolean; dateRange: DateRange; featureDataSeriesName: string; + showFeatureMissingDataPointAnnotation?: boolean; + rawAnomalyResults?: Anomalies; } export const FeatureBreakDown = React.memo((props: FeatureBreakDownProps) => { @@ -46,7 +48,7 @@ export const FeatureBreakDown = React.memo((props: FeatureBreakDownProps) => {

{props.title}

 - + ) : null} @@ -61,6 +63,11 @@ export const FeatureBreakDown = React.memo((props: FeatureBreakDownProps) => { `anomaliesResult.featureData.${feature.featureId}`, [] )} + rawFeatureData={get( + props, + `rawAnomalyResults.featureData.${feature.featureId}`, + [] + )} annotations={props.annotations} isLoading={props.isLoading} dateRange={props.dateRange} @@ -97,8 +104,13 @@ export const FeatureBreakDown = React.memo((props: FeatureBreakDownProps) => { onEdit={() => { focusOnFeatureAccordion(index); }} + detectorInterval={props.detector.detectionInterval.period} + showFeatureMissingDataPointAnnotation={ + props.showFeatureMissingDataPointAnnotation + } + detectorEnabledTime={props.detector.enabledTime} /> - + ) )} diff --git a/public/pages/DetectorResults/containers/AnomalyHistory.tsx b/public/pages/DetectorResults/containers/AnomalyHistory.tsx index f4604c25..1fd87fb4 100644 --- a/public/pages/DetectorResults/containers/AnomalyHistory.tsx +++ b/public/pages/DetectorResults/containers/AnomalyHistory.tsx @@ -22,8 +22,7 @@ import { EuiLoadingSpinner, } from '@elastic/eui'; import moment from 'moment'; -import { useSelector, useDispatch } from 'react-redux'; -import { AppState } from 'public/redux/reducers'; +import { useDispatch } from 'react-redux'; import { AnomalyData, Detector, @@ -33,13 +32,13 @@ import { Anomalies, } from '../../../models/interfaces'; import { - getAnomalyResultsWithDateRange, filterWithDateRange, getAnomalySummaryQuery, getBucketizedAnomalyResultsQuery, parseBucketizedAnomalyResults, parseAnomalySummary, parsePureAnomalies, + buildParamsForGetAnomalyResultsWithDateRange, } from '../../utils/anomalyResultUtils'; import { get } from 'lodash'; import { AnomalyResultsTable } from './AnomalyResultsTable'; @@ -51,6 +50,7 @@ import { searchES } from '../../../redux/reducers/elasticsearch'; import { MIN_IN_MILLI_SECS } from '../../../../server/utils/constants'; import { INITIAL_ANOMALY_SUMMARY } from '../../AnomalyCharts/utils/constants'; import { MAX_ANOMALIES } from '../../../utils/constants'; +import { getDetectorResults } from '../../../redux/reducers/anomalyResults'; interface AnomalyHistoryProps { detector: Detector; @@ -60,9 +60,8 @@ interface AnomalyHistoryProps { export const AnomalyHistory = (props: AnomalyHistoryProps) => { const dispatch = useDispatch(); - const isLoading = useSelector( - (state: AppState) => state.anomalyResults.requesting - ); + + const [isLoading, setIsLoading] = useState(false); const initialStartDate = moment().subtract(7, 'days'); const initialEndDate = moment(); const [dateRange, setDateRange] = useState({ @@ -124,31 +123,69 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { ); } finally { setIsLoadingAnomalyResults(false); + fetchRawAnomalyResults(false); } } if ( dateRange.endDate - dateRange.startDate > - get(props.detector, 'detectionInterval.period.interval', 1) * - MIN_IN_MILLI_SECS * - MAX_ANOMALIES + detectorInterval * MIN_IN_MILLI_SECS * MAX_ANOMALIES ) { getBucketizedAnomalyResults(); } else { setBucketizedAnomalyResults(undefined); - getAnomalyResultsWithDateRange( - dispatch, - dateRange.startDate, - dateRange.endDate, - props.detector.id - ); + fetchRawAnomalyResults(true); } }, [dateRange]); - const atomicAnomalyResults = useSelector( - (state: AppState) => state.anomalyResults + const detectorInterval = get( + props.detector, + 'detectionInterval.period.interval', + 1 ); + const fetchRawAnomalyResults = async ( + shouldSetAtomicAnomalyResults: boolean + ) => { + if (shouldSetAtomicAnomalyResults) { + setIsLoading(true); + } + + try { + const params = buildParamsForGetAnomalyResultsWithDateRange( + dateRange.startDate - detectorInterval * MIN_IN_MILLI_SECS, + dateRange.endDate + ); + const detectorResultResponse = await dispatch( + getDetectorResults(props.detector.id, params) + ); + const anomaliesData = get(detectorResultResponse, 'data.response', []); + + if (shouldSetAtomicAnomalyResults) { + setAtomicAnomalyResults({ + anomalies: get(anomaliesData, 'results', []), + featureData: get(anomaliesData, 'featureResults', []), + }); + } + setRawAnomalyResults({ + anomalies: get(anomaliesData, 'results', []), + featureData: get(anomaliesData, 'featureResults', []), + }); + } catch (err) { + console.error( + `Failed to get atomic anomaly results for ${props.detector.id}`, + err + ); + } finally { + if (shouldSetAtomicAnomalyResults) { + setIsLoading(false); + } + } + }; + + const [atomicAnomalyResults, setAtomicAnomalyResults] = useState(); + const [rawAnomalyResults, setRawAnomalyResults] = useState(); + const anomalyResults = bucketizedAnomalyResults ? bucketizedAnomalyResults : atomicAnomalyResults; @@ -224,7 +261,7 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { dateRange={dateRange} onDateRangeChange={handleDateRangeChange} onZoomRangeChange={handleZoomChange} - anomalies={anomalyResults.anomalies} + anomalies={anomalyResults ? anomalyResults.anomalies : []} bucketizedAnomalies={bucketizedAnomalyResults !== undefined} anomalySummary={bucketizedAnomalySummary} isLoading={isLoading || isLoadingAnomalyResults} @@ -259,20 +296,24 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { detector={props.detector} // @ts-ignore anomaliesResult={anomalyResults} + rawAnomalyResults={rawAnomalyResults} annotations={annotations} isLoading={isLoading} dateRange={zoomRange} featureDataSeriesName="Feature output" + showFeatureMissingDataPointAnnotation={true} /> ) : ( diff --git a/public/pages/DetectorResults/containers/AnomalyResults.tsx b/public/pages/DetectorResults/containers/AnomalyResults.tsx index efb9d8d2..c78ca4fa 100644 --- a/public/pages/DetectorResults/containers/AnomalyResults.tsx +++ b/public/pages/DetectorResults/containers/AnomalyResults.tsx @@ -23,13 +23,18 @@ import { EuiButton, } from '@elastic/eui'; import { get } from 'lodash'; -import React, { useEffect, Fragment } from 'react'; +import React, { useEffect, Fragment, useState } from 'react'; import { useSelector, useDispatch } from 'react-redux'; import { RouteComponentProps } from 'react-router'; //@ts-ignore import chrome from 'ui/chrome'; import { AppState } from '../../../redux/reducers'; -import { BREADCRUMBS, DETECTOR_STATE } from '../../../utils/constants'; +import { + BREADCRUMBS, + DETECTOR_STATE, + FEATURE_DATA_POINTS_WINDOW, + MISSING_FEATURE_DATA_SEVERITY, +} from '../../../utils/constants'; import { AnomalyResultsLiveChart } from './AnomalyResultsLiveChart'; import { AnomalyHistory } from './AnomalyHistory'; import { DetectorStateDetails } from './DetectorStateDetails'; @@ -43,6 +48,15 @@ import { import { getDetector } from '../../../redux/reducers/ad'; import { MIN_IN_MILLI_SECS } from '../../../../server/utils/constants'; import { getInitFailureMessageAndActionItem } from '../../DetectorDetail/utils/helpers'; +import moment from 'moment'; +import { DateRange } from '../../../models/interfaces'; +import { + getFeatureDataPointsForDetector, + buildParamsForGetAnomalyResultsWithDateRange, + getFeatureMissingSeverities, + getFeatureDataMissingMessageAndActionItem, +} from '../../utils/anomalyResultUtils'; +import { getDetectorResults } from '../../../redux/reducers/anomalyResults'; interface AnomalyResultsProps extends RouteComponentProps { detectorId: string; @@ -78,6 +92,20 @@ export function AnomalyResults(props: AnomalyResultsProps) { } }, [detector]); + useEffect(() => { + if ( + detector && + (detector.curState === DETECTOR_STATE.INIT || + detector.curState === DETECTOR_STATE.RUNNING) + ) { + checkLatestFeatureDataPoints(); + const id = setInterval(checkLatestFeatureDataPoints, MIN_IN_MILLI_SECS); + return () => { + clearInterval(id); + }; + } + }, [detector]); + const monitors = useSelector((state: AppState) => state.alerting.monitors); const monitor = get(monitors, `${detectorId}.0`); @@ -105,13 +133,178 @@ export function AnomalyResults(props: AnomalyResultsProps) { const initErrorMessage = get(initDetails, INIT_ERROR_MESSAGE_FIELD, ''); const initActionItem = get(initDetails, INIT_ACTION_ITEM_FIELD, ''); - const isInitializingNormally = isDetectorInitializing && !isInitOvertime; - const isDetectorFailed = detector && (detector.curState === DETECTOR_STATE.INIT_FAILURE || detector.curState === DETECTOR_STATE.UNEXPECTED_FAILURE); + const detectorIntervalInMin = get( + detector, + 'detectionInterval.period.interval', + 1 + ); + + const [featureMissingSeverity, setFeatureMissingSeverity] = useState< + MISSING_FEATURE_DATA_SEVERITY + >(); + + const [featureNamesAtHighSev, setFeatureNamesAtHighSev] = useState( + [] as string[] + ); + + const isDetectorMissingData = featureMissingSeverity + ? (isDetectorInitializing || isDetectorRunning) && + featureMissingSeverity > MISSING_FEATURE_DATA_SEVERITY.GREEN + : undefined; + + const isInitializingNormally = + isDetectorInitializing && !isInitOvertime && !isDetectorMissingData; + + const checkLatestFeatureDataPoints = async () => { + const featureDataPointsRange = { + startDate: Math.max( + moment() + .subtract( + (FEATURE_DATA_POINTS_WINDOW + 1) * detectorIntervalInMin, + 'minutes' + ) + .valueOf(), + //@ts-ignore + detector.enabledTime + ), + endDate: moment().valueOf(), + } as DateRange; + + const params = buildParamsForGetAnomalyResultsWithDateRange( + featureDataPointsRange.startDate, + featureDataPointsRange.endDate + ); + + try { + const detectorResultResponse = await dispatch( + getDetectorResults(detectorId, params) + ); + + const featuresData = get( + detectorResultResponse, + 'data.response.featureResults', + [] + ); + const featureDataPoints = getFeatureDataPointsForDetector( + detector, + featuresData, + detectorIntervalInMin, + featureDataPointsRange + ); + + const featureMissingSeveritiesMap = getFeatureMissingSeverities( + featureDataPoints + ); + let highestSeverity = MISSING_FEATURE_DATA_SEVERITY.GREEN; + let featuresAtHighestSev = [] as string[]; + featureMissingSeveritiesMap.forEach((featureNames, severity, map) => { + if (severity > highestSeverity) { + highestSeverity = severity; + featuresAtHighestSev = featureNames; + } + }); + + setFeatureMissingSeverity(highestSeverity); + setFeatureNamesAtHighSev(featuresAtHighestSev); + } catch (err) { + console.error( + `Failed to get anomaly results for ${detectorId} during getting latest feature data points`, + err + ); + } + }; + + const getCalloutTitle = () => { + if (isDetectorUpdated) { + return 'The detector configuration has changed since it was last stopped.'; + } + if (isDetectorMissingData) { + return get( + getFeatureDataMissingMessageAndActionItem( + featureMissingSeverity, + featureNamesAtHighSev + ), + 'message', + '' + ); + } + if (isInitializingNormally) { + return 'The detector is being initialized based on the latest configuration changes.'; + } + if (isInitOvertime) { + return `Detector initialization is not complete because ${initErrorMessage}.`; + } + if (isDetectorFailed) { + return `The detector is not initialized because ${get( + getInitFailureMessageAndActionItem( + //@ts-ignore + detector.stateError + ), + 'cause', + '' + )}.`; + } + }; + const getCalloutColor = () => { + if ( + isDetectorFailed || + featureMissingSeverity === MISSING_FEATURE_DATA_SEVERITY.RED + ) { + return 'danger'; + } + if ( + isInitOvertime || + isDetectorUpdated || + featureMissingSeverity === MISSING_FEATURE_DATA_SEVERITY.YELLOW + ) { + return 'warning'; + } + if (isInitializingNormally) { + return 'primary'; + } + }; + + const getCalloutContent = () => { + return isDetectorUpdated ? ( +

+ Restart the detector to see accurate anomalies based on configuration + changes. +

+ ) : isDetectorMissingData ? ( +

+ {get( + getFeatureDataMissingMessageAndActionItem( + featureMissingSeverity, + featureNamesAtHighSev + ), + 'actionItem', + '' + )} +

+ ) : isInitializingNormally ? ( +

+ After the initialization is complete, you will see the anomaly results + based on your latest configuration changes. +

+ ) : isInitOvertime ? ( +

{`${initActionItem}`}

+ ) : ( + // detector has failure +

{`${get( + getInitFailureMessageAndActionItem( + //@ts-ignore + detector.stateError + ), + 'actionItem', + '' + )}`}

+ ); + }; return ( @@ -125,70 +318,29 @@ export function AnomalyResults(props: AnomalyResultsProps) { isDetectorFailed ? ( {isDetectorUpdated || - isDetectorInitializing || + (isDetectorInitializing && + isDetectorMissingData != undefined) || isDetectorFailed ? ( - {isDetectorUpdated ? ( -

- Restart the detector to see accurate anomalies based - on configuration changes. -

- ) : isInitializingNormally ? ( -

- After the initialization is complete, you will see the - anomaly results based on your latest configuration - changes. -

- ) : isInitOvertime ? ( -

{`${initActionItem}`}

- ) : ( - // detector has failure -

{`${get( - getInitFailureMessageAndActionItem( - //@ts-ignore - detector.stateError - ), - 'actionItem', - '' - )}`}

- )} + {getCalloutContent()} diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index bb52d647..e7660871 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -22,17 +22,24 @@ import { getDetectorResults } from '../../redux/reducers/anomalyResults'; import { getDetectorLiveResults } from '../../redux/reducers/liveAnomalyResults'; import moment from 'moment'; import { Dispatch } from 'redux'; -import { get } from 'lodash'; +import { get, orderBy, isEmpty } from 'lodash'; import { AnomalyData, DateRange, AnomalySummary, FeatureAggregationData, Anomalies, + Detector, + FeatureAttributes, } from '../../models/interfaces'; -import { MAX_ANOMALIES } from '../../utils/constants'; -import { minuteDateFormatter } from './helpers'; +import { + MAX_ANOMALIES, + MISSING_FEATURE_DATA_SEVERITY, +} from '../../utils/constants'; +import { minuteDateFormatter, dateFormatter } from './helpers'; import { toFixedNumberForAnomaly } from '../../../server/utils/helpers'; +import { getFloorPlotTime } from '../Dashboard/utils/utils'; +import { DETECTOR_INIT_FAILURES } from '../DetectorDetail/utils/constants'; export const getQueryParamsForLiveAnomalyResults = ( detectionInterval: number, @@ -67,13 +74,11 @@ export const getLiveAnomalyResults = ( dispatch(getDetectorLiveResults(detectorId, queryParams)); }; -export const getAnomalyResultsWithDateRange = ( - dispatch: Dispatch, +export const buildParamsForGetAnomalyResultsWithDateRange = ( startTime: number, - endTime: number, - detectorId: string + endTime: number ) => { - const updatedParams = { + return { from: 0, size: MAX_ANOMALIES, sortDirection: SORT_DIRECTION.DESC, @@ -84,21 +89,57 @@ export const getAnomalyResultsWithDateRange = ( fieldName: AD_DOC_FIELDS.DATA_START_TIME, }, }; - dispatch(getDetectorResults(detectorId, updatedParams)); +}; + +export const getAnomalyResultsWithDateRange = ( + dispatch: Dispatch, + startTime: number, + endTime: number, + detectorId: string +) => { + const params = buildParamsForGetAnomalyResultsWithDateRange( + startTime, + endTime + ); + dispatch(getDetectorResults(detectorId, params)); }; const MAX_DATA_POINTS = 1000; +const MAX_FEATURE_ANNOTATIONS = 100; const calculateStep = (total: number): number => { return Math.ceil(total / MAX_DATA_POINTS); }; -// If array size is 100K, `findAnomalyWithMaxAnomalyGrade` -// takes less than 2ms by average, while `Array#reduce` -// takes about 16ms by average and`Array#sort` +const calculateSampleWindowsWithMaxDataPoints = ( + maxDataPoints: number, + dateRange: DateRange +): DateRange[] => { + const resultSampleWindows = [] as DateRange[]; + const rangeInMilliSec = dateRange.endDate - dateRange.startDate; + const windowSizeinMilliSec = Math.max( + Math.ceil(rangeInMilliSec / maxDataPoints), + MIN_IN_MILLI_SECS + ); + for ( + let currentTime = dateRange.startDate; + currentTime <= dateRange.endDate; + currentTime += windowSizeinMilliSec + ) { + resultSampleWindows.push({ + startDate: currentTime, + endDate: Math.min(currentTime + windowSizeinMilliSec, dateRange.endDate), + } as DateRange); + } + return resultSampleWindows; +}; + +// If array size is 100K, `findAnomalyWithMaxAnomalyGrade` +// takes less than 2ms by average, while `Array#reduce` +// takes about 16ms by average and`Array#sort` // takes about 3ms by average. -// If array size is 1M, `findAnomalyWithMaxAnomalyGrade` -// takes less than 6ms by average, while `Array#reduce` +// If array size is 1M, `findAnomalyWithMaxAnomalyGrade` +// takes less than 6ms by average, while `Array#reduce` // takes about 170ms by average and`Array#sort` takes about // 80ms by average. // Considering performance impact, will not change this @@ -161,20 +202,17 @@ export const prepareDataForLiveChart = ( return anomalies; }; -export const prepareDataForChart = ( - data: any[], - dateRange: DateRange, -) => { - if (!data || data.length === 0) { - return []; - } - let anomalies = data.filter( - anomaly => - anomaly.plotTime >= dateRange.startDate && - anomaly.plotTime <= dateRange.endDate - ); - if (anomalies.length > MAX_DATA_POINTS) { - anomalies = sampleMaxAnomalyGrade(anomalies); +export const prepareDataForChart = (data: any[], dateRange: DateRange) => { + let anomalies = []; + if (data && data.length > 0) { + anomalies = data.filter( + anomaly => + anomaly.plotTime >= dateRange.startDate && + anomaly.plotTime <= dateRange.endDate + ); + if (anomalies.length > MAX_DATA_POINTS) { + anomalies = sampleMaxAnomalyGrade(anomalies); + } } anomalies.push({ startTime: dateRange.startDate, @@ -379,7 +417,9 @@ export const parseBucketizedAnomalyResults = (result: any): Anomalies => { const rawAnomaly = get(item, 'top_anomaly_hits.hits.hits.0._source'); if (get(rawAnomaly, 'anomaly_grade') !== undefined) { anomalies.push({ - anomalyGrade: toFixedNumberForAnomaly(get(rawAnomaly, 'anomaly_grade')), + anomalyGrade: toFixedNumberForAnomaly( + get(rawAnomaly, 'anomaly_grade') + ), confidence: toFixedNumberForAnomaly(get(rawAnomaly, 'confidence')), startTime: get(rawAnomaly, 'data_start_time'), endTime: get(rawAnomaly, 'data_end_time'), @@ -483,3 +523,274 @@ export const parsePureAnomalies = ( } return anomalies; }; + +export type FeatureDataPoint = { + isMissing: boolean; + plotTime: number; + startTime: number; + endTime: number; +}; + +export const getFeatureDataPoints = ( + featureData: FeatureAggregationData[], + interval: number, + dateRange: DateRange | undefined +): FeatureDataPoint[] => { + const featureDataPoints = [] as FeatureDataPoint[]; + if (!dateRange) { + return featureDataPoints; + } + const existingTimes = isEmpty(featureData) + ? [] + : featureData + .map(feature => feature.endTime) + .filter(featureTime => featureTime != undefined); + for ( + let currentTime = dateRange.startDate; + currentTime < + getFloorPlotTime(dateRange.endDate - interval * MIN_IN_MILLI_SECS); + currentTime += interval * MIN_IN_MILLI_SECS + ) { + const isExisting = findTimeExistsInWindow( + existingTimes, + currentTime, + currentTime + interval * MIN_IN_MILLI_SECS + ); + featureDataPoints.push({ + isMissing: !isExisting, + plotTime: currentTime + interval * MIN_IN_MILLI_SECS, + startTime: currentTime, + endTime: currentTime + interval * MIN_IN_MILLI_SECS, + }); + } + + return featureDataPoints; +}; + +const findTimeExistsInWindow = ( + timestamps: any[], + startTime: number, + endTime: number +): boolean => { + const result = timestamps.filter( + timestamp => timestamp >= startTime && timestamp <= endTime + ); + return !isEmpty(result); +}; + +const sampleFeatureMissingDataPoints = ( + featureMissingDataPoints: FeatureDataPoint[], + dateRange?: DateRange +): FeatureDataPoint[] => { + if (!dateRange) { + return featureMissingDataPoints; + } + const sampleTimeWindows = calculateSampleWindowsWithMaxDataPoints( + MAX_FEATURE_ANNOTATIONS, + dateRange + ); + + const sampledResults = [] as FeatureDataPoint[]; + for (const timeWindow of sampleTimeWindows) { + const sampledDataPoint = getMiddleDataPoint( + getDataPointsInWindow(featureMissingDataPoints, timeWindow) + ); + if (sampledDataPoint) { + sampledResults.push({ + ...sampledDataPoint, + startTime: Math.max(timeWindow.startDate, sampledDataPoint.startTime), + endTime: Math.max(timeWindow.endDate, sampledDataPoint.endTime), + } as FeatureDataPoint); + } + } + + return sampledResults; +}; + +const getMiddleDataPoint = (arr: any[]) => { + if (arr && arr.length > 0) { + return arr[Math.floor(arr.length / 2)]; + } + return undefined; +}; + +const getDataPointsInWindow = ( + dataPoints: FeatureDataPoint[], + timeWindow: DateRange +) => { + return dataPoints.filter( + dataPoint => + get(dataPoint, 'plotTime', 0) >= timeWindow.startDate && + get(dataPoint, 'plotTime', 0) < timeWindow.endDate + ); +}; + +const generateFeatureMissingAnnotations = ( + featureMissingDataPoints: FeatureDataPoint[] +) => { + return featureMissingDataPoints.map(feature => ({ + dataValue: feature.plotTime, + details: `There is feature data point missing between ${moment( + feature.startTime + ).format('MM/DD/YY h:mm A')} and ${moment(feature.endTime).format( + 'MM/DD/YY h:mm A' + )}`, + header: dateFormatter(feature.plotTime), + })); +}; + +const finalizeFeatureMissingDataAnnotations = ( + featureMissingDataPoints: any[], + dateRange?: DateRange +) => { + const sampledFeatureMissingDataPoints = sampleFeatureMissingDataPoints( + featureMissingDataPoints, + dateRange + ); + + return generateFeatureMissingAnnotations(sampledFeatureMissingDataPoints); +}; + +export const getFeatureMissingDataAnnotations = ( + featureData: FeatureAggregationData[], + interval: number, + queryDateRange?: DateRange, + displayDateRange?: DateRange +) => { + const featureMissingDataPoints = getFeatureDataPoints( + featureData, + interval, + queryDateRange + ).filter(dataPoint => get(dataPoint, 'isMissing', false)); + + const featureMissingAnnotations = finalizeFeatureMissingDataAnnotations( + featureMissingDataPoints, + displayDateRange + ); + return featureMissingAnnotations; +}; + +// returns feature data points(missing/existing both included) for detector in a map like +// { +// 'featureName': data points[] +// } +export const getFeatureDataPointsForDetector = ( + detector: Detector, + featuresData: { [key: string]: FeatureAggregationData[] }, + interval: number, + dateRange?: DateRange +) => { + let featureDataPointsForDetector = {} as { + [key: string]: FeatureDataPoint[]; + }; + + const allFeatures = get( + detector, + 'featureAttributes', + [] as FeatureAttributes[] + ); + allFeatures.forEach(feature => { + //@ts-ignore + const featureData = featuresData[feature.featureId]; + const featureDataPoints = getFeatureDataPoints( + featureData, + interval, + dateRange + ); + + featureDataPointsForDetector = { + ...featureDataPointsForDetector, + [feature.featureName]: featureDataPoints, + }; + }); + return featureDataPointsForDetector; +}; + +export const getFeatureMissingSeverities = (featuresDataPoint: { + [key: string]: FeatureDataPoint[]; +}): Map => { + const featureMissingSeverities = new Map(); + + for (const [featureName, featureDatePoints] of Object.entries( + featuresDataPoint + )) { + // all feature data points should have same length + let featuresWithMissingData = [] as string[]; + if (featureDatePoints.length <= 1) { + // return empty map + return featureMissingSeverities; + } + if ( + featureDatePoints.length === 2 && + featureDatePoints[0].isMissing && + featureDatePoints[1].isMissing + ) { + if (featureMissingSeverities.has(MISSING_FEATURE_DATA_SEVERITY.YELLOW)) { + featuresWithMissingData = featureMissingSeverities.get( + MISSING_FEATURE_DATA_SEVERITY.YELLOW + ); + } + featuresWithMissingData.push(featureName); + featureMissingSeverities.set( + MISSING_FEATURE_DATA_SEVERITY.YELLOW, + featuresWithMissingData + ); + continue; + } + + const orderedFeatureDataPoints = orderBy( + featureDatePoints, + // sort by plot time in desc order + dataPoint => get(dataPoint, 'plotTime', 0), + SORT_DIRECTION.DESC + ); + // feature has >= 3 data points + if ( + orderedFeatureDataPoints.length >= 3 && + orderedFeatureDataPoints[0].isMissing && + orderedFeatureDataPoints[1].isMissing + ) { + // at least latest 2 ones are missing + let currentSeverity = MISSING_FEATURE_DATA_SEVERITY.YELLOW; + if (orderedFeatureDataPoints[2].isMissing) { + // all the latest 3 ones are missing + currentSeverity = MISSING_FEATURE_DATA_SEVERITY.RED; + } + if (featureMissingSeverities.has(currentSeverity)) { + featuresWithMissingData = featureMissingSeverities.get(currentSeverity); + } + featuresWithMissingData.push(featureName); + featureMissingSeverities.set(currentSeverity, featuresWithMissingData); + } + } + + return featureMissingSeverities; +}; + +export const getFeatureDataMissingMessageAndActionItem = ( + featureMissingSev: MISSING_FEATURE_DATA_SEVERITY | undefined, + featuresWithMissingData: string[] +) => { + switch (featureMissingSev) { + case MISSING_FEATURE_DATA_SEVERITY.YELLOW: + return { + message: `Recent data is missing for feature ${featuresWithMissingData.join( + ', ' + )}, anomaly result is missing at the same time because of that.`, + actionItem: + 'Make sure your data is ingested correctly. See the feature data shown below for more details.', + }; + case MISSING_FEATURE_DATA_SEVERITY.RED: + return { + message: `Data ingestion is not going well for feature ${featuresWithMissingData.join( + ', ' + )}, anomaly result is missing at the same time because of that.`, + actionItem: `${DETECTOR_INIT_FAILURES.NO_TRAINING_DATA.actionItem} See the feature data shown below for more details.`, + }; + default: + return { + message: '', + actionItem: '', + }; + } +}; diff --git a/public/utils/constants.ts b/public/utils/constants.ts index 8e796bab..b2ae663b 100644 --- a/public/utils/constants.ts +++ b/public/utils/constants.ts @@ -65,3 +65,14 @@ export const NAME_REGEX = RegExp('^[a-zA-Z0-9._-]+$'); //https://github.com/opendistro-for-elasticsearch/anomaly-detection/blob/master/src/main/java/com/amazon/opendistroforelasticsearch/ad/settings/AnomalyDetectorSettings.java#L186 export const SHINGLE_SIZE = 8; + +export const FEATURE_DATA_POINTS_WINDOW = 3; + +export enum MISSING_FEATURE_DATA_SEVERITY { + // user attention not needed + GREEN = '0', + // needs user attention + YELLOW = '1', + // needs user attention and action + RED = '2', +} From f652ae8d3680039eb728b2491e7cc1b954e3f6e8 Mon Sep 17 00:00:00 2001 From: Yizhe Liu Date: Tue, 7 Jul 2020 18:30:32 -0700 Subject: [PATCH 02/10] Fix bug that wrong missing data point is identified when feature data is a few milli seconds late --- .../components/FeatureChart/FeatureChart.tsx | 22 +++++++++++-------- public/pages/utils/anomalyResultUtils.ts | 19 ++++++++++------ 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx b/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx index 2a5e9a4f..42576f43 100644 --- a/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx +++ b/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx @@ -117,20 +117,24 @@ export const FeatureChart = (props: FeatureChartProps) => { const featureData = prepareDataForChart(props.featureData, props.dateRange); // return undefined if featureMissingDataPointAnnotationStartDate is missing + // OR it is even behind the specified date range const getFeatureMissingAnnotationDateRange = ( dateRange: DateRange, featureMissingDataPointAnnotationStartDate?: number ) => { - if (!featureMissingDataPointAnnotationStartDate) { - return undefined; + if ( + featureMissingDataPointAnnotationStartDate && + dateRange.endDate > featureMissingDataPointAnnotationStartDate + ) { + return { + startDate: Math.max( + dateRange.startDate, + featureMissingDataPointAnnotationStartDate + ), + endDate: dateRange.endDate, + }; } - return { - startDate: Math.max( - dateRange.startDate, - featureMissingDataPointAnnotationStartDate - ), - endDate: dateRange.endDate, - }; + return undefined; }; return ( diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index e7660871..06db47c9 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -123,7 +123,7 @@ const calculateSampleWindowsWithMaxDataPoints = ( ); for ( let currentTime = dateRange.startDate; - currentTime <= dateRange.endDate; + currentTime < dateRange.endDate - windowSizeinMilliSec; currentTime += windowSizeinMilliSec ) { resultSampleWindows.push({ @@ -540,21 +540,22 @@ export const getFeatureDataPoints = ( if (!dateRange) { return featureDataPoints; } + const existingTimes = isEmpty(featureData) ? [] : featureData - .map(feature => feature.endTime) + .map(feature => getFloorPlotTime(feature.startTime)) .filter(featureTime => featureTime != undefined); for ( - let currentTime = dateRange.startDate; + let currentTime = getRoundedTimeInMin(dateRange.startDate); currentTime < - getFloorPlotTime(dateRange.endDate - interval * MIN_IN_MILLI_SECS); + getRoundedTimeInMin(dateRange.endDate - interval * MIN_IN_MILLI_SECS); currentTime += interval * MIN_IN_MILLI_SECS ) { const isExisting = findTimeExistsInWindow( existingTimes, - currentTime, - currentTime + interval * MIN_IN_MILLI_SECS + getRoundedTimeInMin(currentTime), + getRoundedTimeInMin(currentTime) + interval * MIN_IN_MILLI_SECS ); featureDataPoints.push({ isMissing: !isExisting, @@ -573,11 +574,15 @@ const findTimeExistsInWindow = ( endTime: number ): boolean => { const result = timestamps.filter( - timestamp => timestamp >= startTime && timestamp <= endTime + timestamp => timestamp >= startTime && timestamp < endTime ); return !isEmpty(result); }; +const getRoundedTimeInMin = (timestamp: number): number => { + return Math.round(timestamp / MIN_IN_MILLI_SECS) * MIN_IN_MILLI_SECS; +}; + const sampleFeatureMissingDataPoints = ( featureMissingDataPoints: FeatureDataPoint[], dateRange?: DateRange From 71941e3708d9bf0617a670fe9c834a4df406fc28 Mon Sep 17 00:00:00 2001 From: Yizhe Liu Date: Tue, 7 Jul 2020 22:53:24 -0700 Subject: [PATCH 03/10] Fix typo --- public/pages/utils/anomalyResultUtils.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index 06db47c9..9b8cedc2 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -716,19 +716,19 @@ export const getFeatureMissingSeverities = (featuresDataPoint: { }): Map => { const featureMissingSeverities = new Map(); - for (const [featureName, featureDatePoints] of Object.entries( + for (const [featureName, featureDataPoints] of Object.entries( featuresDataPoint )) { // all feature data points should have same length let featuresWithMissingData = [] as string[]; - if (featureDatePoints.length <= 1) { + if (featureDataPoints.length <= 1) { // return empty map return featureMissingSeverities; } if ( - featureDatePoints.length === 2 && - featureDatePoints[0].isMissing && - featureDatePoints[1].isMissing + featureDataPoints.length === 2 && + featureDataPoints[0].isMissing && + featureDataPoints[1].isMissing ) { if (featureMissingSeverities.has(MISSING_FEATURE_DATA_SEVERITY.YELLOW)) { featuresWithMissingData = featureMissingSeverities.get( @@ -744,7 +744,7 @@ export const getFeatureMissingSeverities = (featuresDataPoint: { } const orderedFeatureDataPoints = orderBy( - featureDatePoints, + featureDataPoints, // sort by plot time in desc order dataPoint => get(dataPoint, 'plotTime', 0), SORT_DIRECTION.DESC From 42fd4445469cd8c73ab935e978c3b123ece108d5 Mon Sep 17 00:00:00 2001 From: Yizhe Liu Date: Wed, 8 Jul 2020 10:34:37 -0700 Subject: [PATCH 04/10] Expand annotation window size --- public/pages/utils/anomalyResultUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index 9b8cedc2..a90c70c7 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -603,7 +603,7 @@ const sampleFeatureMissingDataPoints = ( if (sampledDataPoint) { sampledResults.push({ ...sampledDataPoint, - startTime: Math.max(timeWindow.startDate, sampledDataPoint.startTime), + startTime: Math.min(timeWindow.startDate, sampledDataPoint.startTime), endTime: Math.max(timeWindow.endDate, sampledDataPoint.endTime), } as FeatureDataPoint); } From b0ff18f1b8bd4d3a12038d6fb1a4284354202260 Mon Sep 17 00:00:00 2001 From: Yizhe Liu Date: Wed, 8 Jul 2020 16:58:27 -0700 Subject: [PATCH 05/10] Fix wording according to tech writer --- public/pages/utils/anomalyResultUtils.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index a90c70c7..6e848214 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -779,17 +779,21 @@ export const getFeatureDataMissingMessageAndActionItem = ( switch (featureMissingSev) { case MISSING_FEATURE_DATA_SEVERITY.YELLOW: return { - message: `Recent data is missing for feature ${featuresWithMissingData.join( + message: `Recent data is missing for feature${ + featuresWithMissingData.length > 1 ? 's' : '' + }: ${featuresWithMissingData.join( ', ' - )}, anomaly result is missing at the same time because of that.`, + )}. So, anomaly result is missing during this time.`, actionItem: 'Make sure your data is ingested correctly. See the feature data shown below for more details.', }; case MISSING_FEATURE_DATA_SEVERITY.RED: return { - message: `Data ingestion is not going well for feature ${featuresWithMissingData.join( + message: `Data is not being ingested correctly for feature${ + featuresWithMissingData.length > 1 ? 's' : '' + }: ${featuresWithMissingData.join( ', ' - )}, anomaly result is missing at the same time because of that.`, + )}. So, anomaly result is missing during this time.`, actionItem: `${DETECTOR_INIT_FAILURES.NO_TRAINING_DATA.actionItem} See the feature data shown below for more details.`, }; default: From b11b71858487dded8694fe55fcb42d7ed011d220 Mon Sep 17 00:00:00 2001 From: Yizhe Liu Date: Thu, 9 Jul 2020 17:18:55 -0700 Subject: [PATCH 06/10] Add callout saying we show annotation of missing data only after last enabled time; fix some minor bugs --- .../containers/FeatureBreakDown.tsx | 23 +++++++++++++++++-- .../containers/AnomalyHistory.tsx | 10 ++++++-- .../containers/AnomalyResults.tsx | 10 ++++---- public/pages/utils/anomalyResultUtils.ts | 11 ++++++--- 4 files changed, 43 insertions(+), 11 deletions(-) diff --git a/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx b/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx index b5c21132..38c8fe59 100644 --- a/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx +++ b/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx @@ -15,7 +15,13 @@ import React from 'react'; import { get } from 'lodash'; -import { EuiFlexItem, EuiFlexGroup, EuiTitle, EuiSpacer } from '@elastic/eui'; +import { + EuiFlexItem, + EuiFlexGroup, + EuiTitle, + EuiSpacer, + EuiCallOut, +} from '@elastic/eui'; import { FeatureChart } from '../components/FeatureChart/FeatureChart'; import { Detector, @@ -26,6 +32,7 @@ import { } from '../../../models/interfaces'; import { NoFeaturePrompt } from '../components/FeatureChart/NoFeaturePrompt'; import { focusOnFeatureAccordion } from '../../EditFeatures/utils/helpers'; +import moment from 'moment'; interface FeatureBreakDownProps { title?: string; @@ -37,6 +44,7 @@ interface FeatureBreakDownProps { featureDataSeriesName: string; showFeatureMissingDataPointAnnotation?: boolean; rawAnomalyResults?: Anomalies; + isFeatureDataMissing?: boolean; } export const FeatureBreakDown = React.memo((props: FeatureBreakDownProps) => { @@ -52,7 +60,18 @@ export const FeatureBreakDown = React.memo((props: FeatureBreakDownProps) => { ) : null} - + {props.showFeatureMissingDataPointAnnotation && + props.detector.enabledTime && + props.isFeatureDataMissing ? ( + + ) : null} {get(props, 'detector.featureAttributes', []).map( (feature: FeatureAttributes, index: number) => ( diff --git a/public/pages/DetectorResults/containers/AnomalyHistory.tsx b/public/pages/DetectorResults/containers/AnomalyHistory.tsx index 1fd87fb4..9b79b8b5 100644 --- a/public/pages/DetectorResults/containers/AnomalyHistory.tsx +++ b/public/pages/DetectorResults/containers/AnomalyHistory.tsx @@ -39,6 +39,7 @@ import { parseAnomalySummary, parsePureAnomalies, buildParamsForGetAnomalyResultsWithDateRange, + FEATURE_DATA_CHECK_WINDOW_OFFSET, } from '../../utils/anomalyResultUtils'; import { get } from 'lodash'; import { AnomalyResultsTable } from './AnomalyResultsTable'; @@ -56,6 +57,7 @@ interface AnomalyHistoryProps { detector: Detector; monitor: Monitor | undefined; createFeature(): void; + isFeatureDataMissing?: boolean; } export const AnomalyHistory = (props: AnomalyHistoryProps) => { @@ -153,7 +155,10 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { try { const params = buildParamsForGetAnomalyResultsWithDateRange( - dateRange.startDate - detectorInterval * MIN_IN_MILLI_SECS, + dateRange.startDate - + FEATURE_DATA_CHECK_WINDOW_OFFSET * + detectorInterval * + MIN_IN_MILLI_SECS, dateRange.endDate ); const detectorResultResponse = await dispatch( @@ -301,7 +306,8 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { isLoading={isLoading} dateRange={zoomRange} featureDataSeriesName="Feature output" - showFeatureMissingDataPointAnnotation={true} + showFeatureMissingDataPointAnnotation={props.detector.enabled} + isFeatureDataMissing={props.isFeatureDataMissing} /> ) : ( {isDetectorUpdated || - (isDetectorInitializing && - isDetectorMissingData != undefined) || + ((isDetectorInitializing || isDetectorRunning) && + isDetectorMissingData) || isDetectorFailed ? ( props.history.push(`/detectors/${detectorId}/features`) } + isFeatureDataMissing={isDetectorMissingData} /> ) : detector ? ( diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index 6e848214..90d361a4 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -123,7 +123,7 @@ const calculateSampleWindowsWithMaxDataPoints = ( ); for ( let currentTime = dateRange.startDate; - currentTime < dateRange.endDate - windowSizeinMilliSec; + currentTime < dateRange.endDate; currentTime += windowSizeinMilliSec ) { resultSampleWindows.push({ @@ -531,6 +531,8 @@ export type FeatureDataPoint = { endTime: number; }; +export const FEATURE_DATA_CHECK_WINDOW_OFFSET = 2; + export const getFeatureDataPoints = ( featureData: FeatureAggregationData[], interval: number, @@ -549,7 +551,11 @@ export const getFeatureDataPoints = ( for ( let currentTime = getRoundedTimeInMin(dateRange.startDate); currentTime < - getRoundedTimeInMin(dateRange.endDate - interval * MIN_IN_MILLI_SECS); + // skip checking for latest interval as data point for it may not be delivered in time + getRoundedTimeInMin( + dateRange.endDate - + FEATURE_DATA_CHECK_WINDOW_OFFSET * interval * MIN_IN_MILLI_SECS + ); currentTime += interval * MIN_IN_MILLI_SECS ) { const isExisting = findTimeExistsInWindow( @@ -702,7 +708,6 @@ export const getFeatureDataPointsForDetector = ( interval, dateRange ); - featureDataPointsForDetector = { ...featureDataPointsForDetector, [feature.featureName]: featureDataPoints, From 6b19ed141294adf0b4b200a5a81befbe3995b266 Mon Sep 17 00:00:00 2001 From: Yizhe Liu Date: Thu, 9 Jul 2020 17:26:15 -0700 Subject: [PATCH 07/10] Use enabled time to be consistent with detector list page --- public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx b/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx index 38c8fe59..c7a5a4cb 100644 --- a/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx +++ b/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx @@ -64,7 +64,7 @@ export const FeatureBreakDown = React.memo((props: FeatureBreakDownProps) => { props.detector.enabledTime && props.isFeatureDataMissing ? ( Date: Fri, 10 Jul 2020 10:40:02 -0700 Subject: [PATCH 08/10] Modify wording --- public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx b/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx index c7a5a4cb..a09321f0 100644 --- a/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx +++ b/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx @@ -64,7 +64,7 @@ export const FeatureBreakDown = React.memo((props: FeatureBreakDownProps) => { props.detector.enabledTime && props.isFeatureDataMissing ? ( Date: Mon, 13 Jul 2020 11:29:48 -0700 Subject: [PATCH 09/10] use rounded time; use binary search to find appropriate timestamp --- public/pages/utils/anomalyResultUtils.ts | 28 +++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index 90d361a4..acf5a13d 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -546,7 +546,7 @@ export const getFeatureDataPoints = ( const existingTimes = isEmpty(featureData) ? [] : featureData - .map(feature => getFloorPlotTime(feature.startTime)) + .map(feature => getRoundedTimeInMin(feature.startTime)) .filter(featureTime => featureTime != undefined); for ( let currentTime = getRoundedTimeInMin(dateRange.startDate); @@ -579,10 +579,28 @@ const findTimeExistsInWindow = ( startTime: number, endTime: number ): boolean => { - const result = timestamps.filter( - timestamp => timestamp >= startTime && timestamp < endTime - ); - return !isEmpty(result); + // timestamps is in desc order + let result = false; + if (isEmpty(timestamps)) { + return result; + } + + let left = 0; + let right = timestamps.length - 1; + while (left <= right) { + let middle = Math.floor((right + left) / 2); + if (timestamps[middle] >= startTime && timestamps[middle] < endTime) { + result = true; + break; + } + if (timestamps[middle] < startTime) { + right = middle - 1; + } + if (timestamps[middle] >= endTime) { + left = middle + 1; + } + } + return result; }; const getRoundedTimeInMin = (timestamp: number): number => { From 6ac3b5cafbda107639a9d77e82ad0ccfe047d15f Mon Sep 17 00:00:00 2001 From: Yizhe Liu Date: Wed, 15 Jul 2020 10:37:25 -0700 Subject: [PATCH 10/10] Fix minor issue --- .../pages/DetectorResults/containers/AnomalyResults.tsx | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/public/pages/DetectorResults/containers/AnomalyResults.tsx b/public/pages/DetectorResults/containers/AnomalyResults.tsx index da90580b..3d5b88d4 100644 --- a/public/pages/DetectorResults/containers/AnomalyResults.tsx +++ b/public/pages/DetectorResults/containers/AnomalyResults.tsx @@ -159,7 +159,10 @@ export function AnomalyResults(props: AnomalyResultsProps) { : undefined; const isInitializingNormally = - isDetectorInitializing && !isInitOvertime && !isDetectorMissingData; + isDetectorInitializing && + !isInitOvertime && + isDetectorMissingData != undefined && + !isDetectorMissingData; const checkLatestFeatureDataPoints = async () => { const featureDataPointsRange = { @@ -319,8 +322,8 @@ export function AnomalyResults(props: AnomalyResultsProps) { isDetectorFailed ? ( {isDetectorUpdated || - ((isDetectorInitializing || isDetectorRunning) && - isDetectorMissingData) || + isDetectorMissingData || + isInitializingNormally || isDetectorFailed ? (