From 9db643a718cb61668d6b6f467aebc7bd4c137fcb Mon Sep 17 00:00:00 2001 From: Yaliang <49084640+ylwu-amzn@users.noreply.github.com> Date: Tue, 28 Apr 2020 21:50:23 -0700 Subject: [PATCH] modify get ad result to return correct anomaly/feature data (#57) --- server/models/types.ts | 28 +++++++++- server/routes/ad.ts | 88 ++++++++++++++++++++++++-------- server/routes/utils/adHelpers.ts | 12 ++--- server/utils/helpers.ts | 11 ++++ 4 files changed, 109 insertions(+), 30 deletions(-) diff --git a/server/models/types.ts b/server/models/types.ts index 009d7480..6c9f58ee 100644 --- a/server/models/types.ts +++ b/server/models/types.ts @@ -88,26 +88,50 @@ export type GetDetectorsQueryParams = { sortField: string; }; +export type GetAdMonitorsQueryParams = { + from: number; + size: number; + search: string; + indices?: string; + sortDirection: SORT_DIRECTION; + sortField: string; +}; + export type DetectorResultsQueryParams = { from: number; size: number; sortDirection: SORT_DIRECTION; sortField: string; - range?: object; + dateRangeFilter?: DateRangeFilter; }; export type AnomalyResult = { startTime: number; endTime: number; - grade: number; + plotTime: number; + anomalyGrade: number; confidence: number; }; +export type FeatureResult = { + startTime: number; + endTime: number; + plotTime: number; + data: number; +}; + export type AnomalyResultsResponse = { totalAnomalies: number; results: AnomalyResult[]; + featureResults: { [key: string]: FeatureResult[] }; }; export type ServerResponse = | { ok: false; error: string } | { ok: true; response: T }; + +export type DateRangeFilter = { + startTime?: number; + endTime?: number; + fieldName: string; +} \ No newline at end of file diff --git a/server/routes/ad.ts b/server/routes/ad.ts index 9f3bb306..bdb45dd1 100644 --- a/server/routes/ad.ts +++ b/server/routes/ad.ts @@ -27,10 +27,12 @@ import { Detector, GetDetectorsQueryParams, ServerResponse, + FeatureResult, + DateRangeFilter, } from '../models/types'; import { Router } from '../router'; import { SORT_DIRECTION, AD_DOC_FIELDS } from '../utils/constants'; -import { mapKeysDeep, toCamel, toSnake } from '../utils/helpers'; +import { mapKeysDeep, toCamel, toSnake, toFixedNumber } from '../utils/helpers'; import { anomalyResultMapper, convertDetectorKeysToCamelCase, @@ -38,6 +40,7 @@ import { getResultAggregationQuery, getFinalDetectorStates, } from './utils/adHelpers'; +import { set } from 'lodash'; type PutDetectorParams = { detectorId: string; @@ -472,14 +475,14 @@ const getAnomalyResults = async ( size = 20, sortDirection = SORT_DIRECTION.DESC, sortField = AD_DOC_FIELDS.DATA_START_TIME, - range = undefined, + dateRangeFilter = undefined, //@ts-ignore } = req.query as { from: number; size: number; sortDirection: SORT_DIRECTION; sortField?: string; - range?: any; + dateRangeFilter?: string; }; const { detectorId } = req.params; @@ -500,12 +503,6 @@ const getAnomalyResults = async ( sort = sortQuery; } - let rangeObj = range; - - if (range !== undefined && typeof range === 'string') { - rangeObj = JSON.parse(range); - } - //Preparing search request const requestBody = { sort, @@ -519,42 +516,93 @@ const getAnomalyResults = async ( detector_id: detectorId, }, }, - { ...(rangeObj !== undefined && { range: rangeObj }) }, ], }, }, }; + try { + const dateRangeFilterObj = (dateRangeFilter + ? JSON.parse(dateRangeFilter) + : undefined) as DateRangeFilter; + const filterSize = requestBody.query.bool.filter.length; + if (dateRangeFilterObj && dateRangeFilterObj.fieldName) { + (dateRangeFilterObj.startTime || dateRangeFilterObj.endTime) && + set( + requestBody.query.bool.filter, + `${filterSize}.range.${dateRangeFilterObj.fieldName}.format`, + 'epoch_millis' + ); + + dateRangeFilterObj.startTime && + set( + requestBody.query.bool.filter, + `${filterSize}.range.${dateRangeFilterObj.fieldName}.gte`, + dateRangeFilterObj.startTime + ); + + dateRangeFilterObj.endTime && + set( + requestBody.query.bool.filter, + `${filterSize}.range.${dateRangeFilterObj.fieldName}.lte`, + dateRangeFilterObj.endTime + ); + } + } catch (error) { + console.log('wrong date range filter', error); + } + const response = await callWithRequest(req, 'ad.searchResults', { body: requestBody, }); const totalResults: number = get(response, 'hits.total.value', 0); - // Get all detectors from search detector API - const detectorResults: AnomalyResult[] = get(response, 'hits.hits', []).map( - (result: any) => ({ + + const detectorResult: AnomalyResult[] = []; + const featureResult: { [key: string]: FeatureResult[] } = {}; + get(response, 'hits.hits', []).forEach((result: any) => { + detectorResult.push({ startTime: result._source.data_start_time, endTime: result._source.data_end_time, + plotTime: result._source.data_end_time, confidence: - result._source.confidence != null && result._source.confidence > 0 - ? Number.parseFloat(result._source.confidence).toFixed(3) + result._source.confidence != null && + result._source.confidence !== 'NaN' && + result._source.confidence > 0 + ? toFixedNumber(Number.parseFloat(result._source.confidence)) : 0, anomalyGrade: result._source.anomaly_grade != null && + result._source.anomaly_grade !== 'NaN' && result._source.anomaly_grade > 0 - ? Number.parseFloat(result._source.anomaly_grade).toFixed(3) + ? toFixedNumber(Number.parseFloat(result._source.anomaly_grade)) : 0, - }) - ); + }); + result._source.feature_data.forEach((featureData: any) => { + if (!featureResult[featureData.feature_id]) { + featureResult[featureData.feature_id] = []; + } + featureResult[featureData.feature_id].push({ + startTime: result._source.data_start_time, + endTime: result._source.data_end_time, + plotTime: result._source.data_end_time, + data: + featureData.data != null && featureData.data !== 'NaN' + ? toFixedNumber(Number.parseFloat(featureData.data)) + : 0, + }); + }); + }); return { ok: true, response: { totalAnomalies: totalResults, - results: detectorResults, + results: detectorResult, + featureResults: featureResult, }, }; } catch (err) { - console.log('Anomaly detector - Unable get results', err); + console.log('Anomaly detector - Unable to get results', err); return { ok: false, error: err.message }; } }; diff --git a/server/routes/utils/adHelpers.ts b/server/routes/utils/adHelpers.ts index 9f15d005..fed26b58 100644 --- a/server/routes/utils/adHelpers.ts +++ b/server/routes/utils/adHelpers.ts @@ -129,25 +129,21 @@ export const anomalyResultMapper = (anomalyResults: any[]): AnomalyResults => { ...others, anomalyGrade: rest.anomalyGrade != null && rest.anomalyGrade > 0 - ? Number.parseFloat(rest.anomalyGrade).toFixed(3) + ? Number.parseFloat(rest.anomalyGrade).toFixed(2) : 0, confidence: rest.anomalyGrade != null && rest.anomalyGrade > 0 - ? Number.parseFloat(rest.confidence).toFixed(3) + ? Number.parseFloat(rest.confidence).toFixed(2) : 0, startTime: rest.dataStartTime, endTime: rest.dataEndTime, - plotTime: - rest.dataStartTime + - Math.floor((rest.dataEndTime - rest.dataStartTime) / 2), + plotTime: rest.dataEndTime, }); featureData.forEach((feature: any) => { resultData.featureData[feature.featureId].push({ startTime: rest.dataStartTime, endTime: rest.dataEndTime, - plotTime: - rest.dataStartTime + - Math.floor((rest.dataEndTime - rest.dataStartTime) / 2), + plotTime: rest.dataEndTime, data: feature.data, }); }); diff --git a/server/utils/helpers.ts b/server/utils/helpers.ts index 7dd4ffc8..a0f2e9f8 100644 --- a/server/utils/helpers.ts +++ b/server/utils/helpers.ts @@ -22,6 +22,8 @@ import { snakeCase, } from 'lodash'; +import { MIN_IN_MILLI_SECS } from './constants'; + export function mapKeysDeep(obj: object, fn: any): object | any[] { if (Array.isArray(obj)) { return map(obj, innerObj => mapKeysDeep(innerObj, fn)); @@ -36,3 +38,12 @@ export function mapKeysDeep(obj: object, fn: any): object | any[] { export const toSnake = (value: any, key: string) => snakeCase(key); export const toCamel = (value: any, key: string) => camelCase(key); + +export const getFloorPlotTime = (plotTime: number): number => { + return Math.floor(plotTime / MIN_IN_MILLI_SECS) * MIN_IN_MILLI_SECS; +}; + +export const toFixedNumber = (num: number, digits?: number, base?: number) => { + var pow = Math.pow(base || 10, digits || 2); + return Math.round(num * pow) / pow; +};