Fix missing data call out if window delay is large (opensearch-projec…

…t#278) * Fix missing data call out if window delay is large The missing feature callout only shows when fetching feature data points based on the interval and time range, and if there are any empty points Previously, when I configured the window delay to 20 minutes, the missing data call out will show. The reason is that the window delay offset isn't being considered in the time range used in the feature fetching query, or in the displayed chart, such that it shows as empty, when it's actually expected to be empty based on the window delay.This PR fixed the bug by considering window delay in missing feature calculations. Testing done: 1. done e2e testing and verified the problem is fixed. 2. added unit tests. Signed-off-by: Kaituo Li <[email protected]>
jackiehanyang · Jun 20, 2022 · 9d34ba2 · 9d34ba2
1 parent 13c0962
commit 9d34ba2
Show file tree

Hide file tree

Showing 8 changed files with 523 additions and 20 deletions.
diff --git a/opensearch_dashboards.json b/opensearch_dashboards.json
@@ -1,7 +1,7 @@
 {
   "id": "anomalyDetectionDashboards",
-  "version": "2.0.0.0",
-  "opensearchDashboardsVersion": "2.0.0",
+  "version": "2.0.2.0",
+  "opensearchDashboardsVersion": "2.0.2",
   "configPath": ["anomaly_detection_dashboards"],
   "requiredPlugins": ["navigation"],
   "optionalPlugins": [],

diff --git a/package.json b/package.json
@@ -1,10 +1,10 @@
 {
   "name": "anomaly-detection-dashboards",
-  "version": "2.0.0.0",
+  "version": "2.0.2.0",
   "description": "OpenSearch Anomaly Detection Dashboards Plugin",
   "main": "index.js",
   "config": {
-    "plugin_version": "2.0.0.0",
+    "plugin_version": "2.0.2.0",
     "plugin_name": "anomalyDetectionDashboards",
     "plugin_zip_name": "anomaly-detection-dashboards"
   },

diff --git a/public/models/interfaces.ts b/public/models/interfaces.ts
@@ -12,6 +12,8 @@
 import { InitProgress } from '../../server/models/interfaces';
 import { DATA_TYPES } from '../utils/constants';
 import { DETECTOR_STATE } from '../../server/utils/constants';
+import { Duration } from 'moment';
+import moment from 'moment';
 
 export type FieldInfo = {
   label: string;
@@ -78,8 +80,82 @@ export type FeatureAttributes = {
   aggregationQuery: { [key: string]: any };
 };
 
+// all possible valid units accepted by the backend
 export enum UNITS {
+  NANOS = "Nanos",
+  MICROS = "Micros",
+  MILLIS = "Millis",
+  SECONDS = "Seconds",
   MINUTES = 'Minutes',
+  HOURS = "Hours",
+  HALF_DAYS = "HalfDays",
+  DAYS = "Days",
+  WEEKS = "Weeks",
+  MONTHS = "Months",
+  YEARS = "Years",
+  DECADES = "Decades",
+  CENTURIES = "Centuries",
+  MILLENNIA = "Millennia",
+  ERAS = "Eras",
+  FOREVER = "Forever"
+}
+
+// cannot create a method in enum, have to write function separately
+export function toDuration(units: UNITS): Duration {
+  switch(units) {
+    case UNITS.NANOS: {
+      // Duration in moment library does not support
+      return moment.duration(0.000000001, 'seconds');
+    }
+    case UNITS.MICROS: {
+      return moment.duration(0.000001, 'seconds');
+    }
+    case UNITS.MILLIS: {
+      return moment.duration(0.001, 'seconds');
+    }
+    case UNITS.SECONDS: {
+      return moment.duration(1, 'seconds');
+    }
+    case UNITS.MINUTES: {
+      return moment.duration(60, 'seconds');
+    }
+    case UNITS.HOURS: {
+      return moment.duration(3600, 'seconds');
+    }
+    case UNITS.HALF_DAYS: {
+      return moment.duration(43200, 'seconds');
+    }
+    case UNITS.DAYS: {
+      return moment.duration(86400, 'seconds');
+    }
+    case UNITS.WEEKS: {
+      return moment.duration(7 * 86400, 'seconds');
+    }
+    case UNITS.MONTHS: {
+      return moment.duration(31556952 / 12, 'seconds');
+    }
+    case UNITS.YEARS: {
+      return moment.duration(31556952, 'seconds');
+    }
+    case UNITS.DECADES: {
+      return moment.duration(31556952 * 10, 'seconds');
+    }
+    case UNITS.CENTURIES: {
+      return moment.duration(31556952 * 100, 'seconds');
+    }
+    case UNITS.MILLENNIA: {
+      return moment.duration(31556952 * 1000, 'seconds');
+    }
+    case UNITS.ERAS: {
+      return moment.duration(31556952 * 1000000000, 'seconds');
+    }
+    case UNITS.FOREVER: {
+      return moment.duration(Number.MAX_VALUE, 'seconds');
+    }
+    default:
+      break;
+  }
+  throw new Error("Unexpected unit: " + units);
 }
 
 export type Schedule = {

diff --git a/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx b/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx
@@ -68,6 +68,7 @@ interface FeatureChartProps {
   rawFeatureData: FeatureAggregationData[][];
   entityData: EntityData[][];
   isHCDetector?: boolean;
+  windowDelay: Schedule;
 }
 
 export const FeatureChart = (props: FeatureChartProps) => {
@@ -228,11 +229,14 @@ export const FeatureChart = (props: FeatureChartProps) => {
                       ? flattenData(props.rawFeatureData)
                       : flattenData(props.featureData),
                     props.detectorInterval.interval,
+                    props.windowDelay,
                     getFeatureMissingAnnotationDateRange(
                       props.dateRange,
                       props.detectorEnabledTime
                     ),
-                    props.dateRange
+                    props.dateRange,
+                    // date range is selected by customer in UX so window delay time is not considered
+                    false
                   )}
                   marker={<EuiIcon type="alert" />}
                   style={{

diff --git a/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx b/public/pages/AnomalyCharts/containers/FeatureBreakDown.tsx
@@ -27,6 +27,7 @@ import {
   FEATURE_TYPE,
   FeatureAggregationData,
   EntityData,
+  UNITS,
 } from '../../../models/interfaces';
 import { NoFeaturePrompt } from '../components/FeatureChart/NoFeaturePrompt';
 import { focusOnFeatureAccordion } from '../../ConfigureModel/utils/helpers';
@@ -175,6 +176,11 @@ export const FeatureBreakDown = React.memo((props: FeatureBreakDownProps) => {
               detectorEnabledTime={props.detector.enabledTime}
               entityData={getEntityDataForChart(props.anomalyAndFeatureResults)}
               isHCDetector={props.isHCDetector}
+              windowDelay={
+                get(props, `detector.windowDelay.period`,  {
+                  period: { interval: 0, unit: UNITS.MINUTES },
+                })
+              }
             />
             {index + 1 ===
             get(props, 'detector.featureAttributes', []).length ? null : (

diff --git a/public/pages/DetectorResults/containers/AnomalyResults.tsx b/public/pages/DetectorResults/containers/AnomalyResults.tsx
@@ -53,7 +53,7 @@ import {
   getDetectorStateDetails,
 } from '../../DetectorDetail/utils/helpers';
 import moment from 'moment';
-import { DateRange } from '../../../models/interfaces';
+import { DateRange, UNITS, toDuration } from '../../../models/interfaces';
 import {
   getFeatureDataPointsForDetector,
   buildParamsForGetAnomalyResultsWithDateRange,
@@ -204,10 +204,29 @@ export function AnomalyResults(props: AnomalyResultsProps) {
 
   const isHCDetector = !isEmpty(get(detector, 'categoryField', []));
 
+  // ran during componentDidMount and componentDidUpdate
   const checkLatestFeatureDataPoints = async () => {
+    let windowDelayInMinutes = 0;
+    if (detector.windowDelay !== undefined) {
+      const windowDelay = detector.windowDelay.period;
+      const windowDelayUnit = get(windowDelay, 'unit',  UNITS.MINUTES);
+
+      // current time minus window delay
+      const windowDelayInterval = get(windowDelay, 'interval',  0);
+      windowDelayInMinutes = windowDelayInterval * toDuration(windowDelayUnit).asMinutes();
+    }
+
+    // The query in this function uses data start/end time. So we should consider window delay
+    let adjustedCurrentTime = moment().subtract(
+      windowDelayInMinutes,
+      'minutes'
+    );;
+
+    // check from FEATURE_DATA_POINTS_WINDOW + FEATURE_DATA_CHECK_WINDOW_OFFSET (currently 5) intervals to now
     const featureDataPointsRange = {
       startDate: Math.max(
-        moment()
+        // clone since subtract mutates the original moment that we need to use as endData later
+        adjustedCurrentTime.clone()
           .subtract(
             (FEATURE_DATA_POINTS_WINDOW + FEATURE_DATA_CHECK_WINDOW_OFFSET) *
               detectorIntervalInMin,
@@ -217,7 +236,7 @@ export function AnomalyResults(props: AnomalyResultsProps) {
         //@ts-ignore
         detector.enabledTime
       ),
-      endDate: moment().valueOf(),
+      endDate: adjustedCurrentTime.valueOf(),
     } as DateRange;
 
     const params = buildParamsForGetAnomalyResultsWithDateRange(
@@ -239,7 +258,8 @@ export function AnomalyResults(props: AnomalyResultsProps) {
         detector,
         featuresData,
         detectorIntervalInMin,
-        featureDataPointsRange
+        featureDataPointsRange,
+        true
       );
 
       const featureMissingSeveritiesMap =