From 2e7a3fc96a15de1448c4691419ea4ef7ffa9cf57 Mon Sep 17 00:00:00 2001
From: Yizhe Liu <59710443+yizheliu-amazon@users.noreply.github.com>
Date: Mon, 20 Jul 2020 10:58:19 -0700
Subject: [PATCH] Add progress bar for initialization (#253)

---
 public/models/interfaces.ts                   |  6 ++
 .../containers/DetectorDetail.tsx             | 13 +--
 public/pages/DetectorDetail/utils/helpers.ts  |  2 +-
 .../containers/AnomalyResults.tsx             | 79 ++++++++++++++-----
 public/redux/reducers/ad.ts                   |  1 +
 server/cluster/ad/adPlugin.ts                 |  2 +-
 server/routes/ad.ts                           | 25 +++---
 server/routes/utils/adHelpers.ts              | 19 ++++-
 8 files changed, 105 insertions(+), 42 deletions(-)

diff --git a/public/models/interfaces.ts b/public/models/interfaces.ts
index d5207b2e..6f295143 100644
--- a/public/models/interfaces.ts
+++ b/public/models/interfaces.ts
@@ -86,6 +86,11 @@ export type UiMetaData = {
   };
 };
 
+export type InitProgress = {
+  percentageStr: string;
+  estimatedMinutesLeft: number;
+  neededShingles: number;
+};
 export type Detector = {
   primaryTerm: number;
   seqNo: number;
@@ -105,6 +110,7 @@ export type Detector = {
   disabledTime?: number;
   curState: DETECTOR_STATE;
   stateError: string;
+  initProgress?: InitProgress;
 };
 
 export type DetectorListItem = {
diff --git a/public/pages/DetectorDetail/containers/DetectorDetail.tsx b/public/pages/DetectorDetail/containers/DetectorDetail.tsx
index 1d4f7665..d561a54a 100644
--- a/public/pages/DetectorDetail/containers/DetectorDetail.tsx
+++ b/public/pages/DetectorDetail/containers/DetectorDetail.tsx
@@ -290,7 +290,10 @@ export const DetectorDetail = (props: DetectorDetailProps) => {
                   ) : detector.enabled &&
                     detector.curState === DETECTOR_STATE.INIT ? (
                     <EuiHealth color={DETECTOR_STATE_COLOR.INIT}>
-                      Initializing
+                      {detector.initProgress
+                        ? //@ts-ignore
+                          `Initializing (${detector.initProgress.percentageStr} complete)`
+                        : 'Initializing'}
                     </EuiHealth>
                   ) : detector.curState === DETECTOR_STATE.INIT_FAILURE ||
                     detector.curState === DETECTOR_STATE.UNEXPECTED_FAILURE ? (
@@ -343,7 +346,7 @@ export const DetectorDetail = (props: DetectorDetailProps) => {
           <EuiFlexGroup>
             <EuiFlexItem>
               <EuiTabs>
-                {tabs.map(tab => (
+                {tabs.map((tab) => (
                   <EuiTab
                     onClick={() => {
                       handleTabChange(tab.route);
@@ -388,7 +391,7 @@ export const DetectorDetail = (props: DetectorDetailProps) => {
                   <EuiFieldText
                     fullWidth={true}
                     placeholder="delete"
-                    onChange={e => {
+                    onChange={(e) => {
                       if (e.target.value === 'delete') {
                         setDetectorDetailModel({
                           ...detectorDetailModel,
@@ -472,7 +475,7 @@ export const DetectorDetail = (props: DetectorDetailProps) => {
         <Route
           exact
           path="/detectors/:detectorId/results"
-          render={props => (
+          render={(props) => (
             <AnomalyResults
               {...props}
               detectorId={detectorId}
@@ -484,7 +487,7 @@ export const DetectorDetail = (props: DetectorDetailProps) => {
         <Route
           exact
           path="/detectors/:detectorId/configurations"
-          render={props => (
+          render={(props) => (
             <DetectorConfig
               {...props}
               detectorId={detectorId}
diff --git a/public/pages/DetectorDetail/utils/helpers.ts b/public/pages/DetectorDetail/utils/helpers.ts
index 6136d9c3..0bbe7bc2 100644
--- a/public/pages/DetectorDetail/utils/helpers.ts
+++ b/public/pages/DetectorDetail/utils/helpers.ts
@@ -17,7 +17,7 @@ import { DETECTOR_INIT_FAILURES } from './constants';
 
 export const getInitFailureMessageAndActionItem = (error: string): object => {
   const failureDetails = Object.values(DETECTOR_INIT_FAILURES);
-  const failureDetail = failureDetails.find(failure =>
+  const failureDetail = failureDetails.find((failure) =>
     error.includes(failure.keyword)
   );
   if (!failureDetail) {
diff --git a/public/pages/DetectorResults/containers/AnomalyResults.tsx b/public/pages/DetectorResults/containers/AnomalyResults.tsx
index 3d5b88d4..49e34bb4 100644
--- a/public/pages/DetectorResults/containers/AnomalyResults.tsx
+++ b/public/pages/DetectorResults/containers/AnomalyResults.tsx
@@ -21,6 +21,10 @@ import {
   EuiSpacer,
   EuiCallOut,
   EuiButton,
+  EuiProgress,
+  EuiFlexGroup,
+  EuiFlexItem,
+  EuiText,
 } from '@elastic/eui';
 import { get } from 'lodash';
 import React, { useEffect, Fragment, useState } from 'react';
@@ -110,6 +114,13 @@ export function AnomalyResults(props: AnomalyResultsProps) {
   const monitors = useSelector((state: AppState) => state.alerting.monitors);
   const monitor = get(monitors, `${detectorId}.0`);
 
+  const [featureMissingSeverity, setFeatureMissingSeverity] = useState<
+    MISSING_FEATURE_DATA_SEVERITY
+  >();
+
+  const [featureNamesAtHighSev, setFeatureNamesAtHighSev] = useState(
+    [] as string[]
+  );
   const isDetectorRunning =
     detector && detector.curState === DETECTOR_STATE.RUNNING;
 
@@ -127,9 +138,16 @@ export function AnomalyResults(props: AnomalyResultsProps) {
   const isDetectorInitializing =
     detector && detector.curState === DETECTOR_STATE.INIT;
 
-  const initializationInfo = getDetectorInitializationInfo(detector);
+  const isDetectorMissingData = featureMissingSeverity
+    ? (isDetectorInitializing || isDetectorRunning) &&
+      featureMissingSeverity > MISSING_FEATURE_DATA_SEVERITY.GREEN
+    : undefined;
 
-  const isInitOvertime = get(initializationInfo, IS_INIT_OVERTIME_FIELD, false);
+  const initializationInfo = featureMissingSeverity
+    ? getDetectorInitializationInfo(detector)
+    : undefined;
+
+  const isInitOvertime = get(initializationInfo, IS_INIT_OVERTIME_FIELD);
   const initDetails = get(initializationInfo, INIT_DETAILS_FIELD, {});
   const initErrorMessage = get(initDetails, INIT_ERROR_MESSAGE_FIELD, '');
   const initActionItem = get(initDetails, INIT_ACTION_ITEM_FIELD, '');
@@ -145,21 +163,9 @@ export function AnomalyResults(props: AnomalyResultsProps) {
     1
   );
 
-  const [featureMissingSeverity, setFeatureMissingSeverity] = useState<
-    MISSING_FEATURE_DATA_SEVERITY
-  >();
-
-  const [featureNamesAtHighSev, setFeatureNamesAtHighSev] = useState(
-    [] as string[]
-  );
-
-  const isDetectorMissingData = featureMissingSeverity
-    ? (isDetectorInitializing || isDetectorRunning) &&
-      featureMissingSeverity > MISSING_FEATURE_DATA_SEVERITY.GREEN
-    : undefined;
-
   const isInitializingNormally =
     isDetectorInitializing &&
+    isInitOvertime != undefined &&
     !isInitOvertime &&
     isDetectorMissingData != undefined &&
     !isDetectorMissingData;
@@ -273,6 +279,12 @@ export function AnomalyResults(props: AnomalyResultsProps) {
     }
   };
 
+  const getInitProgressMessage = () => {
+    return detector && isDetectorInitializing && detector.initProgress
+      ? `The detector needs ${detector.initProgress.estimatedMinutesLeft} minutes for initializing. If your data stream is not continuous, it may take even longer. `
+      : '';
+  };
+
   const getCalloutContent = () => {
     return isDetectorUpdated ? (
       <p>
@@ -281,6 +293,7 @@ export function AnomalyResults(props: AnomalyResultsProps) {
       </p>
     ) : isDetectorMissingData ? (
       <p>
+        {getInitProgressMessage()}
         {get(
           getFeatureDataMissingMessageAndActionItem(
             featureMissingSeverity,
@@ -292,11 +305,11 @@ export function AnomalyResults(props: AnomalyResultsProps) {
       </p>
     ) : isInitializingNormally ? (
       <p>
-        After the initialization is complete, you will see the anomaly results
-        based on your latest configuration changes.
+        {getInitProgressMessage()}After the initialization is complete, you will
+        see the anomaly results based on your latest configuration changes.
       </p>
     ) : isInitOvertime ? (
-      <p>{`${initActionItem}`}</p>
+      <p>{`${getInitProgressMessage()}${initActionItem}`}</p>
     ) : (
       // detector has failure
       <p>{`${get(
@@ -324,6 +337,7 @@ export function AnomalyResults(props: AnomalyResultsProps) {
                   {isDetectorUpdated ||
                   isDetectorMissingData ||
                   isInitializingNormally ||
+                  isInitOvertime ||
                   isDetectorFailed ? (
                     <EuiCallOut
                       title={getCalloutTitle()}
@@ -332,6 +346,35 @@ export function AnomalyResults(props: AnomalyResultsProps) {
                       style={{ marginBottom: '20px' }}
                     >
                       {getCalloutContent()}
+                      {isDetectorInitializing && detector.initProgress ? (
+                        <div>
+                          <EuiFlexGroup alignItems="center">
+                            <EuiFlexItem
+                              style={{ maxWidth: '20px', marginRight: '5px' }}
+                            >
+                              <EuiText>
+                                {
+                                  //@ts-ignore
+                                  detector.initProgress.percentageStr
+                                }
+                              </EuiText>
+                            </EuiFlexItem>
+                            <EuiFlexItem>
+                              <EuiProgress
+                                //@ts-ignore
+                                value={detector.initProgress.percentageStr.replace(
+                                  '%',
+                                  ''
+                                )}
+                                max={100}
+                                color="primary"
+                                size="xs"
+                              />
+                            </EuiFlexItem>
+                          </EuiFlexGroup>
+                          <EuiSpacer size="l" />
+                        </div>
+                      ) : null}
                       <EuiButton
                         onClick={props.onSwitchToConfiguration}
                         color={
diff --git a/public/redux/reducers/ad.ts b/public/redux/reducers/ad.ts
index c6d79839..9ff9e9ac 100644
--- a/public/redux/reducers/ad.ts
+++ b/public/redux/reducers/ad.ts
@@ -139,6 +139,7 @@ const reducer = handleActions<Detectors>(
             disabledTime: moment().valueOf(),
             curState: DETECTOR_STATE.DISABLED,
             stateError: '',
+            initProgress: undefined,
           },
         },
       }),
diff --git a/server/cluster/ad/adPlugin.ts b/server/cluster/ad/adPlugin.ts
index 2b16729a..f11ca745 100644
--- a/server/cluster/ad/adPlugin.ts
+++ b/server/cluster/ad/adPlugin.ts
@@ -129,7 +129,7 @@ export default function adPlugin(Client: any, config: any, components: any) {
 
   ad.detectorProfile = ca({
     url: {
-      fmt: `${API.DETECTOR_BASE}/<%=detectorId%>/_profile`,
+      fmt: `${API.DETECTOR_BASE}/<%=detectorId%>/_profile/init_progress,state,error`,
       req: {
         detectorId: {
           type: 'string',
diff --git a/server/routes/ad.ts b/server/routes/ad.ts
index 5e32f2e5..053fce72 100644
--- a/server/routes/ad.ts
+++ b/server/routes/ad.ts
@@ -45,6 +45,7 @@ import {
   getResultAggregationQuery,
   getFinalDetectorStates,
   getDetectorsWithJob,
+  getDetectorInitProgress,
 } from './utils/adHelpers';
 import { set } from 'lodash';
 
@@ -55,7 +56,7 @@ type PutDetectorParams = {
   body: string;
 };
 
-export default function(apiRouter: Router) {
+export default function (apiRouter: Router) {
   apiRouter.post('/detectors', putDetector);
   apiRouter.put('/detectors/{detectorId}', putDetector);
   apiRouter.post('/detectors/_search', searchDetector);
@@ -181,7 +182,6 @@ const getDetector = async (
           detectorId: detectorId,
         }
       );
-
       const detectorStates = getFinalDetectorStates(
         [detectorStateResp],
         [convertDetectorKeysToCamelCase(response.anomaly_detector)]
@@ -196,11 +196,12 @@ const getDetector = async (
       primaryTerm: response._primary_term,
       seqNo: response._seq_no,
       adJob: { ...response.anomaly_detector_job },
-      //@ts-ignore
-      ...(detectorState !== undefined ? { curState: detectorState.state } : {}),
       ...(detectorState !== undefined
-        ? //@ts-ignore
-          { stateError: detectorState.error }
+        ? {
+            curState: detectorState.state,
+            stateError: detectorState.error,
+            initProgress: getDetectorInitProgress(detectorState),
+          }
         : {}),
     };
     return {
@@ -333,10 +334,7 @@ const getDetectors = async (
         query_string: {
           fields: ['name', 'description'],
           default_operator: 'AND',
-          query: `*${search
-            .trim()
-            .split(' ')
-            .join('* *')}*`,
+          query: `*${search.trim().split(' ').join('* *')}*`,
         },
       });
     }
@@ -345,10 +343,7 @@ const getDetectors = async (
         query_string: {
           fields: ['indices'],
           default_operator: 'OR',
-          query: `*${indices
-            .trim()
-            .split(' ')
-            .join('* *')}*`,
+          query: `*${indices.trim().split(' ').join('* *')}*`,
         },
       });
     }
@@ -452,7 +447,7 @@ const getDetectors = async (
     }
 
     // Get detector state as well: loop through the ids to get each detector's state using profile api
-    const allIds = finalDetectors.map(detector => detector.id);
+    const allIds = finalDetectors.map((detector) => detector.id);
 
     const detectorStatePromises = allIds.map(async (id: string) => {
       try {
diff --git a/server/routes/utils/adHelpers.ts b/server/routes/utils/adHelpers.ts
index f1dcf05d..4d9ed312 100644
--- a/server/routes/utils/adHelpers.ts
+++ b/server/routes/utils/adHelpers.ts
@@ -18,6 +18,7 @@ import { AnomalyResults } from 'server/models/interfaces';
 import { GetDetectorsQueryParams } from '../../models/types';
 import { mapKeysDeep, toCamel, toSnake } from '../../utils/helpers';
 import { DETECTOR_STATE } from '../../../public/utils/constants';
+import { InitProgress } from 'public/models/interfaces';
 
 export const convertDetectorKeysToSnakeCase = (payload: any) => {
   return {
@@ -151,12 +152,26 @@ export const anomalyResultMapper = (anomalyResults: any[]): AnomalyResults => {
   return resultData;
 };
 
+export const getDetectorInitProgress = (
+  detectorStateResponse: any
+): InitProgress | undefined => {
+  if (detectorStateResponse.init_progress) {
+    return {
+      percentageStr: detectorStateResponse.init_progress.percentage,
+      estimatedMinutesLeft:
+        detectorStateResponse.init_progress.estimated_minutes_left,
+      neededShingles: detectorStateResponse.init_progress.needed_shingles,
+    };
+  }
+  return undefined;
+};
+
 export const getFinalDetectorStates = (
   detectorStateResponses: any[],
   finalDetectors: any[]
 ) => {
   let finalDetectorStates = cloneDeep(detectorStateResponses);
-  finalDetectorStates.forEach(detectorState => {
+  finalDetectorStates.forEach((detectorState) => {
     //@ts-ignore
     detectorState.state = DETECTOR_STATE[detectorState.state];
   });
@@ -198,7 +213,7 @@ export const getDetectorsWithJob = (
 ): any[] => {
   const finalDetectorsWithJobResponses = cloneDeep(detectorsWithJobResponses);
   const resultDetectorWithJobs = [] as any[];
-  finalDetectorsWithJobResponses.forEach(detectorWithJobResponse => {
+  finalDetectorsWithJobResponses.forEach((detectorWithJobResponse) => {
     const resp = {
       ...detectorWithJobResponse.anomaly_detector,
       id: detectorWithJobResponse._id,