Add feature attribution (opensearch-project#296)

Signed-off-by: Jackie Han <[email protected]>
jackiehanyang · Aug 5, 2022 · d973b8e · d973b8e
1 parent 6f736d7
commit d973b8e
Show file tree

Hide file tree

Showing 9 changed files with 239 additions and 40 deletions.
diff --git a/public/models/interfaces.ts b/public/models/interfaces.ts
@@ -234,14 +234,23 @@ export type AnomalyData = {
   plotTime?: number;
   entity?: EntityData[];
   features?: { [key: string]: FeatureAggregationData };
-  aggInterval?: string;
+  contributions?: { [key: string]: FeatureContributionData };
+  aggInterval?: string; 
 };
 
 export type FeatureAggregationData = {
   data: number;
+  name?: string;
   endTime: number;
   startTime: number;
   plotTime?: number;
+  attribution?: number;
+  expectedValue?: number;
+};
+
+export type FeatureContributionData = {
+  name: string;
+  attribution: number;
 };
 
 export type Anomalies = {
@@ -279,6 +288,7 @@ export type AnomalySummary = {
   minConfidence: number;
   maxConfidence: number;
   lastAnomalyOccurrence: string;
+  contributions?: string;
 };
 
 export type DateRange = {

diff --git a/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx b/public/pages/AnomalyCharts/components/FeatureChart/FeatureChart.tsx
@@ -205,6 +205,7 @@ export const FeatureChart = (props: FeatureChartProps) => {
            * and thus show different annotations per feature chart (currently all annotations
            * shown equally across all enabled feature charts for a given detector).
            */}
+
           {props.feature.featureEnabled ? (
             <RectAnnotation
               dataValues={flattenData(props.annotations)}
@@ -257,13 +258,14 @@ export const FeatureChart = (props: FeatureChartProps) => {
           }
           {featureData.map(
             (featureTimeSeries: FeatureAggregationData[], index) => {
+              const timeSeriesList: any[] = [];
               const seriesKey = props.isHCDetector
                 ? `${props.featureDataSeriesName} (${convertToEntityString(
                     props.entityData[index],
                     ', '
                   )})`
                 : props.featureDataSeriesName;
-              return (
+                timeSeriesList.push(
                 <LineSeries
                   id={seriesKey}
                   name={seriesKey}
@@ -278,9 +280,29 @@ export const FeatureChart = (props: FeatureChartProps) => {
                   yAccessors={[CHART_FIELDS.DATA]}
                   data={featureTimeSeries}
                 />
-              );
+              )
+              if (featureTimeSeries.map(
+                (item: FeatureAggregationData) => {
+                  if(item.hasOwnProperty('expectedValue')) {
+                    timeSeriesList.push(
+                      <LineSeries
+                        id={"ExpectedValue"}
+                        name={"Expected Value"}
+                        color={"#0475a2"}
+                        xScaleType={ScaleType.Time}
+                        yScaleType={ScaleType.Linear}
+                        xAccessor={CHART_FIELDS.PLOT_TIME}
+                        yAccessors={[CHART_FIELDS.EXPECTED_VALUE]}
+                        data={featureTimeSeries}
+                      />
+                    )
+                  }
+                }
+              ))
+              return timeSeriesList;
             }
           )}
+
         </Chart>
         {showCustomExpression ? (
           <CodeModal

diff --git a/public/pages/AnomalyCharts/containers/AnomalyDetailsChart.tsx b/public/pages/AnomalyCharts/containers/AnomalyDetailsChart.tsx
@@ -18,6 +18,7 @@ import {
   niceTimeFormatter,
   Position,
   RectAnnotation,
+  RectAnnotationDatum,
   ScaleType,
   Settings,
   XYBrushArea,
@@ -29,8 +30,9 @@ import {
   EuiLoadingChart,
   EuiStat,
   EuiButtonGroup,
+  EuiText,
 } from '@elastic/eui';
-import { get } from 'lodash';
+import { forEach, get } from 'lodash';
 import moment from 'moment';
 import React, { useEffect, useState } from 'react';
 import { useDispatch, useSelector } from 'react-redux';
@@ -41,6 +43,7 @@ import {
   Monitor,
   MonitorAlert,
   AnomalyData,
+  Anomalies,
 } from '../../../models/interfaces';
 import { AppState } from '../../../redux/reducers';
 import { searchAlerts } from '../../../redux/reducers/alerting';
@@ -52,6 +55,8 @@ import {
   getHistoricalAggQuery,
   parseHistoricalAggregatedAnomalies,
   convertToEntityString,
+  flattenData,
+  generateAnomalyAnnotations,
 } from '../../utils/anomalyResultUtils';
 import { AlertsFlyout } from '../components/AlertsFlyout/AlertsFlyout';
 import {
@@ -155,7 +160,6 @@ export const AnomalyDetailsChart = React.memo(
 
     const taskId = get(props, 'detector.taskId');
     const taskState = get(props, 'detector.taskState');
-
     const isRequestingAnomalyResults = useSelector(
       (state: AppState) => state.anomalyResults.requesting
     );
@@ -344,6 +348,69 @@ export const AnomalyDetailsChart = React.memo(
       zoomRange.endDate,
     ]);
 
+
+    const customAnomalyContributionTooltip = (details?: string) => {
+      const anomaly = details ? JSON.parse(details) : undefined;
+      const contributionData = get(anomaly, `contributions`, [])  
+
+      const featureData = get(anomaly, `features`, {}) 
+      let featureAttributionList = [] as any[];
+      if (Array.isArray(contributionData)) {
+        contributionData.map((contribution: any) => {
+          const featureName = get(get(featureData, contribution.feature_id, ""), "name", "") 
+          const dataString = (contribution.data * 100) + "%"
+          featureAttributionList.push(
+            <div>
+              {featureName}: {dataString} <br />
+            </div>
+          )
+        })
+      } else {
+        for (const [, value] of Object.entries(contributionData)) {
+          featureAttributionList.push(
+            <div>
+              {value.name}: {value.attribution} <br />
+            </div>
+          )
+        }
+      }
+      return (
+        <div>
+          <EuiText size="xs">
+            <EuiIcon type="list" /> <b>Feature Contribution: </b>
+            {anomaly ? (
+            <p>
+              <hr style={{ color: '#E0E0E0' }}></hr>
+              {featureAttributionList}
+            </p>
+          ) : null}
+          </EuiText>
+        </div>
+      );
+    };
+
+
+    const generateContributionAnomalyAnnotations = (
+      anomalies: AnomalyData[][]
+    ): any[][] => {
+      let annotations = [] as any[];
+      anomalies.forEach((anomalyTimeSeries: AnomalyData[]) => {
+        annotations.push(
+          anomalyTimeSeries
+            .filter((anomaly: AnomalyData) => anomaly.anomalyGrade > 0)
+            .map((anomaly: AnomalyData) => (              
+              {
+              coordinates: {
+                x0: anomaly.startTime,
+                x1: anomaly.endTime + (anomaly.endTime - anomaly.startTime),
+              },
+              details: `${JSON.stringify(anomaly)}`
+            }))
+        );
+      });
+      return annotations;
+    };
+
     const isLoading =
       props.isLoading || isLoadingAlerts || isRequestingAnomalyResults;
     const isInitializingHistorical = taskState === DETECTOR_STATE.INIT;
@@ -531,7 +598,18 @@ export const AnomalyDetailsChart = React.memo(
                       }}
                     />
                   )}
-
+                  <RectAnnotation
+                    dataValues={flattenData(generateContributionAnomalyAnnotations(zoomedAnomalies))}
+                    id="featureAttributionAnnotation"
+                    style={{
+                      stroke: CHART_COLORS.ANOMALY_GRADE_COLOR,
+                      strokeWidth: 1,
+                      opacity: 0.1,
+                      fill: CHART_COLORS.ANOMALY_GRADE_COLOR,
+                    }}
+                    renderTooltip={customAnomalyContributionTooltip}
+                  />  
+
                   {alertAnnotations ? (
                     <LineAnnotation
                       id="alertAnnotation"
@@ -603,32 +681,31 @@ export const AnomalyDetailsChart = React.memo(
                             get(anomalySeries, '0.entity', []),
                             ', '
                           )})`
-                        : props.anomalyGradeSeriesName;
-
-                      return (
-                        <LineSeries
-                          id={seriesKey}
-                          name={seriesKey}
-                          color={
-                            multipleTimeSeries
-                              ? ENTITY_COLORS[index]
-                              : CHART_COLORS.ANOMALY_GRADE_COLOR
-                          }
-                          data={anomalySeries}
-                          xScaleType={
-                            showAggregateResults
-                              ? ScaleType.Ordinal
-                              : ScaleType.Time
-                          }
-                          yScaleType={ScaleType.Linear}
-                          xAccessor={
-                            showAggregateResults
-                              ? CHART_FIELDS.AGG_INTERVAL
-                              : CHART_FIELDS.PLOT_TIME
-                          }
-                          yAccessors={[CHART_FIELDS.ANOMALY_GRADE]}
-                        />
-                      );
+                        : props.anomalyGradeSeriesName; 
+                        return (
+                          <LineSeries
+                            id={seriesKey}
+                            name={seriesKey}
+                            color={
+                              multipleTimeSeries
+                                ? ENTITY_COLORS[index]
+                                : CHART_COLORS.ANOMALY_GRADE_COLOR
+                            }
+                            data={anomalySeries}
+                            xScaleType={
+                              showAggregateResults
+                                ? ScaleType.Ordinal
+                                : ScaleType.Time
+                            }
+                            yScaleType={ScaleType.Linear}
+                            xAccessor={
+                              showAggregateResults
+                                ? CHART_FIELDS.AGG_INTERVAL
+                                : CHART_FIELDS.PLOT_TIME
+                            }
+                            yAccessors={[CHART_FIELDS.ANOMALY_GRADE]}
+                          />
+                        )
                     }
                   )}
                 </Chart>

diff --git a/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx b/public/pages/AnomalyCharts/utils/anomalyChartUtils.tsx
@@ -640,7 +640,9 @@ export const getFeatureBreakdownWording = (
   return isNotSample ? 'Feature breakdown' : 'Sample feature breakdown';
 };
 
-export const getFeatureDataWording = (isNotSample: boolean | undefined) => {
+export const getFeatureDataWording = (
+  isNotSample: boolean | undefined
+) => {
   return isNotSample ? 'Feature output' : 'Sample feature output';
 };
 

diff --git a/public/pages/AnomalyCharts/utils/constants.ts b/public/pages/AnomalyCharts/utils/constants.ts
@@ -24,11 +24,13 @@ export enum CHART_FIELDS {
   CONFIDENCE = 'confidence',
   DATA = 'data',
   AGG_INTERVAL = 'aggInterval',
+  EXPECTED_VALUE = 'expectedValue'
 }
 
 export enum CHART_COLORS {
   ANOMALY_GRADE_COLOR = '#D13212',
   FEATURE_DATA_COLOR = '#16191F',
+  FEATURE_COLOR = '#fcd529',
   CONFIDENCE_COLOR = '#017F75',
   LIGHT_BACKGROUND = '#FFFFFF',
   DARK_BACKGROUND = '#1D1E24',
@@ -89,7 +91,7 @@ export const DEFAULT_ANOMALY_SUMMARY = {
   maxAnomalyGrade: 0,
   minConfidence: 0,
   maxConfidence: 0,
-  lastAnomalyOccurrence: '-',
+  lastAnomalyOccurrence: '-'
 };
 
 export const HEATMAP_CHART_Y_AXIS_WIDTH = 30;
diff --git a/public/pages/DetectorResults/containers/AnomalyHistory.tsx b/public/pages/DetectorResults/containers/AnomalyHistory.tsx
@@ -136,7 +136,7 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => {
     endDate: initialEndDate,
   });
   const [selectedTabId, setSelectedTabId] = useState<string>(
-    ANOMALY_HISTORY_TABS.ANOMALY_OCCURRENCE
+    ANOMALY_HISTORY_TABS.FEATURE_BREAKDOWN
   );
   const [isLoadingAnomalyResults, setIsLoadingAnomalyResults] =
     useState<boolean>(false);
@@ -425,7 +425,6 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => {
             setIsLoadingAnomalyResults(false);
             errorFetchingResults = true;
           });
-
       const rawAnomaliesData = get(detectorResultResponse, 'response', []);
       const rawAnomaliesResult = {
         anomalies: get(rawAnomaliesData, 'results', []),
@@ -610,6 +609,7 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => {
         fetchBucketizedEntityAnomalyData(entityLists);
       } else {
         fetchAllEntityAnomalyData(dateRange, entityLists);
+        // no visit
         setBucketizedAnomalyResults(undefined);
       }
     } catch (err) {
@@ -807,13 +807,13 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => {
 
   const tabs = [
     {
-      id: ANOMALY_HISTORY_TABS.ANOMALY_OCCURRENCE,
-      name: 'Anomaly occurrences',
+      id: ANOMALY_HISTORY_TABS.FEATURE_BREAKDOWN,
+      name: 'Feature breakdown',
       disabled: false,
     },
     {
-      id: ANOMALY_HISTORY_TABS.FEATURE_BREAKDOWN,
-      name: 'Feature breakdown',
+      id: ANOMALY_HISTORY_TABS.ANOMALY_OCCURRENCE,
+      name: 'Anomaly occurrences',
       disabled: false,
     },
   ];