Explainability: Fairness and Bias Metrics (Phase 0) (opendatahub-io#1001

) (opendatahub-io#1006) (opendatahub-io#1007) (opendatahub-io#1008) - Initial feature set for TrustyAI related UI functionality - Adds tab based navigation to modelServing screen - Adds a bias metrics tab with charts for visualising SPD and DIR metrics - Enhances prometheus query features for accessing TrustyAI data - Enhacements to MetricsChart component making it more configurable
alexcreasy · Apr 21, 2023 · d4f90f6 · d4f90f6
1 parent dfe7ecf
commit d4f90f6
Show file tree

Hide file tree

Showing 22 changed files with 499 additions and 46 deletions.
diff --git a/frontend/src/api/prometheus/serving.ts b/frontend/src/api/prometheus/serving.ts
@@ -5,7 +5,9 @@ import {
   RuntimeMetricType,
 } from '~/pages/modelServing/screens/metrics/ModelServingMetricsContext';
 import { MetricType, TimeframeTitle } from '~/pages/modelServing/screens/types';
-import useQueryRangeResourceData from './useQueryRangeResourceData';
+import useQueryRangeResourceData, {
+  useQueryRangeResourceDataTrusty,
+} from './useQueryRangeResourceData';
 
 export const useModelServingMetrics = (
   type: MetricType,
@@ -61,6 +63,20 @@ export const useModelServingMetrics = (
     timeframe,
   );
 
+  const inferenceTrustyAISPD = useQueryRangeResourceDataTrusty(
+    type === 'inference',
+    queries[InferenceMetricType.TRUSTY_AI_SPD],
+    end,
+    timeframe,
+  );
+
+  const inferenceTrustyAIDIR = useQueryRangeResourceDataTrusty(
+    type === 'inference',
+    queries[InferenceMetricType.TRUSTY_AI_DIR],
+    end,
+    timeframe,
+  );
+
   React.useEffect(() => {
     setLastUpdateTime(Date.now());
     // re-compute lastUpdateTime when data changes
@@ -87,6 +103,8 @@ export const useModelServingMetrics = (
         [RuntimeMetricType.MEMORY_UTILIZATION]: runtimeMemoryUtilization,
         [InferenceMetricType.REQUEST_COUNT_SUCCESS]: inferenceRequestSuccessCount,
         [InferenceMetricType.REQUEST_COUNT_FAILED]: inferenceRequestFailedCount,
+        [InferenceMetricType.TRUSTY_AI_SPD]: inferenceTrustyAISPD,
+        [InferenceMetricType.TRUSTY_AI_DIR]: inferenceTrustyAIDIR,
       },
       refresh: refreshAllMetrics,
     }),
@@ -97,6 +115,8 @@ export const useModelServingMetrics = (
       runtimeMemoryUtilization,
       inferenceRequestSuccessCount,
       inferenceRequestFailedCount,
+      inferenceTrustyAISPD,
+      inferenceTrustyAIDIR,
       refreshAllMetrics,
     ],
   );

diff --git a/frontend/src/api/prometheus/usePrometheusQueryRange.ts b/frontend/src/api/prometheus/usePrometheusQueryRange.ts
@@ -1,19 +1,27 @@
 import * as React from 'react';
 import axios from 'axios';
-import { PrometheusQueryRangeResponse, PrometheusQueryRangeResultValue } from '~/types';
+
 import useFetchState, { FetchState, FetchStateCallbackPromise } from '~/utilities/useFetchState';
+import {
+  PrometheusQueryRangeResponse,
+  PrometheusQueryRangeResponseData,
+  PrometheusQueryRangeResultValue,
+} from '~/types';
+
+export type ResponsePredicate<T = PrometheusQueryRangeResultValue> = (
+  data: PrometheusQueryRangeResponseData,
+) => T[];
 
-const usePrometheusQueryRange = (
+const usePrometheusQueryRange = <T = PrometheusQueryRangeResultValue>(
   active: boolean,
   apiPath: string,
   queryLang: string,
   span: number,
   endInMs: number,
   step: number,
-): FetchState<PrometheusQueryRangeResultValue[]> => {
-  const fetchData = React.useCallback<
-    FetchStateCallbackPromise<PrometheusQueryRangeResultValue[]>
-  >(() => {
+  responsePredicate?: ResponsePredicate<T>,
+): FetchState<T[]> => {
+  const fetchData = React.useCallback<FetchStateCallbackPromise<T[]>>(() => {
     const endInS = endInMs / 1000;
     const start = endInS - span;
 
@@ -22,10 +30,18 @@ const usePrometheusQueryRange = (
         query: `query=${queryLang}&start=${start}&end=${endInS}&step=${step}`,
       })
 
-      .then((response) => response.data?.response.data.result?.[0]?.values || []);
-  }, [queryLang, apiPath, span, endInMs, step]);
+      .then((response) => {
+        let result: T[] | PrometheusQueryRangeResultValue[];
+        if (responsePredicate) {
+          result = responsePredicate(response.data?.response.data);
+        } else {
+          result = response.data?.response.data.result?.[0]?.values || [];
+        }
+        return result as T[];
+      });
+  }, [endInMs, span, apiPath, queryLang, step, responsePredicate]);
 
-  return useFetchState<PrometheusQueryRangeResultValue[]>(fetchData, []);
+  return useFetchState<T[]>(fetchData, []);
 };
 
 export default usePrometheusQueryRange;
diff --git a/frontend/src/api/prometheus/useQueryRangeResourceData.ts b/frontend/src/api/prometheus/useQueryRangeResourceData.ts
@@ -1,8 +1,13 @@
+import * as React from 'react';
 import { TimeframeStep, TimeframeTimeRange } from '~/pages/modelServing/screens/const';
 import { TimeframeTitle } from '~/pages/modelServing/screens/types';
-import { ContextResourceData, PrometheusQueryRangeResultValue } from '~/types';
+import {
+  ContextResourceData,
+  PrometheusQueryRangeResponseDataResult,
+  PrometheusQueryRangeResultValue,
+} from '~/types';
 import { useContextResourceData } from '~/utilities/useContextResourceData';
-import usePrometheusQueryRange from './usePrometheusQueryRange';
+import usePrometheusQueryRange, { ResponsePredicate } from './usePrometheusQueryRange';
 
 const useQueryRangeResourceData = (
   /** Is the query active -- should we be fetching? */
@@ -23,4 +28,30 @@ const useQueryRangeResourceData = (
     5 * 60 * 1000,
   );
 
+type TrustyData = PrometheusQueryRangeResponseDataResult;
+
+export const useQueryRangeResourceDataTrusty = (
+  /** Is the query active -- should we be fetching? */
+  active: boolean,
+  query: string,
+  end: number,
+  timeframe: TimeframeTitle,
+): ContextResourceData<TrustyData> => {
+  const responsePredicate = React.useCallback<ResponsePredicate<TrustyData>>(
+    (data) => data.result,
+    [],
+  );
+  return useContextResourceData<TrustyData>(
+    usePrometheusQueryRange<TrustyData>(
+      active,
+      '/api/prometheus/serving',
+      query,
+      TimeframeTimeRange[timeframe],
+      end,
+      TimeframeStep[timeframe],
+      responsePredicate,
+    ),
+    5 * 60 * 1000,
+  );
+};
 export default useQueryRangeResourceData;
diff --git a/frontend/src/pages/modelServing/ModelServingRoutes.tsx b/frontend/src/pages/modelServing/ModelServingRoutes.tsx
@@ -13,7 +13,7 @@ const ModelServingRoutes: React.FC = () => {
       <Route path="/" element={<ModelServingContextProvider />}>
         <Route index element={<ModelServingGlobal />} />
         <Route
-          path="/metrics/:project/:inferenceService"
+          path="/metrics/:project/:inferenceService/:tab?"
           element={
             modelMetricsEnabled ? <GlobalInferenceMetricsWrapper /> : <Navigate replace to="/" />
           }

diff --git a/frontend/src/pages/modelServing/screens/const.ts b/frontend/src/pages/modelServing/screens/const.ts
@@ -127,6 +127,7 @@ export const DEFAULT_MODEL_SERVING_TEMPLATE: ServingRuntimeKind = {
  * Unit is in seconds
  */
 export const TimeframeTimeRange: TimeframeTimeType = {
+  [TimeframeTitle.FIFTEEN_MINUTES]: 15 * 60,
   [TimeframeTitle.ONE_HOUR]: 60 * 60,
   [TimeframeTitle.ONE_DAY]: 24 * 60 * 60,
   [TimeframeTitle.ONE_WEEK]: 7 * 24 * 60 * 60,
@@ -143,6 +144,7 @@ export const TimeframeTimeRange: TimeframeTimeType = {
  *   86,400 / (24 * 12) => 300 points of prometheus data
  */
 export const TimeframeStep: TimeframeStepType = {
+  [TimeframeTitle.FIFTEEN_MINUTES]: 3,
   [TimeframeTitle.ONE_HOUR]: 12,
   [TimeframeTitle.ONE_DAY]: 24 * 12,
   [TimeframeTitle.ONE_WEEK]: 7 * 24 * 12,

diff --git a/frontend/src/pages/modelServing/screens/metrics/BiasTab.tsx b/frontend/src/pages/modelServing/screens/metrics/BiasTab.tsx
@@ -0,0 +1,25 @@
+import React from 'react';
+import { PageSection, Stack, StackItem } from '@patternfly/react-core';
+import DIRGraph from '~/pages/modelServing/screens/metrics/DIRChart';
+import MetricsPageToolbar from './MetricsPageToolbar';
+import SPDChart from './SPDChart';
+
+const BiasTab = () => (
+  <Stack>
+    <StackItem>
+      <MetricsPageToolbar />
+    </StackItem>
+    <PageSection isFilled>
+      <Stack hasGutter>
+        <StackItem>
+          <SPDChart />
+        </StackItem>
+        <StackItem>
+          <DIRGraph />
+        </StackItem>
+      </Stack>
+    </PageSection>
+  </Stack>
+);
+
+export default BiasTab;
diff --git a/frontend/src/pages/modelServing/screens/metrics/DIRChart.tsx b/frontend/src/pages/modelServing/screens/metrics/DIRChart.tsx
@@ -0,0 +1,40 @@
+import React from 'react';
+import { Stack, StackItem } from '@patternfly/react-core';
+import { InferenceMetricType } from '~/pages/modelServing/screens/metrics/ModelServingMetricsContext';
+import TrustyChart from '~/pages/modelServing/screens/metrics/TrustyChart';
+import { DomainCalculator, MetricsChartTypes } from '~/pages/modelServing/screens/metrics/types';
+
+const DIRChart = () => {
+  const DEFAULT_MAX_THRESHOLD = 1.2;
+  const DEFAULT_MIN_THRESHOLD = 0.8;
+
+  const domainCalc: DomainCalculator = (maxYValue) => ({
+    y: maxYValue > 1.2 ? [0, maxYValue + 0.1] : [0, 1.3],
+  });
+
+  return (
+    <TrustyChart
+      title="Disparate Impact Ratio"
+      abbreviation="DIR"
+      metricType={InferenceMetricType.TRUSTY_AI_DIR}
+      tooltip={
+        <Stack hasGutter>
+          <StackItem>
+            Disparate Impact Ratio (DIR) measures imbalances in classifications by calculating the
+            ratio between the proportion of the majority and protected classes getting a particular
+            outcome.
+          </StackItem>
+          <StackItem>
+            Typically, the further away the DIR is from 1, the more unfair the model. A DIR equal to
+            1 indicates a perfectly fair model for the groups and outcomes in question.
+          </StackItem>
+        </Stack>
+      }
+      domain={domainCalc}
+      thresholds={[DEFAULT_MAX_THRESHOLD, DEFAULT_MIN_THRESHOLD]}
+      type={MetricsChartTypes.LINE}
+    />
+  );
+};
+
+export default DIRChart;
diff --git a/frontend/src/pages/modelServing/screens/metrics/MetricsChart.tsx b/frontend/src/pages/modelServing/screens/metrics/MetricsChart.tsx
@@ -1,18 +1,23 @@
 import * as React from 'react';
 import {
   Card,
+  CardActions,
   CardBody,
+  CardHeader,
   CardTitle,
   EmptyState,
   EmptyStateIcon,
   Spinner,
   Title,
+  Toolbar,
+  ToolbarContent,
 } from '@patternfly/react-core';
 import {
   Chart,
   ChartArea,
   ChartAxis,
   ChartGroup,
+  ChartLine,
   ChartThemeColor,
   ChartThreshold,
   ChartVoronoiContainer,
@@ -21,46 +26,67 @@ import {
 import { CubesIcon } from '@patternfly/react-icons';
 import { TimeframeTimeRange } from '~/pages/modelServing/screens/const';
 import { ModelServingMetricsContext } from './ModelServingMetricsContext';
-import { MetricChartLine, ProcessedMetrics } from './types';
+import {
+  DomainCalculator,
+  MetricChartLine,
+  MetricChartThreshold,
+  MetricsChartTypes,
+  ProcessedMetrics,
+} from './types';
 import {
   convertTimestamp,
+  createGraphMetricLine,
   formatToShow,
   getThresholdData,
-  createGraphMetricLine,
   useStableMetrics,
 } from './utils';
 
+const defaultDomainCalculator: DomainCalculator = (maxYValue) => ({
+  y: maxYValue === 0 ? [0, 1] : [0, maxYValue],
+});
+
 type MetricsChartProps = {
   title: string;
   color?: string;
   metrics: MetricChartLine;
-  threshold?: number;
+  thresholds?: MetricChartThreshold[];
+  domain?: DomainCalculator;
+  toolbar?: React.ReactElement<typeof ToolbarContent>;
+  type?: MetricsChartTypes;
 };
-
 const MetricsChart: React.FC<MetricsChartProps> = ({
   title,
   color,
   metrics: unstableMetrics,
-  threshold,
+  thresholds = [],
+  domain = defaultDomainCalculator,
+  toolbar,
+  type = MetricsChartTypes.AREA,
 }) => {
   const bodyRef = React.useRef<HTMLDivElement>(null);
   const [chartWidth, setChartWidth] = React.useState(0);
   const { currentTimeframe, lastUpdateTime } = React.useContext(ModelServingMetricsContext);
   const metrics = useStableMetrics(unstableMetrics, title);
 
-  const { data: graphLines, maxYValue } = React.useMemo(
+  const {
+    data: graphLines,
+    maxYValue,
+    minYValue,
+  } = React.useMemo(
     () =>
       metrics.reduce<ProcessedMetrics>(
         (acc, metric) => {
           const lineValues = createGraphMetricLine(metric);
           const newMaxValue = Math.max(...lineValues.map((v) => v.y));
+          const newMinValue = Math.min(...lineValues.map((v) => v.y));
 
           return {
             data: [...acc.data, lineValues],
             maxYValue: Math.max(acc.maxYValue, newMaxValue),
+            minYValue: Math.min(acc.minYValue, newMinValue),
           };
         },
-        { data: [], maxYValue: 0 },
+        { data: [], maxYValue: 0, minYValue: 0 },
       ),
     [metrics],
   );
@@ -94,7 +120,14 @@ const MetricsChart: React.FC<MetricsChartProps> = ({
 
   return (
     <Card>
-      <CardTitle>{title}</CardTitle>
+      <CardHeader>
+        <CardTitle>{title}</CardTitle>
+        {toolbar && (
+          <CardActions>
+            <Toolbar>{toolbar}</Toolbar>
+          </CardActions>
+        )}
+      </CardHeader>
       <CardBody style={{ height: hasSomeData ? 400 : 200, padding: 0 }}>
         <div ref={bodyRef}>
           {hasSomeData ? (
@@ -106,7 +139,7 @@ const MetricsChart: React.FC<MetricsChartProps> = ({
                   constrainToVisibleArea
                 />
               }
-              domain={{ y: maxYValue === 0 ? [0, 1] : [0, maxYValue + 1] }}
+              domain={domain(maxYValue, minYValue)}
               height={400}
               width={chartWidth}
               padding={{ left: 70, right: 50, bottom: 70, top: 50 }}
@@ -123,11 +156,25 @@ const MetricsChart: React.FC<MetricsChartProps> = ({
               />
               <ChartAxis dependentAxis tickCount={10} fixLabelOverlap />
               <ChartGroup>
-                {graphLines.map((line, i) => (
-                  <ChartArea key={i} data={line} />
-                ))}
+                {graphLines.map((line, i) => {
+                  switch (type) {
+                    case MetricsChartTypes.AREA:
+                      return <ChartArea key={i} data={line} />;
+                      break;
+                    case MetricsChartTypes.LINE:
+                      return <ChartLine key={i} data={line} />;
+                      break;
+                  }
+                })}
               </ChartGroup>
-              {threshold && <ChartThreshold data={getThresholdData(graphLines, threshold)} />}
+              {thresholds.map((t, i) => (
+                <ChartThreshold
+                  key={i}
+                  data={getThresholdData(graphLines, t.value)}
+                  style={t.color ? { data: { stroke: t.color } } : undefined}
+                  name={t.label}
+                />
+              ))}
             </Chart>
           ) : (
             <EmptyState>