From c8db73510cc586d63aeff545c6d65ad210a605a5 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Sat, 12 Dec 2020 13:46:46 +0000 Subject: [PATCH 1/3] Added constants for env --- components/alibi-detect-server/adserver/constants.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/alibi-detect-server/adserver/constants.py b/components/alibi-detect-server/adserver/constants.py index fe2de0311b..63d7ce3a4a 100644 --- a/components/alibi-detect-server/adserver/constants.py +++ b/components/alibi-detect-server/adserver/constants.py @@ -1,5 +1,7 @@ HEADER_RETURN_INSTANCE_SCORE = "Alibi-Detect-Return-Instance-Score" HEADER_RETURN_FEATURE_SCORE = "Alibi-Detect-Return-Feature-Score" +ENV_RETURN_INSTANCE_SCORE = "ALIBI_DETECT_RETURN_INSTANCE_SCORE" +ENV_RETURN_FEATURE_SCORE = "ALIBI_DETECT_RETURN_FEATURE_SCORE" HEADER_OUTLIER_TYPE = "Alibi-Detect-Outlier-Type" REQUEST_ID_HEADER_NAME = "Ce-Requestid" From 04a32bca03526b14fda54c84c39c81e30f5c15b9 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Sat, 12 Dec 2020 13:48:42 +0000 Subject: [PATCH 2/3] Added od metrics to prometheus --- .../alibi-detect-server/adserver/od_model.py | 40 +++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/components/alibi-detect-server/adserver/od_model.py b/components/alibi-detect-server/adserver/od_model.py index c147f5fb8b..aeba4e2b13 100644 --- a/components/alibi-detect-server/adserver/od_model.py +++ b/components/alibi-detect-server/adserver/od_model.py @@ -1,17 +1,43 @@ import json from typing import List, Dict, Optional, Union +import os import logging import kfserving import numpy as np from .numpy_encoder import NumpyEncoder from adserver.base import CEModel from alibi_detect.utils.saving import load_detector, Data +from seldon_core.user_model import SeldonResponse from adserver.constants import ( HEADER_RETURN_INSTANCE_SCORE, HEADER_RETURN_FEATURE_SCORE, + ENV_RETURN_INSTANCE_SCORE, + ENV_RETURN_FEATURE_SCORE, HEADER_OUTLIER_TYPE, ) +RETURN_INSTANCE_SCORE = os.environ.get(ENV_RETURN_INSTANCE_SCORE, "").upper() == "TRUE" +RETURN_FEATURE_SCORE = os.environ.get(ENV_RETURN_FEATURE_SCORE, "").upper() == "TRUE" + + +def _append_outlier_metrcs(metrics, outlier, name): + metric_found = outlier.get("data", {}).get(name) + + # Assumes metric_found is always float/int or list/np.array when not none + if metric_found is not None: + if not isinstance(metric_found, (list, np.ndarray)): + metric_found = [metric_found] + + for i, instance in enumerate(metric_found): + metrics.append( + { + "key": f"seldon_metric_outlier_{name}", + "value": instance, + "type": "GAUGE", + "tags": {"index": str(i)}, + } + ) + class AlibiDetectOutlierModel(CEModel): # pylint:disable=c-extension-no-member def __init__(self, name: str, storage_uri: str, model: Optional[Data] = None): @@ -70,7 +96,7 @@ def process_event(self, inputs: Union[List, Dict], headers: Dict) -> Dict: if ( HEADER_RETURN_INSTANCE_SCORE in headers and headers[HEADER_RETURN_INSTANCE_SCORE] == "true" - ): + ) or RETURN_INSTANCE_SCORE: ret_instance_score = True outlier_type = "instance" @@ -80,7 +106,7 @@ def process_event(self, inputs: Union[List, Dict], headers: Dict) -> Dict: if ( HEADER_RETURN_FEATURE_SCORE in headers and headers[HEADER_RETURN_FEATURE_SCORE] == "true" - ): + ) or RETURN_FEATURE_SCORE: ret_feature_score = True od_preds = {} name = self.model.meta["name"] @@ -105,6 +131,12 @@ def process_event(self, inputs: Union[List, Dict], headers: Dict) -> Dict: # scores used to determine outliers return_instance_score=ret_instance_score, ) + + # Register metrics + metrics = [] + _append_outlier_metrcs(metrics, od_preds, "is_outlier") + _append_outlier_metrcs(metrics, od_preds, "instance_score") + # clean result if ( "data" in od_preds @@ -119,4 +151,6 @@ def process_event(self, inputs: Union[List, Dict], headers: Dict) -> Dict: ): del od_preds["data"]["feature_score"] - return json.loads(json.dumps(od_preds, cls=NumpyEncoder)) + resp_data = json.loads(json.dumps(od_preds, cls=NumpyEncoder)) + + return SeldonResponse(resp_data, None, metrics) From 730ea862d4624fb565c48b2bb12737c8f217e4fc Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Mon, 14 Dec 2020 09:27:37 +0000 Subject: [PATCH 3/3] Added gauge --- components/alibi-detect-server/adserver/od_model.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/alibi-detect-server/adserver/od_model.py b/components/alibi-detect-server/adserver/od_model.py index aeba4e2b13..092decec63 100644 --- a/components/alibi-detect-server/adserver/od_model.py +++ b/components/alibi-detect-server/adserver/od_model.py @@ -20,7 +20,7 @@ RETURN_FEATURE_SCORE = os.environ.get(ENV_RETURN_FEATURE_SCORE, "").upper() == "TRUE" -def _append_outlier_metrcs(metrics, outlier, name): +def _append_outlier_metrcs(metrics, outlier, name, is_count=True): metric_found = outlier.get("data", {}).get(name) # Assumes metric_found is always float/int or list/np.array when not none @@ -33,7 +33,7 @@ def _append_outlier_metrcs(metrics, outlier, name): { "key": f"seldon_metric_outlier_{name}", "value": instance, - "type": "GAUGE", + "type": "COUNTER" if is_count else "GAUGE", "tags": {"index": str(i)}, } ) @@ -135,7 +135,7 @@ def process_event(self, inputs: Union[List, Dict], headers: Dict) -> Dict: # Register metrics metrics = [] _append_outlier_metrcs(metrics, od_preds, "is_outlier") - _append_outlier_metrcs(metrics, od_preds, "instance_score") + _append_outlier_metrcs(metrics, od_preds, "instance_score", is_count=False) # clean result if (