Merge pull request #3248 from cliveseldon/3247_v2_protocol_explain

Add kfserving protocol to alibi explainer server

components/alibi-explain-server/Makefile

@@ -221,3 +221,32 @@ curl_explain_adult_treeshap:
 cleanup_treeshap:
 	docker rm -f explainer
 
+
+
+#
+# Test Triton Cifar10
+#
+
+test_models/triton/cifar10/tf_cifar10:
+	mkdir -p test_models/triton/tf_cifar10
+	gsutil cp -r gs://seldon-models/triton/tf_cifar10 test_models/triton
+
+
+run_triton_cifar10: test_models/triton/cifar10/tf_cifar10
+	docker run --rm --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 -p9000:9000 -p8001:8001 -p8002:8002 -p5001:5001 -v ${PWD}/test_models/triton/tf_cifar10:/models nvcr.io/nvidia/tritonserver:20.08-py3 /opt/tritonserver/bin/tritonserver --model-repository=/models --http-port=9000 --grpc-port=5001
+
+curl_triton_cifar10:
+	curl -H "Content-Type: application/json" http://0.0.0.0:9000/v2/models/cifar10/infer -d '@tests/data/truck-v2.json'
+
+
+run_explainer_triton_cifar10:
+	python -m alibiexplainer --model_name cifar10 --protocol kfserving.http --storage_uri gs://seldon-models/tfserving/cifar10/explainer-py36-0.5.2 --predictor_host localhost:9000 AnchorImages
+
+
+run_explainer_triton_cifar10_docker:
+	docker run --rm -d --name "explainer" --network=host -p 8080:8080 seldonio/${IMAGE}:${VERSION} --model_name cifar10 --protocol kfserving.http --storage_uri gs://seldon-models/tfserving/cifar10/explainer-py36-0.5.2 --predictor_host localhost:9000 AnchorImages 
+
+
+curl_explain_triton_cifar10_image:
+	curl -d @tests/data/truck-v2.json  -X POST http://localhost:8080/v2/models/cifar10/explain    -H "Content-Type: application/json"
+

components/alibi-explain-server/alibiexplainer/explainer.py

@@ -35,6 +35,7 @@
 import grpc
 import tensorflow as tf
 import alibiexplainer.seldon_http as seldon
+import alibiexplainer.v2_http as v2
 import requests
 import os
 from alibiexplainer.model import ExplainerModel
@@ -45,11 +46,13 @@
 GRPC_MAX_MSG_LEN = 1000000000
 TENSORFLOW_PREDICTOR_URL_FORMAT = "http://{0}/v1/models/:predict"
 SELDON_PREDICTOR_URL_FORMAT = "http://{0}/api/v0.1/predictions"
+V2_PREDICTOR_URL_FORMAT = "http://{0}/v2/models/{1}/infer"
 
 class Protocol(Enum):
     tensorflow_http = "tensorflow.http"
     seldon_http = "seldon.http"
     seldon_grpc = "seldon.grpc"
+    v2_http = "kfserving.http"
 
     def __str__(self):
         return self.value
@@ -83,6 +86,9 @@ def __init__(self,
         self.protocol = protocol
         self.tf_data_type = tf_data_type
         logging.info("Protocol is %s",str(self.protocol))
+        self.v2_name = None
+        self.v2_type = None
+        self.v2_model_name = None
 
         # Add type for first value to help pass mypy type checks
         if self.method is ExplainerMethod.anchor_tabular:
@@ -135,13 +141,37 @@ def _predict_fn(self, arr: Union[np.ndarray, List]) -> np.ndarray:
                 raise Exception(
                     "Failed to get response from model return_code:%d" % response.status_code)
             return np.array(response.json()["predictions"])
+        elif self.protocol == Protocol.v2_http:
+            rh = v2.KFServingV2RequestHandler()
+            request = rh.create_request(arr, self.v2_name, self.v2_type)
+            logging.info("url %s",V2_PREDICTOR_URL_FORMAT.format(self.predictor_host, self.v2_model_name))
+            response = requests.post(
+                V2_PREDICTOR_URL_FORMAT.format(self.predictor_host, self.v2_model_name),
+                json.dumps(request)
+            )
+            if response.status_code != 200:
+                raise Exception(
+                    "Failed to get response from model return_code:%d" % response.status_code)
+            response_json = response.json()
+            arr = rh.extract_response(response_json)
+            return arr
 
-    def explain(self, request: Dict) -> Any:
+
+    def explain(self, request: Dict, model_name=None) -> Any:
         if self.method is ExplainerMethod.anchor_tabular or self.method is ExplainerMethod.anchor_images or \
                 self.method is ExplainerMethod.anchor_text or self.method is ExplainerMethod.kernel_shap or \
                 self.method is ExplainerMethod.integrated_gradients or self.method is ExplainerMethod.tree_shap:
             if self.protocol == Protocol.tensorflow_http:
                 explanation: Explanation = self.wrapper.explain(request["instances"])
+            elif self.protocol == Protocol.v2_http:
+                logging.info("model name %s", model_name)
+                rh = v2.KFServingV2RequestHandler()
+                self.v2_model_name = model_name
+                self.v2_name = rh.extract_name(request)
+                self.v2_type = rh.extract_type(request)
+                logging.info("v2 name from inputs %s:", self.v2_name)
+                response_list = rh.extract_request(request)
+                explanation = self.wrapper.explain(response_list)
             else:
                 rh = seldon.SeldonRequestHandler(request)
                 response_list = rh.extract_request()

components/alibi-explain-server/alibiexplainer/model.py

@@ -11,5 +11,5 @@ def load(self):
         self.ready = True
 
     @abstractmethod
-    def explain(self, request: Dict) -> Dict:
+    def explain(self, request: Dict, model_name = None) -> Dict:
        pass

components/alibi-explain-server/alibiexplainer/proto/tensorflow/core/framework/tensor.proto

@@ -52,7 +52,7 @@ message TensorProto {
   // DT_DOUBLE.
   repeated double double_val = 6 [packed = true];
 
-  // DT_INT32, DT_INT16, DT_INT8, DT_UINT8.
+  // DT_INT32, DT_INT16, DT_UINT16, DT_INT8, DT_UINT8.
   repeated int32 int_val = 7 [packed = true];
 
   // DT_STRING

components/alibi-explain-server/alibiexplainer/proto/tensorflow/core/framework/types.proto

@@ -1,6 +1,7 @@
 syntax = "proto3";
 
 package tensorflow;
+
 option cc_enable_arenas = true;
 option java_outer_classname = "TypesProtos";
 option java_multiple_files = true;
@@ -79,6 +80,7 @@ enum DataType {
 // listed here are a subset of the types in the variant type registry,
 // corresponding to commonly used variants which must occasionally be
 // special-cased.
+// TODO(b/174224459): Merge with FullTypeDef.
 enum SpecializedType {
   // Invalid/unknown specialized type.
   ST_INVALID = 0;

components/alibi-explain-server/alibiexplainer/server.py

@@ -52,6 +52,8 @@ def create_application(self):
                  ExplainHandler, dict(model=self.registered_model)),
                 (r"/api/v1.0/explain",
                  ExplainHandler, dict(model=self.registered_model)),
+                (r"/v2/models/([a-zA-Z0-9_-]+)/explain",
+                 ExplainV2Handler, dict(model=self.registered_model)),
             ]
         )
 
@@ -96,3 +98,18 @@ def post(self):
         self.write(response)
 
 
+class ExplainV2Handler(tornado.web.RequestHandler):
+    def initialize(self, model: ExplainerModel):
+        self.model = model  # pylint:disable=attribute-defined-outside-init
+
+    def post(self, model_name):
+        try:
+            body = json.loads(self.request.body)
+        except json.decoder.JSONDecodeError as e:
+            raise tornado.web.HTTPError(
+                status_code=HTTPStatus.BAD_REQUEST,
+                reason="Unrecognized request format: %s" % e
+            )
+        response =self.model.explain(body, model_name=model_name)
+        self.write(response)
+

components/alibi-explain-server/alibiexplainer/v2_http.py

@@ -0,0 +1,68 @@
+import numpy as np
+from typing import Dict, List, Union
+
+_v2tymap: Dict[str, np.dtype] = {
+    "BOOL": np.dtype("bool"),
+    "UINT8": np.dtype("uint8"),
+    "UINT16": np.dtype("uint16"),
+    "UINT32": np.dtype("uint32"),
+    "UINT64": np.dtype("uint64"),
+    "INT8": np.dtype("int8"),
+    "INT16": np.dtype("int16"),
+    "INT32": np.dtype("int32"),
+    "INT64": np.dtype("int64"),
+    "FP16": np.dtype("float32"),
+    "FP32": np.dtype("float32"),
+    "FP64": np.dtype("float64"),
+}
+
+_nptymap = dict([(value, key) for key, value in _v2tymap.items()])
+_nptymap[np.dtype("float32")] = "FP32"  # Ensure correct mapping for ambiguous type
+
+def _create_np_from_v2(data: list, ty: str, shape: list) -> np.array:
+    npty = _v2tymap[ty]
+    arr = np.array(data, dtype=npty)
+    arr.shape = tuple(shape)
+    return arr
+
+def _create_v2_from_np(arr: np.ndarray, name: str, ty: str) -> Dict:
+    if arr.dtype in _nptymap:
+        return {
+            "name": name,
+            "datatype": ty,
+            "data": arr.flatten().tolist(),
+            "shape": list(arr.shape),
+        }
+    else:
+        raise ValueError(f"Unknown numpy type {arr.dtype}")
+
+# Only handle single input/output payloads
+class KFServingV2RequestHandler():
+
+    def extract_request(self, request: Dict) -> List:
+        inputs = request["inputs"][0]
+        data_type = inputs["datatype"]
+        shape = inputs["shape"]
+        data = inputs["data"]
+        arr = _create_np_from_v2(data, data_type, shape)
+        return arr.tolist()
+
+    def extract_response(self, request: Dict) -> List:
+        inputs = request["outputs"][0]
+        data_type = inputs["datatype"]
+        shape = inputs["shape"]
+        data = inputs["data"]
+        arr = _create_np_from_v2(data, data_type, shape)
+        return arr
+
+    def create_request(self, arr: np.ndarray, name: str, ty: str) -> Dict:
+        req = {}
+        data = _create_v2_from_np(arr, name, ty)
+        req["inputs"] = [data]
+        return req
+
+    def extract_name(self, request: Dict) -> str:
+        return request["inputs"][0]["name"]
+
+    def extract_type(self, request: Dict) -> str:
+        return request["inputs"][0]["datatype"]

components/alibi-explain-server/tests/data/truck-v2-response.json

@@ -0,0 +1,26 @@
+{
+	"model_name": "cifar10",
+	"model_version": "1",
+	"outputs": [
+		{
+			"data": [
+				0.0000012644828757402138,
+				4.881440140991344e-9,
+				1.5153264198985994e-9,
+				8.490542491301767e-9,
+				5.513066114737342e-10,
+				1.1617126149943147e-9,
+				5.772862743391727e-10,
+				2.8839471610808687e-7,
+				0.000614893389865756,
+				0.9993835687637329
+			],
+			"datatype": "FP32",
+			"name": "fc10",
+			"shape": [
+				1,
+				10
+			]
+		}
+	]
+}