Merge pull request #342 from cliveseldon/openvino_demo

Python wrapper update and openvino example

examples/models/tfserving-mnist/tfserving-mnist.ipynb

@@ -24,14 +24,31 @@
    "metadata": {},
    "outputs": [
     {
-     "name": "stderr",
+     "name": "stdout",
      "output_type": "stream",
      "text": [
-      "/home/clive/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
-      "  from ._conv import register_converters as _register_converters\n"
+      "'../../../proto/tensorflow/tensorflow' -> './tensorflow'\r\n",
+      "'../../../proto/tensorflow/tensorflow/core' -> './tensorflow/core'\r\n",
+      "'../../../proto/tensorflow/tensorflow/core/framework' -> './tensorflow/core/framework'\r\n",
+      "'../../../proto/tensorflow/tensorflow/core/framework/types.proto' -> './tensorflow/core/framework/types.proto'\r\n",
+      "'../../../proto/tensorflow/tensorflow/core/framework/resource_handle.proto' -> './tensorflow/core/framework/resource_handle.proto'\r\n",
+      "'../../../proto/tensorflow/tensorflow/core/framework/tensor_shape.proto' -> './tensorflow/core/framework/tensor_shape.proto'\r\n",
+      "'../../../proto/tensorflow/tensorflow/core/framework/tensor.proto' -> './tensorflow/core/framework/tensor.proto'\r\n"
      ]
     }
    ],
+   "source": [
+    "!mkdir -p proto\n",
+    "!cp ../../../proto/prediction.proto ./proto\n",
+    "!cp -vr ../../../proto/tensorflow/tensorflow .\n",
+    "!python -m grpc.tools.protoc -I./ --python_out=./ --grpc_python_out=./ ./proto/prediction.proto"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {},
+   "outputs": [],
    "source": [
     "%matplotlib inline\n",
     "import requests\n",
@@ -42,12 +59,17 @@
     "from tensorflow.examples.tutorials.mnist import input_data\n",
     "import sys\n",
     "sys.path.append(\"../../../notebooks\")\n",
-    "from visualizer import get_graph"
+    "from visualizer import get_graph\n",
+    "from proto import prediction_pb2\n",
+    "from proto import prediction_pb2_grpc\n",
+    "import grpc\n",
+    "import tensorflow as tf\n",
+    "from tensorflow.core.framework.tensor_pb2 import TensorProto"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 33,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -88,6 +110,17 @@
     "    print(response.status_code)\n",
     "    print(response.text)\n",
     "\n",
+    "def grpc_request_internal(data,endpoint=\"localhost:5000\"):\n",
+    "    datadef = prediction_pb2.DefaultData(\n",
+    "            tftensor=tf.make_tensor_proto(data)\n",
+    "        )\n",
+    "\n",
+    "    request = prediction_pb2.SeldonMessage(data = datadef)\n",
+    "    channel = grpc.insecure_channel(endpoint)\n",
+    "    stub = prediction_pb2_grpc.ModelStub(channel)\n",
+    "    response = stub.Predict(request=request)\n",
+    "    return response\n",
+    "\n",
     "\n",
     "def gen_mnist_data(mnist):\n",
     "    batch_xs, batch_ys = mnist.train.next_batch(1)\n",
@@ -99,38 +132,31 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "WARNING:tensorflow:From <ipython-input-2-8029beac92e7>:7: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
+      "WARNING:tensorflow:From <ipython-input-5-d2f9601b3465>:7: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
       "Instructions for updating:\n",
       "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
       "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
       "Instructions for updating:\n",
       "Please write your own downloading logic.\n",
-      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/base.py:252: _internal_retry.<locals>.wrap.<locals>.wrapped_fn (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
-      "Instructions for updating:\n",
-      "Please use urllib or similar directly.\n",
-      "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
       "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
       "Instructions for updating:\n",
       "Please use tf.data to implement this functionality.\n",
       "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
-      "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
       "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
       "Instructions for updating:\n",
       "Please use tf.data to implement this functionality.\n",
       "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
       "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
       "Instructions for updating:\n",
       "Please use tf.one_hot on tensors.\n",
-      "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
       "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
-      "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
       "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
       "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
       "Instructions for updating:\n",

helm-charts/seldon-openvino/templates/openvino_deployment.json

@@ -16,7 +16,7 @@
                     "spec": {
                         "containers": [
                             {
-                                "image": "seldonio/tfserving-proxy:0.1",
+                                "image": "{{ .Values.tfserving_proxy.image }}",
                                 "name": "tfserving-proxy"
                             },
 			    {
@@ -72,7 +72,7 @@
 		}],
                 "graph": {
 		    "name": "tfserving-proxy",
-		    "endpoint": { "type" : "REST" },
+		    "endpoint": { "type" : "GRPC" },
 		    "type": "MODEL",
 		    "children": [],
 		    "parameters":

helm-charts/seldon-openvino/values.yaml

@@ -1,10 +1,11 @@
 openvino:
-  image: ie-serving-py:0.1
+  image: intelaipg/openvino-model-server:0.2
   model:
     path: /opt/ml/squeezenet
     name: squeezenet1.1
     input: data
     output: prob
-  port: 8000
+  port: 8001
   model_volume: hostPath
-
+tfserving_proxy:
+  image: seldonio/tfserving-proxy:0.2

integrations/tfserving/.s2i/environment

@@ -1,4 +1,4 @@
 MODEL_NAME=TfServingProxy
-API_TYPE=REST
+API_TYPE=GRPC
 SERVICE_TYPE=MODEL
 PERSISTENCE=0

integrations/tfserving/Makefile

@@ -1,11 +1,11 @@
-IMAGE_VERSION=0.1
+IMAGE_VERSION=0.2
 IMAGE_NAME = docker.io/seldonio/tfserving-proxy
 
 SELDON_CORE_DIR=../../..
 
 .PHONY: build
 build: 
-	s2i build . seldonio/seldon-core-s2i-python3:0.3 $(IMAGE_NAME):$(IMAGE_VERSION)
+	s2i build . seldonio/seldon-core-s2i-python36:0.4-SNAPSHOT $(IMAGE_NAME):$(IMAGE_VERSION)
 
 push_to_dockerhub:
 	docker push $(IMAGE_NAME):$(IMAGE_VERSION)

integrations/tfserving/TfServingProxy.py

@@ -5,9 +5,13 @@
 from tensorflow.python.saved_model import signature_constants
 from tensorflow_serving.apis import predict_pb2
 from tensorflow_serving.apis import prediction_service_pb2_grpc
+from seldon_core.microservice import get_data_from_proto, array_to_grpc_datadef
+from seldon_core.model_microservice import get_class_names
+from seldon_core.proto import prediction_pb2, prediction_pb2_grpc
 
 import requests
 import json
+import numpy as np
 
 class TensorflowServerError(Exception):
 
@@ -36,14 +40,54 @@ def __init__(self,rest_endpoint=None,grpc_endpoint=None,model_name=None,signatur
             self.signature_name = signature_name
         self.model_input = model_input        
         self.model_output = model_output
+
+
+    # if we have a TFTensor message we got directly without converting the message otherwise we go the usual route
+    def predict_grpc(self,request):
+        print("Predict grpc called")
+        default_data_type = request.data.WhichOneof("data_oneof")
+        print(default_data_type)
+        if default_data_type == "tftensor":
+            tfrequest = predict_pb2.PredictRequest()
+            tfrequest.model_spec.name = self.model_name
+            tfrequest.model_spec.signature_name = self.signature_name
+            tfrequest.inputs[self.model_input].CopyFrom(request.data.tftensor)
+            result = self.stub.Predict(tfrequest)
+            print(result)
+            datadef = prediction_pb2.DefaultData(
+                tftensor=result.outputs[self.model_output]
+            )
+            return prediction_pb2.SeldonMessage(data=datadef)
+
+        else:
+            features = get_data_from_proto(request)
+            datadef = request.data
+            data_type = request.WhichOneof("data_oneof")
+            predictions = self.predict(features, datadef.names)
+
+            predictions = np.array(predictions)
+            if len(predictions.shape) > 1:
+                class_names = get_class_names(
+                    self, predictions.shape[1])
+            else:
+                class_names = []
+
+            if data_type == "data":
+                default_data_type = request.data.WhichOneof("data_oneof")
+            else:
+                default_data_type = "tensor"
+            data = array_to_grpc_datadef(
+                predictions, class_names, default_data_type)
+            return prediction_pb2.SeldonMessage(data=data)
+
+
 
     def predict(self,X,features_names):
         if self.grpc:
             request = predict_pb2.PredictRequest()
             request.model_spec.name = self.model_name
             request.model_spec.signature_name = self.signature_name
             request.inputs[self.model_input].CopyFrom(tf.contrib.util.make_tensor_proto(X.tolist(), shape=X.shape))
-            print(request)
             result = self.stub.Predict(request)
             print(result)
             response = numpy.array(result.outputs[self.model_output].float_val)