From bd5c54193e199709fa57e7a2353b2201e0e344ab Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Sat, 20 Jul 2019 16:36:36 +0100 Subject: [PATCH 1/2] update seldon client to support ssl and provide notebook exmaples --- notebooks/seldon_client.ipynb | 224 ++++++++++++++++++++++++++++ python/requirements.txt | 2 +- python/seldon_core/seldon_client.py | 194 ++++++++++++------------ 3 files changed, 318 insertions(+), 102 deletions(-) create mode 100644 notebooks/seldon_client.ipynb diff --git a/notebooks/seldon_client.ipynb b/notebooks/seldon_client.ipynb new file mode 100644 index 0000000000..2ab6f9a495 --- /dev/null +++ b/notebooks/seldon_client.ipynb @@ -0,0 +1,224 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Advanced Usage Exampes for Seldon Client" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Istio Gateway Request with token over HTTPS - no SSL verification\n", + "\n", + "Test against a current kubeflow cluster with Dex token authentication.\n", + "\n", + " 1. Install kubeflow with Dex authentication" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "INGRESS_HOST=!kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip}'\n", + "INGRESS_PORT=!kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.spec.ports[?(@.name==\"http2\")].port}'\n", + "ISTIO_GATEWAY=INGRESS_HOST[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ISTIO_GATEWAY" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Get a token from the Dex gateway. At present as Dex does not support curl password credentials you will need to get it from your browser logged into the cluster. Open up a browser console and run `document.cookie`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "TOKEN=\"eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE1NjM2MjA0ODYsImlhdCI6MTU2MzUzNDA4NiwiaXNzIjoiMzQuNjUuNzMuMjU1IiwianRpIjoiYjllNDQxOGQtZjNmNC00NTIyLTg5ODEtNDcxOTY0ODNmODg3IiwidWlmIjoiZXlKcGMzTWlPaUpvZEhSd2N6b3ZMek0wTGpZMUxqY3pMakkxTlRvMU5UVTJMMlJsZUNJc0luTjFZaUk2SWtOcFVYZFBSMFUwVG1wbk1GbHBNV3RaYW1jMFRGUlNhVTU2VFhSUFZFSm9UMU13ZWxreVVYaE9hbGw0V21wVk1FNXFXVk5DVjNoMldUSkdjeUlzSW1GMVpDSTZJbXQxWW1WbWJHOTNMV0YxZEdoelpYSjJhV05sTFc5cFpHTWlMQ0psZUhBaU9qRTFOak0yTWpBME9EWXNJbWxoZENJNk1UVTJNelV6TkRBNE5pd2lZWFJmYUdGemFDSTZJbE5OWlZWRGJUQmFOVkZoUTNCdVNHTndRMWgwTVZFaUxDSmxiV0ZwYkNJNkltRmtiV2x1UUhObGJHUnZiaTVwYnlJc0ltVnRZV2xzWDNabGNtbG1hV1ZrSWpwMGNuVmxMQ0p1WVcxbElqb2lZV1J0YVc0aWZRPT0ifQ.7CQIz4A1s9m6lJeWTqpz_JKGArGX4e_zpRCOXXjVRJgguB3z48rSfei_KL7niMCWpruhU11c8UIw9E79PwHNNw\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You will need to install Seldon and ensure its using the kubeflow gateway" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!helm install ../helm-charts/seldon-core-operator --name seldon-core --set istio.enabled=true --set istio.gateway=kubeflow-gateway --set usageMetrics.enabled=true --namespace seldon-system" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "deployment_name=\"test1\"\n", + "namespace=\"default\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from seldon_core.seldon_client import SeldonClient, SeldonChannelCredentials, SeldonCallCredentials\n", + "sc = SeldonClient(deployment_name=deployment_name,namespace=namespace,gateway_endpoint=ISTIO_GATEWAY,debug=True,\n", + " channel_credentials=SeldonChannelCredentials(verify=False),\n", + " call_credentials=SeldonCallCredentials(token=TOKEN))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = sc.predict(gateway=\"istio\",transport=\"rest\",shape=(1,4))\n", + "print(r)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Its not presently possible to use gRPC without getting access to the certificates. We will update this once its clear how to obtain them from a Kubeflow cluser setup." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Istio - SSL Endpoint - Client Side Verification - No Authentication\n", + "\n", + " 1. First run through the [Istio Secure Gateway SDS example](https://istio.io/docs/tasks/traffic-management/ingress/secure-ingress-sds/) and make sure this works for you.\n", + " * This will create certificates for `httpbin.example.com` and test them out.\n", + " 1. Update your `/etc/hosts` file to include an entry for the ingress gateway for `httpbin.example.com` e.g. add a line like: `10.107.247.132 httpbin.example.com` replacing the ip address with your ingress gateway ip address." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Set to folder where the httpbin certificates are\n", + "ISTIO_HTTPBIN_CERT_FOLDER='/home/clive/work/istio/httpbin.example.com'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You will need to install Seldon using the gateway `mygateway` used in the example." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!helm install ../helm-charts/seldon-core-operator --name seldon-core --set istio.enabled=true --set istio.gateway=mygateway --set usageMetrics.enabled=true --namespace seldon-system" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "deployment_name=\"mymodel\"\n", + "namespace=\"default\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from seldon_core.seldon_client import SeldonClient, SeldonChannelCredentials, SeldonCallCredentials\n", + "sc = SeldonClient(deployment_name=deployment_name,namespace=namespace,gateway_endpoint=\"httpbin.example.com\",debug=True,\n", + " channel_credentials=SeldonChannelCredentials(certificate_chain_file=ISTIO_HTTPBIN_CERT_FOLDER+'/2_intermediate/certs/ca-chain.cert.pem',\n", + " root_certificates_file=ISTIO_HTTPBIN_CERT_FOLDER+'/4_client/certs/httpbin.example.com.cert.pem',\n", + " private_key_file=ISTIO_HTTPBIN_CERT_FOLDER+'/4_client/private/httpbin.example.com.key.pem'\n", + " ))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = sc.predict(gateway=\"istio\",transport=\"rest\",shape=(1,4))\n", + "print(r)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "r = sc.predict(gateway=\"istio\",transport=\"grpc\",shape=(1,4))\n", + "print(r)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/python/requirements.txt b/python/requirements.txt index cafc2b8087..81849608e2 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,6 +1,6 @@ numpy==1.14.5 pandas==0.23.4 -grpcio==1.14.0 +grpcio>=1.14.0 Flask==1.0.2 flask-cors==3.0.3 futures diff --git a/python/seldon_core/seldon_client.py b/python/seldon_core/seldon_client.py index 152481ca87..8ff2fae167 100644 --- a/python/seldon_core/seldon_client.py +++ b/python/seldon_core/seldon_client.py @@ -9,6 +9,7 @@ from typing import Tuple, Dict, Union, List, Optional, Iterable import json import logging +import http.client as http_client logger = logging.getLogger(__name__) @@ -23,6 +24,28 @@ def __init__(self, message): Exception.__init__(self) self.message = message +class SeldonChannelCredentials(object): + """ + Channel credentials + Presently just denotes an SSL connection. + """ + + def __init__(self, verify:bool = True, root_certificates_file: str = None, + private_key_file: str = None, certificate_chain_file: str = None): + self.verify = verify + self.root_certificates_file = root_certificates_file + self.private_key_file = private_key_file + self.certificate_chain_file = certificate_chain_file + + + +class SeldonCallCredentials(object): + """ + Credentials for each call + """ + + def __init__(self,token:str = None): + self.token = token class SeldonClientPrediction(object): """ @@ -89,7 +112,9 @@ def __init__(self, gateway: str = "ambassador", transport: str = "rest", namespa seldon_rest_endpoint: str = "localhost:8002", seldon_grpc_endpoint: str = "localhost:8004", gateway_endpoint: str = "localhost:8003", microservice_endpoint: str = "localhost:5000", grpc_max_send_message_length: int = 4 * 1024 * 1024, - grpc_max_receive_message_length: int = 4 * 1024 * 1024): + grpc_max_receive_message_length: int = 4 * 1024 * 1024, + channel_credentials: SeldonChannelCredentials =None, + call_credentials: SeldonCallCredentials = None, debug = False): """ Parameters @@ -121,9 +146,12 @@ def __init__(self, gateway: str = "ambassador", transport: str = "rest", namespa grpc_max_receive_message_length Max grpc receive message size in bytes """ + if debug: + logger.setLevel(logging.DEBUG) + http_client.HTTPConnection.debuglevel = 1 self.config = locals() del self.config["self"] - logging.debug("Configuration:" + str(self.config)) + logger.debug("Configuration:" + str(self.config)) def _gather_args(self, **kwargs): @@ -1002,7 +1030,8 @@ def rest_predict_gateway(deployment_name: str, namespace: str = None, gateway_en data: np.ndarray = None, headers: Dict = None, gateway_prefix: str = None, payload_type: str = "tensor", bin_data: Union[bytes, bytearray] = None, str_data: str = None, - names: Iterable[str] = None, + names: Iterable[str] = None, call_credentials: SeldonCallCredentials = None, + channel_credentials: SeldonChannelCredentials= None, **kwargs) -> SeldonClientPrediction: """ REST request to Gateway Ingress @@ -1031,6 +1060,10 @@ def rest_predict_gateway(deployment_name: str, namespace: str = None, gateway_en String data to send names Column names + call_credentials + Call credentials - see SeldonCallCredentials + channel_credentials + Channel credentials - see SeldonChannelCredentials Returns ------- @@ -1047,100 +1080,44 @@ def rest_predict_gateway(deployment_name: str, namespace: str = None, gateway_en datadef = array_to_grpc_datadef(payload_type, data, names=names) request = prediction_pb2.SeldonMessage(data=datadef) payload = seldon_message_to_json(request) + + if not headers is None: + req_headers = headers.copy() + else: + req_headers = {} + if channel_credentials is None: + scheme = "http" + else: + scheme = "https" + if not call_credentials is None: + if not call_credentials.token is None: + req_headers["X-Auth-Token"] = call_credentials.token if gateway_prefix is None: if namespace is None: - response_raw = requests.post( - "http://" + gateway_endpoint + "/seldon/" + deployment_name + "/api/v0.1/predictions", - json=payload, - headers=headers) + url = scheme + "://" + gateway_endpoint + "/seldon/" + deployment_name + "/api/v0.1/predictions" else: - response_raw = requests.post( - "http://" + gateway_endpoint + "/seldon/" + namespace + "/" + deployment_name + "/api/v0.1/predictions", - json=payload, - headers=headers) - else: - response_raw = requests.post( - "http://" + gateway_endpoint + gateway_prefix + "/api/v0.1/predictions", - json=payload, - headers=headers) - - if response_raw.status_code == 200: - success = True - msg = "" + url = scheme+"://" + gateway_endpoint + "/seldon/" + namespace + "/" + deployment_name + "/api/v0.1/predictions" else: - success = False - msg = str(response_raw.status_code) + ":" + response_raw.reason - try: - if len(response_raw.text) > 0: - try: - response = json_to_seldon_message(response_raw.json()) - except: - response = None + url = scheme+"://" + gateway_endpoint + gateway_prefix + "/api/v0.1/predictions" + verify = True + cert = None + if not channel_credentials is None: + if not channel_credentials.certificate_chain_file is None: + verify = channel_credentials.certificate_chain_file else: - response = None - return SeldonClientPrediction(request, response, success, msg) - except Exception as e: - return SeldonClientPrediction(request, None, False, str(e)) - - -def rest_predict_gateway_basicauth(deployment_name: str, username: str, password: str, namespace: str = None, - gateway_endpoint: str = "localhost:8003", - shape: Tuple[int, int] = (1, 1), data: np.ndarray = None, - payload_type: str = "tensor", - bin_data: Union[bytes, bytearray] = None, str_data: str = None, - names: Iterable[str] = None, - **kwargs) -> SeldonClientPrediction: - """ - REST request with Basic Auth to Gateway Ingress - - Parameters - ---------- - deployment_name - The name of the running deployment - username - Username for basic auth - password - Password for basic auth - namespace - The namespace of the running deployment - gateway_endpoint - The host:port of gateway - shape - The shape of data - data - The numpy data to send - payload_type - payload - tensor, ndarray or tftensor - bin_data - Binary data to send - str_data - names - Column names - - Returns - ------- + verify = channel_credentials.verify + if not channel_credentials.private_key_file is None: + cert = (channel_credentials.root_certificates_file, channel_credentials.private_key_file) + logger.debug("URL is "+url) + logger.debug("verify is "+verify) + logger.debug("cert is %s",cert) + response_raw = requests.post( + url, + json=payload, + headers=req_headers, + verify=verify, + cert=cert) - """ - if bin_data is not None: - request = prediction_pb2.SeldonMessage(binData=bin_data) - elif str_data is not None: - request = prediction_pb2.SeldonMessage(strData=str_data) - else: - if data is None: - data = np.random.rand(*shape) - datadef = array_to_grpc_datadef(payload_type, data, names) - request = prediction_pb2.SeldonMessage(data=datadef) - payload = seldon_message_to_json(request) - if namespace is None: - response_raw = requests.post( - "http://" + gateway_endpoint + "/seldon/" + deployment_name + "/api/v0.1/predictions", - json=payload, - auth=HTTPBasicAuth(username, password)) - else: - response_raw = requests.post( - "http://" + gateway_endpoint + "/seldon/" + namespace + "/" + deployment_name + "/api/v0.1/predictions", - json=payload, - auth=HTTPBasicAuth(username, password)) if response_raw.status_code == 200: success = True msg = "" @@ -1159,7 +1136,6 @@ def rest_predict_gateway_basicauth(deployment_name: str, username: str, password except Exception as e: return SeldonClientPrediction(request, None, False, str(e)) - def grpc_predict_gateway(deployment_name: str, namespace: str = None, gateway_endpoint: str = "localhost:8003", shape: Tuple[int, int] = (1, 1), data: np.ndarray = None, @@ -1167,7 +1143,8 @@ def grpc_predict_gateway(deployment_name: str, namespace: str = None, gateway_en bin_data: Union[bytes, bytearray] = None, str_data: str = None, grpc_max_send_message_length: int = 4 * 1024 * 1024, grpc_max_receive_message_length: int = 4 * 1024 * 1024, - names: Iterable[str] = None, + names: Iterable[str] = None, call_credentials: SeldonCallCredentials = None, + channel_credentials: SeldonChannelCredentials= None, **kwargs) -> SeldonClientPrediction: """ gRPC request to Gateway Ingress @@ -1213,9 +1190,24 @@ def grpc_predict_gateway(deployment_name: str, namespace: str = None, gateway_en data = np.random.rand(*shape) datadef = array_to_grpc_datadef(payload_type, data, names=names) request = prediction_pb2.SeldonMessage(data=datadef) - channel = grpc.insecure_channel(gateway_endpoint, options=[ - ('grpc.max_send_message_length', grpc_max_send_message_length), - ('grpc.max_receive_message_length', grpc_max_receive_message_length)]) + options = [ + ('grpc.max_send_message_length', grpc_max_send_message_length), + ('grpc.max_receive_message_length', grpc_max_receive_message_length)] + if channel_credentials is None: + channel = grpc.insecure_channel(gateway_endpoint, options) + else: + grpc_channel_credentials = grpc.ssl_channel_credentials(root_certificates=open(channel_credentials.certificate_chain_file, 'rb').read(), + private_key=open(channel_credentials.private_key_file, 'rb').read(), + certificate_chain=open(channel_credentials.root_certificates_file, 'rb').read()) + if not call_credentials is None: + #grpc_call_credentials = grpc.access_token_call_credentials(call_credentials.token) + grpc_call_credentials = grpc.metadata_call_credentials( + lambda context, callback: callback((("X-Auth-Token", call_credentials.token),), None)) + credentials = grpc.composite_channel_credentials(grpc_channel_credentials,grpc_call_credentials) + else: + credentials = grpc_channel_credentials + logger.debug(gateway_endpoint) + channel = grpc.secure_channel(gateway_endpoint,credentials, options) stub = prediction_pb2_grpc.SeldonStub(channel) if namespace is None: metadata = [('seldon', deployment_name)] @@ -1224,11 +1216,11 @@ def grpc_predict_gateway(deployment_name: str, namespace: str = None, gateway_en if not headers is None: for k in headers: metadata.append((k, headers[k])) - try: - response = stub.Predict(request=request, metadata=metadata) - return SeldonClientPrediction(request, response, True, "") - except Exception as e: - return SeldonClientPrediction(request, None, False, str(e)) + #try: + response = stub.Predict(request=request, metadata=metadata) + return SeldonClientPrediction(request, response, True, "") + #except Exception as e: + # return SeldonClientPrediction(request, None, False, str(e)) def rest_feedback_seldon_oauth(prediction_request: prediction_pb2.SeldonMessage = None, From 00f62f7e009e850714fd59406799b2fb500bab37 Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Sat, 20 Jul 2019 18:19:08 +0100 Subject: [PATCH 2/2] Add docs for seldon client --- .../examples/alibaba_aks_deep_mnist.nblink | 2 +- doc/source/examples/seldon_client.nblink | 3 + doc/source/index.rst | 1 + doc/source/python/index.rst | 1 + doc/source/python/seldon_client.md | 34 +++ notebooks/helm_examples.ipynb | 223 ++++++++++-------- python/seldon_core/seldon_client.py | 2 - 7 files changed, 161 insertions(+), 105 deletions(-) create mode 100644 doc/source/examples/seldon_client.nblink create mode 100644 doc/source/python/seldon_client.md diff --git a/doc/source/examples/alibaba_aks_deep_mnist.nblink b/doc/source/examples/alibaba_aks_deep_mnist.nblink index b66880a632..ebd664e3aa 100644 --- a/doc/source/examples/alibaba_aks_deep_mnist.nblink +++ b/doc/source/examples/alibaba_aks_deep_mnist.nblink @@ -1,3 +1,3 @@ { - "path": "../../../examples/models/alibaba_aks_deep_mnist/alibaba_cloud_aks_deep_mnist.ipynb" + "path": "../../../examples/models/alibaba_aks_deep_mnist/alibaba_cloud_ack_deep_mnist.ipynb" } diff --git a/doc/source/examples/seldon_client.nblink b/doc/source/examples/seldon_client.nblink new file mode 100644 index 0000000000..c440bbf217 --- /dev/null +++ b/doc/source/examples/seldon_client.nblink @@ -0,0 +1,3 @@ +{ + "path": "../../../notebooks/seldon_client.ipynb" +} diff --git a/doc/source/index.rst b/doc/source/index.rst index 05d03ed31d..e819fc1f54 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -105,6 +105,7 @@ Seldon Core is an open source platform for deploying machine learning models on :maxdepth: 1 :caption: Reference + Python API reference Seldon Microservice API Seldon Orchestrator Benchmarking diff --git a/doc/source/python/index.rst b/doc/source/python/index.rst index 8976730840..233015d273 100644 --- a/doc/source/python/index.rst +++ b/doc/source/python/index.rst @@ -10,5 +10,6 @@ Python Seldon Core Python Module Your python class Wrap using S2I or Docker + Seldon Python Client Python API reference diff --git a/doc/source/python/seldon_client.md b/doc/source/python/seldon_client.md new file mode 100644 index 0000000000..51b20805b8 --- /dev/null +++ b/doc/source/python/seldon_client.md @@ -0,0 +1,34 @@ +# Seldon Python Client + +We provide an example python client for calling the API using REST or gRPC for internal mciroservice testing or for calling the public external API. + +Examples of its use can be found in various notebooks: + + * [Helm based deployment examples](../examples/helm_examples.html) + * [Istio examples](../examples/istio_examples.html) + +To use the client simply create an instance with settings for your use case, for example: + +``` +from seldon_core.seldon_client import SeldonClient +sc = SeldonClient(deployment_name="mymodel",namespace="seldon",gateway_endpoint="localhost:8003",gateway="ambassador") +``` + +In the above we set our deployment_name to "mymodel" and the namespace to "seldon". For the full set of parameters see [here](api/seldon_core.html#seldon_core.seldon_client.SeldonClient) + +To make a REST call with a random payload: + +``` +r = sc.predict(transport="rest") +``` + +To make a gRPC call with random payload: + +``` +r = sc.predict(transport="grpc") +``` + + +## Advanced Examples + + * [SSL and Authentication](../examples/seldon_client.html) diff --git a/notebooks/helm_examples.ipynb b/notebooks/helm_examples.ipynb index f6352ddaff..c07c6e39da 100644 --- a/notebooks/helm_examples.ipynb +++ b/notebooks/helm_examples.ipynb @@ -151,7 +151,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 6, "metadata": { "scrolled": true }, @@ -161,57 +161,61 @@ "output_type": "stream", "text": [ "NAME: seldon-core\n", - "LAST DEPLOYED: Tue Apr 16 07:45:41 2019\n", + "LAST DEPLOYED: Sat Jul 20 17:21:41 2019\n", "NAMESPACE: seldon-system\n", "STATUS: DEPLOYED\n", "\n", "RESOURCES:\n", - "==> v1beta1/ClusterRole\n", - "NAME AGE\n", - "seldon-spartakus-volunteer 0s\n", - "\n", - "==> v1beta1/ClusterRoleBinding\n", - "NAME AGE\n", - "seldon-spartakus-volunteer 0s\n", - "\n", - "==> v1/Pod(related)\n", - "NAME READY STATUS RESTARTS AGE\n", - "seldon-operator-controller-manager-0 0/1 ContainerCreating 0 0s\n", + "==> v1/ClusterRole\n", + "NAME AGE\n", + "seldon-operator-manager-role 0s\n", "\n", "==> v1/ClusterRoleBinding\n", "NAME AGE\n", "seldon-operator-manager-rolebinding 0s\n", "\n", - "==> v1/Service\n", - "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\n", - "seldon-operator-controller-manager-service ClusterIP 10.101.237.184 443/TCP 0s\n", + "==> v1/ConfigMap\n", + "NAME DATA AGE\n", + "seldon-spartakus-config 1 0s\n", "\n", - "==> v1beta1/Deployment\n", - "NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE\n", - "seldon-spartakus-volunteer 1 0 0 0 0s\n", + "==> v1/Pod(related)\n", + "NAME READY STATUS RESTARTS AGE\n", + "seldon-operator-controller-manager-0 0/1 ContainerCreating 0 0s\n", + "seldon-spartakus-volunteer-df68b9d48-979zh 0/1 ContainerCreating 0 0s\n", + "\n", + "==> v1/Secret\n", + "NAME TYPE DATA AGE\n", + "seldon-operator-webhook-server-secret Opaque 0 0s\n", + "\n", + "==> v1/Service\n", + "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\n", + "seldon-operator-controller-manager-service ClusterIP 10.98.43.142 443/TCP 0s\n", + "webhook-server-service ClusterIP 10.104.0.59 443/TCP 0s\n", "\n", "==> v1/ServiceAccount\n", - "NAME SECRETS AGE\n", - "seldon-spartakus-volunteer 1 0s\n", + "NAME SECRETS AGE\n", + "seldon-core-seldon-core-operator 1 0s\n", + "seldon-spartakus-volunteer 1 0s\n", "\n", "==> v1/StatefulSet\n", - "NAME DESIRED CURRENT AGE\n", - "seldon-operator-controller-manager 1 1 0s\n", + "NAME READY AGE\n", + "seldon-operator-controller-manager 0/1 0s\n", "\n", - "==> v1/Secret\n", - "NAME TYPE DATA AGE\n", - "seldon-operator-webhook-server-secret Opaque 0 0s\n", + "==> v1beta1/ClusterRole\n", + "NAME AGE\n", + "seldon-spartakus-volunteer 0s\n", "\n", - "==> v1/ConfigMap\n", - "NAME DATA AGE\n", - "seldon-spartakus-config 3 0s\n", + "==> v1beta1/ClusterRoleBinding\n", + "NAME AGE\n", + "seldon-spartakus-volunteer 0s\n", "\n", "==> v1beta1/CustomResourceDefinition\n", "NAME AGE\n", "seldondeployments.machinelearning.seldon.io 0s\n", "\n", - "==> v1/ClusterRole\n", - "seldon-operator-manager-role 0s\n", + "==> v1beta1/Deployment\n", + "NAME READY UP-TO-DATE AVAILABLE AGE\n", + "seldon-spartakus-volunteer 0/1 1 0 0s\n", "\n", "\n", "NOTES:\n", @@ -227,14 +231,15 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "partitioned roll out complete: 1 new pods have been updated...\r\n" + "Waiting for 1 pods to be ready...\n", + "partitioned roll out complete: 1 new pods have been updated...\n" ] } ], @@ -252,7 +257,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -260,25 +265,25 @@ "output_type": "stream", "text": [ "NAME: ambassador\n", - "LAST DEPLOYED: Tue Apr 16 07:28:49 2019\n", + "LAST DEPLOYED: Sat Jul 20 17:22:00 2019\n", "NAMESPACE: seldon\n", "STATUS: DEPLOYED\n", "\n", "RESOURCES:\n", - "==> v1/Service\n", - "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\n", - "ambassador-admins ClusterIP 10.101.241.19 8877/TCP 0s\n", - "ambassador LoadBalancer 10.98.184.144 80:30382/TCP,443:30204/TCP 0s\n", - "\n", "==> v1/Deployment\n", - "NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE\n", - "ambassador 3 3 3 0 0s\n", + "NAME READY UP-TO-DATE AVAILABLE AGE\n", + "ambassador 0/3 3 0 0s\n", "\n", "==> v1/Pod(related)\n", "NAME READY STATUS RESTARTS AGE\n", - "ambassador-5b89d44544-d4952 0/1 ContainerCreating 0 0s\n", - "ambassador-5b89d44544-jpvc9 0/1 ContainerCreating 0 0s\n", - "ambassador-5b89d44544-sdqtg 0/1 ContainerCreating 0 0s\n", + "ambassador-778b689797-227sm 0/1 ContainerCreating 0 0s\n", + "ambassador-778b689797-lf847 0/1 ContainerCreating 0 0s\n", + "ambassador-778b689797-wq2zk 0/1 ContainerCreating 0 0s\n", + "\n", + "==> v1/Service\n", + "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\n", + "ambassador LoadBalancer 10.110.29.89 80:32337/TCP,443:31543/TCP 0s\n", + "ambassador-admins ClusterIP 10.104.55.130 8877/TCP 0s\n", "\n", "==> v1/ServiceAccount\n", "NAME SECRETS AGE\n", @@ -292,6 +297,16 @@ "NAME AGE\n", "ambassador 0s\n", "\n", + "==> v1beta1/CustomResourceDefinition\n", + "NAME AGE\n", + "authservices.getambassador.io 0s\n", + "mappings.getambassador.io 0s\n", + "modules.getambassador.io 0s\n", + "ratelimitservices.getambassador.io 0s\n", + "tcpmappings.getambassador.io 0s\n", + "tlscontexts.getambassador.io 0s\n", + "tracingservices.getambassador.io 0s\n", + "\n", "\n", "NOTES:\n", "Congratuations! You've successfully installed Ambassador.\n", @@ -319,7 +334,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -357,7 +372,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -365,14 +380,14 @@ "output_type": "stream", "text": [ "NAME: mymodel\n", - "LAST DEPLOYED: Tue Apr 16 07:46:10 2019\n", + "LAST DEPLOYED: Sat Jul 20 18:11:38 2019\n", "NAMESPACE: seldon\n", "STATUS: DEPLOYED\n", "\n", "RESOURCES:\n", "==> v1alpha2/SeldonDeployment\n", "NAME AGE\n", - "mymodel 0s\n", + "mymodel 1s\n", "\n", "\n" ] @@ -449,7 +464,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -473,12 +488,12 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ "from seldon_core.seldon_client import SeldonClient\n", - "sc = SeldonClient(deployment_name=\"mymodel\",namespace=\"seldon\")" + "sc = SeldonClient(deployment_name=\"mymodel\",namespace=\"seldon\",gateway_endpoint=\"localhost:8003\",gateway=\"ambassador\")" ] }, { @@ -490,7 +505,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -503,13 +518,13 @@ " tensor {\n", " shape: 1\n", " shape: 1\n", - " values: 0.7059113295039197\n", + " values: 0.7784058932896504\n", " }\n", "}\n", "\n", "Response:\n", "meta {\n", - " puid: \"nuujtjvmno6b4kl2i9dgim3b7d\"\n", + " puid: \"8l4dem4dik8rnfk3cuscfbe146\"\n", " requestPath {\n", " key: \"classifier\"\n", " value: \"seldonio/mock_classifier:1.0\"\n", @@ -520,7 +535,7 @@ " tensor {\n", " shape: 1\n", " shape: 1\n", - " values: 0.09878880544753225\n", + " values: 0.10543331585480226\n", " }\n", "}\n", "\n" @@ -528,7 +543,7 @@ } ], "source": [ - "r = sc.predict(gateway=\"ambassador\",transport=\"rest\")\n", + "r = sc.predict(transport=\"rest\")\n", "print(r)" ] }, @@ -541,7 +556,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 17, "metadata": { "scrolled": true }, @@ -556,13 +571,13 @@ " tensor {\n", " shape: 1\n", " shape: 1\n", - " values: 0.5960703491429203\n", + " values: 0.6305579397781365\n", " }\n", "}\n", "\n", "Response:\n", "meta {\n", - " puid: \"tcqcav1vdar3t9evqm6ljcmu0d\"\n", + " puid: \"tuco33gp0he65oh36pq9cd5lie\"\n", " requestPath {\n", " key: \"classifier\"\n", " value: \"seldonio/mock_classifier:1.0\"\n", @@ -573,7 +588,7 @@ " tensor {\n", " shape: 1\n", " shape: 1\n", - " values: 0.08943148722225684\n", + " values: 0.09227998331474725\n", " }\n", "}\n", "\n" @@ -581,13 +596,13 @@ } ], "source": [ - "r = sc.predict(gateway=\"ambassador\",transport=\"grpc\")\n", + "r = sc.predict(transport=\"grpc\")\n", "print(r)" ] }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -611,7 +626,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -619,14 +634,14 @@ "output_type": "stream", "text": [ "NAME: myabtest\n", - "LAST DEPLOYED: Tue Apr 16 07:47:07 2019\n", + "LAST DEPLOYED: Sat Jul 20 18:13:29 2019\n", "NAMESPACE: seldon\n", "STATUS: DEPLOYED\n", "\n", "RESOURCES:\n", "==> v1alpha2/SeldonDeployment\n", "NAME AGE\n", - "myabtest 0s\n", + "myabtest 1s\n", "\n", "\n" ] @@ -750,7 +765,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 22, "metadata": {}, "outputs": [ { @@ -776,12 +791,12 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ "from seldon_core.seldon_client import SeldonClient\n", - "sc = SeldonClient(deployment_name=\"myabtest\",namespace=\"seldon\")" + "sc = SeldonClient(deployment_name=\"myabtest\",namespace=\"seldon\",gateway_endpoint=\"localhost:8003\",gateway=\"ambassador\")" ] }, { @@ -793,7 +808,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 30, "metadata": {}, "outputs": [ { @@ -806,23 +821,24 @@ " tensor {\n", " shape: 1\n", " shape: 1\n", - " values: 0.04041233913042219\n", + " values: 0.7647081632646227\n", " }\n", "}\n", "\n", "Response:\n", "meta {\n", - " puid: \"ti07spbas0t9o4uaub3ll72hgs\"\n", + " puid: \"8h4qnnqvo2861abjhsfdv5b267\"\n", " routing {\n", " key: \"myabtest\"\n", - " value: 1\n", + " value: 0\n", " }\n", " requestPath {\n", - " key: \"classifier-2\"\n", + " key: \"classifier-1\"\n", " value: \"seldonio/mock_classifier:1.0\"\n", " }\n", " requestPath {\n", " key: \"myabtest\"\n", + " value: \"\"\n", " }\n", "}\n", "data {\n", @@ -830,7 +846,7 @@ " tensor {\n", " shape: 1\n", " shape: 1\n", - " values: 0.05333995727727314\n", + " values: 0.10414835015130552\n", " }\n", "}\n", "\n" @@ -838,7 +854,7 @@ } ], "source": [ - "r = sc.predict(gateway=\"ambassador\",transport=\"rest\")\n", + "r = sc.predict(transport=\"rest\")\n", "print(r)" ] }, @@ -851,7 +867,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 35, "metadata": { "scrolled": true }, @@ -866,22 +882,24 @@ " tensor {\n", " shape: 1\n", " shape: 1\n", - " values: 0.891513821538451\n", + " values: 0.3556469031281131\n", " }\n", "}\n", "\n", "Response:\n", "meta {\n", - " puid: \"buccus5sv4vioiemig64khl292\"\n", + " puid: \"qbckh3cddrqlc78vvm27id9870\"\n", " routing {\n", " key: \"myabtest\"\n", + " value: 1\n", " }\n", " requestPath {\n", - " key: \"classifier-1\"\n", + " key: \"classifier-2\"\n", " value: \"seldonio/mock_classifier:1.0\"\n", " }\n", " requestPath {\n", " key: \"myabtest\"\n", + " value: \"\"\n", " }\n", "}\n", "data {\n", @@ -889,7 +907,7 @@ " tensor {\n", " shape: 1\n", " shape: 1\n", - " values: 0.11658722600889675\n", + " values: 0.07168964710483954\n", " }\n", "}\n", "\n" @@ -897,13 +915,13 @@ } ], "source": [ - "r = sc.predict(gateway=\"ambassador\",transport=\"grpc\")\n", + "r = sc.predict(transport=\"grpc\")\n", "print(r)" ] }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 36, "metadata": {}, "outputs": [ { @@ -927,7 +945,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 37, "metadata": { "scrolled": true }, @@ -937,7 +955,7 @@ "output_type": "stream", "text": [ "NAME: mymab\n", - "LAST DEPLOYED: Tue Apr 16 07:48:36 2019\n", + "LAST DEPLOYED: Sat Jul 20 18:15:30 2019\n", "NAMESPACE: seldon\n", "STATUS: DEPLOYED\n", "\n", @@ -1081,14 +1099,13 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 38, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Waiting for deployment \"mymab-abtest-41de5b8\" rollout to finish: 0 of 1 updated replicas are available...\n", "deployment \"mymab-abtest-41de5b8\" successfully rolled out\n", "deployment \"mymab-abtest-b8038b2\" successfully rolled out\n", "deployment \"mymab-abtest-df66c5c\" successfully rolled out\n" @@ -1110,12 +1127,12 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 39, "metadata": {}, "outputs": [], "source": [ "from seldon_core.seldon_client import SeldonClient\n", - "sc = SeldonClient(deployment_name=\"mymab\",namespace=\"seldon\")" + "sc = SeldonClient(deployment_name=\"mymab\",namespace=\"seldon\",gateway_endpoint=\"localhost:8003\",gateway=\"ambassador\")" ] }, { @@ -1127,7 +1144,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 40, "metadata": {}, "outputs": [ { @@ -1140,18 +1157,19 @@ " tensor {\n", " shape: 1\n", " shape: 1\n", - " values: 0.7895325000968559\n", + " values: 0.19835639554117446\n", " }\n", "}\n", "\n", "Response:\n", "meta {\n", - " puid: \"3m77cfurv2th42ilqmt84pb9uv\"\n", + " puid: \"15ur5egq097hkrj4kmot0lpsb2\"\n", " routing {\n", " key: \"eg-router\"\n", + " value: 1\n", " }\n", " requestPath {\n", - " key: \"classifier-1\"\n", + " key: \"classifier-2\"\n", " value: \"seldonio/mock_classifier:1.0\"\n", " }\n", " requestPath {\n", @@ -1164,7 +1182,7 @@ " tensor {\n", " shape: 1\n", " shape: 1\n", - " values: 0.10648736208223847\n", + " values: 0.06190151573679043\n", " }\n", "}\n", "\n" @@ -1172,7 +1190,7 @@ } ], "source": [ - "r = sc.predict(gateway=\"ambassador\",transport=\"rest\")\n", + "r = sc.predict(transport=\"rest\")\n", "print(r)" ] }, @@ -1185,7 +1203,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 42, "metadata": { "scrolled": true }, @@ -1200,15 +1218,16 @@ " tensor {\n", " shape: 1\n", " shape: 1\n", - " values: 0.35755536162542667\n", + " values: 0.5350085315378481\n", " }\n", "}\n", "\n", "Response:\n", "meta {\n", - " puid: \"etui7aiog8tqhi1k6a09n3h3ma\"\n", + " puid: \"u6fhq6blf6lrgesmse4tbudo2u\"\n", " routing {\n", " key: \"eg-router\"\n", + " value: 0\n", " }\n", " requestPath {\n", " key: \"classifier-1\"\n", @@ -1224,7 +1243,7 @@ " tensor {\n", " shape: 1\n", " shape: 1\n", - " values: 0.07181675934348077\n", + " values: 0.08458209124227425\n", " }\n", "}\n", "\n" @@ -1232,13 +1251,13 @@ } ], "source": [ - "r = sc.predict(gateway=\"ambassador\",transport=\"grpc\")\n", + "r = sc.predict(transport=\"grpc\")\n", "print(r)" ] }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 43, "metadata": {}, "outputs": [ { @@ -1278,7 +1297,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.4" + "version": "3.6.8" }, "varInspector": { "cols": { diff --git a/python/seldon_core/seldon_client.py b/python/seldon_core/seldon_client.py index 8ff2fae167..892de15c98 100644 --- a/python/seldon_core/seldon_client.py +++ b/python/seldon_core/seldon_client.py @@ -1109,8 +1109,6 @@ def rest_predict_gateway(deployment_name: str, namespace: str = None, gateway_en if not channel_credentials.private_key_file is None: cert = (channel_credentials.root_certificates_file, channel_credentials.private_key_file) logger.debug("URL is "+url) - logger.debug("verify is "+verify) - logger.debug("cert is %s",cert) response_raw = requests.post( url, json=payload,