diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentControllerImpl.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentControllerImpl.java index 638a4a81ae..a4ea766a15 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentControllerImpl.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentControllerImpl.java @@ -26,13 +26,16 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import io.kubernetes.client.ApiClient; import io.kubernetes.client.ApiException; import io.kubernetes.client.ProtoClient; import io.kubernetes.client.ProtoClient.ObjectOrStatus; +import io.kubernetes.client.apis.CoreV1Api; import io.kubernetes.client.models.ExtensionsV1beta1Deployment; import io.kubernetes.client.models.ExtensionsV1beta1DeploymentList; import io.kubernetes.client.models.V1Service; import io.kubernetes.client.models.V1ServiceList; +import io.kubernetes.client.models.V1Status; import io.kubernetes.client.proto.Meta.DeleteOptions; import io.kubernetes.client.proto.V1.Service; import io.kubernetes.client.proto.V1beta1Extensions.Deployment; @@ -139,6 +142,38 @@ private void removeDeployments(ProtoClient client,String namespace,SeldonDeploym } } + private void removeServices(ApiClient client,String namespace,SeldonDeployment seldonDeployment,List services) throws ApiException, IOException, SeldonDeploymentException + { + Set names = getServiceNames(services); + V1ServiceList svcList = crdHandler.getOwnedServices(seldonDeployment.getSpec().getName()); + for(V1Service s : svcList.getItems()) + { + if (!names.contains(s.getMetadata().getName())) + { + CoreV1Api api = new CoreV1Api(client); + V1Status status = api.deleteNamespacedService(s.getMetadata().getName(), namespace, null); + if (!"Success".equals(status.getStatus())) + { + logger.error("Failed to delete service "+s.getMetadata().getName()); + throw new SeldonDeploymentException("Failed to delete service "+s.getMetadata().getName()); + } + else + logger.debug("Deleted deployment "+s.getMetadata().getName()); + + } + } + } + + /** + * Currently Not used as issue with proto client needs further investigation + * @param client + * @param namespace + * @param seldonDeployment + * @param services + * @throws ApiException + * @throws IOException + * @throws SeldonDeploymentException + */ private void removeServices(ProtoClient client,String namespace,SeldonDeployment seldonDeployment,List services) throws ApiException, IOException, SeldonDeploymentException { Set names = getServiceNames(services); @@ -151,9 +186,9 @@ private void removeServices(ProtoClient client,String namespace,SeldonDeployment .replaceAll("\\{" + "name" + "\\}", client.getApiClient().escapeString(s.getMetadata().getName())) .replaceAll("\\{" + "namespace" + "\\}", client.getApiClient().escapeString(namespace)); DeleteOptions options = DeleteOptions.newBuilder().setPropagationPolicy("Foreground").build(); - ObjectOrStatus os = client.delete(Deployment.newBuilder(),deleteApiPath,options); + ObjectOrStatus os = client.delete(Service.newBuilder(),deleteApiPath,options); if (os.status != null) { - logger.error("Error deleting deployment:"+ProtoBufUtils.toJson(os.status)); + logger.error("Error deleting service:"+ProtoBufUtils.toJson(os.status)); throw new SeldonDeploymentException("Failed to delete service "+s.getMetadata().getName()); } else { @@ -245,7 +280,9 @@ public void createOrReplaceSeldonDeployment(SeldonDeployment mlDep) { createDeployments(client, namespace, resources.deployments); removeDeployments(client, namespace, mlDep2, resources.deployments); createServices(client, namespace, resources.services); - removeServices(client,namespace, mlDep2, resources.services); + //removeServices(client,namespace, mlDep2, resources.services); //Proto Client not presently working for deletion + ApiClient client2 = clientProvider.getClient(); + removeServices(client2,namespace, mlDep2, resources.services); if (!mlDep.getSpec().equals(mlDep2.getSpec())) { logger.debug("Pushing updated SeldonDeployment "+mlDep2.getMetadata().getName()+" back to kubectl"); diff --git a/notebooks/helm_minikube_ambassador.ipynb b/notebooks/helm_minikube_ambassador.ipynb new file mode 100644 index 0000000000..53da81f45d --- /dev/null +++ b/notebooks/helm_minikube_ambassador.ipynb @@ -0,0 +1,607 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Deploying Machine Learning Models Minikube with RBAC using kubectl\n", + "This demo shows how you can interact directly with kubernetes using kubectl to create and manage runtime machine learning models. It uses Minikube as the target Kubernetes cluster.\n", + "\"predictor" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prerequistes\n", + "You will need\n", + " - [Git clone of Seldon Core](https://github.com/SeldonIO/seldon-core)\n", + " - [Helm](https://github.com/kubernetes/helm)\n", + " - [Minikube](https://github.com/kubernetes/minikube) version v0.24.0 or greater\n", + " - [python grpc tools](https://grpc.io/docs/quickstart/python.html)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Create Cluster\n", + "\n", + "Start minikube and ensure custom resource validation is activated and there is 5G of memory. \n", + "\n", + "**2018-06-13** : At present we find the most stable version of minikube across platforms is 0.25.2 as there are issues with 0.26 and 0.27 on some systems. We also find the default VirtualBox driver can be problematic on some systems to we suggest using the [KVM2 driver](https://github.com/kubernetes/minikube/blob/master/docs/drivers.md#kvm2-driver).\n", + "\n", + "Your start command would then look like:\n", + "```\n", + "minikube start --vm-driver kvm2 --memory 4096 --feature-gates=CustomResourceValidation=true --extra-config=apiserver.Authorization.Mode=RBAC\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl create namespace seldon" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl create clusterrolebinding kube-system-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Install Helm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl -n kube-system create sa tiller\n", + "!kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller\n", + "!helm init --service-account tiller" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Label the node to allow load testing to run on it" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl label nodes `kubectl get nodes -o jsonpath='{.items[0].metadata.name}'` role=locust --overwrite" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Start seldon-core" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Install the custom resource definition" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!helm install ../helm-charts/seldon-core-crd --name seldon-core-crd --set usage_metrics.enabled=true" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!helm install ../helm-charts/seldon-core --name seldon-core --namespace seldon --set ambassador.enabled=true" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Install prometheus and grafana for analytics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!helm install ../helm-charts/seldon-core-analytics --name seldon-core-analytics \\\n", + " --set grafana_prom_admin_password=password \\\n", + " --set persistence.enabled=false \\\n", + " --namespace seldon" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Check all services are running before proceeding." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl get pods -n seldon" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Set up REST and gRPC methods\n", + "\n", + "**Ensure you port forward ambassador**:\n", + "\n", + "```\n", + "kubectl port-forward $(kubectl get pods -n seldon -l service=ambassador -o jsonpath='{.items[0].metadata.name}') -n seldon 8004:8080\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Install gRPC modules for the prediction protos." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!cp ../proto/prediction.proto ./proto\n", + "!python -m grpc.tools.protoc -I. --python_out=. --grpc_python_out=. ./proto/prediction.proto" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Illustration of both REST and gRPC requests. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "from requests.auth import HTTPBasicAuth\n", + "from proto import prediction_pb2\n", + "from proto import prediction_pb2_grpc\n", + "import grpc\n", + "\n", + "AMBASSADOR_API=\"localhost:8004\"\n", + "\n", + "def rest_request(deploymentName):\n", + " payload = {\"data\":{\"names\":[\"a\",\"b\"],\"tensor\":{\"shape\":[2,2],\"values\":[0,0,1,1]}}}\n", + " response = requests.post(\n", + " \"http://\"+AMBASSADOR_API+\"/seldon/\"+deploymentName+\"/api/v0.1/predictions\",\n", + " json=payload)\n", + " print(response.status_code)\n", + " print(response.text) \n", + " \n", + "def rest_request_auth(deploymentName,username,password):\n", + " payload = {\"data\":{\"names\":[\"a\",\"b\"],\"tensor\":{\"shape\":[2,2],\"values\":[0,0,1,1]}}}\n", + " response = requests.post(\n", + " \"http://\"+AMBASSADOR_API+\"/seldon/\"+deploymentName+\"/api/v0.1/predictions\",\n", + " json=payload,\n", + " auth=HTTPBasicAuth(username, password))\n", + " print(response.status_code)\n", + " print(response.text)\n", + "\n", + "def grpc_request(deploymentName):\n", + " datadef = prediction_pb2.DefaultData(\n", + " names = [\"a\",\"b\"],\n", + " tensor = prediction_pb2.Tensor(\n", + " shape = [3,2],\n", + " values = [1.0,1.0,2.0,3.0,4.0,5.0]\n", + " )\n", + " )\n", + " request = prediction_pb2.SeldonMessage(data = datadef)\n", + " channel = grpc.insecure_channel(AMBASSADOR_API)\n", + " stub = prediction_pb2_grpc.SeldonStub(channel)\n", + " metadata = [('seldon',deploymentName)]\n", + " response = stub.Predict(request=request,metadata=metadata)\n", + " print(response)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Integrating with Kubernetes API" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Validation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Using OpenAPI Schema certain basic validation can be done before the custom resource is accepted." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl create -f resources/model_invalid1.json -n seldon" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Normal Operation\n", + "A simple example is shown below we use a single prepacked model for illustration. The spec contains a set of predictors each of which contains a ***componentSpec*** which is a Kubernetes [PodTemplateSpec](https://kubernetes.io/docs/api-reference/v1.9/#podtemplatespec-v1-core) alongside a ***graph*** which describes how components fit together." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pygmentize resources/model.json" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create Seldon Deployment" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Deploy the runtime graph to kubernetes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl apply -f resources/model.json -n seldon" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl get seldondeployments -n seldon" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl describe seldondeployments seldon-deployment-example -n seldon" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Get the status of the SeldonDeployment. **When ready the replicasAvailable should be 1**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl get seldondeployments seldon-deployment-example -o jsonpath='{.status}' -n seldon" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get predictions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### REST Request" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "rest_request(\"seldon-deployment-example\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### gRPC Request" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "grpc_request(\"seldon-deployment-example\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Update deployment with canary" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will change the deployment to add a \"canary\" deployment. This illustrates:\n", + " - Updating a deployment with no downtime\n", + " - Adding an extra predictor to run alongside th exsting predictor.\n", + " \n", + " You could manage different traffic levels by controlling the number of replicas of each." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pygmentize resources/model_with_canary.json" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl apply -f resources/model_with_canary.json -n seldon" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Check the status of the deployments. Note: **Might need to run several times until replicasAvailable is 1 for both predictors**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl get seldondeployments seldon-deployment-example -o jsonpath='{.status}' -n seldon" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### REST Request" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "rest_request(\"seldon-deployment-example\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### gRPC request" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "grpc_request(\"seldon-deployment-example\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load test" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Start a load test which will post REST requests at 10 requests per second." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!helm install seldon-core-loadtesting --name loadtest \\\n", + " --set locust.host=http://seldon-core-seldon-apiserver:8080 \\\n", + " --set oauth.key=oauth-key \\\n", + " --set oauth.secret=oauth-secret \\\n", + " --namespace seldon \\\n", + " --repo https://storage.googleapis.com/seldon-charts" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You should port-foward the grafana dashboard\n", + "\n", + "```bash\n", + "kubectl port-forward $(kubectl get pods -n seldon -l app=grafana-prom-server -o jsonpath='{.items[0].metadata.name}') -n seldon 3000:3000\n", + "```\n", + "\n", + "You can then iew an analytics dashboard inside the cluster at http://localhost:3000/dashboard/db/prediction-analytics?refresh=5s&orgId=1. Your IP address may be different. get it via minikube ip. Login with:\n", + " - Username : admin\n", + " - password : password (as set when starting seldon-core-analytics above)\n", + " \n", + " The dashboard should look like below:\n", + " \n", + " \n", + " \"predictor" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tear down" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!helm delete loadtest --purge" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl delete -f resources/model_with_canary.json -n seldon" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!helm delete seldon-core-analytics --purge" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!helm delete seldon-core --purge" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!helm delete seldon-core-crd --purge" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +}