Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix outlier detection divide by zero and add initial mnist example (wip) #243

Merged
merged 1 commit into from
Oct 2, 2018
Merged

Fix outlier detection divide by zero and add initial mnist example (wip) #243

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
359 changes: 359 additions & 0 deletions examples/transformers/mnist_outlier/mnist_outlier.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,359 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# MNIST Outlier Detection (WIP)\n",
"\n",
"Skaffolding for MNIST Outlier Detection demo.\n",
"\n",
"Create Minikube:\n",
"\n",
"```\n",
"minikube start --vm-driver kvm2 --memory 4096 --feature-gates=CustomResourceValidation=true --extra-config=apiserver.Authorization.Mode=RBAC\n",
"```"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Setup"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%matplotlib inline\n",
"import requests\n",
"from random import randint,random\n",
"import json\n",
"from matplotlib import pyplot as plt\n",
"import numpy as np\n",
"from tensorflow.examples.tutorials.mnist import input_data\n",
"import sys\n",
"sys.path.append(\"../../../notebooks\")\n",
"from visualizer import get_graph"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def gen_image(arr):\n",
" two_d = (np.reshape(arr, (28, 28)) * 255).astype(np.uint8)\n",
" plt.imshow(two_d,cmap=plt.cm.gray_r, interpolation='nearest')\n",
" return plt\n",
"\n",
"def download_mnist():\n",
" return input_data.read_data_sets(\"MNIST_data/\", one_hot = True)\n",
"\n",
"def rest_predict_request(endpoint,data):\n",
" request = {\"data\":{\"ndarray\":data.tolist()}}\n",
" response = requests.post(\n",
" \"http://\"+endpoint+\"/predict\",\n",
" data={\"json\":json.dumps(request),\"isDefault\":True})\n",
" return response.json() \n",
"\n",
"def rest_transform_input_request(endpoint,data):\n",
" request = {\"data\":{\"ndarray\":data.tolist()}}\n",
" response = requests.post(\n",
" \"http://\"+endpoint+\"/transform-input\",\n",
" data={\"json\":json.dumps(request),\"isDefault\":True})\n",
" return response.json() \n",
"\n",
"def rest_transform_output_request(endpoint,data):\n",
" request = {\"data\":{\"ndarray\":data.tolist()}}\n",
" response = requests.post(\n",
" \"http://\"+endpoint+\"/transform-output\",\n",
" data={\"json\":json.dumps(request),\"isDefault\":True})\n",
" return response.json() \n",
"\n",
"def rest_request_ambassador(deploymentName,endpoint=\"localhost:8003\",arr=None):\n",
" payload = {\"data\":{\"names\":[\"a\",\"b\"],\"tensor\":{\"shape\":[1,784],\"values\":arr.tolist()}}}\n",
" response = requests.post(\n",
" \"http://\"+endpoint+\"/seldon/\"+deploymentName+\"/api/v0.1/predictions\",\n",
" json=payload)\n",
" print(response.status_code)\n",
" print(response.text)\n",
"\n",
"\n",
"def gen_mnist_data(mnist):\n",
" batch_xs, batch_ys = mnist.train.next_batch(1)\n",
" chosen=0\n",
" gen_image(batch_xs[chosen]).show()\n",
" data = batch_xs[chosen].reshape((1,784))\n",
" return data\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"mnist = download_mnist()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Test "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!kubectl create clusterrolebinding kube-system-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!kubectl create namespace seldon"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!kubectl config set-context $(kubectl config current-context) --namespace=seldon"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!kubectl -n kube-system create sa tiller\n",
"!kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller\n",
"!helm init --service-account tiller"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!kubectl rollout status deploy/tiller-deploy -n kube-system"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!helm install ../../../helm-charts/seldon-core-crd --name seldon-core-crd \\\n",
" --set usage_metrics.enabled=true"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!helm install ../../../helm-charts/seldon-core --name seldon-core \\\n",
" --namespace seldon \\\n",
" --set ambassador.enabled=true"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!kubectl rollout status deploy/seldon-core-seldon-cluster-manager -n seldon\n",
"!kubectl rollout status deploy/seldon-core-seldon-apiserver -n seldon"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!helm install ../../../helm-charts/seldon-single-model \\\n",
" --set outlier_detection.enabled=true \\\n",
" --set outlier_detection.image.name=seldonio/outlier_mahalanobis:0.3 \\\n",
" --set outlier_detection.n_components=3 \\\n",
" --name seldon-model --set oauth.key=oauth-key \\\n",
" --set oauth.secret=oauth-secret \\\n",
" --set model.image.name=seldonio/sk-mnist:0.1 \\\n",
" --namespace=seldon"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Port forward Ambassador**\n",
"\n",
"```\n",
"kubectl port-forward $(kubectl get pods -n seldon -l service=ambassador -o jsonpath='{.items[0].metadata.name}') -n seldon 8003:8080\n",
"```"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Test with Real Data"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"data = gen_mnist_data(mnist)\n",
"data = data.reshape((784))\n",
"rest_request_ambassador(\"seldon-model\",endpoint=\"localhost:8003\",arr=data)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Test with Random Data"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"\n",
"def create_random_data(data_size,rows=1):\n",
" shape = [rows,data_size]\n",
" arr = np.random.rand(rows*data_size)\n",
" return (shape,arr)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"shape,data = create_random_data(784)\n",
"rest_request_ambassador(\"seldon-model\",endpoint=\"localhost:8003\",arr=data)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Analytics and Load Test"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!helm install ../../../helm-charts/seldon-core-analytics --name seldon-core-analytics \\\n",
" --set grafana_prom_admin_password=password \\\n",
" --set persistence.enabled=false \\\n",
" --namespace seldon"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!kubectl label nodes $(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') role=locust"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!helm install seldon-core-loadtesting --name loadtest \\\n",
" --namespace seldon \\\n",
" --repo https://storage.googleapis.com/seldon-charts \\\n",
" --set locust.script=mnist_rest_locust.py \\\n",
" --set locust.host=http://test-deployment:8000 \\\n",
" --set oauth.enabled=false \\\n",
" --set locust.hatchRate=1 \\\n",
" --set locust.clients=1 \\\n",
" --set loadtest.sendFeedback=1 \\\n",
" --set locust.minWait=0 \\\n",
" --set locust.maxWait=0 \\\n",
" --set replicaCount=1 \\\n",
" --set data.size=784\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"You should port-foward the grafana dashboard\n",
"\n",
"```\n",
"kubectl port-forward $(kubectl get pods -n seldon -l app=grafana-prom-server -o jsonpath='{.items[0].metadata.name}') -n seldon 3000:3000\n",
"```\n",
"\n",
"You can then view an analytics dashboard inside the cluster at http://localhost:3000/dashboard/db/prediction-analytics?refresh=5s&orgId=1. Your IP address may be different. get it via minikube ip. Login with:\n",
"\n",
" Username : admin\n",
"\n",
" password : password (as set when starting seldon-core-analytics above)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.4"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
10 changes: 10 additions & 0 deletions examples/transformers/outlier_mahalanobis/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
IMAGE_VERSION=0.3
IMAGE_NAME = seldonio/outlier_mahalanobis

build:
s2i build . seldonio/seldon-core-s2i-python3:0.2 $(IMAGE_NAME):$(IMAGE_VERSION)

push_to_dockerhub:
docker push $(IMAGE_NAME):$(IMAGE_VERSION)


4 changes: 3 additions & 1 deletion examples/transformers/outlier_mahalanobis/OutlierMahalanobis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ def score(self,features,feature_names):
else:
n = self.n

print("n=",n,"nb=",nb,"p=",p,"n_components=",self.n_components)

# Tracking the mean and covariance matrix
roll_partial_means = features.cumsum(axis=0)/(np.arange(nb)+1).reshape((nb,1))
coefs = (np.arange(nb)+1.)/(np.arange(nb)+n+1.)
Expand All @@ -31,7 +33,7 @@ def score(self,features,feature_names):

coefs = ((n+np.arange(nb))/(n+np.arange(nb)+1.)).reshape((nb,1,1))
B = coefs*np.matmul((features - new_means_offset)[:,:,None],(features - new_means_offset)[:,None,:])
cov_batch = (n-1.)/(n+nb-1.)*self.C + 1./(n+nb-1.)*B.sum(axis=0)
cov_batch = (n-1.)/(n+max(1,nb-1.))*self.C + 1./(n+max(1,nb-1.))*B.sum(axis=0)

# PCA
eigvals, eigvects = eigh(cov_batch,eigvals=(p-n_components,p-1))
Expand Down
Loading