Merge pull request #143 from cliveseldon/mnist_loadtest

MNIST loadtest

.gitignore

@@ -78,4 +78,6 @@ examples/models/mean_classifier/build/
 notebooks/my-ml-deployment/
 
 wrappers/s2i/python/Dockerfile
-wrappers/s2i/python/_wrappers
+wrappers/s2i/python/_wrappers
+
+.Rhistory

examples/models/deep_mnist/README.md

@@ -45,7 +45,12 @@ python ../../../wrappers/testing/tester.py contract.json 0.0.0.0 5000 -p
 minikube start --memory 4096
 ```
 
-[Install seldon core](/readme.md#install)
+Install seldon core
+
+```
+helm install seldon-core-crd --name seldon-core-crd --repo https://storage.googleapis.com/seldon-charts --set usage_metrics.enabled=true --set rbac.enabled=false
+helm install seldon-core --name seldon-core --repo https://storage.googleapis.com/seldon-charts --set rbac.enabled=false
+```
 
 Connect to Minikube Docker daemon
 
@@ -68,7 +73,7 @@ kubectl create -f deep_mnist.json
 Port forward API server
 
 ```bash
-kubectl port-forward $(kubectl get pods -n seldon -l app=seldon-apiserver-container-app -o jsonpath='{.items[0].metadata.name}') -n seldon 8080:8080
+kubectl port-forward $(kubectl get pods -l app=seldon-apiserver-container-app -o jsonpath='{.items[0].metadata.name}') 8080:8080
 ```
 
 Ensure tester gRPC modules compiled. You will need to install grpc-tools module.

examples/models/keras_mnist/README.md

@@ -46,7 +46,12 @@ python ../../../wrappers/testing/tester.py contract.json 0.0.0.0 5000 -p
 minikube start --memory 4096
 ```
 
-[Install seldon core](/readme.md#install)
+Install seldon core
+
+```
+helm install seldon-core-crd --name seldon-core-crd --repo https://storage.googleapis.com/seldon-charts --set usage_metrics.enabled=true --set rbac.enabled=false
+helm install seldon-core --name seldon-core --repo https://storage.googleapis.com/seldon-charts --set rbac.enabled=false
+```
 
 Connect to Minikube Docker daemon
 
@@ -69,7 +74,7 @@ kubectl create -f keras_mnist_deployment.json
 Port forward API server
 
 ```bash
-kubectl port-forward $(kubectl get pods -n seldon -l app=seldon-apiserver-container-app -o jsonpath='{.items[0].metadata.name}') -n seldon 8080:8080
+kubectl port-forward $(kubectl get pods -l app=seldon-apiserver-container-app -o jsonpath='{.items[0].metadata.name}') 8080:8080
 ```
 
 Ensure tester gRPC modules compiled. 

examples/models/r_iris/README.md

@@ -1,4 +1,4 @@
-# Deep MNIST
+# R model for IRIS dataset
 An R Iris model.
 
 ## Depenencies
@@ -43,7 +43,12 @@ python ../../../wrappers/testing/tester.py contract.json 0.0.0.0 5000 -p
 minikube start --memory 4096
 ```
 
-[Install seldon core](/readme.md#install)
+Install seldon core
+
+```
+helm install seldon-core-crd --name seldon-core-crd --repo https://storage.googleapis.com/seldon-charts --set usage_metrics.enabled=true --set rbac.enabled=false
+helm install seldon-core --name seldon-core --repo https://storage.googleapis.com/seldon-charts --set rbac.enabled=false
+```
 
 Connect to Minikube Docker daemon
 
@@ -66,7 +71,7 @@ kubectl create -f r_iris_deployment.json
 Port forward API server
 
 ```bash
-kubectl port-forward $(kubectl get pods -n seldon -l app=seldon-apiserver-container-app -o jsonpath='{.items[0].metadata.name}') -n seldon 8080:8080
+kubectl port-forward $(kubectl get pods -l app=seldon-apiserver-container-app -o jsonpath='{.items[0].metadata.name}') 8080:8080
 ```
 
 Ensure tester gRPC modules compiled. 

examples/models/r_mnist/.s2i/environment

@@ -0,0 +1,4 @@
+MODEL_NAME=mnist.R
+API_TYPE=REST
+SERVICE_TYPE=MODEL
+PERSISTENCE=0

examples/models/r_mnist/Makefile

@@ -0,0 +1,9 @@
+
+
+train:
+	./get_data.sh
+	Rscript train.R
+
+clean:
+	rm -f *ubyte
+	rm -f model.Rds	

examples/models/r_mnist/README.md

@@ -0,0 +1,88 @@
+# Deep MNIST
+An R MNIST model.
+
+## Depenencies
+
+R
+
+## Train locally
+
+```bash
+make train
+```
+
+## Wrap using [s2i](https://github.com/openshift/source-to-image#installation).
+
+```bash
+s2i build . seldonio/seldon-core-s2i-r r-mnist:0.1
+```
+
+## Local Docker Smoke Test
+
+Run under docker.
+
+```bash
+docker run --rm -p 5000:5000 r-mnist:0.1
+```
+
+Ensure test grpc modules compiled.
+
+```bash
+pushd ../../../wrappers/testing ; make build_protos ; popd
+```
+
+Send a data request using the wrapper tester.
+
+```bash
+python ../../../wrappers/testing/tester.py contract.json 0.0.0.0 5000 -p
+```
+
+## Minikube test
+
+```bash
+minikube start --memory 4096
+```
+
+Install seldon core
+
+```
+helm install seldon-core-crd --name seldon-core-crd --repo https://storage.googleapis.com/seldon-charts --set usage_metrics.enabled=true --set rbac.enabled=false
+helm install seldon-core --name seldon-core --repo https://storage.googleapis.com/seldon-charts --set rbac.enabled=false
+```
+
+Connect to Minikube Docker daemon
+
+```bash
+eval $(minikube docker-env)
+```
+
+Build image using minikube docker daemon.
+
+```bash
+s2i build . seldonio/seldon-core-s2i-r r-mnist:0.1
+```
+
+Launch deployment
+
+```bash
+kubectl create -f r_mnist_deployment.json -n seldon
+```
+
+Port forward API server
+
+```bash
+kubectl port-forward $(kubectl get pods -l app=seldon-apiserver-container-app -o jsonpath='{.items[0].metadata.name}') 8080:8080
+```
+
+Ensure tester gRPC modules compiled. 
+
+```bash
+pushd ../../../util/api_tester ; make build_protos ; popd
+```
+
+Send test request
+```bash
+python ../../../util/api_tester/api-tester.py contract.json 0.0.0.0 8080 --oauth-key oauth-key --oauth-secret oauth-secret -p
+```
+
+

examples/models/r_mnist/contract.json

@@ -0,0 +1,22 @@
+{
+    "features":[
+	{
+	    "name":"X",
+	    "dtype":"FLOAT",
+	    "ftype":"continuous",
+	    "range":[0,1],
+	    "repeat":784
+	}
+    ],
+    "targets":[
+	{
+	    "name":"class",
+	    "dtype":"FLOAT",
+	    "ftype":"continuous",
+	    "range":[0,1],
+	    "repeat":10
+	}
+    ]
+}
+
+

examples/models/r_mnist/get_data.sh

@@ -0,0 +1,10 @@
+wget http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz
+wget http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz
+wget http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz
+wget http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz
+
+
+gunzip train-images-idx3-ubyte.gz
+gunzip train-labels-idx1-ubyte.gz
+gunzip t10k-images-idx3-ubyte.gz
+gunzip t10k-labels-idx1-ubyte.gz

examples/models/r_mnist/install.R

@@ -0,0 +1,2 @@
+install.packages('pls')
+

examples/models/r_mnist/mnist.R

@@ -0,0 +1,17 @@
+library(methods)
+
+predict.mnist <- function(mnist,newdata=list()) {
+  cn <- 1:784
+  for (i in seq_along(cn)){cn[i] <- paste("X",cn[i],sep = "")}
+  colnames(newdata) <- cn
+  predict(mnist$model, newdata = newdata, type='prob')
+}
+
+new_mnist <- function(filename) {
+  model <- readRDS(filename)
+  structure(list(model=model), class = "mnist")
+}
+
+initialise_seldon <- function(params) {
+  new_mnist("model.Rds")
+}

examples/models/r_mnist/r_mnist_deployment.json

@@ -0,0 +1,53 @@
+{
+    "apiVersion": "machinelearning.seldon.io/v1alpha1",
+    "kind": "SeldonDeployment",
+    "metadata": {
+        "labels": {
+            "app": "seldon"
+        },
+        "name": "seldon-deployment-example"
+    },
+    "spec": {
+        "annotations": {
+            "project_name": "Mnist classification",
+            "deployment_version": "0.1"
+        },
+        "name": "r-mnist-deployment",
+        "oauth_key": "oauth-key",
+        "oauth_secret": "oauth-secret",
+        "predictors": [
+            {
+                "componentSpec": {
+                    "spec": {
+                        "containers": [
+                            {
+                                "image": "r-mnist:0.1",
+                                "imagePullPolicy": "IfNotPresent",
+                                "name": "r-mnist-classifier",
+                                "resources": {
+                                    "requests": {
+                                        "memory": "1Mi"
+                                    }
+                                }
+                            }
+                        ],
+                        "terminationGracePeriodSeconds": 20
+                    }
+                },
+                "graph": {
+                    "children": [],
+                    "name": "r-mnist-classifier",
+                    "endpoint": {
+                        "type" : "REST"
+                    },
+                    "type": "MODEL"
+                },
+                "name": "r-mnist-predictor",
+                "replicas": 1,
+                "annotations": {
+                "predictor_version" : "0.1"
+                }
+            }
+        ]
+    }
+}

examples/models/r_mnist/train.R

@@ -0,0 +1,68 @@
+library(caret)
+library(doParallel)
+
+# Enable parallel processing.
+cl <- makeCluster(detectCores())
+registerDoParallel(cl)
+
+# Load the MNIST digit recognition dataset into R
+# http://yann.lecun.com/exdb/mnist/
+# assume you have all 4 files and gunzip'd them
+# creates train$n, train$x, train$y  and test$n, test$x, test$y
+# e.g. train$x is a 60000 x 784 matrix, each row is one digit (28x28)
+# call:  show_digit(train$x[5,])   to see a digit.
+# brendan o'connor - gist.github.com/39760 - anyall.org
+load_mnist <- function() {
+  load_image_file <- function(filename) {
+    ret = list()
+    f = file(filename,'rb')
+    readBin(f,'integer',n=1,size=4,endian='big')
+    ret$n = readBin(f,'integer',n=1,size=4,endian='big')
+    nrow = readBin(f,'integer',n=1,size=4,endian='big')
+    ncol = readBin(f,'integer',n=1,size=4,endian='big')
+    x = readBin(f,'integer',n=ret$n*nrow*ncol,size=1,signed=F)
+    ret$x = matrix(x, ncol=nrow*ncol, byrow=T)
+    close(f)
+    ret
+  }
+  load_label_file <- function(filename) {
+    f = file(filename,'rb')
+    readBin(f,'integer',n=1,size=4,endian='big')
+    n = readBin(f,'integer',n=1,size=4,endian='big')
+    y = readBin(f,'integer',n=n,size=1,signed=F)
+    close(f)
+    y
+  }
+  train <<- load_image_file('train-images-idx3-ubyte')
+  test <<- load_image_file('t10k-images-idx3-ubyte')
+
+  train$y <<- load_label_file('train-labels-idx1-ubyte')
+  test$y <<- load_label_file('t10k-labels-idx1-ubyte')  
+}
+
+show_digit <- function(arr784, col=gray(12:1/12), ...) {
+  image(matrix(arr784, nrow=28)[,28:1], col=col, ...)
+}
+
+train <- data.frame()
+test <- data.frame()
+
+# Load data.
+load_mnist()
+
+# Normalize: X = (X - min) / (max - min) => X = (X - 0) / (255 - 0) => X = X / 255.
+train$x <- train$x / 255
+
+# Setup training data with digit and pixel values with 60/40 split for train/cv.
+inTrain = data.frame(y=train$y, train$x)
+inTrain$y <- as.factor(inTrain$y)
+trainIndex = createDataPartition(inTrain$y, p = 0.60,list=FALSE)
+training = inTrain[trainIndex,]
+cv = inTrain[-trainIndex,]
+
+# SVM. 95/94.
+#fit <- train(y ~ ., data = head(training, 1000), method = 'svmRadial', tuneGrid = data.frame(sigma=0.0107249, C=1))
+fit <- train(y ~ ., data = head(training, 1000), method = 'pls')
+results <- predict(fit, newdata = head(cv, 1000), type='prob')
+#confusionMatrix(results, head(cv$y, 1000))
+saveRDS(fit, file = "model.Rds", compress = TRUE)

examples/models/sklearn_iris/README.md

@@ -46,7 +46,12 @@ python ../../../wrappers/testing/tester.py contract.json 0.0.0.0 5000 -p
 minikube start --memory 4096
 ```
 
-[Install seldon core](/readme.md#install)
+Install seldon core
+
+```
+helm install seldon-core-crd --name seldon-core-crd --repo https://storage.googleapis.com/seldon-charts --set usage_metrics.enabled=true --set rbac.enabled=false
+helm install seldon-core --name seldon-core --repo https://storage.googleapis.com/seldon-charts --set rbac.enabled=false
+```
 
 Connect to Minikube Docker daemon
 
@@ -69,7 +74,7 @@ kubectl create -f sklearn_iris_deployment.json
 Port forward API server
 
 ```bash
-kubectl port-forward $(kubectl get pods -n seldon -l app=seldon-apiserver-container-app -o jsonpath='{.items[0].metadata.name}') -n seldon 8080:8080
+kubectl port-forward $(kubectl get pods -l app=seldon-apiserver-container-app -o jsonpath='{.items[0].metadata.name}') 8080:8080
 ```
 
 Ensure tester gRPC modules compiled. 

helm-charts/seldon-core-loadtesting/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 description: Loadtesting for seldon core
 name: seldon-core-loadtesting
-version: 0.1.0
+version: 0.1.1
 keywords:
 - kubernetes
 - machine-learning