Alibi-detect iris drift detection example (#191)

SeldonIO · May 13, 2022 · 5639aa4 · 5639aa4
1 parent aa6f7fa
commit 5639aa4
Show file tree

Hide file tree

Showing 4 changed files with 324 additions and 1 deletion.
diff --git a/samples/models/iris-drift-detect.yaml b/samples/models/iris-drift-detect.yaml
@@ -0,0 +1,10 @@
+apiVersion: mlops.seldon.io/v1alpha1
+kind: Model
+metadata:
+  name: iris-drift-detect
+  namespace: seldon-mesh
+spec:
+  storageUri: "gs://seldon-models/mlserver/alibi-detect/iris-tabular-drift/alibi-detector-artifacts"
+  requirements:
+    - mlserver
+    - alibi-detect
diff --git a/samples/pipelines/iris-async-drift.yaml b/samples/pipelines/iris-async-drift.yaml
@@ -0,0 +1,17 @@
+apiVersion: mlops.seldon.io/v1alpha1
+kind: Pipeline
+metadata:
+  name: iris-async-drift
+  namespace: seldon-mesh
+spec:
+  steps:
+  - name: iris
+  - name: iris-drift-detect
+    batch:
+      size: 2
+    inputs:
+    - iris-async-drift.inputs
+  output:
+    steps:
+    - iris
+    - iris-drift-detect
diff --git a/scheduler/Makefile b/scheduler/Makefile
@@ -7,7 +7,7 @@ MODELGATEWAY_IMG ?= ${DOCKERHUB_USERNAME}/seldon-modelgateway:${CUSTOM_IMAGE_TAG
 PIPELINEGATEWAY_IMG ?= ${DOCKERHUB_USERNAME}/seldon-pipelinegateway:${CUSTOM_IMAGE_TAG}
 DATAFLOW_IMG ?= ${DOCKERHUB_USERNAME}/seldon-dataflow-engine:${CUSTOM_IMAGE_TAG}
 ENVOY_IMG ?= ${DOCKERHUB_USERNAME}/seldon-envoy:${CUSTOM_IMAGE_TAG}
-MLSERVER_IMG ?= seldonio/mlserver:1.0.1
+MLSERVER_IMG ?= seldonio/mlserver:1.1.0.dev3
 TRITON_IMG ?= nvcr.io/nvidia/tritonserver:21.12-py3
 KIND_NAME=ansible
 

diff --git a/scheduler/notebooks/pipeline-alibi-detect.ipynb b/scheduler/notebooks/pipeline-alibi-detect.ipynb
@@ -0,0 +1,296 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "lesbian-springer",
+   "metadata": {},
+   "source": [
+    "# Seldon V2 Pipeline async drift detection example\n",
+    "\n",
+    "\n",
+    " * Build if needed and place `seldon` binary in your path\n",
+    "    * run `make build-seldon` from operator folder and add bin folder to PATH\n",
+    " * Run Seldon V2 `make deploy-local` from top level folder\n",
+    " * Install `kafkacat` for debugging kafka topics & messages"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "modular-digest",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!which seldon"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b78ea55e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!kafkacat -b localhost:9092 -L | grep -i seldon -A1"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ed1c8627",
+   "metadata": {},
+   "source": [
+    "### Train the drift detector for an iris model using alibi detect"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3a8536b8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn import datasets\n",
+    "iris = datasets.load_iris()\n",
+    "X, y = iris.data, iris.target"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "91b6a687",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from alibi_detect.cd import TabularDrift\n",
+    "cd_tabular = TabularDrift(X, p_val=.05)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "947acfa8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from alibi_detect.utils.saving import save_detector\n",
+    "filepath = \"alibi-detector-artifacts\"\n",
+    "save_detector(cd_tabular, filepath)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7854e60a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# !gsutil -m cp ./alibi-detector-artifacts/ gs://seldon-models/mlserver/alibi-detect/iris-tabular-drift/"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "running-antarctica",
+   "metadata": {},
+   "source": [
+    "## Model Batched Chaining based on batch size"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0f2ab2a3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!cat ../../samples/models/sklearn-iris-gs.yaml\n",
+    "!cat ../../samples/models/iris-drift-detect.yaml"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f9e073d7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!seldon model load -f ../../samples/models/sklearn-iris-gs.yaml \n",
+    "!seldon model load -f ../../samples/models/iris-drift-detect.yaml"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "997b4028",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!seldon model status --model-name iris -w ModelAvailable | jq .\n",
+    "!seldon model status --model-name iris-drift-detect -w ModelAvailable | jq ."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3d017ede",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!cat ../../samples/pipelines/iris-async-drift.yaml"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "following-winning",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!seldon pipeline load -f ../../samples/pipelines/iris-async-drift.yaml"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "artistic-kentucky",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!seldon pipeline status -p iris-async-drift -w PipelineReady| jq ."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5c33133e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!seldon pipeline infer -p iris-async-drift --inference-mode grpc \\\n",
+    "    '{\"model_name\":\"iris\",\"inputs\":[{\"name\":\"INPUT0\",\"contents\":{\"int_contents\":[1,2,3,4]},\"datatype\":\"INT32\",\"shape\":[1,4]}]}' | jq ."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4c2f6d4a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!kafkacat -b localhost:9092 -t seldon.default.model.iris-drift-detect.outputs -C -o beginning -e -f 'Offset %o\\tlength %S\\tcontents %s\\n' -s value=i"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "553c8b9a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!kafkacat -b localhost:9092 -t seldon.default.model.iris-drift-detect.inputs -C -o beginning -e -f 'Offset %o\\tlength %S\\tcontents %s\\n' -s value=i"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "93f472fc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!seldon pipeline infer -p iris-async-drift --inference-mode grpc \\\n",
+    "    '{\"model_name\":\"iris\",\"inputs\":[{\"name\":\"INPUT0\",\"contents\":{\"int_contents\":[1,2,3,4]},\"datatype\":\"INT32\",\"shape\":[1,4]}]}' | jq ."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cd635024",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!kafkacat -b localhost:9092 -L | grep -i seldon -A1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "95f63878",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "!kafkacat -b localhost:9092 -t seldon.default.pipeline.iris-async-drift.inputs -C -o beginning -e -f 'Offset %o\\tlength %S\\tcontents %s\\n' -s value=i"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c37ed57e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!kafkacat -b localhost:9092 -t seldon.default.model.iris.inputs -C -o beginning -e -f 'Offset %o\\tlength %S\\tcontents %s\\n' -s value=i"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8d9dbbfb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!kafkacat -b localhost:9092 -t seldon.default.model.iris.outputs -C -o beginning -e -f 'Offset %o\\tlength %S\\tcontents %s\\n' -s value=i"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2e51e17f",
+   "metadata": {},
+   "source": [
+    "### Cleanup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dimensional-hours",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!seldon pipeline unload -p iris-async-drift"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "outside-inspiration",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!seldon model unload --model-name iris\n",
+    "!seldon model unload --model-name iris-drift-detect"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "alibi",
+   "language": "python",
+   "name": "alibi"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}