From 5639aa4aafe761b97fa4376f18eb1808ccee0468 Mon Sep 17 00:00:00 2001 From: Sachin Varghese Date: Fri, 13 May 2022 19:08:18 +0100 Subject: [PATCH] Alibi-detect iris drift detection example (#191) --- samples/models/iris-drift-detect.yaml | 10 + samples/pipelines/iris-async-drift.yaml | 17 + scheduler/Makefile | 2 +- .../notebooks/pipeline-alibi-detect.ipynb | 296 ++++++++++++++++++ 4 files changed, 324 insertions(+), 1 deletion(-) create mode 100644 samples/models/iris-drift-detect.yaml create mode 100644 samples/pipelines/iris-async-drift.yaml create mode 100644 scheduler/notebooks/pipeline-alibi-detect.ipynb diff --git a/samples/models/iris-drift-detect.yaml b/samples/models/iris-drift-detect.yaml new file mode 100644 index 0000000000..9966d55259 --- /dev/null +++ b/samples/models/iris-drift-detect.yaml @@ -0,0 +1,10 @@ +apiVersion: mlops.seldon.io/v1alpha1 +kind: Model +metadata: + name: iris-drift-detect + namespace: seldon-mesh +spec: + storageUri: "gs://seldon-models/mlserver/alibi-detect/iris-tabular-drift/alibi-detector-artifacts" + requirements: + - mlserver + - alibi-detect diff --git a/samples/pipelines/iris-async-drift.yaml b/samples/pipelines/iris-async-drift.yaml new file mode 100644 index 0000000000..838779e962 --- /dev/null +++ b/samples/pipelines/iris-async-drift.yaml @@ -0,0 +1,17 @@ +apiVersion: mlops.seldon.io/v1alpha1 +kind: Pipeline +metadata: + name: iris-async-drift + namespace: seldon-mesh +spec: + steps: + - name: iris + - name: iris-drift-detect + batch: + size: 2 + inputs: + - iris-async-drift.inputs + output: + steps: + - iris + - iris-drift-detect \ No newline at end of file diff --git a/scheduler/Makefile b/scheduler/Makefile index f7949167fa..90cf4624a9 100644 --- a/scheduler/Makefile +++ b/scheduler/Makefile @@ -7,7 +7,7 @@ MODELGATEWAY_IMG ?= ${DOCKERHUB_USERNAME}/seldon-modelgateway:${CUSTOM_IMAGE_TAG PIPELINEGATEWAY_IMG ?= ${DOCKERHUB_USERNAME}/seldon-pipelinegateway:${CUSTOM_IMAGE_TAG} DATAFLOW_IMG ?= ${DOCKERHUB_USERNAME}/seldon-dataflow-engine:${CUSTOM_IMAGE_TAG} ENVOY_IMG ?= ${DOCKERHUB_USERNAME}/seldon-envoy:${CUSTOM_IMAGE_TAG} -MLSERVER_IMG ?= seldonio/mlserver:1.0.1 +MLSERVER_IMG ?= seldonio/mlserver:1.1.0.dev3 TRITON_IMG ?= nvcr.io/nvidia/tritonserver:21.12-py3 KIND_NAME=ansible diff --git a/scheduler/notebooks/pipeline-alibi-detect.ipynb b/scheduler/notebooks/pipeline-alibi-detect.ipynb new file mode 100644 index 0000000000..e2ddc7cda0 --- /dev/null +++ b/scheduler/notebooks/pipeline-alibi-detect.ipynb @@ -0,0 +1,296 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "lesbian-springer", + "metadata": {}, + "source": [ + "# Seldon V2 Pipeline async drift detection example\n", + "\n", + "\n", + " * Build if needed and place `seldon` binary in your path\n", + " * run `make build-seldon` from operator folder and add bin folder to PATH\n", + " * Run Seldon V2 `make deploy-local` from top level folder\n", + " * Install `kafkacat` for debugging kafka topics & messages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "modular-digest", + "metadata": {}, + "outputs": [], + "source": [ + "!which seldon" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b78ea55e", + "metadata": {}, + "outputs": [], + "source": [ + "!kafkacat -b localhost:9092 -L | grep -i seldon -A1" + ] + }, + { + "cell_type": "markdown", + "id": "ed1c8627", + "metadata": {}, + "source": [ + "### Train the drift detector for an iris model using alibi detect" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3a8536b8", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn import datasets\n", + "iris = datasets.load_iris()\n", + "X, y = iris.data, iris.target" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "91b6a687", + "metadata": {}, + "outputs": [], + "source": [ + "from alibi_detect.cd import TabularDrift\n", + "cd_tabular = TabularDrift(X, p_val=.05)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "947acfa8", + "metadata": {}, + "outputs": [], + "source": [ + "from alibi_detect.utils.saving import save_detector\n", + "filepath = \"alibi-detector-artifacts\"\n", + "save_detector(cd_tabular, filepath)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7854e60a", + "metadata": {}, + "outputs": [], + "source": [ + "# !gsutil -m cp ./alibi-detector-artifacts/ gs://seldon-models/mlserver/alibi-detect/iris-tabular-drift/" + ] + }, + { + "cell_type": "markdown", + "id": "running-antarctica", + "metadata": {}, + "source": [ + "## Model Batched Chaining based on batch size" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0f2ab2a3", + "metadata": {}, + "outputs": [], + "source": [ + "!cat ../../samples/models/sklearn-iris-gs.yaml\n", + "!cat ../../samples/models/iris-drift-detect.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f9e073d7", + "metadata": {}, + "outputs": [], + "source": [ + "!seldon model load -f ../../samples/models/sklearn-iris-gs.yaml \n", + "!seldon model load -f ../../samples/models/iris-drift-detect.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "997b4028", + "metadata": {}, + "outputs": [], + "source": [ + "!seldon model status --model-name iris -w ModelAvailable | jq .\n", + "!seldon model status --model-name iris-drift-detect -w ModelAvailable | jq ." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3d017ede", + "metadata": {}, + "outputs": [], + "source": [ + "!cat ../../samples/pipelines/iris-async-drift.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "following-winning", + "metadata": {}, + "outputs": [], + "source": [ + "!seldon pipeline load -f ../../samples/pipelines/iris-async-drift.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "artistic-kentucky", + "metadata": {}, + "outputs": [], + "source": [ + "!seldon pipeline status -p iris-async-drift -w PipelineReady| jq ." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5c33133e", + "metadata": {}, + "outputs": [], + "source": [ + "!seldon pipeline infer -p iris-async-drift --inference-mode grpc \\\n", + " '{\"model_name\":\"iris\",\"inputs\":[{\"name\":\"INPUT0\",\"contents\":{\"int_contents\":[1,2,3,4]},\"datatype\":\"INT32\",\"shape\":[1,4]}]}' | jq ." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4c2f6d4a", + "metadata": {}, + "outputs": [], + "source": [ + "!kafkacat -b localhost:9092 -t seldon.default.model.iris-drift-detect.outputs -C -o beginning -e -f 'Offset %o\\tlength %S\\tcontents %s\\n' -s value=i" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "553c8b9a", + "metadata": {}, + "outputs": [], + "source": [ + "!kafkacat -b localhost:9092 -t seldon.default.model.iris-drift-detect.inputs -C -o beginning -e -f 'Offset %o\\tlength %S\\tcontents %s\\n' -s value=i" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "93f472fc", + "metadata": {}, + "outputs": [], + "source": [ + "!seldon pipeline infer -p iris-async-drift --inference-mode grpc \\\n", + " '{\"model_name\":\"iris\",\"inputs\":[{\"name\":\"INPUT0\",\"contents\":{\"int_contents\":[1,2,3,4]},\"datatype\":\"INT32\",\"shape\":[1,4]}]}' | jq ." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cd635024", + "metadata": {}, + "outputs": [], + "source": [ + "!kafkacat -b localhost:9092 -L | grep -i seldon -A1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "95f63878", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "!kafkacat -b localhost:9092 -t seldon.default.pipeline.iris-async-drift.inputs -C -o beginning -e -f 'Offset %o\\tlength %S\\tcontents %s\\n' -s value=i" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c37ed57e", + "metadata": {}, + "outputs": [], + "source": [ + "!kafkacat -b localhost:9092 -t seldon.default.model.iris.inputs -C -o beginning -e -f 'Offset %o\\tlength %S\\tcontents %s\\n' -s value=i" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8d9dbbfb", + "metadata": {}, + "outputs": [], + "source": [ + "!kafkacat -b localhost:9092 -t seldon.default.model.iris.outputs -C -o beginning -e -f 'Offset %o\\tlength %S\\tcontents %s\\n' -s value=i" + ] + }, + { + "cell_type": "markdown", + "id": "2e51e17f", + "metadata": {}, + "source": [ + "### Cleanup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dimensional-hours", + "metadata": {}, + "outputs": [], + "source": [ + "!seldon pipeline unload -p iris-async-drift" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "outside-inspiration", + "metadata": {}, + "outputs": [], + "source": [ + "!seldon model unload --model-name iris\n", + "!seldon model unload --model-name iris-drift-detect" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "alibi", + "language": "python", + "name": "alibi" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}