diff --git a/doc/source/graph/protocols.md b/doc/source/graph/protocols.md index 66c0f887ce..8f13e78a2f 100644 --- a/doc/source/graph/protocols.md +++ b/doc/source/graph/protocols.md @@ -84,5 +84,6 @@ In particular, | [TRITON_SERVER](../servers/triton.md) | ✅ | [NVIDIA Triton](https://github.com/triton-inference-server/server) | | [SKLEARN_SERVER](../servers/sklearn.md) | ✅ | [Seldon MLServer](https://github.com/seldonio/mlserver) | | [XGBOOST_SERVER](../servers/xgboost.md) | ✅ | [Seldon MLServer](https://github.com/seldonio/mlserver) | +| [MLFLOW_SERVER](../servers/mlflow.md) | ✅ | [Seldon MLServer](https://github.com/seldonio/mlserver) | You can try out the `kfserving` in [this example notebook](../examples/protocol_examples.html). diff --git a/doc/source/servers/mlflow.md b/doc/source/servers/mlflow.md index 4c322e6a1f..2f39c32206 100644 --- a/doc/source/servers/mlflow.md +++ b/doc/source/servers/mlflow.md @@ -81,3 +81,38 @@ You can also try out a [worked notebook](../examples/server_examples.html#Serve-MLflow-Elasticnet-Wines-Model) or check our [talk at the Spark + AI Summit 2019](https://www.youtube.com/watch?v=D6eSfd9w9eA). + +## V2 KFServing protocol [Incubating] + +.. Warning:: + Support for the V2 KFServing protocol is still considered an incubating + feature. + This means that some parts of Seldon Core may still not be supported (e.g. + tracing, graphs, etc.). + +The MLFlow server can also be used to expose an API compatible with the [V2 +KFServing Protocol](../graph/protocols.md#v2-kfserving-protocol). +Note that, under the hood, it will use the [Seldon +MLServer](https://github.com/SeldonIO/MLServer) runtime. + +In order to enable support for the V2 KFServing protocol, it's enough to +specify the `protocol` of the `SeldonDeployment` to use `kfserving`. +For example, + +```yaml +apiVersion: machinelearning.seldon.io/v1alpha2 +kind: SeldonDeployment +metadata: + name: mlflow +spec: + protocol: kfserving # Activate the v2 protocol + name: wines + predictors: + - graph: + children: [] + implementation: MLFLOW_SERVER + modelUri: gs://seldon-models/v1.10.0-dev/mlflow/elasticnet_wine + name: classifier + name: default + replicas: 1 +``` \ No newline at end of file diff --git a/helm-charts/seldon-core-operator/values.yaml b/helm-charts/seldon-core-operator/values.yaml index f5cef42c15..4997c9ad5a 100644 --- a/helm-charts/seldon-core-operator/values.yaml +++ b/helm-charts/seldon-core-operator/values.yaml @@ -113,6 +113,9 @@ predictor_servers: seldon: defaultImageVersion: "1.10.0-dev" image: seldonio/mlflowserver + kfserving: + defaultImageVersion: "0.4.0.dev1" + image: seldonio/mlserver SKLEARN_SERVER: protocols: seldon: diff --git a/notebooks/server_examples.ipynb b/notebooks/server_examples.ipynb index f75aaa3b26..2a040e9bbb 100644 --- a/notebooks/server_examples.ipynb +++ b/notebooks/server_examples.ipynb @@ -981,11 +981,200 @@ "source": [ "!kubectl delete -f ./resources/elasticnet_wine.yaml" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## MLFlow kfserving v2 protocol" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%writefile ./resources/elasticnet_wine_v2.yaml\n", + "apiVersion: machinelearning.seldon.io/v1alpha2\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " name: mlflow\n", + "spec:\n", + " protocol: kfserving # Activate v2 protocol\n", + " name: wines\n", + " predictors:\n", + " - graph:\n", + " children: []\n", + " implementation: MLFLOW_SERVER\n", + " modelUri: gs://seldon-models/v1.10.0-dev/mlflow/elasticnet_wine\n", + " name: classifier\n", + " name: default\n", + " replicas: 1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl apply -f ./resources/elasticnet_wine_v2.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=mlflow -o jsonpath='{.items[0].metadata.name}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## REST requests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "import requests\n", + "\n", + "inference_request = {\n", + " \"parameters\": {\n", + " \"content_type\": \"pd\"\n", + " },\n", + " \"inputs\": [\n", + " {\n", + " \"name\": \"fixed acidity\",\n", + " \"shape\": [1],\n", + " \"datatype\": \"FP32\",\n", + " \"data\": [7.4],\n", + " \"parameters\": {\n", + " \"content_type\": \"np\"\n", + " }\n", + " },\n", + " {\n", + " \"name\": \"volatile acidity\",\n", + " \"shape\": [1],\n", + " \"datatype\": \"FP32\",\n", + " \"data\": [0.7000],\n", + " \"parameters\": {\n", + " \"content_type\": \"np\"\n", + " }\n", + " },\n", + " {\n", + " \"name\": \"citric acidity\",\n", + " \"shape\": [1],\n", + " \"datatype\": \"FP32\",\n", + " \"data\": [0],\n", + " \"parameters\": {\n", + " \"content_type\": \"np\"\n", + " }\n", + " },\n", + " {\n", + " \"name\": \"residual sugar\",\n", + " \"shape\": [1],\n", + " \"datatype\": \"FP32\",\n", + " \"data\": [1.9],\n", + " \"parameters\": {\n", + " \"content_type\": \"np\"\n", + " }\n", + " },\n", + " {\n", + " \"name\": \"chlorides\",\n", + " \"shape\": [1],\n", + " \"datatype\": \"FP32\",\n", + " \"data\": [0.076],\n", + " \"parameters\": {\n", + " \"content_type\": \"np\"\n", + " }\n", + " },\n", + " {\n", + " \"name\": \"free sulfur dioxide\",\n", + " \"shape\": [1],\n", + " \"datatype\": \"FP32\",\n", + " \"data\": [11],\n", + " \"parameters\": {\n", + " \"content_type\": \"np\"\n", + " }\n", + " },\n", + " {\n", + " \"name\": \"total sulfur dioxide\",\n", + " \"shape\": [1],\n", + " \"datatype\": \"FP32\",\n", + " \"data\": [34],\n", + " \"parameters\": {\n", + " \"content_type\": \"np\"\n", + " }\n", + " },\n", + " {\n", + " \"name\": \"density\",\n", + " \"shape\": [1],\n", + " \"datatype\": \"FP32\",\n", + " \"data\": [0.9978],\n", + " \"parameters\": {\n", + " \"content_type\": \"np\"\n", + " }\n", + " },\n", + " {\n", + " \"name\": \"pH\",\n", + " \"shape\": [1],\n", + " \"datatype\": \"FP32\",\n", + " \"data\": [3.51],\n", + " \"parameters\": {\n", + " \"content_type\": \"np\"\n", + " }\n", + " },\n", + " {\n", + " \"name\": \"sulphates\",\n", + " \"shape\": [1],\n", + " \"datatype\": \"FP32\",\n", + " \"data\": [0.56],\n", + " \"parameters\": {\n", + " \"content_type\": \"np\"\n", + " }\n", + " },\n", + " {\n", + " \"name\": \"alcohol\",\n", + " \"shape\": [1],\n", + " \"datatype\": \"FP32\",\n", + " \"data\": [9.4],\n", + " \"parameters\": {\n", + " \"content_type\": \"np\"\n", + " }\n", + " },\n", + " ]\n", + "}\n", + "\n", + "endpoint = \"http://localhost:8003/seldon/seldon/mlflow/v2/models/infer\"\n", + "response = requests.post(endpoint, json=inference_request)\n", + "\n", + "print(json.dumps(response.json(), indent=2))\n", + "assert response.ok" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl delete -f ./resources/elasticnet_wine_v2.yaml" + ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -999,7 +1188,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.10" + "version": "3.8.10" }, "varInspector": { "cols": { diff --git a/operator/config/manager/configmap.yaml b/operator/config/manager/configmap.yaml index 4facf7218c..c6dcd6895e 100644 --- a/operator/config/manager/configmap.yaml +++ b/operator/config/manager/configmap.yaml @@ -61,7 +61,7 @@ data: }, "kfserving": { "image": "seldonio/mlserver", - "defaultImageVersion": "0.3.2" + "defaultImageVersion": "0.4.0.dev1" } } }, diff --git a/operator/controllers/mlserver.go b/operator/controllers/mlserver.go index 8815bed1cb..4500578897 100644 --- a/operator/controllers/mlserver.go +++ b/operator/controllers/mlserver.go @@ -16,6 +16,7 @@ const ( MLServerSKLearnImplementation = "mlserver_sklearn.SKLearnModel" MLServerXGBoostImplementation = "mlserver_xgboost.XGBoostModel" MLServerTempoImplementation = "tempo.mlserver.InferenceRuntime" + MLServerMLFlowImplementation = "mlserver_mlflow.MLflowRuntime" MLServerHTTPPortEnv = "MLSERVER_HTTP_PORT" MLServerGRPCPortEnv = "MLSERVER_GRPC_PORT" @@ -215,6 +216,8 @@ func getMLServerModelImplementation(pu *machinelearningv1.PredictiveUnit) (strin return MLServerXGBoostImplementation, nil case machinelearningv1.PrepackTempoName: return MLServerTempoImplementation, nil + case machinelearningv1.PrepackMlflowName: + return MLServerMLFlowImplementation, nil default: return "", nil } diff --git a/operator/controllers/mlserver_test.go b/operator/controllers/mlserver_test.go index 577d44b065..8f263573f2 100644 --- a/operator/controllers/mlserver_test.go +++ b/operator/controllers/mlserver_test.go @@ -157,6 +157,7 @@ var _ = Describe("MLServer helpers", func() { Entry("sklearn", machinelearningv1.PrepackSklearnName, MLServerSKLearnImplementation), Entry("xgboost", machinelearningv1.PrepackXgboostName, MLServerXGBoostImplementation), Entry("tempo", machinelearningv1.PrepackTempoName, MLServerTempoImplementation), + Entry("mlserver", machinelearningv1.PrepackMlflowName, MLServerMLFlowImplementation), Entry("unknown", "foo", ""), ) }) diff --git a/testing/scripts/test_prepackaged_servers.py b/testing/scripts/test_prepackaged_servers.py index 6440c0b2bc..31c1f02364 100644 --- a/testing/scripts/test_prepackaged_servers.py +++ b/testing/scripts/test_prepackaged_servers.py @@ -199,6 +199,110 @@ def test_mlflow(self, namespace): run(f"kubectl delete -f {spec} -n {namespace}", shell=True) + # test mlflow with kfserving (v2) protocol + def test_mlflow_v2(self, namespace): + tag = "mlflow" + deploy_model( + tag, + namespace=namespace, + protocol="kfserving", + model_implementation="MLFLOW_SERVER", + model_uri="gs://seldon-models/v1.10.0-dev/mlflow/elasticnet_wine", + ) + wait_for_status(tag, namespace) + wait_for_rollout(tag, namespace) + time.sleep(1) + + logging.warning("Initial request") + r = v2_protocol.inference_request( + deployment_name=tag, + model_name="model", + namespace=namespace, + payload={ + "parameters": {"content_type": "pd"}, + "inputs": [ + { + "name": "fixed acidity", + "shape": [1], + "datatype": "FP32", + "data": [7.4], + "parameters": {"content_type": "np"}, + }, + { + "name": "volatile acidity", + "shape": [1], + "datatype": "FP32", + "data": [0.7000], + "parameters": {"content_type": "np"}, + }, + { + "name": "citric acidity", + "shape": [1], + "datatype": "FP32", + "data": [0], + "parameters": {"content_type": "np"}, + }, + { + "name": "residual sugar", + "shape": [1], + "datatype": "FP32", + "data": [1.9], + "parameters": {"content_type": "np"}, + }, + { + "name": "chlorides", + "shape": [1], + "datatype": "FP32", + "data": [0.076], + "parameters": {"content_type": "np"}, + }, + { + "name": "free sulfur dioxide", + "shape": [1], + "datatype": "FP32", + "data": [11], + "parameters": {"content_type": "np"}, + }, + { + "name": "total sulfur dioxide", + "shape": [1], + "datatype": "FP32", + "data": [34], + "parameters": {"content_type": "np"}, + }, + { + "name": "density", + "shape": [1], + "datatype": "FP32", + "data": [0.9978], + "parameters": {"content_type": "np"}, + }, + { + "name": "pH", + "shape": [1], + "datatype": "FP32", + "data": [3.51], + "parameters": {"content_type": "np"}, + }, + { + "name": "sulphates", + "shape": [1], + "datatype": "FP32", + "data": [0.56], + "parameters": {"content_type": "np"}, + }, + { + "name": "alcohol", + "shape": [1], + "datatype": "FP32", + "data": [9.4], + "parameters": {"content_type": "np"}, + }, + ], + }, + ) + assert r.status_code == 200 + # Test prepackaged Text SKLearn Alibi Explainer def test_text_alibi_explainer(self, namespace): spec = "../resources/movies-text-explainer.yaml"