Skip to content

Commit

Permalink
Respect MLServer content type in pipeline gateway http server respons…
Browse files Browse the repository at this point in the history 
…es (#600)

* Respect MLServer content type

* update huggingface example notebook as returned prediction is not base64 encoded anymore with this change
  • Loading branch information
@ukclivecox
ukclivecox authored Nov 18, 2022
1 parent d55b925 commit 2299529
Show file tree
Hide file tree
Showing 5 changed files with 174 additions and 80 deletions.
100 changes: 63 additions & 37 deletions samples/examples/huggingface/huggingface.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
},
{
"cell_type": "code",
"execution_count": 96,
"execution_count": 1,
"id": "3154774d",
"metadata": {},
"outputs": [],
Expand All @@ -44,7 +44,7 @@
},
{
"cell_type": "code",
"execution_count": 97,
"execution_count": 24,
"id": "526b0678",
"metadata": {},
"outputs": [],
Expand All @@ -63,15 +63,13 @@
" headers = {\"Content-Type\": \"application/json\", \"seldon-model\": resource}\n",
" response_raw = requests.post(url, json=reqJson, headers=headers)\n",
" j = response_raw.json()\n",
" predictionEncoded = j[\"outputs\"][0][\"data\"][0]\n",
" decodedSentiment = base64.b64decode(predictionEncoded)\n",
" textEncoded = j[\"outputs\"][1][\"data\"][0]\n",
" decodedText = base64.b64decode(textEncoded)\n",
" sentiment = j[\"outputs\"][0][\"data\"][0]\n",
" text = j[\"outputs\"][1][\"data\"][0]\n",
" reqId = response_raw.headers[\"x-request-id\"]\n",
" print(reqId)\n",
" os.environ[\"REQUEST_ID\"]=reqId\n",
" print(decodedText)\n",
" print(decodedSentiment)"
" print(text)\n",
" print(sentiment)"
]
},
{
Expand All @@ -86,7 +84,7 @@
},
{
"cell_type": "code",
"execution_count": 98,
"execution_count": 3,
"id": "43b7a0a1",
"metadata": {},
"outputs": [
Expand Down Expand Up @@ -122,7 +120,7 @@
},
{
"cell_type": "code",
"execution_count": 99,
"execution_count": 4,
"id": "9aacd53d",
"metadata": {},
"outputs": [
Expand All @@ -142,7 +140,7 @@
},
{
"cell_type": "code",
"execution_count": 100,
"execution_count": 5,
"id": "b13a6082",
"metadata": {},
"outputs": [
Expand Down Expand Up @@ -177,14 +175,28 @@
},
{
"cell_type": "code",
"execution_count": 101,
"execution_count": 6,
"id": "6d64d3bf",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[37m# Copyright 2022 Seldon Technologies Ltd.\u001b[39;49;00m\r\n",
"\u001b[37m#\u001b[39;49;00m\r\n",
"\u001b[37m# Licensed under the Apache License, Version 2.0 (the \"License\");\u001b[39;49;00m\r\n",
"\u001b[37m# you may not use this file except in compliance with the License.\u001b[39;49;00m\r\n",
"\u001b[37m# You may obtain a copy of the License at\u001b[39;49;00m\r\n",
"\u001b[37m#\u001b[39;49;00m\r\n",
"\u001b[37m# http://www.apache.org/licenses/LICENSE-2.0\u001b[39;49;00m\r\n",
"\u001b[37m#\u001b[39;49;00m\r\n",
"\u001b[37m# Unless required by applicable law or agreed to in writing, software\u001b[39;49;00m\r\n",
"\u001b[37m# distributed under the License is distributed on an \"AS IS\" BASIS,\u001b[39;49;00m\r\n",
"\u001b[37m# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\u001b[39;49;00m\r\n",
"\u001b[37m# See the License for the specific language governing permissions and\u001b[39;49;00m\r\n",
"\u001b[37m# limitations under the License.\u001b[39;49;00m\r\n",
"\r\n",
"\u001b[34mfrom\u001b[39;49;00m \u001b[04m\u001b[36mmlserver\u001b[39;49;00m \u001b[34mimport\u001b[39;49;00m MLModel\r\n",
"\u001b[34mfrom\u001b[39;49;00m \u001b[04m\u001b[36mmlserver\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mtypes\u001b[39;49;00m \u001b[34mimport\u001b[39;49;00m InferenceRequest, InferenceResponse, ResponseOutput\r\n",
"\u001b[34mfrom\u001b[39;49;00m \u001b[04m\u001b[36mmlserver\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mcodecs\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mstring\u001b[39;49;00m \u001b[34mimport\u001b[39;49;00m StringRequestCodec\r\n",
Expand Down Expand Up @@ -218,14 +230,28 @@
},
{
"cell_type": "code",
"execution_count": 102,
"execution_count": 7,
"id": "251ae9ab",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[37m# Copyright 2022 Seldon Technologies Ltd.\u001b[39;49;00m\r\n",
"\u001b[37m#\u001b[39;49;00m\r\n",
"\u001b[37m# Licensed under the Apache License, Version 2.0 (the \"License\");\u001b[39;49;00m\r\n",
"\u001b[37m# you may not use this file except in compliance with the License.\u001b[39;49;00m\r\n",
"\u001b[37m# You may obtain a copy of the License at\u001b[39;49;00m\r\n",
"\u001b[37m#\u001b[39;49;00m\r\n",
"\u001b[37m# http://www.apache.org/licenses/LICENSE-2.0\u001b[39;49;00m\r\n",
"\u001b[37m#\u001b[39;49;00m\r\n",
"\u001b[37m# Unless required by applicable law or agreed to in writing, software\u001b[39;49;00m\r\n",
"\u001b[37m# distributed under the License is distributed on an \"AS IS\" BASIS,\u001b[39;49;00m\r\n",
"\u001b[37m# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\u001b[39;49;00m\r\n",
"\u001b[37m# See the License for the specific language governing permissions and\u001b[39;49;00m\r\n",
"\u001b[37m# limitations under the License.\u001b[39;49;00m\r\n",
"\r\n",
"\u001b[34mfrom\u001b[39;49;00m \u001b[04m\u001b[36mmlserver\u001b[39;49;00m \u001b[34mimport\u001b[39;49;00m MLModel\r\n",
"\u001b[34mfrom\u001b[39;49;00m \u001b[04m\u001b[36mmlserver\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mtypes\u001b[39;49;00m \u001b[34mimport\u001b[39;49;00m InferenceRequest, InferenceResponse, ResponseOutput\r\n",
"\u001b[34mfrom\u001b[39;49;00m \u001b[04m\u001b[36mmlserver\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mcodecs\u001b[39;49;00m \u001b[34mimport\u001b[39;49;00m StringCodec, Base64Codec, NumpyRequestCodec\r\n",
Expand Down Expand Up @@ -264,7 +290,7 @@
},
{
"cell_type": "code",
"execution_count": 103,
"execution_count": 8,
"id": "26c94fb2",
"metadata": {},
"outputs": [
Expand Down Expand Up @@ -302,7 +328,7 @@
},
{
"cell_type": "code",
"execution_count": 104,
"execution_count": 9,
"id": "64fc938f",
"metadata": {},
"outputs": [
Expand All @@ -322,7 +348,7 @@
},
{
"cell_type": "code",
"execution_count": 105,
"execution_count": 10,
"id": "d6cd0078",
"metadata": {},
"outputs": [
Expand All @@ -342,7 +368,7 @@
},
{
"cell_type": "code",
"execution_count": 108,
"execution_count": 11,
"id": "2ff58083",
"metadata": {},
"outputs": [
Expand Down Expand Up @@ -374,7 +400,7 @@
},
{
"cell_type": "code",
"execution_count": 109,
"execution_count": 12,
"id": "830d6bb1",
"metadata": {},
"outputs": [
Expand All @@ -392,7 +418,7 @@
},
{
"cell_type": "code",
"execution_count": 110,
"execution_count": 13,
"id": "95fa2faf",
"metadata": {},
"outputs": [
Expand All @@ -406,7 +432,7 @@
" {\r\n",
" \"pipeline\": {\r\n",
" \"name\": \"sentiment-explain\",\r\n",
" \"uid\": \"cddsn9atsj0s738f5ut0\",\r\n",
" \"uid\": \"cdrn2c5jr36s73eg9om0\",\r\n",
" \"version\": 1,\r\n",
" \"steps\": [\r\n",
" {\r\n",
Expand All @@ -433,7 +459,7 @@
" \"pipelineVersion\": 1,\r\n",
" \"status\": \"PipelineReady\",\r\n",
" \"reason\": \"created pipeline\",\r\n",
" \"lastChangeTimestamp\": \"2022-10-28T12:31:34.304608492Z\"\r\n",
" \"lastChangeTimestamp\": \"2022-11-18T11:49:04.697311598Z\"\r\n",
" }\r\n",
" }\r\n",
" ]\r\n",
Expand All @@ -447,7 +473,7 @@
},
{
"cell_type": "code",
"execution_count": 111,
"execution_count": 14,
"id": "229c61af",
"metadata": {},
"outputs": [
Expand All @@ -473,7 +499,7 @@
},
{
"cell_type": "code",
"execution_count": 112,
"execution_count": 15,
"id": "7ba179b0",
"metadata": {},
"outputs": [
Expand All @@ -491,7 +517,7 @@
},
{
"cell_type": "code",
"execution_count": 113,
"execution_count": 16,
"id": "a1efedc0",
"metadata": {},
"outputs": [
Expand Down Expand Up @@ -519,7 +545,7 @@
},
{
"cell_type": "code",
"execution_count": 114,
"execution_count": 17,
"id": "d787e1ed",
"metadata": {},
"outputs": [
Expand Down Expand Up @@ -558,7 +584,7 @@
},
{
"cell_type": "code",
"execution_count": 115,
"execution_count": 18,
"id": "22d5997a",
"metadata": {},
"outputs": [
Expand All @@ -576,7 +602,7 @@
},
{
"cell_type": "code",
"execution_count": 116,
"execution_count": 19,
"id": "8bad7972",
"metadata": {},
"outputs": [
Expand All @@ -590,7 +616,7 @@
" {\r\n",
" \"pipeline\": {\r\n",
" \"name\": \"speech-to-sentiment\",\r\n",
" \"uid\": \"cddsnjqtsj0s738f5utg\",\r\n",
" \"uid\": \"cdrn2htjr36s73eg9omg\",\r\n",
" \"version\": 1,\r\n",
" \"steps\": [\r\n",
" {\r\n",
Expand Down Expand Up @@ -630,7 +656,7 @@
" \"pipelineVersion\": 1,\r\n",
" \"status\": \"PipelineReady\",\r\n",
" \"reason\": \"created pipeline\",\r\n",
" \"lastChangeTimestamp\": \"2022-10-28T12:32:16.212929516Z\"\r\n",
" \"lastChangeTimestamp\": \"2022-11-18T11:49:27.793654254Z\"\r\n",
" }\r\n",
" }\r\n",
" ]\r\n",
Expand All @@ -652,14 +678,14 @@
},
{
"cell_type": "code",
"execution_count": 117,
"execution_count": 20,
"id": "85884fb2",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "a7ea34b58d49461cbe60a343af53e7ba",
"model_id": "04e2a186856546d8a6430f86f3044936",
"version_major": 2,
"version_minor": 0
},
Expand All @@ -679,17 +705,17 @@
},
{
"cell_type": "code",
"execution_count": 120,
"execution_count": 25,
"id": "6b8984ba",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"cddsonmq0ics73e6eqo0\n",
"b'{\"text\": \" Cambridge is wonderful and beautiful.\"}'\n",
"b'{\"label\": \"POSITIVE\", \"score\": 0.9998691082000732}'\n"
"cdrn3iits0ps73bgtaeg\n",
"{\"text\": \" Cambridge is wonderful and beautiful.\"}\n",
"{\"label\": \"POSITIVE\", \"score\": 0.9998691082000732}\n"
]
}
],
Expand All @@ -707,15 +733,15 @@
},
{
"cell_type": "code",
"execution_count": 121,
"execution_count": 26,
"id": "81107af6",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"........\n",
".....\n",
"Explanation anchors: ['beautiful']\n"
]
}
Expand Down
Loading

0 comments on commit 2299529

Please sign in to comment.