Fixing Alibi Detect Server response cloud event data is json marshall…

…ed string (#3907) * Removing cloud event data json stringify * updating tests for alibi-detect server response * updating tests for alibi-detect server response * updating tests * Updating notebooks and upgrading
SeldonIO · Feb 3, 2022 · f8e08f7 · f8e08f7
1 parent fd79a63
commit f8e08f7
Show file tree

Hide file tree

Showing 7 changed files with 26 additions and 68 deletions.
diff --git a/components/alibi-detect-server/adserver/server.py b/components/alibi-detect-server/adserver/server.py
@@ -253,7 +253,6 @@ def post(self):
                 logging.error("Metrics returned are invalid: " + str(runtime_metrics))
 
         if response.data is not None:
-            responseStr = json.dumps(response.data)
 
             # Create event from response if reply_url is active
             if not self.reply_url == "":
@@ -264,7 +263,7 @@ def post(self):
                 revent = (
                     v1.Event()
                     .SetContentType("application/json")
-                    .SetData(responseStr)
+                    .SetData(response.data)
                     .SetEventID(resp_event_id)
                     .SetSource(self.event_source)
                     .SetEventType(self.event_type)

diff --git a/components/alibi-detect-server/adserver/tests/test_server.py b/components/alibi-detect-server/adserver/tests/test_server.py
@@ -73,11 +73,9 @@ def test_basic(self):
                 },
             )
             self.assertEqual(response.code, 200)
-            expectedResponse = json.dumps(DummyModel.getResponse().data)
-            self.assertEqual(response.body.decode("utf-8"), expectedResponse)
-            self.assertEqual(
-                m.request_history[0].json(), json.dumps(DummyModel.getResponse().data)
-            )
+            expectedResponse = DummyModel.getResponse().data
+            self.assertEqual(response.body.decode("utf-8"), json.dumps(expectedResponse))
+            self.assertEqual(m.request_history[0].json(), expectedResponse)
             headers: Dict = m.request_history[0]._request.headers
             self.assertEqual(headers["ce-source"], self.eventSource)
             self.assertEqual(headers["ce-type"], self.eventType)
@@ -130,11 +128,9 @@ def test_basic(self):
                 },
             )
             self.assertEqual(response.code, 200)
-            expectedResponse = json.dumps(DummyModel.getResponse().data)
-            self.assertEqual(response.body.decode("utf-8"), expectedResponse)
-            self.assertEqual(
-                m.request_history[0].json(), json.dumps(DummyModel.getResponse().data)
-            )
+            expectedResponse = DummyModel.getResponse().data
+            self.assertEqual(response.body.decode("utf-8"), json.dumps(expectedResponse))
+            self.assertEqual(m.request_history[0].json(), expectedResponse)
             headers: Dict = m.request_history[0]._request.headers
             self.assertEqual(headers["ce-source"], self.eventSource)
             self.assertEqual(headers["ce-type"], self.eventType)

diff --git a/components/drift-detection/cifar10/cifar10_drift.ipynb b/components/drift-detection/cifar10/cifar10_drift.ipynb
@@ -519,13 +519,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "res = !kubectl logs -n cifar10drift $(kubectl get pod -n cifar10drift -l app=hello-display -o jsonpath='{.items[0].metadata.name}')\n",
-    "data = []\n",
-    "for i in range(0, len(res)):\n",
-    "    if res[i] == \"Data,\":\n",
-    "        data.append(res[i + 1])\n",
-    "j = json.loads(json.loads(data[0]))\n",
-    "print(\"Drift\", j[\"data\"][\"is_drift\"] == 1)"
+    "!kubectl logs -n cifar10drift $(kubectl get pod -n cifar10drift -l app=hello-display -o jsonpath='{.items[0].metadata.name}')"
    ]
   },
   {
@@ -590,13 +584,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "res = !kubectl logs -n cifar10drift $(kubectl get pod -n cifar10drift -l app=hello-display -o jsonpath='{.items[0].metadata.name}')\n",
-    "data = []\n",
-    "for i in range(0, len(res)):\n",
-    "    if res[i] == \"Data,\":\n",
-    "        data.append(res[i + 1])\n",
-    "j = json.loads(json.loads(data[-1]))\n",
-    "print(\"Drift\", j[\"data\"][\"is_drift\"] == 1)"
+    "!kubectl logs -n cifar10drift $(kubectl get pod -n cifar10drift -l app=hello-display -o jsonpath='{.items[0].metadata.name}')"
    ]
   },
   {

diff --git a/components/drift-detection/nvidia-triton-cifar10/cifar10_drift.ipynb b/components/drift-detection/nvidia-triton-cifar10/cifar10_drift.ipynb
@@ -696,13 +696,7 @@
     }
    ],
    "source": [
-    "res = !kubectl logs -n cifar10drift $(kubectl get pod -n cifar10drift -l app=hello-display -o jsonpath='{.items[0].metadata.name}')\n",
-    "data = []\n",
-    "for i in range(0, len(res)):\n",
-    "    if res[i] == \"Data,\":\n",
-    "        data.append(res[i + 1])\n",
-    "j = json.loads(json.loads(data[0]))\n",
-    "print(\"Drift\", j[\"data\"][\"is_drift\"] == 1)"
+    "!kubectl logs -n cifar10drift $(kubectl get pod -n cifar10drift -l app=hello-display -o jsonpath='{.items[0].metadata.name}')"
    ]
   },
   {
@@ -829,13 +823,7 @@
     }
    ],
    "source": [
-    "res = !kubectl logs -n cifar10drift $(kubectl get pod -n cifar10drift -l app=hello-display -o jsonpath='{.items[0].metadata.name}')\n",
-    "data = []\n",
-    "for i in range(0, len(res)):\n",
-    "    if res[i] == \"Data,\":\n",
-    "        data.append(res[i + 1])\n",
-    "j = json.loads(json.loads(data[-1]))\n",
-    "print(\"Drift\", j[\"data\"][\"is_drift\"] == 1)"
+    "!kubectl logs -n cifar10drift $(kubectl get pod -n cifar10drift -l app=hello-display -o jsonpath='{.items[0].metadata.name}')"
    ]
   },
   {

diff --git a/components/outlier-detection/cifar10/cifar10_outlier.ipynb b/components/outlier-detection/cifar10/cifar10_outlier.ipynb
@@ -507,13 +507,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "res = !kubectl logs -n cifar10 $(kubectl get pod -n cifar10 -l app=hello-display -o jsonpath='{.items[0].metadata.name}')\n",
-    "data = []\n",
-    "for i in range(0, len(res)):\n",
-    "    if res[i] == \"Data,\":\n",
-    "        data.append(res[i + 1])\n",
-    "j = json.loads(json.loads(data[0]))\n",
-    "print(\"Outlier\", j[\"data\"][\"is_outlier\"] == [1])"
+    "!kubectl logs -n cifar10 $(kubectl get pod -n cifar10 -l app=hello-display -o jsonpath='{.items[0].metadata.name}')"
    ]
   },
   {
@@ -564,13 +558,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "res = !kubectl logs -n cifar10 $(kubectl get pod -n cifar10 -l app=hello-display -o jsonpath='{.items[0].metadata.name}')\n",
-    "data = []\n",
-    "for i in range(0, len(res)):\n",
-    "    if res[i] == \"Data,\":\n",
-    "        data.append(res[i + 1])\n",
-    "j = json.loads(json.loads(data[-1]))\n",
-    "print(\"Outlier\", j[\"data\"][\"is_outlier\"] == [1])"
+    "!kubectl logs -n cifar10 $(kubectl get pod -n cifar10 -l app=hello-display -o jsonpath='{.items[0].metadata.name}')"
    ]
   },
   {

diff --git a/components/outlier-detection/nvidia-triton-cifar10/cifar10_outlier.ipynb b/components/outlier-detection/nvidia-triton-cifar10/cifar10_outlier.ipynb
@@ -644,13 +644,7 @@
     }
    ],
    "source": [
-    "res = !kubectl logs -n cifar10 $(kubectl get pod -n cifar10 -l app=hello-display -o jsonpath='{.items[0].metadata.name}')\n",
-    "data = []\n",
-    "for i in range(0, len(res)):\n",
-    "    if res[i] == \"Data,\":\n",
-    "        data.append(res[i + 1])\n",
-    "j = json.loads(json.loads(data[0]))\n",
-    "print(\"Outlier\", j[\"data\"][\"is_outlier\"] == [1])"
+    "!kubectl logs -n cifar10 $(kubectl get pod -n cifar10 -l app=hello-display -o jsonpath='{.items[0].metadata.name}')"
    ]
   },
   {
@@ -732,13 +726,7 @@
     }
    ],
    "source": [
-    "res = !kubectl logs -n cifar10 $(kubectl get pod -n cifar10 -l app=hello-display -o jsonpath='{.items[0].metadata.name}')\n",
-    "data = []\n",
-    "for i in range(0, len(res)):\n",
-    "    if res[i] == \"Data,\":\n",
-    "        data.append(res[i + 1])\n",
-    "j = json.loads(json.loads(data[-1]))\n",
-    "print(\"Outlier\", j[\"data\"][\"is_outlier\"] == [1])"
+    "!kubectl logs -n cifar10 $(kubectl get pod -n cifar10 -l app=hello-display -o jsonpath='{.items[0].metadata.name}')"
    ]
   },
   {

diff --git a/doc/source/reference/upgrading.md b/doc/source/reference/upgrading.md
@@ -5,6 +5,17 @@ This page provides with instructions on how to upgrade from previous versions. E
 If you were running our Openshift 0.4.2 certified operator and are looking to upgrade to our 1.1 certified operator, you will also need to follow the "upgrading process" steps in the "Upgrading to 0.5.2 from previous versions" section.
 
 Make sure you also [read the CHANGELOG](./changelog.html) to see the detailed features and bug-fixes in each version.
+
+## Upgrading to 1.13
+
+### Seldon Inference Payload Logging Changes
+
+The seldon executor logs the prediction request and respose payload to a configured URL post during the inference at each stage of a inference graph. These request/response pairs are now matched correctly to reflect the component's input/output at each node level as compared to pairing based on the tree structure of the inference graph previously. See [relevant issue](https://github.com/SeldonIO/seldon-core/issues/3873) for more details.
+
+### Alibi Detect Server Event Update
+
+The cloud event data that the alibi detect server sends to a configured replyURL post the outlier/drift detection was a JSON string. This is now updated to reflect the content type as JSON correctly.
+
 ## Upgrading to 1.12
 
 ### Support for Kubernetes 1.22