From 5e9aa5c7f7f210f47b17068d4c5493f69e11206e Mon Sep 17 00:00:00 2001 From: SachinVarghese Date: Tue, 1 Feb 2022 22:59:54 +0530 Subject: [PATCH 1/5] Removing cloud event data json stringify --- components/alibi-detect-server/adserver/server.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/alibi-detect-server/adserver/server.py b/components/alibi-detect-server/adserver/server.py index 126122e06f..2e4c0c46ff 100644 --- a/components/alibi-detect-server/adserver/server.py +++ b/components/alibi-detect-server/adserver/server.py @@ -253,7 +253,6 @@ def post(self): logging.error("Metrics returned are invalid: " + str(runtime_metrics)) if response.data is not None: - responseStr = json.dumps(response.data) # Create event from response if reply_url is active if not self.reply_url == "": @@ -264,7 +263,7 @@ def post(self): revent = ( v1.Event() .SetContentType("application/json") - .SetData(responseStr) + .SetData(response.data) .SetEventID(resp_event_id) .SetSource(self.event_source) .SetEventType(self.event_type) From d3f5fe5d90037a0071c07bd3f1653c6eb6c81d3a Mon Sep 17 00:00:00 2001 From: SachinVarghese Date: Wed, 2 Feb 2022 16:27:22 +0530 Subject: [PATCH 2/5] updating tests for alibi-detect server response --- .../adserver/tests/test_server.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/components/alibi-detect-server/adserver/tests/test_server.py b/components/alibi-detect-server/adserver/tests/test_server.py index 4410293f3c..61a6af0460 100644 --- a/components/alibi-detect-server/adserver/tests/test_server.py +++ b/components/alibi-detect-server/adserver/tests/test_server.py @@ -73,11 +73,9 @@ def test_basic(self): }, ) self.assertEqual(response.code, 200) - expectedResponse = json.dumps(DummyModel.getResponse().data) + expectedResponse = DummyModel.getResponse().data self.assertEqual(response.body.decode("utf-8"), expectedResponse) - self.assertEqual( - m.request_history[0].json(), json.dumps(DummyModel.getResponse().data) - ) + self.assertEqual(m.request_history[0].json(), DummyModel.getResponse().data) headers: Dict = m.request_history[0]._request.headers self.assertEqual(headers["ce-source"], self.eventSource) self.assertEqual(headers["ce-type"], self.eventType) @@ -130,11 +128,9 @@ def test_basic(self): }, ) self.assertEqual(response.code, 200) - expectedResponse = json.dumps(DummyModel.getResponse().data) + expectedResponse = DummyModel.getResponse().data self.assertEqual(response.body.decode("utf-8"), expectedResponse) - self.assertEqual( - m.request_history[0].json(), json.dumps(DummyModel.getResponse().data) - ) + self.assertEqual(m.request_history[0].json(), DummyModel.getResponse().data) headers: Dict = m.request_history[0]._request.headers self.assertEqual(headers["ce-source"], self.eventSource) self.assertEqual(headers["ce-type"], self.eventType) From 0c2cae921419e09cb806ecc164de1b706fb0ecdc Mon Sep 17 00:00:00 2001 From: SachinVarghese Date: Wed, 2 Feb 2022 16:35:36 +0530 Subject: [PATCH 3/5] updating tests for alibi-detect server response --- .../alibi-detect-server/adserver/tests/test_server.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/alibi-detect-server/adserver/tests/test_server.py b/components/alibi-detect-server/adserver/tests/test_server.py index 61a6af0460..c0a75ca567 100644 --- a/components/alibi-detect-server/adserver/tests/test_server.py +++ b/components/alibi-detect-server/adserver/tests/test_server.py @@ -73,9 +73,9 @@ def test_basic(self): }, ) self.assertEqual(response.code, 200) - expectedResponse = DummyModel.getResponse().data + expectedResponse = json.dumps(DummyModel.getResponse().data) self.assertEqual(response.body.decode("utf-8"), expectedResponse) - self.assertEqual(m.request_history[0].json(), DummyModel.getResponse().data) + self.assertEqual(m.request_history[0].json(), expectedResponse) headers: Dict = m.request_history[0]._request.headers self.assertEqual(headers["ce-source"], self.eventSource) self.assertEqual(headers["ce-type"], self.eventType) @@ -128,9 +128,9 @@ def test_basic(self): }, ) self.assertEqual(response.code, 200) - expectedResponse = DummyModel.getResponse().data + expectedResponse = json.dumps(DummyModel.getResponse().data) self.assertEqual(response.body.decode("utf-8"), expectedResponse) - self.assertEqual(m.request_history[0].json(), DummyModel.getResponse().data) + self.assertEqual(m.request_history[0].json(), expectedResponse) headers: Dict = m.request_history[0]._request.headers self.assertEqual(headers["ce-source"], self.eventSource) self.assertEqual(headers["ce-type"], self.eventType) From a396d309bc52db10839e5a629e0f9ad2f113c9ec Mon Sep 17 00:00:00 2001 From: SachinVarghese Date: Wed, 2 Feb 2022 17:15:56 +0530 Subject: [PATCH 4/5] updating tests --- .../alibi-detect-server/adserver/tests/test_server.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/alibi-detect-server/adserver/tests/test_server.py b/components/alibi-detect-server/adserver/tests/test_server.py index c0a75ca567..6ca27b4f4a 100644 --- a/components/alibi-detect-server/adserver/tests/test_server.py +++ b/components/alibi-detect-server/adserver/tests/test_server.py @@ -73,8 +73,8 @@ def test_basic(self): }, ) self.assertEqual(response.code, 200) - expectedResponse = json.dumps(DummyModel.getResponse().data) - self.assertEqual(response.body.decode("utf-8"), expectedResponse) + expectedResponse = DummyModel.getResponse().data + self.assertEqual(response.body.decode("utf-8"), json.dumps(expectedResponse)) self.assertEqual(m.request_history[0].json(), expectedResponse) headers: Dict = m.request_history[0]._request.headers self.assertEqual(headers["ce-source"], self.eventSource) @@ -128,8 +128,8 @@ def test_basic(self): }, ) self.assertEqual(response.code, 200) - expectedResponse = json.dumps(DummyModel.getResponse().data) - self.assertEqual(response.body.decode("utf-8"), expectedResponse) + expectedResponse = DummyModel.getResponse().data + self.assertEqual(response.body.decode("utf-8"), json.dumps(expectedResponse)) self.assertEqual(m.request_history[0].json(), expectedResponse) headers: Dict = m.request_history[0]._request.headers self.assertEqual(headers["ce-source"], self.eventSource) From f2e63f0c84d315394070f1e2075d9cff77a1fa7a Mon Sep 17 00:00:00 2001 From: SachinVarghese Date: Thu, 3 Feb 2022 18:23:58 +0530 Subject: [PATCH 5/5] Updating notebooks and upgrading --- .../drift-detection/cifar10/cifar10_drift.ipynb | 16 ++-------------- .../nvidia-triton-cifar10/cifar10_drift.ipynb | 16 ++-------------- .../cifar10/cifar10_outlier.ipynb | 16 ++-------------- .../nvidia-triton-cifar10/cifar10_outlier.ipynb | 16 ++-------------- doc/source/reference/upgrading.md | 11 +++++++++++ 5 files changed, 19 insertions(+), 56 deletions(-) diff --git a/components/drift-detection/cifar10/cifar10_drift.ipynb b/components/drift-detection/cifar10/cifar10_drift.ipynb index bcada6c63d..eecaba51d5 100644 --- a/components/drift-detection/cifar10/cifar10_drift.ipynb +++ b/components/drift-detection/cifar10/cifar10_drift.ipynb @@ -519,13 +519,7 @@ "metadata": {}, "outputs": [], "source": [ - "res = !kubectl logs -n cifar10drift $(kubectl get pod -n cifar10drift -l app=hello-display -o jsonpath='{.items[0].metadata.name}')\n", - "data = []\n", - "for i in range(0, len(res)):\n", - " if res[i] == \"Data,\":\n", - " data.append(res[i + 1])\n", - "j = json.loads(json.loads(data[0]))\n", - "print(\"Drift\", j[\"data\"][\"is_drift\"] == 1)" + "!kubectl logs -n cifar10drift $(kubectl get pod -n cifar10drift -l app=hello-display -o jsonpath='{.items[0].metadata.name}')" ] }, { @@ -590,13 +584,7 @@ "metadata": {}, "outputs": [], "source": [ - "res = !kubectl logs -n cifar10drift $(kubectl get pod -n cifar10drift -l app=hello-display -o jsonpath='{.items[0].metadata.name}')\n", - "data = []\n", - "for i in range(0, len(res)):\n", - " if res[i] == \"Data,\":\n", - " data.append(res[i + 1])\n", - "j = json.loads(json.loads(data[-1]))\n", - "print(\"Drift\", j[\"data\"][\"is_drift\"] == 1)" + "!kubectl logs -n cifar10drift $(kubectl get pod -n cifar10drift -l app=hello-display -o jsonpath='{.items[0].metadata.name}')" ] }, { diff --git a/components/drift-detection/nvidia-triton-cifar10/cifar10_drift.ipynb b/components/drift-detection/nvidia-triton-cifar10/cifar10_drift.ipynb index 029604b73a..2b88b9390a 100644 --- a/components/drift-detection/nvidia-triton-cifar10/cifar10_drift.ipynb +++ b/components/drift-detection/nvidia-triton-cifar10/cifar10_drift.ipynb @@ -696,13 +696,7 @@ } ], "source": [ - "res = !kubectl logs -n cifar10drift $(kubectl get pod -n cifar10drift -l app=hello-display -o jsonpath='{.items[0].metadata.name}')\n", - "data = []\n", - "for i in range(0, len(res)):\n", - " if res[i] == \"Data,\":\n", - " data.append(res[i + 1])\n", - "j = json.loads(json.loads(data[0]))\n", - "print(\"Drift\", j[\"data\"][\"is_drift\"] == 1)" + "!kubectl logs -n cifar10drift $(kubectl get pod -n cifar10drift -l app=hello-display -o jsonpath='{.items[0].metadata.name}')" ] }, { @@ -829,13 +823,7 @@ } ], "source": [ - "res = !kubectl logs -n cifar10drift $(kubectl get pod -n cifar10drift -l app=hello-display -o jsonpath='{.items[0].metadata.name}')\n", - "data = []\n", - "for i in range(0, len(res)):\n", - " if res[i] == \"Data,\":\n", - " data.append(res[i + 1])\n", - "j = json.loads(json.loads(data[-1]))\n", - "print(\"Drift\", j[\"data\"][\"is_drift\"] == 1)" + "!kubectl logs -n cifar10drift $(kubectl get pod -n cifar10drift -l app=hello-display -o jsonpath='{.items[0].metadata.name}')" ] }, { diff --git a/components/outlier-detection/cifar10/cifar10_outlier.ipynb b/components/outlier-detection/cifar10/cifar10_outlier.ipynb index 5bc0bbb485..a08901f041 100644 --- a/components/outlier-detection/cifar10/cifar10_outlier.ipynb +++ b/components/outlier-detection/cifar10/cifar10_outlier.ipynb @@ -507,13 +507,7 @@ "metadata": {}, "outputs": [], "source": [ - "res = !kubectl logs -n cifar10 $(kubectl get pod -n cifar10 -l app=hello-display -o jsonpath='{.items[0].metadata.name}')\n", - "data = []\n", - "for i in range(0, len(res)):\n", - " if res[i] == \"Data,\":\n", - " data.append(res[i + 1])\n", - "j = json.loads(json.loads(data[0]))\n", - "print(\"Outlier\", j[\"data\"][\"is_outlier\"] == [1])" + "!kubectl logs -n cifar10 $(kubectl get pod -n cifar10 -l app=hello-display -o jsonpath='{.items[0].metadata.name}')" ] }, { @@ -564,13 +558,7 @@ "metadata": {}, "outputs": [], "source": [ - "res = !kubectl logs -n cifar10 $(kubectl get pod -n cifar10 -l app=hello-display -o jsonpath='{.items[0].metadata.name}')\n", - "data = []\n", - "for i in range(0, len(res)):\n", - " if res[i] == \"Data,\":\n", - " data.append(res[i + 1])\n", - "j = json.loads(json.loads(data[-1]))\n", - "print(\"Outlier\", j[\"data\"][\"is_outlier\"] == [1])" + "!kubectl logs -n cifar10 $(kubectl get pod -n cifar10 -l app=hello-display -o jsonpath='{.items[0].metadata.name}')" ] }, { diff --git a/components/outlier-detection/nvidia-triton-cifar10/cifar10_outlier.ipynb b/components/outlier-detection/nvidia-triton-cifar10/cifar10_outlier.ipynb index 428409a84e..63f58bc354 100644 --- a/components/outlier-detection/nvidia-triton-cifar10/cifar10_outlier.ipynb +++ b/components/outlier-detection/nvidia-triton-cifar10/cifar10_outlier.ipynb @@ -644,13 +644,7 @@ } ], "source": [ - "res = !kubectl logs -n cifar10 $(kubectl get pod -n cifar10 -l app=hello-display -o jsonpath='{.items[0].metadata.name}')\n", - "data = []\n", - "for i in range(0, len(res)):\n", - " if res[i] == \"Data,\":\n", - " data.append(res[i + 1])\n", - "j = json.loads(json.loads(data[0]))\n", - "print(\"Outlier\", j[\"data\"][\"is_outlier\"] == [1])" + "!kubectl logs -n cifar10 $(kubectl get pod -n cifar10 -l app=hello-display -o jsonpath='{.items[0].metadata.name}')" ] }, { @@ -732,13 +726,7 @@ } ], "source": [ - "res = !kubectl logs -n cifar10 $(kubectl get pod -n cifar10 -l app=hello-display -o jsonpath='{.items[0].metadata.name}')\n", - "data = []\n", - "for i in range(0, len(res)):\n", - " if res[i] == \"Data,\":\n", - " data.append(res[i + 1])\n", - "j = json.loads(json.loads(data[-1]))\n", - "print(\"Outlier\", j[\"data\"][\"is_outlier\"] == [1])" + "!kubectl logs -n cifar10 $(kubectl get pod -n cifar10 -l app=hello-display -o jsonpath='{.items[0].metadata.name}')" ] }, { diff --git a/doc/source/reference/upgrading.md b/doc/source/reference/upgrading.md index 43b4ef38a3..3221f549f2 100644 --- a/doc/source/reference/upgrading.md +++ b/doc/source/reference/upgrading.md @@ -5,6 +5,17 @@ This page provides with instructions on how to upgrade from previous versions. E If you were running our Openshift 0.4.2 certified operator and are looking to upgrade to our 1.1 certified operator, you will also need to follow the "upgrading process" steps in the "Upgrading to 0.5.2 from previous versions" section. Make sure you also [read the CHANGELOG](./changelog.html) to see the detailed features and bug-fixes in each version. + +## Upgrading to 1.13 + +### Seldon Inference Payload Logging Changes + +The seldon executor logs the prediction request and respose payload to a configured URL post during the inference at each stage of a inference graph. These request/response pairs are now matched correctly to reflect the component's input/output at each node level as compared to pairing based on the tree structure of the inference graph previously. See [relevant issue](https://github.com/SeldonIO/seldon-core/issues/3873) for more details. + +### Alibi Detect Server Event Update + +The cloud event data that the alibi detect server sends to a configured replyURL post the outlier/drift detection was a JSON string. This is now updated to reflect the content type as JSON correctly. + ## Upgrading to 1.12 ### Support for Kubernetes 1.22