From 30511c77c0390265d5fbebb7ceeb98cf265aa234 Mon Sep 17 00:00:00 2001 From: harshita-meena <70981082+harshita-meena@users.noreply.github.com> Date: Thu, 24 Mar 2022 16:52:24 -0700 Subject: [PATCH 01/13] Update seldon_methods.py --- python/seldon_core/seldon_methods.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/seldon_core/seldon_methods.py b/python/seldon_core/seldon_methods.py index f8ae4b1a37..8ed6dc7332 100644 --- a/python/seldon_core/seldon_methods.py +++ b/python/seldon_core/seldon_methods.py @@ -59,7 +59,10 @@ def handle_raw_custom_metrics( """ metrics = [] if is_proto: - metrics = seldon_message_to_json(msg.meta).get("metrics", []) + # proto to json extracts dictionary in no particular order + # sorting it here to ensure unique key in metrics.py _generate_tags_key + unordered_metrics = seldon_message_to_json(msg.meta).get("metrics", []) + metrics = dict(sorted(unordered_metrics.items(), key=lambda item: item[1]) if metrics and not INCLUDE_METRICS_IN_CLIENT_RESPONSE: del msg.meta.metrics[:] elif isinstance(msg, dict): From 810ed62b8e7d699da7d403dc0b41aa9b6b2c5a34 Mon Sep 17 00:00:00 2001 From: harshita-meena <70981082+harshita-meena@users.noreply.github.com> Date: Thu, 24 Mar 2022 16:55:28 -0700 Subject: [PATCH 02/13] Update seldon_methods.py --- python/seldon_core/seldon_methods.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/seldon_core/seldon_methods.py b/python/seldon_core/seldon_methods.py index 8ed6dc7332..e48e05f5b7 100644 --- a/python/seldon_core/seldon_methods.py +++ b/python/seldon_core/seldon_methods.py @@ -60,9 +60,9 @@ def handle_raw_custom_metrics( metrics = [] if is_proto: # proto to json extracts dictionary in no particular order - # sorting it here to ensure unique key in metrics.py _generate_tags_key - unordered_metrics = seldon_message_to_json(msg.meta).get("metrics", []) - metrics = dict(sorted(unordered_metrics.items(), key=lambda item: item[1]) + # sorting tags it here to ensure unique key in metrics.py _generate_tags_key + metrics = seldon_message_to_json(msg.meta).get("metrics", []) + metrics["tags"] = dict(sorted(metrics["tags"].items(), key=lambda item: item[1]) if metrics and not INCLUDE_METRICS_IN_CLIENT_RESPONSE: del msg.meta.metrics[:] elif isinstance(msg, dict): From 48e4a11ba6639f3582a0e1ed3cee77782bdd9108 Mon Sep 17 00:00:00 2001 From: harshita-meena <70981082+harshita-meena@users.noreply.github.com> Date: Fri, 25 Mar 2022 09:03:59 -0700 Subject: [PATCH 03/13] Update seldon_methods.py --- python/seldon_core/seldon_methods.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/seldon_core/seldon_methods.py b/python/seldon_core/seldon_methods.py index e48e05f5b7..b63da553d1 100644 --- a/python/seldon_core/seldon_methods.py +++ b/python/seldon_core/seldon_methods.py @@ -62,7 +62,7 @@ def handle_raw_custom_metrics( # proto to json extracts dictionary in no particular order # sorting tags it here to ensure unique key in metrics.py _generate_tags_key metrics = seldon_message_to_json(msg.meta).get("metrics", []) - metrics["tags"] = dict(sorted(metrics["tags"].items(), key=lambda item: item[1]) + metrics["tags"] = dict(sorted(metrics["tags"].items(), key=lambda item: item[1])) if metrics and not INCLUDE_METRICS_IN_CLIENT_RESPONSE: del msg.meta.metrics[:] elif isinstance(msg, dict): From 5d28a3663f1c939bbc99c0c41bdfa8e30b6374cf Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Tue, 29 Mar 2022 10:25:14 +0100 Subject: [PATCH 04/13] Test commit to trigger CI --- python/seldon_core/seldon_methods.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/seldon_core/seldon_methods.py b/python/seldon_core/seldon_methods.py index b63da553d1..9ddb865088 100644 --- a/python/seldon_core/seldon_methods.py +++ b/python/seldon_core/seldon_methods.py @@ -63,6 +63,7 @@ def handle_raw_custom_metrics( # sorting tags it here to ensure unique key in metrics.py _generate_tags_key metrics = seldon_message_to_json(msg.meta).get("metrics", []) metrics["tags"] = dict(sorted(metrics["tags"].items(), key=lambda item: item[1])) + if metrics and not INCLUDE_METRICS_IN_CLIENT_RESPONSE: del msg.meta.metrics[:] elif isinstance(msg, dict): From b95ed54dedf66d185f41e81d782ccbb3d86831e5 Mon Sep 17 00:00:00 2001 From: hmeena Date: Tue, 29 Mar 2022 13:45:08 +0000 Subject: [PATCH 05/13] iterate all tags in list of metrics --- python/seldon_core/seldon_methods.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/seldon_core/seldon_methods.py b/python/seldon_core/seldon_methods.py index b63da553d1..05d1b230a9 100644 --- a/python/seldon_core/seldon_methods.py +++ b/python/seldon_core/seldon_methods.py @@ -62,7 +62,8 @@ def handle_raw_custom_metrics( # proto to json extracts dictionary in no particular order # sorting tags it here to ensure unique key in metrics.py _generate_tags_key metrics = seldon_message_to_json(msg.meta).get("metrics", []) - metrics["tags"] = dict(sorted(metrics["tags"].items(), key=lambda item: item[1])) + for metric in metrics: + metric["tags"] = dict(sorted(metric["tags"].items(), key=lambda item: item[1])) if metrics and not INCLUDE_METRICS_IN_CLIENT_RESPONSE: del msg.meta.metrics[:] elif isinstance(msg, dict): From b32212db55234a7ea59c932fc5caa43f41edc4dd Mon Sep 17 00:00:00 2001 From: hmeena Date: Wed, 4 May 2022 05:42:11 +0000 Subject: [PATCH 06/13] remove spaces --- python/seldon_core/seldon_methods.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/seldon_core/seldon_methods.py b/python/seldon_core/seldon_methods.py index 002745b4a2..eea8b3cb6d 100644 --- a/python/seldon_core/seldon_methods.py +++ b/python/seldon_core/seldon_methods.py @@ -64,7 +64,7 @@ def handle_raw_custom_metrics( metrics = seldon_message_to_json(msg.meta).get("metrics", []) for metric in metrics: metric["tags"] = dict(sorted(metric["tags"].items(), key=lambda item: item[1])) - + if metrics and not INCLUDE_METRICS_IN_CLIENT_RESPONSE: del msg.meta.metrics[:] elif isinstance(msg, dict): From c3d514add8ff6bef46415fb4e6b633f20960191f Mon Sep 17 00:00:00 2001 From: harshita-meena <70981082+harshita-meena@users.noreply.github.com> Date: Thu, 12 May 2022 10:44:18 -0700 Subject: [PATCH 07/13] Update seldon_methods.py --- python/seldon_core/seldon_methods.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/python/seldon_core/seldon_methods.py b/python/seldon_core/seldon_methods.py index eea8b3cb6d..c853101343 100644 --- a/python/seldon_core/seldon_methods.py +++ b/python/seldon_core/seldon_methods.py @@ -59,11 +59,7 @@ def handle_raw_custom_metrics( """ metrics = [] if is_proto: - # proto to json extracts dictionary in no particular order - # sorting tags it here to ensure unique key in metrics.py _generate_tags_key metrics = seldon_message_to_json(msg.meta).get("metrics", []) - for metric in metrics: - metric["tags"] = dict(sorted(metric["tags"].items(), key=lambda item: item[1])) if metrics and not INCLUDE_METRICS_IN_CLIENT_RESPONSE: del msg.meta.metrics[:] From 6d0703ae63f0fa59627e63ae01663f1921cac7aa Mon Sep 17 00:00:00 2001 From: harshita-meena <70981082+harshita-meena@users.noreply.github.com> Date: Thu, 12 May 2022 10:45:45 -0700 Subject: [PATCH 08/13] Making global change to generate tags --- python/seldon_core/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/seldon_core/metrics.py b/python/seldon_core/metrics.py index 15fe3a26b7..713fd1f8c3 100644 --- a/python/seldon_core/metrics.py +++ b/python/seldon_core/metrics.py @@ -200,7 +200,7 @@ def _merge_labels(self, worker, tags): @staticmethod def _generate_tags_key(tags): - return "_".join(["-".join(i) for i in tags.items()]) + return "_".join(["-".join(i) for i in sorted(tags.items())]) @staticmethod def _update_hist(x, vals, sumv): From b5b22c5ee74cda19cc4dd62dc90c046d11b0ccf2 Mon Sep 17 00:00:00 2001 From: harshita-meena <70981082+harshita-meena@users.noreply.github.com> Date: Thu, 12 May 2022 10:46:36 -0700 Subject: [PATCH 09/13] Remove extra line --- python/seldon_core/seldon_methods.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/seldon_core/seldon_methods.py b/python/seldon_core/seldon_methods.py index c853101343..f8ae4b1a37 100644 --- a/python/seldon_core/seldon_methods.py +++ b/python/seldon_core/seldon_methods.py @@ -60,7 +60,6 @@ def handle_raw_custom_metrics( metrics = [] if is_proto: metrics = seldon_message_to_json(msg.meta).get("metrics", []) - if metrics and not INCLUDE_METRICS_IN_CLIENT_RESPONSE: del msg.meta.metrics[:] elif isinstance(msg, dict): From 1c452f862dd85b07b2bcc04315cbf17cd2ee505f Mon Sep 17 00:00:00 2001 From: harshita-meena Date: Thu, 12 May 2022 18:58:32 -0700 Subject: [PATCH 10/13] add unit tests --- python/tests/test_runtime_metrics_tags.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/python/tests/test_runtime_metrics_tags.py b/python/tests/test_runtime_metrics_tags.py index 571ff1df81..96ec82144f 100644 --- a/python/tests/test_runtime_metrics_tags.py +++ b/python/tests/test_runtime_metrics_tags.py @@ -5,6 +5,7 @@ import numpy as np import pytest from google.protobuf import json_format +from collections import OrderedDict from seldon_core.flask_utils import SeldonMicroserviceException from seldon_core.metrics import ( @@ -85,6 +86,8 @@ def verify_seldon_metrics(data, mycounter_value, histogram_entries, method): base_tags_key = SeldonMetrics._generate_tags_key(expected_base_tags) expected_custom_tags = {"mytag": "mytagvalue", "method": method} custom_tags_key = SeldonMetrics._generate_tags_key(expected_custom_tags) + expected_tags_key = f"method-{method}_mytag-mytagvalue" + assert custom_tags_key == expected_tags_key assert data["GAUGE", "runtime_gauge", base_tags_key]["value"] == 42 assert data["GAUGE", "mygauge", base_tags_key]["value"] == 100 assert data["GAUGE", "customtag", custom_tags_key]["value"] == 200 @@ -99,6 +102,19 @@ def verify_seldon_metrics(data, mycounter_value, histogram_entries, method): ) +def test_generate_tags_key(): + # initializing two different kinds of dictionary + insertion_order = OrderedDict({'b': 'b', 'a': 'a'}) + sorted_order = {'a': 'a', 'b': 'b'} + # assert the items in the list will differ based on order + assert list(insertion_order.items()) != list(sorted_order.items()) + + insertion_order_tag = SeldonMetrics._generate_tags_key(insertion_order) + sorted_order_tag = SeldonMetrics._generate_tags_key(sorted_order) + # same tag generated irrespective of order + assert insertion_order_tag == sorted_order_tag + + @pytest.mark.parametrize("cls", [UserObject]) def test_seldon_runtime_data_predict(cls, client_gets_metrics): user_object = cls() From 089459830ea6bd40ad802998ec3a7f772031079f Mon Sep 17 00:00:00 2001 From: harshita-meena Date: Thu, 12 May 2022 19:01:30 -0700 Subject: [PATCH 11/13] comment fix --- python/tests/test_runtime_metrics_tags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tests/test_runtime_metrics_tags.py b/python/tests/test_runtime_metrics_tags.py index 96ec82144f..45ee43eee5 100644 --- a/python/tests/test_runtime_metrics_tags.py +++ b/python/tests/test_runtime_metrics_tags.py @@ -106,7 +106,7 @@ def test_generate_tags_key(): # initializing two different kinds of dictionary insertion_order = OrderedDict({'b': 'b', 'a': 'a'}) sorted_order = {'a': 'a', 'b': 'b'} - # assert the items in the list will differ based on order + # assert the items in the list differ based on order assert list(insertion_order.items()) != list(sorted_order.items()) insertion_order_tag = SeldonMetrics._generate_tags_key(insertion_order) From 6e34822780667ba32e11af858688299f027c9d4b Mon Sep 17 00:00:00 2001 From: harshita-meena Date: Fri, 13 May 2022 13:57:09 -0700 Subject: [PATCH 12/13] fix formatting --- python/tests/test_runtime_metrics_tags.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/tests/test_runtime_metrics_tags.py b/python/tests/test_runtime_metrics_tags.py index 45ee43eee5..b2d774b037 100644 --- a/python/tests/test_runtime_metrics_tags.py +++ b/python/tests/test_runtime_metrics_tags.py @@ -104,8 +104,8 @@ def verify_seldon_metrics(data, mycounter_value, histogram_entries, method): def test_generate_tags_key(): # initializing two different kinds of dictionary - insertion_order = OrderedDict({'b': 'b', 'a': 'a'}) - sorted_order = {'a': 'a', 'b': 'b'} + insertion_order = OrderedDict({"b": "b", "a": "a"}) + sorted_order = {"a": "a", "b": "b"} # assert the items in the list differ based on order assert list(insertion_order.items()) != list(sorted_order.items()) From 55869dfd1a29aa8a4e1e30f9cb4cb4a59ab680cb Mon Sep 17 00:00:00 2001 From: harshita-meena Date: Fri, 13 May 2022 16:03:42 -0700 Subject: [PATCH 13/13] fix formatting --- python/tests/test_runtime_metrics_tags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tests/test_runtime_metrics_tags.py b/python/tests/test_runtime_metrics_tags.py index b2d774b037..3f9160a695 100644 --- a/python/tests/test_runtime_metrics_tags.py +++ b/python/tests/test_runtime_metrics_tags.py @@ -1,11 +1,11 @@ import json import logging import os +from collections import OrderedDict import numpy as np import pytest from google.protobuf import json_format -from collections import OrderedDict from seldon_core.flask_utils import SeldonMicroserviceException from seldon_core.metrics import (