Clear methods when Gunicorn worker exits

python/licenses/license.txt

@@ -147,7 +147,7 @@ SOFTWARE.
 
 
 Markdown
-3.3.3
+3.3.4
 BSD License
 Copyright 2007, 2008 The Python Markdown Project (v. 1.7 and later)
 Copyright 2004, 2005, 2006 Yuri Takhteyev (v. 0.2-1.6b)
@@ -1248,7 +1248,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 google-auth
-1.27.0
+1.27.1
 Apache Software License
                                  Apache License
                            Version 2.0, January 2004
@@ -1454,7 +1454,7 @@ Apache Software License
 
 
 google-auth-oauthlib
-0.4.2
+0.4.3
 Apache Software License
                             Apache License
                            Version 2.0, January 2004
@@ -1866,7 +1866,7 @@ Apache Software License
 
 
 grpcio
-1.35.0
+1.36.1
 Apache Software License
 
                                  Apache License
@@ -2558,7 +2558,7 @@ modification, are permitted provided that the following conditions are met:
 
 
 importlib-metadata
-3.4.0
+3.7.0
 Apache Software License
 Copyright 2017-2019 Jason R. Coombs, Barry Warsaw
 
@@ -4043,7 +4043,7 @@ Apache Software License
 
 
 protobuf
-3.15.1
+3.15.5
 3-Clause BSD License
 UNKNOWN
 
@@ -4391,7 +4391,7 @@ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
 
 rsa
-4.7.1
+4.7.2
 Apache Software License
 Copyright 2011 Sybren A. Stüvel <[email protected]>
 
@@ -6302,7 +6302,7 @@ POSSIBILITY OF SUCH DAMAGE.
 
 
 zipp
-3.4.0
+3.4.1
 MIT License
 Copyright Jason R. Coombs
 

python/licenses/license_info.csv

@@ -4,7 +4,7 @@
 "Flask-OpenTracing","1.1.0","BSD License"
 "Jinja2","2.11.3","BSD License"
 "Keras-Preprocessing","1.1.2","MIT License"
-"Markdown","3.3.3","BSD License"
+"Markdown","3.3.4","BSD License"
 "MarkupSafe","1.1.1","BSD License"
 "PyYAML","5.4.1","MIT License"
 "Werkzeug","1.0.1","BSD License"
@@ -19,16 +19,16 @@
 "cryptography","3.4","Apache Software License, BSD License"
 "flatbuffers","1.12","Apache Software License"
 "gast","0.3.3","BSD License"
-"google-auth","1.27.0","Apache Software License"
-"google-auth-oauthlib","0.4.2","Apache Software License"
+"google-auth","1.27.1","Apache Software License"
+"google-auth-oauthlib","0.4.3","Apache Software License"
 "google-pasta","0.2.0","Apache Software License"
-"grpcio","1.35.0","Apache Software License"
+"grpcio","1.36.1","Apache Software License"
 "grpcio-opentracing","1.1.4","Apache Software License"
 "grpcio-reflection","1.34.1","Apache Software License"
 "gunicorn","20.0.4","MIT License"
 "h5py","2.10.0","BSD License"
 "idna","2.10","BSD License"
-"importlib-metadata","3.4.0","Apache Software License"
+"importlib-metadata","3.7.0","Apache Software License"
 "itsdangerous","1.1.0","BSD License"
 "jaeger-client","4.4.0","Apache Software License"
 "jsonschema","3.2.0","MIT License"
@@ -37,15 +37,15 @@
 "opentracing","2.4.0","Apache Software License"
 "opt-einsum","3.3.0","MIT"
 "prometheus-client","0.8.0","Apache Software License"
-"protobuf","3.15.1","3-Clause BSD License"
+"protobuf","3.15.5","3-Clause BSD License"
 "pyasn1","0.4.8","BSD License"
 "pyasn1-modules","0.2.8","BSD License"
 "pycparser","2.20","BSD License"
 "pyrsistent","0.17.3","MIT License"
 "redis","3.5.3","MIT License"
 "requests","2.25.1","Apache Software License"
 "requests-oauthlib","1.3.0","BSD License"
-"rsa","4.7.1","Apache Software License"
+"rsa","4.7.2","Apache Software License"
 "seldon-core","1.7.0.dev0","Apache 2.0"
 "semantic-version","2.8.5","BSD License"
 "setuptools-rust","0.11.6","MIT License"
@@ -62,4 +62,4 @@
 "typing-extensions","3.7.4.3","Python Software Foundation License"
 "urllib3","1.25.9","MIT License"
 "wrapt","1.12.1","BSD License"
-"zipp","3.4.0","MIT License"
+"zipp","3.4.1","MIT License"

python/licenses/license_info.no_versions.csv

@@ -32,7 +32,7 @@
 "itsdangerous","BSD License"
 "jaeger-client","Apache Software License"
 "jsonschema","MIT License"
-"numpy","BSD"
+"numpy","BSD License"
 "oauthlib","BSD License"
 "opentracing","Apache Software License"
 "opt-einsum","MIT"

python/seldon_core/app.py → python/seldon_core/gunicorn_utils.py

@@ -6,6 +6,7 @@
 
 from gunicorn.app.base import BaseApplication
 
+from seldon_core.metrics import SeldonMetrics
 from seldon_core.utils import setup_tracing
 
 logger = logging.getLogger(__name__)
@@ -17,6 +18,11 @@ def post_worker_init(worker):
     atexit.unregister(_exit_function)
 
 
+def worker_exit(server, worker, seldon_metrics: SeldonMetrics):
+    # Clear all metrics from dying worker
+    seldon_metrics.clear()
+
+
 def accesslog(flag: bool) -> Union[str, None]:
     """
     Enable / disable access log in Gunicorn depending on the flag.

python/seldon_core/metrics.py

@@ -184,6 +184,16 @@ def generate_metrics(self):
             exposition.CONTENT_TYPE_LATEST,
         )
 
+    def clear(self):
+        """
+        Clear all metrics from current worker.
+        """
+        worker_id = self.worker_id_func()
+        logger.debug(f"Clearing metrics from worker #{worker_id}")
+        with self._lock:
+            if worker_id in self.data:
+                del self.data[worker_id]
+
     def _merge_labels(self, worker, tags):
         labels = {
             **tags,

python/seldon_core/microservice.py

@@ -5,21 +5,22 @@
 import multiprocessing as mp
 import os
 import sys
-import threading
 import time
 from distutils.util import strtobool
+from functools import partial
 from typing import Callable, Dict
 
 from seldon_core import __version__, persistence
 from seldon_core import wrapper as seldon_microservice
-from seldon_core.app import (
+from seldon_core.flask_utils import ANNOTATIONS_FILE, SeldonMicroserviceException
+from seldon_core.gunicorn_utils import (
     StandaloneApplication,
     UserModelApplication,
     accesslog,
     post_worker_init,
     threads,
+    worker_exit,
 )
-from seldon_core.flask_utils import ANNOTATIONS_FILE, SeldonMicroserviceException
 from seldon_core.metrics import SeldonMetrics
 from seldon_core.utils import getenv_as_bool, setup_tracing
 
@@ -404,6 +405,7 @@ def rest_prediction_server():
                 "max_requests": args.max_requests,
                 "max_requests_jitter": args.max_requests_jitter,
                 "post_worker_init": post_worker_init,
+                "worker_exit": partial(worker_exit, seldon_metrics=seldon_metrics),
             }
             if args.pidfile is not None:
                 options["pidfile"] = args.pidfile

python/tests/resources/custom-metrics-model/.s2i/environment

@@ -0,0 +1,4 @@
+MODEL_NAME=MyModel
+API_TYPE=REST
+SERVICE_TYPE=MODEL
+PERSISTENCE=0

python/tests/resources/custom-metrics-model/MyModel.py

@@ -0,0 +1,33 @@
+import logging
+
+from seldon_core.user_model import SeldonResponse
+
+
+def reshape(x):
+    if len(x.shape) < 2:
+        return x.reshape(1, -1)
+    else:
+        return x
+
+
+class MyModel:
+    def predict(self, features, names=[], meta={}):
+        X = reshape(features)
+
+        logging.info(f"model features: {features}")
+        logging.info(f"model names: {names}")
+        logging.info(f"model meta: {meta}")
+
+        logging.info(f"model X: {X}")
+
+        runtime_metrics = [
+            {"type": "COUNTER", "key": "instance_counter", "value": len(X)}
+        ]
+        runtime_tags = {"runtime": "tag", "shared": "right one"}
+        return SeldonResponse(data=X, metrics=runtime_metrics, tags=runtime_tags)
+
+    def metrics(self):
+        return [{"type": "COUNTER", "key": "requests_counter", "value": 1}]
+
+    def tags(self):
+        return {"static": "tag", "shared": "not right one"}

python/tests/test_gunicorn_utils.py

@@ -0,0 +1,51 @@
+import os
+import signal
+import time
+from typing import Set
+
+import pytest
+import requests
+from prometheus_client.parser import text_string_to_metric_families
+
+
+def _get_workers(
+    metrics_endpoint: str = "http://127.0.0.1:6005/metrics-endpoint",
+) -> Set[str]:
+    workers = set()
+
+    res = requests.get("http://127.0.0.1:6005/metrics-endpoint")
+    families = text_string_to_metric_families(res.content.decode())
+    for fam in families:
+        for sample in fam.samples:
+            worker_id = sample.labels["worker_id"]
+            workers.add(worker_id)
+
+    return workers
+
+
+@pytest.mark.parametrize(
+    "microservice", [{"app_name": "custom-metrics-model"}], indirect=True
+)
+def test_worker_exit(microservice):
+    # Warm up the custom metrics
+    for _ in range(5):
+        res = requests.post(
+            "http://127.0.0.1:9000/api/v1.0/predictions",
+            json={"data": {"ndarray": [[1, 2, 3]]}},
+        )
+        res.raise_for_status()
+
+    # Fetch metrics and get current list of workers
+    workers = _get_workers(metrics_endpoint="http://127.0.0.1:6005/metrics-endpoint")
+    assert len(workers) != 0
+
+    # Ask Gunicorn to restart all workers (through a HUP)
+    server_pid = microservice.p.pid
+    os.kill(server_pid, signal.SIGHUP)
+    time.sleep(1)
+
+    # Metrics should now be empty
+    empty_workers = _get_workers(
+        metrics_endpoint="http://127.0.0.1:6005/metrics-endpoint"
+    )
+    assert len(empty_workers) == 0

python/tests/test_metrics.py

@@ -141,6 +141,15 @@ def test_component_bad():
         client_custom_metrics(c, SeldonMetrics(), TEST_METRIC_METHOD_TAG)
 
 
+def test_metrics_clear():
+    metrics = SeldonMetrics()
+    metrics.update([RAW_COUNTER_METRIC], method="predict")
+    assert len(metrics.data) > 0
+
+    metrics.clear()
+    assert len(metrics.data) == 0
+
+
 def test_proto_metrics():
     metrics = [{"type": "COUNTER", "key": "a", "value": 1}]
     meta = prediction_pb2.Meta()