diff --git a/autopush/main.py b/autopush/main.py index 5e24f924..ca5afb4e 100644 --- a/autopush/main.py +++ b/autopush/main.py @@ -34,6 +34,7 @@ from autopush.haproxy import HAProxyServerEndpoint from autopush.logging import PushLogger from autopush.main_argparse import parse_connection, parse_endpoint +from autopush.metrics import periodic_reporter from autopush.router import routers_from_config from autopush.ssl import ( monkey_patch_ssl_wrap_socket, @@ -189,6 +190,8 @@ def setup(self, rotate_tables=True): # Start the table rotation checker/updater if rotate_tables: self.add_timer(60, self.db.update_rotating_tables) + self.add_timer(15, periodic_reporter, self.db.metrics, + prefix='autoendpoint') def add_endpoint(self): """Start the Endpoint HTTP router""" @@ -259,6 +262,7 @@ def setup(self, rotate_tables=True): # Start the table rotation checker/updater if rotate_tables: self.add_timer(60, self.db.update_rotating_tables) + self.add_timer(15, periodic_reporter, self.db.metrics) def add_internal_router(self): """Start the internal HTTP notification router""" diff --git a/autopush/metrics.py b/autopush/metrics.py index 38766e91..6becd51f 100644 --- a/autopush/metrics.py +++ b/autopush/metrics.py @@ -129,3 +129,21 @@ def from_config(conf): return TwistedMetrics(conf.statsd_host, conf.statsd_port) else: return SinkMetrics() + + +def periodic_reporter(metrics, prefix=''): + # type: (IMetrics, Optional[str]) -> None + """Emit metrics on twisted's thread pool. + + Only meant to be called via a LoopingCall (TimerService). + + """ + # unfortunately stats only available via the private '_team' + stats = reactor.getThreadPool()._team.statistics() + for attr in ('idleWorkerCount', 'busyWorkerCount', 'backloggedWorkCount'): + name = '{}{}twisted.threadpool.{}'.format( + prefix, + '.' if prefix else '', + attr + ) + metrics.gauge(name, getattr(stats, attr)) diff --git a/autopush/tests/test_metrics.py b/autopush/tests/test_metrics.py index ef33896a..f3f076a0 100644 --- a/autopush/tests/test_metrics.py +++ b/autopush/tests/test_metrics.py @@ -3,13 +3,14 @@ import twisted.internet.base import pytest -from mock import Mock, patch +from mock import Mock, call, patch from autopush.metrics import ( IMetrics, DatadogMetrics, TwistedMetrics, SinkMetrics, + periodic_reporter ) @@ -71,3 +72,19 @@ def test_basic(self, mock_dog): m.timing("lifespan", 113) m._client.timing.assert_called_with("testpush.lifespan", value=113, host=hostname) + + +class PeriodicReporterTestCase(unittest.TestCase): + + def test_periodic_reporter(self): + metrics = Mock(spec=SinkMetrics) + periodic_reporter(metrics) + periodic_reporter(metrics, prefix='foo') + metrics.gauge.assert_has_calls([ + call('twisted.threadpool.idleWorkerCount', 0), + call('twisted.threadpool.busyWorkerCount', 0), + call('twisted.threadpool.backloggedWorkCount', 0), + call('foo.twisted.threadpool.idleWorkerCount', 0), + call('foo.twisted.threadpool.busyWorkerCount', 0), + call('foo.twisted.threadpool.backloggedWorkCount', 0), + ])