feat: add some metrics for twisted's threadpool

emitted every 15 seconds per datadog's preferred minimum

Closes #1093

(cherry picked from commit 6a968c6)

autopush/main.py

@@ -34,6 +34,7 @@
 from autopush.haproxy import HAProxyServerEndpoint
 from autopush.logging import PushLogger
 from autopush.main_argparse import parse_connection, parse_endpoint
+from autopush.metrics import periodic_reporter
 from autopush.router import routers_from_config
 from autopush.ssl import (
     monkey_patch_ssl_wrap_socket,
@@ -189,6 +190,8 @@ def setup(self, rotate_tables=True):
         # Start the table rotation checker/updater
         if rotate_tables:
             self.add_timer(60, self.db.update_rotating_tables)
+        self.add_timer(15, periodic_reporter, self.db.metrics,
+                       prefix='autoendpoint')
 
     def add_endpoint(self):
         """Start the Endpoint HTTP router"""
@@ -259,6 +262,7 @@ def setup(self, rotate_tables=True):
         # Start the table rotation checker/updater
         if rotate_tables:
             self.add_timer(60, self.db.update_rotating_tables)
+        self.add_timer(15, periodic_reporter, self.db.metrics)
 
     def add_internal_router(self):
         """Start the internal HTTP notification router"""

autopush/metrics.py

@@ -129,3 +129,21 @@ def from_config(conf):
         return TwistedMetrics(conf.statsd_host, conf.statsd_port)
     else:
         return SinkMetrics()
+
+
+def periodic_reporter(metrics, prefix=''):
+    # type: (IMetrics, Optional[str]) -> None
+    """Emit metrics on twisted's thread pool.
+
+    Only meant to be called via a LoopingCall (TimerService).
+
+    """
+    # unfortunately stats only available via the private '_team'
+    stats = reactor.getThreadPool()._team.statistics()
+    for attr in ('idleWorkerCount', 'busyWorkerCount', 'backloggedWorkCount'):
+        name = '{}{}twisted.threadpool.{}'.format(
+            prefix,
+            '.' if prefix else '',
+            attr
+        )
+        metrics.gauge(name, getattr(stats, attr))

autopush/tests/test_metrics.py

@@ -3,13 +3,14 @@
 import twisted.internet.base
 
 import pytest
-from mock import Mock, patch
+from mock import Mock, call, patch
 
 from autopush.metrics import (
     IMetrics,
     DatadogMetrics,
     TwistedMetrics,
     SinkMetrics,
+    periodic_reporter
 )
 
 
@@ -71,3 +72,19 @@ def test_basic(self, mock_dog):
         m.timing("lifespan", 113)
         m._client.timing.assert_called_with("testpush.lifespan", value=113,
                                             host=hostname)
+
+
+class PeriodicReporterTestCase(unittest.TestCase):
+
+    def test_periodic_reporter(self):
+        metrics = Mock(spec=SinkMetrics)
+        periodic_reporter(metrics)
+        periodic_reporter(metrics, prefix='foo')
+        metrics.gauge.assert_has_calls([
+            call('twisted.threadpool.idleWorkerCount', 0),
+            call('twisted.threadpool.busyWorkerCount', 0),
+            call('twisted.threadpool.backloggedWorkCount', 0),
+            call('foo.twisted.threadpool.idleWorkerCount', 0),
+            call('foo.twisted.threadpool.busyWorkerCount', 0),
+            call('foo.twisted.threadpool.backloggedWorkCount', 0),
+        ])