feat: include glibc malloc & jemalloc stats in memusage

Closes #1101

autopush/memusage.py

@@ -11,6 +11,19 @@
 
 from autopush.gcdump import Stat
 
+from cffi import FFI
+
+
+# cffi's ABI mode is preferable but it would assume jemalloc is always
+# available (and we LD_PRELOAD it)
+ffi = FFI()
+ffi.cdef("""
+int malloc_info(int options, FILE *stream);
+void malloc_stats_print(void (*write_cb) (void *, const char *),
+                        void *cbopaque, const char *opts);
+""")
+lib = ffi.dlopen(None)
+
 
 def memusage(do_dump_rpy_heap=True, do_objgraph=True):
     """Returning a str of memory usage stats"""
@@ -26,6 +39,8 @@ def trap_err(func, *args, **kwargs):
     rusage = trap_err(resource.getrusage, resource.RUSAGE_SELF)
     buf.writelines([repr(rusage), '\n\n'])
     trap_err(pmap_extended, buf)
+    trap_err(jemalloc_stats, buf)
+    trap_err(glibc_malloc_info, buf)
     if do_dump_rpy_heap:
         # dump rpython's heap before objgraph potentially pollutes the
         # heap with its heavy workload
@@ -90,3 +105,27 @@ def get_stats_asmmemmgr(stream):  # pragma: nocover
     stream.write('\n\nget_stats_asmmemmgr: ')
     stream.write(repr(pypyjit.get_stats_asmmemmgr()))
     stream.write('\n')
+
+
+def glibc_malloc_info(stream):
+    """Write glib malloc's malloc_info(3)"""
+    with tempfile.NamedTemporaryFile('wb+') as fp:
+        if not lib.malloc_info(0, fp.file):
+            fp.seek(0)
+            stream.writelines(fp.readlines())
+
+
+def jemalloc_stats(stream):
+    """Write jemalloc's malloc_stats_print()"""
+    try:
+        malloc_stats_print = lib.malloc_stats_print
+    except AttributeError:
+        # not using jemalloc
+        return
+
+    @ffi.callback("void (*write_cb) (void *, const char *)")
+    def write_cb(handle, msg):
+        stream = ffi.from_handle(handle)
+        stream.write(ffi.string(msg))
+    malloc_stats_print(write_cb, ffi.new_handle(stream), ffi.NULL)
+    stream.write('\n')

autopush/tests/test_integration.py

@@ -2110,6 +2110,7 @@ def test_memusage(self):
         )
         assert response.code == 200
         assert 'ru_maxrss=' in body
+        assert '<malloc ' in body
         assert 'Logger' in body
         if find_executable('pmap'):
             assert 'RSS' in body or 'Rss' in body  # pmap -x or -XX/X output
@@ -2126,6 +2127,7 @@ def test_memusage_options(self):
         response, body = yield _agent('GET', url)
         assert response.code == 200
         assert 'ru_maxrss=' in body
+        assert '<malloc ' in body
         assert 'Logger' not in body
         if find_executable('pmap'):
             assert 'RSS' in body or 'Rss' in body  # pmap -x or -XX/X output

requirements.in

@@ -3,6 +3,7 @@ attrs
 autobahn[twisted]
 boto
 boto3
+cffi
 click
 configargparse
 cryptography