diff --git a/acceptance_tests/tests/tests/test_debug.py b/acceptance_tests/tests/tests/test_debug.py index cfed2fd71..abcf2680e 100644 --- a/acceptance_tests/tests/tests/test_debug.py +++ b/acceptance_tests/tests/tests/test_debug.py @@ -77,7 +77,7 @@ def test_headers(app_connection): def _check_leak_there(response): print("response=" + json.dumps(response, indent=4)) - leaked = {v[0]: v[2] for v in response} + leaked = {v[0]: v[2] for v in response["objgraph.growth"]} assert "c2cwsgiutils_app.services.LeakedObject" in leaked, leaked.keys() assert leaked["c2cwsgiutils_app.services.LeakedObject"] == 1 diff --git a/c2cwsgiutils/debug/_views.py b/c2cwsgiutils/debug/_views.py index 649a60735..b91fcdf12 100644 --- a/c2cwsgiutils/debug/_views.py +++ b/c2cwsgiutils/debug/_views.py @@ -1,5 +1,6 @@ import gc import logging +import os import re import time from collections.abc import Mapping @@ -8,6 +9,7 @@ from typing import Any, Callable, cast import objgraph +import psutil import pyramid.config import pyramid.request import pyramid.response @@ -87,6 +89,9 @@ def _dump_memory_diff(request: pyramid.request.Request) -> list[Any]: gc.collect(i) objgraph.growth(limit=limit, peak_stats=peak_stats, shortnames=False) + process = psutil.Process(os.getpid()) + mem_before = process.memory_info() + start_time = time.time() response = None try: @@ -96,12 +101,26 @@ def _dump_memory_diff(request: pyramid.request.Request) -> list[Any]: except HTTPException as ex: _LOG.debug("response was %s", str(ex)) + elapsed_time = time.time() - start_time del response for i in range(3): gc.collect(i) - return objgraph.growth(limit=limit, peak_stats=peak_stats, shortnames=False) # type: ignore + mem_after = process.memory_info() + return { + "memory.growth": { + "rss_kb": (mem_after.rss - mem_before.rss) / 1024, + "vms_kb": (mem_after.vms - mem_before.vms) / 1024, + "shared_kb": (mem_after.shared - mem_before.shared) / 1024, + "text_kb": (mem_after.text - mem_before.text) / 1024, + "lib_kb": (mem_after.lib - mem_before.lib) / 1024, + "data_kb": (mem_after.data - mem_before.data) / 1024, + "dirty_kb": (mem_after.dirty - mem_before.dirty) / 1024, + }, + "elapsed_time": elapsed_time, + "objgraph.growth": objgraph.growth(limit=limit, peak_stats=peak_stats, shortnames=False), # type: ignore + } def _sleep(request: pyramid.request.Request) -> pyramid.response.Response: diff --git a/poetry.lock b/poetry.lock index caedc77ac..fe6e43b75 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. [[package]] name = "alembic" @@ -1612,6 +1612,36 @@ files = [ {file = "prospector_profile_utils-1.9.1.tar.gz", hash = "sha256:008efa6797a85233fd8093dcb9d86f5fa5d89673e431c15cb1496a91c9b2c601"}, ] +[[package]] +name = "psutil" +version = "6.1.0" +description = "Cross-platform lib for process and system monitoring in Python." +optional = true +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "psutil-6.1.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ff34df86226c0227c52f38b919213157588a678d049688eded74c76c8ba4a5d0"}, + {file = "psutil-6.1.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:c0e0c00aa18ca2d3b2b991643b799a15fc8f0563d2ebb6040f64ce8dc027b942"}, + {file = "psutil-6.1.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:000d1d1ebd634b4efb383f4034437384e44a6d455260aaee2eca1e9c1b55f047"}, + {file = "psutil-6.1.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:5cd2bcdc75b452ba2e10f0e8ecc0b57b827dd5d7aaffbc6821b2a9a242823a76"}, + {file = "psutil-6.1.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:045f00a43c737f960d273a83973b2511430d61f283a44c96bf13a6e829ba8fdc"}, + {file = "psutil-6.1.0-cp27-none-win32.whl", hash = "sha256:9118f27452b70bb1d9ab3198c1f626c2499384935aaf55388211ad982611407e"}, + {file = "psutil-6.1.0-cp27-none-win_amd64.whl", hash = "sha256:a8506f6119cff7015678e2bce904a4da21025cc70ad283a53b099e7620061d85"}, + {file = "psutil-6.1.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6e2dcd475ce8b80522e51d923d10c7871e45f20918e027ab682f94f1c6351688"}, + {file = "psutil-6.1.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:0895b8414afafc526712c498bd9de2b063deaac4021a3b3c34566283464aff8e"}, + {file = "psutil-6.1.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9dcbfce5d89f1d1f2546a2090f4fcf87c7f669d1d90aacb7d7582addece9fb38"}, + {file = "psutil-6.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:498c6979f9c6637ebc3a73b3f87f9eb1ec24e1ce53a7c5173b8508981614a90b"}, + {file = "psutil-6.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d905186d647b16755a800e7263d43df08b790d709d575105d419f8b6ef65423a"}, + {file = "psutil-6.1.0-cp36-cp36m-win32.whl", hash = "sha256:6d3fbbc8d23fcdcb500d2c9f94e07b1342df8ed71b948a2649b5cb060a7c94ca"}, + {file = "psutil-6.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:1209036fbd0421afde505a4879dee3b2fd7b1e14fee81c0069807adcbbcca747"}, + {file = "psutil-6.1.0-cp37-abi3-win32.whl", hash = "sha256:1ad45a1f5d0b608253b11508f80940985d1d0c8f6111b5cb637533a0e6ddc13e"}, + {file = "psutil-6.1.0-cp37-abi3-win_amd64.whl", hash = "sha256:a8fb3752b491d246034fa4d279ff076501588ce8cbcdbb62c32fd7a377d996be"}, + {file = "psutil-6.1.0.tar.gz", hash = "sha256:353815f59a7f64cdaca1c0307ee13558a0512f6db064e92fe833784f08539c7a"}, +] + +[package.extras] +dev = ["black", "check-manifest", "coverage", "packaging", "pylint", "pyperf", "pypinfo", "pytest-cov", "requests", "rstcheck", "ruff", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "virtualenv", "wheel"] +test = ["pytest", "pytest-xdist", "setuptools"] + [[package]] name = "psycopg2" version = "2.9.10" @@ -2963,9 +2993,9 @@ test = ["zope.testing"] [extras] alembic = ["alembic"] -all = ["SQLAlchemy", "SQLAlchemy-Utils", "alembic", "boltons", "cornice", "gunicorn", "lxml", "objgraph", "prometheus-client", "psycopg2", "pyjwt", "pyramid", "pyramid-tm", "pyramid_mako", "redis", "requests-oauthlib", "sentry-sdk", "waitress", "zope.interface", "zope.sqlalchemy"] +all = ["SQLAlchemy", "SQLAlchemy-Utils", "alembic", "boltons", "cornice", "gunicorn", "lxml", "objgraph", "prometheus-client", "psutil", "psycopg2", "pyjwt", "pyramid", "pyramid-tm", "pyramid_mako", "redis", "requests-oauthlib", "sentry-sdk", "waitress", "zope.interface", "zope.sqlalchemy"] broadcast = ["redis"] -debug = ["objgraph"] +debug = ["objgraph", "psutil"] dev = ["waitress"] oauth2 = ["pyjwt", "requests-oauthlib"] sentry = ["sentry-sdk"] @@ -2977,4 +3007,4 @@ webserver = ["SQLAlchemy", "SQLAlchemy-Utils", "cornice", "gunicorn", "prometheu [metadata] lock-version = "2.0" python-versions = ">=3.10,<4.0" -content-hash = "d2a9cde784ec645710bf3014913b30faac9f0ea48a5307df23a6bd1417ccd2c5" +content-hash = "a7a7c5a95650883dd8ab72624f370b3b8c6dafae7227e0b88389879b0fb46851" diff --git a/pyproject.toml b/pyproject.toml index 49f75a5da..7160bcfb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,6 +92,7 @@ waitress = { version = "3.0.1", optional = true } scikit-image = { version = "0.24.0", optional = true } prometheus-client = { version = "0.21.0", optional = true} pyramid_mako = { version = "1.1.0", optional = true} +psutil = { version = "6.1.0", optional = true} [tool.poetry.extras] standard = [ @@ -118,7 +119,7 @@ standard = [ "pyramid_mako", ] alembic = ["alembic"] -debug = ["objgraph"] +debug = ["objgraph", "psutil"] oauth2 = ["pyjwt", "requests-oauthlib"] sentry = ["sentry-sdk"] dev = ["waitress"] @@ -141,6 +142,7 @@ all = [ "alembic", # debug "objgraph", + "psutil", # oauth2 "pyjwt", "requests-oauthlib",