✨ Add profiling middleware to simcore services

.env-devel

@@ -18,6 +18,7 @@ AGENT_VOLUMES_CLEANUP_S3_BUCKET=simcore-volume-backups
 AGENT_VOLUMES_CLEANUP_S3_PROVIDER=MINIO
 
 API_SERVER_DEV_FEATURES_ENABLED=0
+API_SERVER_PROFILING_MIDDLEWARE_ENABLED=1
 
 AUTOSCALING_DASK=null
 AUTOSCALING_DRAIN_NODES_WITH_LABELS=False
@@ -34,6 +35,7 @@ BF_API_SECRET=none
 CATALOG_DEV_FEATURES_ENABLED=0
 CATALOG_SERVICES_DEFAULT_RESOURCES='{"CPU": {"limit": 0.1, "reservation": 0.1}, "RAM": {"limit": 2147483648, "reservation": 2147483648}}'
 CATALOG_SERVICES_DEFAULT_SPECIFICATIONS='{}'
+CATALOG_PROFILING_MIDDLEWARE_ENABLED=1
 
 CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG=master-github-latest
 CLUSTERS_KEEPER_DASK_NTHREADS=0
@@ -63,6 +65,7 @@ COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_FILE_LINK_TYPE=S3
 COMPUTATIONAL_BACKEND_DEFAULT_FILE_LINK_TYPE=PRESIGNED
 COMPUTATIONAL_BACKEND_ON_DEMAND_CLUSTERS_FILE_LINK_TYPE=PRESIGNED
 DIRECTOR_V2_DEV_FEATURES_ENABLED=0
+DIRECTOR_V2_PROFILING_MIDDLEWARE_ENABLED=1
 
 DYNAMIC_SIDECAR_ENDPOINT_SPECS_MODE_DNSRR_ENABLED=0
 DYNAMIC_SIDECAR_IMAGE=${DOCKER_REGISTRY:-itisfoundation}/dynamic-sidecar:${DOCKER_IMAGE_TAG:-latest}
@@ -163,6 +166,7 @@ SIMCORE_SERVICES_NETWORK_NAME=interactive_services_subnet
 STORAGE_ENDPOINT=storage:8080
 STORAGE_HOST=storage
 STORAGE_PORT=8080
+STORAGE_PROFILING_MIDDLEWARE_ENABLED=1
 
 SWARM_STACK_NAME=master-simcore
 
@@ -184,6 +188,7 @@ PROJECTS_INACTIVITY_INTERVAL=20
 WEBSERVER_ANNOUNCEMENTS=1
 
 WEBSERVER_DEV_FEATURES_ENABLED=0
+WEBSERVER_PROFILING_MIDDLEWARE_ENABLED=1
 
 WEBSERVER_HOST=webserver
 WEBSERVER_PORT=8080
@@ -228,6 +233,7 @@ DIRECTOR_V2_PUBLIC_API_BASE_URL=http://127.0.0.1:8006
 DYNAMIC_SIDECAR_ENABLE_VOLUME_LIMITS=False
 
 DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=3600
+DYNAMIC_SCHEDULER_PROFILING_MIDDLEWARE_ENABLED=1
 
 WEBSERVER_DIAGNOSTICS={}
 DIAGNOSTICS_MAX_AVG_LATENCY=10

packages/service-library/src/servicelib/_utils_profiling_middleware.py

@@ -0,0 +1,24 @@
+import json
+
+
+def append_profile(body: str, profile: str) -> str:
+    try:
+        json.loads(body)
+        body += "\n" if not body.endswith("\n") else ""
+    except json.decoder.JSONDecodeError:
+        pass
+    body += json.dumps({"profile": profile})
+    return body
+
+
+def check_response_headers(
+    response_headers: dict[bytes, bytes]
+) -> list[tuple[bytes, bytes]]:
+    original_content_type: str = response_headers[b"content-type"].decode()
+    assert original_content_type in {
+        "application/x-ndjson",
+        "application/json",
+    }  # nosec
+    headers: dict = {}
+    headers[b"content-type"] = b"application/x-ndjson"
+    return list(headers.items())

packages/service-library/src/servicelib/aiohttp/profiler_middleware.py

@@ -0,0 +1,42 @@
+from aiohttp.web import Request, StreamResponse, middleware
+from pyinstrument import Profiler
+from servicelib.aiohttp import status
+
+from .._utils_profiling_middleware import append_profile
+
+
+def create_profiling_middleware(app_name: str):
+    @middleware
+    async def profiling_middleware(request: Request, handler):
+        profiler: Profiler | None = None
+        if request.headers.get(f"x-profile-{app_name}") is not None:
+            profiler = Profiler(async_mode="enabled")
+            profiler.start()
+
+        response = await handler(request)
+
+        if profiler is None:
+            return response
+        if response.content_type != "application/json":
+            return StreamResponse(
+                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                reason=f"Profiling middleware is not compatible with {response.content_type=}",
+                headers={},
+            )
+
+        stream_response = StreamResponse(
+            status=response.status,
+            reason=response.reason,
+            headers=response.headers,
+        )
+        stream_response.content_type = "application/x-ndjson"
+        await stream_response.prepare(request)
+        await stream_response.write(response.body)
+        profiler.stop()
+        await stream_response.write(
+            append_profile("", profiler.output_text(unicode=True, color=True)).encode()
+        )
+        await stream_response.write_eof()
+        return stream_response
+
+    return profiling_middleware

services/api-server/src/simcore_service_api_server/core/_profiler_middleware.py → packages/service-library/src/servicelib/fastapi/profiler_middleware.py

@@ -1,32 +1,10 @@
-import json
 from typing import Any
 
 from fastapi import FastAPI
 from pyinstrument import Profiler
 from starlette.requests import Request
 
-
-def _check_response_headers(
-    response_headers: dict[bytes, bytes]
-) -> list[tuple[bytes, bytes]]:
-    original_content_type: str = response_headers[b"content-type"].decode()
-    assert original_content_type in {
-        "application/x-ndjson",
-        "application/json",
-    }  # nosec
-    headers: dict = {}
-    headers[b"content-type"] = b"application/x-ndjson"
-    return list(headers.items())
-
-
-def _append_profile(body: str, profile: str) -> str:
-    try:
-        json.loads(body)
-        body += "\n" if not body.endswith("\n") else ""
-    except json.decoder.JSONDecodeError:
-        pass
-    body += json.dumps({"profile": profile})
-    return body
+from .._utils_profiling_middleware import append_profile, check_response_headers
 
 
 def is_last_response(response_headers: dict[bytes, bytes], message: dict[str, Any]):
@@ -40,16 +18,16 @@ def is_last_response(response_headers: dict[bytes, bytes], message: dict[str, An
     raise RuntimeError(msg)
 
 
-class ApiServerProfilerMiddleware:
+class ProfilerMiddleware:
     """Following
     https://www.starlette.io/middleware/#cleanup-and-error-handling
     https://www.starlette.io/middleware/#reusing-starlette-components
     https://fastapi.tiangolo.com/advanced/middleware/#advanced-middleware
     """
 
-    def __init__(self, app: FastAPI):
+    def __init__(self, app: FastAPI, app_name: str):
         self._app: FastAPI = app
-        self._profile_header_trigger: str = "x-profile-api-server"
+        self._profile_header_trigger: str = f"x-profile-{app_name}"
 
     async def __call__(self, scope, receive, send):
         if scope["type"] != "http":
@@ -61,7 +39,7 @@ async def __call__(self, scope, receive, send):
         request_headers = dict(request.headers)
         response_headers: dict[bytes, bytes] = {}
 
-        if request_headers.get(self._profile_header_trigger) == "true":
+        if request_headers.get(self._profile_header_trigger) is not None:
             request_headers.pop(self._profile_header_trigger)
             scope["headers"] = [
                 (k.encode("utf8"), v.encode("utf8")) for k, v in request_headers.items()
@@ -74,11 +52,11 @@ async def _send_wrapper(message):
                 nonlocal response_headers
                 if message["type"] == "http.response.start":
                     response_headers = dict(message.get("headers"))
-                    message["headers"] = _check_response_headers(response_headers)
+                    message["headers"] = check_response_headers(response_headers)
                 elif message["type"] == "http.response.body":
                     if is_last_response(response_headers, message):
                         profiler.stop()
-                        message["body"] = _append_profile(
+                        message["body"] = append_profile(
                             message["body"].decode(),
                             profiler.output_text(unicode=True, color=True),
                         ).encode()

services/api-server/requirements/_test.in

@@ -19,7 +19,6 @@ docker
 faker
 jsonref
 moto[server] # mock out tests based on AWS-S3
-pyinstrument
 pytest
 pytest-asyncio
 pytest-cov

services/api-server/requirements/_test.txt

@@ -20,19 +20,19 @@ aws-sam-translator==1.55.0
     # via cfn-lint
 aws-xray-sdk==2.13.0
     # via moto
-boto3==1.34.76
+boto3==1.34.93
     # via
     #   aws-sam-translator
     #   moto
-boto3-stubs==1.34.76
+boto3-stubs==1.34.93
     # via types-boto3
-botocore==1.34.76
+botocore==1.34.93
     # via
     #   aws-xray-sdk
     #   boto3
     #   moto
     #   s3transfer
-botocore-stubs==1.34.69
+botocore-stubs==1.34.93
     # via boto3-stubs
 certifi==2024.2.2
     # via
@@ -47,7 +47,7 @@ charset-normalizer==3.3.2
     # via requests
 click==8.1.7
     # via flask
-coverage==7.4.4
+coverage==7.5.0
     # via pytest-cov
 cryptography==42.0.5
     # via
@@ -56,7 +56,7 @@ cryptography==42.0.5
     #   sshpubkeys
 docker==7.0.0
     # via moto
-ecdsa==0.18.0
+ecdsa==0.19.0
     # via
     #   moto
     #   python-jose
@@ -65,7 +65,7 @@ exceptiongroup==1.2.0
     # via
     #   anyio
     #   pytest
-faker==24.4.0
+faker==24.14.1
 flask==2.1.3
     # via
     #   flask-cors
@@ -111,7 +111,7 @@ jsondiff==2.0.0
     # via moto
 jsonpatch==1.33
     # via cfn-lint
-jsonpickle==3.0.3
+jsonpickle==3.0.4
     # via jschema-to-python
 jsonpointer==2.4
     # via jsonpatch
@@ -136,7 +136,7 @@ multidict==6.0.5
     # via
     #   aiohttp
     #   yarl
-mypy==1.9.0
+mypy==1.10.0
     # via sqlalchemy
 mypy-extensions==1.0.0
     # via mypy
@@ -154,20 +154,19 @@ pbr==6.0.0
     # via
     #   jschema-to-python
     #   sarif-om
-pluggy==1.4.0
+pluggy==1.5.0
     # via pytest
 pyasn1==0.6.0
     # via
     #   python-jose
     #   rsa
 pycparser==2.22
     # via cffi
-pyinstrument==4.6.2
 pyparsing==3.1.2
     # via moto
 pyrsistent==0.20.0
     # via jsonschema
-pytest==8.1.1
+pytest==8.2.0
     # via
     #   pytest-asyncio
     #   pytest-cov
@@ -234,10 +233,10 @@ tomli==2.0.1
     #   coverage
     #   mypy
     #   pytest
-types-awscrt==0.20.5
+types-awscrt==0.20.9
     # via botocore-stubs
 types-boto3==1.0.2
-types-s3transfer==0.10.0
+types-s3transfer==0.10.1
     # via boto3-stubs
 typing-extensions==4.10.0
     # via

services/api-server/requirements/_tools.txt

@@ -1,6 +1,6 @@
 astroid==3.1.0
     # via pylint
-black==24.3.0
+black==24.4.2
 build==1.2.1
     # via pip-tools
 bump2version==1.0.1
@@ -15,9 +15,9 @@ dill==0.3.8
     # via pylint
 distlib==0.3.8
     # via virtualenv
-filelock==3.13.3
+filelock==3.13.4
     # via virtualenv
-identify==2.5.35
+identify==2.5.36
     # via pre-commit
 isort==5.13.2
     # via pylint
@@ -39,22 +39,22 @@ pathspec==0.12.1
 pip==24.0
     # via pip-tools
 pip-tools==7.4.1
-platformdirs==4.2.0
+platformdirs==4.2.1
     # via
     #   black
     #   pylint
     #   virtualenv
 pre-commit==3.7.0
 pylint==3.1.0
-pyproject-hooks==1.0.0
+pyproject-hooks==1.1.0
     # via
     #   build
     #   pip-tools
 pyyaml==6.0.1
     # via
     #   pre-commit
     #   watchdog
-ruff==0.3.5
+ruff==0.4.2
 setuptools==69.2.0
     # via
     #   nodeenv
@@ -65,14 +65,13 @@ tomli==2.0.1
     #   build
     #   pip-tools
     #   pylint
-    #   pyproject-hooks
 tomlkit==0.12.4
     # via pylint
 typing-extensions==4.10.0
     # via
     #   astroid
     #   black
-virtualenv==20.25.1
+virtualenv==20.26.0
     # via pre-commit
 watchdog==4.0.0
 wheel==0.43.0

services/api-server/src/simcore_service_api_server/core/application.py

@@ -120,10 +120,10 @@ def init_app(settings: ApplicationSettings | None = None) -> FastAPI:
             add_oec_to_message=True,
         ),
     )
-    if settings.API_SERVER_DEV_FEATURES_ENABLED:
-        from ._profiler_middleware import ApiServerProfilerMiddleware
+    if settings.API_SERVER_PROFILING_MIDDLEWARE_ENABLED:
+        from servicelib.fastapi.profiler_middleware import ProfilerMiddleware
 
-        app.add_middleware(ApiServerProfilerMiddleware)
+        app.add_middleware(ProfilerMiddleware, app_name="api-server")
 
     if app.state.settings.API_SERVER_PROMETHEUS_INSTRUMENTATION_ENABLED:
         setup_prometheus_instrumentation(app)

services/api-server/src/simcore_service_api_server/core/settings.py

@@ -88,6 +88,7 @@ class ApplicationSettings(BasicSettings):
     API_SERVER_HEALTH_CHECK_TASK_TIMEOUT_SECONDS: PositiveInt = 10
     API_SERVER_ALLOWED_HEALTH_CHECK_FAILURES: PositiveInt = 5
     API_SERVER_PROMETHEUS_INSTRUMENTATION_COLLECT_SECONDS: PositiveInt = 5
+    API_SERVER_PROFILING_MIDDLEWARE_ENABLED: bool = False
     # DEV-TOOLS
     API_SERVER_DEV_HTTP_CALLS_LOGS_PATH: Path | None = Field(
         default=None,