diff --git a/.env-devel b/.env-devel index 5a0ef089f87..e2a2e63fd87 100644 --- a/.env-devel +++ b/.env-devel @@ -18,6 +18,7 @@ AGENT_VOLUMES_CLEANUP_S3_BUCKET=simcore-volume-backups AGENT_VOLUMES_CLEANUP_S3_PROVIDER=MINIO API_SERVER_DEV_FEATURES_ENABLED=0 +API_SERVER_PROFILING=1 AUTOSCALING_DASK=null AUTOSCALING_DRAIN_NODES_WITH_LABELS=False @@ -34,6 +35,7 @@ BF_API_SECRET=none CATALOG_DEV_FEATURES_ENABLED=0 CATALOG_SERVICES_DEFAULT_RESOURCES='{"CPU": {"limit": 0.1, "reservation": 0.1}, "RAM": {"limit": 2147483648, "reservation": 2147483648}}' CATALOG_SERVICES_DEFAULT_SPECIFICATIONS='{}' +CATALOG_PROFILING=1 CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG=master-github-latest CLUSTERS_KEEPER_DASK_NTHREADS=0 @@ -63,6 +65,7 @@ COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_FILE_LINK_TYPE=S3 COMPUTATIONAL_BACKEND_DEFAULT_FILE_LINK_TYPE=PRESIGNED COMPUTATIONAL_BACKEND_ON_DEMAND_CLUSTERS_FILE_LINK_TYPE=PRESIGNED DIRECTOR_V2_DEV_FEATURES_ENABLED=0 +DIRECTOR_V2_PROFILING=1 DYNAMIC_SIDECAR_ENDPOINT_SPECS_MODE_DNSRR_ENABLED=0 DYNAMIC_SIDECAR_IMAGE=${DOCKER_REGISTRY:-itisfoundation}/dynamic-sidecar:${DOCKER_IMAGE_TAG:-latest} @@ -163,6 +166,7 @@ SIMCORE_SERVICES_NETWORK_NAME=interactive_services_subnet STORAGE_ENDPOINT=storage:8080 STORAGE_HOST=storage STORAGE_PORT=8080 +STORAGE_PROFILING=1 SWARM_STACK_NAME=master-simcore @@ -184,6 +188,7 @@ PROJECTS_INACTIVITY_INTERVAL=20 WEBSERVER_ANNOUNCEMENTS=1 WEBSERVER_DEV_FEATURES_ENABLED=0 +WEBSERVER_PROFILING=1 WEBSERVER_HOST=webserver WEBSERVER_PORT=8080 @@ -228,6 +233,7 @@ DIRECTOR_V2_PUBLIC_API_BASE_URL=http://127.0.0.1:8006 DYNAMIC_SIDECAR_ENABLE_VOLUME_LIMITS=False DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=3600 +DYNAMIC_SCHEDULER_PROFILING=1 WEBSERVER_DIAGNOSTICS={} DIAGNOSTICS_MAX_AVG_LATENCY=10 diff --git a/packages/service-library/src/servicelib/_utils_profiling_middleware.py b/packages/service-library/src/servicelib/_utils_profiling_middleware.py new file mode 100644 index 00000000000..bf477300298 --- /dev/null +++ b/packages/service-library/src/servicelib/_utils_profiling_middleware.py @@ -0,0 +1,29 @@ +import json + +from servicelib.mimetype_constants import ( + MIMETYPE_APPLICATION_JSON, + MIMETYPE_APPLICATION_ND_JSON, +) + + +def append_profile(body: str, profile: str) -> str: + try: + json.loads(body) + body += "\n" if not body.endswith("\n") else "" + except json.decoder.JSONDecodeError: + pass + body += json.dumps({"profile": profile}) + return body + + +def check_response_headers( + response_headers: dict[bytes, bytes] +) -> list[tuple[bytes, bytes]]: + original_content_type: str = response_headers[b"content-type"].decode() + assert original_content_type in { + MIMETYPE_APPLICATION_ND_JSON.encode(), + MIMETYPE_APPLICATION_JSON.encode(), + } # nosec + headers: dict = {} + headers[b"content-type"] = MIMETYPE_APPLICATION_ND_JSON.encode() + return list(headers.items()) diff --git a/packages/service-library/src/servicelib/aiohttp/profiler_middleware.py b/packages/service-library/src/servicelib/aiohttp/profiler_middleware.py new file mode 100644 index 00000000000..cca9e8a0de1 --- /dev/null +++ b/packages/service-library/src/servicelib/aiohttp/profiler_middleware.py @@ -0,0 +1,41 @@ +from aiohttp.web import HTTPInternalServerError, Request, StreamResponse, middleware +from pyinstrument import Profiler +from servicelib.mimetype_constants import ( + MIMETYPE_APPLICATION_JSON, + MIMETYPE_APPLICATION_ND_JSON, +) + +from .._utils_profiling_middleware import append_profile + + +@middleware +async def profiling_middleware(request: Request, handler): + profiler: Profiler | None = None + if request.headers.get("x-profile") is not None: + profiler = Profiler(async_mode="enabled") + profiler.start() + + response = await handler(request) + + if profiler is None: + return response + if response.content_type != MIMETYPE_APPLICATION_JSON: + raise HTTPInternalServerError( + reason=f"Profiling middleware is not compatible with {response.content_type=}", + headers={}, + ) + + stream_response = StreamResponse( + status=response.status, + reason=response.reason, + headers=response.headers, + ) + stream_response.content_type = MIMETYPE_APPLICATION_ND_JSON + await stream_response.prepare(request) + await stream_response.write(response.body) + profiler.stop() + await stream_response.write( + append_profile("\n", profiler.output_text(unicode=True, color=True)).encode() + ) + await stream_response.write_eof() + return stream_response diff --git a/services/api-server/src/simcore_service_api_server/core/_profiler_middleware.py b/packages/service-library/src/servicelib/fastapi/profiler_middleware.py similarity index 66% rename from services/api-server/src/simcore_service_api_server/core/_profiler_middleware.py rename to packages/service-library/src/servicelib/fastapi/profiler_middleware.py index 3ec39d5ebcc..15eccd038e8 100644 --- a/services/api-server/src/simcore_service_api_server/core/_profiler_middleware.py +++ b/packages/service-library/src/servicelib/fastapi/profiler_middleware.py @@ -1,39 +1,17 @@ -import json from typing import Any from fastapi import FastAPI -from models_library.utils.json_serialization import json_dumps from pyinstrument import Profiler +from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON from starlette.requests import Request - -def _check_response_headers( - response_headers: dict[bytes, bytes] -) -> list[tuple[bytes, bytes]]: - original_content_type: str = response_headers[b"content-type"].decode() - assert original_content_type in { - "application/x-ndjson", - "application/json", - } # nosec - headers: dict = {} - headers[b"content-type"] = b"application/x-ndjson" - return list(headers.items()) - - -def _append_profile(body: str, profile: str) -> str: - try: - json.loads(body) - body += "\n" if not body.endswith("\n") else "" - except json.decoder.JSONDecodeError: - pass - body += json_dumps({"profile": profile}) - return body +from .._utils_profiling_middleware import append_profile, check_response_headers def is_last_response(response_headers: dict[bytes, bytes], message: dict[str, Any]): if ( content_type := response_headers.get(b"content-type") - ) and content_type == b"application/json": + ) and content_type == MIMETYPE_APPLICATION_JSON.encode(): return True if (more_body := message.get("more_body")) is not None: return not more_body @@ -41,7 +19,7 @@ def is_last_response(response_headers: dict[bytes, bytes], message: dict[str, An raise RuntimeError(msg) -class ApiServerProfilerMiddleware: +class ProfilerMiddleware: """Following https://www.starlette.io/middleware/#cleanup-and-error-handling https://www.starlette.io/middleware/#reusing-starlette-components @@ -50,7 +28,7 @@ class ApiServerProfilerMiddleware: def __init__(self, app: FastAPI): self._app: FastAPI = app - self._profile_header_trigger: str = "x-profile-api-server" + self._profile_header_trigger: str = "x-profile" async def __call__(self, scope, receive, send): if scope["type"] != "http": @@ -62,7 +40,7 @@ async def __call__(self, scope, receive, send): request_headers = dict(request.headers) response_headers: dict[bytes, bytes] = {} - if request_headers.get(self._profile_header_trigger) == "true": + if request_headers.get(self._profile_header_trigger) is not None: request_headers.pop(self._profile_header_trigger) scope["headers"] = [ (k.encode("utf8"), v.encode("utf8")) for k, v in request_headers.items() @@ -75,11 +53,11 @@ async def _send_wrapper(message): nonlocal response_headers if message["type"] == "http.response.start": response_headers = dict(message.get("headers")) - message["headers"] = _check_response_headers(response_headers) + message["headers"] = check_response_headers(response_headers) elif message["type"] == "http.response.body": if is_last_response(response_headers, message): profiler.stop() - message["body"] = _append_profile( + message["body"] = append_profile( message["body"].decode(), profiler.output_text(unicode=True, color=True), ).encode() diff --git a/packages/service-library/src/servicelib/mimetype_constants.py b/packages/service-library/src/servicelib/mimetype_constants.py index 718f72d6f17..ffde9a3f7b3 100644 --- a/packages/service-library/src/servicelib/mimetype_constants.py +++ b/packages/service-library/src/servicelib/mimetype_constants.py @@ -12,6 +12,7 @@ # NOTE: mimetypes (https://docs.python.org/3/library/mimetypes.html) is already a module in python MIMETYPE_APPLICATION_JSON: Final[str] = "application/json" +MIMETYPE_APPLICATION_ND_JSON: Final[str] = "application/x-ndjson" MIMETYPE_APPLICATION_ZIP: Final[str] = "application/zip" MIMETYPE_TEXT_HTML: Final[str] = "text/html" MIMETYPE_TEXT_PLAIN: Final[str] = "text/plain" diff --git a/services/api-server/src/simcore_service_api_server/core/application.py b/services/api-server/src/simcore_service_api_server/core/application.py index 66ea7448711..09259f0c88e 100644 --- a/services/api-server/src/simcore_service_api_server/core/application.py +++ b/services/api-server/src/simcore_service_api_server/core/application.py @@ -5,6 +5,7 @@ from fastapi_pagination import add_pagination from httpx import HTTPError as HttpxException from models_library.basic_types import BootModeEnum +from servicelib.fastapi.profiler_middleware import ProfilerMiddleware from servicelib.logging_utils import config_all_loggers from starlette import status from starlette.exceptions import HTTPException @@ -120,10 +121,8 @@ def init_app(settings: ApplicationSettings | None = None) -> FastAPI: add_oec_to_message=True, ), ) - if settings.API_SERVER_DEV_FEATURES_ENABLED: - from ._profiler_middleware import ApiServerProfilerMiddleware - - app.add_middleware(ApiServerProfilerMiddleware) + if settings.API_SERVER_PROFILING: + app.add_middleware(ProfilerMiddleware) if app.state.settings.API_SERVER_PROMETHEUS_INSTRUMENTATION_ENABLED: setup_prometheus_instrumentation(app) diff --git a/services/api-server/src/simcore_service_api_server/core/settings.py b/services/api-server/src/simcore_service_api_server/core/settings.py index b04116e1e00..9074833c298 100644 --- a/services/api-server/src/simcore_service_api_server/core/settings.py +++ b/services/api-server/src/simcore_service_api_server/core/settings.py @@ -88,6 +88,7 @@ class ApplicationSettings(BasicSettings): API_SERVER_HEALTH_CHECK_TASK_TIMEOUT_SECONDS: PositiveInt = 10 API_SERVER_ALLOWED_HEALTH_CHECK_FAILURES: PositiveInt = 5 API_SERVER_PROMETHEUS_INSTRUMENTATION_COLLECT_SECONDS: PositiveInt = 5 + API_SERVER_PROFILING: bool = False # DEV-TOOLS API_SERVER_DEV_HTTP_CALLS_LOGS_PATH: Path | None = Field( default=None, diff --git a/services/catalog/src/simcore_service_catalog/core/application.py b/services/catalog/src/simcore_service_catalog/core/application.py index 425d4579397..6a0ec7f882c 100644 --- a/services/catalog/src/simcore_service_catalog/core/application.py +++ b/services/catalog/src/simcore_service_catalog/core/application.py @@ -7,6 +7,7 @@ from fastapi.middleware.gzip import GZipMiddleware from models_library.basic_types import BootModeEnum from servicelib.fastapi.openapi import override_fastapi_openapi_method +from servicelib.fastapi.profiler_middleware import ProfilerMiddleware from servicelib.fastapi.prometheus_instrumentation import ( setup_prometheus_instrumentation, ) @@ -67,6 +68,9 @@ def init_app(settings: ApplicationSettings | None = None) -> FastAPI: if app.state.settings.CATALOG_PROMETHEUS_INSTRUMENTATION_ENABLED: setup_prometheus_instrumentation(app) + if app.state.settings.CATALOG_PROFILING: + app.add_middleware(ProfilerMiddleware) + # EVENTS async def _on_startup() -> None: print(APP_STARTED_BANNER_MSG, flush=True) diff --git a/services/catalog/src/simcore_service_catalog/core/settings.py b/services/catalog/src/simcore_service_catalog/core/settings.py index d034575aeb2..24902065c94 100644 --- a/services/catalog/src/simcore_service_catalog/core/settings.py +++ b/services/catalog/src/simcore_service_catalog/core/settings.py @@ -71,6 +71,8 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): CATALOG_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True + CATALOG_PROFILING: bool = False + # BACKGROUND TASK CATALOG_BACKGROUND_TASK_REST_TIME: PositiveInt = 60 CATALOG_BACKGROUND_TASK_WAIT_AFTER_FAILURE: PositiveInt = 5 # secs diff --git a/services/director-v2/src/simcore_service_director_v2/core/application.py b/services/director-v2/src/simcore_service_director_v2/core/application.py index 077d25f6f19..7969143c2c2 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/application.py +++ b/services/director-v2/src/simcore_service_director_v2/core/application.py @@ -6,6 +6,7 @@ get_common_oas_options, override_fastapi_openapi_method, ) +from servicelib.fastapi.profiler_middleware import ProfilerMiddleware from servicelib.fastapi.prometheus_instrumentation import ( setup_prometheus_instrumentation, ) @@ -185,6 +186,9 @@ def init_app(settings: AppSettings | None = None) -> FastAPI: if settings.DIRECTOR_V2_PROMETHEUS_INSTRUMENTATION_ENABLED: setup_prometheus_instrumentation(app) + if settings.DIRECTOR_V2_PROFILING: + app.add_middleware(ProfilerMiddleware) + # setup app -- app.add_event_handler("startup", on_startup) app.add_event_handler("shutdown", on_shutdown) diff --git a/services/director-v2/src/simcore_service_director_v2/core/settings.py b/services/director-v2/src/simcore_service_director_v2/core/settings.py index 9988a7c3290..58d36071410 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/settings.py +++ b/services/director-v2/src/simcore_service_director_v2/core/settings.py @@ -154,6 +154,7 @@ class AppSettings(BaseCustomSettings, MixinLoggingSettings): description="Filepath to self-signed osparc.crt file *as mounted inside the container*, empty strings disables it", ) DIRECTOR_V2_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True + DIRECTOR_V2_PROFILING: bool = False DIRECTOR_V2_REMOTE_DEBUGGING_PORT: PortInt | None diff --git a/services/docker-compose.devel.yml b/services/docker-compose.devel.yml index d6e40f80fa0..014b554f152 100644 --- a/services/docker-compose.devel.yml +++ b/services/docker-compose.devel.yml @@ -11,6 +11,7 @@ services: - SC_BOOT_MODE=debug - LOG_LEVEL=DEBUG - DEBUG=true + - API_SERVER_PROFILING=${API_SERVER_PROFILING} volumes: - ./api-server:/devel/services/api-server - ../packages:/devel/packages @@ -47,6 +48,7 @@ services: - SC_BOOT_MODE=debug - DYNAMIC_SCHEDULER_LOGLEVEL=DEBUG - DEBUG=true + - DYNAMIC_SCHEDULER_PROFILING=${DYNAMIC_SCHEDULER_PROFILING} volumes: - ./dynamic-scheduler:/devel/services/dynamic-scheduler - ../packages:/devel/packages @@ -59,6 +61,7 @@ services: - LOG_LEVEL=DEBUG - DEBUG=true - DYNAMIC_SIDECAR_MOUNT_PATH_DEV=${PWD}/services/dynamic-sidecar + - CATALOG_PROFILING=${CATALOG_PROFILING} volumes: - ./catalog:/devel/services/catalog - ../packages:/devel/packages @@ -95,6 +98,7 @@ services: - LOG_LEVEL=debug - DEBUG=true - DYNAMIC_SIDECAR_MOUNT_PATH_DEV=${PWD}/services/dynamic-sidecar + - DIRECTOR_V2_PROFILING=${DIRECTOR_V2_PROFILING} volumes: - ./director-v2:/devel/services/director-v2 - ../packages:/devel/packages @@ -115,6 +119,8 @@ services: WEBSERVER_REMOTE_DEBUGGING_PORT: 3000 SC_BOOT_MODE: debug WEBSERVER_LOGLEVEL: ${LOG_LEVEL:-DEBUG} + WEBSERVER_PROFILING: ${WEBSERVER_PROFILING} + wb-db-event-listener: volumes: *webserver-volumes-dev @@ -186,6 +192,7 @@ services: environment: - SC_BOOT_MODE=debug - STORAGE_LOGLEVEL=DEBUG + - STORAGE_PROFILING=${STORAGE_PROFILING} agent: environment: diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 352c44942e8..a13c537c9ea 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -16,6 +16,7 @@ services: - API_SERVER_DEV_FEATURES_ENABLED=${API_SERVER_DEV_FEATURES_ENABLED} - API_SERVER_DEV_HTTP_CALLS_LOGS_PATH=${API_SERVER_DEV_HTTP_CALLS_LOGS_PATH} - API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED=${LOG_FORMAT_LOCAL_DEV_ENABLED} + - API_SERVER_PROFILING=${API_SERVER_PROFILING} - CATALOG_HOST=${CATALOG_HOST} - CATALOG_PORT=${CATALOG_PORT} @@ -133,6 +134,7 @@ services: - CATALOG_DEV_FEATURES_ENABLED=${CATALOG_DEV_FEATURES_ENABLED} - CATALOG_SERVICES_DEFAULT_RESOURCES=${CATALOG_SERVICES_DEFAULT_RESOURCES} - CATALOG_SERVICES_DEFAULT_SPECIFICATIONS=${CATALOG_SERVICES_DEFAULT_SPECIFICATIONS} + - CATALOG_PROFILING=${CATALOG_PROFILING} - DIRECTOR_HOST=${DIRECTOR_HOST:-director} - DIRECTOR_PORT=${DIRECTOR_PORT:-8080} - LOG_LEVEL=${LOG_LEVEL:-WARNING} @@ -271,6 +273,7 @@ services: - DIRECTOR_V2_DYNAMIC_SCHEDULER_CLOSE_SERVICES_VIA_FRONTEND_WHEN_CREDITS_LIMIT_REACHED=${DIRECTOR_V2_DYNAMIC_SCHEDULER_CLOSE_SERVICES_VIA_FRONTEND_WHEN_CREDITS_LIMIT_REACHED} - DIRECTOR_V2_PUBLIC_API_BASE_URL=${DIRECTOR_V2_PUBLIC_API_BASE_URL} - DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS=${DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS} + - DIRECTOR_V2_PROFILING=${DIRECTOR_V2_PROFILING} - DYNAMIC_SIDECAR_ENDPOINT_SPECS_MODE_DNSRR_ENABLED=${DYNAMIC_SIDECAR_ENDPOINT_SPECS_MODE_DNSRR_ENABLED} - DYNAMIC_SIDECAR_ENABLE_VOLUME_LIMITS=${DYNAMIC_SIDECAR_ENABLE_VOLUME_LIMITS} @@ -442,6 +445,7 @@ services: - DIRECTOR_V2_HOST=${DIRECTOR_V2_HOST} - DIRECTOR_V2_PORT=${DIRECTOR_V2_PORT} - DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=${DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT} + - DYNAMIC_SCHEDULER_PROFILING=${DYNAMIC_SCHEDULER_PROFILING} static-webserver: image: ${DOCKER_REGISTRY:-itisfoundation}/static-webserver:${DOCKER_IMAGE_TAG:-latest} @@ -507,6 +511,7 @@ services: WEBSERVER_DEV_FEATURES_ENABLED: ${WEBSERVER_DEV_FEATURES_ENABLED} WEBSERVER_LOGLEVEL: ${WEBSERVER_LOGLEVEL} + WEBSERVER_PROFILING: ${WEBSERVER_PROFILING} WEBSERVER_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} @@ -968,6 +973,7 @@ services: - STORAGE_LOGLEVEL=${LOG_LEVEL:-WARNING} - STORAGE_MONITORING_ENABLED=1 - TRACING_ZIPKIN_ENDPOINT=${TRACING_ZIPKIN_ENDPOINT:-http://jaeger:9411} + - STORAGE_PROFILING=${STORAGE_PROFILING} networks: - default - interactive_services_subnet diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/application.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/application.py index a94a3014cff..62f07ea31fc 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/application.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/application.py @@ -1,5 +1,6 @@ from fastapi import FastAPI from servicelib.fastapi.openapi import override_fastapi_openapi_method +from servicelib.fastapi.profiler_middleware import ProfilerMiddleware from servicelib.fastapi.prometheus_instrumentation import ( setup_prometheus_instrumentation, ) @@ -42,6 +43,9 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: if app.state.settings.DYNAMIC_SCHEDULER_PROMETHEUS_INSTRUMENTATION_ENABLED: setup_prometheus_instrumentation(app) + if app.state.settings.DYNAMIC_SCHEDULER_PROFILING: + app.add_middleware(ProfilerMiddleware) + # PLUGINS SETUP setup_director_v2(app) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py index 4ee31f77619..15c6dd7a9e4 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py @@ -78,3 +78,5 @@ class ApplicationSettings(_BaseApplicationSettings): ) DYNAMIC_SCHEDULER_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True + + DYNAMIC_SCHEDULER_PROFILING: bool = False diff --git a/services/storage/src/simcore_service_storage/application.py b/services/storage/src/simcore_service_storage/application.py index ae1207fb07d..3cda74cd3ab 100644 --- a/services/storage/src/simcore_service_storage/application.py +++ b/services/storage/src/simcore_service_storage/application.py @@ -10,6 +10,7 @@ from servicelib.aiohttp.application import APP_CONFIG_KEY, create_safe_application from servicelib.aiohttp.dev_error_logger import setup_dev_error_logger from servicelib.aiohttp.monitoring import setup_monitoring +from servicelib.aiohttp.profiler_middleware import profiling_middleware from servicelib.aiohttp.tracing import setup_tracing from ._meta import APP_NAME, APP_STARTED_BANNER_MSG, VERSION @@ -72,6 +73,10 @@ def create(settings: Settings) -> web.Application: app.middlewares.append(dsm_exception_handler) + if settings.STORAGE_PROFILING: + + app.middlewares.append(profiling_middleware) + if settings.LOG_LEVEL == "DEBUG": setup_dev_error_logger(app) diff --git a/services/storage/src/simcore_service_storage/settings.py b/services/storage/src/simcore_service_storage/settings.py index db9db7984ce..5d68069082a 100644 --- a/services/storage/src/simcore_service_storage/settings.py +++ b/services/storage/src/simcore_service_storage/settings.py @@ -26,6 +26,7 @@ class Settings(BaseCustomSettings, MixinLoggingSettings): ) STORAGE_MONITORING_ENABLED: bool = False + STORAGE_PROFILING: bool = False BF_API_KEY: str | None = Field( None, description="Pennsieve API key ONLY for testing purposes" diff --git a/services/web/server/src/simcore_service_webserver/application.py b/services/web/server/src/simcore_service_webserver/application.py index 60e5475844e..8b2d9c83bcc 100644 --- a/services/web/server/src/simcore_service_webserver/application.py +++ b/services/web/server/src/simcore_service_webserver/application.py @@ -17,7 +17,7 @@ from .clusters.plugin import setup_clusters from .db.plugin import setup_db from .db_listener.plugin import setup_db_listener -from .diagnostics.plugin import setup_diagnostics +from .diagnostics.plugin import setup_diagnostics, setup_profiling_middleware from .director_v2.plugin import setup_director_v2 from .dynamic_scheduler.plugin import setup_dynamic_scheduler from .email.plugin import setup_email @@ -107,6 +107,7 @@ def create_application() -> web.Application: setup_notifications(app) setup_socketio(app) setup_db_listener(app) + setup_profiling_middleware(app) # login setup_email(app) diff --git a/services/web/server/src/simcore_service_webserver/application_settings.py b/services/web/server/src/simcore_service_webserver/application_settings.py index a0ab44e18e4..49e78f6a518 100644 --- a/services/web/server/src/simcore_service_webserver/application_settings.py +++ b/services/web/server/src/simcore_service_webserver/application_settings.py @@ -231,6 +231,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): WEBSERVER_TAGS: bool = True WEBSERVER_VERSION_CONTROL: bool = True WEBSERVER_WALLETS: bool = True + WEBSERVER_PROFILING: bool = False # WEBSERVER_SECURITY: bool = Field( diff --git a/services/web/server/src/simcore_service_webserver/diagnostics/plugin.py b/services/web/server/src/simcore_service_webserver/diagnostics/plugin.py index 8252d43dc8b..d457c92943e 100644 --- a/services/web/server/src/simcore_service_webserver/diagnostics/plugin.py +++ b/services/web/server/src/simcore_service_webserver/diagnostics/plugin.py @@ -5,6 +5,8 @@ from aiohttp import web from servicelib.aiohttp import monitor_slow_callbacks from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup +from servicelib.aiohttp.profiler_middleware import profiling_middleware +from simcore_service_webserver.application_settings import get_application_settings from ..rest.healthcheck import HealthCheck from ..rest.plugin import setup_rest @@ -56,3 +58,10 @@ async def _on_healthcheck_async_adapter(app: web.Application) -> None: app.router.add_routes(_handlers.routes) app[HEALTH_PLUGIN_START_TIME] = time.time() + + +def setup_profiling_middleware( + app: web.Application, +) -> None: + if get_application_settings(app).WEBSERVER_PROFILING: + app.middlewares.append(profiling_middleware)