diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/6e91067932f2_adding_resource_tracker_container_table.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/6e91067932f2_adding_resource_tracker_container_table.py new file mode 100644 index 00000000000..29db67d4af6 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/6e91067932f2_adding_resource_tracker_container_table.py @@ -0,0 +1,57 @@ +"""adding resource tracker container table + +Revision ID: 6e91067932f2 +Revises: 52cf00912ad9 +Create Date: 2023-06-21 14:12:40.292816+00:00 + +""" +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "6e91067932f2" +down_revision = "52cf00912ad9" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "resource_tracker_container", + sa.Column("container_id", sa.String(), nullable=False), + sa.Column("image", sa.String(), nullable=False), + sa.Column("user_id", sa.BigInteger(), nullable=False), + sa.Column("product_name", sa.String(), nullable=False), + sa.Column( + "service_settings_reservation_nano_cpus", sa.BigInteger(), nullable=True + ), + sa.Column( + "service_settings_reservation_memory_bytes", sa.BigInteger(), nullable=True + ), + sa.Column( + "service_settings_reservation_additional_info", + postgresql.JSONB(astext_type=sa.Text()), + nullable=False, + ), + sa.Column("container_cpu_usage_seconds_total", sa.Float(), nullable=False), + sa.Column("prometheus_created", sa.DateTime(timezone=True), nullable=False), + sa.Column( + "prometheus_last_scraped", sa.DateTime(timezone=True), nullable=False + ), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.PrimaryKeyConstraint("container_id", name="resource_tracker_container_pkey"), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("resource_tracker_container") + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker.py b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker.py new file mode 100644 index 00000000000..3fbb2d5a75b --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker.py @@ -0,0 +1,74 @@ +""" resource_tracker_container table + + - Table where we store the resource usage of each container that + we scrape via resource-usage-tracker service +""" + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import JSONB + +from ._common import column_modified_datetime +from .base import metadata + +resource_tracker_container = sa.Table( + "resource_tracker_container", + metadata, + sa.Column( + "container_id", + sa.String, + nullable=False, + doc="Refers to container id scraped via Prometheus", + ), + sa.Column( + "image", + sa.String, + nullable=False, + doc="image label scraped via Prometheus (taken from container labels), ex. registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.9", + ), + sa.Column( + "user_id", + sa.BigInteger, + nullable=False, + doc="user_id label scraped via Prometheus (taken from container labels)", + ), + sa.Column( + "product_name", + sa.String, + nullable=False, + doc="product_name label scraped via Prometheus (taken from container labels)", + ), + sa.Column( + "service_settings_reservation_nano_cpus", + sa.BigInteger, + nullable=True, + doc="CPU resource allocated to a container, ex.500000000 means that the container is allocated 0.5 CPU shares", + ), + sa.Column( + "service_settings_reservation_memory_bytes", + sa.BigInteger, + nullable=True, + doc="memory limit in bytes scraped via Prometheus", + ), + sa.Column( + "service_settings_reservation_additional_info", + JSONB, + nullable=False, + doc="storing additional information about the reservation settings", + ), + sa.Column("container_cpu_usage_seconds_total", sa.Float, nullable=False), + sa.Column( + "prometheus_created", + sa.DateTime(timezone=True), + nullable=False, + doc="First container creation timestamp (UTC timestamp)", + ), + sa.Column( + "prometheus_last_scraped", + sa.DateTime(timezone=True), + nullable=False, + doc="Last prometheus scraped timestamp (UTC timestamp)", + ), + column_modified_datetime(timezone=True), + # --------------------------- + sa.PrimaryKeyConstraint("container_id", name="resource_tracker_container_pkey"), +) diff --git a/packages/service-library/src/servicelib/db_async_engine.py b/packages/service-library/src/servicelib/db_async_engine.py new file mode 100644 index 00000000000..2d8351dc7a8 --- /dev/null +++ b/packages/service-library/src/servicelib/db_async_engine.py @@ -0,0 +1,58 @@ +import logging + +from fastapi import FastAPI +from settings_library.postgres import PostgresSettings +from simcore_postgres_database.utils_aiosqlalchemy import ( + get_pg_engine_stateinfo, + raise_if_migration_not_ready, +) +from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine +from tenacity import retry + +from .retry_policies import PostgresRetryPolicyUponInitialization + +logger = logging.getLogger(__name__) + + +@retry(**PostgresRetryPolicyUponInitialization(logger).kwargs) +async def connect_to_db(app: FastAPI, cfg: PostgresSettings) -> None: + logger.debug("Connecting db ...") + + engine: AsyncEngine = create_async_engine( + cfg.dsn_with_async_sqlalchemy, + pool_size=cfg.POSTGRES_MINSIZE, + max_overflow=cfg.POSTGRES_MAXSIZE - cfg.POSTGRES_MINSIZE, + connect_args={ + "server_settings": {"application_name": cfg.POSTGRES_CLIENT_NAME} + }, + pool_pre_ping=True, # https://docs.sqlalchemy.org/en/14/core/pooling.html#dealing-with-disconnects + future=True, # this uses sqlalchemy 2.0 API, shall be removed when sqlalchemy 2.0 is released + ) + + logger.debug("Connected to %s", engine.url) # pylint: disable=no-member + + logger.debug("Checking db migration...") + try: + await raise_if_migration_not_ready(engine) + except Exception: + # NOTE: engine must be closed because retry will create a new engine + await engine.dispose() + raise + + logger.debug("Migration up-to-date") + + app.state.engine = engine + + logger.debug( + "Setup engine: %s", + await get_pg_engine_stateinfo(engine), + ) + + +async def close_db_connection(app: FastAPI) -> None: + logger.debug("Disconnecting db ...") + + if engine := app.state.engine: + await engine.dispose() + + logger.debug("Disconnected from %s", engine.url) # pylint: disable=no-member diff --git a/services/catalog/src/simcore_service_catalog/core/events.py b/services/catalog/src/simcore_service_catalog/core/events.py index 24679d26900..74f3a2f8b4b 100644 --- a/services/catalog/src/simcore_service_catalog/core/events.py +++ b/services/catalog/src/simcore_service_catalog/core/events.py @@ -3,8 +3,9 @@ from fastapi import FastAPI from models_library.basic_types import BootModeEnum +from servicelib.db_async_engine import close_db_connection, connect_to_db -from ..db.events import close_db_connection, connect_to_db, setup_default_product +from ..db.events import setup_default_product from ..services.director import close_director, setup_director from ..services.remote_debug import setup_remote_debugging from .background_tasks import start_registry_sync_task, stop_registry_sync_task @@ -23,7 +24,7 @@ async def start_app() -> None: # setup connection to pg db if app.state.settings.CATALOG_POSTGRES: - await connect_to_db(app) + await connect_to_db(app, app.state.settings.CATALOG_POSTGRES) await setup_default_product(app) if app.state.settings.CATALOG_DIRECTOR: diff --git a/services/catalog/src/simcore_service_catalog/db/events.py b/services/catalog/src/simcore_service_catalog/db/events.py index 3e7e075c151..c917ade7240 100644 --- a/services/catalog/src/simcore_service_catalog/db/events.py +++ b/services/catalog/src/simcore_service_catalog/db/events.py @@ -1,65 +1,12 @@ import logging from fastapi import FastAPI -from servicelib.retry_policies import PostgresRetryPolicyUponInitialization -from settings_library.postgres import PostgresSettings -from simcore_postgres_database.utils_aiosqlalchemy import ( - get_pg_engine_stateinfo, - raise_if_migration_not_ready, -) -from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine -from tenacity import retry from .repositories.products import ProductsRepository logger = logging.getLogger(__name__) -@retry(**PostgresRetryPolicyUponInitialization(logger).kwargs) -async def connect_to_db(app: FastAPI) -> None: - logger.debug("Connecting db ...") - cfg: PostgresSettings = app.state.settings.CATALOG_POSTGRES - - engine: AsyncEngine = create_async_engine( - cfg.dsn_with_async_sqlalchemy, - pool_size=cfg.POSTGRES_MINSIZE, - max_overflow=cfg.POSTGRES_MAXSIZE - cfg.POSTGRES_MINSIZE, - connect_args={ - "server_settings": {"application_name": cfg.POSTGRES_CLIENT_NAME} - }, - pool_pre_ping=True, # https://docs.sqlalchemy.org/en/14/core/pooling.html#dealing-with-disconnects - future=True, # this uses sqlalchemy 2.0 API, shall be removed when sqlalchemy 2.0 is released - ) - - logger.debug("Connected to %s", engine.url) # pylint: disable=no-member - - logger.debug("Checking db migration...") - try: - await raise_if_migration_not_ready(engine) - except Exception: - # NOTE: engine must be closed because retry will create a new engine - await engine.dispose() - raise - - logger.debug("Migration up-to-date") - - app.state.engine = engine - - logger.debug( - "Setup engine: %s", - await get_pg_engine_stateinfo(engine), - ) - - -async def close_db_connection(app: FastAPI) -> None: - logger.debug("Disconnecting db ...") - - if engine := app.state.engine: - await engine.dispose() - - logger.debug("Disconnected from %s", engine.url) # pylint: disable=no-member - - async def setup_default_product(app: FastAPI): repo = ProductsRepository(db_engine=app.state.engine) app.state.default_product_name = await repo.get_default_product_name() diff --git a/services/docker-compose.devel.yml b/services/docker-compose.devel.yml index 600d670960f..8df034fb255 100644 --- a/services/docker-compose.devel.yml +++ b/services/docker-compose.devel.yml @@ -124,6 +124,16 @@ services: - SC_BOOT_MODE=debug-ptvsd - RESOURCE_USAGE_TRACKER_LOGLEVEL=DEBUG - DEBUG=true + - LOG_FORMAT_LOCAL_DEV_ENABLED=${LOG_FORMAT_LOCAL_DEV_ENABLED} + - POSTGRES_DB=${POSTGRES_DB} + - POSTGRES_ENDPOINT=${POSTGRES_ENDPOINT} + - POSTGRES_HOST=${POSTGRES_HOST} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_PORT=${POSTGRES_PORT} + - POSTGRES_USER=${POSTGRES_USER} + - PROMETHEUS_URL=${RESOURCE_USAGE_TRACKER_PROMETHEUS_URL} + - PROMETHEUS_USERNAME=${RESOURCE_USAGE_TRACKER_PROMETHEUS_USERNAME} + - PROMETHEUS_PASSWORD=${RESOURCE_USAGE_TRACKER_PROMETHEUS_PASSWORD} volumes: - ./resource-usage-tracker:/devel/services/resource-usage-tracker - ../packages:/devel/packages diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 94e209a2e2e..b8db5a62832 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -194,9 +194,9 @@ services: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - POSTGRES_PORT=${POSTGRES_PORT} - POSTGRES_USER=${POSTGRES_USER} - - PROMETHEUS_URL=${PROMETHEUS_URL} - - PROMETHEUS_USERNAME=${PROMETHEUS_USERNAME} - - PROMETHEUS_PASSWORD=${PROMETHEUS_PASSWORD} + - PROMETHEUS_URL=${RESOURCE_USAGE_TRACKER_PROMETHEUS_URL} + - PROMETHEUS_USERNAME=${RESOURCE_USAGE_TRACKER_PROMETHEUS_USERNAME} + - PROMETHEUS_PASSWORD=${RESOURCE_USAGE_TRACKER_PROMETHEUS_PASSWORD} - RESOURCE_USAGE_TRACKER_LOGLEVEL=${LOG_LEVEL:-INFO} static-webserver: diff --git a/services/resource-usage-tracker/.env-devel b/services/resource-usage-tracker/.env-devel new file mode 100644 index 00000000000..86d332191aa --- /dev/null +++ b/services/resource-usage-tracker/.env-devel @@ -0,0 +1,8 @@ +RESOURCE_USAGE_TRACKER_DEV_FEATURES_ENABLED=1 + +LOG_LEVEL=DEBUG + +POSTGRES_USER=test +POSTGRES_PASSWORD=test +POSTGRES_DB=test +POSTGRES_HOST=localhost diff --git a/services/resource-usage-tracker/requirements/_base.in b/services/resource-usage-tracker/requirements/_base.in index c4709b81f8a..79ec5217795 100644 --- a/services/resource-usage-tracker/requirements/_base.in +++ b/services/resource-usage-tracker/requirements/_base.in @@ -8,6 +8,7 @@ # intra-repo required dependencies --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in +--requirement ../../../packages/postgres-database/requirements/_base.in # service-library[fastapi] --requirement ../../../packages/service-library/requirements/_base.in --requirement ../../../packages/service-library/requirements/_fastapi.in diff --git a/services/resource-usage-tracker/requirements/_base.txt b/services/resource-usage-tracker/requirements/_base.txt index bfd64ed7253..cb0f43c5aaf 100644 --- a/services/resource-usage-tracker/requirements/_base.txt +++ b/services/resource-usage-tracker/requirements/_base.txt @@ -2,32 +2,39 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --output-file=requirements/_base.txt --resolver=backtracking --strip-extras requirements/_base.in +# pip-compile _base.in # aio-pika==9.1.2 # via - # -c requirements/../../../packages/service-library/requirements/./_base.in - # -r requirements/../../../packages/service-library/requirements/_base.in + # -c ../../../packages/service-library/requirements/./_base.in + # -r ../../../packages/service-library/requirements/_base.in aiodebug==2.3.0 # via - # -c requirements/../../../packages/service-library/requirements/./_base.in - # -r requirements/../../../packages/service-library/requirements/_base.in + # -c ../../../packages/service-library/requirements/./_base.in + # -r ../../../packages/service-library/requirements/_base.in aiodocker==0.21.0 # via - # -c requirements/../../../packages/service-library/requirements/./_base.in - # -r requirements/../../../packages/service-library/requirements/_base.in + # -c ../../../packages/service-library/requirements/./_base.in + # -r ../../../packages/service-library/requirements/_base.in aiofiles==23.1.0 # via - # -c requirements/../../../packages/service-library/requirements/./_base.in - # -r requirements/../../../packages/service-library/requirements/_base.in + # -c ../../../packages/service-library/requirements/./_base.in + # -r ../../../packages/service-library/requirements/_base.in aiohttp==3.8.4 # via - # -c requirements/../../../packages/service-library/requirements/./../../../requirements/constraints.txt + # -c ../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c ../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c ../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c ../../../packages/service-library/requirements/./../../../requirements/constraints.txt + # -c ../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c ../../../requirements/constraints.txt # aiodocker aiormq==6.7.6 # via aio-pika aiosignal==1.3.1 # via aiohttp +alembic==1.11.1 + # via -r ../../../packages/postgres-database/requirements/_base.in anyio==3.6.2 # via # httpcore @@ -35,15 +42,19 @@ anyio==3.6.2 # watchfiles arrow==1.2.3 # via - # -c requirements/../../../packages/service-library/requirements/./_base.in - # -r requirements/../../../packages/models-library/requirements/_base.in + # -c ../../../packages/service-library/requirements/./_base.in + # -r ../../../packages/models-library/requirements/_base.in + # -r ../../../packages/service-library/requirements/_base.in async-timeout==4.0.2 # via # aiohttp # redis +asyncpg==0.27.0 + # via sqlalchemy attrs==21.4.0 # via - # -c requirements/../../../packages/service-library/requirements/././constraints.txt + # -c ../../../packages/service-library/requirements/././constraints.txt + # -c ../../../packages/service-library/requirements/./constraints.txt # aiohttp # jsonschema certifi==2023.5.7 @@ -73,14 +84,16 @@ email-validator==2.0.0.post2 # via pydantic fastapi==0.96.0 # via - # -r requirements/../../../packages/service-library/requirements/_fastapi.in - # -r requirements/_base.in + # -r ../../../packages/service-library/requirements/_fastapi.in + # -r _base.in fonttools==4.39.4 # via matplotlib frozenlist==1.3.3 # via # aiohttp # aiosignal +greenlet==2.0.2 + # via sqlalchemy h11==0.14.0 # via # httpcore @@ -92,7 +105,14 @@ httpcore==0.17.1 httptools==0.5.0 # via uvicorn httpx==0.24.1 - # via -r requirements/../../../packages/service-library/requirements/_fastapi.in + # via + # -c ../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c ../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c ../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c ../../../packages/service-library/requirements/./../../../requirements/constraints.txt + # -c ../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c ../../../requirements/constraints.txt + # -r ../../../packages/service-library/requirements/_fastapi.in idna==3.4 # via # anyio @@ -102,12 +122,24 @@ idna==3.4 # yarl jsonschema==3.2.0 # via - # -c requirements/../../../packages/service-library/requirements/././constraints.txt - # -r requirements/../../../packages/models-library/requirements/_base.in + # -c ../../../packages/service-library/requirements/././constraints.txt + # -c ../../../packages/service-library/requirements/./constraints.txt + # -r ../../../packages/models-library/requirements/_base.in kiwisolver==1.4.4 # via matplotlib +mako==1.2.4 + # via + # -c ../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c ../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c ../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c ../../../packages/service-library/requirements/./../../../requirements/constraints.txt + # -c ../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c ../../../requirements/constraints.txt + # alembic markdown-it-py==2.2.0 # via rich +markupsafe==2.1.3 + # via mako matplotlib==3.7.1 # via prometheus-api-client mdurl==0.1.2 @@ -124,7 +156,7 @@ numpy==1.24.3 # prometheus-api-client packaging==23.1 # via - # -r requirements/_base.in + # -r _base.in # matplotlib pamqp==3.2.1 # via aiormq @@ -133,18 +165,28 @@ pandas==2.0.1 pillow==9.5.0 # via matplotlib prometheus-api-client==0.5.3 - # via -r requirements/_base.in -pydantic==1.10.7 - # via - # -c requirements/../../../packages/service-library/requirements/./_base.in - # -r requirements/../../../packages/models-library/requirements/_base.in + # via -r _base.in +psycopg2-binary==2.9.6 + # via sqlalchemy +pydantic[email]==1.10.7 + # via + # -c ../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c ../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c ../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c ../../../packages/service-library/requirements/./../../../requirements/constraints.txt + # -c ../../../packages/service-library/requirements/./_base.in + # -c ../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c ../../../requirements/constraints.txt + # -r ../../../packages/models-library/requirements/_base.in + # -r ../../../packages/service-library/requirements/_base.in + # -r ../../../packages/settings-library/requirements/_base.in # fastapi pygments==2.15.1 # via rich pyinstrument==4.4.0 # via - # -c requirements/../../../packages/service-library/requirements/./_base.in - # -r requirements/../../../packages/service-library/requirements/_base.in + # -c ../../../packages/service-library/requirements/./_base.in + # -r ../../../packages/service-library/requirements/_base.in pyparsing==3.0.9 # via matplotlib pyrsistent==0.19.3 @@ -163,13 +205,27 @@ pytz==2023.3 # pandas pyyaml==5.4.1 # via - # -c requirements/../../../packages/service-library/requirements/./_base.in - # -r requirements/../../../packages/service-library/requirements/_base.in + # -c ../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c ../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c ../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c ../../../packages/service-library/requirements/./../../../requirements/constraints.txt + # -c ../../../packages/service-library/requirements/././constraints.txt + # -c ../../../packages/service-library/requirements/./_base.in + # -c ../../../packages/service-library/requirements/./constraints.txt + # -c ../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c ../../../requirements/constraints.txt + # -r ../../../packages/service-library/requirements/_base.in # uvicorn redis==4.5.5 # via - # -c requirements/../../../packages/service-library/requirements/./_base.in - # -r requirements/../../../packages/service-library/requirements/_base.in + # -c ../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c ../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c ../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c ../../../packages/service-library/requirements/./../../../requirements/constraints.txt + # -c ../../../packages/service-library/requirements/./_base.in + # -c ../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c ../../../requirements/constraints.txt + # -r ../../../packages/service-library/requirements/_base.in regex==2023.5.5 # via dateparser requests==2.30.0 @@ -177,7 +233,9 @@ requests==2.30.0 # httmock # prometheus-api-client rich==13.3.5 - # via typer + # via + # -r ../../../packages/settings-library/requirements/_base.in + # typer shellingham==1.5.0.post1 # via typer six==1.16.0 @@ -189,26 +247,42 @@ sniffio==1.3.0 # anyio # httpcore # httpx +sqlalchemy[postgresql_asyncpg,postgresql_psycopg2binary]==1.4.48 + # via + # -c ../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c ../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c ../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c ../../../packages/service-library/requirements/./../../../requirements/constraints.txt + # -c ../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c ../../../requirements/constraints.txt + # -r ../../../packages/postgres-database/requirements/_base.in + # alembic starlette==0.27.0 # via - # -c requirements/../../../packages/service-library/requirements/./../../../requirements/constraints.txt + # -c ../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c ../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c ../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c ../../../packages/service-library/requirements/./../../../requirements/constraints.txt + # -c ../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c ../../../requirements/constraints.txt # fastapi tenacity==8.2.2 # via - # -c requirements/../../../packages/service-library/requirements/./_base.in - # -r requirements/../../../packages/service-library/requirements/_base.in + # -c ../../../packages/service-library/requirements/./_base.in + # -r ../../../packages/service-library/requirements/_base.in tqdm==4.65.0 # via - # -c requirements/../../../packages/service-library/requirements/./_base.in - # -r requirements/../../../packages/service-library/requirements/_base.in -typer==0.9.0 + # -c ../../../packages/service-library/requirements/./_base.in + # -r ../../../packages/service-library/requirements/_base.in +typer[all]==0.9.0 # via - # -r requirements/../../../packages/settings-library/requirements/_base.in - # -r requirements/_base.in + # -r ../../../packages/settings-library/requirements/_base.in + # -r _base.in typing-extensions==4.5.0 # via # aiodebug # aiodocker + # alembic # pydantic # typer tzdata==2023.3 @@ -217,12 +291,17 @@ tzlocal==5.0.1 # via dateparser urllib3==2.0.2 # via - # -c requirements/../../../packages/service-library/requirements/./../../../requirements/constraints.txt + # -c ../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c ../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c ../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c ../../../packages/service-library/requirements/./../../../requirements/constraints.txt + # -c ../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c ../../../requirements/constraints.txt # requests -uvicorn==0.22.0 +uvicorn[standard]==0.22.0 # via - # -r requirements/../../../packages/service-library/requirements/_fastapi.in - # -r requirements/_base.in + # -r ../../../packages/service-library/requirements/_fastapi.in + # -r _base.in uvloop==0.17.0 # via uvicorn watchfiles==0.19.0 @@ -231,6 +310,7 @@ websockets==11.0.3 # via uvicorn yarl==1.9.2 # via + # -r ../../../packages/postgres-database/requirements/_base.in # aio-pika # aiohttp # aiormq diff --git a/services/resource-usage-tracker/requirements/_test.in b/services/resource-usage-tracker/requirements/_test.in index 533d615c682..2050838d9ac 100644 --- a/services/resource-usage-tracker/requirements/_test.in +++ b/services/resource-usage-tracker/requirements/_test.in @@ -10,8 +10,10 @@ # --constraint _base.txt +alembic # migration due to pytest_simcore.postgres_service asgi-lifespan coverage +docker faker httpx pytest @@ -23,3 +25,4 @@ pytest-sugar python-dotenv requests-mock respx +sqlalchemy[mypy] # adds Mypy / Pep-484 Support for ORM Mappings SEE https://docs.sqlalchemy.org/en/20/orm/extensions/mypy.html diff --git a/services/resource-usage-tracker/requirements/_test.txt b/services/resource-usage-tracker/requirements/_test.txt index 4c5c187ae7f..02b483d3ebb 100644 --- a/services/resource-usage-tracker/requirements/_test.txt +++ b/services/resource-usage-tracker/requirements/_test.txt @@ -2,109 +2,161 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --output-file=requirements/_test.txt --resolver=backtracking --strip-extras requirements/_test.in +# pip-compile _test.in # +alembic==1.11.1 + # via + # -c _base.txt + # -r _test.in anyio==3.6.2 # via - # -c requirements/_base.txt + # -c _base.txt # httpcore asgi-lifespan==2.1.0 - # via -r requirements/_test.in + # via -r _test.in certifi==2023.5.7 # via - # -c requirements/_base.txt + # -c _base.txt # httpcore # httpx # requests charset-normalizer==3.1.0 # via - # -c requirements/_base.txt + # -c _base.txt # requests -coverage==7.2.7 +coverage[toml]==7.2.7 # via - # -r requirements/_test.in + # -r _test.in # pytest-cov +docker==6.1.3 + # via -r _test.in exceptiongroup==1.1.1 # via pytest faker==18.10.1 - # via -r requirements/_test.in + # via -r _test.in +greenlet==2.0.2 + # via + # -c _base.txt + # sqlalchemy h11==0.14.0 # via - # -c requirements/_base.txt + # -c _base.txt # httpcore httpcore==0.17.1 # via - # -c requirements/_base.txt + # -c _base.txt # httpx httpx==0.24.1 # via - # -r requirements/_test.in + # -c ../../../requirements/constraints.txt + # -c _base.txt + # -r _test.in # respx idna==3.4 # via - # -c requirements/_base.txt + # -c _base.txt # anyio # httpx # requests iniconfig==2.0.0 # via pytest +mako==1.2.4 + # via + # -c ../../../requirements/constraints.txt + # -c _base.txt + # alembic +markupsafe==2.1.3 + # via + # -c _base.txt + # mako +mypy==1.3.0 + # via sqlalchemy +mypy-extensions==1.0.0 + # via mypy packaging==23.1 # via - # -c requirements/_base.txt + # -c _base.txt + # docker # pytest # pytest-sugar pluggy==1.0.0 # via pytest +psycopg2-binary==2.9.6 + # via + # -c _base.txt + # sqlalchemy pytest==7.3.1 # via - # -r requirements/_test.in + # -r _test.in # pytest-asyncio # pytest-cov # pytest-mock # pytest-sugar pytest-asyncio==0.21.0 - # via -r requirements/_test.in + # via -r _test.in pytest-cov==4.1.0 - # via -r requirements/_test.in + # via -r _test.in pytest-mock==3.10.0 - # via -r requirements/_test.in + # via -r _test.in pytest-runner==6.0.0 - # via -r requirements/_test.in + # via -r _test.in pytest-sugar==0.9.7 - # via -r requirements/_test.in + # via -r _test.in python-dateutil==2.8.2 # via - # -c requirements/_base.txt + # -c _base.txt # faker python-dotenv==1.0.0 - # via -r requirements/_test.in + # via + # -c _base.txt + # -r _test.in requests==2.30.0 # via - # -c requirements/_base.txt + # -c _base.txt + # docker # requests-mock requests-mock==1.10.0 - # via -r requirements/_test.in + # via -r _test.in respx==0.20.1 - # via -r requirements/_test.in + # via -r _test.in six==1.16.0 # via - # -c requirements/_base.txt + # -c _base.txt # python-dateutil # requests-mock sniffio==1.3.0 # via - # -c requirements/_base.txt + # -c _base.txt # anyio # asgi-lifespan # httpcore # httpx +sqlalchemy[asyncio,mypy,postgresql_psycopg2binary]==1.4.48 + # via + # -c ../../../requirements/constraints.txt + # -c _base.txt + # -r _test.in + # alembic +sqlalchemy2-stubs==0.0.2a34 + # via sqlalchemy termcolor==2.3.0 # via pytest-sugar tomli==2.0.1 # via # coverage + # mypy # pytest +typing-extensions==4.5.0 + # via + # -c _base.txt + # alembic + # mypy + # sqlalchemy2-stubs urllib3==2.0.2 # via - # -c requirements/_base.txt + # -c ../../../requirements/constraints.txt + # -c _base.txt + # docker # requests +websocket-client==1.6.0 + # via docker diff --git a/services/resource-usage-tracker/requirements/_tools.txt b/services/resource-usage-tracker/requirements/_tools.txt index c51656306ce..747a527de3b 100644 --- a/services/resource-usage-tracker/requirements/_tools.txt +++ b/services/resource-usage-tracker/requirements/_tools.txt @@ -2,34 +2,34 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --output-file=requirements/_tools.txt --resolver=backtracking --strip-extras requirements/_tools.in +# pip-compile _tools.in # astroid==2.15.5 # via pylint black==23.3.0 - # via -r requirements/../../../requirements/devenv.txt + # via -r ../../../requirements/devenv.txt build==0.10.0 # via pip-tools bump2version==1.0.1 - # via -r requirements/../../../requirements/devenv.txt + # via -r ../../../requirements/devenv.txt cfgv==3.3.1 # via pre-commit click==8.1.3 # via - # -c requirements/_base.txt + # -c _base.txt # black # pip-tools dill==0.3.6 # via pylint distlib==0.3.6 # via virtualenv -filelock==3.12.0 +filelock==3.12.2 # via virtualenv identify==2.5.24 # via pre-commit isort==5.12.0 # via - # -r requirements/../../../requirements/devenv.txt + # -r ../../../requirements/devenv.txt # pylint lazy-object-proxy==1.9.0 # via astroid @@ -41,46 +41,47 @@ nodeenv==1.8.0 # via pre-commit packaging==23.1 # via - # -c requirements/_test.txt + # -c _base.txt + # -c _test.txt # black # build pathspec==0.11.1 # via black pip-tools==6.13.0 - # via -r requirements/../../../requirements/devenv.txt -platformdirs==3.5.1 + # via -r ../../../requirements/devenv.txt +platformdirs==3.5.3 # via # black # pylint # virtualenv pre-commit==3.3.2 - # via -r requirements/../../../requirements/devenv.txt + # via -r ../../../requirements/devenv.txt pylint==2.17.4 - # via -r requirements/../../../requirements/devenv.txt + # via -r ../../../requirements/devenv.txt pyproject-hooks==1.0.0 # via build pyyaml==5.4.1 # via - # -c requirements/_base.txt + # -c ../../../requirements/constraints.txt + # -c _base.txt # pre-commit # watchdog tomli==2.0.1 # via - # -c requirements/_test.txt + # -c _test.txt # black # build # pylint - # pyproject-hooks tomlkit==0.11.8 # via pylint typing-extensions==4.5.0 # via - # -c requirements/_base.txt + # -c _base.txt # astroid virtualenv==20.23.0 # via pre-commit -watchdog==3.0.0 - # via -r requirements/_tools.in +watchdog[watchmedo]==3.0.0 + # via -r _tools.in wheel==0.40.0 # via pip-tools wrapt==1.15.0 diff --git a/services/resource-usage-tracker/requirements/ci.txt b/services/resource-usage-tracker/requirements/ci.txt index a9b821dd6ca..0cc5a0b7f67 100644 --- a/services/resource-usage-tracker/requirements/ci.txt +++ b/services/resource-usage-tracker/requirements/ci.txt @@ -15,6 +15,7 @@ ../../packages/pytest-simcore ../../packages/service-library[fastapi] ../../packages/settings-library +../../packages/postgres-database # installs current package . diff --git a/services/resource-usage-tracker/requirements/dev.txt b/services/resource-usage-tracker/requirements/dev.txt index 6d8bfca9475..cbaea17f45e 100644 --- a/services/resource-usage-tracker/requirements/dev.txt +++ b/services/resource-usage-tracker/requirements/dev.txt @@ -16,6 +16,7 @@ --editable ../../packages/pytest-simcore --editable ../../packages/service-library[fastapi] --editable ../../packages/settings-library +--editable ../../packages/postgres-database # installs current package --editable . diff --git a/services/resource-usage-tracker/requirements/prod.txt b/services/resource-usage-tracker/requirements/prod.txt index 29aeba2dd1e..bb9863c8d60 100644 --- a/services/resource-usage-tracker/requirements/prod.txt +++ b/services/resource-usage-tracker/requirements/prod.txt @@ -13,5 +13,6 @@ ../../packages/models-library ../../packages/service-library[fastapi] ../../packages/settings-library +../../packages/postgres-database # installs current package . diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/cli.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/cli.py index 17fcd4c3c07..741724d4d5c 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/cli.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/cli.py @@ -9,7 +9,7 @@ from settings_library.utils_cli import create_settings_command from simcore_service_resource_usage_tracker.core.errors import ConfigurationError from simcore_service_resource_usage_tracker.modules.prometheus import create_client -from simcore_service_resource_usage_tracker.resource_tracker_core import ( +from simcore_service_resource_usage_tracker.resource_tracker_cli_placeholder import ( collect_and_return_service_resource_usage, ) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py index efeca84fbc8..95115c161a7 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py @@ -12,6 +12,7 @@ SUMMARY, ) from ..api.routes import setup_api_routes +from ..modules.db import setup as setup_db from ..modules.prometheus import setup as setup_prometheus_api_client from ..resource_tracker import setup as setup_background_task from .settings import ApplicationSettings @@ -43,6 +44,9 @@ def create_app(settings: ApplicationSettings) -> FastAPI: # ERROR HANDLERS # ... add here ... setup_prometheus_api_client(app) + if settings.RESOURCE_USAGE_TRACKER_POSTGRES: + setup_db(app) + setup_background_task(app) # EVENTS diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py index 4d82ddde791..91d34a2b7fa 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py @@ -39,6 +39,11 @@ class _BaseApplicationSettings(BaseCustomSettings, MixinLoggingSettings): SC_USER_NAME: str | None = None # RUNTIME ----------------------------------------------------------- + MACHINE_FQDN: str = Field( + default="osparc-master.speag.com", + description="Machine's fully qualified domain name", + env=["MACHINE_FQDN"], + ) RESOURCE_USAGE_TRACKER_DEBUG: bool = Field( default=False, description="Debug mode", @@ -91,14 +96,6 @@ class ApplicationSettings(MinimalApplicationSettings): """ RESOURCE_USAGE_TRACKER_EVALUATION_INTERVAL_SEC: datetime.timedelta = Field( - default=datetime.timedelta(minutes=5), + default=datetime.timedelta(minutes=15), description="Interval to evaluate the resource usage (default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)", ) - RESOURCE_USAGE_TRACKER_GRANULARITY_SEC: int = Field( - default=60, - description="Granularity to fetch data from prometheus. This should be larger than prometheus scraping interval.", - ) - RESOURCE_USAGE_TRACKER_CONTAINER_LABEL_USER_ID_REGEX: str = Field( - default=".*", - description="Regex for the prometheus timeseries label `CONTAINER_LABEL_USER_ID`.", - ) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/__init__.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_container.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_container.py new file mode 100644 index 00000000000..4ee8f685274 --- /dev/null +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_container.py @@ -0,0 +1,31 @@ +from typing import Any + +from arrow import Arrow +from models_library.products import ProductName +from models_library.users import UserID +from pydantic import BaseModel + + +class ContainerResourceUsageMetric(BaseModel): + container_id: str + image: str + user_id: UserID + product_name: ProductName + service_settings_reservation_nano_cpus: int | None + service_settings_reservation_memory_bytes: int | None + service_settings_reservation_additional_info: dict[str, Any] = {} + + +class ContainerResourceUsageValues(BaseModel): + container_cpu_usage_seconds_total: float + prometheus_created: Arrow + prometheus_last_scraped: Arrow + + class Config: + arbitrary_types_allowed = True + + +class ContainerResourceUsage( + ContainerResourceUsageMetric, ContainerResourceUsageValues +): + ... diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/__init__.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/__init__.py new file mode 100644 index 00000000000..bca3083383c --- /dev/null +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/__init__.py @@ -0,0 +1,13 @@ +from fastapi import FastAPI +from servicelib.db_async_engine import close_db_connection, connect_to_db + + +def setup(app: FastAPI): + async def on_startup() -> None: + await connect_to_db(app, app.state.settings.RESOURCE_USAGE_TRACKER_POSTGRES) + + async def on_shutdown() -> None: + await close_db_connection(app) + + app.add_event_handler("startup", on_startup) + app.add_event_handler("shutdown", on_shutdown) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/__init__.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/__init__.py new file mode 100644 index 00000000000..93da4003de3 --- /dev/null +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/__init__.py @@ -0,0 +1,3 @@ +from ._base import BaseRepository + +__all__: tuple[str, ...] = ("BaseRepository",) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/_base.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/_base.py new file mode 100644 index 00000000000..4a20b37c735 --- /dev/null +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/_base.py @@ -0,0 +1,12 @@ +from dataclasses import dataclass + +from sqlalchemy.ext.asyncio import AsyncEngine + + +@dataclass +class BaseRepository: + """ + Repositories are pulled at every request + """ + + db_engine: AsyncEngine diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py new file mode 100644 index 00000000000..89cbadc42f5 --- /dev/null +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py @@ -0,0 +1,53 @@ +import logging + +from simcore_postgres_database.models.resource_tracker import resource_tracker_container +from sqlalchemy import func +from sqlalchemy.dialects.postgresql import insert as pg_insert + +from ....models.resource_tracker_container import ContainerResourceUsage +from ._base import BaseRepository + +_logger = logging.getLogger(__name__) + + +class ResourceTrackerRepository(BaseRepository): + async def upsert_resource_tracker_container_data_( + self, data: ContainerResourceUsage + ) -> None: + async with self.db_engine.begin() as conn: + insert_stmt = pg_insert(resource_tracker_container).values( + container_id=data.container_id, + image=data.image, + user_id=data.user_id, + product_name=data.product_name, + service_settings_reservation_nano_cpus=data.service_settings_reservation_nano_cpus, + service_settings_reservation_memory_bytes=data.service_settings_reservation_memory_bytes, + service_settings_reservation_additional_info=data.service_settings_reservation_additional_info, + container_cpu_usage_seconds_total=data.container_cpu_usage_seconds_total, + prometheus_created=data.prometheus_created.datetime, + prometheus_last_scraped=data.prometheus_last_scraped.datetime, + modified=func.now(), + ) + + on_update_stmt = insert_stmt.on_conflict_do_update( + index_elements=[ + resource_tracker_container.c.container_id, + ], + set_={ + "container_cpu_usage_seconds_total": func.greatest( + resource_tracker_container.c.container_cpu_usage_seconds_total, + insert_stmt.excluded.container_cpu_usage_seconds_total, + ), + "prometheus_created": func.least( + resource_tracker_container.c.prometheus_created, + insert_stmt.excluded.prometheus_created, + ), + "prometheus_last_scraped": func.greatest( + resource_tracker_container.c.prometheus_last_scraped, + insert_stmt.excluded.prometheus_last_scraped, + ), + "modified": func.now(), + }, + ) + + await conn.execute(on_update_stmt) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker.py index a867142300b..340c8920934 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker.py @@ -6,7 +6,7 @@ from settings_library.prometheus import PrometheusSettings from .core.settings import ApplicationSettings -from .resource_tracker_core import collect_service_resource_usage_task +from .resource_tracker_core import collect_container_resource_usage_task _TASK_NAME = "periodic_prometheus_polling" @@ -24,7 +24,7 @@ async def _startup() -> None: _logger.warning("Prometheus API client is de-activated in the settings") return app.state.resource_tracker_task = start_periodic_task( - collect_service_resource_usage_task, + collect_container_resource_usage_task, interval=app_settings.RESOURCE_USAGE_TRACKER_EVALUATION_INTERVAL_SEC, task_name=_TASK_NAME, app=app, diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_cli_placeholder.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_cli_placeholder.py new file mode 100644 index 00000000000..4a97ff8b5f4 --- /dev/null +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_cli_placeholder.py @@ -0,0 +1,136 @@ +import datetime +import json +import logging +from typing import Any + +import arrow +from fastapi import FastAPI +from models_library.users import UserID +from prometheus_api_client import PrometheusConnect +from simcore_service_resource_usage_tracker.modules.prometheus import ( + get_prometheus_api_client, +) + +##### +# This is just a placeholder script with original DevOps provided script, will be modified in upcoming PRs +# Unit tests are already prepared and running. +##### + +_logger = logging.getLogger(__name__) + + +def _assure_dict_entry_exists( + metric_data, max_values_per_docker_id, image, userid +) -> None: + for metric in metric_data: + current_id = metric["metric"]["id"] + if current_id not in max_values_per_docker_id.keys(): + max_values_per_docker_id[current_id] = { + "container_uuid": metric["metric"]["container_label_uuid"], + "cpu_seconds": 0, + "uptime_minutes": 0, + "nano_cpu_limits": 0, + "egress_bytes": 0, + "image": image, + "user_id": userid, + } + + +async def _evaluate_service_resource_usage( + prometheus_client: PrometheusConnect, + start_time: datetime.datetime, + stop_time: datetime.datetime, + user_id: UserID, + uuid: str = ".*", + image: str = "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.9", +) -> dict[str, Any]: + max_values_per_docker_id: dict[str, Any] = {} + time_delta = stop_time - start_time + minutes = round(time_delta.total_seconds() / 60) + + for current_datetime in [ + stop_time - datetime.timedelta(minutes=i) for i in range(minutes) + ]: + rfc3339_str = current_datetime.isoformat("T") + # Query CPU seconds + promql_cpu_query = f"sum without (cpu) (container_cpu_usage_seconds_total{{container_label_user_id='{user_id}',image='{image}',container_label_uuid=~'{uuid}'}})" + container_cpu_seconds_usage = prometheus_client.custom_query( + promql_cpu_query, params={"time": rfc3339_str} + ) + # Query network egress + promql_network_query = f"container_network_transmit_bytes_total{{container_label_user_id='{user_id}',image='{image}',container_label_uuid=~'{uuid}'}}" + container_network_egress = prometheus_client.custom_query( + promql_network_query, params={"time": rfc3339_str} + ) + + if container_cpu_seconds_usage: + _assure_dict_entry_exists( + container_cpu_seconds_usage, max_values_per_docker_id, image, user_id + ) + metric_data = container_cpu_seconds_usage + for metric in metric_data: + current_id = metric["metric"]["id"] + if ( + float(metric["value"][-1]) + > max_values_per_docker_id[current_id]["cpu_seconds"] + ): + max_values_per_docker_id[current_id]["cpu_seconds"] = float( + metric["value"][-1] + ) + if container_network_egress: + _assure_dict_entry_exists( + container_network_egress, max_values_per_docker_id, image, user_id + ) + metric_data = container_network_egress + for metric in metric_data: + current_id = metric["metric"]["id"] + if ( + float(metric["value"][-1]) + > max_values_per_docker_id[current_id]["cpu_seconds"] + ): + max_values_per_docker_id[current_id]["egress_bytes"] = float( + metric["value"][-1] + ) + if container_network_egress or container_cpu_seconds_usage: + metric_data = container_network_egress + for metric in metric_data: + current_id = metric["metric"]["id"] + if float(metric["value"][-1]): + # For every point in time (granularity: minutes) where we find a timeseries, we assume the + # service ran for the incremental unit of 1 minute + max_values_per_docker_id[current_id]["uptime_minutes"] += 1 + + # Get CPU Limits from docker container labels + simcore_service_settings = metric["metric"][ + "container_label_simcore_service_settings" + ] + simcore_service_settings = json.loads(simcore_service_settings) + simcore_service_settings_resources = [ + i + for i in simcore_service_settings + if "name" in i.keys() and "Resources" in i.values() + ][0] + nano_cpu_limits = int( + simcore_service_settings_resources["value"]["Limits"]["NanoCPUs"] + ) + max_values_per_docker_id[current_id][ + "nano_cpu_limits" + ] = nano_cpu_limits + return max_values_per_docker_id + + +async def collect_and_return_service_resource_usage( + prometheus_client: PrometheusConnect, user_id: UserID +) -> dict[str, Any]: + now = arrow.utcnow().datetime + data = await _evaluate_service_resource_usage( + prometheus_client, now - datetime.timedelta(hours=1), now, user_id=user_id + ) + _logger.info(json.dumps(data, indent=2, sort_keys=True)) + return data + + +async def collect_service_resource_usage_task(app: FastAPI) -> None: + await collect_and_return_service_resource_usage( + get_prometheus_api_client(app), 43817 + ) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_core.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_core.py index a22a0c3f4bc..134691a947f 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_core.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_core.py @@ -1,135 +1,111 @@ -import datetime +import asyncio import json import logging from typing import Any import arrow from fastapi import FastAPI -from models_library.users import UserID from prometheus_api_client import PrometheusConnect from simcore_service_resource_usage_tracker.modules.prometheus import ( get_prometheus_api_client, ) -# This script assumes everywhere that the minimum granularity of the data is 1 minute -# Setting it smaller is unreasonable at least for prometheus, as the scraping interval is apprx. eq. to 1 h - +from .models.resource_tracker_container import ContainerResourceUsage +from .modules.db.repositories.resource_tracker import ResourceTrackerRepository _logger = logging.getLogger(__name__) -def _assure_dict_entry_exists( - metric_data, max_values_per_docker_id, image, userid -) -> None: - for metric in metric_data: - current_id = metric["metric"]["id"] - if current_id not in max_values_per_docker_id.keys(): - max_values_per_docker_id[current_id] = { - "container_uuid": metric["metric"]["container_label_uuid"], - "cpu_seconds": 0, - "uptime_minutes": 0, - "nano_cpu_limits": 0, - "egress_bytes": 0, - "image": image, - "user_id": userid, - } +async def _prometheus_client_custom_query( + prometheus_client: PrometheusConnect, promql_cpu_query: str +) -> list[dict]: + _logger.info("Querying prometheus with: %s", promql_cpu_query) + data: list[dict] = await asyncio.get_event_loop().run_in_executor( + None, prometheus_client.custom_query(promql_cpu_query) + ) + return data -async def _evaluate_service_resource_usage( +async def _scrape_and_upload_container_resource_usage( prometheus_client: PrometheusConnect, - start_time: datetime.datetime, - stop_time: datetime.datetime, - user_id: UserID, - uuid: str = ".*", - image: str = "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.9", -) -> dict[str, Any]: - max_values_per_docker_id: dict[str, Any] = {} - time_delta = stop_time - start_time - minutes = round(time_delta.total_seconds() / 60) + resource_tracker_repo: ResourceTrackerRepository, + image_regex: str, +) -> None: + # Query CPU seconds + promql_cpu_query = f"sum without (cpu) (container_cpu_usage_seconds_total{{image=~'{image_regex}'}})[30m:1m]" + containers_cpu_seconds_usage: list = await _prometheus_client_custom_query( + prometheus_client, promql_cpu_query + ) + _logger.info( + "Received %s containers from Prometheus", len(containers_cpu_seconds_usage) + ) - for current_datetime in [ - stop_time - datetime.timedelta(minutes=i) for i in range(minutes) - ]: - rfc3339_str = current_datetime.isoformat("T") - # Query CPU seconds - promql_cpu_query = f"sum without (cpu) (container_cpu_usage_seconds_total{{container_label_user_id='{user_id}',image='{image}',container_label_uuid=~'{uuid}'}})" - container_cpu_seconds_usage = prometheus_client.custom_query( - promql_cpu_query, params={"time": rfc3339_str} - ) - # Query network egress - promql_network_query = f"container_network_transmit_bytes_total{{container_label_user_id='{user_id}',image='{image}',container_label_uuid=~'{uuid}'}}" - container_network_egress = prometheus_client.custom_query( - promql_network_query, params={"time": rfc3339_str} + for item in containers_cpu_seconds_usage: + # Prepare metric + metric: dict[str, Any] = item["metric"] + container_label_simcore_service_settings: list[dict[str, Any]] = json.loads( + metric["container_label_simcore_service_settings"] ) + nano_cpus: int | None = None + memory_bytes: int | None = None + for setting in container_label_simcore_service_settings: + if setting.get("type") == "Resources": + nano_cpus = ( + setting.get("value", {}) + .get("Reservations", {}) + .get("NanoCPUs", None) + ) + memory_bytes = ( + setting.get("value", {}) + .get("Reservations", {}) + .get("MemoryBytes", None) + ) + break - if container_cpu_seconds_usage: - _assure_dict_entry_exists( - container_cpu_seconds_usage, max_values_per_docker_id, image, user_id - ) - metric_data = container_cpu_seconds_usage - for metric in metric_data: - current_id = metric["metric"]["id"] - if ( - float(metric["value"][-1]) - > max_values_per_docker_id[current_id]["cpu_seconds"] - ): - max_values_per_docker_id[current_id]["cpu_seconds"] = float( - metric["value"][-1] - ) - if container_network_egress: - _assure_dict_entry_exists( - container_network_egress, max_values_per_docker_id, image, user_id - ) - metric_data = container_network_egress - for metric in metric_data: - current_id = metric["metric"]["id"] - if ( - float(metric["value"][-1]) - > max_values_per_docker_id[current_id]["cpu_seconds"] - ): - max_values_per_docker_id[current_id]["egress_bytes"] = float( - metric["value"][-1] - ) - if container_network_egress or container_cpu_seconds_usage: - metric_data = container_network_egress - for metric in metric_data: - current_id = metric["metric"]["id"] - if float(metric["value"][-1]): - # For every point in time (granularity: minutes) where we find a timeseries, we assume the - # service ran for the incremental unit of 1 minute - max_values_per_docker_id[current_id]["uptime_minutes"] += 1 + # Prepare values + values: list[list] = item["values"] + first_value: list = values[0] + last_value: list = values[-1] + assert len(first_value) == 2 # nosec + assert len(last_value) == 2 # nosec - # Get CPU Limits from docker container labels - simcore_service_settings = metric["metric"][ - "container_label_simcore_service_settings" - ] - simcore_service_settings = json.loads(simcore_service_settings) - simcore_service_settings_resources = [ - i - for i in simcore_service_settings - if "name" in i.keys() and "Resources" in i.values() - ][0] - nano_cpu_limits = int( - simcore_service_settings_resources["value"]["Limits"]["NanoCPUs"] - ) - max_values_per_docker_id[current_id][ - "nano_cpu_limits" - ] = nano_cpu_limits - return max_values_per_docker_id + container_resource_usage = ContainerResourceUsage( + container_id=metric["id"], + image=metric["image"], + user_id=metric["container_label_user_id"], + product_name=metric["container_label_product_name"], + service_settings_reservation_nano_cpus=int(nano_cpus) + if nano_cpus + else None, + service_settings_reservation_memory_bytes=int(memory_bytes) + if memory_bytes + else None, + service_settings_reservation_additional_info={}, + container_cpu_usage_seconds_total=last_value[1], + prometheus_created=arrow.get(first_value[0]), + prometheus_last_scraped=arrow.get(last_value[0]), + ) + + await resource_tracker_repo.upsert_resource_tracker_container_data_( + container_resource_usage + ) -async def collect_and_return_service_resource_usage( - prometheus_client: PrometheusConnect, user_id: UserID -) -> dict[str, Any]: - now = arrow.utcnow().datetime - data = await _evaluate_service_resource_usage( - prometheus_client, now - datetime.timedelta(hours=1), now, user_id=user_id +async def collect_container_resource_usage( + prometheus_client: PrometheusConnect, + resource_tracker_repo: ResourceTrackerRepository, + machine_fqdn: str, +) -> None: + await _scrape_and_upload_container_resource_usage( + prometheus_client=prometheus_client, + resource_tracker_repo=resource_tracker_repo, + image_regex=f"registry.{machine_fqdn}/simcore/services/dynamic/jupyter-smash:.*", ) - _logger.info(json.dumps(data, indent=2, sort_keys=True)) - return data -async def collect_service_resource_usage_task(app: FastAPI) -> None: - await collect_and_return_service_resource_usage( - get_prometheus_api_client(app), 43817 +async def collect_container_resource_usage_task(app: FastAPI) -> None: + await collect_container_resource_usage( + get_prometheus_api_client(app), + ResourceTrackerRepository(db_engine=app.state.engine), + app.state.settings.MACHINE_FQDN, ) diff --git a/services/resource-usage-tracker/tests/__init__.py b/services/resource-usage-tracker/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/resource-usage-tracker/tests/integration/test_none.py b/services/resource-usage-tracker/tests/integration/test_none.py new file mode 100644 index 00000000000..297cb2d7ad5 --- /dev/null +++ b/services/resource-usage-tracker/tests/integration/test_none.py @@ -0,0 +1,5 @@ +# added as minimal integration tests + + +def test_mock(): + assert True diff --git a/services/resource-usage-tracker/tests/unit/api/test__oas_spec.py b/services/resource-usage-tracker/tests/unit/api/test__oas_spec.py index bf0a72c9ae6..c69c230ca67 100644 --- a/services/resource-usage-tracker/tests/unit/api/test__oas_spec.py +++ b/services/resource-usage-tracker/tests/unit/api/test__oas_spec.py @@ -9,7 +9,10 @@ def test_openapi_json_is_in_sync_with_app_oas( - disabled_prometheus: None, client: TestClient, project_slug_dir: Path + disabled_database: None, + disabled_prometheus: None, + client: TestClient, + project_slug_dir: Path, ): """ If this test fails, just 'make openapi.json' diff --git a/services/resource-usage-tracker/tests/unit/api/test_api_meta.py b/services/resource-usage-tracker/tests/unit/api/test_api_meta.py index 86f4492de4c..a3f1b8ce94e 100644 --- a/services/resource-usage-tracker/tests/unit/api/test_api_meta.py +++ b/services/resource-usage-tracker/tests/unit/api/test_api_meta.py @@ -3,14 +3,15 @@ # pylint: disable=unused-variable # pylint: disable=too-many-arguments - from fastapi import status from fastapi.testclient import TestClient from simcore_service_resource_usage_tracker._meta import API_VTAG from simcore_service_resource_usage_tracker.api._meta import _Meta -def test_healthcheck(disabled_prometheus: None, client: TestClient): +def test_healthcheck( + disabled_database: None, disabled_prometheus: None, client: TestClient +): response = client.get("/") assert response.status_code == status.HTTP_200_OK assert response.text.startswith( @@ -18,7 +19,7 @@ def test_healthcheck(disabled_prometheus: None, client: TestClient): ) -def test_meta(disabled_prometheus: None, client: TestClient): +def test_meta(disabled_database: None, disabled_prometheus: None, client: TestClient): response = client.get(f"/{API_VTAG}/meta") assert response.status_code == status.HTTP_200_OK meta = _Meta.parse_obj(response.json()) diff --git a/services/resource-usage-tracker/tests/unit/conftest.py b/services/resource-usage-tracker/tests/unit/conftest.py index 760d7cf54b1..aa780f3626b 100644 --- a/services/resource-usage-tracker/tests/unit/conftest.py +++ b/services/resource-usage-tracker/tests/unit/conftest.py @@ -25,9 +25,18 @@ from simcore_service_resource_usage_tracker.core.settings import ApplicationSettings pytest_plugins = [ + "pytest_simcore.docker_compose", + "pytest_simcore.docker_registry", + "pytest_simcore.docker_swarm", + "pytest_simcore.monkeypatch_extra", + "pytest_simcore.postgres_service", + "pytest_simcore.pydantic_models", + "pytest_simcore.repository_paths", + "pytest_simcore.schemas", + "pytest_simcore.tmp_path_extra", + "pytest_simcore.pytest_global_environs", "pytest_simcore.cli_runner", "pytest_simcore.environment_configs", - "pytest_simcore.repository_paths", ] @@ -47,11 +56,11 @@ def app_environment(monkeypatch: MonkeyPatch, faker: Faker) -> EnvVarsDict: { "POSTGRES_HOST": faker.domain_name(), "POSTGRES_USER": faker.user_name(), - "POSTGRES_PASSWORD": faker.password(), + "POSTGRES_PASSWORD": faker.password(special_chars=False), "POSTGRES_DB": faker.pystr(), - "PROMETHEUS_URL": f"{choice(['http', 'https'])}://{faker.domain_name()}:{faker.port_number()}", + "PROMETHEUS_URL": f"{choice(['http', 'https'])}://{faker.domain_name()}", "PROMETHEUS_USERNAME": faker.user_name(), - "PROMETHEUS_PASSWORD": faker.password(), + "PROMETHEUS_PASSWORD": faker.password(special_chars=False), }, ) @@ -68,7 +77,19 @@ def disabled_prometheus( @pytest.fixture -def app_settings(app_environment: EnvVarsDict) -> ApplicationSettings: +def disabled_database( + app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("POSTGRES_HOST") + monkeypatch.delenv("POSTGRES_USER") + monkeypatch.delenv("POSTGRES_PASSWORD") + monkeypatch.delenv("POSTGRES_DB") + + +@pytest.fixture +def app_settings( + app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch +) -> ApplicationSettings: settings = ApplicationSettings.create_from_envs() return settings diff --git a/services/resource-usage-tracker/tests/unit/modules/test_prometheus.py b/services/resource-usage-tracker/tests/unit/modules/test_prometheus.py index cba5cb3fbdb..4933547265e 100644 --- a/services/resource-usage-tracker/tests/unit/modules/test_prometheus.py +++ b/services/resource-usage-tracker/tests/unit/modules/test_prometheus.py @@ -27,6 +27,7 @@ def mocked_prometheus_fail_response( def test_prometheus_does_not_initialize_if_deactivated( + disabled_database: None, disabled_prometheus: None, initialized_app: FastAPI, ): @@ -38,7 +39,7 @@ def test_prometheus_does_not_initialize_if_deactivated( def test_mocked_prometheus_initialize( - mocked_prometheus: None, initialized_app: FastAPI + disabled_database, mocked_prometheus: None, initialized_app: FastAPI ): assert get_prometheus_api_client(initialized_app) diff --git a/services/resource-usage-tracker/tests/unit/test_cli.py b/services/resource-usage-tracker/tests/unit/test_cli.py index ed08738b2d8..ec4eb8ee9f0 100644 --- a/services/resource-usage-tracker/tests/unit/test_cli.py +++ b/services/resource-usage-tracker/tests/unit/test_cli.py @@ -43,5 +43,5 @@ def test_evaluate( app_environment: EnvVarsDict, mocked_prometheus_with_query: requests_mock.Mocker, ): - result = cli_runner.invoke(app, ["evaluate", "1234"]) + result = cli_runner.invoke(app, ["evaluate", "43817"]) assert result.exit_code == os.EX_OK, result.output diff --git a/services/resource-usage-tracker/tests/unit/test_resource_tracker.py b/services/resource-usage-tracker/tests/unit/test_resource_tracker.py index 559636d5d55..744658d427f 100644 --- a/services/resource-usage-tracker/tests/unit/test_resource_tracker.py +++ b/services/resource-usage-tracker/tests/unit/test_resource_tracker.py @@ -32,7 +32,7 @@ def app_environment( @pytest.fixture def mock_background_task(mocker: MockerFixture) -> mock.Mock: mocked_task = mocker.patch( - "simcore_service_resource_usage_tracker.resource_tracker.collect_service_resource_usage_task", + "simcore_service_resource_usage_tracker.resource_tracker.collect_container_resource_usage_task", autospec=True, ) return mocked_task @@ -40,6 +40,7 @@ def mock_background_task(mocker: MockerFixture) -> mock.Mock: async def test_resource_tracker_disabled_if_prometheus_disabled_task_created_and_deleted( app_environment: EnvVarsDict, + disabled_database: None, disabled_prometheus: None, mock_background_task: mock.Mock, initialized_app: FastAPI, @@ -56,6 +57,7 @@ async def test_resource_tracker_disabled_if_prometheus_disabled_task_created_and async def test_resource_tracker_task_created_and_deleted( + disabled_database: None, app_environment: EnvVarsDict, mocked_prometheus: requests_mock.Mocker, mock_background_task: mock.Mock, diff --git a/services/resource-usage-tracker/tests/unit/test_resource_tracker_core.py b/services/resource-usage-tracker/tests/unit/test_resource_tracker_core.py index 589f12abd5f..4cd0819b232 100644 --- a/services/resource-usage-tracker/tests/unit/test_resource_tracker_core.py +++ b/services/resource-usage-tracker/tests/unit/test_resource_tracker_core.py @@ -8,7 +8,7 @@ import pytest import requests_mock from fastapi import FastAPI -from simcore_service_resource_usage_tracker.resource_tracker_core import ( +from simcore_service_resource_usage_tracker.resource_tracker_cli_placeholder import ( collect_service_resource_usage_task, ) @@ -34,6 +34,7 @@ async def _triggerer() -> None: async def test_triggering( + disabled_database: None, minimal_configuration: None, mocked_prometheus_with_query: requests_mock.Mocker, trigger_collect_service_resource_usage: Callable[[], Awaitable[None]], diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py b/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py new file mode 100644 index 00000000000..b820de25af3 --- /dev/null +++ b/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py @@ -0,0 +1,64 @@ +# pylint: disable=not-context-manager +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + +from typing import AsyncIterable +from unittest import mock + +import httpx +import pytest +import sqlalchemy as sa +from asgi_lifespan import LifespanManager +from fastapi import FastAPI +from pytest import MonkeyPatch +from pytest_mock import MockerFixture +from pytest_simcore.helpers.typing_env import EnvVarsDict +from pytest_simcore.helpers.utils_envs import setenvs_from_dict +from simcore_service_resource_usage_tracker.core.application import create_app +from simcore_service_resource_usage_tracker.core.settings import ApplicationSettings + + +@pytest.fixture(scope="function") +def mock_env(monkeypatch: MonkeyPatch) -> EnvVarsDict: + """This is the base mock envs used to configure the app. + + Do override/extend this fixture to change configurations + """ + env_vars: EnvVarsDict = { + "SC_BOOT_MODE": "production", + "POSTGRES_CLIENT_NAME": "postgres_test_client", + } + setenvs_from_dict(monkeypatch, env_vars) + return env_vars + + +@pytest.fixture(scope="function") +async def initialized_app( + mock_env: EnvVarsDict, + postgres_db: sa.engine.Engine, + postgres_host_config: dict[str, str], +) -> AsyncIterable[FastAPI]: + settings = ApplicationSettings.create_from_envs() + app = create_app(settings) + async with LifespanManager(app): + yield app + + +@pytest.fixture(scope="function") +async def async_client(initialized_app: FastAPI) -> AsyncIterable[httpx.AsyncClient]: + async with httpx.AsyncClient( + app=initialized_app, + base_url="http://resource-usage-tracker.testserver.io", + headers={"Content-Type": "application/json"}, + ) as client: + yield client + + +@pytest.fixture +def mocked_prometheus(mocker: MockerFixture) -> mock.Mock: + mocked_get_prometheus_api_client = mocker.patch( + "simcore_service_resource_usage_tracker.resource_tracker_core.get_prometheus_api_client", + autospec=True, + ) + return mocked_get_prometheus_api_client diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/data/list_of_prometheus_mocked_outputs.json b/services/resource-usage-tracker/tests/unit/with_dbs/data/list_of_prometheus_mocked_outputs.json new file mode 100644 index 00000000000..01b7f82a95a --- /dev/null +++ b/services/resource-usage-tracker/tests/unit/with_dbs/data/list_of_prometheus_mocked_outputs.json @@ -0,0 +1,474 @@ +[ + { + "metric": { + "container_label_com_docker_compose_oneoff": "False", + "container_label_com_docker_compose_project_working_dir": "/tmp/tmpb1ztyf6m", + "container_label_com_docker_compose_version": "1.29.1", + "container_label_product_name": "osparc", + "container_label_simcore_service_settings": "[{\"name\": \"ports\", \"type\": \"int\", \"value\": 8888}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"DISPLAY=:0\"]}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"SYM_SERVER_HOSTNAME=sym-server_%service_uuid%\"]}, {\"name\": \"mount\", \"type\": \"object\", \"value\": [{\"ReadOnly\": true, \"Source\": \"/tmp/.X11-unix\", \"Target\": \"/tmp/.X11-unix\", \"Type\": \"bind\"}]}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}, {\"name\": \"Resources\", \"type\": \"Resources\", \"value\": {\"Limits\": {\"NanoCPUs\": 4000000000, \"MemoryBytes\": 17179869184}, \"Reservations\": {\"NanoCPUs\": 100000000, \"MemoryBytes\": 536870912, \"GenericResources\": [{\"DiscreteResourceSpec\": {\"Kind\": \"VRAM\", \"Value\": 1}}]}}}]", + "container_label_simcore_user_agent": "puppeteer", + "container_label_study_id": "ac62ed72-0c26-11ee-994f-02420a0b0fc8", + "container_label_user_id": "43818", + "container_label_uuid": "11177106-2302-5461-9017-e495145858b6", + "id": "/docker/ec08331c8658286a384cfa6e57f3b4daa59c9e9a369ed42401f621c142042f40", + "image": "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.7", + "instance": "gpu1", + "job": "cadvisor", + "name": "dy-sidecar-11177106-2302-5461-9017-e495145858b6-0-jupyter-smash" + }, + "values": [ + [ + 1686907140, + "2.5380915280000007" + ], + [ + 1686907200, + "18.057210595000004" + ], + [ + 1686907260, + "18.057210595000004" + ], + [ + 1686907320, + "18.057210595000004" + ], + [ + 1686907380, + "18.057210595000004" + ], + [ + 1686907440, + "18.057210595000004" + ] + ] + }, + { + "metric": { + "container_label_com_docker_compose_oneoff": "False", + "container_label_com_docker_compose_project_working_dir": "/tmp/tmpqg_rpq7e", + "container_label_com_docker_compose_version": "1.29.1", + "container_label_product_name": "osparc", + "container_label_simcore_service_settings": "[{\"name\": \"ports\", \"type\": \"int\", \"value\": 8888}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"DISPLAY=:0\"]}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"SYM_SERVER_HOSTNAME=sym-server_%service_uuid%\"]}, {\"name\": \"mount\", \"type\": \"object\", \"value\": [{\"ReadOnly\": true, \"Source\": \"/tmp/.X11-unix\", \"Target\": \"/tmp/.X11-unix\", \"Type\": \"bind\"}]}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}, {\"name\": \"Resources\", \"type\": \"Resources\", \"value\": {\"Limits\": {\"NanoCPUs\": 4000000000, \"MemoryBytes\": 17179869184}, \"Reservations\": {\"NanoCPUs\": 100000000, \"MemoryBytes\": 536870912, \"GenericResources\": [{\"DiscreteResourceSpec\": {\"Kind\": \"VRAM\", \"Value\": 1}}]}}}]", + "container_label_simcore_user_agent": "puppeteer", + "container_label_study_id": "92acac24-0c26-11ee-98b2-02420a0b0fc9", + "container_label_user_id": "43817", + "container_label_uuid": "9069c179-c3d7-58a6-8431-8cb19339c847", + "id": "/docker/2873c8bee46543f059d4c2c2500ba658d24bdf0abdfcdc2b7eccc955e9538c43", + "image": "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.7", + "instance": "gpu1", + "job": "cadvisor", + "name": "dy-sidecar-9069c179-c3d7-58a6-8431-8cb19339c847-0-jupyter-smash" + }, + "values": [ + [ + 1686907080, + "2.5288082430000003" + ], + [ + 1686907140, + "16.675726328" + ], + [ + 1686907200, + "17.503259114" + ], + [ + 1686907260, + "17.503259114" + ], + [ + 1686907320, + "17.503259114" + ], + [ + 1686907380, + "17.503259114" + ], + [ + 1686907440, + "17.503259114" + ] + ] + }, + { + "metric": { + "container_label_com_docker_compose_oneoff": "False", + "container_label_com_docker_compose_project_working_dir": "/tmp/tmp0m7ijyoy", + "container_label_com_docker_compose_version": "1.29.1", + "container_label_product_name": "osparc", + "container_label_simcore_service_settings": "[{\"name\": \"ports\", \"type\": \"int\", \"value\": 8888}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"DISPLAY=:0\"]}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"SYM_SERVER_HOSTNAME=sym-server_%service_uuid%\"]}, {\"name\": \"mount\", \"type\": \"object\", \"value\": [{\"ReadOnly\": true, \"Source\": \"/tmp/.X11-unix\", \"Target\": \"/tmp/.X11-unix\", \"Type\": \"bind\"}]}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}, {\"name\": \"Resources\", \"type\": \"Resources\", \"value\": {\"Limits\": {\"NanoCPUs\": 4000000000, \"MemoryBytes\": 17179869184}, \"Reservations\": {\"NanoCPUs\": 100000000, \"MemoryBytes\": 536870912, \"GenericResources\": [{\"DiscreteResourceSpec\": {\"Kind\": \"VRAM\", \"Value\": 1}}]}}}]", + "container_label_simcore_user_agent": "puppeteer", + "container_label_study_id": "ac62ed72-0c26-11ee-994f-02420a0b0fc8", + "container_label_user_id": "43818", + "container_label_uuid": "8f1a908e-8b4b-5011-a42c-98abd141fc50", + "id": "/docker/b6cd6d013763a34795aaac8402b7f8943df80c2f63ba4f90f95c5b7235dbbc4e", + "image": "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.7", + "instance": "gpu1", + "job": "cadvisor", + "name": "dy-sidecar-8f1a908e-8b4b-5011-a42c-98abd141fc50-0-jupyter-smash" + }, + "values": [ + [ + 1686907140, + "5.765171485000001" + ], + [ + 1686907200, + "5.835658748999999" + ], + [ + 1686907260, + "5.835658748999999" + ], + [ + 1686907320, + "5.835658748999999" + ], + [ + 1686907380, + "5.835658748999999" + ], + [ + 1686907440, + "5.835658748999999" + ] + ] + }, + { + "metric": { + "container_label_com_docker_compose_oneoff": "False", + "container_label_com_docker_compose_project_working_dir": "/tmp/tmp3o6q4v5e", + "container_label_com_docker_compose_version": "1.29.1", + "container_label_product_name": "osparc", + "container_label_simcore_service_settings": "[{\"name\": \"ports\", \"type\": \"int\", \"value\": 8888}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"DISPLAY=:0\"]}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"SYM_SERVER_HOSTNAME=sym-server_%service_uuid%\"]}, {\"name\": \"mount\", \"type\": \"object\", \"value\": [{\"ReadOnly\": true, \"Source\": \"/tmp/.X11-unix\", \"Target\": \"/tmp/.X11-unix\", \"Type\": \"bind\"}]}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}, {\"name\": \"Resources\", \"type\": \"Resources\", \"value\": {\"Limits\": {\"NanoCPUs\": 4000000000, \"MemoryBytes\": 17179869184}, \"Reservations\": {\"NanoCPUs\": 100000000, \"MemoryBytes\": 536870912, \"GenericResources\": [{\"DiscreteResourceSpec\": {\"Kind\": \"VRAM\", \"Value\": 1}}]}}}]", + "container_label_simcore_user_agent": "puppeteer", + "container_label_study_id": "df604058-0c26-11ee-994f-02420a0b0fc8", + "container_label_user_id": "43819", + "container_label_uuid": "14387f03-4a37-5187-b8e0-0e491389ade4", + "id": "/docker/22394ffafb00907311fc6b76423875411f1084009f89fe77bc618f48f06053eb", + "image": "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.7", + "instance": "gpu1", + "job": "cadvisor", + "name": "dy-sidecar-14387f03-4a37-5187-b8e0-0e491389ade4-0-jupyter-smash" + }, + "values": [ + [ + 1686907200, + "2.500054865" + ], + [ + 1686907260, + "7.3528712789999995" + ], + [ + 1686907320, + "17.330580469999997" + ], + [ + 1686907380, + "17.330580469999997" + ], + [ + 1686907440, + "17.330580469999997" + ], + [ + 1686907500, + "17.330580469999997" + ], + [ + 1686907560, + "17.330580469999997" + ] + ] + }, + { + "metric": { + "container_label_com_docker_compose_oneoff": "False", + "container_label_com_docker_compose_project_working_dir": "/tmp/tmpfgfyp7n5", + "container_label_com_docker_compose_version": "1.29.1", + "container_label_product_name": "osparc", + "container_label_simcore_service_settings": "[{\"name\": \"ports\", \"type\": \"int\", \"value\": 8888}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"DISPLAY=:0\"]}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"SYM_SERVER_HOSTNAME=sym-server_%service_uuid%\"]}, {\"name\": \"mount\", \"type\": \"object\", \"value\": [{\"ReadOnly\": true, \"Source\": \"/tmp/.X11-unix\", \"Target\": \"/tmp/.X11-unix\", \"Type\": \"bind\"}]}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}, {\"name\": \"Resources\", \"type\": \"Resources\", \"value\": {\"Limits\": {\"NanoCPUs\": 4000000000, \"MemoryBytes\": 17179869184}, \"Reservations\": {\"NanoCPUs\": 100000000, \"MemoryBytes\": 536870912, \"GenericResources\": [{\"DiscreteResourceSpec\": {\"Kind\": \"VRAM\", \"Value\": 1}}]}}}]", + "container_label_simcore_user_agent": "puppeteer", + "container_label_study_id": "98f516f8-0c25-11ee-bec2-02420a0b0fc7", + "container_label_user_id": "11367", + "container_label_uuid": "32f3c80d-7cbc-58d3-840e-fedae5252dd0", + "id": "/docker/fecc93800827e3fbdaee9a51842ff2001d010f809c46e806f74611bada02b7b1", + "image": "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.7", + "instance": "gpu1", + "job": "cadvisor", + "name": "dy-sidecar-32f3c80d-7cbc-58d3-840e-fedae5252dd0-0-jupyter-smash" + }, + "values": [ + [ + 1686906660, + "4.4991374429999995" + ], + [ + 1686906720, + "5.768837835" + ], + [ + 1686906780, + "5.815829203" + ], + [ + 1686906840, + "5.815829203" + ], + [ + 1686906900, + "5.815829203" + ], + [ + 1686906960, + "5.815829203" + ], + [ + 1686907020, + "5.815829203" + ] + ] + }, + { + "metric": { + "container_label_com_docker_compose_oneoff": "False", + "container_label_com_docker_compose_project_working_dir": "/tmp/tmpyugvc3z8", + "container_label_com_docker_compose_version": "1.29.1", + "container_label_product_name": "osparc", + "container_label_simcore_service_settings": "[{\"name\": \"ports\", \"type\": \"int\", \"value\": 8888}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"DISPLAY=:0\"]}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"SYM_SERVER_HOSTNAME=sym-server_%service_uuid%\"]}, {\"name\": \"mount\", \"type\": \"object\", \"value\": [{\"ReadOnly\": true, \"Source\": \"/tmp/.X11-unix\", \"Target\": \"/tmp/.X11-unix\", \"Type\": \"bind\"}]}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}, {\"name\": \"Resources\", \"type\": \"Resources\", \"value\": {\"Limits\": {\"NanoCPUs\": 4000000000, \"MemoryBytes\": 17179869184}, \"Reservations\": {\"NanoCPUs\": 100000000, \"MemoryBytes\": 536870912, \"GenericResources\": [{\"DiscreteResourceSpec\": {\"Kind\": \"VRAM\", \"Value\": 1}}]}}}]", + "container_label_simcore_user_agent": "puppeteer", + "container_label_study_id": "92acac24-0c26-11ee-98b2-02420a0b0fc9", + "container_label_user_id": "43817", + "container_label_uuid": "f7c6e801-d8a0-5f57-8283-850adf128613", + "id": "/docker/38179a390b273788cba9a7b33860f0606c4d4c45d13d63f9d1920509e05d2054", + "image": "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.7", + "instance": "gpu1", + "job": "cadvisor", + "name": "dy-sidecar-f7c6e801-d8a0-5f57-8283-850adf128613-0-jupyter-smash" + }, + "values": [ + [ + 1686907080, + "5.086436862999999" + ], + [ + 1686907140, + "5.778702072999999" + ], + [ + 1686907200, + "5.795705571" + ], + [ + 1686907260, + "5.795705571" + ], + [ + 1686907320, + "5.795705571" + ], + [ + 1686907380, + "5.795705571" + ], + [ + 1686907440, + "5.795705571" + ] + ] + }, + { + "metric": { + "container_label_com_docker_compose_oneoff": "False", + "container_label_com_docker_compose_project_working_dir": "/tmp/tmpehms8dgc", + "container_label_com_docker_compose_version": "1.29.1", + "container_label_product_name": "osparc", + "container_label_simcore_service_settings": "[{\"name\": \"ports\", \"type\": \"int\", \"value\": 8888}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"DISPLAY=:0\"]}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"SYM_SERVER_HOSTNAME=sym-server_%service_uuid%\"]}, {\"name\": \"mount\", \"type\": \"object\", \"value\": [{\"ReadOnly\": true, \"Source\": \"/tmp/.X11-unix\", \"Target\": \"/tmp/.X11-unix\", \"Type\": \"bind\"}]}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}, {\"name\": \"Resources\", \"type\": \"Resources\", \"value\": {\"Limits\": {\"NanoCPUs\": 4000000000, \"MemoryBytes\": 17179869184}, \"Reservations\": {\"NanoCPUs\": 100000000, \"MemoryBytes\": 536870912, \"GenericResources\": [{\"DiscreteResourceSpec\": {\"Kind\": \"VRAM\", \"Value\": 1}}]}}}]", + "container_label_simcore_user_agent": "puppeteer", + "container_label_study_id": "52d7e1a8-0c27-11ee-bec2-02420a0b0fc7", + "container_label_user_id": "43820", + "container_label_uuid": "d283967b-11db-5d91-a484-af570f32d5e7", + "id": "/docker/98f90bfd2e28861247f550da8dcf09ebcfa3e5304d15f470708f67f84e724a8f", + "image": "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.7", + "instance": "gpu1", + "job": "cadvisor", + "name": "dy-sidecar-d283967b-11db-5d91-a484-af570f32d5e7-0-jupyter-smash" + }, + "values": [ + [ + 1686907440, + "5.755467759" + ], + [ + 1686907500, + "5.801798311000001" + ], + [ + 1686907560, + "5.801798311000001" + ] + ] + }, + { + "metric": { + "container_label_com_docker_compose_oneoff": "False", + "container_label_com_docker_compose_project_working_dir": "/tmp/tmpdkm5klhn", + "container_label_com_docker_compose_version": "1.29.1", + "container_label_product_name": "osparc", + "container_label_simcore_service_settings": "[{\"name\": \"ports\", \"type\": \"int\", \"value\": 8888}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"DISPLAY=:0\"]}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"SYM_SERVER_HOSTNAME=sym-server_%service_uuid%\"]}, {\"name\": \"mount\", \"type\": \"object\", \"value\": [{\"ReadOnly\": true, \"Source\": \"/tmp/.X11-unix\", \"Target\": \"/tmp/.X11-unix\", \"Type\": \"bind\"}]}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}, {\"name\": \"Resources\", \"type\": \"Resources\", \"value\": {\"Limits\": {\"NanoCPUs\": 4000000000, \"MemoryBytes\": 17179869184}, \"Reservations\": {\"NanoCPUs\": 100000000, \"MemoryBytes\": 536870912, \"GenericResources\": [{\"DiscreteResourceSpec\": {\"Kind\": \"VRAM\", \"Value\": 1}}]}}}]", + "container_label_simcore_user_agent": "puppeteer", + "container_label_study_id": "df604058-0c26-11ee-994f-02420a0b0fc8", + "container_label_user_id": "43819", + "container_label_uuid": "fee6dbfe-529f-5c57-8121-af8a507326f0", + "id": "/docker/e468fa476815d843e3d3dcf4c7988c8846107be02959a2ef1fd41584b18c18ee", + "image": "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.7", + "instance": "gpu1", + "job": "cadvisor", + "name": "dy-sidecar-fee6dbfe-529f-5c57-8121-af8a507326f0-0-jupyter-smash" + }, + "values": [ + [ + 1686907200, + "2.623185233" + ], + [ + 1686907260, + "5.775293023000001" + ], + [ + 1686907320, + "5.818879645000001" + ], + [ + 1686907380, + "5.818879645000001" + ], + [ + 1686907440, + "5.818879645000001" + ], + [ + 1686907500, + "5.818879645000001" + ], + [ + 1686907560, + "5.818879645000001" + ] + ] + }, + { + "metric": { + "container_label_com_docker_compose_oneoff": "False", + "container_label_com_docker_compose_project_working_dir": "/tmp/tmp6y2a68v3", + "container_label_com_docker_compose_version": "1.29.1", + "container_label_product_name": "osparc", + "container_label_simcore_service_settings": "[{\"name\": \"ports\", \"type\": \"int\", \"value\": 8888}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"DISPLAY=:0\"]}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"SYM_SERVER_HOSTNAME=sym-server_%service_uuid%\"]}, {\"name\": \"mount\", \"type\": \"object\", \"value\": [{\"ReadOnly\": true, \"Source\": \"/tmp/.X11-unix\", \"Target\": \"/tmp/.X11-unix\", \"Type\": \"bind\"}]}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}, {\"name\": \"Resources\", \"type\": \"Resources\", \"value\": {\"Limits\": {\"NanoCPUs\": 4000000000, \"MemoryBytes\": 17179869184}, \"Reservations\": {\"NanoCPUs\": 100000000, \"MemoryBytes\": 536870912, \"GenericResources\": [{\"DiscreteResourceSpec\": {\"Kind\": \"VRAM\", \"Value\": 1}}]}}}]", + "container_label_simcore_user_agent": "puppeteer", + "container_label_study_id": "98f516f8-0c25-11ee-bec2-02420a0b0fc7", + "container_label_user_id": "11367", + "container_label_uuid": "88ebaebf-c50e-5b33-a3ce-fe7ed2ad9668", + "id": "/docker/7f7d260357ee8c80f69b148bc090068df4bc1c9aafad909dc1557934d0366967", + "image": "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.7", + "instance": "gpu1", + "job": "cadvisor", + "name": "dy-sidecar-88ebaebf-c50e-5b33-a3ce-fe7ed2ad9668-0-jupyter-smash" + }, + "values": [ + [ + 1686906660, + "2.4900833509999996" + ], + [ + 1686906720, + "10.969049619000002" + ], + [ + 1686906780, + "17.436269327999998" + ], + [ + 1686906840, + "17.436269327999998" + ], + [ + 1686906900, + "17.436269327999998" + ], + [ + 1686906960, + "17.436269327999998" + ], + [ + 1686907020, + "17.436269327999998" + ] + ] + }, + { + "metric": { + "container_label_com_docker_compose_oneoff": "False", + "container_label_com_docker_compose_project_working_dir": "/tmp/tmp_3seh6kp", + "container_label_com_docker_compose_version": "1.29.1", + "container_label_product_name": "osparc", + "container_label_simcore_service_settings": "[{\"name\": \"ports\", \"type\": \"int\", \"value\": 8888}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"DISPLAY=:0\"]}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"SYM_SERVER_HOSTNAME=sym-server_%service_uuid%\"]}, {\"name\": \"mount\", \"type\": \"object\", \"value\": [{\"ReadOnly\": true, \"Source\": \"/tmp/.X11-unix\", \"Target\": \"/tmp/.X11-unix\", \"Type\": \"bind\"}]}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}, {\"name\": \"Resources\", \"type\": \"Resources\", \"value\": {\"Limits\": {\"NanoCPUs\": 4000000000, \"MemoryBytes\": 17179869184}, \"Reservations\": {\"NanoCPUs\": 100000000, \"MemoryBytes\": 536870912, \"GenericResources\": [{\"DiscreteResourceSpec\": {\"Kind\": \"VRAM\", \"Value\": 1}}]}}}]", + "container_label_simcore_user_agent": "puppeteer", + "container_label_study_id": "52d7e1a8-0c27-11ee-bec2-02420a0b0fc7", + "container_label_user_id": "43820", + "container_label_uuid": "2b231c38-0ebc-5cc0-9030-1ffe573f54e9", + "id": "/docker/58e1138d51eb5eafd737024d0df0b01ef88f2087e5a3922565c59130d57ac7a3", + "image": "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.7", + "instance": "gpu1", + "job": "cadvisor", + "name": "dy-sidecar-2b231c38-0ebc-5cc0-9030-1ffe573f54e9-0-jupyter-smash" + }, + "values": [ + [ + 1686907440, + "5.157543565" + ], + [ + 1686907500, + "17.386926716" + ], + [ + 1686907560, + "17.386926716" + ] + ] + }, + { + "metric": { + "container_label_com_docker_compose_oneoff": "False", + "container_label_com_docker_compose_project_working_dir": "/tmp/tmp_3seh6kp", + "container_label_com_docker_compose_version": "1.29.1", + "container_label_product_name": "TEST", + "container_label_simcore_service_settings": "[{\"name\": \"ports\", \"type\": \"int\", \"value\": 8888}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"DISPLAY=:0\"]}, {\"name\": \"env\", \"type\": \"string\", \"value\": [\"SYM_SERVER_HOSTNAME=sym-server_%service_uuid%\"]}, {\"name\": \"mount\", \"type\": \"object\", \"value\": [{\"ReadOnly\": true, \"Source\": \"/tmp/.X11-unix\", \"Target\": \"/tmp/.X11-unix\", \"Type\": \"bind\"}]}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}, {\"name\": \"Resources\", \"type\": \"Resources\", \"value\": {\"Limits\": {\"NanoCPUs\": 4000000000, \"MemoryBytes\": 17179869184}, \"Reservations\": {\"NanoCPUs\": 100000000, \"MemoryBytes\": 536870912, \"GenericResources\": [{\"DiscreteResourceSpec\": {\"Kind\": \"VRAM\", \"Value\": 1}}]}}}]", + "container_label_simcore_user_agent": "puppeteer", + "container_label_study_id": "52d7e1a8-0c27-11ee-bec2-02420a0b0fc7", + "container_label_user_id": "43820", + "container_label_uuid": "2b231c38-0ebc-5cc0-9030-1ffe573f54e9", + "id": "/docker/58e1138d51eb5eafd737024d0df0b01ef88f2087e5a3922565c59130d57ac7a3", + "image": "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.7", + "instance": "gpu1", + "job": "cadvisor", + "name": "dy-sidecar-2b231c38-0ebc-5cc0-9030-1ffe573f54e9-0-jupyter-smash" + }, + "values": [ + [ + 1686907440, + "5.157543565" + ], + [ + 1686907500, + "17.386926716" + ], + [ + 1686908560, + "20.846512345" + ] + ] + } +] diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_collect_container_resource_usage_task.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_collect_container_resource_usage_task.py new file mode 100644 index 00000000000..9e939b80676 --- /dev/null +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_collect_container_resource_usage_task.py @@ -0,0 +1,62 @@ +import json +from pathlib import Path +from unittest import mock + +import pytest +import sqlalchemy as sa +from fastapi import FastAPI +from pytest_mock import MockerFixture +from simcore_postgres_database.models.resource_tracker import resource_tracker_container +from simcore_service_resource_usage_tracker.resource_tracker_core import ( + collect_container_resource_usage_task, +) + +pytest_simcore_core_services_selection = [ + "postgres", +] +pytest_simcore_ops_services_selection = [ + "adminer", +] + + +@pytest.fixture +def mocked_prometheus_client_custom_query( + mocker: MockerFixture, project_slug_dir: Path +) -> mock.MagicMock: + with open( + project_slug_dir + / "tests" + / "unit" + / "with_dbs" + / "data" + / "list_of_prometheus_mocked_outputs.json" + ) as file: + data = json.load(file) + + mocked_get_prometheus_api_client = mocker.patch( + "simcore_service_resource_usage_tracker.resource_tracker_core._prometheus_client_custom_query", + autospec=True, + return_value=data, + ) + return mocked_get_prometheus_api_client + + +async def test_collect_container_resource_usage_task( + mocked_prometheus: mock.Mock, + mocked_prometheus_client_custom_query: mock.MagicMock, + initialized_app: FastAPI, + postgres_db: sa.engine.Engine, +): + await collect_container_resource_usage_task(initialized_app) + + expected_query = "sum without (cpu) (container_cpu_usage_seconds_total{image=~'registry.osparc-master.speag.com/simcore/services/dynamic/jupyter-smash:.*'})[30m:1m]" + mocked_prometheus_client_custom_query.assert_called_once_with( + mocked_prometheus.return_value, expected_query + ) + + db_rows = [] + with postgres_db.connect() as con: + result = con.execute(sa.select(resource_tracker_container)) + for row in result: + db_rows.append(row) + assert len(db_rows) == 10 diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_collect_container_resource_usage_task__on_update_set.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_collect_container_resource_usage_task__on_update_set.py new file mode 100644 index 00000000000..7b58427a631 --- /dev/null +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_collect_container_resource_usage_task__on_update_set.py @@ -0,0 +1,147 @@ +import logging +import random +from pathlib import Path +from unittest import mock + +import arrow +import pytest +import sqlalchemy as sa +from faker import Faker +from fastapi import FastAPI +from pytest_mock import MockerFixture +from simcore_postgres_database.models.resource_tracker import resource_tracker_container +from simcore_service_resource_usage_tracker.resource_tracker_core import ( + collect_container_resource_usage_task, +) + +pytest_simcore_core_services_selection = [ + "postgres", +] +pytest_simcore_ops_services_selection = [ + "adminer", +] + +_logger = logging.getLogger(__name__) + +_END_DATETIME = 1687132800 +_START_DATETIME = 1687046400 +_NUM_OF_GENERATED_OUTPUTS = 20 + + +def _update_variable_if_smaller(existing_value, new_value): + if existing_value is None or new_value < existing_value: + existing_value = new_value + return existing_value + + +def _update_variable_if_bigger(existing_value, new_value): + if existing_value is None or new_value > existing_value: + existing_value = new_value + return existing_value + + +@pytest.fixture +def random_promql_output_generator(): + random_seed = random.randint(0, 100) + _logger.info("Random seed %s", random_seed) + Faker.seed(random_seed) + faker = Faker() + + generated_data: list = [] + min_timestamp_value: int | None = None + max_timestamp_value: int | None = None + max_float_value: float | None = None + + for _ in range(_NUM_OF_GENERATED_OUTPUTS): + a = faker.unix_time(end_datetime=_END_DATETIME, start_datetime=_START_DATETIME) + b = faker.unix_time(end_datetime=_END_DATETIME, start_datetime=_START_DATETIME) + (smaller_timestamp, bigger_timestamp) = (a, b) if a < b else (b, a) + + random_float = faker.pyfloat( + positive=True, min_value=0.157543565, max_value=500 + ) + + min_timestamp_value = _update_variable_if_smaller( + min_timestamp_value, smaller_timestamp + ) + max_timestamp_value = _update_variable_if_bigger( + max_timestamp_value, bigger_timestamp + ) + max_float_value = _update_variable_if_bigger(max_float_value, random_float) + + data_point = { + "metric": { + "container_label_com_docker_compose_oneoff": "False", + "container_label_com_docker_compose_project_working_dir": "/tmp/tmp_3seh6kp", + "container_label_com_docker_compose_version": "1.29.1", + "container_label_product_name": "osparc", + "container_label_simcore_service_settings": '[{"name": "ports", "type": "int", "value": 8888}, {"name": "env", "type": "string", "value": ["DISPLAY=:0"]}, {"name": "env", "type": "string", "value": ["SYM_SERVER_HOSTNAME=sym-server_%service_uuid%"]}, {"name": "mount", "type": "object", "value": [{"ReadOnly": true, "Source": "/tmp/.X11-unix", "Target": "/tmp/.X11-unix", "Type": "bind"}]}, {"name": "constraints", "type": "string", "value": ["node.platform.os == linux"]}, {"name": "Resources", "type": "Resources", "value": {"Limits": {"NanoCPUs": 4000000000, "MemoryBytes": 17179869184}, "Reservations": {"NanoCPUs": 100000000, "MemoryBytes": 536870912, "GenericResources": [{"DiscreteResourceSpec": {"Kind": "VRAM", "Value": 1}}]}}}]', + "container_label_simcore_user_agent": "puppeteer", + "container_label_study_id": "52d7e1a8-0c27-11ee-bec2-024201234c7", + "container_label_user_id": "43820", + "container_label_uuid": "2b231c38-0ebc-5cc0-1234-1ffe573f54e9", + "id": "/docker/58e1138d51eb5eafd737024d0df0b01ef88f2087e5a3922565c59130d57ac7a3", + "image": "registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.7", + "instance": "gpu1", + "job": "cadvisor", + "name": "dy-sidecar-2b231c38-0ebc-5cc0-1234-1ffe573f54e9-0-jupyter-smash", + }, + "values": [ + [smaller_timestamp, "0.157543565"], + [bigger_timestamp, random_float], + ], + } + generated_data.append(data_point) + + return { + "data": generated_data, + "min_timestamp": min_timestamp_value, + "max_timestamp": max_timestamp_value, + "max_float": max_float_value, + } + + +@pytest.fixture +def mocked_prometheus_client_custom_query( + mocker: MockerFixture, project_slug_dir: Path, random_promql_output_generator +) -> dict[str, mock.Mock]: + mocked_get_prometheus_api_client = mocker.patch( + "simcore_service_resource_usage_tracker.resource_tracker_core._prometheus_client_custom_query", + autospec=True, + return_value=random_promql_output_generator["data"], + ) + return mocked_get_prometheus_api_client + + +async def test_collect_container_resource_usage_task( + mocked_prometheus, + mocked_prometheus_client_custom_query, + initialized_app: FastAPI, + postgres_db, + random_promql_output_generator, +): + await collect_container_resource_usage_task(initialized_app) + + expected_query = "sum without (cpu) (container_cpu_usage_seconds_total{image=~'registry.osparc-master.speag.com/simcore/services/dynamic/jupyter-smash:.*'})[30m:1m]" + mocked_prometheus_client_custom_query.assert_called_once_with( + mocked_prometheus.return_value, expected_query + ) + + db_rows = [] + with postgres_db.connect() as con: + result = con.execute(sa.select(resource_tracker_container)) + for row in result: + db_rows.append(row) + assert len(db_rows) == 1 + + assert ( + random_promql_output_generator["max_float"] == db_rows[0][7] + ) # <-- container_cpu_usage_seconds_total + assert ( + arrow.get(random_promql_output_generator["min_timestamp"]).datetime + == db_rows[0][8] + ) # <-- prometheus_created + assert ( + arrow.get(random_promql_output_generator["max_timestamp"]).datetime + == db_rows[0][9] + ) # <-- prometheus_last_scraped