diff --git a/lessons/232/python-litestar-app/.dockerignore b/lessons/232/python-litestar-app/.dockerignore new file mode 100644 index 00000000..06e47050 --- /dev/null +++ b/lessons/232/python-litestar-app/.dockerignore @@ -0,0 +1,27 @@ +*.pyc +*.pyo +*.mo +*.db +*.css.map +*.egg-info +*.sql.gz +.cache +.project +.idea +.pydevproject +.idea/workspace.xml +.DS_Store +.git/ +.sass-cache +.vagrant/ +__pycache__ +dist +docs +env +logs +src/{{ project_name }}/settings/local.py +src/node_modules +web/media +web/static/CACHE +stats +Dockerfile \ No newline at end of file diff --git a/lessons/232/python-litestar-app/.gitignore b/lessons/232/python-litestar-app/.gitignore new file mode 100644 index 00000000..9cc511ec --- /dev/null +++ b/lessons/232/python-litestar-app/.gitignore @@ -0,0 +1,168 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ \ No newline at end of file diff --git a/lessons/232/python-litestar-app/Dockerfile b/lessons/232/python-litestar-app/Dockerfile new file mode 100644 index 00000000..6b64e558 --- /dev/null +++ b/lessons/232/python-litestar-app/Dockerfile @@ -0,0 +1,27 @@ +FROM python:3.12-slim-bookworm AS build + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +RUN pip install --upgrade pip +COPY ./requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +FROM python:3.12-slim-bookworm + +ENV PYTHONUNBUFFERED=1 + +WORKDIR /app + +COPY --from=build /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages +COPY --from=build /usr/local/bin /usr/local/bin + +COPY . /app + +CMD ["gunicorn", "-w", "4", "-k", "uvicorn.workers.UvicornWorker", "--timeout", "60", "--graceful-timeout", "60", "--log-level", "error", "main:app", "--bind", "0.0.0.0:8000"] diff --git a/lessons/232/python-litestar-app/db.py b/lessons/232/python-litestar-app/db.py new file mode 100644 index 00000000..57cb55bb --- /dev/null +++ b/lessons/232/python-litestar-app/db.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import logging +import os +from contextlib import asynccontextmanager +from typing import AsyncGenerator + +import aiomcache +import asyncpg + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +POSTGRES_URI = os.environ["POSTGRES_URI"] +POSTGRES_POOL_SIZE = int(os.environ["POSTGRES_POOL_SIZE"]) +MEMCACHED_HOST = os.environ["MEMCACHED_HOST"] +MEMCACHED_POOL_SIZE = int(os.environ["MEMCACHED_POOL_SIZE"]) + + +class Database: + __slots__ = ("_pool",) + + def __init__(self, pool: asyncpg.Pool): + self._pool = pool + + @staticmethod + async def from_postgres() -> Database: + """Create connection pool if it doesn't exist""" + try: + pool = await asyncpg.create_pool( + POSTGRES_URI, + min_size=10, + max_size=POSTGRES_POOL_SIZE, + max_inactive_connection_lifetime=300, + ) + logger.info("Database pool created: %s", pool) + + return Database(pool) + except asyncpg.exceptions.PostgresError as e: + logging.error(f"Error creating PostgreSQL connection pool: {e}") + raise ValueError("Failed to create PostgreSQL connection pool") + except Exception as e: + logging.error(f"Unexpected error while creating connection pool: {e}") + raise + + @asynccontextmanager + async def get_connection(self) -> AsyncGenerator[asyncpg.Connection, None]: + """Get database connection from pool""" + async with self._pool.acquire() as connection: + logger.info("Connection acquired from pool") + yield connection + logger.info("Connection released back to pool") + + async def close(self): + """Close the pool when shutting down""" + await self._pool.close() + logger.info("Database pool closed") + + +class MemcachedClient: + __slots__ = ("_client",) + + def __init__(self, client: aiomcache.Client): + self._client = client + + @staticmethod + async def initialize() -> MemcachedClient: + """Initialize the Memcached client with connection pooling""" + try: + client = aiomcache.Client( + host=MEMCACHED_HOST, pool_size=MEMCACHED_POOL_SIZE + ) + logger.info(f"Memcached client created: {client}") + return MemcachedClient(client) + except Exception: + logging.exception("Error creating Memcached client") + raise ValueError("Failed to create Memcached client") + + async def close(self): + """Close the Memcached client""" + await self._client.close() + logger.info("Memcached client closed") + + def get_client(self) -> aiomcache.Client: + """Get the Memcached client instance""" + return self._client diff --git a/lessons/232/python-litestar-app/main.py b/lessons/232/python-litestar-app/main.py new file mode 100644 index 00000000..ff14a721 --- /dev/null +++ b/lessons/232/python-litestar-app/main.py @@ -0,0 +1,187 @@ +import datetime +import logging +import time +import uuid +from contextlib import asynccontextmanager +from dataclasses import dataclass + +import aiomcache +import orjson +from asyncpg import PostgresError +from litestar import Litestar, Request, get, post +from litestar.exceptions import HTTPException +from litestar.logging import LoggingConfig +from litestar.openapi.config import OpenAPIConfig +from litestar.openapi.plugins import SwaggerRenderPlugin +from pydantic import BaseModel + +from db import Database, MemcachedClient +from metrics import H + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + +logging_config = LoggingConfig( + root={"level": "ERROR", "handlers": ["queue_listener"]}, + formatters={ + "standard": {"format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s"} + }, + log_exceptions="always", +) + + +@dataclass(slots=True) +class DeviceRequest(BaseModel): + mac: str + firmware: str + + +H_MEMCACHED_LABEL = H.labels(op="set", db="memcache") +H_POSTGRES_LABEL = H.labels(op="insert", db="postgres") +INSERT_QUERY = """ + INSERT INTO python_device (uuid, mac, firmware, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5) + RETURNING id; +""" + + +@get("/healthz") +async def healthz_handler(request: Request) -> str: + return "OK" + + +@get("/api/devices") +async def get_devices_handler(request: Request) -> list[dict]: + return [ + { + "id": 1, + "uuid": "9add349c-c35c-4d32-ab0f-53da1ba40a2a", + "mac": "EF-2B-C4-F5-D6-34", + "firmware": "2.1.5", + "created_at": "2024-05-28T15:21:51.137Z", + "updated_at": "2024-05-28T15:21:51.137Z", + }, + { + "id": 2, + "uuid": "d2293412-36eb-46e7-9231-af7e9249fffe", + "mac": "E7-34-96-33-0C-4C", + "firmware": "1.0.3", + "created_at": "2024-01-28T15:20:51.137Z", + "updated_at": "2024-01-28T15:20:51.137Z", + }, + { + "id": 3, + "uuid": "eee58ca8-ca51-47a5-ab48-163fd0e44b77", + "mac": "68-93-9B-B5-33-B9", + "firmware": "4.3.1", + "created_at": "2024-08-28T15:18:21.137Z", + "updated_at": "2024-08-28T15:18:21.137Z", + }, + ] + + +@post("/api/devices") +async def create_device_handler(data: DeviceRequest, request: Request) -> dict: + device_request = data + try: + now = datetime.datetime.now(datetime.timezone.utc) + device_uuid = uuid.uuid4() + + start_time = time.perf_counter() + async with request.app.state.db.get_connection() as conn: + row = await conn.fetchrow( + INSERT_QUERY, + device_uuid, + device_request.mac, + device_request.firmware, + now, + now, + ) + + H_POSTGRES_LABEL.observe(time.perf_counter() - start_time) + + if not row: + raise HTTPException( + status_code=500, detail="Failed to create device record" + ) + + device_dict = { + "id": row["id"], + "uuid": str(device_uuid), + "mac": device_request.mac, + "firmware": device_request.firmware, + "created_at": now, + "updated_at": now, + } + + start_time = time.perf_counter() + cache_client = request.app.state.memcached.get_client() + await cache_client.set( + device_uuid.hex.encode(), + orjson.dumps(device_dict), + exptime=20, + ) + H_MEMCACHED_LABEL.observe(time.perf_counter() - start_time) + + return device_dict + + except PostgresError: + logger.exception("Postgres error") + raise HTTPException(status_code=500, detail="Database error occurred") + except aiomcache.exceptions.ClientException: + logger.exception("Memcached error") + raise HTTPException(status_code=500, detail="Memcached error occurred") + except Exception: + logger.exception("Unknown error") + raise HTTPException(status_code=500, detail="Unexpected error occurred") + + +@get("/api/devices/stats") +async def get_device_stats_handler(request: Request) -> dict: + try: + cache_client = request.app.state.memcached.get_client() + stats = await cache_client.stats() + return { + "curr_items": stats.get(b"curr_items", 0), + "total_items": stats.get(b"total_items", 0), + "bytes": stats.get(b"bytes", 0), + "curr_connections": stats.get(b"curr_connections", 0), + "get_hits": stats.get(b"get_hits", 0), + "get_misses": stats.get(b"get_misses", 0), + } + except aiomcache.exceptions.ClientException: + logger.exception("Memcached error") + raise HTTPException(status_code=500, detail="Memcached error occurred") + except Exception: + logger.exception("Unknown error") + raise HTTPException(status_code=500, detail="Unexpected error occurred") + + +@asynccontextmanager +async def lifespan(app: Litestar): + app.state.db = await Database.from_postgres() + app.state.memcached = await MemcachedClient.initialize() + try: + yield + finally: + await app.state.db.close() + await app.state.memcached.close() + + +app = Litestar( + route_handlers=[ + healthz_handler, + get_devices_handler, + create_device_handler, + get_device_stats_handler, + ], + lifespan=[lifespan], + openapi_config=OpenAPIConfig( + title="Litestar Example", + description="Example of Litestar with Scalar OpenAPI docs", + version="0.0.1", + render_plugins=[SwaggerRenderPlugin(version="5.1.3")], + path="/docs", + ), + logging_config=logging_config, +) diff --git a/lessons/232/python-litestar-app/metrics.py b/lessons/232/python-litestar-app/metrics.py new file mode 100644 index 00000000..c4cd59f1 --- /dev/null +++ b/lessons/232/python-litestar-app/metrics.py @@ -0,0 +1,43 @@ +from prometheus_client import Histogram + +# Exactly the same histogram buckets as in Go. +buckets = ( + 0.00001, 0.000015, 0.00002, 0.000025, 0.00003, 0.000035, 0.00004, 0.000045, + 0.00005, 0.000055, 0.00006, 0.000065, 0.00007, 0.000075, 0.00008, 0.000085, + 0.00009, 0.000095, 0.0001, 0.000101, 0.000102, 0.000103, 0.000104, 0.000105, + 0.000106, 0.000107, 0.000108, 0.000109, 0.00011, 0.000111, 0.000112, 0.000113, + 0.000114, 0.000115, 0.000116, 0.000117, 0.000118, 0.000119, 0.00012, 0.000121, + 0.000122, 0.000123, 0.000124, 0.000125, 0.000126, 0.000127, 0.000128, + 0.000129, 0.00013, 0.000131, 0.000132, 0.000133, 0.000134, 0.000135, 0.000136, + 0.000137, 0.000138, 0.000139, 0.00014, 0.000141, 0.000142, 0.000143, 0.000144, + 0.000145, 0.000146, 0.000147, 0.000148, 0.000149, 0.00015, 0.000151, 0.000152, + 0.000153, 0.000154, 0.000155, 0.000156, 0.000157, 0.000158, 0.000159, 0.00016, + 0.000161, 0.000162, 0.000163, 0.000164, 0.000165, 0.000166, 0.000167, + 0.000168, 0.000169, 0.00017, 0.000171, 0.000172, 0.000173, 0.000174, 0.000175, + 0.000176, 0.000177, 0.000178, 0.000179, 0.00018, 0.000181, 0.000182, 0.000183, + 0.000184, 0.000185, 0.000186, 0.000187, 0.000188, 0.000189, 0.00019, 0.000191, + 0.000192, 0.000193, 0.000194, 0.000195, 0.000196, 0.000197, 0.000198, + 0.000199, 0.0002, 0.00021, 0.00022, 0.00023, 0.00024, 0.00025, 0.00026, + 0.00027, 0.00028, 0.00029, 0.0003, 0.00031, 0.00032, 0.00033, 0.00034, + 0.00035, 0.00036, 0.00037, 0.00038, 0.00039, 0.0004, 0.00041, 0.00042, + 0.00043, 0.00044, 0.00045, 0.00046, 0.00047, 0.00048, 0.00049, 0.0005, + 0.00051, 0.00052, 0.00053, 0.00054, 0.00055, 0.00056, 0.00057, 0.00058, + 0.00059, 0.0006, 0.00061, 0.00062, 0.00063, 0.00064, 0.00065, 0.00066, + 0.00067, 0.00068, 0.00069, 0.0007, 0.00071, 0.00072, 0.00073, 0.00074, + 0.00075, 0.00076, 0.00077, 0.00078, 0.00079, 0.0008, 0.00081, 0.00082, + 0.00083, 0.00084, 0.00085, 0.00086, 0.00087, 0.00088, 0.00089, 0.0009, + 0.00091, 0.00092, 0.00093, 0.00094, 0.00095, 0.00096, 0.00097, 0.00098, + 0.00099, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.0035, 0.004, 0.0045, 0.005, + 0.0055, 0.006, 0.0065, 0.007, 0.0075, 0.008, 0.0085, 0.009, 0.0095, 0.01, + 0.015, 0.02, 0.025, 0.03, 0.035, 0.04, 0.045, 0.05, 0.055, 0.06, 0.065, 0.07, + 0.075, 0.08, 0.085, 0.09, 0.095, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, + 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1.0, 1.5, 2.0, 2.5, + 3.0, 3.5, 4.0, 4.5, 5.0, +) + +H = Histogram( + "myapp_request_duration_seconds", + "Duration of the request", + labelnames=("op", "db"), + buckets=buckets +) diff --git a/lessons/232/python-litestar-app/requirements.txt b/lessons/232/python-litestar-app/requirements.txt new file mode 100644 index 00000000..afb9b55f --- /dev/null +++ b/lessons/232/python-litestar-app/requirements.txt @@ -0,0 +1,8 @@ +aiomcache==0.8.2 +orjson==3.10.12 +asyncpg==0.30.0 +litestar==2.13.0 +pydantic==2.10.4 +prometheus_client==0.21.1 +gunicorn==23.0.0 +uvicorn==0.34.0 diff --git a/lessons/232/python-litestar-app/schema.sql b/lessons/232/python-litestar-app/schema.sql new file mode 100644 index 00000000..3a68c1c9 --- /dev/null +++ b/lessons/232/python-litestar-app/schema.sql @@ -0,0 +1,14 @@ +CREATE TABLE IF NOT EXISTS python_device ( + id SERIAL PRIMARY KEY, + uuid UUID DEFAULT NULL, + mac VARCHAR(255) DEFAULT NULL, + firmware VARCHAR(255) DEFAULT NULL, + created_at TIMESTAMP + WITH + TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP + WITH + TIME ZONE DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_device_uuid ON python_device (uuid); \ No newline at end of file