diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..580576e --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +* @timuram + diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..0d14a1c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,16 @@ +version: 2 +updates: + - package-ecosystem: pip + directory: "/" + schedule: + interval: weekly + time: "00:00" + open-pull-requests-limit: 10 + groups: + python-packages: + patterns: + - "*" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: weekly diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..4f6f99b --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,65 @@ +name: CI + +on: + push: + branches: + - master + tags: [ 'v*' ] + pull_request: + branches: + - master + +jobs: + test: + name: Test + runs-on: ubuntu-latest + timeout-minutes: 15 + + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12"] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + make deps + - name: Lint + run: | + make lint + - name: Tests + run: | + make test + + deploy: + name: Deploy + runs-on: ubuntu-latest + needs: test + if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') + environment: + name: pypi + url: https://pypi.org/p/pytest-service/ + permissions: + id-token: write + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install dependencies + run: + python -m pip install -U pip wheel twine build + - name: Make dists + run: + python -m build + - name: Check dists + run: + twine check dist/* + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d01d2d5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,132 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +.idea/ +.git_old diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..53ecd7d --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,8 @@ +## v0.0.2 (2024-18-06) + +* Add `redis` service + + +## v0.0.1 (2024-05-11) + +* First version diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f31169e --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Timur Ozheghin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..19fed2b --- /dev/null +++ b/Makefile @@ -0,0 +1,27 @@ +all: deps lint test + +deps: + @python3 -m pip install --upgrade pip && pip3 install -r requirements-dev.txt + +black: + @black --line-length 120 pytest_service tests + +isort: + @isort --line-length 120 --use-parentheses --multi-line 3 --combine-as --trailing-comma pytest_service tests + +flake8: + @flake8 --max-line-length 120 --ignore C901,C812,E203,E704 --extend-ignore W503 pytest_service tests + +pyright: + @pyright pytest_service tests + +lint: black isort flake8 pyright + +test: + @python3 -m pytest -vv --rootdir tests . + +pyenv: + echo pytest_service > .python-version && pyenv install -s 3.11.1 && pyenv virtualenv -f 3.11.1 pytest_service + +pyenv-delete: + pyenv virtualenv-delete -f pytest_service diff --git a/README.md b/README.md new file mode 100644 index 0000000..f772206 --- /dev/null +++ b/README.md @@ -0,0 +1,43 @@ +# pytest-service + +Inspired by [pytest-pg](https://pypi.org/project/pytest-pg/) but with `MongoDB` onboard + +### How to use + +#### Postgres + +```python +@pytest.fixture(scope="session") +def pg_14_local() -> Iterator: + with pytest_service.PGService("postgres:14.4-alpine").run() as pg: + yield pg + + +@pytest.fixture(scope="session", autouse=True) +def init_env(pg_14_local: pytest_service.PG) -> None: + if not pg_14_local: + return + os.environ["POSTGRES_DBNAME"] = pg_14_local.database + os.environ["POSTGRES_USER"] = pg_14_local.user + os.environ["POSTGRES_PASSWORD"] = pg_14_local.password + os.environ["POSTGRES_HOST"] = pg_14_local.host + os.environ["POSTGRES_PORT"] = str(pg_14_local.port) + +``` + +#### MongoDB + +```python +@pytest.fixture(scope="session") +def mongo_6_local() -> Iterator: + with pytest_service.MongoDBService("mongo:6").run() as mongo: + yield mongo + + +@pytest.fixture(scope="session", autouse=True) +def init_env(mongo_6_local: pytest_service.Mongo) -> None: + if not mongo_6_local: + return + os.environ["MONGODB_CONNECTION_STRING"] = mongo_6_local.connection_string + +``` diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e5ecc64 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +[build-system] +requires = ["setuptools>=68.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "pytest_service" +dynamic = ["version"] +requires-python = ">=3.10" +dependencies = [ + "pytest>=6.0.0", + "docker>=6.1.0", + "requests<2.33.0", +] +classifiers = [ + "License :: OSI Approved :: MIT License", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Operating System :: OS Independent", + "Development Status :: 5 - Production/Stable", + "Framework :: Pytest" +] +license = { file = "LICENSE" } +readme = "README.md" + +[project.urls] +homepage = "https://github.com/anna-money/pytest-service" +changelog = "https://github.com/anna-money/pytest-service/blob/master/CHANGELOG.md" + +[project.entry-points.pytest11] +pytest_service = "pytest_service" + +[tool.setuptools] +packages = ["pytest_service"] + +[tool.setuptools.package-data] +pytest_service = ["py.typed", "VERSION"] + +[tool.setuptools.dynamic] +version = { file = ["pytest_service/VERSION"] } diff --git a/pytest_service/VERSION b/pytest_service/VERSION new file mode 100644 index 0000000..4e379d2 --- /dev/null +++ b/pytest_service/VERSION @@ -0,0 +1 @@ +0.0.2 diff --git a/pytest_service/__init__.py b/pytest_service/__init__.py new file mode 100644 index 0000000..5d0945b --- /dev/null +++ b/pytest_service/__init__.py @@ -0,0 +1,54 @@ +import collections +import re +import sys +from typing import Tuple + +from .mongo import Mongo, MongoDBService, mongo, mongo_5, mongo_6 # noqa +from .pg import PG, PGService, pg, pg_11, pg_12, pg_13, pg_14, pg_15, pg_16 # noqa +from .redis import Redis, RedisService, redis, redis_7 # noqa + +__version__ = "0.0.0" +__all__: Tuple[str, ...] = ( + "PG", + "PGService", + "pg", + "pg_11", + "pg_12", + "pg_13", + "pg_14", + "pg_15", + "pg_16", + "Mongo", + "MongoDBService", + "mongo", + "mongo_6", + "mongo_5", + "Redis", + "RedisService", + "redis", + "redis_7", +) # + +version = f"{__version__}, Python {sys.version}" + +VersionInfo = collections.namedtuple("VersionInfo", "major minor micro release_level serial") + + +def _parse_version(v: str) -> VersionInfo: + version_re = r"^(?P\d+)\.(?P\d+)\.(?P\d+)" r"((?P[a-z]+)(?P\d+)?)?$" + match = re.match(version_re, v) + if not match: + raise ImportError(f"Invalid package version {v}") + try: + major = int(match.group("major")) + minor = int(match.group("minor")) + micro = int(match.group("micro")) + levels = {"rc": "candidate", "a": "alpha", "b": "beta", None: "final"} + release_level = levels[match.group("release_level")] + serial = int(match.group("serial")) if match.group("serial") else 0 + return VersionInfo(major, minor, micro, release_level, serial) + except Exception as e: + raise ImportError(f"Invalid package version {v}") from e + + +version_info = _parse_version(__version__) diff --git a/pytest_service/base.py b/pytest_service/base.py new file mode 100644 index 0000000..220ae68 --- /dev/null +++ b/pytest_service/base.py @@ -0,0 +1,96 @@ +import abc +import contextlib +import socket +import time +import uuid +from typing import Any, Dict, Generator, Generic, TypeVar, Union, cast + +import docker.types +import pytest +from docker.models.containers import Container + +LOCALHOST = "127.0.0.1" +T = TypeVar("T") + + +class AbstractService(abc.ABC, Generic[T]): + __slots__ = ("_image", "_host", "_ready_timeout", "__client") + + service_name: str + service_port: int + data_path: str + + @contextlib.contextmanager + def run(self) -> Generator[T, None, None]: + unused_port = self.find_unused_local_port() + try: + container = cast( + Container, + self._client.containers.run( + name=f"pytest-service-{self.service_name}-{uuid.uuid4()}", + image=self._image, + environment=self._get_container_environment(), + command=self._get_container_command(), + ports={str(self.service_port): unused_port}, + detach=True, + tmpfs={self.data_path: ""}, + stderr=True, + **self._get_container_kwargs(), + ), + ) + + started_at = time.time() + while time.time() - started_at < self._ready_timeout: + container.reload() + if container.status == "running" and self._is_ready(container): + break + + time.sleep(0.5) + else: + assert container.id + raw_logs = cast(bytes, self._client.api.logs(container.id)) + pytest.fail( + f"Failed to start {self.service_name} using {self._image} in {self._ready_timeout}" + f" seconds: {raw_logs.decode()}" + ) + + yield self._get_success_instance(unused_port) + + container.reload() + if container.status == "running": + container.kill() + container.remove(v=True, force=True) + finally: + self._client.close() + + def _get_container_environment(self) -> Union[Dict[str, str], None]: + return None + + def _get_container_command(self) -> Union[str, None]: + return None + + def _get_container_kwargs(self) -> Dict[str, Any]: + return {} + + @abc.abstractmethod + def _is_ready(self, container: Container) -> bool: ... + + @abc.abstractmethod + def _get_success_instance(self, port: int) -> T: ... + + def find_unused_local_port(self) -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind((self._host, 0)) + return s.getsockname()[1] # type: ignore + + @property + def _client(self) -> docker.DockerClient: + if self.__client is None: + self.__client = docker.from_env() + return self.__client + + def __init__(self, image: str, *, host: str = LOCALHOST, ready_timeout: float = 30.0) -> None: + self._image = image + self._host = host + self._ready_timeout = ready_timeout + self.__client = None diff --git a/pytest_service/mongo.py b/pytest_service/mongo.py new file mode 100644 index 0000000..dcda91b --- /dev/null +++ b/pytest_service/mongo.py @@ -0,0 +1,43 @@ +import dataclasses +from typing import Generator + +import pytest +from docker.models.containers import Container + +from .base import AbstractService + + +@dataclasses.dataclass(frozen=True) +class Mongo: + connection_string: str + + +class MongoDBService(AbstractService[Mongo]): + service_name = "mongodb" + service_port = 27017 + data_path = "/data" + + def _is_ready(self, container: Container) -> bool: + result = container.exec_run("echo 'db.runCommand(\"ping\").ok' | mongosh mongo:27017/test --quiet") + return result.exit_code == 0 + + def _get_success_instance(self, port: int) -> Mongo: + return Mongo(connection_string=f"{self._host}:{port}") + + +@pytest.fixture(scope="session") +def mongo() -> Generator[Mongo, None, None]: + with MongoDBService("mongo:latest").run() as pg: + yield pg + + +@pytest.fixture(scope="session") +def mongo_5() -> Generator[Mongo, None, None]: + with MongoDBService("mongo:5").run() as pg: + yield pg + + +@pytest.fixture(scope="session") +def mongo_6() -> Generator[Mongo, None, None]: + with MongoDBService("mongo:6").run() as pg: + yield pg diff --git a/pytest_service/pg.py b/pytest_service/pg.py new file mode 100644 index 0000000..1fd326e --- /dev/null +++ b/pytest_service/pg.py @@ -0,0 +1,101 @@ +import dataclasses +from typing import Dict, Generator, Union + +import pytest +from docker.models.containers import Container + +from .base import AbstractService +from .utils import is_pg_ready + + +@dataclasses.dataclass(frozen=True) +class PG: + host: str + port: int + user: str + password: str + database: str + + +class PGService(AbstractService[PG]): + __slots__ = () + + DEFAULT_PG_USER = "postgres" + DEFAULT_PG_PASSWORD = "my-secret-password" + DEFAULT_PG_DATABASE = "postgres" + + service_name = "postgres" + service_port = 5432 + data_path = "/var/lib/postgresql/data" + + def _get_container_environment(self) -> Union[Dict[str, str], None]: + return { + "POSTGRES_HOST_AUTH_METHOD": "trust", + "PGDATA": self.data_path, + } + + def _get_container_command(self) -> Union[str, None]: + return "-c fsync=off -c full_page_writes=off -c synchronous_commit=off -c bgwriter_lru_maxpages=0 -c jit=off" + + def _is_ready(self, container: Container) -> bool: + bindings = (container.attrs or {}).get("HostConfig", {}).get("PortBindings", {}) + assert bindings + port = bindings[f"{self.service_port}/tcp"][0]["HostPort"] + return is_pg_ready( + host=self._host, + port=port, + database=self.DEFAULT_PG_DATABASE, + user=self.DEFAULT_PG_USER, + password=self.DEFAULT_PG_PASSWORD, + ) + + def _get_success_instance(self, port: int) -> PG: + return PG( + host=self._host, + port=port, + user=self.DEFAULT_PG_USER, + password=self.DEFAULT_PG_PASSWORD, + database=self.DEFAULT_PG_DATABASE, + ) + + +@pytest.fixture(scope="session") +def pg() -> Generator[PG, None, None]: + with PGService("postgres:latest").run() as pg: + yield pg + + +@pytest.fixture(scope="session") +def pg_11() -> Generator[PG, None, None]: + with PGService("postgres:11").run() as pg: + yield pg + + +@pytest.fixture(scope="session") +def pg_12() -> Generator[PG, None, None]: + with PGService("postgres:12").run() as pg: + yield pg + + +@pytest.fixture(scope="session") +def pg_13() -> Generator[PG, None, None]: + with PGService("postgres:13").run() as pg: + yield pg + + +@pytest.fixture(scope="session") +def pg_14() -> Generator[PG, None, None]: + with PGService("postgres:14").run() as pg: + yield pg + + +@pytest.fixture(scope="session") +def pg_15() -> Generator[PG, None, None]: + with PGService("postgres:15").run() as pg: + yield pg + + +@pytest.fixture(scope="session") +def pg_16() -> Generator[PG, None, None]: + with PGService("postgres:16").run() as pg: + yield pg diff --git a/pytest_service/py.typed b/pytest_service/py.typed new file mode 100644 index 0000000..20a7439 --- /dev/null +++ b/pytest_service/py.typed @@ -0,0 +1 @@ +Marker \ No newline at end of file diff --git a/pytest_service/redis.py b/pytest_service/redis.py new file mode 100644 index 0000000..960c999 --- /dev/null +++ b/pytest_service/redis.py @@ -0,0 +1,38 @@ +import dataclasses +from typing import Generator + +import pytest +from docker.models.containers import Container + +from .base import AbstractService + + +@dataclasses.dataclass(frozen=True) +class Redis: + connection_string: str + + +class RedisService(AbstractService[Redis]): + service_name = "redis" + service_port = 6379 + data_path = "/data" + proto = "redis" + + def _is_ready(self, container: Container) -> bool: + result = container.exec_run("redis-cli ping | grep PONG") + return result.exit_code == 0 + + def _get_success_instance(self, port: int) -> Redis: + return Redis(connection_string=f"{self.proto}://{self._host}:{port}") + + +@pytest.fixture(scope="session") +def redis() -> Generator[Redis, None, None]: + with RedisService("redis:alpine").run() as r: + yield r + + +@pytest.fixture(scope="session") +def redis_7() -> Generator[Redis, None, None]: + with RedisService("redis:7-alpine").run() as pg: + yield pg diff --git a/pytest_service/utils.py b/pytest_service/utils.py new file mode 100644 index 0000000..c12d128 --- /dev/null +++ b/pytest_service/utils.py @@ -0,0 +1,67 @@ +import asyncio +from typing import Any, Optional, Protocol + + +class IsPostgresReadyFunc(Protocol): + def __call__( + self, + *, + host: str, + port: int, + database: str, + user: str, + password: str, + ) -> bool: ... + + +def _try_get_is_postgres_ready_based_on_psycopg2() -> Optional[IsPostgresReadyFunc]: + try: + # noinspection PyPackageRequirements + import psycopg2 # type: ignore[reportMissingModuleSource] + + def _is_postgres_ready(**params: Any) -> bool: + try: + with psycopg2.connect(**params): + return True + except psycopg2.OperationalError: + return False + + return _is_postgres_ready + except ImportError: + return None + + +def _try_get_is_postgres_ready_based_on_asyncpg() -> Optional[IsPostgresReadyFunc]: + try: + # noinspection PyPackageRequirements + import asyncpg # type: ignore[reportMissingImports] + + def _is_postgres_ready(**params: Any) -> bool: + async def _is_postgres_ready_async() -> bool: + try: + connection = await asyncpg.connect(**params) + await connection.close() + return True + except (asyncpg.exceptions.PostgresError, OSError): + return False + + return asyncio.run(_is_postgres_ready_async()) + + return _is_postgres_ready + + except ImportError: + return None + + +def _get_dummy_is_postgresql_ready() -> IsPostgresReadyFunc: + def _is_postgres_ready(**_: Any) -> bool: + return True + + return _is_postgres_ready + + +is_pg_ready = ( + _try_get_is_postgres_ready_based_on_asyncpg() + or _try_get_is_postgres_ready_based_on_psycopg2() + or _get_dummy_is_postgresql_ready() +) diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..04ba946 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,12 @@ +pytest==8.3.2 +isort==5.13.2 +flake8==7.1.1 +black==24.8.0 +setuptools==72.1.0 +wheel==0.44.0 +twine==5.1.1 +pyright==1.1.374 + +docker==7.1.0 + +-e . diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..05c09a7 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1 @@ +pytest_plugins = ["pytest_service"] diff --git a/tests/test_mongo.py b/tests/test_mongo.py new file mode 100644 index 0000000..590a31a --- /dev/null +++ b/tests/test_mongo.py @@ -0,0 +1,13 @@ +import pytest_service + + +def test_mongo(mongo: pytest_service.Mongo) -> None: + assert mongo + + +def test_mongo_5(mongo_5: pytest_service.Mongo) -> None: + assert mongo_5 + + +def test_mongo_6(mongo_6: pytest_service.Mongo) -> None: + assert mongo_6 diff --git a/tests/test_pg.py b/tests/test_pg.py new file mode 100644 index 0000000..a4406d1 --- /dev/null +++ b/tests/test_pg.py @@ -0,0 +1,29 @@ +import pytest_service + + +def test_pg(pg: pytest_service.PG) -> None: + assert pg + + +def test_pg_11(pg_11: pytest_service.PG) -> None: + assert pg_11 + + +def test_pg_12(pg_12: pytest_service.PG) -> None: + assert pg_12 + + +def test_pg_13(pg_13: pytest_service.PG) -> None: + assert pg_13 + + +def test_pg_14(pg_14: pytest_service.PG) -> None: + assert pg_14 + + +def test_pg_15(pg_15: pytest_service.PG) -> None: + assert pg_15 + + +def test_pg_16(pg_16: pytest_service.PG) -> None: + assert pg_16 diff --git a/tests/test_redis.py b/tests/test_redis.py new file mode 100644 index 0000000..f65df96 --- /dev/null +++ b/tests/test_redis.py @@ -0,0 +1,9 @@ +import pytest_service + + +def test_redis(redis: pytest_service.Redis) -> None: + assert redis + + +def test_redis_7(redis_7: pytest_service.Redis) -> None: + assert redis_7