chore: migrate to pyproject.toml

.dockerignore

@@ -5,9 +5,8 @@
 # Include source directories and files required for building.
 !karapace
 !requirements/*.txt
-!setup.py
-!version.py
 !README.rst
+!pyproject.toml
 !container/start.sh
 !container/healthcheck.py
 

.github/workflows/container-smoke-test.yml

@@ -13,12 +13,23 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
+        # Need fetch-depth 0 to fetch tags, see https://github.com/actions/checkout/issues/701
+        with:
+          fetch-depth: 0
 
-      - name: Build karapace/version.py
-        run: python version.py
+      - name: Install requirements
+        run: pip install .
 
-      - name: Build and start services
-        run: docker compose --file=container/compose.yml up --build --wait --detach
+      - name: Resolve Karapace version
+        run: |
+          KARAPACE_VERSION=$(python -c "from karapace import version; print(version.__version__)")
+          echo KARAPACE_VERSION=$KARAPACE_VERSION >> $GITHUB_ENV
+
+      - name: Build container
+        run: docker build --build-arg KARAPACE_VERSION=${{ env.KARAPACE_VERSION }} --file=container/Dockerfile .
+
+      - name: Run container
+        run: docker compose --file=container/compose.yml up --wait --detach
 
       - name: Smoke test registry
         run: bin/smoke-test-registry.sh

.github/workflows/lint.yml

@@ -26,8 +26,6 @@ jobs:
         python-version: '3.12'
     - name: Install libsnappy-dev
       run: sudo apt install libsnappy-dev
-    # required for pylint
-    - run: make karapace/version.py
     - run: pip install pre-commit
     - uses: actions/cache@v3
       with:
@@ -45,6 +43,6 @@ jobs:
         python-version: '3.12'
     - name: Install libsnappy-dev
       run: sudo apt install libsnappy-dev
-    - run: pip install -r requirements/requirements.txt -r requirements/requirements-typing.txt
-    - run: make karapace/version.py
+    - name: Install requirements and typing requirements
+      run: pip install .[typing]
     - run: mypy

.github/workflows/schema.yml

@@ -21,7 +21,8 @@ jobs:
             requirements.txt
       - name: Install libsnappy-dev
         run: sudo apt install libsnappy-dev
-      - run: pip install -r requirements/requirements.txt
+      - name: Install requirements
+      - run: pip install .
       # Compare with latest release when running on main.
       - run: make schema against=$(git describe --abbrev=0 --tags)
         if: github.ref == 'refs/heads/main'

.github/workflows/tests.yml

@@ -74,8 +74,6 @@ jobs:
         pattern: coverage-*
         merge-multiple: true
 
-    - run: make karapace/version.py
-
     - name: Post coverage comment
       id: post_coverage_comment
       uses: py-cov-action/python-coverage-comment-action@v3

CONTRIBUTING.md

@@ -19,8 +19,8 @@ There is very little you need to get started coding for Karapace:
 ```python
 python -m venv <path_to_venv>
 source <path_to_venv>/bin/activate
-pip install -r ./requirements/requirements-dev.txt
-pip install -e .
+pip install .
+pip install -e .[dev,typing]
 ```
 
 ## Website
@@ -55,7 +55,6 @@ To run the tests use the binary `pytest` available in the virtualenv. It will do
 used in the tests for you:
 
 ```sh
-make karapace/version.py
 pytest tests/unit
 pytest tests/integration
 ```

GNUmakefile

@@ -11,6 +11,14 @@ endif
 
 PYTHON_VERSION ?= 3.8
 
+define PIN_VERSIONS_COMMAND
+pip install pip-tools && \
+	python -m piptools compile -o /karapace/requirements/requirements.txt /karapace/pyproject.toml && \
+	python -m piptools compile --extra dev -o /karapace/requirements/requirements-dev.txt /karapace/pyproject.toml && \
+	python -m piptools compile --extra typing -o /karapace/requirements/requirements-typing.txt /karapace/pyproject.toml
+endef
+
+
 export PATH   := $(VENV_DIR)/bin:$(PATH)
 export PS4    := \e[0m\e[32m==> \e[0m
 export LC_ALL := C
@@ -40,31 +48,26 @@ venv/.deps: requirements/requirements-dev.txt requirements/requirements.txt | ve
 	source ./bin/get-protoc
 	source ./bin/get-snappy
 	set -x
-	$(PIP) install --use-pep517 -r '$(<)'
-	$(PIP) install --use-pep517 .
+	$(PIP) install --use-pep517 . .[dev]
 	$(PIP) check
 	touch '$(@)'
 
-
-karapace/version.py:
-	$(PYTHON) version.py
-
 .PHONY: version
-version: venv/.make | karapace/version.py
+version: venv/.make
 
 .PHONY: test
 tests: unit-tests integration-tests
 
 .PHONY: unit-tests
 unit-tests: export PYTEST_ARGS ?=
-unit-tests: karapace/version.py venv/.deps
+unit-tests: venv/.deps
 	rm -fr runtime/*
 	$(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/unit/
 	rm -fr runtime/*
 
 .PHONY: integration-tests
 unit-tests: export PYTEST_ARGS ?=
-integration-tests: karapace/version.py venv/.deps
+integration-tests: venv/.deps
 	rm -fr runtime/*
 	$(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/integration/
 	rm -fr runtime/*
@@ -84,12 +87,14 @@ cleanest: cleaner
 .PHONY: requirements
 requirements: export CUSTOM_COMPILE_COMMAND='make requirements'
 requirements:
-	pip install --upgrade pip setuptools pip-tools
-	cd requirements && pip-compile --upgrade --resolver=backtracking requirements.in -o requirements.txt
-	cd requirements && pip-compile --upgrade --resolver=backtracking requirements-dev.in -o requirements-dev.txt
-	cd requirements && pip-compile --upgrade --resolver=backtracking requirements-typing.in -o requirements-typing.txt
+	$(PIP) install --upgrade pip setuptools pip-tools
+	$(PIP) install . .[dev,typing]
 
 .PHONY: schema
 schema: against := origin/main
 schema:
-	python3 -m karapace.backup.backends.v3.schema_tool --against=$(against)
+	$(PYTHON) -m karapace.backup.backends.v3.schema_tool --against=$(against)
+
+.PHONY: pin-requirements
+pin-requirements:
+	docker run -it -v .:/karapace --security-opt label=disable python:$(PYTHON_VERSION)-bullseye /bin/bash -c "$(PIN_VERSIONS_COMMAND)"

MANIFEST.in

@@ -4,9 +4,7 @@ include karapace.unit
 include tests/*.py
 include scripts/*
 include README.rst
-include version.py
-include setup.py
-include setup.cfg
+include pyproject.toml
 include LICENSE
 include MANIFEST.in
 

README.rst

@@ -88,7 +88,7 @@ Source install
 
 Alternatively you can do a source install using::
 
-  python setup.py install
+  pip install .
 
 Quickstart
 ==========
@@ -648,7 +648,7 @@ If you don't need or want to have the Karapace images around you can now proceed
 Installed from Sources
 ----------------------
 
-If you installed Karapace from the sources via ``python setup.py install``, it can be uninstalled with the following ``pip`` command::
+Karapace is installed ``pip install .``, it can be uninstalled with the following ``pip`` command::
 
     pip uninstall karapace
 
@@ -692,11 +692,8 @@ targets that correctly clean the ``runtime`` directory without deleting it, but
 keep this in mind whenever you are not using ``make`` (e.g. running tests from
 your IDE).
 
-Note that the pre-commit checks are currently not working with the default
-Python version. This is because isort dropped Python 3.7 support. You have to
-use at least Python 3.8 for the pre-commit checks. Use ``pipx`` or ``brew`` or
-… to install pre-commit and use the global installation, there is also no
-dependency on it.
+Use ``pipx`` or ``brew`` or to install pre-commit and use the global installation,
+there is also no dependency on it.
 
 License
 =======

container/Dockerfile

@@ -19,8 +19,14 @@ RUN --mount=type=cache,target=/root/.cache/pip \
     python3 -m pip install -r /build/requirements.txt
 
 COPY . /build/karapace-repo
+WORKDIR /build/karapace-repo
 RUN --mount=type=cache,target=/root/.cache/pip \
-    python3 -m pip install /build/karapace-repo
+    if [ -z "${KARAPACE_VERSION}" ]; then \
+        PRETEND_VERSION="$(python -c 'from karapace import version; print(version.__version__)')"; \
+    else \
+        PRETEND_VERSION=$KARAPACE_VERSION; \
+    fi; \
+    SETUPTOOLS_SCM_PRETEND_VERSION=$PRETEND_VERSION python3 -m pip install .
 
 # Karapace image, i.e. production.
 FROM python:3.10.11-slim-bullseye AS karapace

pyproject.toml

@@ -1,3 +1,111 @@
+[build-system]
+requires = ["setuptools > 64", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "karapace"
+requires-python = ">= 3.8"
+dynamic = ["version"]
+readme = "README.rst"
+license = {file = "LICENSE"}
+dependencies = [
+  "accept-types < 1",
+  "aiohttp < 4",
+  "aiokafka == 0.10.0",
+  "cachetools == 5.3.3",
+  "confluent-kafka == 2.4.0",
+  "isodate < 1",
+  "jsonschema < 5",
+  "lz4",
+  "networkx < 4",
+  "protobuf < 4",
+  "pyjwt >= 2.4.0 , < 3",
+  "python-dateutil < 3",
+  "python-snappy",
+  "rich ~= 13.7.1",
+  "tenacity < 10",
+  "typing-extensions",
+  "ujson < 6",
+  "watchfiles < 1",
+  "xxhash ~= 3.3",
+  "zstandard",
+  "prometheus-client == 0.20.0",
+
+  # Patched dependencies
+  #
+  # Note: It is important to use commits to reference patched dependencies. This
+  # has two advantages:
+  # - Reproducible builds
+  # - The contents of the file change, which invalidates the existing docker
+  #   images and forces a new image generation.
+  #
+  "avro @ https://github.com/aiven/avro/archive/5a82d57f2a650fd87c819a30e433f1abb2c76ca2.tar.gz#subdirectory=lang/py",
+]
+
+classifiers=[
+  "Development Status :: 5 - Production/Stable",
+  "Intended Audience :: Developers",
+  "Intended Audience :: Information Technology",
+  "Intended Audience :: System Administrators",
+  "License :: OSI Approved :: Apache Software License",
+  "Programming Language :: Python :: 3.8",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Topic :: Database :: Database Engines/Servers",
+  "Topic :: Software Development :: Libraries",
+]
+
+[project.scripts]
+karapace = "karapace.karapace_all:main"
+karapace_schema_backup = "karapace.backup.cli:main"
+karapace_mkpasswd = "karapace.auth:main"
+
+[project.urls]
+Homepage = "https://karapace.io"
+Repository = "https://github.com/Aiven-Open/karapace/"
+Issues = "https://github.com/Aiven-Open/karapace/issues"
+
+[project.optional-dependencies]
+sentry-sdk = ["sentry-sdk>=1.6.0"]
+ujson = ["ujson"]
+dev = [
+  # Developer QoL
+  "pdbpp",
+
+  # testing
+  "filelock",
+  "hypothesis",
+  "psutil",
+  "pytest",
+  "pytest-cov",
+  "pytest-random-order",
+  "pytest-timeout",
+  "pytest-xdist[psutil]",
+  "requests",
+
+  # performance test
+  "locust",
+
+  # Sentry SDK
+  "sentry-sdk"
+]
+typing = [
+  "mypy",
+  "sentry-sdk",
+  "types-cachetools",
+  "types-jsonschema",
+  "types-protobuf < 4"
+]
+
+
+[tool.setuptools.packages.find]
+include = ["karapace"]
+
+[tool.setuptools_scm]
+version_file = "karapace/version.py"
+
 [tool.black]
 target-version = ["py38"]
 line-length = 125