ci: multi-python docker image for GHA matrix

Aiven-Open · Dec 9, 2024 · 59d3a1a · 59d3a1a
1 parent 552187c
commit 59d3a1a
Show file tree

Hide file tree

Showing 12 changed files with 46 additions and 31 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -4,7 +4,8 @@
 
 # Include source directories and files required for building.
 !go
-!src
+!src/karapace
+!src/schema_registry
 !requirements/*.txt
 !README.rst
 !LICENSE

diff --git a/.github/workflows/container-smoke-test.yml b/.github/workflows/container-smoke-test.yml
@@ -8,6 +8,9 @@ on:
 jobs:
   smoke-test-container:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ '3.10', '3.11', '3.12' ]
     env:
       BUILDKIT_PROGRESS: plain
     steps:
@@ -18,7 +21,7 @@ jobs:
           fetch-depth: 0
 
       - name: Install requirements
-        run: make install
+        run: make install-dev
 
       - name: Resolve Karapace version
         run: |
@@ -32,6 +35,7 @@ jobs:
       - name: Run container
         run: make start-karapace-docker-resources
         env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
           KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
           RUNNER_UID: ${{ env.RUNNER_UID }}
           RUNNER_GID: ${{ env.RUNNER_GID }}

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ '3.9', '3.10', '3.11', '3.12' ]
+        python-version: [ '3.10', '3.11', '3.12' ]
     env:
       PYTEST_ADDOPTS: >-
         --log-dir=/tmp/ci-logs
@@ -40,14 +40,21 @@ jobs:
       with:
         go-version: '1.21.0'
 
+    - name: Install requirements
+      run: make install-dev
+
     - name: Resolve Karapace version
-      run: echo KARAPACE_VERSION=4.1.1.dev44+gac20eeed.d20241205 >> $GITHUB_ENV
+      run: |
+        source ./venv/bin/activate
+        KARAPACE_VERSION=$(python -c "from karapace import version; print(version.__version__)")
+        echo KARAPACE_VERSION=$KARAPACE_VERSION >> $GITHUB_ENV
 
     - run: echo "RUNNER_UID=$(id -u)" >> $GITHUB_ENV
     - run: echo "RUNNER_GID=$(id -g)" >> $GITHUB_ENV
 
     - run: make unit-tests-in-docker
       env:
+        PYTHON_VERSION: ${{ matrix.python-version }}
         KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
         RUNNER_UID: ${{ env.RUNNER_UID }}
         RUNNER_GID: ${{ env.RUNNER_GID }}
@@ -56,6 +63,7 @@ jobs:
 
     - run: make e2e-tests-in-docker
       env:
+        PYTHON_VERSION: ${{ matrix.python-version }}
         KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
         RUNNER_UID: ${{ env.RUNNER_UID }}
         RUNNER_GID: ${{ env.RUNNER_GID }}
@@ -64,6 +72,7 @@ jobs:
 
     - run: make integration-tests-in-docker
       env:
+        PYTHON_VERSION: ${{ matrix.python-version }}
         KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
         RUNNER_UID: ${{ env.RUNNER_UID }}
         RUNNER_GID: ${{ env.RUNNER_GID }}

diff --git a/GNUmakefile b/GNUmakefile
@@ -3,7 +3,7 @@ SHELL := /usr/bin/env bash
 VENV_DIR ?= $(CURDIR)/venv
 PIP      ?= pip3 --disable-pip-version-check --no-input --require-virtualenv
 PYTHON   ?= python3
-PYTHON_VERSION ?= 3.9
+PYTHON_VERSION ?= 3.10
 DOCKER_COMPOSE ?= docker compose
 KARAPACE-CLI   ?= $(DOCKER_COMPOSE) -f container/compose.yml run --rm karapace-cli
 
@@ -105,6 +105,10 @@ schema:
 pin-requirements:
 	docker run -e CUSTOM_COMPILE_COMMAND='make pin-requirements' -it -v .:/karapace --security-opt label=disable python:$(PYTHON_VERSION)-bullseye /bin/bash -c "$(PIN_VERSIONS_COMMAND)"
 
+.PHONY: stop-karapace-docker-resources
+stop-karapace-docker-resources:
+	$(DOCKER_COMPOSE) -f container/compose.yml down -v --remove-orphans
+
 .PHONY: start-karapace-docker-resources
 start-karapace-docker-resources: export KARAPACE_VERSION ?= 4.1.1.dev44+gac20eeed.d20241205
 start-karapace-docker-resources:
@@ -121,15 +125,15 @@ unit-tests-in-docker: start-karapace-docker-resources
 
 .PHONY: e2e-tests-in-docker
 e2e-tests-in-docker: export PYTEST_ARGS ?=
-e2e-tests-in-docker: start-karapace-docker-resources
+e2e-tests-in-docker: stop-karapace-docker-resources start-karapace-docker-resources
 	rm -fr runtime/*
 	sleep 10
-	$(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/e2e/test_karapace.py
+	$(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/e2e/
 	rm -fr runtime/*
 
 .PHONY: integration-tests-in-docker
 integration-tests-in-docker: export PYTEST_ARGS ?=
-integration-tests-in-docker: start-karapace-docker-resources
+integration-tests-in-docker: stop-karapace-docker-resources start-karapace-docker-resources
 	rm -fr runtime/*
 	sleep 10
 	$(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/integration/

diff --git a/container/Dockerfile.dev b/container/Dockerfile.dev
@@ -1,5 +1,7 @@
+ARG PYTHON_VERSION
+
 # Current versions of avro and zstandard don't yet have wheels for 3.11.
-FROM python:3.10.11-bullseye AS builder
+FROM python:${PYTHON_VERSION}-bullseye AS builder
 
 ARG KARAPACE_VERSION
 ARG RUNNER_UID

diff --git a/container/compose.yml b/container/compose.yml
@@ -124,6 +124,7 @@ services:
       dockerfile: container/Dockerfile.dev
       args:
         KARAPACE_VERSION: $KARAPACE_VERSION
+        PYTHON_VERSION: $PYTHON_VERSION
         RUNNER_UID: $RUNNER_UID
         RUNNER_GID: $RUNNER_GID
     tty: true

diff --git a/mypy.ini b/mypy.ini
@@ -1,6 +1,6 @@
 [mypy]
 mypy_path = $MYPY_CONFIG_FILE_DIR/stubs
-python_version = 3.9
+python_version = 3.10
 packages = karapace
 show_error_codes = True
 pretty = True

diff --git a/pyproject.toml b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "karapace"
-requires-python = ">= 3.9"
+requires-python = ">= 3.10"
 dynamic = ["version"]
 readme = "README.rst"
 license = {file = "LICENSE"}
@@ -56,7 +56,6 @@ classifiers=[
   "Intended Audience :: Information Technology",
   "Intended Audience :: System Administrators",
   "License :: OSI Approved :: Apache Software License",
-  "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
@@ -114,5 +113,5 @@ include-package-data = true
 version_file = "src/karapace/version.py"
 
 [tool.black]
-target-version = ["py39"]
+target-version = ["py310", "py311", "py312"]
 line-length = 125
diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.9
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 #    make pin-requirements
@@ -119,9 +119,9 @@ httpcore==1.0.7
     # via httpx
 httptools==0.6.4
     # via uvicorn
-httpx==0.28.0
+httpx==0.28.1
     # via fastapi
-hypothesis==6.122.1
+hypothesis==6.122.3
     # via karapace (/karapace/pyproject.toml)
 idna==3.10
     # via
@@ -131,9 +131,7 @@ idna==3.10
     #   requests
     #   yarl
 importlib-metadata==8.5.0
-    # via
-    #   flask
-    #   opentelemetry-api
+    # via opentelemetry-api
 iniconfig==2.0.0
     # via pytest
 isodate==0.7.2
@@ -166,7 +164,7 @@ multidict==6.1.0
     # via
     #   aiohttp
     #   yarl
-networkx==3.2.1
+networkx==3.4.2
     # via karapace (/karapace/pyproject.toml)
 opentelemetry-api==1.28.2
     # via
@@ -314,7 +312,6 @@ typing-extensions==4.12.2
     #   pydantic
     #   pydantic-core
     #   rich-toolkit
-    #   starlette
     #   typer
     #   uvicorn
 ujson==5.10.0

diff --git a/requirements/requirements-typing.txt b/requirements/requirements-typing.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.9
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 #    make pin-requirements
@@ -78,7 +78,7 @@ httpcore==1.0.7
     # via httpx
 httptools==0.6.4
     # via uvicorn
-httpx==0.28.0
+httpx==0.28.1
     # via fastapi
 idna==3.10
     # via
@@ -112,7 +112,7 @@ mypy==1.13.0
     # via karapace (/karapace/pyproject.toml)
 mypy-extensions==1.0.0
     # via mypy
-networkx==3.2.1
+networkx==3.4.2
     # via karapace (/karapace/pyproject.toml)
 opentelemetry-api==1.28.2
     # via
@@ -211,7 +211,7 @@ typer==0.15.1
     # via fastapi-cli
 types-cachetools==5.5.0.20240820
     # via karapace (/karapace/pyproject.toml)
-types-jsonschema==4.23.0.20240813
+types-jsonschema==4.23.0.20241208
     # via karapace (/karapace/pyproject.toml)
 types-protobuf==3.20.4.6
     # via karapace (/karapace/pyproject.toml)
@@ -227,7 +227,6 @@ typing-extensions==4.12.2
     #   pydantic
     #   pydantic-core
     #   rich-toolkit
-    #   starlette
     #   typer
     #   uvicorn
 ujson==5.10.0

diff --git a/requirements/requirements.txt b/requirements/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.9
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 #    make pin-requirements
@@ -77,7 +77,7 @@ httpcore==1.0.7
     # via httpx
 httptools==0.6.4
     # via uvicorn
-httpx==0.28.0
+httpx==0.28.1
     # via fastapi
 idna==3.10
     # via
@@ -107,7 +107,7 @@ multidict==6.1.0
     # via
     #   aiohttp
     #   yarl
-networkx==3.2.1
+networkx==3.4.2
     # via karapace (/karapace/pyproject.toml)
 opentelemetry-api==1.28.2
     # via
@@ -210,7 +210,6 @@ typing-extensions==4.12.2
     #   pydantic
     #   pydantic-core
     #   rich-toolkit
-    #   starlette
     #   typer
     #   uvicorn
 ujson==5.10.0

diff --git a/website/README.rst b/website/README.rst
@@ -6,7 +6,7 @@ A static HTML site, generated with Sphinx. You can find the website source in th
 Dependencies
 ------------
 
-You need Python 3.9+. Install the dependencies with ``pip``::
+You need Python 3.10+. Install the dependencies with ``pip``::
 
     pip install -r requirements.txt