ci: migrate to a modern Python stack

.github/workflows/ci.yml

@@ -1,76 +1,116 @@
-name: Continous Integration (CI)
+name: Continuous Integration
 
 on:
   pull_request:
 
 jobs:
-  lint:
+  unit:
+    runs-on: ubuntu-latest
     strategy:
-      fail-fast: false
       matrix:
-        os: [ubuntu-latest]
-        architecture: ['x64']
-        python-version: ['3.8', '3.9', '3.10', '3.11']
-    runs-on: ${{ matrix.os }}
+        # TODO: Test Hatch matrices again.
+        # They could be as fast as concurrent GitHub jobs thanks to caching of dependencies and type check results.
+        # Then it is always possible to target a specific version of Python, for example with +py3.9 in the Dockerfile.
+        # Hatch matrices also vastly reduce the size of this workflow, with a single step for lint, test and audit.
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          architecture: ${{ matrix.architecture }}
-
-      - name: Install dependencies
-        run: make install
-
-      - name: Lint
-        run: make lint
+      - name: Set up pip cache
+        if: runner.os == 'Linux'
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }}
+          restore-keys: ${{ runner.os }}-pip-
+      - name: Install Hatch
+        run: pipx install hatch
+      - name: List Hatch environments
+        run: hatch env show --ascii
+      - name: Run unit tests with coverage
+        run: hatch run test:unit-cov
+        env:
+          MIN_VALID_EVENTS: 10
+          GOOGLE_APPLICATION_CREDENTIALS: tests/unit/fixtures/fake_credentials.json
 
-  unit:
+  lint:
+    runs-on: ubuntu-latest
     strategy:
-      fail-fast: false
       matrix:
-        os: [ubuntu-latest]
-        architecture: ['x64']
-        python-version: ['3.8', '3.9', '3.10', '3.11']
-    runs-on: ${{ matrix.os }}
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          architecture: ${{ matrix.architecture }}
-
-      - name: Install dependencies
-        run: make install
-
-      - name: Run unit tests
-        run: make unit
-        env:
-          MIN_VALID_EVENTS: "10"
-          GOOGLE_APPLICATION_CREDENTIALS: tests/unit/fixtures/fake_credentials.json
-
-      - name: Run coverage report
-        run: make coverage
+      - name: Set up pip cache
+        if: runner.os == 'Linux'
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }}
+          restore-keys: ${{ runner.os }}-pip-
+      - name: Install Hatch
+        run: pipx install hatch
+      - name: List Hatch environments
+        run: hatch env show --ascii
+      - name: Lint
+        run: hatch run lint:all
 
   audit:
     runs-on: ubuntu-latest
+    # TODO: Not sure `audit` is required to run on multiple versions of Python.
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.9"  # same as Dockerfile
-
-      - name: Install dependencies
-        run: make install
-
-      - name: Audit for CVEs
-        run: make audit
+          python-version: ${{ matrix.python-version }}
+      - name: Set up pip cache
+        if: runner.os == 'Linux'
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }}
+          restore-keys: ${{ runner.os }}-pip-
+      - name: Install Hatch
+        run: pipx install hatch
+      - name: List Hatch environments
+        run: hatch env show --ascii
+      - name: Audit
+        run: hatch run audit:all
 
-  docker:
+  build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
     steps:
       - uses: actions/checkout@v4
-      - uses: docker-practice/actions-setup-docker@master
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Set up pip cache
+        if: runner.os == 'Linux'
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }}
+          restore-keys: ${{ runner.os }}-pip-
+      - name: Install Hatch
+        run: pipx install hatch
+      - name: List Hatch environments
+        run: hatch env show --ascii
+      - name: Build
+        run: hatch build
 
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: docker/build-push-action@v5
       - name: Build Docker image
         run: make docker_build

.gitignore

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# FIXME: Clean up unused entries and sort by categories
 __pycache__/
 *.py[cod]
 *$py.class
@@ -50,4 +51,4 @@ reports/
 .pytest_cache/
 .pytype/
 .pip-wheel-metadata/
-.ruff_cache
+.ruff_cache/

.pre-commit-config.yaml

@@ -22,3 +22,7 @@ repos:
     - id: ruff
     # Run the formatter.
     - id: ruff-format
+- repo: https://github.com/PyCQA/bandit
+  rev: 1.7.8
+  hooks:
+    - id: bandit

Makefile

@@ -1,4 +1,4 @@
-#!/usr/bin/make
+#!/usr/bin/env make
 # WARN: gmake syntax
 ########################################################
 # Makefile for $(NAME)
@@ -12,25 +12,29 @@
 #	make audit -- run CVE scan separately
 #	make integration -- run integration tests
 ########################################################
-# variable section
+
+# Variables
 
 NAME = slo_generator
 
+HATCH = hatch
 PIP = pip3
 PYTHON = python3
-TWINE = twine
 COVERAGE = coverage
-SITELIB = $(shell $(PYTHON) -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")
 
-VERSION ?= $(shell grep "version = " setup.cfg | cut -d ' ' -f 3)
+VERSION ?= $(shell grep "version = " pyproject.toml | cut -d ' ' -f 3)
 
 ########################################################
 
 all: clean install test
 
+##############
+
 info:
 	@echo "slo-generator version: ${VERSION}"
 
+##############
+
 clean:
 	@echo "Cleaning up distutils stuff"
 	rm -rf build
@@ -46,46 +50,64 @@ clean:
 	@echo "Cleaning up test reports"
 	rm -rf report/*
 
+##############
+
 build: clean
-	$(PYTHON) setup.py sdist bdist_wheel
+	$(HATCH) build
 
-deploy: clean install_twine build
-	$(TWINE) upload dist/*
+deploy: build
+	$(HATCH) publish
 
-install_twine:
-	$(PIP) install twine
-
-develop: install
-	pre-commit install
+##############
 
+#TODO: Replace with Hatch.
 install: clean
+#FIXME: Are all dependencies and features requested for all these targets?
 	$(PIP) install -e ."[api, datadog, prometheus, elasticsearch, opensearch, splunk, pubsub, cloud_monitoring, bigquery, dev]"
 
 uninstall: clean
-	$(PIP) freeze --exclude-editable | xargs $(PIP) uninstall -y
+	$(HATCH) env prune
+
+##############
+
+#TODO: How to handle pre-commit with Hatch environments? 
+develop: install
+	pre-commit install
 
+##############
+
+format:
+	$(HATCH) fmt
+
+##############
+
+#FIXME: Are all dependencies and features requested for all these targets?
 test: install unit lint audit
 
+##############
+
 unit: clean
-	pytest --cov=$(NAME) tests -p no:warnings
+	$(HATCH) run test:unit
 
 coverage:
-	$(COVERAGE) report --rcfile=".coveragerc"
+	$(HATCH) run test:cov
 
-format:
-	ruff format
+##############
 
 lint: ruff pytype mypy
 
 ruff:
-	ruff format --check
-	ruff check
+	$(HATCH) fmt
 
 pytype:
-	pytype
+	$(HATCH) run lint:pytype
 
 mypy:
-	mypy --show-error-codes $(NAME)
+	$(HATCH) run lint:mypy
+
+##############
+
+audit: bandit safety
 
 audit: bandit safety
 
@@ -97,6 +119,8 @@ safety:
 	# We do not use the `--extra-index-url` option, and the behavior is intended anyway.
 	safety check --ignore 67599
 
+##############
+
 integration: int_cm int_csm int_custom int_dd int_dt int_es int_prom int_sp int_os
 
 int_cm:
@@ -126,6 +150,8 @@ int_sp:
 int_os:
 	slo-generator compute -f samples/opensearch -c samples/config.yaml
 
+##############
+
 # Run API locally
 run_api:
 	slo-generator api --target=run_compute --signature-type=http -c samples/config.yaml