feat: e2e tests within docker

- we split out e2e tests into own folder - e2e tests will run against the service in docker compose - we start by splitting out the prometheus and kafka e2e tests which do not need local running services
Aiven-Open · Dec 4, 2024 · 6f968ae · 6f968ae
1 parent c36507e
commit 6f968ae
Show file tree

Hide file tree

Showing 65 changed files with 769 additions and 1,190 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -1,4 +1,10 @@
 [run]
-branch = true
-relative_files = true
-source = src/karapace
+branch = True
+relative_files = True
+source = karapace,schema_registry
+
+[report]
+skip_empty = True
+
+[combine]
+keep = True
diff --git a/.dockerignore b/.dockerignore
@@ -10,7 +10,6 @@
 !LICENSE
 !pyproject.toml
 !setup.py
-!container/start.sh
 !container/healthcheck.py
 
 # Ignore some files in source directories.

diff --git a/.github/workflows/container-smoke-test.yml b/.github/workflows/container-smoke-test.yml
@@ -32,8 +32,12 @@ jobs:
       - name: Run container
         run: docker compose --file=container/compose.yml up --build --wait --detach
 
-      - name: Smoke test registry
-        run: bin/smoke-test-registry.sh
+      - name: Smoke test schema registry
+        run: bin/smoke-test-schema-registry.sh
+        env:
+          KARAPACE_PORT: 8081
 
       - name: Smoke test REST proxy
-        run: bin/smoke-test-rest.sh
+        run: bin/smoke-test-rest-proxy.sh
+        env:
+          KARAPACE_PORT: 8082
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -14,14 +14,13 @@ env:
   FORCE_COLOR: 1
   PIP_PROGRESS_BAR: off
   PYTHONUNBUFFERED: 1
-  KARAPACE_DOTENV: ${{ github.workspace }}/karapace.config.env
 
 jobs:
-  unit-tests:
+  tests:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ '3.10', '3.11', '3.12' ]
+        python-version: [ '3.9', '3.10', '3.11', '3.12' ]
     env:
       PYTEST_ADDOPTS: >-
         --log-dir=/tmp/ci-logs
@@ -41,53 +40,34 @@ jobs:
       with:
         go-version: '1.21.0'
 
-    - name: Install requirements
-      run: make install
+    - name: Create virtual environment & install requirements
+      run: make install-dev
 
     - name: Resolve Karapace version
       run: |
         source ./venv/bin/activate
         KARAPACE_VERSION=$(python -c "from karapace import version; print(version.__version__)")
         echo KARAPACE_VERSION=$KARAPACE_VERSION >> $GITHUB_ENV
 
-    - name: Run containers
-      run: KARAPACE_VERSION=${{ env.KARAPACE_VERSION }} docker compose --file=container/compose.yml up --build --wait --detach
-
-    - run: make install-dev
     - run: make unit-tests-in-docker
       env:
-        COVERAGE_FILE: ".coverage.${{ matrix.python-version }}"
-        PYTEST_ARGS: "--cov=src --cov-append --numprocesses 4"
-        KARAPACE_VERSION=: ${{ env.KARAPACE_VERSION }}
+        KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
+        RUNNER_UID: 1001
+        COVERAGE_FILE: "/opt/karapace/.coverage.${{ matrix.python-version }}"
+        PYTEST_ARGS: "--cov=karapace --cov=schema_registry --cov-append --numprocesses 4"
 
-  integration-tests:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [ '3.10', '3.11', '3.12' ]
-    env:
-      PYTEST_ADDOPTS: >-
-        --log-dir=/tmp/ci-logs
-        --log-file=/tmp/ci-logs/pytest.log
-        --showlocals
-    steps:
-    - uses: actions/checkout@v4
-
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        cache: pip
-        python-version: ${{ matrix.python-version }}
-
-    - name: Set up Go
-      uses: actions/setup-go@v5
-      with:
-        go-version: '1.21.0'
+    - run: make e2e-tests-in-docker
+      env:
+        KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
+        RUNNER_UID: 1001
+        COVERAGE_FILE: "/opt/karapace/.coverage.${{ matrix.python-version }}"
+        PYTEST_ARGS: "--cov=karapace --cov=schema_registry --cov-append --numprocesses 4"
 
     - run: make integration-tests
       env:
+        KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
         COVERAGE_FILE: ".coverage.${{ matrix.python-version }}"
-        PYTEST_ARGS: "--cov=src --cov-append --random-order --numprocesses 4"
+        PYTEST_ARGS: "--cov=karapace --cov=schema_registry --cov-append --random-order --numprocesses 4"
 
     - name: Archive logs
       uses: actions/upload-artifact@v4

diff --git a/.gitignore b/.gitignore
@@ -10,12 +10,13 @@
 __pycache__/
 /build/
 /dist/
-/karapace.egg-info/
+src/karapace.egg-info/
 /karapace-rpm-src.tar
 /kafka_*.tgz
 /kafka_*/
 venv
-/karapace/version.py
+*.so
+src/karapace/version.py
 .run
 .python-version
 .hypothesis/

diff --git a/GNUmakefile b/GNUmakefile
@@ -5,7 +5,7 @@ PIP      ?= pip3 --disable-pip-version-check --no-input --require-virtualenv
 PYTHON   ?= python3
 PYTHON_VERSION ?= 3.10
 DOCKER_COMPOSE ?= docker compose
-KARAPACE-CLI   ?= $(DOCKER_COMPOSE) -f container/compose.yml run karapace-cli
+KARAPACE-CLI   ?= $(DOCKER_COMPOSE) -f container/compose.yml run --rm karapace-cli
 
 define PIN_VERSIONS_COMMAND
 pip install pip-tools && \
@@ -105,9 +105,42 @@ schema:
 pin-requirements:
 	docker run -e CUSTOM_COMPILE_COMMAND='make pin-requirements' -it -v .:/karapace --security-opt label=disable python:$(PYTHON_VERSION)-bullseye /bin/bash -c "$(PIN_VERSIONS_COMMAND)"
 
+.PHONY: start-karapace-docker-resources
+start-karapace-docker-resources:
+	sudo touch .coverage.3.9 .coverage.3.10 .coverage.3.11 .coverage.3.12
+	sudo chown ${RUNNER_UID} .coverage.3.9 .coverage.3.10 .coverage.3.11 .coverage.3.12
+	$(DOCKER_COMPOSE) -f container/compose.yml up -d --build --wait --detach
+	sleep 30
+
 .PHONY: unit-tests-in-docker
 unit-tests-in-docker: export PYTEST_ARGS ?=
-unit-tests-in-docker:
+unit-tests-in-docker: start-karapace-docker-resources
+	rm -fr runtime/*
+	$(KARAPACE-CLI) $(PYTHON) -m coverage run --append -m pytest -s -vvv tests/unit/
+	$(KARAPACE-CLI) $(PYTHON) -m coverage report
+	rm -fr runtime/*
+
+.PHONY: e2e-tests-in-docker
+e2e-tests-in-docker: export PYTEST_ARGS ?=
+e2e-tests-in-docker: start-karapace-docker-resources
+	rm -fr runtime/*
+	$(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/e2e/test_karapace.py
+	$(KARAPACE-CLI) $(PYTHON) -m coverage report
 	rm -fr runtime/*
-	$(KARAPACE-CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/unit/
+
+.PHONY: integration-tests-in-docker
+integration-tests-in-docker: export PYTEST_ARGS ?=
+integration-tests-in-docker: start-karapace-docker-resources
+	rm -fr runtime/*
+	$(KARAPACE-CLI) $(PYTHON) -m coverage run --append -m pytest -s -vvv $(PYTEST_ARGS) tests/integration/test_health_check.py
+	$(KARAPACE-CLI) $(PYTHON) -m coverage report
+	rm -fr runtime/*
+
+.PHONY: cli
+cli: export PYTEST_ARGS ?=
+cli: start-karapace-docker-resources
+	rm .coverage
+	touch .coverage
+	$(KARAPACE-CLI) $(PYTHON) -m coverage run --append -m pytest -s -vvv $(PYTEST_ARGS) tests/integration/test_health_check.py
+	$(KARAPACE-CLI) $(PYTHON) -m coverage report
 	rm -fr runtime/*
diff --git a/bin/smoke-test-rest.sh → bin/smoke-test-rest-proxy.sh b/bin/smoke-test-rest.sh → bin/smoke-test-rest-proxy.sh
@@ -3,7 +3,7 @@
 retries=5
 
 for ((i = 0; i <= retries; i++)); do
-    response=$(curl --silent --verbose --fail http://localhost:8082/topics)
+    response=$(curl --silent --verbose --fail "http://localhost:$KARAPACE_PORT/topics")
 
     if [[ $response == '["_schemas","__consumer_offsets"]' ]]; then
         echo "Ok!"

diff --git a/bin/smoke-test-registry.sh → bin/smoke-test-schema-registry.sh b/bin/smoke-test-registry.sh → bin/smoke-test-schema-registry.sh
@@ -8,7 +8,7 @@ for ((i = 0; i <= retries; i++)); do
             --header 'Content-Type: application/vnd.schemaregistry.v1+json' \
             --header 'Authorization: Basic Og==' \
             --data '{"schema": "{\"type\": \"record\", \"name\": \"Obj\", \"fields\":[{\"name\": \"age\", \"type\": \"int\"}]}"}' \
-            http://localhost:8081/subjects/test-key/versions
+            "http://localhost:$KARAPACE_PORT/subjects/test-key/versions"
     )
 
     if [[ $response == '{"id":1}' ]]; then

diff --git a/container/Dockerfile b/container/Dockerfile
@@ -55,10 +55,6 @@ RUN apt-get update \
 COPY --from=builder /venv /venv
 ENV PATH="/venv/bin:$PATH"
 
-COPY ./container/start.sh /opt/karapace
-RUN chmod 500 /opt/karapace/start.sh \
- && chown karapace:karapace /opt/karapace/start.sh
-
 COPY ./container/healthcheck.py /opt/karapace
 
 WORKDIR /opt/karapace

diff --git a/container/Dockerfile.dev b/container/Dockerfile.dev
@@ -41,9 +41,9 @@ FROM python:3.10.11-slim-bullseye AS karapace
 
 # Setup user and directories.
 RUN groupadd --system karapace \
- && useradd --system --gid karapace karapace \
- && mkdir /opt/karapace /opt/karapace/runtime /var/log/karapace \
- && chown --recursive karapace:karapace /opt/karapace /var/log/karapace
+&& useradd --system --gid karapace karapace \
+&& mkdir /opt/karapace /opt/karapace/runtime /var/log/karapace \
+&& chown --recursive karapace:karapace /opt/karapace /var/log/karapace
 
 # Install protobuf compiler.
 ARG PROTOBUF_COMPILER_VERSION="3.12.4-1+deb11u1"
@@ -52,6 +52,11 @@ RUN apt-get update \
     protobuf-compiler=$PROTOBUF_COMPILER_VERSION \
  && rm -rf /var/lib/apt/lists/*
 
+# Install Java via openjdk-11
+COPY --from=openjdk:11 /usr/local/openjdk-11 /usr/local/openjdk-11
+ENV JAVA_HOME /usr/local/openjdk-11
+RUN update-alternatives --install /usr/bin/java java /usr/local/openjdk-11/bin/java 1
+
 # Copy virtualenv from builder and activate it.
 COPY --from=builder /venv /venv
 ENV PATH="/venv/bin:$PATH"

diff --git a/container/compose.yml b/container/compose.yml
@@ -67,11 +67,22 @@ services:
       - kafka
     ports:
       - 8081:8081
-    volumes:
-    - ./karapace.registry.env:/opt/karapace/karapace.env
     environment:
-      KARAPACE_DOTENV: /opt/karapace/karapace.env
+      KARAPACE_KARAPACE_REGISTRY: true
+      KARAPACE_ADVERTISED_HOSTNAME: karapace-schema-registry
+      KARAPACE_BOOTSTRAP_URI: kafka:29092
       KARAPACE_PORT: 8081
+      KARAPACE_HOST: 0.0.0.0
+      KARAPACE_CLIENT_ID: karapace-schema-registry
+      KARAPACE_GROUP_ID: karapace-schema-registry
+      KARAPACE_MASTER_ELIGIBILITY: true
+      KARAPACE_TOPIC_NAME: _schemas
+      KARAPACE_LOG_LEVEL: DEBUG
+      KARAPACE_COMPATIBILITY: FULL
+      KARAPACE_STATSD_HOST: statsd-exporter
+      KARAPACE_STATSD_PORT: 8125
+      KARAPACE_KAFKA_SCHEMA_READER_STRICT_MODE: false
+      KARAPACE_KAFKA_RETRIABLE_ERRORS_SILENCED: true
 
   karapace-rest-proxy:
     image: ghcr.io/aiven-open/karapace:develop
@@ -87,11 +98,20 @@ services:
       - karapace-schema-registry
     ports:
       - 8082:8082
-    volumes:
-    - ./karapace.rest.env:/opt/karapace/karapace.env
     environment:
-      KARAPACE_DOTENV: /opt/karapace/karapace.env
+      KARAPACE_KARAPACE_REST: true
       KARAPACE_PORT: 8082
+      KARAPACE_HOST: 0.0.0.0
+      KARAPACE_ADVERTISED_HOSTNAME: karapace-rest-proxy
+      KARAPACE_BOOTSTRAP_URI: kafka:29092
+      KARAPACE_REGISTRY_HOST: karapace-schema-registry
+      KARAPACE_REGISTRY_PORT: 8081
+      KARAPACE_ADMIN_METADATA_MAX_AGE: 0
+      KARAPACE_LOG_LEVEL: DEBUG
+      KARAPACE_STATSD_HOST: statsd-exporter
+      KARAPACE_STATSD_PORT: 8125
+      KARAPACE_KAFKA_SCHEMA_READER_STRICT_MODE: false
+      KARAPACE_KAFKA_RETRIABLE_ERRORS_SILENCED: true
 
   karapace-cli:
     image: ghcr.io/aiven-open/karapace:cli
@@ -104,10 +124,15 @@ services:
       - karapace-schema-registry
       - karapace-rest-proxy
     volumes:
+    - ../.coverage.3.9:/opt/karapace/.coverage.3.9
+    - ../.coverage.3.10:/opt/karapace/.coverage.3.10
+    - ../.coverage.3.11:/opt/karapace/.coverage.3.11
+    - ../.coverage.3.12:/opt/karapace/.coverage.3.12
+    - ../.coveragerc:/opt/karapace/.coveragerc
     - ../tests:/opt/karapace/tests
-    - ../karapace.config.env:/opt/karapace/karapace.env
     environment:
-      KARAPACE_DOTENV: /opt/karapace/karapace.env
+      - COVERAGE_FILE
+      - COVERAGE_RCFILE=/opt/karapace/.coveragerc
 
   prometheus:
     image: prom/prometheus

diff --git a/container/karapace.registry.env b/container/karapace.registry.env