Merge pull request #579 from aiven/chore/optimize-container-build

Optimize container build time

.dockerignore

@@ -1,11 +1,19 @@
-# Ignoring files that are specific to a given checkout, these change based on
-# the user commands and not on the repository history. They are not important
-# to determine the state of the repository and would invalidate the cache
-# layer.
-#
-# - .git/logs/HEAD - command history
-# - .git/index - binary file for the current index, very important for a
-#    working repository, not interesting for our image
-#
-.git/logs/HEAD
-.git/index
+# Ignore everything by default. Making as few files as possible part of default context
+# ensures only relevant changes will evict layer cache.
+*
+
+# Include source directories and files required for building.
+!karapace
+!requirements/*.txt
+!setup.py
+!version.py
+!README.rst
+!container/start.sh
+!container/healthcheck.py
+
+# Ignore some files in source directories.
+**/.DS_Store
+**/Thumbs.db
+**/*.pyc
+**/*.pyo
+**/__pycache__

.github/workflows/container-smoke-test.yml

@@ -14,6 +14,9 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v3
 
+      - name: Build karapace/version.py
+        run: python version.py
+
       - name: Build and start services
         run: docker compose --file=container/compose.yml up --build --wait --detach
 

.pre-commit-config.yaml

@@ -66,6 +66,16 @@ repos:
   - id: mypy
     pass_filenames: false
 
+- repo: https://github.com/hadolint/hadolint
+  rev: v2.12.0
+  hooks:
+  - id: hadolint-docker
+    alias: hadolint
+    args:
+    # This rule has false positives when using a mounted cache volume.
+    # https://github.com/hadolint/hadolint/issues/497
+    - --ignore=DL3042
+
 - repo: https://github.com/PyCQA/pylint
   # Note: pre-commit autoupdate changes to an alpha version. Instead, manually find the
   # latest stable version here: https://github.com/pylint-dev/pylint/releases

container/Dockerfile

@@ -1,56 +1,50 @@
-# Builder image contains header files and additional dependencies necessary to
-# generate wheel files.
-FROM debian:stable-slim AS builder
+# Current versions of avro and zstandard don't yet have wheels for 3.11.
+FROM python:3.10.11-bullseye AS builder
 
 ARG KARAPACE_VERSION
 
-# Build dependencies that need to be installed:
-# - git: Used to install dependencies directly from their public repos (release
-#   not on PyPI).
-# - python3-devel: Python .h files, used to compile C extensions (e.g. multidict)
-#
-# Build dependencies that need to be installed because of `--no-install-recommends`:
-# - gcc: g++ and gcc to compile C extensions
-# - python3-wheel: Library to generate .whl files
-# - python3-setuptools: Packaging library
-#
-RUN apt-get update && \
-    apt-get -y install --no-install-recommends git python3-dev python3-pip python3-setuptools python3-wheel gcc && \
-    rm -rf /var/lib/apt/lists/*
+# Create, activate, and enforce usage of virtualenv.
+RUN python3 -m venv /venv
+ENV PATH="/venv/bin:$PATH"
+ENV PIP_REQUIRE_VIRTUALENV=true
 
-# Copy the requirements.txt and generate wheels for each dependency. Using a
-# separate command to use layer caching.
+# Copy the requirements.txt and install dependencies in venv. Using a separate
+# command to use layer caching.
 #
 # Note: the requirements.txt is pinned, if any of the dependencies is updated
 # the cache will be invalidated and the image regenerated, which is the
 # intended behavior.
-#
 COPY ./requirements/requirements.txt /build/
-RUN pip3 wheel --requirement /build/requirements.txt --wheel-dir /build/dependencies-wheels
+RUN --mount=type=cache,target=/root/.cache/pip \
+    python3 -m pip install -r /build/requirements.txt
 
 COPY . /build/karapace-repo
-RUN pip3 wheel --no-deps /build/karapace-repo --wheel-dir /build/karapace-wheel
-
-# Karapace image.
-FROM debian:stable-slim AS karapace
+RUN --mount=type=cache,target=/root/.cache/pip \
+    python3 -m pip install /build/karapace-repo
 
-RUN groupadd --system karapace && \
-    useradd --system --gid karapace karapace && \
-    mkdir /opt/karapace /opt/karapace/runtime /var/log/karapace && \
-    chown --recursive karapace:karapace /opt/karapace /var/log/karapace
+# Karapace image, i.e. production.
+FROM python:3.10.11-slim-bullseye AS karapace
 
-RUN apt-get update && \
-    apt-get -y install --no-install-recommends python3-pip protobuf-compiler && \
-    rm -rf /var/lib/apt/lists/*
+# Setup user and directories.
+RUN groupadd --system karapace \
+ && useradd --system --gid karapace karapace \
+ && mkdir /opt/karapace /opt/karapace/runtime /var/log/karapace \
+ && chown --recursive karapace:karapace /opt/karapace /var/log/karapace
 
-COPY --from=builder /build/dependencies-wheels/*.whl /build/dependencies-wheels/
-RUN pip3 install --no-deps /build/dependencies-wheels/*.whl && rm -rf /build/dependencies-wheels/
+# Install protobuf compiler.
+ARG PROTOBUF_COMPILER_VERSION="3.12.4-1"
+RUN apt-get update \
+ && apt-get install --assume-yes --no-install-recommends \
+    protobuf-compiler=$PROTOBUF_COMPILER_VERSION \
+ && rm -rf /var/lib/apt/lists/*
 
-COPY --from=builder /build/karapace-wheel/*.whl /build/karapace-wheel/
-RUN pip3 install --no-deps /build/karapace-wheel/*.whl && rm -rf /build/karapace-wheel/
+# Copy virtualenv from builder and activate it.
+COPY --from=builder /venv /venv
+ENV PATH="/venv/bin:$PATH"
 
 COPY ./container/start.sh /opt/karapace
-RUN chmod 500 /opt/karapace/start.sh && chown karapace:karapace /opt/karapace/start.sh
+RUN chmod 500 /opt/karapace/start.sh \
+ && chown karapace:karapace /opt/karapace/start.sh
 
 COPY ./container/healthcheck.py /opt/karapace
 

container/start.sh

@@ -35,7 +35,7 @@ registry)
     [[ -n ${KARAPACE_REGISTRY_PORT+isset} ]] && export KARAPACE_PORT="${KARAPACE_REGISTRY_PORT}"
     [[ -n ${KARAPACE_REGISTRY_CLIENT_ID+isset} ]] && export KARAPACE_CLIENT_ID="${KARAPACE_REGISTRY_CLIENT_ID}"
     [[ -n ${KARAPACE_REGISTRY_GROUP_ID+isset} ]] && export KARAPACE_GROUP_ID="${KARAPACE_REGISTRY_GROUP_ID}"
-    # Map misspelt environment variable to correct spelling for backwards compatibility.
+    # Map misspelled environment variables to correct spelling for backwards compatibility.
     [[ -n ${KARAPACE_REGISTRY_MASTER_ELIGIBITY+isset} ]] && export KARAPACE_MASTER_ELIGIBILITY="${KARAPACE_REGISTRY_MASTER_ELIGIBITY}"
     [[ -n ${KARAPACE_REGISTRY_MASTER_ELIGIBILITY+isset} ]] && export KARAPACE_MASTER_ELIGIBILITY="${KARAPACE_REGISTRY_MASTER_ELIGIBILITY}"
     [[ -n ${KARAPACE_REGISTRY_TOPIC_NAME+isset} ]] && export KARAPACE_TOPIC_NAME="${KARAPACE_REGISTRY_TOPIC_NAME}"

setup.py

@@ -13,7 +13,7 @@
 with open(readme_path, encoding="utf8") as fp:
     readme_text = fp.read()
 
-version_for_setup_py = version.get_project_version("karapace/version.py")
+version_for_setup_py = version.get_project_version()
 version_for_setup_py = ".dev".join(version_for_setup_py.split("-", 2)[:2])
 
 setup(

version.py

@@ -4,26 +4,35 @@
 Copyright (c) 2023 Aiven Ltd
 See LICENSE for details
 """
-import importlib.util
+from __future__ import annotations
+
+from typing import Final
+
 import os
+import pathlib
 import subprocess
 
+version_file: Final = pathlib.Path(__file__).parent.resolve() / "karapace/version.py"
+
 
-def save_version(new_ver, old_ver, version_file):
+def save_version(new_ver, old_ver):
     if not new_ver:
         return False
-    version_file = os.path.join(os.path.dirname(__file__), version_file)
     if not old_ver or new_ver != old_ver:
-        with open(version_file, mode="w", encoding="utf8") as fp:
-            fp.write(f'"""{__doc__}"""\n__version__ = "{new_ver}"\n')
+        version_file.write_text(f'"""{__doc__}"""\n__version__ = "{new_ver}"\n')
     return True
 
 
-def get_project_version(version_file: str) -> str:
-    version_file_full_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), version_file)
-    module_spec = importlib.util.spec_from_file_location("verfile", version_file_full_path)
-    module = importlib.util.module_from_spec(module_spec)
-    file_ver = getattr(module, "__version__", None)
+def from_version_file() -> str | None:
+    try:
+        import karapace.version
+    except ImportError:
+        return None
+    return karapace.version.__version__
+
+
+def get_project_version() -> str:
+    file_ver = from_version_file()
 
     version = os.getenv("KARAPACE_VERSION")
     if version is None:
@@ -40,16 +49,14 @@ def get_project_version(version_file: str) -> str:
                 git_ver = f"0.0.1-0-unknown-{git_ver}"
             version = git_ver
 
-    if save_version(version, file_ver, version_file):
+    if save_version(version, file_ver):
         return version
 
     if not file_ver:
-        raise RuntimeError(f"version not available from git or from file {version_file!r}")
+        raise RuntimeError(f"version not available from git or from file {str(version_file)!r}")
 
     return file_ver
 
 
 if __name__ == "__main__":
-    import sys
-
-    get_project_version(sys.argv[1])
+    get_project_version()