From 517fcf8581079f7c66f946687e04a3f32c593dfa Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Sat, 11 Jun 2022 15:43:28 -0400 Subject: [PATCH 1/9] Enable BuildKit in GitHub Actions https://github.com/moby/buildkit/blob/HEAD/frontend/dockerfile/docs/syntax.md --- .github/workflows/builds.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 2c785d7..747ff07 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -9,6 +9,7 @@ on: workflow_dispatch: env: + DOCKER_BUILDKIT: "1" PIPX_VERSION: "1.1.0" POETRY_VERSION: "1.1.11" From 64fc4a815dc6a457bae148397f8a1eef4a25c21b Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Sat, 11 Jun 2022 16:01:29 -0400 Subject: [PATCH 2/9] Add Docker BuildKit info to CONTRIBUTING.md --- docs/contributing.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/contributing.md b/docs/contributing.md index ec13d0a..31a1cd5 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -167,6 +167,8 @@ docker cp [container_name]:/path/to/file destination.file ### Building development images +Note that Docker builds use BuildKit. See the [BuildKit docs](https://github.com/moby/buildkit/blob/HEAD/frontend/dockerfile/docs/syntax.md) and [Docker docs](https://docs.docker.com/develop/develop-images/build_enhancements/). + To build the Docker images for each stage: ```sh @@ -174,6 +176,8 @@ git clone git@github.com:br3ndonland/inboard.git cd inboard +export DOCKER_BUILDKIT=1 + docker build . --rm --target base -t localhost/br3ndonland/inboard:base && \ docker build . --rm --target fastapi -t localhost/br3ndonland/inboard:fastapi && \ docker build . --rm --target starlette -t localhost/br3ndonland/inboard:starlette From cd22c8dee2c7133440571873bd43b5a6d6bb0c82 Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Sat, 11 Jun 2022 15:45:11 -0400 Subject: [PATCH 3/9] Refactor Dockerfile with heredoc syntax https://github.com/moby/buildkit/blob/HEAD/frontend/dockerfile/docs/syntax.md https://docs.docker.com/develop/develop-images/build_enhancements/ --- Dockerfile | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index ae2cdb5..420edb2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1.4 ARG PYTHON_VERSION=3.10 LINUX_VERSION= FROM python:${PYTHON_VERSION}${LINUX_VERSION:+-$LINUX_VERSION} AS base LABEL org.opencontainers.image.authors="Brendon Smith " @@ -10,11 +11,31 @@ ARG LINUX_VERSION PIPX_VERSION=1.1.0 POETRY_VERSION=1.1.11 ENV APP_MODULE=inboard.app.main_base:app LINUX_VERSION=$LINUX_VERSION PATH=/opt/pipx/bin:/app/.venv/bin:$PATH PIPX_BIN_DIR=/opt/pipx/bin PIPX_HOME=/opt/pipx/home PIPX_VERSION=$PIPX_VERSION POETRY_VERSION=$POETRY_VERSION PYTHONPATH=/app COPY poetry.lock poetry.toml pyproject.toml /app/ WORKDIR /app -RUN sh -c 'if [ "$LINUX_VERSION" = "slim" ]; then apt-get update -qy && apt-get install -qy --no-install-recommends gcc libc-dev make wget; fi' && \ - sh -c '. /etc/os-release; if [ "$ID" = "alpine" ]; then apk add --no-cache --virtual .build-deps gcc libc-dev libffi-dev make openssl-dev; fi' && \ - python -m pip install --no-cache-dir --upgrade pip "pipx==$PIPX_VERSION" && pipx install "poetry==$POETRY_VERSION" && poetry install --no-dev --no-interaction --no-root && \ - sh -c 'if [ "$LINUX_VERSION" = "slim" ]; then apt-get purge --auto-remove -qy gcc libc-dev make wget; fi' && \ - sh -c '. /etc/os-release; if [ "$ID" = "alpine" ]; then apk del .build-deps; fi' +RUN < Date: Thu, 7 Jul 2022 16:13:41 -0400 Subject: [PATCH 4/9] Break heredoc across multiple stages Previously, build-time dependencies for building Python packages with binary extensions were installed and uninstalled in one `RUN` command, as they were only needed to install some dependencies on Alpine Linux. However, this also meant that build-time dependencies were not available to later installation steps, such as installing FastAPI dependencies. pydantic has binary extensions, and although it now provides wheels with the `musllinux` tag (for Alpine), inboard should account for the possibility that packages like pydantic might need to build from source. This commit will refactor build-time dependency installation into two steps. The dependencies will be installed in the new `builder` stage, then uninstalled at the end of each subsequent stage. --- Dockerfile | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 420edb2..2cfe842 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1.4 ARG PYTHON_VERSION=3.10 LINUX_VERSION= -FROM python:${PYTHON_VERSION}${LINUX_VERSION:+-$LINUX_VERSION} AS base +FROM python:${PYTHON_VERSION}${LINUX_VERSION:+-$LINUX_VERSION} AS builder LABEL org.opencontainers.image.authors="Brendon Smith " LABEL org.opencontainers.image.description="Docker images and utilities to power your Python APIs and help you ship faster." LABEL org.opencontainers.image.licenses="MIT" @@ -12,9 +12,7 @@ ENV APP_MODULE=inboard.app.main_base:app LINUX_VERSION=$LINUX_VERSION PATH=/opt/ COPY poetry.lock poetry.toml pyproject.toml /app/ WORKDIR /app RUN < Date: Thu, 7 Jul 2022 17:02:49 -0400 Subject: [PATCH 5/9] Add heredoc examples and info to docs This commit will refactor the Dockerfile examples in the docs to use heredoc syntax, and will also add a tip explaining the syntax. --- docs/docker.md | 56 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/docs/docker.md b/docs/docker.md index c1922b6..d80fd18 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -241,14 +241,29 @@ The basic build dependencies used by inboard include `gcc`, `libc-dev`, and `mak !!! example "Example Alpine Linux _Dockerfile_ for PostgreSQL project" ```dockerfile + # syntax=docker/dockerfile:1.4 ARG INBOARD_DOCKER_TAG=fastapi-alpine FROM ghcr.io/br3ndonland/inboard:${INBOARD_DOCKER_TAG} ENV APP_MODULE=mypackage.main:app COPY poetry.lock pyproject.toml /app/ WORKDIR /app - RUN sh -c '. /etc/os-release; if [ "$ID" = "alpine" ]; then apk add --no-cache --virtual .build-project build-base freetype-dev gcc libc-dev libpng-dev make openblas-dev postgresql-dev; fi' && \ - poetry install --no-dev --no-interaction --no-root && \ - sh -c '. /etc/os-release; if [ "$ID" = "alpine" ]; then apk del .build-project && apk add --no-cache libpq; fi' + RUN < Date: Thu, 7 Jul 2022 17:08:11 -0400 Subject: [PATCH 6/9] Use `COPY --link` to improve layer caching This commit will add `--link` to `COPY` commands in the Dockerfile. This allows re-use of these layers, even when layers built previously change. https://github.com/moby/buildkit/blob/HEAD/frontend/dockerfile/docs/syntax.md --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2cfe842..b53ad6e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ LABEL org.opencontainers.image.title="inboard" LABEL org.opencontainers.image.url="https://github.com/br3ndonland/inboard/pkgs/container/inboard" ARG LINUX_VERSION PIPX_VERSION=1.1.0 POETRY_VERSION=1.1.11 ENV APP_MODULE=inboard.app.main_base:app LINUX_VERSION=$LINUX_VERSION PATH=/opt/pipx/bin:/app/.venv/bin:$PATH PIPX_BIN_DIR=/opt/pipx/bin PIPX_HOME=/opt/pipx/home PIPX_VERSION=$PIPX_VERSION POETRY_VERSION=$POETRY_VERSION PYTHONPATH=/app -COPY poetry.lock poetry.toml pyproject.toml /app/ +COPY --link poetry.lock poetry.toml pyproject.toml /app/ WORKDIR /app RUN < Date: Fri, 8 Jul 2022 13:15:49 -0400 Subject: [PATCH 7/9] Add `BUILDKIT_INLINE_CACHE` Docker build argument The `BUILDKIT_INLINE_CACHE` build argument tells Docker to write cache metadata into the image during `docker build`. The image can then be used as a cache source for subsequent builds. The build argument will be provided with the `docker build` command. The build argument could be provided directly within the Dockerfile, but BuildKit does not offer any guidance on where it goes. It could either be supplied before the first `FROM`, or after the first `FROM`, and could work differently in each case. https://docs.docker.com/engine/reference/commandline/build/#specifying-external-cache-sources --- .github/workflows/builds.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 747ff07..cc5e6d1 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -110,6 +110,7 @@ jobs: - name: Build Docker images run: | docker build . --rm --target base \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ --build-arg LINUX_VERSION="$LINUX_VERSION" \ --build-arg PIPX_VERSION="$PIPX_VERSION" \ --build-arg POETRY_VERSION="$POETRY_VERSION" \ @@ -117,12 +118,14 @@ jobs: --cache-from python:"$PYTHON_VERSION$LINUX_TAG" \ -t ghcr.io/br3ndonland/inboard:base"$LINUX_TAG" docker build . --rm --target starlette \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ --build-arg LINUX_VERSION="$LINUX_VERSION" \ --build-arg PIPX_VERSION="$PIPX_VERSION" \ --build-arg POETRY_VERSION="$POETRY_VERSION" \ --build-arg PYTHON_VERSION="$PYTHON_VERSION" \ -t ghcr.io/br3ndonland/inboard:starlette"$LINUX_TAG" docker build . --rm --target fastapi \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ --build-arg LINUX_VERSION="$LINUX_VERSION" \ --build-arg PIPX_VERSION="$PIPX_VERSION" \ --build-arg POETRY_VERSION="$POETRY_VERSION" \ From 8cc569f3502de1e76ec6dc10e51d25f905e66f62 Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Sat, 9 Jul 2022 13:58:25 -0400 Subject: [PATCH 8/9] Update `docker build --cache-from` for inboard `--cache-from` allows build commands to specify external cache sources. The Docker build was previously caching from the official Python image. With the BuildKit inline cache metadata provided by images built with `BUILDKIT_INLINE_CACHE=1`, layers from inboard can now also be cached. The inboard caching may not kick in until after this PR is merged, because images must first be pushed to a registry before cache metadata can be read. The standard caching mechanism may not be effective for the multi-stage builds used here, so `--cache-from` may not end up reducing build times. https://docs.docker.com/engine/reference/commandline/build/#specifying-external-cache-sources --- .github/workflows/builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index cc5e6d1..0d28923 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -115,7 +115,7 @@ jobs: --build-arg PIPX_VERSION="$PIPX_VERSION" \ --build-arg POETRY_VERSION="$POETRY_VERSION" \ --build-arg PYTHON_VERSION="$PYTHON_VERSION" \ - --cache-from python:"$PYTHON_VERSION$LINUX_TAG" \ + --cache-from ghcr.io/br3ndonland/inboard \ -t ghcr.io/br3ndonland/inboard:base"$LINUX_TAG" docker build . --rm --target starlette \ --build-arg BUILDKIT_INLINE_CACHE=1 \ From 79ce14eee673970701c760a2ed25c2985bddcdd7 Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Sat, 9 Jul 2022 14:00:04 -0400 Subject: [PATCH 9/9] Relax Dockerfile syntax version specifier Now that Dockerfile 1.4 is in stable release, this commit will relax the Dockerfile version from 1.4 to 1, to allow further 1.x releases. https://github.com/moby/buildkit/blob/f4eb826799e53547c793bfa83a035b8e24a2b88d/frontend/dockerfile/docs/reference.md --- Dockerfile | 2 +- docs/docker.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index b53ad6e..66a5709 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:1.4 +# syntax=docker/dockerfile:1 ARG PYTHON_VERSION=3.10 LINUX_VERSION= FROM python:${PYTHON_VERSION}${LINUX_VERSION:+-$LINUX_VERSION} AS builder LABEL org.opencontainers.image.authors="Brendon Smith " diff --git a/docs/docker.md b/docs/docker.md index d80fd18..68a3ea0 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -241,7 +241,7 @@ The basic build dependencies used by inboard include `gcc`, `libc-dev`, and `mak !!! example "Example Alpine Linux _Dockerfile_ for PostgreSQL project" ```dockerfile - # syntax=docker/dockerfile:1.4 + # syntax=docker/dockerfile:1 ARG INBOARD_DOCKER_TAG=fastapi-alpine FROM ghcr.io/br3ndonland/inboard:${INBOARD_DOCKER_TAG} ENV APP_MODULE=mypackage.main:app @@ -292,7 +292,7 @@ A _Dockerfile_ equivalent to the Alpine Linux example might look like the follow !!! example "Example Debian Linux slim _Dockerfile_ for PostgreSQL project" ```dockerfile - # syntax=docker/dockerfile:1.4 + # syntax=docker/dockerfile:1 ARG INBOARD_DOCKER_TAG=fastapi-slim FROM ghcr.io/br3ndonland/inboard:${INBOARD_DOCKER_TAG} ENV APP_MODULE=mypackage.main:app