Refactoring multiple parts of the dockerfile (pypi#10004)

* Remove the Dockerfile.static, replace with a --target

* Use APT cache

* Headers

* Switch to set -eux

* Add things to dockerignore

* Add lockfile for pip & the likes

* Add pip req alias files

* Improve APT cache

* Reorgnize apt deps

* Make dev image, use pip req aliases, and envvars

* Add base image to limit repetition

* docker-compose use dev images

* docker-compose use simple python images when possible

* Dev is not pinned, must be installed separately

* Remove unused DEVEL references

* Build static image first, though it's not used by the dev image

* Reorganize dev

Co-authored-by: Ee Durbin <[email protected]>

.dockerignore

@@ -1,6 +1,10 @@
-.git/*
+.git
 node_modules
-dev/*
+dev
 **/*.pyc
 htmlcov
 warehouse/static/dist
+.mypy_cache
+.state
+tests
+.github

Dockerfile

@@ -1,18 +1,28 @@
+# ---------------------------------- STATIC ----------------------------------
+
 # First things first, we build an image which is where we're going to compile
-# our static assets with. It is important that the steps in this remain the
-# same as the steps in Dockerfile.static, EXCEPT this may include additional
-# steps appended onto the end.
+# our static assets with.
 FROM node:14.4.0 as static
 
 WORKDIR /opt/warehouse/src/
 
+# By default, Docker has special steps to avoid keeping APT caches in the layers, which
+# is good, but in our case, we're going to mount a special cache volume (kept between
+# builds), so we WANT the cache to persist.
+RUN set -eux; \
+    rm -f /etc/apt/apt.conf.d/docker-clean; \
+    echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache;
+
 # The list of C packages we need are almost never going to change, so installing
 # them first, right off the bat lets us cache that and having node.js level
 # dependency changes not trigger a reinstall.
-RUN set -x \
-    && apt-get update \
-    && apt-get install --no-install-recommends -y \
-        libjpeg-dev nasm
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    set -eux; \
+    apt-get update; \
+    apt-get install --no-install-recommends -y \
+        libjpeg-dev \
+        nasm
 
 # However, we do want to trigger a reinstall of our node.js dependencies anytime
 # our package.json changes, so we'll ensure that we're copying that into our
@@ -22,10 +32,10 @@ COPY package.json package-lock.json .babelrc /opt/warehouse/src/
 # Installing npm dependencies is done as a distinct step and *prior* to copying
 # over our static files so that, you guessed it, we don't invalidate the cache
 # of installed dependencies just because files have been modified.
-RUN set -x \
-    && npm install -g npm@latest \
-    && npm install -g gulp-cli \
-    && npm ci
+RUN set -eux \
+    npm install -g npm@latest; \
+    npm install -g gulp-cli; \
+    npm ci;
 
 # Actually copy over our static files, we only copy over the static files to
 # save a small amount of space in our image and because we don't need them. We
@@ -39,109 +49,124 @@ COPY Gulpfile.babel.js /opt/warehouse/src/
 RUN gulp dist
 
 
+# ---------------------------------- BASE -----------------------------------
+FROM python:3.8.2-slim-buster as base
+
+# Setup some basic environment variables that are ~never going to change.
+ENV PYTHONUNBUFFERED 1
+ENV PYTHONPATH /opt/warehouse/src/
+ENV PATH="/opt/warehouse/bin:${PATH}"
 
+WORKDIR /opt/warehouse/src/
 
+# By default, Docker has special steps to avoid keeping APT caches in the layers, which
+# is good, but in our case, we're going to mount a special cache volume (kept between
+# builds), so we WANT the cache to persist.
+RUN set -eux \
+    rm -f /etc/apt/apt.conf.d/docker-clean; \
+    echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
 
+# Install System level Warehouse requirements, this is done before everything
+# else because these are rarely ever going to change.
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    set -eux; \
+    apt-get update; \
+    apt-get install --no-install-recommends -y \
+        libpq5 \
+        libxml2 \
+        libxslt1.1 \
+        libcurl4 \
+    ;
+
+# ---------------------------------- BUILD ----------------------------------
 
 # Now we're going to build our actual application, but not the actual production
 # image that it gets deployed into.
-FROM python:3.8.2-slim-buster as build
-
-# Define whether we're building a production or a development image. This will
-# generally be used to control whether or not we install our development and
-# test dependencies.
-ARG DEVEL=no
-
-# To enable Ipython in the development environment set to yes (for using ipython
-# as the warehouse shell interpreter,
-# i.e. 'docker-compose run --rm web python -m warehouse shell --type=ipython')
-ARG IPYTHON=no
+FROM base as build
 
 # Install System level Warehouse build requirements, this is done before
 # everything else because these are rarely ever going to change.
-RUN set -x \
-    && apt-get update \
-    && apt-get install --no-install-recommends -y \
-        build-essential libffi-dev libxml2-dev libxslt-dev libpq-dev libcurl4-openssl-dev libssl-dev \
-        $(if [ "$DEVEL" = "yes" ]; then echo 'libjpeg-dev'; fi)
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    set -eux; \
+    apt-get update; \
+    apt-get install --no-install-recommends -y \
+        build-essential \
+        libcurl4-openssl-dev \
+        libffi-dev \
+        libpq-dev \
+        libssl-dev \
+        libxml2-dev \
+        libxslt-dev \
+    ;
 
 # We create an /opt directory with a virtual environment in it to store our
 # application in.
-RUN set -x \
-    && python3 -m venv /opt/warehouse
-
+RUN python3 -m venv /opt/warehouse
 
-# Now that we've created our virtual environment, we'll go ahead and update
-# our $PATH to refer to it first.
-ENV PATH="/opt/warehouse/bin:${PATH}"
-
-# Next, we want to update pip, setuptools, and wheel inside of this virtual
-# environment to ensure that we have the latest versions of them.
-# TODO: We use --require-hashes in our requirements files, but not here, making
-#       the ones in the requirements files kind of a moot point. We should
-#       probably pin these too, and update them as we do anything else.
-RUN pip --no-cache-dir --disable-pip-version-check install --upgrade pip setuptools wheel
+# Pip configuration (https://github.com/pypa/warehouse/pull/4584)
+ENV PIP_NO_BINARY=hiredis PIP_DISABLE_PIP_VERSION_CHECK=1
 
 # We copy this into the docker container prior to copying in the rest of our
 # application so that we can skip installing requirements if the only thing
 # that has changed is the Warehouse code itself.
 COPY requirements /tmp/requirements
 
-# Install our development dependencies if we're building a development install
-# otherwise this will do nothing.
-RUN set -x \
-    && if [ "$DEVEL" = "yes" ]; then pip --no-cache-dir --disable-pip-version-check install -r /tmp/requirements/dev.txt; fi
-
-RUN set -x \
-    && if [ "$DEVEL" = "yes" ] && [ "$IPYTHON" = "yes" ]; then pip --no-cache-dir --disable-pip-version-check install -r /tmp/requirements/ipython.txt; fi
+# Next, we want to update pip, setuptools, and wheel inside of this virtual
+# environment to ensure that we have the latest versions of them.
+RUN --mount=type=cache,target=/root/.cache \
+    pip install -r /tmp/requirements/pip.txt
 
 # Install the Python level Warehouse requirements, this is done after copying
 # the requirements but prior to copying Warehouse itself into the container so
 # that code changes don't require triggering an entire install of all of
 # Warehouse's dependencies.
-RUN set -x \
-    && pip --no-cache-dir --disable-pip-version-check \
-            install --no-binary hiredis \
-                    -r /tmp/requirements/deploy.txt \
-                    -r /tmp/requirements/main.txt \
-                    $(if [ "$DEVEL" = "yes" ]; then echo '-r /tmp/requirements/tests.txt -r /tmp/requirements/lint.txt'; fi) \
-    && find /opt/warehouse -name '*.pyc' -delete
-
-
+RUN --mount=type=cache,target=/root/.cache \
+    set -eux; \
+    pip install -r /tmp/requirements/all-base.txt; \
+    find /opt/warehouse -name '*.pyc' -delete;
 
+# ---------------------------------- DEV ----------------------------------
 
+FROM build as dev
 
-# Now we're going to build our actual application image, which will eventually
-# pull in the static files that were built above.
-FROM python:3.8.2-slim-buster
-
-# Setup some basic environment variables that are ~never going to change.
-ENV PYTHONUNBUFFERED 1
-ENV PYTHONPATH /opt/warehouse/src/
-ENV PATH="/opt/warehouse/bin:${PATH}"
-
-WORKDIR /opt/warehouse/src/
-
-# Define whether we're building a production or a development image. This will
-# generally be used to control whether or not we install our development and
-# test dependencies.
-ARG DEVEL=no
+# To enable Ipython in the development environment set to yes (for using ipython
+# as the warehouse shell interpreter,
+# i.e. 'docker-compose run --rm web python -m warehouse shell --type=ipython')
+ARG IPYTHON=no
 
 # This is a work around because otherwise postgresql-client bombs out trying
 # to create symlinks to these directories.
-RUN set -x \
-    && mkdir -p /usr/share/man/man1 \
-    && mkdir -p /usr/share/man/man7
+RUN set -eux; \
+    mkdir -p /usr/share/man/man1; \
+    mkdir -p /usr/share/man/man7
 
-# Install System level Warehouse requirements, this is done before everything
-# else because these are rarely ever going to change.
-RUN set -x \
-    && apt-get update \
-    && apt-get install --no-install-recommends -y \
-        libpq5 libxml2 libxslt1.1 libcurl4  \
-        $(if [ "$DEVEL" = "yes" ]; then echo 'bash libjpeg62 postgresql-client'; fi) \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+# Install System level Warehouse build requirements, this is done before
+# everything else because these are rarely ever going to change.
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    set -eux; \
+    apt-get update; \
+    apt-get install --no-install-recommends -y \
+        bash \
+        libjpeg-dev \
+        libjpeg62 \
+        postgresql-client \
+    ;
+
+# Install our development dependencies
+RUN set -eux; \
+    pip install -r /tmp/requirements/dev.txt; \
+    if [ "$IPYTHON" = "yes" ]; then pip install -r /tmp/requirements/all-ipython.txt; fi;
+
+RUN pip install -r /tmp/requirements/all-lint-test.txt;
+
+
+# ---------------------------------- APP ----------------------------------
+FROM base as app
+# Now we're going to build our actual application image, which will eventually
+# pull in the static files that were built above.
 
 # Copy the directory into the container, this is done last so that changes to
 # Warehouse itself require the least amount of layers being invalidated from

Makefile

@@ -64,9 +64,9 @@ endif
 
 .state/docker-build: Dockerfile package.json package-lock.json requirements/main.txt requirements/deploy.txt
 	# Build our docker containers for this project.
+	docker-compose build --force-rm static
 	docker-compose build --build-arg IPYTHON=$(IPYTHON) --force-rm web
 	docker-compose build --force-rm worker
-	docker-compose build --force-rm static
 
 	# Mark the state so we don't rebuild this needlessly.
 	mkdir -p .state

docker-compose.yml

@@ -61,8 +61,8 @@ services:
   web:
     build:
       context: .
+      target: dev
       args:
-        DEVEL: "yes"
         IPYTHON: "no"
     command: gunicorn --reload -b 0.0.0.0:8000 warehouse.wsgi:application
     env_file: dev/environment
@@ -88,8 +88,7 @@ services:
       - "80:8000"
 
   files:
-    build:
-      context: .
+    image: python:3.8.2-slim-buster
     working_dir: /var/opt/warehouse
     command: python -m http.server 9001
     volumes:
@@ -101,8 +100,7 @@ services:
   worker:
     build:
       context: .
-      args:
-        DEVEL: "yes"
+      target: dev
     command: hupper -m celery -A warehouse worker -B -S redbeat.RedBeatScheduler -l info
     volumes:
       - ./warehouse:/opt/warehouse/src/warehouse:z
@@ -114,7 +112,7 @@ services:
   static:
     build:
       context: .
-      dockerfile: Dockerfile.static
+      target: static
     command: bash -c "node --trace-warnings `which gulp` watch"
     volumes:
       - ./warehouse:/opt/warehouse/src/warehouse:z
@@ -130,8 +128,7 @@ services:
       - "1080:80"
 
   notdatadog:
-    build:
-      context: .
+    image: python:3.8.2-slim-buster
     command: python /opt/warehouse/dev/notdatadog.py 0.0.0.0:8125
     ports:
       - "8125:8125/udp"

requirements/all-base.txt

@@ -0,0 +1,2 @@
+-r main.txt
+-r deploy.txt

requirements/all-ipython.txt

@@ -0,0 +1,2 @@
+-r dev.txt
+-r ipython.txt

requirements/all-lint-test.txt

@@ -0,0 +1,3 @@
+-r all-base.txt
+-r tests.txt
+-r lint.txt

requirements/deploy.in

@@ -1 +1,2 @@
+-r pip.txt
 gunicorn==20.1.0

requirements/deploy.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile
+# This file is autogenerated by pip-compile with python 3.8
 # To update, run:
 #
 #    pip-compile --allow-unsafe --generate-hashes --output-file=requirements/deploy.txt requirements/deploy.in
@@ -8,9 +8,19 @@ gunicorn==20.1.0 \
     --hash=sha256:9dcc4547dbb1cb284accfb15ab5667a0e5d1881cc443e0677b4882a4067a807e \
     --hash=sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8
     # via -r requirements/deploy.in
+wheel==0.37.0 \
+    --hash=sha256:21014b2bd93c6d0034b6ba5d35e4eb284340e09d63c59aef6fc14b0f346146fd \
+    --hash=sha256:e2ef7239991699e3355d54f8e968a21bb940a1dbf34a4d226741e64462516fad
+    # via -r requirements/pip.txt
 
 # The following packages are considered to be unsafe in a requirements file:
-setuptools==57.4.0 \
-    --hash=sha256:6bac238ffdf24e8806c61440e755192470352850f3419a52f26ffe0a1a64f465 \
-    --hash=sha256:a49230977aa6cfb9d933614d2f7b79036e9945c4cdd7583163f4e920b83418d6
-    # via gunicorn
+pip==21.2.4 \
+    --hash=sha256:0eb8a1516c3d138ae8689c0c1a60fde7143310832f9dc77e11d8a4bc62de193b \
+    --hash=sha256:fa9ebb85d3fd607617c0c44aca302b1b45d87f9c2a1649b46c26167ca4296323
+    # via -r requirements/pip.txt
+setuptools==57.5.0 \
+    --hash=sha256:60d78588f15b048f86e35cdab73003d8b21dd45108ee61a6693881a427f22073 \
+    --hash=sha256:d9d3266d50f59c6967b9312844470babbdb26304fe740833a5f8d89829ba3a24
+    # via
+    #   -r requirements/pip.txt
+    #   gunicorn

requirements/docs.in

@@ -1,3 +1,4 @@
+-r pip.txt
 Sphinx
 sphinx_rtd_theme
 sphinxcontrib-httpdomain