From df23e17cde33162c818ae2eb934cd6cbb3ef7dc6 Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Wed, 8 Sep 2021 12:10:45 -0400 Subject: [PATCH] Revert "Refactoring multiple parts of the dockerfile (#10004)" This reverts commit 6581b0225686270140415b35dfdaa4a00da15674. --- .dockerignore | 8 +- Dockerfile | 191 ++++++++++++++------------------- Dockerfile.static | 24 +++++ Makefile | 2 +- docker-compose.yml | 13 ++- requirements/all-base.txt | 2 - requirements/all-ipython.txt | 2 - requirements/all-lint-test.txt | 3 - requirements/deploy.in | 1 - requirements/deploy.txt | 20 +--- requirements/docs.in | 1 - requirements/docs.txt | 20 +--- requirements/main.in | 1 - requirements/main.txt | 17 +-- requirements/pip.in | 3 - requirements/pip.txt | 20 ---- 16 files changed, 132 insertions(+), 196 deletions(-) create mode 100644 Dockerfile.static delete mode 100644 requirements/all-base.txt delete mode 100644 requirements/all-ipython.txt delete mode 100644 requirements/all-lint-test.txt delete mode 100644 requirements/pip.in delete mode 100644 requirements/pip.txt diff --git a/.dockerignore b/.dockerignore index 572dd7dd3ece..deb5ab58042b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,10 +1,6 @@ -.git +.git/* node_modules -dev +dev/* **/*.pyc htmlcov warehouse/static/dist -.mypy_cache -.state -tests -.github diff --git a/Dockerfile b/Dockerfile index 771ab8bdfa0a..270224805d87 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,28 +1,18 @@ -# ---------------------------------- STATIC ---------------------------------- - # First things first, we build an image which is where we're going to compile -# our static assets with. +# our static assets with. It is important that the steps in this remain the +# same as the steps in Dockerfile.static, EXCEPT this may include additional +# steps appended onto the end. FROM node:14.4.0 as static WORKDIR /opt/warehouse/src/ -# By default, Docker has special steps to avoid keeping APT caches in the layers, which -# is good, but in our case, we're going to mount a special cache volume (kept between -# builds), so we WANT the cache to persist. -RUN set -eux; \ - rm -f /etc/apt/apt.conf.d/docker-clean; \ - echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache; - # The list of C packages we need are almost never going to change, so installing # them first, right off the bat lets us cache that and having node.js level # dependency changes not trigger a reinstall. -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - set -eux; \ - apt-get update; \ - apt-get install --no-install-recommends -y \ - libjpeg-dev \ - nasm +RUN set -x \ + && apt-get update \ + && apt-get install --no-install-recommends -y \ + libjpeg-dev nasm # However, we do want to trigger a reinstall of our node.js dependencies anytime # our package.json changes, so we'll ensure that we're copying that into our @@ -32,10 +22,10 @@ COPY package.json package-lock.json .babelrc /opt/warehouse/src/ # Installing npm dependencies is done as a distinct step and *prior* to copying # over our static files so that, you guessed it, we don't invalidate the cache # of installed dependencies just because files have been modified. -RUN set -eux \ - npm install -g npm@latest; \ - npm install -g gulp-cli; \ - npm ci; +RUN set -x \ + && npm install -g npm@latest \ + && npm install -g gulp-cli \ + && npm ci # Actually copy over our static files, we only copy over the static files to # save a small amount of space in our image and because we don't need them. We @@ -49,124 +39,109 @@ COPY Gulpfile.babel.js /opt/warehouse/src/ RUN gulp dist -# ---------------------------------- BASE ----------------------------------- -FROM python:3.8.2-slim-buster as base - -# Setup some basic environment variables that are ~never going to change. -ENV PYTHONUNBUFFERED 1 -ENV PYTHONPATH /opt/warehouse/src/ -ENV PATH="/opt/warehouse/bin:${PATH}" -WORKDIR /opt/warehouse/src/ -# By default, Docker has special steps to avoid keeping APT caches in the layers, which -# is good, but in our case, we're going to mount a special cache volume (kept between -# builds), so we WANT the cache to persist. -RUN set -eux \ - rm -f /etc/apt/apt.conf.d/docker-clean; \ - echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache -# Install System level Warehouse requirements, this is done before everything -# else because these are rarely ever going to change. -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - set -eux; \ - apt-get update; \ - apt-get install --no-install-recommends -y \ - libpq5 \ - libxml2 \ - libxslt1.1 \ - libcurl4 \ - ; - -# ---------------------------------- BUILD ---------------------------------- # Now we're going to build our actual application, but not the actual production # image that it gets deployed into. -FROM base as build +FROM python:3.8.2-slim-buster as build + +# Define whether we're building a production or a development image. This will +# generally be used to control whether or not we install our development and +# test dependencies. +ARG DEVEL=no + +# To enable Ipython in the development environment set to yes (for using ipython +# as the warehouse shell interpreter, +# i.e. 'docker-compose run --rm web python -m warehouse shell --type=ipython') +ARG IPYTHON=no # Install System level Warehouse build requirements, this is done before # everything else because these are rarely ever going to change. -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - set -eux; \ - apt-get update; \ - apt-get install --no-install-recommends -y \ - build-essential \ - libcurl4-openssl-dev \ - libffi-dev \ - libpq-dev \ - libssl-dev \ - libxml2-dev \ - libxslt-dev \ - ; +RUN set -x \ + && apt-get update \ + && apt-get install --no-install-recommends -y \ + build-essential libffi-dev libxml2-dev libxslt-dev libpq-dev libcurl4-openssl-dev libssl-dev \ + $(if [ "$DEVEL" = "yes" ]; then echo 'libjpeg-dev'; fi) # We create an /opt directory with a virtual environment in it to store our # application in. -RUN python3 -m venv /opt/warehouse +RUN set -x \ + && python3 -m venv /opt/warehouse + -# Pip configuration (https://github.com/pypa/warehouse/pull/4584) -ENV PIP_NO_BINARY=hiredis PIP_DISABLE_PIP_VERSION_CHECK=1 +# Now that we've created our virtual environment, we'll go ahead and update +# our $PATH to refer to it first. +ENV PATH="/opt/warehouse/bin:${PATH}" + +# Next, we want to update pip, setuptools, and wheel inside of this virtual +# environment to ensure that we have the latest versions of them. +# TODO: We use --require-hashes in our requirements files, but not here, making +# the ones in the requirements files kind of a moot point. We should +# probably pin these too, and update them as we do anything else. +RUN pip --no-cache-dir --disable-pip-version-check install --upgrade pip setuptools wheel # We copy this into the docker container prior to copying in the rest of our # application so that we can skip installing requirements if the only thing # that has changed is the Warehouse code itself. COPY requirements /tmp/requirements -# Next, we want to update pip, setuptools, and wheel inside of this virtual -# environment to ensure that we have the latest versions of them. -RUN --mount=type=cache,target=/root/.cache \ - pip install -r /tmp/requirements/pip.txt +# Install our development dependencies if we're building a development install +# otherwise this will do nothing. +RUN set -x \ + && if [ "$DEVEL" = "yes" ]; then pip --no-cache-dir --disable-pip-version-check install -r /tmp/requirements/dev.txt; fi + +RUN set -x \ + && if [ "$DEVEL" = "yes" ] && [ "$IPYTHON" = "yes" ]; then pip --no-cache-dir --disable-pip-version-check install -r /tmp/requirements/ipython.txt; fi # Install the Python level Warehouse requirements, this is done after copying # the requirements but prior to copying Warehouse itself into the container so # that code changes don't require triggering an entire install of all of # Warehouse's dependencies. -RUN --mount=type=cache,target=/root/.cache \ - set -eux; \ - pip install -r /tmp/requirements/all-base.txt; \ - find /opt/warehouse -name '*.pyc' -delete; +RUN set -x \ + && pip --no-cache-dir --disable-pip-version-check \ + install --no-binary hiredis \ + -r /tmp/requirements/deploy.txt \ + -r /tmp/requirements/main.txt \ + $(if [ "$DEVEL" = "yes" ]; then echo '-r /tmp/requirements/tests.txt -r /tmp/requirements/lint.txt'; fi) \ + && find /opt/warehouse -name '*.pyc' -delete -# ---------------------------------- DEV ---------------------------------- -FROM build as dev -# To enable Ipython in the development environment set to yes (for using ipython -# as the warehouse shell interpreter, -# i.e. 'docker-compose run --rm web python -m warehouse shell --type=ipython') -ARG IPYTHON=no -# This is a work around because otherwise postgresql-client bombs out trying -# to create symlinks to these directories. -RUN set -eux; \ - mkdir -p /usr/share/man/man1; \ - mkdir -p /usr/share/man/man7 -# Install System level Warehouse build requirements, this is done before -# everything else because these are rarely ever going to change. -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - set -eux; \ - apt-get update; \ - apt-get install --no-install-recommends -y \ - bash \ - libjpeg-dev \ - libjpeg62 \ - postgresql-client \ - ; - -# Install our development dependencies -RUN set -eux; \ - pip install -r /tmp/requirements/dev.txt; \ - if [ "$IPYTHON" = "yes" ]; then pip install -r /tmp/requirements/all-ipython.txt; fi; - -RUN pip install -r /tmp/requirements/all-lint-test.txt; - - -# ---------------------------------- APP ---------------------------------- -FROM base as app # Now we're going to build our actual application image, which will eventually # pull in the static files that were built above. +FROM python:3.8.2-slim-buster + +# Setup some basic environment variables that are ~never going to change. +ENV PYTHONUNBUFFERED 1 +ENV PYTHONPATH /opt/warehouse/src/ +ENV PATH="/opt/warehouse/bin:${PATH}" + +WORKDIR /opt/warehouse/src/ + +# Define whether we're building a production or a development image. This will +# generally be used to control whether or not we install our development and +# test dependencies. +ARG DEVEL=no + +# This is a work around because otherwise postgresql-client bombs out trying +# to create symlinks to these directories. +RUN set -x \ + && mkdir -p /usr/share/man/man1 \ + && mkdir -p /usr/share/man/man7 + +# Install System level Warehouse requirements, this is done before everything +# else because these are rarely ever going to change. +RUN set -x \ + && apt-get update \ + && apt-get install --no-install-recommends -y \ + libpq5 libxml2 libxslt1.1 libcurl4 \ + $(if [ "$DEVEL" = "yes" ]; then echo 'bash libjpeg62 postgresql-client'; fi) \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # Copy the directory into the container, this is done last so that changes to # Warehouse itself require the least amount of layers being invalidated from diff --git a/Dockerfile.static b/Dockerfile.static new file mode 100644 index 000000000000..4d49609fe684 --- /dev/null +++ b/Dockerfile.static @@ -0,0 +1,24 @@ +FROM node:14.4.0 as static + +WORKDIR /opt/warehouse/src/ + +# The list of C packages we need are almost never going to change, so installing +# them first, right off the bat lets us cache that and having node.js level +# dependency changes not trigger a reinstall. +RUN set -x \ + && apt-get update \ + && apt-get install --no-install-recommends -y \ + libjpeg-dev nasm + +# However, we do want to trigger a reinstall of our node.js dependencies anytime +# our package.json changes, so we'll ensure that we're copying that into our +# static container prior to actually installing the npm dependencies. +COPY package.json package-lock.json .babelrc /opt/warehouse/src/ + +# Installing npm dependencies is done as a distinct step and *prior* to copying +# over our static files so that, you guessed it, we don't invalidate the cache +# of installed dependencies just because files have been modified. +RUN set -x \ + && npm install -g npm@latest \ + && npm install -g gulp-cli \ + && npm ci diff --git a/Makefile b/Makefile index 13c68d53e4bb..798dc896e1ad 100644 --- a/Makefile +++ b/Makefile @@ -64,9 +64,9 @@ endif .state/docker-build: Dockerfile package.json package-lock.json requirements/main.txt requirements/deploy.txt # Build our docker containers for this project. - docker-compose build --force-rm static docker-compose build --build-arg IPYTHON=$(IPYTHON) --force-rm web docker-compose build --force-rm worker + docker-compose build --force-rm static # Mark the state so we don't rebuild this needlessly. mkdir -p .state diff --git a/docker-compose.yml b/docker-compose.yml index 845138e895af..d17d3f953025 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -61,8 +61,8 @@ services: web: build: context: . - target: dev args: + DEVEL: "yes" IPYTHON: "no" command: gunicorn --reload -b 0.0.0.0:8000 warehouse.wsgi:application env_file: dev/environment @@ -88,7 +88,8 @@ services: - "80:8000" files: - image: python:3.8.2-slim-buster + build: + context: . working_dir: /var/opt/warehouse command: python -m http.server 9001 volumes: @@ -100,7 +101,8 @@ services: worker: build: context: . - target: dev + args: + DEVEL: "yes" command: hupper -m celery -A warehouse worker -B -S redbeat.RedBeatScheduler -l info volumes: - ./warehouse:/opt/warehouse/src/warehouse:z @@ -112,7 +114,7 @@ services: static: build: context: . - target: static + dockerfile: Dockerfile.static command: bash -c "node --trace-warnings `which gulp` watch" volumes: - ./warehouse:/opt/warehouse/src/warehouse:z @@ -128,7 +130,8 @@ services: - "1080:80" notdatadog: - image: python:3.8.2-slim-buster + build: + context: . command: python /opt/warehouse/dev/notdatadog.py 0.0.0.0:8125 ports: - "8125:8125/udp" diff --git a/requirements/all-base.txt b/requirements/all-base.txt deleted file mode 100644 index 5de0ace106b6..000000000000 --- a/requirements/all-base.txt +++ /dev/null @@ -1,2 +0,0 @@ --r main.txt --r deploy.txt diff --git a/requirements/all-ipython.txt b/requirements/all-ipython.txt deleted file mode 100644 index f59afd794905..000000000000 --- a/requirements/all-ipython.txt +++ /dev/null @@ -1,2 +0,0 @@ --r dev.txt --r ipython.txt diff --git a/requirements/all-lint-test.txt b/requirements/all-lint-test.txt deleted file mode 100644 index cec67e080989..000000000000 --- a/requirements/all-lint-test.txt +++ /dev/null @@ -1,3 +0,0 @@ --r all-base.txt --r tests.txt --r lint.txt diff --git a/requirements/deploy.in b/requirements/deploy.in index 33ea5b2074d1..9d41f264a678 100644 --- a/requirements/deploy.in +++ b/requirements/deploy.in @@ -1,2 +1 @@ --r pip.txt gunicorn==20.1.0 diff --git a/requirements/deploy.txt b/requirements/deploy.txt index c3d1cc429421..411601cc075f 100644 --- a/requirements/deploy.txt +++ b/requirements/deploy.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile # To update, run: # # pip-compile --allow-unsafe --generate-hashes --output-file=requirements/deploy.txt requirements/deploy.in @@ -8,19 +8,9 @@ gunicorn==20.1.0 \ --hash=sha256:9dcc4547dbb1cb284accfb15ab5667a0e5d1881cc443e0677b4882a4067a807e \ --hash=sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8 # via -r requirements/deploy.in -wheel==0.37.0 \ - --hash=sha256:21014b2bd93c6d0034b6ba5d35e4eb284340e09d63c59aef6fc14b0f346146fd \ - --hash=sha256:e2ef7239991699e3355d54f8e968a21bb940a1dbf34a4d226741e64462516fad - # via -r requirements/pip.txt # The following packages are considered to be unsafe in a requirements file: -pip==21.2.4 \ - --hash=sha256:0eb8a1516c3d138ae8689c0c1a60fde7143310832f9dc77e11d8a4bc62de193b \ - --hash=sha256:fa9ebb85d3fd607617c0c44aca302b1b45d87f9c2a1649b46c26167ca4296323 - # via -r requirements/pip.txt -setuptools==57.5.0 \ - --hash=sha256:60d78588f15b048f86e35cdab73003d8b21dd45108ee61a6693881a427f22073 \ - --hash=sha256:d9d3266d50f59c6967b9312844470babbdb26304fe740833a5f8d89829ba3a24 - # via - # -r requirements/pip.txt - # gunicorn +setuptools==57.4.0 \ + --hash=sha256:6bac238ffdf24e8806c61440e755192470352850f3419a52f26ffe0a1a64f465 \ + --hash=sha256:a49230977aa6cfb9d933614d2f7b79036e9945c4cdd7583163f4e920b83418d6 + # via gunicorn diff --git a/requirements/docs.in b/requirements/docs.in index a614f395f38b..6ff9a8162b58 100644 --- a/requirements/docs.in +++ b/requirements/docs.in @@ -1,4 +1,3 @@ --r pip.txt Sphinx sphinx_rtd_theme sphinxcontrib-httpdomain diff --git a/requirements/docs.txt b/requirements/docs.txt index 3265c3c134e2..097be7b1ef86 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile # To update, run: # # pip-compile --allow-unsafe --generate-hashes --output-file=requirements/docs.txt requirements/docs.in @@ -145,19 +145,9 @@ urllib3==1.26.6 \ --hash=sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4 \ --hash=sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f # via requests -wheel==0.37.0 \ - --hash=sha256:21014b2bd93c6d0034b6ba5d35e4eb284340e09d63c59aef6fc14b0f346146fd \ - --hash=sha256:e2ef7239991699e3355d54f8e968a21bb940a1dbf34a4d226741e64462516fad - # via -r requirements/pip.txt # The following packages are considered to be unsafe in a requirements file: -pip==21.2.4 \ - --hash=sha256:0eb8a1516c3d138ae8689c0c1a60fde7143310832f9dc77e11d8a4bc62de193b \ - --hash=sha256:fa9ebb85d3fd607617c0c44aca302b1b45d87f9c2a1649b46c26167ca4296323 - # via -r requirements/pip.txt -setuptools==57.5.0 \ - --hash=sha256:60d78588f15b048f86e35cdab73003d8b21dd45108ee61a6693881a427f22073 \ - --hash=sha256:d9d3266d50f59c6967b9312844470babbdb26304fe740833a5f8d89829ba3a24 - # via - # -r requirements/pip.txt - # sphinx +setuptools==57.4.0 \ + --hash=sha256:6bac238ffdf24e8806c61440e755192470352850f3419a52f26ffe0a1a64f465 \ + --hash=sha256:a49230977aa6cfb9d933614d2f7b79036e9945c4cdd7583163f4e920b83418d6 + # via sphinx diff --git a/requirements/main.in b/requirements/main.in index d0d8345876ed..c5736101b26b 100644 --- a/requirements/main.in +++ b/requirements/main.in @@ -1,4 +1,3 @@ --r pip.txt alembic>=0.7.0 Automat argon2-cffi diff --git a/requirements/main.txt b/requirements/main.txt index 03a554b649e6..5c20ac35db64 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile # To update, run: # # pip-compile --allow-unsafe --generate-hashes --output-file=requirements/main.txt requirements/main.in @@ -1062,10 +1062,6 @@ webob==1.8.7 \ --hash=sha256:73aae30359291c14fa3b956f8b5ca31960e420c28c1bec002547fb04928cf89b \ --hash=sha256:b64ef5141be559cfade448f044fa45c2260351edcb6a8ef6b7e00c7dcef0c323 # via pyramid -wheel==0.37.0 \ - --hash=sha256:21014b2bd93c6d0034b6ba5d35e4eb284340e09d63c59aef6fc14b0f346146fd \ - --hash=sha256:e2ef7239991699e3355d54f8e968a21bb940a1dbf34a4d226741e64462516fad - # via -r requirements/pip.txt whitenoise==5.3.0 \ --hash=sha256:d234b871b52271ae7ed6d9da47ffe857c76568f11dd30e28e18c5869dbd11e12 \ --hash=sha256:d963ef25639d1417e8a247be36e6aedd8c7c6f0a08adcb5a89146980a96b577c @@ -1159,16 +1155,11 @@ zxcvbn==4.4.28 \ # via -r requirements/main.in # The following packages are considered to be unsafe in a requirements file: -pip==21.2.4 \ - --hash=sha256:0eb8a1516c3d138ae8689c0c1a60fde7143310832f9dc77e11d8a4bc62de193b \ - --hash=sha256:fa9ebb85d3fd607617c0c44aca302b1b45d87f9c2a1649b46c26167ca4296323 - # via -r requirements/pip.txt -setuptools==57.5.0 \ - --hash=sha256:60d78588f15b048f86e35cdab73003d8b21dd45108ee61a6693881a427f22073 \ - --hash=sha256:d9d3266d50f59c6967b9312844470babbdb26304fe740833a5f8d89829ba3a24 +setuptools==57.4.0 \ + --hash=sha256:6bac238ffdf24e8806c61440e755192470352850f3419a52f26ffe0a1a64f465 \ + --hash=sha256:a49230977aa6cfb9d933614d2f7b79036e9945c4cdd7583163f4e920b83418d6 # via # -r requirements/main.in - # -r requirements/pip.txt # google-api-core # google-auth # pastedeploy diff --git a/requirements/pip.in b/requirements/pip.in deleted file mode 100644 index 7015e2e2f3a7..000000000000 --- a/requirements/pip.in +++ /dev/null @@ -1,3 +0,0 @@ -pip -setuptools -wheel diff --git a/requirements/pip.txt b/requirements/pip.txt deleted file mode 100644 index ad3730213a97..000000000000 --- a/requirements/pip.txt +++ /dev/null @@ -1,20 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: -# -# pip-compile --allow-unsafe --generate-hashes --output-file=requirements/pip.txt requirements/pip.in -# -wheel==0.37.0 \ - --hash=sha256:21014b2bd93c6d0034b6ba5d35e4eb284340e09d63c59aef6fc14b0f346146fd \ - --hash=sha256:e2ef7239991699e3355d54f8e968a21bb940a1dbf34a4d226741e64462516fad - # via -r requirements/pip.in - -# The following packages are considered to be unsafe in a requirements file: -pip==21.2.4 \ - --hash=sha256:0eb8a1516c3d138ae8689c0c1a60fde7143310832f9dc77e11d8a4bc62de193b \ - --hash=sha256:fa9ebb85d3fd607617c0c44aca302b1b45d87f9c2a1649b46c26167ca4296323 - # via -r requirements/pip.in -setuptools==57.5.0 \ - --hash=sha256:60d78588f15b048f86e35cdab73003d8b21dd45108ee61a6693881a427f22073 \ - --hash=sha256:d9d3266d50f59c6967b9312844470babbdb26304fe740833a5f8d89829ba3a24 - # via -r requirements/pip.in