diff --git a/Dockerfile b/Dockerfile index 7e12814f44ce..cff5aff52138 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,9 @@ ENV OLYMPIA_UID=9500 \ RUN groupadd -g ${OLYMPIA_GID} olympia && useradd -u ${OLYMPIA_UID} -g ${OLYMPIA_GID} -s /sbin/nologin -d /data/olympia olympia # Add support for https apt repos and gpg signed repos -RUN apt-get update && apt-get install -y \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && apt-get install -y \ apt-transport-https \ gnupg2 \ && rm -rf /var/lib/apt/lists/* @@ -21,7 +23,9 @@ COPY docker/*.list /etc/apt/sources.list.d/ # Allow scripts to detect we're running in our own container and install # packages. -RUN touch /addons-server-docker-container \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + touch /addons-server-docker-container \ && apt-get update && apt-get install -y \ # General (dev-) dependencies bash-completion \ @@ -63,15 +67,16 @@ ENV HOME /data/olympia # The pipeline v2 standard requires the existence of /app/version.json # inside the docker image, thus it's copied there. COPY version.json /app/version.json -COPY --chown=olympia:olympia . ${HOME} WORKDIR ${HOME} +# give olympia access to the HOME directory +RUN chown -R olympia:olympia ${HOME} # Set up directories and links that we'll need later, before switching to the # olympia user. RUN mkdir /deps \ - && chown olympia:olympia /deps \ + && chown -R olympia:olympia /deps \ && rm -rf ${HOME}/src/olympia.egg-info \ - && mkdir ${HOME}/src/olympia.egg-info \ + && mkdir -p ${HOME}/src/olympia.egg-info \ && chown olympia:olympia ${HOME}/src/olympia.egg-info \ # For backwards-compatibility purposes, set up links to uwsgi. Note that # the target doesn't exist yet at this point, but it will later. @@ -88,18 +93,35 @@ ENV PIP_SRC=/deps/src/ ENV PYTHONUSERBASE=/deps ENV PATH $PYTHONUSERBASE/bin:$PATH ENV NPM_CONFIG_PREFIX=/deps/ -RUN ln -s ${HOME}/package.json /deps/package.json \ +ENV NPM_CACHE_DIR=/deps/cache/npm +ENV NPM_DEBUG=true + +RUN \ + # Files needed to run the make command + --mount=type=bind,source=Makefile,target=${HOME}/Makefile \ + --mount=type=bind,source=Makefile-docker,target=${HOME}/Makefile-docker \ + # Files required to install pip dependencies + --mount=type=bind,source=setup.py,target=${HOME}/setup.py \ + --mount=type=bind,source=./requirements,target=${HOME}/requirements \ + # Files required to install npm dependencies + --mount=type=bind,source=package.json,target=${HOME}/package.json \ + --mount=type=bind,source=package-lock.json,target=${HOME}/package-lock.json \ + # Mounts for caching pip/npm dependencies + --mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_GID} \ + --mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_GID} \ + # Command to install pip/npm dependencies + ln -s ${HOME}/package.json /deps/package.json \ && ln -s ${HOME}/package-lock.json /deps/package-lock.json \ && make update_deps_prod +# Only copy our source files after we have installed all dependencies +# TODO: split this into a separate stage to make even blazingly faster WORKDIR ${HOME} +COPY --chown=olympia:olympia . ${HOME} # Build locales, assets, build id. -RUN echo "from olympia.lib.settings_base import *\n" \ -> settings_local.py && DJANGO_SETTINGS_MODULE='settings_local' locale/compile-mo.sh locale \ - && DJANGO_SETTINGS_MODULE='settings_local' python manage.py compress_assets \ - && DJANGO_SETTINGS_MODULE='settings_local' python manage.py generate_jsi18n_files \ - && DJANGO_SETTINGS_MODULE='settings_local' python manage.py collectstatic --noinput \ - && npm prune --production \ +RUN echo "from olympia.lib.settings_base import *" > settings_local.py \ + # TODO: extract this into a separate stage to prevent recompiling locales on any file change + && DJANGO_SETTINGS_MODULE='settings_local' make update_assets \ && ./scripts/generate_build.py > build.py \ && rm -f settings_local.py settings_local.pyc diff --git a/Makefile-docker b/Makefile-docker index badf73089174..ff4083fb360e 100644 --- a/Makefile-docker +++ b/Makefile-docker @@ -17,6 +17,14 @@ ifneq ($(NPM_CONFIG_PREFIX),) NPM_ARGS := --prefix $(NPM_CONFIG_PREFIX) endif +ifneq ($(NPM_CACHE_DIR),) + NPM_ARGS := $(NPM_ARGS) --cache $(NPM_CACHE_DIR) +endif + +ifneq ($(NPM_DEBUG),) + NPM_ARGS := $(NPM_ARGS) --loglevel verbose +endif + NODE_MODULES := $(NPM_CONFIG_PREFIX)node_modules/ STATIC_CSS := static/css/node_lib/ STATIC_JS := static/js/node_lib/ @@ -84,18 +92,14 @@ update_deps_base: ## update the python and node dependencies $(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/pip.txt $(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt - npm install $(NPM_ARGS) - for dest in $(NODE_LIBS_CSS) ; do cp $(NODE_MODULES)$$dest $(STATIC_CSS) ; done - for dest in $(NODE_LIBS_JS) ; do cp $(NODE_MODULES)$$dest $(STATIC_JS) ; done - for dest in $(NODE_LIBS_JQUERY_UI) ; do cp $(NODE_MODULES)$$dest $(STATIC_JQUERY_UI) ; done - .PHONY: update_deps update_deps: update_deps_base ## update the python and node dependencies for development $(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/dev.txt + npm install $(NPM_ARGS) .PHONY: update_deps_prod update_deps_prod: update_deps_base ## update the python and node dependencies for production - npm prune --omit=dev + npm install $(NPM_ARGS) --omit=dev --no-save --no-package-lock .PHONY: update_db update_db: ## run the database migrations @@ -103,10 +107,17 @@ update_db: ## run the database migrations .PHONY: update_assets update_assets: + # Copy files required in compress_assets to the static folder + mkdir -p $(STATIC_CSS) $(STATIC_JS) $(STATIC_JQUERY_UI) + for dest in $(NODE_LIBS_CSS) ; do cp $(NODE_MODULES)$$dest $(STATIC_CSS) ; done + for dest in $(NODE_LIBS_JS) ; do cp $(NODE_MODULES)$$dest $(STATIC_JS) ; done + for dest in $(NODE_LIBS_JQUERY_UI) ; do cp $(NODE_MODULES)$$dest $(STATIC_JQUERY_UI) ; done # If changing this here, make sure to adapt tests in amo/test_commands.py $(PYTHON_COMMAND) manage.py compress_assets $(PYTHON_COMMAND) manage.py collectstatic --noinput $(PYTHON_COMMAND) manage.py generate_jsi18n_files + # Compile locale binaries + locale/compile-mo.sh locale .PHONY: update update: update_deps update_db update_assets ## update the dependencies, the database, and assets