Split CI dependencies and add ./deps:/deps mount. (#22934)

* Test checkout remote

* Fix broken checks

.dockerignore

@@ -28,7 +28,7 @@
 backups
 build*.py
 buildx-bake-metadata.json
-deps
+deps/*
 docker*.yml
 docker/artifacts/*
 docs/_build

.github/actions/build-docker/action.yml

@@ -10,9 +10,6 @@ inputs:
   version:
     required: true
     description: The image version to tag with
-  target:
-    required: true
-    description: The stage to target in the build
   push:
     required: false
     description: Push the image?
@@ -57,7 +54,8 @@ runs:
     - name: Create .env and version.json files
       shell: bash
       run: |
-        echo "DOCKER_TARGET=${{ inputs.target }}" >> $GITHUB_ENV
+        # We only build the production image in CI
+        echo "DOCKER_TARGET=production" >> $GITHUB_ENV
         echo "DOCKER_VERSION=${{ steps.meta.outputs.version }}" >> $GITHUB_ENV
         echo "DOCKER_COMMIT=${{ steps.context.outputs.git_sha }}" >> $GITHUB_ENV
         echo "DOCKER_BUILD=${{ steps.context.outputs.git_build_url }}" >> $GITHUB_ENV

.github/actions/run-docker/action.yml

@@ -19,8 +19,16 @@ inputs:
     description: 'Skip data backup'
     required: false
     default: 'true'
+  target:
+    description: 'Docker target to run (development|production)'
+    required: false
+    default: 'production'
   mount:
-    description: 'Mount olympia files from host'
+    description: 'Mount host files at runtime? (development|production)'
+    required: false
+    default: 'production'
+  deps:
+    description: 'Which dependencies to install at runtime? (development|production)'
     required: false
     default: 'production'
 
@@ -34,8 +42,10 @@ runs:
         make up \
           DOCKER_VERSION="${{ inputs.version }}" \
           DOCKER_DIGEST="${{ inputs.digest }}" \
+          DOCKER_TARGET="${{ inputs.target }}" \
           OLYMPIA_UID="$(id -u)" \
           OLYMPIA_MOUNT="${{ inputs.mount }}" \
+          OLYMPIA_DEPS="${{ inputs.deps }}" \
           DATA_BACKUP_SKIP="${{ inputs.data_backup_skip }}" \
           DOCKER_WAIT="true"
 
@@ -47,6 +57,6 @@ runs:
         EOF
 
     - name: Logs
-      shell: bash
       if: ${{ inputs.logs }}
+      shell: bash
       run: docker compose logs

.github/workflows/_test.yml

@@ -68,4 +68,5 @@ jobs:
           version: ${{ inputs.version }}
           digest: ${{ inputs.digest }}
           services: ${{ matrix.services }}
+          deps: development
           run: ${{ matrix.run }}

.github/workflows/_test_check.yml

@@ -45,16 +45,24 @@ jobs:
     runs-on: ubuntu-latest
     name: |
       version: '${{ matrix.version }}' |
-      mount: '${{ matrix.mount }}'
+      target: '${{ matrix.target }}' |
+      mount: '${{ matrix.mount }}' |
+      deps: '${{ matrix.deps }}'
     strategy:
       fail-fast: false
       matrix:
         version:
           - local
           - ${{ inputs.version }}
+        target:
+          - development
+          - production
         mount:
           - development
           - production
+        deps:
+          - development
+          - production
     steps:
       - uses: actions/checkout@v4
       - shell: bash
@@ -63,7 +71,9 @@ jobs:
           cat <<EOF
             Values passed to the action:
             version: ${{ matrix.version }}
+            target: ${{ matrix.target }}
             mount: ${{ matrix.mount }}
+            deps: ${{ matrix.deps }}
           EOF
       - name: ${{ matrix.version == 'local' && 'Uncached Build' || 'Pull' }} Check
         uses: ./.github/actions/run-docker
@@ -73,14 +83,18 @@ jobs:
           DOCKER_VERSION: 'not-expected'
         with:
           version: ${{ matrix.version }}
+          target: ${{ matrix.target }}
           mount: ${{ matrix.mount }}
+          deps: ${{ matrix.deps }}
           run: make check
       - name: Cached Build Check
         uses: ./.github/actions/run-docker
         if: ${{ matrix.version == 'local' }}
         with:
           version: ${{ matrix.version }}
+          target: ${{ matrix.target }}
           mount: ${{ matrix.mount }}
+          deps: ${{ matrix.deps }}
           run: echo true
 
   test_make_docker_configuration:
@@ -102,10 +116,11 @@ jobs:
         with:
           digest: ${{ inputs.digest }}
           version: ${{ inputs.version }}
+          deps: development
           run: |
             pytest tests/make/
 
-  test_run_docker_action:
+  test_verify_failure:
     runs-on: ubuntu-latest
     needs: context
 
@@ -131,6 +146,10 @@ jobs:
             exit 1
           fi
 
+  test_special_characters:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
       - name: Check (special characters in command)
         uses: ./.github/actions/run-docker
         with:

.github/workflows/_test_main.yml

@@ -89,6 +89,7 @@ jobs:
           digest: ${{ inputs.digest }}
           version: ${{ inputs.version }}
           mount: development
+          deps: development
           run: |
             split="--splits ${{ needs.test_config.outputs.splits }}"
             group="--group ${{ matrix.group }}"

.github/workflows/ci.yml

@@ -72,7 +72,6 @@ jobs:
           registry: ${{ steps.docker_hub.outputs.registry }}
           image: ${{ steps.docker_hub.outputs.image }}
           version: ci-${{ needs.context.outputs.docker_version }}
-          target: development
           push: true
 
   docs_build:
@@ -89,6 +88,8 @@ jobs:
           digest: ${{ needs.build.outputs.digest }}
           version: ${{ needs.build.outputs.version }}
           mount: development
+          deps: development
+          target: development
           run: |
             make docs
 
@@ -140,6 +141,7 @@ jobs:
           digest: ${{ needs.build.outputs.digest }}
           version: ${{ needs.build.outputs.version }}
           mount: development
+          deps: development
           run: make extract_locales
 
       - name: Push Locales
@@ -213,7 +215,6 @@ jobs:
           registry: ${{ steps.docker_hub.outputs.registry }}
           image: ${{ steps.docker_hub.outputs.image }}
           version: ${{ needs.context.outputs.docker_version }}
-          target: production
           push: true
 
   push_gar:
@@ -245,5 +246,4 @@ jobs:
           registry: ${{ steps.docker_gar.outputs.registry }}
           image: ${{ steps.docker_gar.outputs.image }}
           version: ${{ needs.context.outputs.docker_version }}
-          target: production
           push: true

.gitignore

@@ -27,7 +27,7 @@
 backups
 build*.py
 buildx-bake-metadata.json
-deps
+deps/*
 docker*.yml
 docker/artifacts/*
 docs/_build
@@ -56,3 +56,4 @@ tmp/*
 !docker-compose.private.yml
 !private/README.md
 !storage/.gitignore
+!deps/.gitkeep

Dockerfile

@@ -109,26 +109,29 @@ ENV NPM_ARGS="--prefix ${NPM_CONFIG_PREFIX} --cache ${NPM_CACHE_DIR} --loglevel
 # All we need in "base" is pip to be installed
 #this let's other layers install packages using the correct version.
 RUN \
+    --mount=type=bind,source=scripts/install_deps.py,target=${HOME}/scripts/install_deps.py \
     # Files required to install pip dependencies
     --mount=type=bind,source=./requirements/pip.txt,target=${HOME}/requirements/pip.txt \
     --mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
 <<EOF
-# Work arounds "Multiple .dist-info directories" issue.
-rm -rf /deps/build/*
-${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/pip.txt
+${HOME}/scripts/install_deps.py pip
 EOF
 
 # TODO: we should remove dependency on the environment variable
 # and instead read from the /build-info file
 ARG DOCKER_TARGET
 ENV DOCKER_TARGET=${DOCKER_TARGET}
 
+# Add our custom mime types (required for for ts/json/md files)
+COPY docker/etc/mime.types /etc/mime.types
+
 # Define production dependencies as a single layer
 # let's the rest of the stages inherit prod dependencies
 # and makes copying the /deps dir to the final layer easy.
 FROM base AS pip_production
 
 RUN \
+    --mount=type=bind,source=scripts/install_deps.py,target=${HOME}/scripts/install_deps.py \
     # Files required to install pip dependencies
     --mount=type=bind,source=./requirements/prod.txt,target=${HOME}/requirements/prod.txt \
     # Files required to install npm dependencies
@@ -138,27 +141,10 @@ RUN \
     --mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
     --mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
 <<EOF
-${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
-npm ci ${NPM_ARGS} --include=prod
+${HOME}/scripts/install_deps.py prod
 EOF
 
-FROM pip_production AS pip_development
-
-RUN \
-    # Files required to install pip dependencies
-    --mount=type=bind,source=./requirements/prod.txt,target=${HOME}/requirements/prod.txt \
-    --mount=type=bind,source=./requirements/dev.txt,target=${HOME}/requirements/dev.txt \
-    # Files required to install npm dependencies
-    --mount=type=bind,source=package.json,target=/deps/package.json \
-    --mount=type=bind,source=package-lock.json,target=/deps/package-lock.json \
-    # Mounts for caching dependencies
-    --mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
-    --mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
-<<EOF
-${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
-${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/dev.txt
-npm install ${NPM_ARGS} --no-save
-EOF
+FROM base AS development
 
 FROM base AS locales
 ARG LOCALE_DIR=${HOME}/locale
@@ -195,32 +181,17 @@ echo "from olympia.lib.settings_base import *" > settings_local.py
 DJANGO_SETTINGS_MODULE="settings_local" make -f Makefile-docker update_assets
 EOF
 
-FROM base AS sources
-
-
-
-# Add our custom mime types (required for for ts/json/md files)
-COPY docker/etc/mime.types /etc/mime.types
+FROM base AS production
 # Copy the rest of the source files from the host
 COPY --chown=olympia:olympia . ${HOME}
+# Copy compiled locales from builder
+COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 # Copy assets from assets
 COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-static
 COPY --from=assets --chown=olympia:olympia ${HOME}/static-build ${HOME}/static-build
+# Copy build info from info
 COPY --from=info ${BUILD_INFO} ${BUILD_INFO}
-
-# Set shell back to sh until we can prove we can use bash at runtime
-SHELL ["/bin/sh", "-c"]
-
-FROM sources AS development
-
-# Copy dependencies from `pip_development`
-COPY --from=pip_development --chown=olympia:olympia /deps /deps
-
-FROM sources AS production
-
 # Copy compiled locales from builder
 COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 # Copy dependencies from `pip_production`
 COPY --from=pip_production --chown=olympia:olympia /deps /deps
-
-

Makefile-docker

@@ -18,6 +18,13 @@ REQUIRED_FILES := \
 	/deps/package-lock.json \
 	/addons-server-docker-container \
 
+# Build list of dependencies to install
+DEPS = pip prod
+# If we're running a development image, then we should install the development dependencies
+ifeq ($(OLYMPIA_DEPS), development)
+DEPS += dev
+endif
+
 .PHONY: help_redirect
 help_redirect:
 	@$(MAKE) help --no-print-directory
@@ -28,12 +35,9 @@ check_debian_packages: ## check the existence of multiple debian packages
 
 .PHONY: check_pip_packages
 check_pip_packages: ## check the existence of multiple python packages
-	@ ./scripts/check_pip_packages.sh prod.txt
-# "production" corresponds to the "propduction" DOCKER_TARGET defined in the Dockerfile
-# When the target is "production" it means we cannot expect dev.txt dependencies to be installed.
-	@if [ "$(DOCKER_TARGET)" != "production" ]; then \
-		./scripts/check_pip_packages.sh dev.txt; \
-	fi
+	@for dep in $(DEPS); do \
+		./scripts/check_pip_packages.sh $$dep.txt; \
+	done
 
 .PHONY: check_files
 check_files: ## check the existence of multiple files
@@ -77,6 +81,10 @@ update_assets:
 	$(PYTHON_COMMAND) manage.py collectstatic --noinput
 
 
+.PHONY: update_deps
+update_deps: ## Update the dependencies
+	$(HOME)/scripts/install_deps.py $(DEPS)
+
 # TOOD: remove this after we migrate addons-frontned to not depend on it.
 .PHONY: setup-ui-tests
 setup-ui-tests:

Makefile-os

@@ -58,7 +58,6 @@ CLEAN_PATHS := \
 	version.json \
 	logs \
 	buildx-bake-metadata.json \
-	deps \
 
 .PHONY: help_redirect
 help_redirect:
@@ -154,8 +153,17 @@ docker_clean_build_cache: ## Remove buildx build cache
 .PHONY: clean_docker
 clean_docker: docker_compose_down docker_mysqld_volume_remove docker_clean_images docker_clean_volumes docker_clean_build_cache ## Remove all docker resources taking space on the host machine
 
+.PHONY: docker_update_deps
+docker_update_deps: docker_mysqld_volume_create ## Update the dependencies in the container based on the docker tag and target
+	docker compose run \
+		--rm \
+		--no-deps \
+		$(DOCKER_RUN_ARGS) \
+		web \
+		make update_deps
+
 .PHONY: up_pre
-up_pre: setup docker_pull_or_build ## Pre-up the environment, setup files, volumes and host state
+up_pre: setup docker_pull_or_build docker_update_deps ## Pre-up the environment, setup files, volumes and host state
 
 .PHONY: up_start
 up_start: docker_mysqld_volume_create ## Start the docker containers