chore(ci): Add capability to build and push test images to GCR. (#812)

Adds the steps needed to build and push our containerized tests to GCR. [SYNC-4559](https://mozilla-hub.atlassian.net/browse/SYNC-4559) [SYNC-4559]: https://mozilla-hub.atlassian.net/browse/SYNC-559?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ
mozilla-services · Dec 20, 2024 · 6f614f9 · 6f614f9
2 parents bd756fb + e4815a9
commit 6f614f9
Show file tree

Hide file tree

Showing 5 changed files with 153 additions and 67 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -162,7 +162,14 @@ jobs:
       - checkout
       - setup_remote_docker:
           docker_layer_caching: true
+      - attach_workspace:
+          at: /tmp/cache
       - create_test_result_workspace
+      - run:
+          name: Restore Docker image cache
+          command: |
+            docker load -i /tmp/cache/autopush-integration-tests.tar
+            docker tag autopush-integration-tests:build integration-tests
       - run:
           name: Integration tests (Bigtable)
           command: |
@@ -303,51 +310,66 @@ jobs:
       # required since Workflows do not have the same remote docker instance.
       - run:
           name: docker save <<parameters.image>>
-          command: mkdir -p /cache; docker save -o /cache/docker.tar "<<parameters.image>>"
+          command: mkdir -p /tmp/cache; docker save -o /tmp/cache/<<parameters.image>>.tar "<<parameters.image>>"
       - persist_to_workspace:
-          root: /cache
+          root: /tmp/cache
           paths:
-            - docker.tar
+            - <<parameters.image>>.tar
 
-  build-load-test:
+  build-test-container:
     docker:
       - image: cimg/base:2024.06
+    parameters:
+      image:
+        type: string
+      path:
+        type: string
     steps:
       - checkout
       - setup_remote_docker:
           docker_layer_caching: true
       - run:
           name: Build Image
-          command: docker build -t autopush-locust -f ./tests/load/Dockerfile .
+          command: docker build -t << parameters.image >>:build -f ./tests/<< parameters.path >>/Dockerfile .
       - run:
           name: Save Docker Image to Workspace
           command: |
-            mkdir -p /tmp/workspace
-            docker save -o /tmp/workspace/autopush-locust.tar autopush-locust
+            mkdir -p /tmp/cache
+            docker save -o /tmp/cache/<< parameters.image >>.tar << parameters.image >>:build
       - persist_to_workspace:
-          root: /tmp/workspace
+          root: /tmp/cache
           paths:
-            - autopush-locust.tar
+            - << parameters.image >>.tar
 
-  build-integration-test:
+  build-integration-test-container:
     docker:
       - image: cimg/base:2024.06
+    resource_class: large
+    parameters:
+      image:
+        type: string
+      path:
+        type: string
     steps:
       - checkout
       - setup_remote_docker:
           docker_layer_caching: true
       - run:
           name: Build Image
-          command: docker build -t autopush-integration-tests -f ./tests/integration/Dockerfile .
+          command: |
+            docker buildx build --target planner -f ./tests/<< parameters.path >>/Dockerfile .
+            docker buildx build --target cacher -f ./tests/<< parameters.path >>/Dockerfile .
+            docker buildx build --target builder -f ./tests/<< parameters.path >>/Dockerfile .
+            docker buildx build --target integration-tests -t << parameters.image >>:build -f ./tests/<< parameters.path >>/Dockerfile .
       - run:
           name: Save Docker Image to Workspace
           command: |
-            mkdir -p /tmp/workspace
-            docker save -o /tmp/workspace/autopush-integration-tests.tar autopush-integration-tests
+            mkdir -p /tmp/cache
+            docker save -o /tmp/cache/<< parameters.image >>.tar << parameters.image >>:build
       - persist_to_workspace:
-          root: /tmp/workspace
+          root: /tmp/cache
           paths:
-            - autopush-integration-tests.tar
+            - << parameters.image >>.tar
 
   deploy:
     executor: gcp-gcr/default
@@ -376,7 +398,7 @@ jobs:
           at: /tmp/cache
       - run:
           name: Restore Docker image cache
-          command: docker load -i /tmp/cache/docker.tar
+          command: docker load -i /tmp/cache/<<parameters.image>>.tar
       # This is the easiest way to tag multiple images using different
       # conditions for the GAR_TAG variable in the smallest amount of code.
       #
@@ -403,29 +425,6 @@ jobs:
           registry-url: <<parameters.registry-url>>
           tag: $GAR_TAG,latest
 
-  deploy-load-test:
-    docker:
-      - image: cimg/base:2024.06
-    steps:
-      - checkout
-      - attach_workspace:
-          at: /tmp/workspace
-      - setup_remote_docker
-      - run:
-          name: Load Docker Image From Workspace
-          command: docker load -i /tmp/workspace/autopush-locust.tar
-      - docker_login:
-          load: True
-      - run:
-          name: Push to Docker Hub
-          command: |
-            echo ${DOCKERHUB_LOAD_TEST_REPO}:${CIRCLE_SHA1}
-            docker tag autopush-locust ${DOCKERHUB_LOAD_TEST_REPO}:${CIRCLE_SHA1}
-            docker tag autopush-locust ${DOCKERHUB_LOAD_TEST_REPO}:latest
-            docker images
-            docker push "${DOCKERHUB_LOAD_TEST_REPO}:${CIRCLE_SHA1}"
-            docker push "${DOCKERHUB_LOAD_TEST_REPO}:latest"
-
 workflows:
   build-test-deploy:
     jobs:
@@ -439,6 +438,9 @@ workflows:
               only: /.*/
       - test-integration:
           name: Integration Tests
+          requires:
+            - Build Integration Test Image
+            - python-checks
           filters:
             tags:
               only: /.*/
@@ -470,18 +472,29 @@ workflows:
           filters:
             tags:
               only: /.*/
-
-      - build-load-test:
+      - build-test-container:
+          name: Build Load Test Image
+          image: autopush-load-tests
+          path: load
           filters:
             tags:
               only: /.*/
-
-      - build-integration-test:
+      - build-test-container:
+          name: Build End-To-End Test Image
+          image: autopush-end-to-end-tests
+          path: end-to-end
           filters:
             tags:
               only: /.*/
             branches:
               only: master
+      - build-integration-test-container:
+          name: Build Integration Test Image
+          image: autopush-integration-tests
+          path: integration
+          filters:
+            tags:
+              only: /.*/
 
       # Comment out the following two sections for local CircleCI testing.
       - deploy:
@@ -511,10 +524,34 @@ workflows:
               only: /.*/
             branches:
               only: master
-
-      - deploy-load-test:
+      - deploy:
+          name: Push Load Test Image
+          image: autopush-load-tests
           requires:
-            - build-load-test
+            - Build Load Test Image
+            - python-checks
+          filters:
+            tags:
+              only: /.*/
+            branches:
+              only: master
+      - deploy:
+          name: Push End-To-End Test Image
+          image: autopush-end-to-end-tests
+          requires:
+            - Build End-To-End Test Image
+            - python-checks
+          filters:
+            tags:
+              only: /.*/
+            branches:
+              only: master
+      - deploy:
+          name: Push Integration Test Image
+          image: autopush-integration-tests
+          requires:
+            - Build Integration Test Image
+            - Integration Tests
             - python-checks
           filters:
             tags:

diff --git a/Makefile b/Makefile
@@ -29,8 +29,10 @@ upgrade:
 	$(CARGO) upgrade
 	$(CARGO) update
 
-integration-test:
+build-integration-test:
 	$(DOCKER_COMPOSE) -f $(INTEGRATION_TEST_DIR)/docker-compose.yml build
+
+integration-test:
 	$(DOCKER_COMPOSE) -f $(INTEGRATION_TEST_DIR)/docker-compose.yml run -it --name integration-tests tests
 	docker cp integration-tests:/code/integration_test_results.xml $(INTEGRATION_TEST_DIR)
 

diff --git a/tests/integration/Dockerfile b/tests/integration/Dockerfile
@@ -2,7 +2,64 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-FROM python:3.12-slim-bookworm
+# =============================================================================
+# Pull in the version of cargo-chef we plan to use, so that all the below steps
+# use a consistent set of versions.
+FROM lukemathwalker/cargo-chef:0.1.68-rust-slim-bookworm AS chef
+WORKDIR /app
+
+# =============================================================================
+# Analyze the project, and produce a plan to compile its dependcies. This will
+# be run every time. The output should only change if the dependencies of the
+# project change, or if significant details of the build process change.
+FROM chef AS planner
+COPY . .
+RUN cargo chef prepare --recipe-path recipe.json
+
+# =============================================================================
+# Use the plan from above to build only the dependencies of the project. This
+# should almost always be pulled straight from cache unless dependencies or the
+# build process change.
+FROM chef AS cacher
+COPY --from=planner /app/recipe.json recipe.json
+
+RUN \
+    apt-get -qq update && \
+    apt-get -qq install --no-install-recommends -y \
+      cmake \
+      libssl-dev \
+      ca-certificates \
+      pkg-config \
+      build-essential
+
+RUN cargo chef cook --recipe-path recipe.json
+
+# =============================================================================
+# Now build the project, taking advantage of the cached dependencies from above.
+FROM chef AS builder
+ARG APT_CACHE_BUST
+
+RUN mkdir -m 755 bin
+RUN apt-get -qq update && \
+    apt-get -qq upgrade && apt-get -qq install --no-install-recommends -y \
+      cmake \
+      libssl-dev \
+      ca-certificates \
+      libstdc++6 \
+      libstdc++-12-dev
+
+RUN cargo --version && \
+    rustc --version
+COPY . .
+COPY --from=cacher /app/target target
+COPY --from=cacher $CARGO_HOME $CARGO_HOME
+
+RUN cargo build --features=emulator
+
+FROM python:3.12-slim-bookworm AS integration-tests
+
+# =============================================================================
+# Setup Integration test image
 
 LABEL org.opencontainers.image.authors="[email protected]"
 
@@ -19,25 +76,15 @@ ENV DB_DSN=grpc://localhost:8086
 # (g++/make for gevent on pypy)
 RUN apt-get update && apt install -y --no-install-recommends \
     git \
-    gpg \
     build-essential \
-    python3-dev \
     curl \
-    libstdc++6 \
-    libstdc++-12-dev \
-    libssl-dev \
-    pkg-config \
-    cmake
+    gpg
 
 RUN python -m venv ${PYTHON_VENV}
 ENV PATH="${PYTHON_VENV}/bin:${PATH}"
 
 RUN python -m pip install --upgrade pip
 
-# Install Rust
-RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.81 -y
-RUN rustc --version
-
 # Setup poetry and install requirements
 ENV POETRY_VIRTUALENVS_CREATE=false \
   POETRY_VERSION=1.7.0
@@ -52,10 +99,10 @@ RUN apt-get update -y && apt install google-cloud-cli-cbt -y
 
 COPY . /code
 
-WORKDIR /code
+COPY --from=builder /app/target/debug /code/target/debug
+COPY --from=builder /app/version.json /code
 
-# Build app
-RUN cargo build --features=emulator
+WORKDIR /code
 
 RUN chmod +x scripts/setup_bt.sh
 

diff --git a/tests/notification/Dockerfile b/tests/notification/Dockerfile
@@ -56,9 +56,9 @@ RUN wget https://github.com/mozilla/geckodriver/releases/download/v${GECKODRIVER
 RUN curl -sSL https://install.python-poetry.org | python3 -
 
 WORKDIR /code
-ADD notification/ /code
-ADD ../poetry.lock /code
-ADD ../pyproject.toml /code
+ADD ./tests/notification/ /code
+ADD ./tests/poetry.lock /code
+ADD ./tests/pyproject.toml /code
 
 RUN poetry install --only=notification
 

diff --git a/tests/notification/docker-compose.yml b/tests/notification/docker-compose.yml
@@ -10,8 +10,8 @@ services:
     environment:
       SERVER_URL: server
     build:
-      context: ..
-      dockerfile: notification/Dockerfile
+      context: ../..
+      dockerfile: tests/notification/Dockerfile
     depends_on: 
       - server
     network_mode: host