feat: Containerize integration tests.

.circleci/config.yml

@@ -154,50 +154,25 @@ jobs:
 
   test-integration:
     docker:
-      - image: python:3.12-slim-bookworm
-        auth:
-          username: $DOCKER_USER
-          password: $DOCKER_PASS
+      - image: cimg/base:2024.06
         environment:
           RUST_BACKTRACE: 1
-      - image: google/cloud-sdk:latest
-        auth:
-          username: $DOCKER_USER
-          password: $DOCKER_PASS
-        command: gcloud beta emulators bigtable start --host-port=localhost:8086
     resource_class: large
-    environment:
-      BIGTABLE_EMULATOR_HOST: localhost:8086
     steps:
       - checkout
-      - restore_test_cache:
-          cache_key: rust-v1-integration-test-{{ checksum "Cargo.lock" }}
+      - setup_remote_docker:
+          docker_layer_caching: true
       - create_test_result_workspace
-      - setup_rust
-      - build_applications
-      - run:
-          name: Set up system
-          command: |
-            apt update
-            apt install libssl-dev apt-transport-https ca-certificates gnupg curl cmake -y
-      - setup_cbt
-      - setup_python
-      - setup_bigtable
       - run:
           name: Integration tests (Bigtable)
-          command: make integration-test
-          environment:
-            RUST_LOG: autopush=debug,autopush_common=debug,autoendpoint=debug,autoconnect=debug,slog_mozlog_json=info,warn
-            # PYTEST_ARGS: -sv
-            DB_DSN: grpc://localhost:8086
-            TEST_RESULTS_DIR: workspace/test-results
+          command: |
+            make integration-test
+            cp -r ~/project/tests/integration ~/project/workspace/test-results
       - store_artifacts:
           path: ~/project/workspace/test-results/integration_test_results.xml
           destination: integration_test_results.xml
       - store_test_results:
           path: workspace/test-results
-      - save_test_cache:
-          cache_key: rust-v1-integration-test-cache-{{ checksum "Cargo.lock" }}
 
   test-unit:
     docker:
@@ -354,6 +329,26 @@ jobs:
           paths:
             - autopush-locust.tar
 
+  build-integration-test:
+    docker:
+      - image: cimg/base:2024.06
+    steps:
+      - checkout
+      - setup_remote_docker:
+          docker_layer_caching: true
+      - run:
+          name: Build Image
+          command: docker build -t autopush-integration-tests -f ./tests/integration/Dockerfile .
+      - run:
+          name: Save Docker Image to Workspace
+          command: |
+            mkdir -p /tmp/workspace
+            docker save -o /tmp/workspace/autopush-integration-tests.tar autopush-integration-tests
+      - persist_to_workspace:
+          root: /tmp/workspace
+          paths:
+            - autopush-integration-tests.tar
+
   deploy:
     executor: gcp-gcr/default
     parameters:
@@ -481,6 +476,13 @@ workflows:
             tags:
               only: /.*/
 
+      - build-integration-test:
+          filters:
+            tags:
+              only: /.*/
+            branches:
+              only: master
+
       # Comment out the following two sections for local CircleCI testing.
       - deploy:
           name: deploy-autoconnect

Makefile

@@ -3,7 +3,8 @@ CARGO = cargo
 TESTS_DIR := tests
 TEST_RESULTS_DIR ?= workspace/test-results
 PYTEST_ARGS ?= $(if $(SKIP_SENTRY),-m "not sentry") $(if $(TEST_STUB),,-m "not stub") # Stub tests do not work in CI
-INTEGRATION_TEST_FILE := $(TESTS_DIR)/integration/test_integration_all_rust.py
+INTEGRATION_TEST_DIR := $(TESTS_DIR)/integration
+INTEGRATION_TEST_FILE := $(INTEGRATION_TEST_DIR)/test_integration_all_rust.py
 NOTIFICATION_TEST_DIR := $(TESTS_DIR)/notification
 LOAD_TEST_DIR := $(TESTS_DIR)/load
 POETRY := poetry --directory $(TESTS_DIR)
@@ -28,14 +29,23 @@ upgrade:
 	$(CARGO) upgrade
 	$(CARGO) update
 
+integration-test:
+	$(DOCKER_COMPOSE) -f $(INTEGRATION_TEST_DIR)/docker-compose.yml build
+	$(DOCKER_COMPOSE) -f $(INTEGRATION_TEST_DIR)/docker-compose.yml run -it --name integration-tests tests
+	docker cp integration-tests:/code/integration_test_results.xml $(INTEGRATION_TEST_DIR)
+
+integration-test-clean:
+	$(DOCKER_COMPOSE) -f $(INTEGRATION_TEST_DIR)/docker-compose.yml down
+	docker rm integration-tests
+
 integration-test-legacy:
 	$(POETRY) -V
 	$(POETRY) install --without dev,load,notification --no-root
 	$(POETRY) run pytest $(INTEGRATION_TEST_FILE) \
 		--junit-xml=$(TEST_RESULTS_DIR)/integration_test_legacy_results.xml \
 		-v $(PYTEST_ARGS)
 
-integration-test:
+integration-test-local:
 	$(POETRY) -V
 	$(POETRY) install --without dev,load,notification --no-root
 		$(POETRY) run pytest $(INTEGRATION_TEST_FILE) \

docs/src/testing.md

@@ -78,15 +78,15 @@ $ pyenv activate push-312
 5. Run `poetry install` to install all dependencies for testing.
 
 ### Running Integration Tests
-To run the integration tests, simply run `make integration-tests` from your terminal at the root of the project.
+To run the integration tests, simply run `make integration-tests-local` from your terminal at the root of the project.
 
 You can alter the verbosity and logging output by adding command line flags to the `PYTEST_ARGS ?=` variable in the root project Makefile. For example, for greater verbosity and stdout printing, add `-vv -s`.
 
 The test output is then emitted in your terminal instance. This includes the name of the tests, whether they pass or fail and any exceptions that are triggered during the test run.
 
 The integration tests make use of [pytest markers][pytest_markers] for filtering tests. These can be
 used with the `-m` pytest option, or can be used through the following environment variables and
-`integration-test` make command.
+`integration-test-local` make command.
 
 | ENVIRONMENT VARIABLE | RELATED MARKER | DESCRIPTION                                                       |
 |----------------------|----------------|-------------------------------------------------------------------|
@@ -95,6 +95,9 @@ used with the `-m` pytest option, or can be used through the following environme
 
 Integration tests in CI will be triggered automatically whenever a commit is pushed to a branch as a part of the CI PR workflow.
 
+#### Using Docker to run the Integration Tests.
+If you aren't needing to run specific tests you can use the containerized version of the integration tests with the following make command: `make integration-test`. This will build a docker image and set up the Big Table emulator as well as execute a default set of tests with the `not stub` marker.
+
 ### Debugging
 In some instances after making test changes, the test client can potentially hang in a dangling process. This can result in inaccurate results or tests not running correctly. You can run the following commands to determine the PID's of the offending processes and terminate them:
 ```shell
@@ -145,4 +148,4 @@ For more details see the [README.md][load_tests_docs] file in the `tests/load` d
 [integration_tests_docs]: ./testing.md#integration-tests
 [load_tests]: https://github.com/mozilla-services/autopush-rs/tree/master/tests/load
 [load_tests_docs]: https://github.com/mozilla-services/autopush-rs/blob/master/tests/load/README.md
-[pytest_markers]: https://docs.pytest.org/en/stable/example/markers.html
+[pytest_markers]: https://docs.pytest.org/en/stable/example/markers.html

tests/integration/Dockerfile

@@ -0,0 +1,62 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+FROM python:3.12-slim-bookworm
+
+LABEL org.opencontainers.image.authors="[email protected]"
+
+ENV LANG=C.UTF-8
+ENV PYTHONUNBUFFERED=1
+
+ENV PATH=$PATH:/root/.cargo/bin
+ENV PYTHON_VENV=/venv
+ENV PYTEST_ARGS=""
+ENV RUST_LOG="autopush=debug,autopush_common=debug,autoendpoint=debug,autoconnect=debug,slog_mozlog_json=info,warn"
+ENV DB_DSN=grpc://localhost:8086
+
+# Add gcc since there are no wheels for some packages for arm64/aarch64
+# (g++/make for gevent on pypy)
+RUN apt-get update && apt install -y --no-install-recommends \
+    git \
+    gpg \
+    build-essential \
+    python3-dev \
+    curl \
+    libstdc++6 \
+    libstdc++-12-dev \
+    libssl-dev \
+    pkg-config \
+    cmake
+
+RUN python -m venv ${PYTHON_VENV}
+ENV PATH="${PYTHON_VENV}/bin:${PATH}"
+
+RUN python -m pip install --upgrade pip
+
+# Install Rust
+RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.81 -y
+RUN rustc --version
+
+# Setup poetry and install requirements
+ENV POETRY_VIRTUALENVS_CREATE=false \
+  POETRY_VERSION=1.7.0
+RUN python -m pip install --no-cache-dir --quiet poetry
+COPY ./tests/pyproject.toml ./tests/poetry.lock ./
+RUN poetry install --only=integration --no-interaction --no-ansi
+
+# Setup cloud big table
+RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg
+RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+RUN apt-get update -y && apt install google-cloud-cli-cbt -y
+
+COPY . /code
+
+WORKDIR /code
+
+# Build app
+RUN cargo build --features=emulator
+
+RUN chmod +x scripts/setup_bt.sh
+
+CMD ["sh", "-c", "./scripts/setup_bt.sh && poetry run pytest tests/integration/test_integration_all_rust.py --junit-xml=integration_test_results.xml -v -m 'not stub' ${PYTEST_ARGS}"]

tests/integration/docker-compose.yml

@@ -0,0 +1,21 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+services:
+  google_cbt:
+    image: google/cloud-sdk:latest
+    network_mode: host
+    platform: linux/amd64
+    command: gcloud beta emulators bigtable start --host-port=localhost:8086
+  tests:
+    environment:
+      - BIGTABLE_EMULATOR_HOST=localhost:8086
+      - DB_DSN=grpc://localhost:8086
+    build:
+      context: ../..
+      dockerfile: tests/integration/Dockerfile
+    depends_on:
+      - google_cbt
+    network_mode: host
+    platform: linux/amd64