Initial commit adding autopush loadtesting config. (#391)

tests/load/.flake8

@@ -0,0 +1,5 @@
+[flake8]
+exclude =
+    .git,
+    __pycache__,
+max-line-length = 88

tests/load/Dockerfile

@@ -0,0 +1,41 @@
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+FROM python:3.11-slim
+
+LABEL org.opencontainers.image.authors="[email protected]"
+
+# Add gcc since there are no wheels for some packages for arm64/aarch64
+RUN apt-get update && apt-get install -y git && \
+  if [ -n "$(arch | grep 'arm64\|aarch64')" ]; then \
+    apt install -y --no-install-recommends gcc python3-dev; \
+  fi
+
+ENV LANG=C.UTF-8
+ENV PYTHONUNBUFFERED=1
+
+ENV PYTHON_VENV=/venv
+RUN python -m venv ${PYTHON_VENV}
+ENV PATH="${PYTHON_VENV}/bin:${PATH}"
+
+RUN python -m pip install --upgrade pip
+
+# Setup poetry and extract requirements
+RUN python -m pip install --no-cache-dir --quiet poetry
+WORKDIR /tmp
+COPY pyproject.toml poetry.lock /tmp/
+RUN poetry export --no-interaction --output requirements.txt --without-hashes
+
+WORKDIR /
+RUN python -m pip install -r /tmp/requirements.txt
+
+RUN useradd --create-home locust
+WORKDIR /home/locust
+
+COPY ./locustfile.py locustfile.py
+
+# Expose ports for the web UI and the locust master
+EXPOSE 8089 5557
+
+USER locust
+ENTRYPOINT locust -f locustfile.py --websocket_url ${SERVER_URL} --endpoint_url ${ENDPOINT_URL}

tests/load/README.md

@@ -0,0 +1,86 @@
+# Autopush Load (Locust) Tests
+
+This directory contains the automated load test suite for autopush. These tests are run using a tool named [locust](https://locust.io/).
+
+## Contributing
+
+This project uses [Poetry][1] for dependency management. For environment setup it is 
+recommended to use [pyenv][2] and [pyenv-virtualenv][3], as they work nicely with 
+Poetry.
+
+Project dependencies are listed in the `pyproject.toml` file.
+To install the dependencies execute:
+```shell
+poetry install
+```
+
+Contributors to this project are expected to execute the following tools for import 
+sorting, linting, style guide enforcement and static type checking.
+Configurations are set in the `pyproject.toml` and `.flake8` files.
+
+**[isort][4]**
+ ```shell
+poetry run isort locustfile.py
+ ```
+
+**[black][5]**
+ ```shell
+poetry run black locustfile.py
+ ```
+
+**[flake8][6]**
+ ```shell
+poetry run flake8 locustfile.py
+ ```
+
+
+## Local Execution
+
+There are 3 docker-compose files pertaining to the different environments autopush can run on:
+`docker-compose.prod.yml`
+`docker-compose.stage.yml`
+`docker-compose.dev.yml`
+
+You can select which environment you want to run against by choosing the appropriate file. The environment will be setup for you with the correct URLs.
+
+Ex:
+```shell
+docker-compose -f docker-compose.stage.yml up --build --scale locust_worker=1
+```
+
+This will run build and start the locust session for the Stage environment.
+
+
+### Run Test Session
+
+#### 1. Start Load Test
+
+* In a browser navigate to `http://localhost:8089/`
+* Select "Start Swarming"
+
+#### 2. Stop Load Test
+
+Select the 'Stop' button in the top right hand corner of the Locust UI, after the 
+desired test duration has elapsed. If the 'Run time' or 'Duration' is set in step 1, 
+the load test will stop automatically.
+
+#### 3. Analyse Results
+
+* Only client-side measures, provided by Locust, are available for local execution
+
+### Clean-up Environment
+
+#### 1. Remove Load Test Docker Containers
+
+Execute the following from the `load` directory:
+```shell
+docker-compose -f docker-compose.stage.yml down
+docker rmi locust
+```
+
+[1]: https://python-poetry.org/docs/#installation
+[2]: https://github.com/pyenv/pyenv#installation
+[3]: https://github.com/pyenv/pyenv-virtualenv#installation
+[4]: https://pycqa.github.io/isort/
+[5]: https://black.readthedocs.io/en/stable/
+[6]: https://flake8.pycqa.org/en/latest/

tests/load/docker-compose.yml

@@ -0,0 +1,29 @@
+version: "3"
+services:
+  locust_master:
+    image: locust
+    build: .
+    container_name: locust_master
+    ports:
+      - "8089:8089"
+    environment:
+      LOCUST_HOST: ${AUTOPUSH_HOST}
+      LOCUST_USERS: "20"
+      LOCUST_SPAWN_RATE: "1"
+      LOCUST_RUN_TIME: "10m"
+      LOCUST_LOGLEVEL: "INFO"
+      SERVER_URL: ${SERVER_URL}
+      ENDPOINT_URL: ${ENDPOINT_URL}
+    command: >
+      --master
+
+  locust_worker:
+    image: locust
+    build: .
+    environment:
+      LOCUST_MASTER_NODE_HOST: locust_master
+      LOCUST_LOGLEVEL: "INFO"
+      SERVER_URL: ${SERVER_URL}
+      ENDPOINT_URL: ${ENDPOINT_URL}
+    command: >
+      --worker

tests/load/kubernetes-config/locust-master-controller.yml

@@ -0,0 +1,46 @@
+apiVersion: "apps/v1"
+kind: "Deployment"
+metadata:
+  name: locust-master
+  labels:
+    name: locust-master
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: locust-master
+  template:
+    metadata:
+      labels:
+        app: locust-master
+    spec:
+      containers:
+        - name: locust-master
+          image: gcr.io/[PROJECT_ID]/locust-autopush:[LOCUST_IMAGE_TAG]
+          env:
+            - name: LOCUST_MODE_MASTER
+              value: "true"
+            - name: LOCUST_CSV
+              value: [LOCUST_CSV]
+            - name: LOCUST_USERS
+              value: "[LOCUST_USERS]"
+            - name: LOCUST_SPAWN_RATE
+              value: "[LOCUST_SPAWN_RATE]"
+            - name: LOCUST_RUN_TIME
+              value: "[LOCUST_RUN_TIME]"
+            - name: LOCUST_LOGLEVEL
+              value: [LOCUST_LOGLEVEL]
+            - name: SERVER_URL
+              value: [SERVER_URL]
+            - name: ENDPOINT_URL
+              value: [ENDPOINT_URL]
+          ports:
+            - name: loc-master-web
+              containerPort: 8089
+              protocol: TCP
+            - name: loc-master-p1
+              containerPort: 5557
+              protocol: TCP
+            - name: loc-master-p2
+              containerPort: 5558
+              protocol: TCP

tests/load/kubernetes-config/locust-master-service.yml

@@ -0,0 +1,23 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: locust-master
+  labels:
+    app: locust-master
+spec:
+  ports:
+    - port: 8089
+      targetPort: loc-master-web
+      protocol: TCP
+      name: loc-master-web
+    - port: 5557
+      targetPort: loc-master-p1
+      protocol: TCP
+      name: loc-master-p1
+    - port: 5558
+      targetPort: loc-master-p2
+      protocol: TCP
+      name: loc-master-p2
+  selector:
+    app: locust-master
+  type: LoadBalancer

tests/load/kubernetes-config/locust-worker-controller.yml

@@ -0,0 +1,30 @@
+apiVersion: "apps/v1"
+kind: "Deployment"
+metadata:
+  name: locust-worker
+  labels:
+    name: locust-worker
+spec:
+  replicas: [WORKER_COUNT]
+  selector:
+    matchLabels:
+      app: locust-worker
+  template:
+    metadata:
+      labels:
+        app: locust-worker
+    spec:
+      containers:
+        - name: locust-worker
+          image: gcr.io/[PROJECT_ID]/locust-autopush:[LOCUST_IMAGE_TAG]
+          env:
+            - name: LOCUST_MODE_WORKER
+              value: "true"
+            - name: LOCUST_MASTER_NODE_HOST
+              value: locust-master
+            - name: LOCUST_LOGLEVEL
+              value: [LOCUST_LOGLEVEL]
+            - name: SERVER_URL
+              value: [SERVER_URL]
+            - name: ENDPOINT_URL
+              value: [ENDPOINT_URL]