✨ Add metamodeling load tests

tests/environment-setup/test_used_docker_compose.py

@@ -76,7 +76,9 @@ def ensure_env_file(env_devel_file: Path) -> Iterable[Path]:
 
 
 def _skip_not_useful_docker_composes(p) -> bool:
-    return "osparc-gateway-server" not in f"{p}" and "manual" not in f"{p}"
+    result = "osparc-gateway-server" not in f"{p}" and "manual" not in f"{p}"
+    result &= "tests/performance" not in f"{p}"
+    return result
 
 
 compose_paths = filter(

tests/performance/Dockerfile

@@ -6,6 +6,9 @@ FROM locustio/locust:${LOCUST_VERSION}
 RUN pip3 --version && \
   pip3 install \
   faker \
+  locust-plugins==2.1.1 \
+  pydantic \
+  pydantic-settings \
   python-dotenv \
-  locust-plugins==2.1.1 &&\
+  tenacity && \
   pip3 freeze --verbose

tests/performance/Makefile

@@ -3,7 +3,7 @@
 #
 include ../../scripts/common.Makefile
 
-LOCUST_VERSION=2.5.1
+LOCUST_VERSION=2.29.1
 export LOCUST_VERSION
 
 # UTILS
@@ -42,9 +42,9 @@ down: ## stops and removes osparc locust containers
 	docker compose --file docker-compose.yml down
 
 .PHONY: test
-test: ## runs osparc locust with target=locust_test_file.py in headless mode for a minute. Will fail if 5% more fail requests or average response time is above 50ms, optional host can be set
+test: ## runs osparc locust. locust and test confiuration are specified in .env file next to target file
 	@$(call check_defined, target, please define target file when calling $@ - e.g. ```make $@ target=MY_LOCUST_FILE.py```)
 	@export LOCUST_FILE=$(target); \
-	export TARGET_URL=$(if $(host),$(host),"http://$(get_my_ip):9081"); \
-	export LOCUST_OPTIONS="--headless --print-stats --users=100 --spawn-rate=20 --run-time=1m --check-fail-ratio=0.01 --check-avg-response-time=$(if $(resp_time),$(resp_time),200)"; \
+	export ENV_FILE=$$(dirname $$(realpath locust_files/$${LOCUST_FILE}))/.env; \
+	if [ ! -f $${ENV_FILE} ]; then cp $$(dirname $${ENV_FILE})/.env-devel $${ENV_FILE}; fi; \
 	docker compose --file docker-compose.yml up --scale worker=4 --exit-code-from=master

tests/performance/docker-compose.yml

@@ -7,14 +7,14 @@ services:
       - ./locust_files:/mnt/locust
       - ./locust_report:/reporting
     command: >
-      -f /mnt/locust/${LOCUST_FILE} --host ${TARGET_URL} --html
-      /reporting/locust_html.html ${LOCUST_OPTIONS} --master
+      -f /mnt/locust/${LOCUST_FILE} --html /reporting/locust_html.html --master
+    env_file:
+      - ${ENV_FILE}
 
   worker:
     image: itisfoundation/locust:${LOCUST_VERSION}
     volumes:
       - ./locust_files:/mnt/locust
     command: -f /mnt/locust/${LOCUST_FILE} --worker --master-host master
-    environment:
-      - SC_USER_NAME=${SC_USER_NAME}
-      - SC_PASSWORD=${SC_PASSWORD}
+    env_file:
+      - ${ENV_FILE}

tests/performance/locust_files/.env-devel

@@ -0,0 +1,11 @@
+# user
+SC_USER_NAME=<username>
+SC_PASSWORD=<password>
+
+# locust settings
+LOCUST_HOST=<host (eg https://api.osparc-master.speag.com/)>
+LOCUST_USERS=100
+LOCUST_HEADLESS=true
+LOCUST_PRINT_STATS=true
+LOCUST_SPAWN_RATE=20
+LOCUST_RUN_TIME=1m

tests/performance/locust_files/metamodeling/.env-devel

@@ -0,0 +1,12 @@
+# user
+OSPARC_API_KEY=<osparc api key>
+OSPARC_API_SECRET=<osparc api secret>
+TEMPLATE_UUID=<template uuid>
+
+# locust settings
+LOCUST_HOST=<api server url>
+LOCUST_USERS=100
+LOCUST_HEADLESS=true
+LOCUST_PRINT_STATS=true
+LOCUST_SPAWN_RATE=20
+LOCUST_RUN_TIME=1m

tests/performance/locust_files/metamodeling/passer.py

@@ -0,0 +1,22 @@
+import json
+import os
+from pathlib import Path
+
+
+def main():
+
+    input_path = Path(os.environ["INPUT_FOLDER"])
+    output_path = Path(os.environ["OUTPUT_FOLDER"])
+
+    input_file_path = input_path / "input.json"
+    output_file_path = output_path / "output.json"
+
+    input_content = json.loads(input_file_path.read_text())
+
+    print(input_content)
+
+    output_file_path.write_text(json.dumps(input_content))
+
+
+if __name__ == "__main__":
+    main()

tests/performance/locust_files/metamodeling/workflow.py

@@ -0,0 +1,163 @@
+from datetime import timedelta
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Final
+from uuid import UUID
+
+from locust import HttpUser, task
+from pydantic import Field
+from pydantic_settings import BaseSettings
+from requests.auth import HTTPBasicAuth
+from tenacity import (
+    Retrying,
+    retry_if_exception_type,
+    stop_after_delay,
+    wait_exponential,
+)
+from urllib3 import PoolManager, Retry
+
+_MAX_WAIT_SECONDS: Final[int] = 60
+
+
+# Perform the following setup in order to run this load test:
+# 1. Copy .env-devel to .env in this directory and add your osparc keys to .env.
+# 2. Construct a study **template** according to study_template.png. passer.py is the file next to this file.
+# 3. Setup the locust settings in the .env file (see https://docs.locust.io/en/stable/configuration.html#all-available-configuration-options)
+# run 'make test target=metamodeling/workflow.py' in your terminal and watch the magic happen 🤩
+
+
+class UserSettings(BaseSettings):
+    osparc_api_key: str = Field(default=...)
+    osparc_api_secret: str = Field(default=...)
+
+    template_uuid: UUID = Field(default=...)
+
+
+class MetaModelingUser(HttpUser):
+    def __init__(self, *args, **kwargs):
+        self._user_settings = UserSettings()
+        self._auth = HTTPBasicAuth(
+            username=self._user_settings.osparc_api_key,
+            password=self._user_settings.osparc_api_secret,
+        )
+        retry_strategy = Retry(
+            total=4,
+            backoff_factor=4.0,
+            status_forcelist={429, 503, 504},
+            allowed_methods={
+                "DELETE",
+                "GET",
+                "HEAD",
+                "OPTIONS",
+                "PUT",
+                "TRACE",
+                "POST",
+                "PATCH",
+                "CONNECT",
+            },
+            respect_retry_after_header=True,
+            raise_on_status=True,
+        )
+        self.pool_manager = PoolManager(retries=retry_strategy)
+
+        self._input_json_uuid = None
+        self._job_uuid = None
+
+        super().__init__(*args, **kwargs)
+
+    def on_start(self) -> None:
+        self.client.get("/v0/me", auth=self._auth)  # fail fast
+
+    def on_stop(self) -> None:
+        if self._input_json_uuid is not None:
+            response = self.client.delete(
+                f"/v0/files/{self._input_json_uuid}", name="/v0/files/[file_id]"
+            )
+            response.raise_for_status()
+        if self._job_uuid is not None:
+            response = self.client.delete(
+                f"/v0/studies/{self._user_settings.template_uuid}/jobs/{self._job_uuid}",
+                name="/v0/studies/[study_id]/jobs/[job_id]",
+            )
+            response.raise_for_status()
+
+    @task
+    def create_and_run_job(self):
+        # upload file
+        with TemporaryDirectory() as tmp_dir:
+            file = Path(tmp_dir) / "input.json"
+            file.write_text(
+                """
+                {
+                    "f1": 3
+                }
+                """
+            )
+            self._input_json_uuid = self.upload_file(file)
+
+        # create job
+        response = self.client.post(
+            f"/v0/studies/{self._user_settings.template_uuid}/jobs",
+            json={
+                "values": {"InputFile1": f"{self._input_json_uuid}"},
+            },
+            auth=self._auth,
+            name="/v0/studies/[study_id]/jobs",
+        )
+        response.raise_for_status()
+        job_uuid = response.json().get("id")
+        assert job_uuid is not None
+        self._job_uuid = UUID(job_uuid)
+
+        # start job
+        response = self.client.post(
+            f"/v0/studies/{self._user_settings.template_uuid}/jobs/{self._job_uuid}:start",
+            auth=self._auth,
+            name="/v0/studies/[study_id]/jobs/[job_id]:start",
+        )
+        response.raise_for_status()
+        state = response.json().get("state")
+        for attempt in Retrying(
+            stop=stop_after_delay(timedelta(seconds=_MAX_WAIT_SECONDS)),
+            wait=wait_exponential(),
+            retry=retry_if_exception_type(RuntimeError),
+        ):
+            with attempt:
+                response = self.client.post(
+                    f"/v0/studies/{self._user_settings.template_uuid}/jobs/{self._job_uuid}:inspect",
+                    auth=self._auth,
+                    name="/v0/studies/[study_id]/jobs/[job_id]:inspect",
+                )
+                response.raise_for_status()
+                state = response.json().get("state")
+                if not state in {"SUCCESS", "FAILED"}:
+                    raise RuntimeError(
+                        f"Computation not finished after attempt {attempt.retry_state.attempt_number}"
+                    )
+
+        assert state == "SUCCESS"
+
+        response = self.client.post(
+            f"/v0/studies/{self._user_settings.template_uuid}/jobs/{self._job_uuid}/outputs",
+            auth=self._auth,
+            name="/v0/studies/[study_id]/jobs/[job_id]/outputs",
+        )
+        response.raise_for_status()
+        results = response.json()
+        assert results is not None
+        output_file = results.get("OutputFile1")
+        assert output_file is not None
+        output_file_uuid = output_file.get("id")
+        assert output_file_uuid is not None
+
+    def upload_file(self, file: Path) -> UUID:
+        assert file.is_file()
+        with open(f"{file.resolve()}", "rb") as f:
+            files = {"file": f}
+            response = self.client.put(
+                "/v0/files/content", files=files, auth=self._auth
+            )
+            response.raise_for_status()
+            file_uuid = response.json().get("id")
+        assert file_uuid is not None
+        return UUID(file_uuid)