✨ Add metamodeling load tests

tests/performance/Dockerfile

@@ -6,6 +6,9 @@ FROM locustio/locust:${LOCUST_VERSION}
 RUN pip3 --version && \
   pip3 install \
   faker \
+  pydantic \
+  pydantic-settings \
   python-dotenv \
+  tenacity \
   locust-plugins==2.1.1 &&\
   pip3 freeze --verbose

tests/performance/Makefile

@@ -3,7 +3,7 @@
 #
 include ../../scripts/common.Makefile
 
-LOCUST_VERSION=2.5.1
+LOCUST_VERSION=2.29.1
 export LOCUST_VERSION
 
 # UTILS
@@ -42,9 +42,9 @@ down: ## stops and removes osparc locust containers
 	docker compose --file docker-compose.yml down
 
 .PHONY: test
-test: ## runs osparc locust with target=locust_test_file.py in headless mode for a minute. Will fail if 5% more fail requests or average response time is above 50ms, optional host can be set
+test: ## runs osparc locust. locust and test confiuration are specified in .env file next to target file
 	@$(call check_defined, target, please define target file when calling $@ - e.g. ```make $@ target=MY_LOCUST_FILE.py```)
 	@export LOCUST_FILE=$(target); \
-	export TARGET_URL=$(if $(host),$(host),"http://$(get_my_ip):9081"); \
-	export LOCUST_OPTIONS="--headless --print-stats --users=100 --spawn-rate=20 --run-time=1m --check-fail-ratio=0.01 --check-avg-response-time=$(if $(resp_time),$(resp_time),200)"; \
+	export ENV_FILE=$$(dirname $$(realpath locust_files/$${LOCUST_FILE}))/.env; \
+	if [ ! -f $${ENV_FILE} ]; then cp $$(dirname $${ENV_FILE})/.env-devel $${ENV_FILE}; fi; \
 	docker compose --file docker-compose.yml up --scale worker=4 --exit-code-from=master

tests/performance/docker-compose.yml

@@ -7,14 +7,14 @@ services:
       - ./locust_files:/mnt/locust
       - ./locust_report:/reporting
     command: >
-      -f /mnt/locust/${LOCUST_FILE} --host ${TARGET_URL} --html
-      /reporting/locust_html.html ${LOCUST_OPTIONS} --master
+      -f /mnt/locust/${LOCUST_FILE} --html /reporting/locust_html.html --master
+    env_file:
+      - ${ENV_FILE}
 
   worker:
     image: itisfoundation/locust:${LOCUST_VERSION}
     volumes:
       - ./locust_files:/mnt/locust
     command: -f /mnt/locust/${LOCUST_FILE} --worker --master-host master
-    environment:
-      - SC_USER_NAME=${SC_USER_NAME}
-      - SC_PASSWORD=${SC_PASSWORD}
+    env_file:
+      - ${ENV_FILE}

tests/performance/locust_files/.env-devel

@@ -0,0 +1,11 @@
+# user
+SC_USER_NAME=
+SC_PASSWORD=
+
+# locust settings
+LOCUST_HOST=http://10.43.103.193.nip.io:8005
+LOCUST_USERS=100
+LOCUST_HEADLESS=true
+LOCUST_PRINT_STATS=true
+LOCUST_SPAWN_RATE=20
+LOCUST_RUN_TIME=1m

tests/performance/locust_files/metamodeling/.env-devel

@@ -0,0 +1,12 @@
+# user
+OSPARC_API_KEY=
+OSPARC_API_SECRET=
+TEMPLATE_UUID=
+
+# locust settings
+LOCUST_HOST=http://10.43.103.193:9081
+LOCUST_USERS=100
+LOCUST_HEADLESS=true
+LOCUST_PRINT_STATS=true
+LOCUST_SPAWN_RATE=20
+LOCUST_RUN_TIME=1m

tests/performance/locust_files/metamodeling/passer.py

@@ -0,0 +1,15 @@
+import json
+import os
+import pathlib as pl
+
+input_path = pl.Path(os.environ["INPUT_FOLDER"])
+output_path = pl.Path(os.environ["OUTPUT_FOLDER"])
+
+input_file_path = input_path / "input.json"
+output_file_path = output_path / "output.json"
+
+input_content = json.loads(input_file_path.read_text())
+
+print(input_content)
+
+output_file_path.write_text(json.dumps(input_content))

tests/performance/locust_files/metamodeling/workflow.py

@@ -0,0 +1,159 @@
+from datetime import timedelta
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Final
+from uuid import UUID
+
+from locust import HttpUser, task
+from pydantic import Field
+from pydantic_settings import BaseSettings
+from requests.auth import HTTPBasicAuth
+from tenacity import (
+    Retrying,
+    retry_if_exception_type,
+    stop_after_delay,
+    wait_exponential,
+)
+from urllib3 import PoolManager, Retry
+
+_MAX_WAIT_SECONDS: Final[int] = 60
+
+
+# Perform the following setup in order to run this load test:
+# 1. Copy .env-devel to .env in this directory and add your osparc keys to .env.
+# 2. Construct a study **template** according to study_template.png. passer.py is the file next to this file.
+# 3. Setup the locust settings in the .env file (see https://docs.locust.io/en/stable/configuration.html#all-available-configuration-options)
+# run 'make test target=metamodeling/workflow.py' in your terminal and watch the magic happen 🤩
+
+
+class UserSettings(BaseSettings):
+    osparc_api_key: str = Field(default=...)
+    osparc_api_secret: str = Field(default=...)
+
+    template_uuid: UUID = Field(default=...)
+
+
+class MetaModelingUser(HttpUser):
+    def __init__(self, *args, **kwargs):
+        self._user_settings = UserSettings()
+        self._auth = HTTPBasicAuth(
+            username=self._user_settings.osparc_api_key,
+            password=self._user_settings.osparc_api_secret,
+        )
+        retry_strategy = Retry(
+            total=4,
+            backoff_factor=4.0,
+            status_forcelist={429, 503, 504},
+            allowed_methods={
+                "DELETE",
+                "GET",
+                "HEAD",
+                "OPTIONS",
+                "PUT",
+                "TRACE",
+                "POST",
+                "PATCH",
+                "CONNECT",
+            },
+            respect_retry_after_header=True,
+            raise_on_status=True,
+        )
+        self.pool_manager = PoolManager(retries=retry_strategy)
+
+        super().__init__(*args, **kwargs)
+
+    def on_start(self) -> None:
+        self.client.get("/v0/me", auth=self._auth)  # fail fast
+        self._input_json_uuid = None
+        self._job_uuid = None
+
+    def on_stop(self) -> None:
+        if self._input_json_uuid is not None:
+            response = self.client.delete(
+                f"/v0/files/{self._input_json_uuid}", name="/v0/files/[file_id]"
+            )
+            response.raise_for_status()
+        if self._job_uuid is not None:
+            response = self.client.delete(
+                f"/v0/studies/{self._user_settings.template_uuid}/jobs/{self._job_uuid}",
+                name="/v0/studies/[study_id]/jobs/[job_id]",
+            )
+            response.raise_for_status()
+
+    @task
+    def create_and_run_job(self):
+        # upload file
+        with TemporaryDirectory() as tmp_dir:
+            file = Path(tmp_dir) / "input.json"
+            file.write_text(
+                """
+                {
+                    "f1": 3
+                }
+                """
+            )
+            self._input_json_uuid = self.upload_file(file)
+
+        # create job
+        response = self.client.post(
+            f"/v0/studies/{self._user_settings.template_uuid}/jobs",
+            json={
+                "values": {"InputFile1": f"{self._input_json_uuid}"},
+            },
+            auth=self._auth,
+            name="/v0/studies/[study_id]/jobs",
+        )
+        response.raise_for_status()
+        job_uuid = response.json().get("id")
+        assert job_uuid is not None
+        self._job_uuid = UUID(job_uuid)
+
+        # start job
+        response = self.client.post(
+            f"/v0/studies/{self._user_settings.template_uuid}/jobs/{self._job_uuid}:start",
+            auth=self._auth,
+            name="/v0/studies/[study_id]/jobs/[job_id]:start",
+        )
+        response.raise_for_status()
+        state = response.json().get("state")
+        for attempt in Retrying(
+            stop=stop_after_delay(timedelta(seconds=_MAX_WAIT_SECONDS)),
+            wait=wait_exponential(),
+            retry=retry_if_exception_type(RuntimeError),
+        ):
+            with attempt:
+                response = self.client.post(
+                    f"/v0/studies/{self._user_settings.template_uuid}/jobs/{self._job_uuid}:inspect",
+                    auth=self._auth,
+                    name="/v0/studies/[study_id]/jobs/[job_id]:inspect",
+                )
+                response.raise_for_status()
+                state = response.json().get("state")
+                if not state in {"SUCCESS", "FAILED"}:
+                    raise RuntimeError(
+                        f"Computation not finished after attempt {attempt.retry_state.attempt_number}"
+                    )
+
+        assert state == "SUCCESS"
+
+        response = self.client.post(
+            f"/v0/studies/{self._user_settings.template_uuid}/jobs/{self._job_uuid}/outputs",
+            auth=self._auth,
+            name="/v0/studies/[study_id]/jobs/[job_id]/outputs",
+        )
+        response.raise_for_status()
+        results = response.json()
+        assert results is not None
+        output_file = results.get("OutputFile1")
+        assert output_file is not None
+        output_file_uuid = output_file.get("id")
+        assert output_file_uuid is not None
+
+    def upload_file(self, file: Path) -> UUID:
+        assert file.is_file()
+        files = {"file": open(f"{file.resolve()}", "rb")}
+        response = self.client.put("/v0/files/content", files=files, auth=self._auth)
+        response.raise_for_status()
+        file_uuid = response.json().get("id")
+        assert file_uuid is not None
+        return UUID(file_uuid)