From a010de5e7240c2f8fecf7797cd643ed21a07d15b Mon Sep 17 00:00:00 2001 From: Adrien4193 <39053578+Adrien4193@users.noreply.github.com> Date: Wed, 31 Jul 2024 07:54:53 +0200 Subject: [PATCH] BRAYNS-653 Add object endpoints (#1275) --- python/.gitignore | 1 + python/README.md | 90 +++++++++- python/brayns/__init__.py | 52 +++++- .../api/__init__.py} | 23 --- python/brayns/api/core/__init__.py | 19 ++ python/brayns/api/core/service.py | 162 ++++++++++++++++++ python/brayns/network/connection.py | 129 +++++++++++--- python/brayns/network/json_rpc.py | 113 ++++++------ python/brayns/network/websocket.py | 13 +- python/brayns/utils/__init__.py | 19 ++ python/brayns/utils/parsing.py | 80 +++++++++ python/brayns/version.py | 4 +- python/requirements-dev.txt | 2 + python/tests/integration/__init__.py | 19 ++ python/tests/integration/conftest.py | 69 ++++++++ python/tests/integration/core/__init__.py | 19 ++ python/tests/integration/core/test_service.py | 74 ++++++++ python/tests/unit/__init__.py | 19 ++ python/tests/unit/network/__init__.py | 19 ++ python/tests/unit/network/mock_websocket.py | 94 ++++++++++ python/tests/unit/network/test_connection.py | 90 ++++++++++ python/tests/unit/network/test_json_rpc.py | 77 +++++++++ src/brayns/core/Launcher.cpp | 9 +- src/brayns/core/api/Api.cpp | 34 ++-- src/brayns/core/api/Api.h | 2 +- src/brayns/core/api/ApiBuilder.h | 115 +++++++------ src/brayns/core/api/Endpoint.h | 6 +- src/brayns/core/api/Progress.cpp | 46 +++-- src/brayns/core/api/Progress.h | 43 +++-- src/brayns/core/api/Task.h | 13 +- src/brayns/core/endpoints/ObjectEndpoints.cpp | 113 ++++++++++++ src/brayns/core/endpoints/ObjectEndpoints.h | 46 +++++ ...CoreEndpoints.cpp => ServiceEndpoints.cpp} | 16 +- .../{CoreEndpoints.h => ServiceEndpoints.h} | 3 +- src/brayns/core/json/types/Objects.h | 22 ++- src/brayns/core/jsonrpc/Messages.h | 45 ++--- src/brayns/core/jsonrpc/Parser.cpp | 10 +- src/brayns/core/jsonrpc/Parser.h | 8 +- src/brayns/core/objects/ObjectManager.cpp | 12 +- src/brayns/core/objects/ObjectManager.h | 9 +- src/brayns/core/service/Service.cpp | 116 +++++++------ tests/core/api/TestApi.cpp | 12 +- tests/core/json/TestJsonReflection.cpp | 4 + tests/core/jsonrpc/TestJsonRpc.cpp | 16 +- tests/core/objects/TestObjectManager.cpp | 4 +- 45 files changed, 1528 insertions(+), 363 deletions(-) rename python/{tests/test_json_rpc.py => brayns/api/__init__.py} (58%) create mode 100644 python/brayns/api/core/__init__.py create mode 100644 python/brayns/api/core/service.py create mode 100644 python/brayns/utils/__init__.py create mode 100644 python/brayns/utils/parsing.py create mode 100644 python/tests/integration/__init__.py create mode 100644 python/tests/integration/conftest.py create mode 100644 python/tests/integration/core/__init__.py create mode 100644 python/tests/integration/core/test_service.py create mode 100644 python/tests/unit/__init__.py create mode 100644 python/tests/unit/network/__init__.py create mode 100644 python/tests/unit/network/mock_websocket.py create mode 100644 python/tests/unit/network/test_connection.py create mode 100644 python/tests/unit/network/test_json_rpc.py create mode 100644 src/brayns/core/endpoints/ObjectEndpoints.cpp create mode 100644 src/brayns/core/endpoints/ObjectEndpoints.h rename src/brayns/core/endpoints/{CoreEndpoints.cpp => ServiceEndpoints.cpp} (90%) rename src/brayns/core/endpoints/{CoreEndpoints.h => ServiceEndpoints.h} (90%) diff --git a/python/.gitignore b/python/.gitignore index e15eb2541..34c1fba7b 100644 --- a/python/.gitignore +++ b/python/.gitignore @@ -2,6 +2,7 @@ .pytest_cache .ruff_cache .vscode +.env venv __pycache__ *.pyc diff --git a/python/README.md b/python/README.md index 41771b37a..af1aeac35 100644 --- a/python/README.md +++ b/python/README.md @@ -21,7 +21,7 @@ You can install this package from [PyPI](https://pypi.org/): pip install brayns ``` -Or from source: +The package can also be installed from source: ```bash git clone https://github.com/BlueBrain/Brayns.git @@ -29,11 +29,99 @@ cd Brayns/python pip install . ``` +To install packages that are only required for development: + +```bash +pip install -r requirements-dev.txt +``` + ## Usage -------- TODO +## Tests + +To run the tests, use + +```bash +pytest tests +``` + +## Lint + +To format and lint use + +```bash +ruff format brayns +ruff check brayns +mypy brayns +``` + +## VSCode integration + +From the current directory (Brayns/python): + +1. Create a venv + +```bash +python3.11 -m venv venv +source venv/bin/activate +``` + +2. Install requirements for development: + +```bash +pip install -r requirements.txt +pip install -r requirements-dev.txt +``` + +3. For integration testing, create a `.env` file: + +```bash +BRAYNS_HOST=localhost +BRAYNS_PORT=5000 +BRAYNS_EXECUTABLE=path/to/braynsService +LD_LIBRARY_PATH=path/to/additional/libs +``` + +Note: integration testing can be disable using the pytest --without-integration flag. + +4. Create a .vscode folder and create a `launch.json` inside to use to debug tests: + +```json +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Debug Tests", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "purpose": [ + "debug-test" + ], + "console": "integratedTerminal", + "justMyCode": false + } + ] +} +``` + +5. In the same folder, create a `settings.json` to configure pytest: + +```json +{ + "python.analysis.typeCheckingMode": "basic", + "python.testing.pytestArgs": [ + "tests" + ], + "python.envFile": "${workspaceFolder}/.env", + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} +``` + ## Documentation ----------------- diff --git a/python/brayns/__init__.py b/python/brayns/__init__.py index 2a56e2b77..5c7b9e971 100644 --- a/python/brayns/__init__.py +++ b/python/brayns/__init__.py @@ -21,28 +21,64 @@ """ Brayns Python package. -This package provides a high level API to interact with an instance of Brayns -instance through websockets. - -The low level JSON-RPC API is also available using the instance directly. +This package provides an API to interact with Brayns service. """ -from .version import VERSION -from .network.connection import Connection, connect -from .network.json_rpc import JsonRpcError, JsonRpcErrorResponse, JsonRpcId, JsonRpcRequest, JsonRpcResponse +from .api.core.service import ( + Endpoint, + Task, + TaskInfo, + TaskOperation, + Version, + cancel_task, + get_endpoint, + get_methods, + get_task, + get_task_result, + get_tasks, + get_version, + stop_service, +) +from .network.connection import Connection, FutureResponse, Request, Response, connect +from .network.json_rpc import ( + JsonRpcError, + JsonRpcErrorResponse, + JsonRpcId, + JsonRpcRequest, + JsonRpcResponse, + JsonRpcSuccessResponse, +) from .network.websocket import ServiceUnavailable, WebSocketError +from .version import VERSION __version__ = VERSION """Version tag of brayns Python package (major.minor.patch).""" __all__ = [ - "Connection", + "cancel_task", "connect", + "Connection", + "Endpoint", + "FutureResponse", + "get_endpoint", + "get_methods", + "get_task_result", + "get_task", + "get_tasks", + "get_version", "JsonRpcError", "JsonRpcErrorResponse", "JsonRpcId", "JsonRpcRequest", "JsonRpcResponse", + "JsonRpcSuccessResponse", + "Request", + "Response", "ServiceUnavailable", + "stop_service", + "Task", + "TaskInfo", + "TaskOperation", + "Version", "WebSocketError", ] diff --git a/python/tests/test_json_rpc.py b/python/brayns/api/__init__.py similarity index 58% rename from python/tests/test_json_rpc.py rename to python/brayns/api/__init__.py index 44a7637fd..66765f01e 100644 --- a/python/tests/test_json_rpc.py +++ b/python/brayns/api/__init__.py @@ -17,26 +17,3 @@ # You should have received a copy of the GNU Lesser General Public License # along with this library; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -from brayns.network.json_rpc import JsonRpcRequest, compose_request - - -def test_compose_request() -> None: - request = JsonRpcRequest(0, "test", 123) - data = compose_request(request) - text = """{"jsonrpc":"2.0","id":0,"method":"test","params":123}""" - assert data == text - - request.binary = b"123" - - data = compose_request(request) - - assert len(data) == len(text) + 4 + 3 - - assert isinstance(data, bytes) - - size = int.from_bytes(data[:4], byteorder="little", signed=False) - - assert size == len(text) - assert data[4 : size + 4].decode() == text - assert data[size + 4 :] == request.binary diff --git a/python/brayns/api/core/__init__.py b/python/brayns/api/core/__init__.py new file mode 100644 index 000000000..66765f01e --- /dev/null +++ b/python/brayns/api/core/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) 2015-2024 EPFL/Blue Brain Project +# All rights reserved. Do not distribute without permission. +# +# Responsible Author: adrien.fleury@epfl.ch +# +# This file is part of Brayns +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License version 3.0 as published +# by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. diff --git a/python/brayns/api/core/service.py b/python/brayns/api/core/service.py new file mode 100644 index 000000000..0664664a8 --- /dev/null +++ b/python/brayns/api/core/service.py @@ -0,0 +1,162 @@ +# Copyright (c) 2015-2024 EPFL/Blue Brain Project +# All rights reserved. Do not distribute without permission. +# +# Responsible Author: adrien.fleury@epfl.ch +# +# This file is part of Brayns +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License version 3.0 as published +# by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any, Generic, TypeVar + +from brayns.network.connection import Connection, Response +from brayns.network.json_rpc import get + + +@dataclass +class Version: + major: int + minor: int + patch: int + pre_release: int + tag: str + + +async def get_version(connection: Connection) -> Version: + result = await connection.get_result("get-version") + + return Version( + major=get(result, "major", int), + minor=get(result, "minor", int), + patch=get(result, "patch", int), + pre_release=get(result, "pre_release", int), + tag=get(result, "tag", str), + ) + + +async def get_methods(connection: Connection) -> list[str]: + result = await connection.get_result("get-methods") + + return get(result, "methods", list[str]) + + +@dataclass +class Endpoint: + method: str + description: str + params_schema: dict[str, Any] + result_schema: dict[str, Any] + asynchronous: bool + + +async def get_endpoint(connection: Connection, method: str) -> Endpoint: + result = await connection.get_result("get-schema", {"method": method}) + + return Endpoint( + method=get(result, "method", str), + description=get(result, "description", str), + params_schema=get(result, "params", dict[str, Any]), + result_schema=get(result, "result", dict[str, Any]), + asynchronous=get(result, "async", bool), + ) + + +@dataclass +class TaskOperation: + description: str + index: int + completion: float + + +@dataclass +class TaskInfo: + id: int + operation_count: int + current_operation: TaskOperation + + @property + def done(self) -> bool: + index = self.current_operation.index + completion = self.current_operation.completion + return index == self.operation_count - 1 and completion == 1.0 + + +T = TypeVar("T") + + +def deserialize_task(message: dict[str, Any]) -> TaskInfo: + operation = get(message, "current_operation", dict[str, Any]) + + return TaskInfo( + id=get(message, "id", int), + operation_count=get(message, "operation_count", int), + current_operation=TaskOperation( + description=get(operation, "description", str), + index=get(operation, "index", int), + completion=get(operation, "completion", float), + ), + ) + + +async def get_tasks(connection: Connection) -> list[TaskInfo]: + result = await connection.get_result("get-tasks") + + tasks: list[dict[str, Any]] = get(result, "tasks", list[dict[str, Any]]) + + return [deserialize_task(task) for task in tasks] + + +async def get_task(connection: Connection, task_id: int) -> TaskInfo: + result = await connection.get_result("get-task", {"task_id": task_id}) + + return deserialize_task(result) + + +async def cancel_task(connection: Connection, task_id: int) -> None: + await connection.get_result("cancel-task", {"task_id": task_id}) + + +async def get_task_result(connection: Connection, task_id: int) -> Response: + return await connection.request("get-task-result", {"task_id": task_id}) + + +class Task(Generic[T]): + def __init__(self, connection: Connection, id: int, parser: Callable[[Response], T]) -> None: + self._connection = connection + self._id = id + self._parser = parser + + @property + def id(self) -> int: + return self._id + + async def get_status(self) -> TaskInfo: + return await get_task(self._connection, self._id) + + async def is_done(self) -> bool: + status = await self.get_status() + return status.done + + async def cancel(self) -> None: + await cancel_task(self._connection, self._id) + + async def wait(self) -> T: + result = await get_task_result(self._connection, self._id) + return self._parser(result) + + +async def stop_service(connection: Connection) -> None: + await connection.get_result("stop") diff --git a/python/brayns/network/connection.py b/python/brayns/network/connection.py index b70bf45f3..42b468229 100644 --- a/python/brayns/network/connection.py +++ b/python/brayns/network/connection.py @@ -21,14 +21,24 @@ import asyncio from logging import Logger from ssl import SSLContext -from typing import Any, NamedTuple -from .json_rpc import JsonRpcErrorResponse, JsonRpcId, JsonRpcRequest, JsonRpcResponse, compose_request, parse_response +from typing import Any, NamedTuple, Self + +from .json_rpc import ( + JsonRpcError, + JsonRpcErrorResponse, + JsonRpcId, + JsonRpcRequest, + JsonRpcResponse, + JsonRpcSuccessResponse, + compose_request, + parse_response, +) from .websocket import ServiceUnavailable, WebSocket, connect_websocket -class JsonRpcBuffer: +class ResponseBuffer: def __init__(self) -> None: - self._responses = dict[JsonRpcId, JsonRpcResponse | JsonRpcErrorResponse | None]() + self._responses = dict[JsonRpcId, JsonRpcResponse | None]() def is_running(self, request_id: JsonRpcId) -> bool: return request_id in self._responses @@ -40,9 +50,22 @@ def is_done(self, request_id: JsonRpcId) -> bool: return self._responses[request_id] is not None def add_request(self, request_id: JsonRpcId) -> None: + if request_id in self._responses: + raise ValueError("A request with same ID is already running") + self._responses[request_id] = None - def add_response(self, message: JsonRpcResponse | JsonRpcErrorResponse) -> None: + def add_global_error(self, error: JsonRpcError) -> None: + response = JsonRpcErrorResponse(None, error) + + for request_id in self._responses: + self._responses[request_id] = response + + def add_response(self, message: JsonRpcResponse) -> None: + if message.id is None: + self.add_global_error(message.error) + return + if message.id not in self._responses: raise ValueError("No requests match given response ID") @@ -51,38 +74,55 @@ def add_response(self, message: JsonRpcResponse | JsonRpcErrorResponse) -> None: self._responses[message.id] = message - def get_response(self, request_id: JsonRpcId) -> JsonRpcResponse | JsonRpcErrorResponse | None: + def get_response(self, request_id: JsonRpcId) -> JsonRpcResponse | None: if not self.is_done(request_id): return None return self._responses.pop(request_id) -class Result(NamedTuple): - value: Any +class Request(NamedTuple): + method: str + params: Any = None + binary: bytes = b"" + + +class Response(NamedTuple): + result: Any binary: bytes -class JsonRpcFuture: - def __init__(self, request_id: JsonRpcId, websocket: WebSocket, buffer: JsonRpcBuffer) -> None: +class FutureResponse: + def __init__(self, request_id: JsonRpcId | None, websocket: WebSocket, buffer: ResponseBuffer) -> None: self._request_id = request_id self._websocket = websocket self._buffer = buffer @property - def request_id(self) -> JsonRpcId: + def request_id(self) -> JsonRpcId | None: return self._request_id @property def done(self) -> bool: + if self._request_id is None: + return True + return self._buffer.is_done(self._request_id) async def poll(self) -> None: + if self.done: + return + data = await self._websocket.receive() + message = parse_response(data) + self._buffer.add_response(message) - async def wait_for_result(self) -> Result: + async def wait(self) -> Response: + if self._request_id is None: + raise ValueError("Cannot wait for result of requests without ID") + while True: response = self._buffer.get_response(self._request_id) @@ -90,8 +130,8 @@ async def wait_for_result(self) -> Result: await self.poll() continue - if isinstance(response, JsonRpcResponse): - return Result(response.result, response.binary) + if isinstance(response, JsonRpcSuccessResponse): + return Response(response.result, response.binary) if isinstance(response, JsonRpcErrorResponse): raise response.error @@ -102,39 +142,72 @@ async def wait_for_result(self) -> Result: class Connection: def __init__(self, websocket: WebSocket) -> None: self._websocket = websocket - self._buffer = JsonRpcBuffer() + self._buffer = ResponseBuffer() + + async def __aenter__(self) -> Self: + return self + + async def __aexit__(self, *args) -> None: + await self._websocket.close() + + async def send_json_rpc(self, request: JsonRpcRequest) -> FutureResponse: + if request.id is not None and self._buffer.is_running(request.id): + raise ValueError(f"A request with ID {request.id} is already running") - async def send(self, request: JsonRpcRequest) -> JsonRpcFuture: data = compose_request(request) + await self._websocket.send(data) - return JsonRpcFuture(request.id, self._websocket, self._buffer) - async def start(self, method: str, params: Any = None, binary: bytes = b"") -> JsonRpcFuture: + if request.id is not None: + self._buffer.add_request(request.id) + + return FutureResponse(request.id, self._websocket, self._buffer) + + async def send(self, request: Request) -> FutureResponse: request_id = 0 while self._buffer.is_running(request_id): request_id += 1 - request = JsonRpcRequest(request_id, method, params, binary) + method, params, binary = request + json_rpc_request = JsonRpcRequest(method, params, binary, request_id) + + return await self.send_json_rpc(json_rpc_request) + + async def task(self, method: str, params: Any = None, binary: bytes = b"") -> FutureResponse: + request = Request(method, params, binary) return await self.send(request) - async def request(self, method: str, params: Any = None, binary: bytes = b"") -> Result: - future = await self.start(method, params, binary) - return await future.wait_for_result() + async def request(self, method: str, params: Any = None, binary: bytes = b"") -> Response: + future = await self.task(method, params, binary) + + return await future.wait() + + async def get_result(self, method: str, params: Any = None, binary: bytes = b"") -> Any: + result, binary = await self.request(method, params, binary) + + if binary: + raise ValueError(f"Unexpected binary data in response of method {method}") + + return result async def connect( - url: str, - ssl: SSLContext, - max_frame_size: int, + host: str, + port: int = 5000, + ssl: SSLContext | None = None, + max_frame_size: int = 2**31, max_attempts: int | None = 1, - sleep_between_attempts: float = 1, + sleep_between_attempts: float = 0.1, logger: Logger | None = None, ) -> Connection: if logger is None: logger = Logger("Brayns") + protocol = "ws" if ssl is None else "wss" + url = f"{protocol}://{host}:{port}" + attempt = 0 while True: @@ -143,9 +216,9 @@ async def connect( websocket = await connect_websocket(url, ssl, max_frame_size, logger) break except ServiceUnavailable as e: - logger.warn("Connection attempt failed: %s", e) + logger.warning("Connection attempt failed: %s", e) if max_attempts is not None and attempt >= max_attempts: - logger.warn("Max connection attempts reached, aborted") + logger.warning("Max connection attempts reached, aborted") raise await asyncio.sleep(sleep_between_attempts) diff --git a/python/brayns/network/json_rpc.py b/python/brayns/network/json_rpc.py index f115f973d..449602985 100644 --- a/python/brayns/network/json_rpc.py +++ b/python/brayns/network/json_rpc.py @@ -18,23 +18,25 @@ # along with this library; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -from dataclasses import dataclass import json -from typing import Any, TypeVar +from dataclasses import dataclass +from typing import Any -JsonRpcId = int | str | None +from brayns.utils.parsing import get, try_get + +JsonRpcId = int | str @dataclass class JsonRpcRequest: - id: JsonRpcId method: str params: Any = None binary: bytes = b"" + id: JsonRpcId | None = None @dataclass -class JsonRpcResponse: +class JsonRpcSuccessResponse: id: JsonRpcId result: Any = None binary: bytes = b"" @@ -49,124 +51,111 @@ class JsonRpcError(Exception): @dataclass class JsonRpcErrorResponse: - id: JsonRpcId + id: JsonRpcId | None error: JsonRpcError +JsonRpcResponse = JsonRpcSuccessResponse | JsonRpcErrorResponse + + def serialize_request(request: JsonRpcRequest) -> dict[str, Any]: - return { + message = { "jsonrpc": "2.0", - "id": request.id, "method": request.method, "params": request.params, } + if request.id is not None: + message["id"] = request.id -def compose_text_request(request: JsonRpcRequest) -> str: - message = serialize_request(request) + return message + + +def compose_text(message: Any) -> str: return json.dumps(message, separators=(",", ":")) -def compose_binary_request(request: JsonRpcRequest) -> bytes: - json_part = compose_text_request(request).encode() +def compose_binary(message: Any, binary: bytes) -> bytes: + text = compose_text(message) + json_part = text.encode() header = len(json_part).to_bytes(4, byteorder="little", signed=False) - binary_part = request.binary - - return b"".join([header, json_part, binary_part]) + return header + json_part + binary def compose_request(request: JsonRpcRequest) -> bytes | str: - if request.binary: - return compose_binary_request(request) - return compose_text_request(request) - - -T = TypeVar("T") - - -def _get(message: dict[str, Any], key: str, t: type[T]) -> T: - value = message[key] - - if not isinstance(value, t): - raise ValueError("Invalid JSON-RPC message value type") - - return value - - -def _get_id(message: dict[str, Any]) -> JsonRpcId: - id = message.get("id") + message = serialize_request(request) - if id is None: - return None + if request.binary: + return compose_binary(message, request.binary) - if isinstance(id, int | str): - return id + return compose_text(message) - raise ValueError("Invalid JSON-RPC ID type") +def deserialize_result(message: dict[str, Any], binary: bytes = b"") -> JsonRpcSuccessResponse: + response_id = get(message, "id", int | str) -def deserialize_result(message: dict[str, Any], binary: bytes = b"") -> JsonRpcResponse: - return JsonRpcResponse( - id=_get_id(message), + return JsonRpcSuccessResponse( + id=response_id, result=message["result"], + binary=binary, ) def deserialize_error(message: dict[str, Any]) -> JsonRpcErrorResponse: - error = _get(message, "error", dict) + error: dict[str, Any] = get(message, "error", dict[str, Any]) return JsonRpcErrorResponse( - id=_get_id(message), + id=try_get(message, "id", int | str | None, None), error=JsonRpcError( - code=_get(error, "code", int), - message=_get(error, "message", str), + code=get(error, "code", int), + message=get(error, "message", str), data=error.get("data"), ), ) -def deserialize_response(message: dict[str, Any], binary: bytes = b"") -> JsonRpcResponse | JsonRpcErrorResponse: +def deserialize_response(message: dict[str, Any], binary: bytes = b"") -> JsonRpcResponse: if "result" in message: return deserialize_result(message, binary) if "error" not in message: - raise ValueError("Invalid JSON-RPC message") + raise ValueError("Invalid JSON-RPC message without 'result' and 'error'") if binary: - raise ValueError("Invalid binary in error message") + raise ValueError("Invalid error message with binary") return deserialize_error(message) -def parse_text_response(data: str) -> JsonRpcResponse | JsonRpcErrorResponse: - message = json.loads(data) - return deserialize_response(message) - - -def parse_binary_response(data: bytes) -> JsonRpcResponse | JsonRpcErrorResponse: +def parse_binary(data: bytes) -> tuple[Any, bytes]: size = len(data) if size < 4: - raise ValueError("Invalid binary frame header < 4 bytes") + raise ValueError("Invalid binary frame with header < 4 bytes") header = data[:4] json_size = int.from_bytes(header, byteorder="little", signed=False) + + if json_size > size - 4: + raise ValueError(f"Invalid JSON size: {json_size} > {size - 4}") + json_part = data[4 : 4 + json_size].decode("utf-8") message = json.loads(json_part) binary_part = data[4 + json_size :] - return deserialize_response(message, binary_part) + return message, binary_part -def parse_response(data: bytes | str) -> JsonRpcResponse | JsonRpcErrorResponse: +def parse_response(data: bytes | str) -> JsonRpcResponse: if isinstance(data, bytes): - return parse_binary_response(data) + message, binary = parse_binary(data) - if isinstance(data, str): - return parse_text_response(data) + return deserialize_response(message, binary) - raise TypeError("Invalid data type") + message = json.loads(data) + + return deserialize_response(message) diff --git a/python/brayns/network/websocket.py b/python/brayns/network/websocket.py index 535cc62d5..bf80f37c0 100644 --- a/python/brayns/network/websocket.py +++ b/python/brayns/network/websocket.py @@ -20,10 +20,11 @@ from logging import Logger from ssl import SSLContext -from websockets.client import connect, WebSocketClientProtocol -from websockets.exceptions import WebSocketException from typing import Protocol, Self +from websockets.client import WebSocketClientProtocol, connect +from websockets.exceptions import WebSocketException + class WebSocketError(Exception): ... @@ -80,7 +81,7 @@ async def close(self) -> None: try: await self._websocket.close() except WebSocketException as e: - self._logger.warn("Failed to close connection: %s", e) + self._logger.warning("Failed to close connection: %s", e) raise WebSocketError(str(e)) self._logger.info("Websocket connection closed") @@ -97,7 +98,7 @@ async def send(self, data: bytes | str) -> None: try: await self._websocket.send(sliced) except WebSocketException as e: - self._logger.warn("Failed to send frame: %s", e) + self._logger.warning("Failed to send frame: %s", e) raise WebSocketError(str(e)) self._logger.info("Frame sent") @@ -123,10 +124,10 @@ async def connect_websocket(url: str, ssl: SSLContext | None, max_frame_size: in try: websocket = await connect(url, ssl=ssl, max_size=max_frame_size) except WebSocketException as e: - logger.warn("Connection failed: %s", e) + logger.warning("Connection failed: %s", e) raise WebSocketError(str(e)) except OSError as e: - logger.warn("Service not found (probably not ready): %s", e) + logger.warning("Service not found (probably not ready): %s", e) raise ServiceUnavailable(str(e)) wrapper = _WebSocket(websocket, max_frame_size, logger) diff --git a/python/brayns/utils/__init__.py b/python/brayns/utils/__init__.py new file mode 100644 index 000000000..66765f01e --- /dev/null +++ b/python/brayns/utils/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) 2015-2024 EPFL/Blue Brain Project +# All rights reserved. Do not distribute without permission. +# +# Responsible Author: adrien.fleury@epfl.ch +# +# This file is part of Brayns +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License version 3.0 as published +# by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. diff --git a/python/brayns/utils/parsing.py b/python/brayns/utils/parsing.py new file mode 100644 index 000000000..23ea7816c --- /dev/null +++ b/python/brayns/utils/parsing.py @@ -0,0 +1,80 @@ +# Copyright (c) 2015-2024 EPFL/Blue Brain Project +# All rights reserved. Do not distribute without permission. +# +# Responsible Author: adrien.fleury@epfl.ch +# +# This file is part of Brayns +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License version 3.0 as published +# by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from types import UnionType +from typing import Any, cast, get_args, get_origin + + +def has_type(value: Any, t: Any) -> bool: + if t is Any: + return True + + if t is None: + return value is None + + if t is float and isinstance(value, int): + return True + + origin = get_origin(t) + + if origin is UnionType: + args = get_args(t) + return any(has_type(value, arg) for arg in args) + + if origin is None: + origin = t + + if not isinstance(value, origin): + return False + + if origin is list: + items = cast(list, value) + (arg,) = get_args(t) + return all(has_type(item, arg) for item in items) + + if origin is dict: + object = cast(dict, value) + keytype, valuetype = get_args(t) + return all(has_type(key, keytype) and has_type(item, valuetype) for key, item in object.items()) + + return True + + +def check_type(value: Any, t: Any) -> None: + if not has_type(value, t): + raise TypeError("Invalid type in JSON-RPC message") + + +def try_get(message: dict[str, Any], key: str, t: Any, default: Any = None) -> Any: + value = message.get(key, default) + + check_type(value, t) + + return value + + +def get(message: dict[str, Any], key: str, t: Any) -> Any: + if key not in message: + raise KeyError(f"Missing mandatory key in JSON-RPC message {key}") + + value = message[key] + check_type(value, t) + + return value diff --git a/python/brayns/version.py b/python/brayns/version.py index 5c8a98425..32bda7eb6 100644 --- a/python/brayns/version.py +++ b/python/brayns/version.py @@ -18,5 +18,5 @@ # along with this library; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -VERSION = "4.0.0" -"""Version tag of brayns Python package (major.minor.patch).""" +VERSION = "4.0.0-1" +"""Version tag of brayns Python package (major.minor.patch[-pre-release]).""" diff --git a/python/requirements-dev.txt b/python/requirements-dev.txt index 470b48ef7..122c4df2d 100644 --- a/python/requirements-dev.txt +++ b/python/requirements-dev.txt @@ -20,4 +20,6 @@ mypy==1.10.1 pytest==8.2.2 +pytest-asyncio==0.23.8 +pytest-integration==0.2.3 ruff==0.5.0 diff --git a/python/tests/integration/__init__.py b/python/tests/integration/__init__.py new file mode 100644 index 000000000..66765f01e --- /dev/null +++ b/python/tests/integration/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) 2015-2024 EPFL/Blue Brain Project +# All rights reserved. Do not distribute without permission. +# +# Responsible Author: adrien.fleury@epfl.ch +# +# This file is part of Brayns +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License version 3.0 as published +# by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. diff --git a/python/tests/integration/conftest.py b/python/tests/integration/conftest.py new file mode 100644 index 000000000..574ab1a07 --- /dev/null +++ b/python/tests/integration/conftest.py @@ -0,0 +1,69 @@ +# Copyright (c) 2015-2024 EPFL/Blue Brain Project +# All rights reserved. Do not distribute without permission. +# +# Responsible Author: adrien.fleury@epfl.ch +# +# This file is part of Brayns +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License version 3.0 as published +# by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +from collections.abc import AsyncIterator, Iterator +from subprocess import PIPE, STDOUT, Popen + +import pytest +import pytest_asyncio + +from brayns import Connection, connect + +HOST = os.getenv("BRAYNS_HOST", "localhost") +PORT = int(os.getenv("BRAYNS_PORT", "5000")) +EXECUTABLE = os.getenv("BRAYNS_EXECUTABLE", "") + + +def start_service() -> Popen[str]: + return Popen( + args=[ + EXECUTABLE, + "--host", + HOST, + "--port", + str(PORT), + ], + stdin=PIPE, + stdout=PIPE, + stderr=STDOUT, + text=True, + ) + + +async def connect_to_service() -> Connection: + return await connect(HOST, PORT, max_attempts=100) + + +@pytest.fixture(scope="session") +def service() -> Iterator[None]: + if not EXECUTABLE: + yield + return + + with start_service() as process: + yield + process.terminate() + + +@pytest_asyncio.fixture +async def connection(service) -> AsyncIterator[Connection]: + async with await connect_to_service() as connection: + yield connection diff --git a/python/tests/integration/core/__init__.py b/python/tests/integration/core/__init__.py new file mode 100644 index 000000000..66765f01e --- /dev/null +++ b/python/tests/integration/core/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) 2015-2024 EPFL/Blue Brain Project +# All rights reserved. Do not distribute without permission. +# +# Responsible Author: adrien.fleury@epfl.ch +# +# This file is part of Brayns +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License version 3.0 as published +# by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. diff --git a/python/tests/integration/core/test_service.py b/python/tests/integration/core/test_service.py new file mode 100644 index 000000000..70c89fe45 --- /dev/null +++ b/python/tests/integration/core/test_service.py @@ -0,0 +1,74 @@ +# Copyright (c) 2015-2024 EPFL/Blue Brain Project +# All rights reserved. Do not distribute without permission. +# +# Responsible Author: adrien.fleury@epfl.ch +# +# This file is part of Brayns +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License version 3.0 as published +# by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import pytest + +from brayns import ( + VERSION, + Connection, + JsonRpcError, + cancel_task, + get_endpoint, + get_methods, + get_task, + get_task_result, + get_tasks, + get_version, +) + + +@pytest.mark.integration_test +@pytest.mark.asyncio +async def test_version(connection: Connection) -> None: + version = await get_version(connection) + assert version.tag == VERSION + + +@pytest.mark.integration_test +@pytest.mark.asyncio +async def test_methods(connection: Connection) -> None: + methods = await get_methods(connection) + assert all(isinstance(method, str) for method in methods) + + +@pytest.mark.integration_test +@pytest.mark.asyncio +async def test_schema(connection: Connection) -> None: + endpoint = await get_endpoint(connection, "get-version") + assert endpoint.method == "get-version" + assert isinstance(endpoint.description, str) + assert isinstance(endpoint.params_schema, dict) + assert isinstance(endpoint.result_schema, dict) + + +@pytest.mark.integration_test +@pytest.mark.asyncio +async def test_tasks(connection: Connection) -> None: + tasks = await get_tasks(connection) + assert not tasks + + with pytest.raises(JsonRpcError): + await get_task(connection, 0) + + with pytest.raises(JsonRpcError): + await cancel_task(connection, 0) + + with pytest.raises(JsonRpcError): + await get_task_result(connection, 0) diff --git a/python/tests/unit/__init__.py b/python/tests/unit/__init__.py new file mode 100644 index 000000000..66765f01e --- /dev/null +++ b/python/tests/unit/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) 2015-2024 EPFL/Blue Brain Project +# All rights reserved. Do not distribute without permission. +# +# Responsible Author: adrien.fleury@epfl.ch +# +# This file is part of Brayns +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License version 3.0 as published +# by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. diff --git a/python/tests/unit/network/__init__.py b/python/tests/unit/network/__init__.py new file mode 100644 index 000000000..66765f01e --- /dev/null +++ b/python/tests/unit/network/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) 2015-2024 EPFL/Blue Brain Project +# All rights reserved. Do not distribute without permission. +# +# Responsible Author: adrien.fleury@epfl.ch +# +# This file is part of Brayns +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License version 3.0 as published +# by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. diff --git a/python/tests/unit/network/mock_websocket.py b/python/tests/unit/network/mock_websocket.py new file mode 100644 index 000000000..34cabf852 --- /dev/null +++ b/python/tests/unit/network/mock_websocket.py @@ -0,0 +1,94 @@ +# Copyright (c) 2015-2024 EPFL/Blue Brain Project +# All rights reserved. Do not distribute without permission. +# +# Responsible Author: adrien.fleury@epfl.ch +# +# This file is part of Brayns +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License version 3.0 as published +# by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from dataclasses import dataclass, field +from brayns.network.json_rpc import ( + JsonRpcErrorResponse, + JsonRpcResponse, + JsonRpcSuccessResponse, + compose_binary, + compose_text, +) +from brayns.network.websocket import WebSocket +from brayns.network.connection import Connection + + +@dataclass +class MockWebSocket(WebSocket): + responses: list[bytes | str] + closed: bool = field(default=False, init=False) + sent_packets: list[bytes | str] = field(default_factory=list, init=False) + + @property + def host(self) -> str: + return "localhost" + + @property + def port(self) -> int: + return 5000 + + async def close(self) -> None: + self.closed = True + + async def send(self, data: bytes | str) -> None: + self.sent_packets.append(data) + + async def receive(self) -> bytes | str: + return self.responses.pop(0) + + +def compose_error(response: JsonRpcErrorResponse) -> str: + message = { + "id": response.id, + "error": { + "code": response.error.code, + "message": response.error.message, + "data": response.error.data, + }, + } + + return compose_text(message) + + +def compose_result(response: JsonRpcSuccessResponse) -> bytes | str: + message = { + "id": response.id, + "result": response.result, + } + + if response.binary: + return compose_binary(message, response.binary) + + return compose_text(message) + + +def compose_response(response: JsonRpcResponse) -> bytes | str: + if isinstance(response, JsonRpcErrorResponse): + return compose_error(response) + + return compose_result(response) + + +def mock_connection(responses: list[JsonRpcResponse]) -> tuple[Connection, MockWebSocket]: + packets = [compose_response(response) for response in responses] + + websocket = MockWebSocket(packets) + + return Connection(websocket), websocket diff --git a/python/tests/unit/network/test_connection.py b/python/tests/unit/network/test_connection.py new file mode 100644 index 000000000..defb454c4 --- /dev/null +++ b/python/tests/unit/network/test_connection.py @@ -0,0 +1,90 @@ +# Copyright (c) 2015-2024 EPFL/Blue Brain Project +# All rights reserved. Do not distribute without permission. +# +# Responsible Author: adrien.fleury@epfl.ch +# +# This file is part of Brayns +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License version 3.0 as published +# by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import pytest + +from brayns.network.connection import Response +from brayns.network.json_rpc import ( + JsonRpcError, + JsonRpcErrorResponse, + JsonRpcRequest, + JsonRpcSuccessResponse, + compose_request, +) +from .mock_websocket import mock_connection + + +def mock_request() -> JsonRpcRequest: + return JsonRpcRequest("test", 456, b"456", 0) + + +def mock_response() -> JsonRpcSuccessResponse: + return JsonRpcSuccessResponse(0, 123, b"123") + + +def mock_error() -> JsonRpcErrorResponse: + return JsonRpcErrorResponse(0, JsonRpcError(0, "test")) + + +@pytest.mark.asyncio +async def test_request() -> None: + response = mock_response() + connection, websocket = mock_connection([response]) + + request = mock_request() + + result, binary = await connection.request(request.method, request.params, request.binary) + + assert websocket.sent_packets == [compose_request(request)] + + assert result == response.result + assert binary == response.binary + + +@pytest.mark.asyncio +async def test_json_rpc() -> None: + response = mock_response() + connection, _ = mock_connection([response]) + + request = mock_request() + + future = await connection.send_json_rpc(request) + + assert not future.done + + await future.poll() + + assert future.done + assert await future.wait() == Response(response.result, response.binary) + + +@pytest.mark.asyncio +async def test_error() -> None: + jsonrpc = mock_error() + connection, _ = mock_connection([jsonrpc]) + + request = mock_request() + + future = await connection.send_json_rpc(request) + + with pytest.raises(JsonRpcError) as e: + await future.wait() + + assert e.value == jsonrpc.error diff --git a/python/tests/unit/network/test_json_rpc.py b/python/tests/unit/network/test_json_rpc.py new file mode 100644 index 000000000..cd31085c2 --- /dev/null +++ b/python/tests/unit/network/test_json_rpc.py @@ -0,0 +1,77 @@ +# Copyright (c) 2015-2024 EPFL/Blue Brain Project +# All rights reserved. Do not distribute without permission. +# +# Responsible Author: adrien.fleury@epfl.ch +# +# This file is part of Brayns +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License version 3.0 as published +# by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from brayns.network.json_rpc import ( + JsonRpcError, + JsonRpcErrorResponse, + JsonRpcRequest, + JsonRpcSuccessResponse, + compose_request, + parse_response, + serialize_request, +) + + +def test_compose_request() -> None: + request = JsonRpcRequest("test", 123, id=0) + + data = compose_request(request) + + text = """{"jsonrpc":"2.0","method":"test","params":123,"id":0}""" + assert data == text + + request.binary = b"123" + + data = compose_request(request) + + assert isinstance(data, bytes) + assert len(data) == 4 + len(text) + len(request.binary) + + size = int.from_bytes(data[:4], byteorder="little", signed=False) + + assert size == len(text) + assert data[4 : size + 4].decode() == text + assert data[size + 4 :] == request.binary + + +def test_no_id() -> None: + request = JsonRpcRequest("test") + + message = serialize_request(request) + + assert "id" not in message + + +def test_parse_response() -> None: + text = """{"jsonrpc":"2.0","id":0,"result":123}""" + + assert parse_response(text) == JsonRpcSuccessResponse(0, 123) + + binary = b"123" + header = len(text).to_bytes(4, byteorder="little", signed=False) + binary = header + text.encode() + binary + + assert parse_response(binary) == JsonRpcSuccessResponse(0, 123, b"123") + + +def test_parse_error() -> None: + text = """{"jsonrpc":"2.0","id":0,"error":{"code":0,"message": "test"}}""" + + assert parse_response(text) == JsonRpcErrorResponse(0, JsonRpcError(0, "test")) diff --git a/src/brayns/core/Launcher.cpp b/src/brayns/core/Launcher.cpp index 653a1ffbd..a577e1a04 100644 --- a/src/brayns/core/Launcher.cpp +++ b/src/brayns/core/Launcher.cpp @@ -23,7 +23,8 @@ #include -#include +#include +#include #include #include #include @@ -73,7 +74,11 @@ void startServerAndRunService(const ServiceSettings &settings, Logger &logger) auto builder = ApiBuilder(); - addCoreEndpoints(builder, api, token); + addServiceEndpoints(builder, api, token); + + auto objects = ObjectManager(); + + addObjectEndpoints(builder, objects); api = builder.build(); diff --git a/src/brayns/core/api/Api.cpp b/src/brayns/core/api/Api.cpp index df882319d..5403ab229 100644 --- a/src/brayns/core/api/Api.cpp +++ b/src/brayns/core/api/Api.cpp @@ -30,7 +30,7 @@ namespace { using namespace brayns; -const RawTask &getTask(const std::map &tasks, TaskId id) +const RawTask &getRawTask(const std::map &tasks, TaskId id) { auto i = tasks.find(id); @@ -121,18 +121,18 @@ RawResult Api::execute(const std::string &method, RawParams params) if (endpoint.schema.async) { - const auto &launcher = std::get(endpoint.handler); + const auto &handler = std::get(endpoint.handler); - auto task = launcher(std::move(params)); + auto task = handler(std::move(params)); auto id = addTask(std::move(task), _tasks, _ids); return {serializeToJson(TaskResult{id})}; } - const auto &runner = std::get(endpoint.handler); + const auto &handler = std::get(endpoint.handler); - return runner(std::move(params)); + return handler(std::move(params)); } std::vector Api::getTasks() const @@ -142,31 +142,43 @@ std::vector Api::getTasks() const for (const auto &[id, task] : _tasks) { - infos.push_back({id, task.getProgress()}); + auto operationCount = task.operationCount; + auto currentOperation = task.getCurrentOperation(); + + infos.push_back({id, operationCount, std::move(currentOperation)}); } return infos; } -ProgressInfo Api::getTaskProgress(TaskId id) const +TaskInfo Api::getTask(TaskId id) const { - const auto &task = getTask(_tasks, id); - return task.getProgress(); + const auto &task = getRawTask(_tasks, id); + + auto operationCount = task.operationCount; + auto currentOperation = task.getCurrentOperation(); + + return {id, operationCount, std::move(currentOperation)}; } RawResult Api::waitForTaskResult(TaskId id) { - const auto &task = getTask(_tasks, id); + const auto &task = getRawTask(_tasks, id); + auto result = task.wait(); + _tasks.erase(id); _ids.recycle(id); + return result; } void Api::cancelTask(TaskId id) { - const auto &task = getTask(_tasks, id); + const auto &task = getRawTask(_tasks, id); + task.cancel(); + _tasks.erase(id); _ids.recycle(id); } diff --git a/src/brayns/core/api/Api.h b/src/brayns/core/api/Api.h index 977a055a8..d4bb86f52 100644 --- a/src/brayns/core/api/Api.h +++ b/src/brayns/core/api/Api.h @@ -45,7 +45,7 @@ class Api const EndpointSchema &getSchema(const std::string &method) const; RawResult execute(const std::string &method, RawParams params); std::vector getTasks() const; - ProgressInfo getTaskProgress(TaskId id) const; + TaskInfo getTask(TaskId id) const; RawResult waitForTaskResult(TaskId id); void cancelTask(TaskId id); diff --git a/src/brayns/core/api/ApiBuilder.h b/src/brayns/core/api/ApiBuilder.h index 3a8666e51..7719ab76f 100644 --- a/src/brayns/core/api/ApiBuilder.h +++ b/src/brayns/core/api/ApiBuilder.h @@ -46,7 +46,8 @@ using GetResultType = std::decay_t>; template struct Task { - std::function getProgress; + std::size_t operationCount; + std::function getCurrentOperation; std::function wait; std::function cancel; }; @@ -64,7 +65,7 @@ template concept ReflectedTask = requires { typename TaskReflector::Result; }; template -using GetTaskResultType = typename TaskReflector::Result; +using GetTaskResult = typename TaskReflector::Result; template concept WithReflectedParams = getArgCount == 1 && ApiReflected>; @@ -73,7 +74,7 @@ template concept WithoutParams = getArgCount == 0; template -concept WithReflectableParams = WithReflectedParams || WithoutParams; +concept WithParams = WithReflectedParams || WithoutParams; template concept WithReflectedResult = ApiReflected>; @@ -82,67 +83,67 @@ template concept WithoutResult = std::is_void_v>; template -concept WithReflectableResult = WithReflectedResult || WithoutResult; +concept WithResult = WithReflectedResult || WithoutResult; template -concept WithReflectedTaskResult = ApiReflected>>; +concept WithReflectedTaskResult = ApiReflected>>; template -concept WithoutTaskResult = std::is_void_v>>; +concept WithoutTaskResult = std::is_void_v>>; template -concept WithReflectableTaskResult = WithReflectedTaskResult || WithoutTaskResult; +concept WithTaskResult = WithReflectedTaskResult || WithoutTaskResult; template -concept ReflectedLauncher = WithReflectedParams && WithReflectedTaskResult; +concept ReflectedAsyncHandler = WithReflectedParams && WithReflectedTaskResult; template -concept ReflectableLauncher = WithReflectableParams && WithReflectableTaskResult; +concept AsyncHandler = WithParams && WithTaskResult; template -concept ReflectedRunner = WithReflectedParams && WithReflectedResult; +concept ReflectedSyncHandler = WithReflectedParams && WithReflectedResult; template -concept ReflectableRunner = WithReflectableParams && WithReflectableResult; +concept SyncHandler = WithParams && WithResult; -auto ensureHasParams(WithReflectedParams auto launcher) +auto ensureHasParams(WithReflectedParams auto handler) { - return launcher; + return handler; } -auto ensureHasParams(WithoutParams auto launcher) +auto ensureHasParams(WithoutParams auto handler) { - return [launcher = std::move(launcher)](NullJson) { return launcher(); }; + return [handler = std::move(handler)](NullJson) { return handler(); }; } -auto ensureHasResult(WithReflectedResult auto launcher) +auto ensureHasResult(WithReflectedResult auto handler) { - return launcher; + return handler; } -auto ensureHasResult(WithoutResult auto launcher) +auto ensureHasResult(WithoutResult auto handler) { - using Params = GetParamsType; + using Params = GetParamsType; - return [launcher = std::move(launcher)](Params params) + return [handler = std::move(handler)](Params params) { - launcher(std::move(params)); + handler(std::move(params)); return NullJson(); }; } -auto ensureHasTaskResult(WithReflectedTaskResult auto launcher) +auto ensureHasTaskResult(WithReflectedTaskResult auto handler) { - return launcher; + return handler; } -auto ensureHasTaskResult(WithoutTaskResult auto launcher) +auto ensureHasTaskResult(WithoutTaskResult auto handler) { - using Params = GetParamsType; + using Params = GetParamsType; - return [launcher = std::move(launcher)](Params params) + return [handler = std::move(handler)](Params params) { - auto task = launcher(std::move(params)); + auto task = handler(std::move(params)); auto waitAndReturnNull = [wait = std::move(task.wait)] { @@ -151,7 +152,8 @@ auto ensureHasTaskResult(WithoutTaskResult auto launcher) }; return Task{ - .getProgress = std::move(task.getProgress), + .operationCount = task.operationCount, + .getCurrentOperation = std::move(task.getCurrentOperation), .wait = std::move(waitAndReturnNull), .cancel = std::move(task.cancel), }; @@ -164,30 +166,31 @@ RawTask addParsingToTask(Task task) using ResultReflector = ApiReflector; return { - .getProgress = std::move(task.getProgress), + .operationCount = task.operationCount, + .getCurrentOperation = std::move(task.getCurrentOperation), .wait = [wait = std::move(task.wait)] { return ResultReflector::serialize(wait()); }, .cancel = std::move(task.cancel), }; } -template -TaskLauncher addParsingToTaskLauncher(T launcher) +template +AsyncEndpointHandler addParsingToAsyncHandler(T handler) { using ParamsReflector = ApiReflector>; - return [launcher = std::move(launcher)](auto rawParams) + return [handler = std::move(handler)](auto rawParams) { auto params = ParamsReflector::deserialize(std::move(rawParams)); - auto task = launcher(std::move(params)); + auto task = handler(std::move(params)); return addParsingToTask(std::move(task)); }; } -template +template EndpointSchema reflectAsyncEndpointSchema(std::string method) { using ParamsReflector = ApiReflector>; - using ResultReflector = ApiReflector>>; + using ResultReflector = ApiReflector>>; return { .method = std::move(method), @@ -197,8 +200,8 @@ EndpointSchema reflectAsyncEndpointSchema(std::string method) }; } -template -TaskRunner addParsingToTaskRunner(T handler) +template +SyncEndpointHandler addParsingToSyncHandler(T handler) { using ParamsReflector = ApiReflector>; using ResultReflector = ApiReflector>; @@ -211,8 +214,8 @@ TaskRunner addParsingToTaskRunner(T handler) }; } -template -EndpointSchema reflectEndpointSchema(std::string method) +template +EndpointSchema reflectSyncEndpointSchema(std::string method) { using ParamsReflector = ApiReflector>; using ResultReflector = ApiReflector>; @@ -224,18 +227,28 @@ EndpointSchema reflectEndpointSchema(std::string method) }; } +struct TaskSettings +{ + std::size_t operationCount; + std::string initialOperation; +}; + template Handler> -Task> startTask(Handler handler, ParamsType params) +Task> startTask( + Handler handler, + ParamsType params, + TaskSettings settings) { - auto state = std::make_shared(); + auto monitor = std::make_shared(settings.operationCount, std::move(settings.initialOperation)); - auto future = std::async(std::launch::async, std::move(handler), Progress(state), std::move(params)); + auto future = std::async(std::launch::async, std::move(handler), Progress(monitor), std::move(params)); auto shared = std::make_shared(std::move(future)); return { - .getProgress = [=] { return state->get(); }, + .operationCount = settings.operationCount, + .getCurrentOperation = [=] { return monitor->getCurrentOperation(); }, .wait = [=] { return shared->get(); }, - .cancel = [=] { state->cancel(); }, + .cancel = [=] { monitor->cancel(); }, }; } @@ -260,19 +273,19 @@ class EndpointBuilder class ApiBuilder { public: - EndpointBuilder task(std::string method, ReflectableLauncher auto launcher) + EndpointBuilder task(std::string method, AsyncHandler auto handler) { - auto reflected = ensureHasTaskResult(ensureHasParams(std::move(launcher))); + auto reflected = ensureHasTaskResult(ensureHasParams(std::move(handler))); auto schema = reflectAsyncEndpointSchema(std::move(method)); - auto startTask = addParsingToTaskLauncher(std::move(reflected)); + auto startTask = addParsingToAsyncHandler(std::move(reflected)); return add({std::move(schema), std::move(startTask)}); } - EndpointBuilder endpoint(std::string method, ReflectableRunner auto runner) + EndpointBuilder endpoint(std::string method, SyncHandler auto handler) { - auto reflected = ensureHasResult(ensureHasParams(std::move(runner))); - auto schema = reflectEndpointSchema(std::move(method)); - auto runTask = addParsingToTaskRunner(std::move(reflected)); + auto reflected = ensureHasResult(ensureHasParams(std::move(handler))); + auto schema = reflectSyncEndpointSchema(std::move(method)); + auto runTask = addParsingToSyncHandler(std::move(reflected)); return add({std::move(schema), std::move(runTask)}); } diff --git a/src/brayns/core/api/Endpoint.h b/src/brayns/core/api/Endpoint.h index e414bb2bd..1c60776f0 100644 --- a/src/brayns/core/api/Endpoint.h +++ b/src/brayns/core/api/Endpoint.h @@ -62,9 +62,9 @@ struct JsonObjectReflector } }; -using TaskLauncher = std::function; -using TaskRunner = std::function; -using EndpointHandler = std::variant; +using SyncEndpointHandler = std::function; +using AsyncEndpointHandler = std::function; +using EndpointHandler = std::variant; struct Endpoint { diff --git a/src/brayns/core/api/Progress.cpp b/src/brayns/core/api/Progress.cpp index 3dc46a387..b9dbc1fc1 100644 --- a/src/brayns/core/api/Progress.cpp +++ b/src/brayns/core/api/Progress.cpp @@ -30,14 +30,25 @@ TaskCancelledException::TaskCancelledException(): { } -ProgressInfo ProgressState::get() +TaskMonitor::TaskMonitor(std::size_t operationCount, std::string initialOperation): + _operationCount(operationCount), + _currentOperation{std::move(initialOperation)} +{ +} + +std::size_t TaskMonitor::getOperationCount() const +{ + return _operationCount; +} + +TaskOperation TaskMonitor::getCurrentOperation() { auto lock = std::lock_guard(_mutex); - return _info; + return _currentOperation; } -void ProgressState::update(float currentOperationProgress) +void TaskMonitor::update(float completion) { auto lock = std::lock_guard(_mutex); @@ -46,11 +57,12 @@ void ProgressState::update(float currentOperationProgress) throw TaskCancelledException(); } - assert(currentOperationProgress >= 0.0F && currentOperationProgress <= 1.0F); - _info.currentOperationProgress = currentOperationProgress; + assert(completion >= 0.0F && completion <= 1.0F); + + _currentOperation.completion = completion; } -void ProgressState::nextOperation(std::string value) +void TaskMonitor::nextOperation(std::string description) { auto lock = std::lock_guard(_mutex); @@ -59,30 +71,34 @@ void ProgressState::nextOperation(std::string value) throw TaskCancelledException(); } - _info.currentOperation = std::move(value); - _info.currentOperationProgress = 0.0F; + _currentOperation.description = std::move(description); + _currentOperation.completion = 0.0F; + _currentOperation.index += 1; + + assert(_currentOperation.index < _operationCount); } -void ProgressState::cancel() +void TaskMonitor::cancel() { auto lock = std::lock_guard(_mutex); assert(!_cancelled); + _cancelled = true; } -Progress::Progress(std::shared_ptr state): - _state(std::move(state)) +Progress::Progress(std::shared_ptr monitor): + _monitor(std::move(monitor)) { } -void Progress::update(float currentOperationProgress) +void Progress::update(float completion) { - _state->update(currentOperationProgress); + _monitor->update(completion); } -void Progress::nextOperation(std::string value) +void Progress::nextOperation(std::string description) { - _state->nextOperation(std::move(value)); + _monitor->nextOperation(std::move(description)); } } diff --git a/src/brayns/core/api/Progress.h b/src/brayns/core/api/Progress.h index 204dd488c..f35820df8 100644 --- a/src/brayns/core/api/Progress.h +++ b/src/brayns/core/api/Progress.h @@ -31,22 +31,25 @@ namespace brayns { -struct ProgressInfo +struct TaskOperation { - std::string currentOperation = "Task startup"; - float currentOperationProgress = 0.0F; + std::string description; + float completion = 0.0F; + std::size_t index = 0; }; template<> -struct JsonObjectReflector +struct JsonObjectReflector { static auto reflect() { - auto builder = JsonBuilder(); - builder.field("current_operation", [](auto &object) { return &object.currentOperation; }) - .description("Description of the current operation"); - builder.field("current_operation_progress", [](auto &object) { return &object.currentOperationProgress; }) - .description("Progress of the current operation between 0 and 1"); + auto builder = JsonBuilder(); + builder.field("description", [](auto &object) { return &object.description; }) + .description("Operation description"); + builder.field("completion", [](auto &object) { return &object.completion; }) + .description("Operation completion between 0 and 1"); + builder.field("index", [](auto &object) { return &object.index; }) + .description("Operation index between 0 and operation_count"); return builder.build(); } }; @@ -57,29 +60,33 @@ class TaskCancelledException : public JsonRpcException explicit TaskCancelledException(); }; -class ProgressState +class TaskMonitor { public: - ProgressInfo get(); - void update(float currentOperationProgress); - void nextOperation(std::string value); + explicit TaskMonitor(std::size_t operationCount, std::string initialOperation); + + std::size_t getOperationCount() const; + TaskOperation getCurrentOperation(); + void update(float completion); + void nextOperation(std::string description); void cancel(); private: std::mutex _mutex; - ProgressInfo _info; + std::size_t _operationCount; + TaskOperation _currentOperation; bool _cancelled = false; }; class Progress { public: - explicit Progress(std::shared_ptr state); + explicit Progress(std::shared_ptr monitor); - void update(float currentOperationProgress); - void nextOperation(std::string value); + void update(float completion); + void nextOperation(std::string description); private: - std::shared_ptr _state; + std::shared_ptr _monitor; }; } diff --git a/src/brayns/core/api/Task.h b/src/brayns/core/api/Task.h index cd4ed58f3..936f2615d 100644 --- a/src/brayns/core/api/Task.h +++ b/src/brayns/core/api/Task.h @@ -44,7 +44,8 @@ struct RawResult struct RawTask { - std::function getProgress; + std::size_t operationCount; + std::function getCurrentOperation; std::function wait; std::function cancel; }; @@ -71,7 +72,8 @@ struct JsonObjectReflector struct TaskInfo { TaskId id; - ProgressInfo progress; + std::size_t operationCount; + TaskOperation currentOperation; }; template<> @@ -80,8 +82,11 @@ struct JsonObjectReflector static auto reflect() { auto builder = JsonBuilder(); - builder.field("id", [](auto &object) { return &object.id; }).description("Task ID to monitor it"); - builder.field("progress", [](auto &object) { return &object.progress; }).description("Current task progress"); + builder.field("id", [](auto &object) { return &object.id; }).description("Task ID"); + builder.field("operation_count", [](auto &object) { return &object.operationCount; }) + .description("Number of operations the task will perform"); + builder.field("current_operation", [](auto &object) { return &object.currentOperation; }) + .description("Current task operation"); return builder.build(); } }; diff --git a/src/brayns/core/endpoints/ObjectEndpoints.cpp b/src/brayns/core/endpoints/ObjectEndpoints.cpp new file mode 100644 index 000000000..4586bb3a0 --- /dev/null +++ b/src/brayns/core/endpoints/ObjectEndpoints.cpp @@ -0,0 +1,113 @@ +/* Copyright (c) 2015-2024 EPFL/Blue Brain Project + * All rights reserved. Do not distribute without permission. + * + * Responsible Author: adrien.fleury@epfl.ch + * + * This file is part of Brayns + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 3.0 as published + * by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "ObjectEndpoints.h" + +namespace brayns +{ +struct TagList +{ + std::vector tags; +}; + +template<> +struct JsonObjectReflector +{ + static auto reflect() + { + auto builder = JsonBuilder(); + builder.field("tags", [](auto &object) { return &object.tags; }).description("List of object tag"); + return builder.build(); + } +}; + +struct MetadataList +{ + std::vector objects; +}; + +template<> +struct JsonObjectReflector +{ + static auto reflect() + { + auto builder = JsonBuilder(); + builder.field("objects", [](auto &object) { return &object.objects; }) + .description("List of object generic properties"); + return builder.build(); + } +}; + +std::vector getMetadata(ObjectManager &objects, const std::vector &ids) +{ + auto metadatas = std::vector(); + metadatas.reserve(ids.size()); + + for (auto id : ids) + { + const auto &metadata = objects.getMetadata(id); + metadatas.push_back(metadata); + } + + return metadatas; +} + +std::vector getIdsFromTags(ObjectManager &objects, const std::vector &tags) +{ + auto ids = std::vector(); + ids.reserve(tags.size()); + + for (const auto &tag : tags) + { + auto id = objects.getId(tag); + ids.push_back(id); + } + + return ids; +} + +void removeObjects(ObjectManager &objects, const std::vector &ids) +{ + for (auto id : ids) + { + objects.remove(id); + } +} + +void addObjectEndpoints(ApiBuilder &builder, ObjectManager &objects) +{ + builder.endpoint("get-all-objects", [&] { return MetadataList{objects.getAllMetadata()}; }) + .description("Return the generic properties of all objects, use get-{type} to get specific properties"); + + builder.endpoint("get-objects", [&](IdList params) { return MetadataList{getMetadata(objects, params.ids)}; }) + .description("Get generic object properties from given object IDs"); + + builder.endpoint("get-object-ids", [&](TagList params) { return IdList{getIdsFromTags(objects, params.tags)}; }) + .description("Map given list of tags to object IDs (result is an array in the same order as params)"); + + builder.endpoint("remove-objects", [&](IdList params) { removeObjects(objects, params.ids); }) + .description( + "Remove objects from the registry, the ID can be reused by future objects. Note that the object can stay " + "in memory as long as it is used by other objects (using a ref-counted system)"); + + builder.endpoint("clear-objects", [&] { objects.clear(); }).description("Remove all objects currently in registry"); +} +} diff --git a/src/brayns/core/endpoints/ObjectEndpoints.h b/src/brayns/core/endpoints/ObjectEndpoints.h new file mode 100644 index 000000000..dfbd52643 --- /dev/null +++ b/src/brayns/core/endpoints/ObjectEndpoints.h @@ -0,0 +1,46 @@ +/* Copyright (c) 2015-2024 EPFL/Blue Brain Project + * All rights reserved. Do not distribute without permission. + * + * Responsible Author: adrien.fleury@epfl.ch + * + * This file is part of Brayns + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 3.0 as published + * by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#pragma once + +#include +#include + +namespace brayns +{ +struct IdList +{ + std::vector ids; +}; + +template<> +struct JsonObjectReflector +{ + static auto reflect() + { + auto builder = JsonBuilder(); + builder.field("ids", [](auto &object) { return &object.ids; }).description("List of objects ID"); + return builder.build(); + } +}; + +void addObjectEndpoints(ApiBuilder &builder, ObjectManager &objects); +} diff --git a/src/brayns/core/endpoints/CoreEndpoints.cpp b/src/brayns/core/endpoints/ServiceEndpoints.cpp similarity index 90% rename from src/brayns/core/endpoints/CoreEndpoints.cpp rename to src/brayns/core/endpoints/ServiceEndpoints.cpp index 88c106b20..6cd471c93 100644 --- a/src/brayns/core/endpoints/CoreEndpoints.cpp +++ b/src/brayns/core/endpoints/ServiceEndpoints.cpp @@ -19,7 +19,7 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ -#include "CoreEndpoints.h" +#include "ServiceEndpoints.h" #include @@ -118,23 +118,27 @@ struct JsonObjectReflector } }; -void addCoreEndpoints(ApiBuilder &builder, Api &api, StopToken &token) +void addServiceEndpoints(ApiBuilder &builder, Api &api, StopToken &token) { builder.endpoint("get-version", [] { return VersionResult(); }) .description("Get the build version of the service currently running"); builder.endpoint("get-methods", [&] { return MethodsResult{api.getMethods()}; }) .description("Get available JSON-RPC methods"); + builder.endpoint("get-schema", [&](SchemaParams params) { return api.getSchema(params.method); }) .description("Get the schema of the given JSON-RPC method"); builder.endpoint("get-tasks", [&] { return TasksResult{api.getTasks()}; }) - .description("Get tasks that are currently running"); - builder.endpoint("get-task-progress", [&](TaskParams params) { return api.getTaskProgress(params.taskId); }) - .description("Get progress of the given task"); + .description("Get tasks which result has not been retreived with wait-for-task-result"); + + builder.endpoint("get-task", [&](TaskParams params) { return api.getTask(params.taskId); }) + .description("Get current state of the given task"); + builder.endpoint("cancel-task", [&](TaskParams params) { api.cancelTask(params.taskId); }) .description("Cancel given task"); - builder.endpoint("wait-for-task-result", [&](TaskParams params) { return api.waitForTaskResult(params.taskId); }) + + builder.endpoint("get-task-result", [&](TaskParams params) { return api.waitForTaskResult(params.taskId); }) .description("Wait for given task to finish and return its result"); builder.endpoint("stop", [&] { token.stop(); }) diff --git a/src/brayns/core/endpoints/CoreEndpoints.h b/src/brayns/core/endpoints/ServiceEndpoints.h similarity index 90% rename from src/brayns/core/endpoints/CoreEndpoints.h rename to src/brayns/core/endpoints/ServiceEndpoints.h index fa8bdaafe..752bde709 100644 --- a/src/brayns/core/endpoints/CoreEndpoints.h +++ b/src/brayns/core/endpoints/ServiceEndpoints.h @@ -21,11 +21,10 @@ #pragma once -#include #include #include namespace brayns { -void addCoreEndpoints(ApiBuilder &builder, Api &api, StopToken &token); +void addServiceEndpoints(ApiBuilder &builder, Api &api, StopToken &token); } diff --git a/src/brayns/core/json/types/Objects.h b/src/brayns/core/json/types/Objects.h index 5f2d63383..721c85406 100644 --- a/src/brayns/core/json/types/Objects.h +++ b/src/brayns/core/json/types/Objects.h @@ -44,33 +44,40 @@ template class JsonObjectInfo { public: - explicit JsonObjectInfo(std::vector> fields): - _fields(std::move(fields)) + explicit JsonObjectInfo(std::vector> fields, std::string description): + _fields(std::move(fields)), + _description(std::move(description)) { } JsonSchema getSchema() const { - auto schema = JsonSchema{.type = JsonType::Object}; + auto schema = JsonSchema{.description = _description, .type = JsonType::Object}; + for (const auto &field : _fields) { schema.properties[field.name] = field.schema; } + return schema; } JsonValue serialize(const T &value) const { auto object = createJsonObject(); + for (const auto &field : _fields) { auto jsonItem = field.serialize(value); + if (jsonItem.isEmpty() && !field.schema.required) { continue; } + object->set(field.name, jsonItem); } + return object; } @@ -102,6 +109,7 @@ class JsonObjectInfo private: std::vector> _fields; + std::string _description; }; template @@ -215,6 +223,11 @@ template class JsonBuilder { public: + void description(std::string value) + { + _description = std::move(value); + } + template U> JsonFieldBuilder field(std::string name, U getFieldPtr) { @@ -267,10 +280,11 @@ class JsonBuilder JsonObjectInfo build() { - return JsonObjectInfo(std::exchange(_fields, {})); + return JsonObjectInfo(std::exchange(_fields, {}), std::exchange(_description, {})); } private: std::vector> _fields; + std::string _description; }; } diff --git a/src/brayns/core/jsonrpc/Messages.h b/src/brayns/core/jsonrpc/Messages.h index 71b964744..75911b3c0 100644 --- a/src/brayns/core/jsonrpc/Messages.h +++ b/src/brayns/core/jsonrpc/Messages.h @@ -21,34 +21,36 @@ #pragma once +#include #include - -#include +#include #include +#include + namespace brayns { -using JsonRpcId = std::variant; +using JsonRpcId = std::variant; inline std::string toString(const JsonRpcId &id) { - if (std::get_if(&id)) - { - return "null"; - } - if (const auto *value = std::get_if(&id)) { return std::to_string(*value); } - return std::get(id); + return fmt::format("'{}'", std::get(id)); +} + +inline std::string toString(const std::optional &id) +{ + return id ? toString(*id) : "null"; } struct JsonRpcRequest { - JsonRpcId id; + std::optional id; std::string method; JsonValue params; std::string binary = {}; @@ -61,14 +63,14 @@ struct JsonObjectReflector { auto builder = JsonBuilder(); builder.constant("jsonrpc", "2.0"); - builder.field("id", [](auto &object) { return &object.id; }).required(false); + builder.field("id", [](auto &object) { return &object.id; }); builder.field("method", [](auto &object) { return &object.method; }); builder.field("params", [](auto &object) { return &object.params; }).required(false); return builder.build(); } }; -struct JsonRpcResponse +struct JsonRpcSuccessResponse { JsonRpcId id; JsonValue result; @@ -76,11 +78,11 @@ struct JsonRpcResponse }; template<> -struct JsonObjectReflector +struct JsonObjectReflector { static auto reflect() { - auto builder = JsonBuilder(); + auto builder = JsonBuilder(); builder.constant("jsonrpc", "2.0"); builder.field("id", [](auto &object) { return &object.id; }); builder.field("result", [](auto &object) { return &object.result; }); @@ -110,7 +112,7 @@ struct JsonObjectReflector struct JsonRpcErrorResponse { - JsonRpcId id; + std::optional id; JsonRpcError error; }; @@ -126,17 +128,6 @@ struct JsonObjectReflector return builder.build(); } }; -} -namespace fmt -{ -template<> -struct formatter : formatter -{ - template - auto format(const brayns::JsonRpcId &id, FmtContext &context) - { - return formatter::format(brayns::toString(id), context); - } -}; +using JsonRpcResponse = std::variant; } diff --git a/src/brayns/core/jsonrpc/Parser.cpp b/src/brayns/core/jsonrpc/Parser.cpp index 71b18c5e2..2f1a227d3 100644 --- a/src/brayns/core/jsonrpc/Parser.cpp +++ b/src/brayns/core/jsonrpc/Parser.cpp @@ -30,7 +30,7 @@ namespace brayns { -JsonRpcRequest parseJsonRpcRequest(const std::string &text) +JsonRpcRequest parseTextJsonRpcRequest(const std::string &text) { auto json = JsonValue(); @@ -73,19 +73,19 @@ JsonRpcRequest parseBinaryJsonRpcRequest(const std::string &binary) auto text = extractBytes(data, textSize); - auto request = parseJsonRpcRequest(std::string(text)); + auto request = parseTextJsonRpcRequest(std::string(text)); request.binary = binary.substr(4 + textSize); return request; } -std::string composeAsText(const JsonRpcResponse &response) +std::string composeAsText(const JsonRpcSuccessResponse &response) { return stringifyToJson(response); } -std::string composeAsBinary(const JsonRpcResponse &response) +std::string composeAsBinary(const JsonRpcSuccessResponse &response) { auto text = composeAsText(response); @@ -106,7 +106,7 @@ std::string composeError(const JsonRpcErrorResponse &response) return stringifyToJson(response); } -std::string composeError(const JsonRpcId &id, const JsonRpcException &e) +std::string composeError(const std::optional &id, const JsonRpcException &e) { return composeError({ .id = id, diff --git a/src/brayns/core/jsonrpc/Parser.h b/src/brayns/core/jsonrpc/Parser.h index 605d2611a..9bb33cac7 100644 --- a/src/brayns/core/jsonrpc/Parser.h +++ b/src/brayns/core/jsonrpc/Parser.h @@ -28,10 +28,10 @@ namespace brayns { -JsonRpcRequest parseJsonRpcRequest(const std::string &text); +JsonRpcRequest parseTextJsonRpcRequest(const std::string &text); JsonRpcRequest parseBinaryJsonRpcRequest(const std::string &binary); -std::string composeAsText(const JsonRpcResponse &response); -std::string composeAsBinary(const JsonRpcResponse &response); +std::string composeAsText(const JsonRpcSuccessResponse &response); +std::string composeAsBinary(const JsonRpcSuccessResponse &response); std::string composeError(const JsonRpcErrorResponse &response); -std::string composeError(const JsonRpcId &id, const JsonRpcException &e); +std::string composeError(const std::optional &id, const JsonRpcException &e); } diff --git a/src/brayns/core/objects/ObjectManager.cpp b/src/brayns/core/objects/ObjectManager.cpp index b474f6cd6..0a4ede43c 100644 --- a/src/brayns/core/objects/ObjectManager.cpp +++ b/src/brayns/core/objects/ObjectManager.cpp @@ -67,23 +67,22 @@ ObjectManager::ObjectManager() disableNullId(_ids); } -std::vector ObjectManager::getIds() const +std::vector ObjectManager::getAllMetadata() const { - auto ids = std::vector(); - ids.reserve(_objects.size()); + auto objects = std::vector(); + objects.reserve(_objects.size()); for (const auto &[id, object] : _objects) { - ids.push_back(id); + objects.push_back(*object.getMetadata()); } - return ids; + return objects; } const Metadata &ObjectManager::getMetadata(ObjectId id) const { auto i = getObjectIterator(_objects, id); - return *i->second.getMetadata(); } @@ -153,7 +152,6 @@ void ObjectManager::addEntry(ObjectId id, ObjectManagerEntry entry) if (!tag.empty()) { checkTagIsNotAlreadyUsed(_idsByTag, tag); - _idsByTag.emplace(tag, id); } diff --git a/src/brayns/core/objects/ObjectManager.h b/src/brayns/core/objects/ObjectManager.h index dead9d752..9f489db89 100644 --- a/src/brayns/core/objects/ObjectManager.h +++ b/src/brayns/core/objects/ObjectManager.h @@ -50,7 +50,7 @@ struct Metadata template<> struct JsonObjectReflector { - static auto build() + static auto reflect() { auto builder = JsonBuilder(); builder.field("id", [](auto &object) { return &object.id; }) @@ -75,7 +75,7 @@ struct UserObject template struct JsonObjectReflector> { - static auto build() + static auto reflect() { auto builder = JsonBuilder>(); builder.field("metadata", [](auto &object) { return &object.metadata; }) @@ -119,7 +119,7 @@ class ObjectManager public: explicit ObjectManager(); - std::vector getIds() const; + std::vector getAllMetadata() const; const Metadata &getMetadata(ObjectId id) const; ObjectId getId(const std::string &tag) const; void remove(ObjectId id); @@ -129,9 +129,7 @@ class ObjectManager const std::shared_ptr &getShared(ObjectId id) const { auto &entry = getEntry(id); - checkType(entry, typeid(std::shared_ptr)); - return std::any_cast &>(entry.object); } @@ -149,6 +147,7 @@ class ObjectManager try { auto &[type, tag, userData] = settings; + auto metadata = Metadata{id, std::move(type), std::move(tag), userData}; auto object = T{std::move(metadata), std::move(properties)}; auto ptr = std::make_shared(std::move(object)); diff --git a/src/brayns/core/service/Service.cpp b/src/brayns/core/service/Service.cpp index f4dfa935e..93664b60d 100644 --- a/src/brayns/core/service/Service.cpp +++ b/src/brayns/core/service/Service.cpp @@ -42,7 +42,7 @@ JsonRpcRequest parseRequest(const RawRequest &request) return parseBinaryJsonRpcRequest(request.data); } - return parseJsonRpcRequest(request.data); + return parseTextJsonRpcRequest(request.data); } ResponseData composeResponse(const JsonRpcId &id, RawResult result) @@ -57,94 +57,104 @@ ResponseData composeResponse(const JsonRpcId &id, RawResult result) return {.data = std::move(data), .binary = true}; } -std::optional tryParseRequest(const RawRequest &request, Logger &logger) +void sendExecutionError( + const std::optional &id, + const JsonRpcException &e, + const ResponseHandler &respond, + Logger &logger) { - try - { - logger.info("Parsing JSON-RPC request from client {}", request.clientId); - auto jsonRpcRequest = parseRequest(request); - logger.info("Successfully parsed request"); - - return jsonRpcRequest; - } - catch (const JsonRpcException &e) - { - logger.warn("Error while parsing request: '{}'", e.what()); - request.respond({composeError(NullJson(), e)}); - } - catch (const std::exception &e) + if (!id) { - logger.error("Unexpected error while parsing request: '{}'", e.what()); - request.respond({composeError(NullJson(), InternalError(e.what()))}); - } - catch (...) - { - logger.error("Unknown error while parsing request"); - request.respond({composeError(NullJson(), InternalError("Unknown parsing error"))}); + logger.info("No ID in request, skipping error response"); + return; } - return std::nullopt; + logger.info("Composing execution error"); + auto text = composeError(*id, e); + logger.info("Execution error composed"); + + logger.info("Sending execution error"); + respond({std::move(text)}); + logger.info("Execution error sent"); } -ResponseData executeRequest(JsonRpcRequest request, Api &api, Logger &logger) +void sendParsingError(const JsonRpcException &e, const ResponseHandler &respond, Logger &logger) { - auto params = RawParams{std::move(request.params), std::move(request.binary)}; - - logger.info("Calling endpoint for request {}", request.id); - auto result = api.execute(request.method, std::move(params)); - logger.info("Successfully called endpoint"); - - if (std::holds_alternative(request.id)) - { - logger.info("No ID in request, skipping response"); - return {}; - } + logger.info("Composing parsing error"); + auto text = composeError(std::nullopt, e); + logger.info("Parsing error composed"); - logger.info("Composing response"); - auto response = composeResponse(request.id, std::move(result)); - logger.info("Successfully composed response"); - - return response; + logger.info("Sending parsing error"); + respond({std::move(text)}); + logger.info("Parsing error sent"); } -void tryExecuteRequest(JsonRpcRequest request, const ResponseHandler &respond, Api &api, Logger &logger) +void executeRequest(JsonRpcRequest request, const ResponseHandler &respond, Api &api, Logger &logger) { try { - auto [data, binary] = executeRequest(std::move(request), api, logger); + auto params = RawParams{std::move(request.params), std::move(request.binary)}; + + logger.info("Calling endpoint for request {}", toString(request.id)); + auto result = api.execute(request.method, std::move(params)); + logger.info("Successfully called endpoint"); - if (!data.empty()) + if (!request.id) { - respond({data, binary}); + logger.info("No ID in request, skipping response"); + return; } + + logger.info("Composing response"); + auto response = composeResponse(*request.id, std::move(result)); + logger.info("Successfully composed response"); + + logger.info("Sending response"); + respond({response.data, response.binary}); + logger.info("Response sent"); } catch (const JsonRpcException &e) { logger.warn("Error during request execution: '{}'", e.what()); - respond({composeError(request.id, e)}); + sendExecutionError(request.id, e, respond, logger); } catch (const std::exception &e) { logger.error("Unexpected error during request execution: '{}'", e.what()); - respond({composeError(request.id, InternalError(e.what()))}); + sendExecutionError(request.id, InternalError(e.what()), respond, logger); } catch (...) { logger.error("Unknown error during request execution"); - respond({composeError(request.id, InternalError("Unknown handling error"))}); + sendExecutionError(request.id, InternalError("Unknown execution error"), respond, logger); } } void handleRequest(const RawRequest &request, Api &api, Logger &logger) { - auto jsonRpcRequest = tryParseRequest(request, logger); + try + { + logger.info("Parsing JSON-RPC request from client {}", request.clientId); + auto jsonRpcRequest = parseRequest(request); + logger.info("Successfully parsed request"); - if (!jsonRpcRequest) + executeRequest(std::move(jsonRpcRequest), request.respond, api, logger); + } + catch (const JsonRpcException &e) { - return; + logger.warn("Error while parsing request: '{}'", e.what()); + sendParsingError(e, request.respond, logger); + } + catch (const std::exception &e) + { + logger.error("Unexpected error while parsing request: '{}'", e.what()); + sendParsingError(InternalError(e.what()), request.respond, logger); + } + catch (...) + { + logger.error("Unknown error while parsing request"); + sendParsingError(InternalError("Unknown parsing error"), request.respond, logger); } - - tryExecuteRequest(std::move(*jsonRpcRequest), request.respond, api, logger); } } diff --git a/tests/core/api/TestApi.cpp b/tests/core/api/TestApi.cpp index 40ed831e4..0acbc2dbb 100644 --- a/tests/core/api/TestApi.cpp +++ b/tests/core/api/TestApi.cpp @@ -174,7 +174,7 @@ TEST_CASE("Task") return value + offset; }; - builder.task("test", [=](int value) { return startTask(worker, value); }); + builder.task("test", [=](int value) { return startTask(worker, value, {2, "Test"}); }); auto api = builder.build(); @@ -195,12 +195,12 @@ TEST_CASE("Task") while (true) { - auto progress = api.getTaskProgress(taskId); + auto info = api.getTask(taskId); - if (progress.currentOperationProgress != 0.0F) + if (info.currentOperation.completion != 0.0F) { - CHECK_EQ(progress.currentOperation, "1"); - CHECK_EQ(progress.currentOperationProgress, 0.5F); + CHECK_EQ(info.currentOperation.description, "1"); + CHECK_EQ(info.currentOperation.completion, 0.5F); break; } } @@ -212,7 +212,7 @@ TEST_CASE("Task") CHECK_THROWS_AS(api.cancelTask(taskId), InvalidParams); CHECK_THROWS_AS(api.waitForTaskResult(taskId), InvalidParams); - CHECK_THROWS_AS(api.getTaskProgress(taskId), InvalidParams); + CHECK_THROWS_AS(api.getTask(taskId), InvalidParams); CHECK(api.getTasks().empty()); } diff --git a/tests/core/json/TestJsonReflection.cpp b/tests/core/json/TestJsonReflection.cpp index 528fd86a6..a087133d4 100644 --- a/tests/core/json/TestJsonReflection.cpp +++ b/tests/core/json/TestJsonReflection.cpp @@ -55,6 +55,7 @@ struct JsonObjectReflector static auto reflect() { auto builder = JsonBuilder(); + builder.description("Test child"); builder.field("value", [](auto &object) { return &object.value; }); return builder.build(); } @@ -78,6 +79,7 @@ struct JsonObjectReflector static auto reflect() { auto builder = JsonBuilder(); + builder.description("Test parent"); builder.constant("constant", "test"); builder.field("required", [](auto &object) { return &object.required; }); builder.field("bounded", [](auto &object) { return &object.bounded; }).minimum(1).maximum(3); @@ -257,6 +259,7 @@ TEST_CASE("JsonReflection") { const auto &schema = getJsonSchema(); + CHECK_EQ(schema.description, "Test parent"); CHECK_EQ(schema.type, JsonType::Object); const auto &properties = schema.properties; @@ -281,6 +284,7 @@ TEST_CASE("JsonReflection") CHECK_EQ( properties.at("internal"), JsonSchema{ + .description = "Test child", .type = JsonType::Object, .properties = {{"value", getJsonSchema()}}, }); diff --git a/tests/core/jsonrpc/TestJsonRpc.cpp b/tests/core/jsonrpc/TestJsonRpc.cpp index 701ab4bd7..2a10e4228 100644 --- a/tests/core/jsonrpc/TestJsonRpc.cpp +++ b/tests/core/jsonrpc/TestJsonRpc.cpp @@ -37,9 +37,10 @@ TEST_CASE("JsonRpcParser") SUBCASE("Text") { - auto request = parseJsonRpcRequest(json); + auto request = parseTextJsonRpcRequest(json); - CHECK_EQ(std::get(request.id), 1); + CHECK(request.id); + CHECK_EQ(std::get(*request.id), 1); CHECK_EQ(request.method, "test"); CHECK_EQ(request.params, 123); CHECK_EQ(request.binary, ""); @@ -52,7 +53,8 @@ TEST_CASE("JsonRpcParser") auto request = parseBinaryJsonRpcRequest(data); - CHECK_EQ(std::get(request.id), 1); + CHECK(request.id); + CHECK_EQ(std::get(*request.id), 1); CHECK_EQ(request.method, "test"); CHECK_EQ(request.params, 123); CHECK_EQ(request.binary, "binary"); @@ -60,7 +62,7 @@ TEST_CASE("JsonRpcParser") SUBCASE("Invalid JSON") { auto data = "{\"test"; - CHECK_THROWS_AS(parseJsonRpcRequest(data), ParseError); + CHECK_THROWS_AS(parseTextJsonRpcRequest(data), ParseError); } SUBCASE("Invalid schema") { @@ -69,7 +71,7 @@ TEST_CASE("JsonRpcParser") "method": "test", "params": 123 })"; - CHECK_THROWS_AS(parseJsonRpcRequest(data), InvalidRequest); + CHECK_THROWS_AS(parseTextJsonRpcRequest(data), InvalidRequest); } SUBCASE("Invalid JSON-RPC version") { @@ -79,7 +81,7 @@ TEST_CASE("JsonRpcParser") "method": "test", "params": 123 })"; - CHECK_THROWS_AS(parseJsonRpcRequest(data), InvalidRequest); + CHECK_THROWS_AS(parseTextJsonRpcRequest(data), InvalidRequest); } SUBCASE("No methods") { @@ -88,7 +90,7 @@ TEST_CASE("JsonRpcParser") "id": 1, "params": 123 })"; - CHECK_THROWS_AS(parseJsonRpcRequest(data), InvalidRequest); + CHECK_THROWS_AS(parseTextJsonRpcRequest(data), InvalidRequest); } SUBCASE("Binary without size") { diff --git a/tests/core/objects/TestObjectManager.cpp b/tests/core/objects/TestObjectManager.cpp index de03b66ee..3a26407b0 100644 --- a/tests/core/objects/TestObjectManager.cpp +++ b/tests/core/objects/TestObjectManager.cpp @@ -61,7 +61,7 @@ TEST_CASE("Create and remove objects") auto &another = objects.create({"type"}, {}); CHECK_EQ(another.metadata.id, 2); - CHECK_EQ(objects.getIds(), std::vector{1, 2}); + CHECK_EQ(objects.getAllMetadata().size(), 2); CHECK_EQ(&objects.get(1), &object); @@ -78,7 +78,7 @@ TEST_CASE("Create and remove objects") objects.clear(); - CHECK(objects.getIds().empty()); + CHECK(objects.getAllMetadata().empty()); } TEST_CASE("Errors")