From 230304ad726c6b3946d0a64da7b974a5e50f0a2a Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 24 Feb 2022 19:33:41 +0100 Subject: [PATCH 01/19] drafted iteration results table --- .../test_meta_modeling_results.py | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_results.py diff --git a/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_results.py b/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_results.py new file mode 100644 index 00000000000..b93a7ada876 --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_results.py @@ -0,0 +1,161 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + + +import json + + +def test_it(): + + workbench = { + "0f1e38c9-dcb7-443c-a745-91b97ac28ccc": { + "key": "simcore/services/frontend/data-iterator/funky-range", + "version": "1.0.0", + "label": "Integer iterator", + "inputs": {"linspace_start": 0, "linspace_stop": 2, "linspace_step": 1}, + "inputNodes": [], + # some funky output of iterator/param, + "outputs": {"out_1": 1, "out_2": [3, 4]}, + }, + "e33c6880-1b1d-4419-82d7-270197738aa9": { + "key": "simcore/services/comp/itis/sleeper", + "version": "2.0.0", + "label": "sleeper", + "inputs": { + "input_2": { + "nodeUuid": "0f1e38c9-dcb7-443c-a745-91b97ac28ccc", + "output": "out_1", + }, + "input_3": False, + }, + "inputNodes": ["0f1e38c9-dcb7-443c-a745-91b97ac28ccc"], + "state": { + "currentStatus": "SUCCESS", + "modified": False, + "dependencies": [], + }, + "progress": 100, + "outputs": { + "output_1": { + "store": "0", + "path": "30359da5-ca4d-3288-a553-5f426a204fe6/e33c6880-1b1d-4419-82d7-270197738aa9/single_number.txt", + "eTag": "a87ff679a2f3e71d9181a67b7542122c", + }, + "output_2": 7, + }, + "runHash": "f92d1836aa1b6b1b031f9e1b982e631814708675c74ba5f02161e0f256382b2b", + }, + "4c08265a-427b-4ac3-9eab-1d11c822ada4": { + "key": "simcore/services/comp/itis/sleeper", + "version": "2.0.0", + "label": "sleeper", + "inputNodes": [], + }, + "2d0ce8b9-c9c3-43ce-ad2f-ad493898de37": { + "key": "simcore/services/frontend/iterator-consumer/probe/int", + "version": "1.0.0", + "label": "Probe Sensor - Integer", + "inputs": { + "in_1": { + "nodeUuid": "e33c6880-1b1d-4419-82d7-270197738aa9", + "output": "output_2", + } + }, + "inputNodes": ["e33c6880-1b1d-4419-82d7-270197738aa9"], + }, + "445b44d1-59b3-425c-ac48-7c13e0f2ea5b": { + "key": "simcore/services/frontend/iterator-consumer/probe/int", + "version": "1.0.0", + "label": "Probe Sensor - Integer_2", + "inputs": { + "in_1": { + "nodeUuid": "0f1e38c9-dcb7-443c-a745-91b97ac28ccc", + "output": "out_1", + } + }, + "inputNodes": ["0f1e38c9-dcb7-443c-a745-91b97ac28ccc"], + }, + "d76fca06-f050-4790-88a8-0aac10c87b39": { + "key": "simcore/services/frontend/parameter/boolean", + "version": "1.0.0", + "label": "Boolean Parameter", + "inputs": {}, + "inputNodes": [], + "outputs": {"out_1": True}, + }, + } + + # table : Name, Pogress, Labels.... , + # all projects are guaranted to be topologically identical (i.e. same node uuids ) + + progress = {} + + labels = {} # nodeid -> label # NOTE labels are not uniqu + results = ( + {} + ) # nodeid -> { port: value , ...} # results have two levels deep: node/port + for noid, node in workbench.items(): + key_parts = node["key"].split("/") + + # evaluate progress + if "comp" in key_parts: + progress[noid] = node.get("progress", 0) + + # evaluate results + if "probe" in key_parts: + label = node["label"] + values = {} + for port_name, node_input in node["inputs"].items(): + try: + values[port_name] = workbench[node_input["nodeUuid"]]["outputs"][ + node_input["output"] + ] + except KeyError: + # if not run, we know name but NOT value + values[port_name] = "n/a" + results[noid], labels[noid] = values, label + + elif "data-iterator" in key_parts: + label = node["label"] + try: + values = node["outputs"] # {oid: value, ...} + except KeyError: + # if not iterated, we do not know NEITHER name NOT values + values = {} + results[noid], labels[noid] = values, label + + elif "parameter" in key_parts: + label = node["label"] + values = node["outputs"] + results[noid], labels[noid] = values, label + + print(json.dumps(labels, indent=1)) + print(json.dumps(results, indent=1)) + + # this has to be something that shall be deployable in a table + assert progress == { + "4c08265a-427b-4ac3-9eab-1d11c822ada4": 0, + "e33c6880-1b1d-4419-82d7-270197738aa9": 100, + } + + # labels are not unique, so there is a map to nodeids + assert labels == { + "0f1e38c9-dcb7-443c-a745-91b97ac28ccc": "Integer iterator", + "2d0ce8b9-c9c3-43ce-ad2f-ad493898de37": "Probe Sensor - Integer", + "445b44d1-59b3-425c-ac48-7c13e0f2ea5b": "Probe Sensor - Integer_2", + "d76fca06-f050-4790-88a8-0aac10c87b39": "Boolean Parameter", + } + # this is basically a tree that defines columns + assert results == { + "0f1e38c9-dcb7-443c-a745-91b97ac28ccc": {"out_1": 1, "out_2": [3, 4]}, + "2d0ce8b9-c9c3-43ce-ad2f-ad493898de37": {"in_1": 7}, + "445b44d1-59b3-425c-ac48-7c13e0f2ea5b": {"in_1": 1}, + "d76fca06-f050-4790-88a8-0aac10c87b39": {"out_1": True}, + } + + +# t +# Labels = Dict[NodeID, str] +# NodeResults = Dict[PortName, Any] +# ProjectResults = Dict[NodeID, NodeResults] From 5cbbf6dd10a72b0799add920f360591df55cc716 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 24 Feb 2022 21:40:38 +0100 Subject: [PATCH 02/19] meta results --- .../meta_modeling_results.py | 77 +++++++++++++++++++ .../test_meta_modeling_results.py | 68 ++++------------ 2 files changed, 91 insertions(+), 54 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/meta_modeling_results.py diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling_results.py b/services/web/server/src/simcore_service_webserver/meta_modeling_results.py new file mode 100644 index 00000000000..0714e29be24 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/meta_modeling_results.py @@ -0,0 +1,77 @@ +""" Access to the to projects module + + - Adds a middleware to intercept /projects/* requests + - Implements a MetaProjectRunPolicy policy (see director_v2_abc.py) to define how meta-projects run + +""" + + +import logging +from typing import Any, Dict + +from models_library.projects_nodes import Outputs +from models_library.projects_nodes_io import NodeIDStr +from pydantic import BaseModel, Field, conint + +log = logging.getLogger(__name__) + + +class ExtractedResults(BaseModel): + progress: Dict[NodeIDStr, conint(ge=0, lt=100)] = Field( + ..., description="Progress in each computational node" + ) + labels: Dict[NodeIDStr, str] = Field( + ..., description="Maps captured node with a label" + ) + values: Dict[NodeIDStr, Outputs] = Field( + ..., description="Captured outputs per node" + ) + + +def extract_project_results(workbench: Dict[str, Any]) -> ExtractedResults: + # table : Name, Pogress, Labels.... , + # all projects are guaranted to be topologically identical (i.e. same node uuids ) + + progress = {} + # nodeid -> label # NOTE labels are not uniqu + labels = {} + # nodeid -> { port: value , ...} # results have two levels deep: node/port + results = {} + + for noid, node in workbench.items(): + key_parts = node["key"].split("/") + + # evaluate progress + if "comp" in key_parts: + progress[noid] = node.get("progress", 0) + + # evaluate results + if "probe" in key_parts: + label = node["label"] + values = {} + for port_name, node_input in node["inputs"].items(): + try: + values[port_name] = workbench[node_input["nodeUuid"]]["outputs"][ + node_input["output"] + ] + except KeyError: + # if not run, we know name but NOT value + values[port_name] = "n/a" + results[noid], labels[noid] = values, label + + elif "data-iterator" in key_parts: + label = node["label"] + try: + values = node["outputs"] # {oid: value, ...} + except KeyError: + # if not iterated, we do not know NEITHER name NOT values + values = {} + results[noid], labels[noid] = values, label + + elif "parameter" in key_parts: + label = node["label"] + values = node["outputs"] + results[noid], labels[noid] = values, label + + res = ExtractedResults(progress=progress, labels=labels, values=results) + return res diff --git a/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_results.py b/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_results.py index b93a7ada876..ee39096bc66 100644 --- a/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_results.py +++ b/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_results.py @@ -4,11 +4,15 @@ import json +from typing import Any, Dict +import pytest +from simcore_service_webserver.meta_modeling_results import extract_project_results -def test_it(): - workbench = { +@pytest.fixture +def fake_workbench() -> Dict[str, Any]: + return { "0f1e38c9-dcb7-443c-a745-91b97ac28ccc": { "key": "simcore/services/frontend/data-iterator/funky-range", "version": "1.0.0", @@ -86,76 +90,32 @@ def test_it(): }, } - # table : Name, Pogress, Labels.... , - # all projects are guaranted to be topologically identical (i.e. same node uuids ) - progress = {} +def test_extract_project_results(fake_workbench: Dict[str, Any]): - labels = {} # nodeid -> label # NOTE labels are not uniqu - results = ( - {} - ) # nodeid -> { port: value , ...} # results have two levels deep: node/port - for noid, node in workbench.items(): - key_parts = node["key"].split("/") + results = extract_project_results(fake_workbench) - # evaluate progress - if "comp" in key_parts: - progress[noid] = node.get("progress", 0) - - # evaluate results - if "probe" in key_parts: - label = node["label"] - values = {} - for port_name, node_input in node["inputs"].items(): - try: - values[port_name] = workbench[node_input["nodeUuid"]]["outputs"][ - node_input["output"] - ] - except KeyError: - # if not run, we know name but NOT value - values[port_name] = "n/a" - results[noid], labels[noid] = values, label - - elif "data-iterator" in key_parts: - label = node["label"] - try: - values = node["outputs"] # {oid: value, ...} - except KeyError: - # if not iterated, we do not know NEITHER name NOT values - values = {} - results[noid], labels[noid] = values, label - - elif "parameter" in key_parts: - label = node["label"] - values = node["outputs"] - results[noid], labels[noid] = values, label - - print(json.dumps(labels, indent=1)) - print(json.dumps(results, indent=1)) + print(json.dumps(results.progress, indent=1)) + print(json.dumps(results.labels, indent=1)) + print(json.dumps(results.values, indent=1)) # this has to be something that shall be deployable in a table - assert progress == { + assert results.progress == { "4c08265a-427b-4ac3-9eab-1d11c822ada4": 0, "e33c6880-1b1d-4419-82d7-270197738aa9": 100, } # labels are not unique, so there is a map to nodeids - assert labels == { + assert results.labels == { "0f1e38c9-dcb7-443c-a745-91b97ac28ccc": "Integer iterator", "2d0ce8b9-c9c3-43ce-ad2f-ad493898de37": "Probe Sensor - Integer", "445b44d1-59b3-425c-ac48-7c13e0f2ea5b": "Probe Sensor - Integer_2", "d76fca06-f050-4790-88a8-0aac10c87b39": "Boolean Parameter", } # this is basically a tree that defines columns - assert results == { + assert results.values == { "0f1e38c9-dcb7-443c-a745-91b97ac28ccc": {"out_1": 1, "out_2": [3, 4]}, "2d0ce8b9-c9c3-43ce-ad2f-ad493898de37": {"in_1": 7}, "445b44d1-59b3-425c-ac48-7c13e0f2ea5b": {"in_1": 1}, "d76fca06-f050-4790-88a8-0aac10c87b39": {"out_1": True}, } - - -# t -# Labels = Dict[NodeID, str] -# NodeResults = Dict[PortName, Any] -# ProjectResults = Dict[NodeID, NodeResults] From 515d986c5af50f81b5a6988ed7c4fc65455705ef Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 24 Feb 2022 21:42:55 +0100 Subject: [PATCH 03/19] adding handler to fetch meta results --- .../meta_modeling_handlers.py | 112 +++++++++++++++++- .../meta_modeling_version_control.py | 43 +++---- .../test_meta_modeling_iterations.py | 9 +- 3 files changed, 133 insertions(+), 31 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py b/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py index 694ec652463..37e39ef15f7 100644 --- a/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py +++ b/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py @@ -2,7 +2,7 @@ """ import logging -from typing import List, NamedTuple, Optional, Tuple +from typing import Any, Dict, List, NamedTuple, Optional, Tuple from aiohttp import web from models_library.projects import ProjectID @@ -14,6 +14,7 @@ from ._meta import api_version_prefix as VTAG from .meta_modeling_iterations import ProjectIteration +from .meta_modeling_results import ExtractedResults, extract_project_results from .meta_modeling_version_control import VersionControlForMetaModeling from .rest_constants import RESPONSE_MODEL_POLICY from .security_decorators import permission_required @@ -138,7 +139,7 @@ class ParentMetaProjectRef(BaseModel): ref_id: CheckpointID -class ProjectIterationAsItem(BaseModel): +class BaseMetaProjectIteration(BaseModel): name: str = Field( ..., description="Iteration's resource name [AIP-122](https://google.aip.dev/122)", @@ -153,12 +154,20 @@ class ProjectIterationAsItem(BaseModel): "A working copy is a real project where this iteration is run", ) + +class ProjectIterationItem(BaseMetaProjectIteration): + workcopy_project_url: HttpUrl = Field( ..., description="reference to a working copy project" ) url: HttpUrl = Field(..., description="self reference") +class ProjectIterationResultItem(BaseMetaProjectIteration): + results: ExtractedResults + url: HttpUrl = Field(..., description="self reference") + + # ROUTES ------------------------------------------------------------ @@ -206,7 +215,7 @@ async def _list_meta_project_iterations_handler(request: web.Request) -> web.Res # parse and validate response ---- page_items = [ - ProjectIterationAsItem( + ProjectIterationItem( name=f"projects/{_project_uuid}/checkpoint/{commit_id}/iterations/{iter_id}", parent=ParentMetaProjectRef(project_id=_project_uuid, ref_id=commit_id), workcopy_project_id=wcp_id, @@ -223,7 +232,7 @@ async def _list_meta_project_iterations_handler(request: web.Request) -> web.Res for wcp_id, iter_id in iterations.items ] - page = Page[ProjectIterationAsItem].parse_obj( + page = Page[ProjectIterationItem].parse_obj( paginate_data( chunk=page_items, request_url=request.url, @@ -270,7 +279,7 @@ async def _create_meta_project_iterations_handler(request: web.Request) -> web.R # parse and validate response ---- iterations_items = [ - ProjectIterationAsItem( + ProjectIterationItem( name=f"projects/{_project_uuid}/checkpoint/{commit_id}/iterations/{iter_id}", parent=ParentMetaProjectRef(project_id=_project_uuid, ref_id=commit_id), workcopy_project_id=wcp_id, @@ -298,3 +307,96 @@ async def _create_meta_project_iterations_handler(request: web.Request) -> web.R # SEE https://github.com/ITISFoundation/osparc-simcore/issues/2735 async def _get_meta_project_iterations_handler(request: web.Request) -> web.Response: raise NotImplementedError + + +@routes.get( + f"/{VTAG}/projects/{{project_uuid}}/checkpoint/{{ref_id}}/iterations/-/results", + name=f"{__name__}._list_meta_project_iterations_results_handler", +) +async def _list_meta_project_iterations_results_handler( + request: web.Request, +) -> web.Response: + # parse and validate request ---- + url_for = create_url_for_function(request) + vc_repo = VersionControlForMetaModeling(request) + + _project_uuid = ProjectID(request.match_info["project_uuid"]) + _ref_id = request.match_info["ref_id"] + + _limit = int(request.query.get("limit", DEFAULT_NUMBER_OF_ITEMS_PER_PAGE)) + _offset = int(request.query.get("offset", 0)) + + try: + commit_id = CommitID(_ref_id) + except ValueError as err: + # e.g. HEAD + raise NotImplementedError( + "cannot convert ref (e.g. HEAD) -> commit id" + ) from err + + # core function ---- + iterations = await _get_project_iterations_range( + vc_repo, _project_uuid, commit_id, offset=_offset, limit=_limit + ) + + if iterations.total_count == 0: + raise web.HTTPNotFound( + reason=f"No iterations found for project {_project_uuid=}/{commit_id=}" + ) + + assert len(iterations.items) <= _limit # nosec + + # get every project from the database and extract results + # TODO: fetch ALL project iterations at once. Otherwise they will have different results + projects_data: List[Dict[str, Any]] = [] + for project_id, commit_id in iterations.items: + prj = await vc_repo.get_project( + f"{project_id}", include=["uuid", "name", "workbench"] + ) + projects_data.append(prj) + + results: List[ExtractedResults] = [] + for prj in projects_data: + # TODO: if raises? + res = extract_project_results(prj) + results.append(res) + + # parse and validate response ---- + page_items = [ + ProjectIterationResultItem( + name=f"projects/{_project_uuid}/checkpoint/{commit_id}/iterations/{iter_id}/results", + parent=ParentMetaProjectRef(project_id=_project_uuid, ref_id=commit_id), + workcopy_project_id=wcp_id, + results=res, + url=url_for( + f"{__name__}._list_meta_project_iterations_results_handler", + project_uuid=_project_uuid, + ref_id=commit_id, + ), + ) + for (wcp_id, iter_id), res in zip(iterations.items, results) + ] + + page = Page[ProjectIterationResultItem].parse_obj( + paginate_data( + chunk=page_items, + request_url=request.url, + total=iterations.total_count, + limit=_limit, + offset=_offset, + ) + ) + return web.Response( + text=page.json(**RESPONSE_MODEL_POLICY), + content_type="application/json", + ) + + +# @routes.get( +# f"/{VTAG}/projects/{{project_uuid}}/checkpoint/{{ref_id}}/iterations/{{iter_id}}/results", +# name=f"{__name__}._get_meta_project_iterations_handler", +# ) +async def _get_meta_project_iteration_results_handler( + request: web.Request, +) -> web.Response: + raise NotImplementedError diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling_version_control.py b/services/web/server/src/simcore_service_webserver/meta_modeling_version_control.py index aff42f0efd1..cad0b62a425 100644 --- a/services/web/server/src/simcore_service_webserver/meta_modeling_version_control.py +++ b/services/web/server/src/simcore_service_webserver/meta_modeling_version_control.py @@ -43,32 +43,35 @@ async def get_workcopy_project(self, repo_id: int, commit_id: int) -> ProjectDic assert project # nosec return dict(project.items()) - async def get_project(self, project_id: ProjectIDStr) -> ProjectDict: + async def get_project( + self, project_id: ProjectIDStr, *, include: Optional[List[str]] = None + ) -> ProjectDict: async with self.engine.acquire() as conn: if self.user_id is None: raise UserUndefined() + if include is None: + include = [ + "type", + "uuid", + "name", + "description", + "thumbnail", + "prj_owner", + "access_rights", + "workbench", + "ui", + "classifiers", + "dev", + "quality", + "published", + "hidden", + ] + project = ( await self.ProjectsOrm(conn) - .set_filter(uuid=str(project_id), prj_owner=self.user_id) - .fetch( - [ - "type", - "uuid", - "name", - "description", - "thumbnail", - "prj_owner", - "access_rights", - "workbench", - "ui", - "classifiers", - "dev", - "quality", - "published", - "hidden", - ] - ) + .set_filter(uuid=f"{project_id}", prj_owner=self.user_id) + .fetch(include) ) assert project # nosec project_as_dict = dict(project.items()) diff --git a/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_iterations.py b/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_iterations.py index eba7bc2374b..55913cf5eeb 100644 --- a/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_iterations.py +++ b/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_iterations.py @@ -20,10 +20,7 @@ from simcore_postgres_database.models.projects import projects from simcore_service_webserver._constants import APP_DB_ENGINE_KEY from simcore_service_webserver.director_v2_api import get_project_run_policy -from simcore_service_webserver.meta_modeling_handlers import ( - Page, - ProjectIterationAsItem, -) +from simcore_service_webserver.meta_modeling_handlers import Page, ProjectIterationItem from simcore_service_webserver.meta_modeling_projects import ( meta_project_policy, projects_redirection_middleware, @@ -138,7 +135,7 @@ async def _mock_start(project_id, user_id, **options): f"/v0/projects/{project_uuid}/checkpoint/{head_ref_id}/iterations?offset=0" ) body = await resp.json() - first_iterlist = Page[ProjectIterationAsItem].parse_obj(body).data + first_iterlist = Page[ProjectIterationItem].parse_obj(body).data assert len(first_iterlist) == 3 @@ -228,7 +225,7 @@ async def _mock_catalog_get(app, user_id, product_name, only_key_versions): f"/v0/projects/{project_uuid}/checkpoint/{head_ref_id}/iterations?offset=0" ) body = await resp.json() - second_iterlist = Page[ProjectIterationAsItem].parse_obj(body).data + second_iterlist = Page[ProjectIterationItem].parse_obj(body).data assert len(second_iterlist) == 4 assert len(set(it.workcopy_project_id for it in second_iterlist)) == len( From cf0282ae72af76822ef57a2cd0dc846f7a8e89e3 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Feb 2022 19:29:50 +0100 Subject: [PATCH 04/19] adds iteration index --- .../meta_modeling_iterations.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling_iterations.py b/services/web/server/src/simcore_service_webserver/meta_modeling_iterations.py index 28f05e4b3f8..245a2ad56a7 100644 --- a/services/web/server/src/simcore_service_webserver/meta_modeling_iterations.py +++ b/services/web/server/src/simcore_service_webserver/meta_modeling_iterations.py @@ -17,6 +17,7 @@ from models_library.services import ServiceDockerData from pydantic import BaseModel, ValidationError from pydantic.fields import Field +from pydantic.types import PositiveInt from .meta_modeling_function_nodes import ( FUNCTION_SERVICE_TO_CALLABLE, @@ -124,6 +125,8 @@ def extract_parameters( # DOMAIN MODEL for project iteration ------------------------------------------------------------ +IterationID = PositiveInt + class ProjectIteration(BaseModel): """ @@ -135,10 +138,16 @@ class ProjectIteration(BaseModel): # version-control info repo_id: Optional[int] = None - repo_commit_id: CommitID = Field(...) + repo_commit_id: CommitID = Field( + ..., + description="this id makes it unique but does not guarantees order. See iter_index for that", + ) # iteration info - iter_index: int = Field(...) + iteration_index: IterationID = Field( + ..., + description="Index that allows iterations to be sortable", + ) total_count: Union[int, Literal["unbound"]] = "unbound" parameters_checksum: SHA1Str = Field(...) @@ -159,7 +168,7 @@ def to_tag_name(self) -> str: """Composes unique tag name for this iteration""" return compose_iteration_tag_name( repo_commit_id=self.repo_commit_id, - iter_index=self.iter_index, + iter_index=self.iteration_index, total_count=self.total_count, parameters_checksum=self.parameters_checksum, ) @@ -282,7 +291,7 @@ async def get_or_create_runnable_projects( project_iteration = ProjectIteration( repo_id=repo_id, repo_commit_id=main_commit_id, - iter_index=iter_index, + iteration_index=iter_index, total_count=total_count, parameters_checksum=_compute_params_checksum(parameters), ) From af12e0f3734082d2f6eaee9f6f05e3d69af6171a Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Feb 2022 19:39:54 +0100 Subject: [PATCH 05/19] adds utils to simplify handlers implementation --- .../meta_modeling_handlers.py | 246 ++++++++++-------- 1 file changed, 138 insertions(+), 108 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py b/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py index 37e39ef15f7..b7db6e72a61 100644 --- a/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py +++ b/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py @@ -2,18 +2,18 @@ """ import logging -from typing import Any, Dict, List, NamedTuple, Optional, Tuple +from typing import List, NamedTuple, Optional from aiohttp import web from models_library.projects import ProjectID from models_library.rest_pagination import DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, Page from models_library.rest_pagination_utils import paginate_data -from pydantic import BaseModel +from pydantic import BaseModel, ValidationError, validator from pydantic.fields import Field from pydantic.networks import HttpUrl from ._meta import api_version_prefix as VTAG -from .meta_modeling_iterations import ProjectIteration +from .meta_modeling_iterations import IterationID, ProjectIteration from .meta_modeling_results import ExtractedResults, extract_project_results from .meta_modeling_version_control import VersionControlForMetaModeling from .rest_constants import RESPONSE_MODEL_POLICY @@ -24,10 +24,33 @@ log = logging.getLogger(__name__) - # HANDLER'S CORE IMPLEMENTATION ------------------------------------------------------------ -IterationTuple = Tuple[ProjectID, CommitID] + +class _QueryParametersModel(BaseModel): + project_uuid: ProjectID + ref_id: CommitID + limit: int = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE + offset: int = 0 + + @validator("ref_id", pre=True) + @classmethod + def tags_as_refid_not_implemented(cls, v): + try: + return CommitID(v) + except ValueError as err: + # e.g. HEAD + raise NotImplementedError( + "cannot convert ref (e.g. HEAD) -> commit id" + ) from err + + +def parse_query_parameters(request: web.Request) -> _QueryParametersModel: + try: + return _QueryParametersModel(**request.match_info) + except ValidationError as err: + # TODO: compose reason message better + raise web.HTTPUnprocessableEntity(reason=f"Invalid query parameters: {err}") class _NotTaggedAsIteration(Exception): @@ -38,8 +61,14 @@ class _NotTaggedAsIteration(Exception): ... +class IterationItem(NamedTuple): + project_id: ProjectID + commit_id: CommitID + iteration_index: IterationID + + class _IterationsRange(NamedTuple): - items: List[IterationTuple] + items: List[IterationItem] total_count: int @@ -65,7 +94,7 @@ async def _get_project_iterations_range( repo_id, commit_id ) - iterations: List[Tuple[ProjectID, CommitID]] = [] + iter_items: List[IterationItem] = [] for n, tags in enumerate(tags_per_child): try: iteration: Optional[ProjectIteration] = None @@ -94,7 +123,13 @@ async def _get_project_iterations_range( if not iteration: raise _NotTaggedAsIteration(f"No iteration tag found in {tags=}") - iterations.append((workcopy_id, iteration.iter_index)) + iter_items.append( + IterationItem( + project_id=workcopy_id, + commit_id=iteration.repo_commit_id, + iteration_index=iteration.iteration_index, + ) + ) except _NotTaggedAsIteration as err: log.warning( @@ -106,18 +141,19 @@ async def _get_project_iterations_range( ) # Selects range on those tagged as iterations and returned their assigned workcopy id - total_number_of_iterations = len(iterations) + total_number_of_iterations = len(iter_items) # sort and select. If requested interval is outside of range, it returns empty - iterations.sort(key=lambda tup: tup[1]) + iter_items.sort(key=lambda item: item.iteration_index) if limit is None: return _IterationsRange( - items=iterations[offset:], total_count=total_number_of_iterations + items=iter_items[offset:], + total_count=total_number_of_iterations, ) return _IterationsRange( - items=iterations[offset : (offset + limit)], + items=iter_items[offset : (offset + limit)], total_count=total_number_of_iterations, ) @@ -126,7 +162,7 @@ async def create_or_get_project_iterations( vc_repo: VersionControlForMetaModeling, project_uuid: ProjectID, commit_id: CommitID, -) -> List[IterationTuple]: +) -> List[IterationItem]: raise NotImplementedError() @@ -139,33 +175,38 @@ class ParentMetaProjectRef(BaseModel): ref_id: CheckpointID -class BaseMetaProjectIteration(BaseModel): +class _BaseModelGet(BaseModel): name: str = Field( ..., - description="Iteration's resource name [AIP-122](https://google.aip.dev/122)", + description="Iteration's resource API name", + x_mark_resouce_name=True, # [AIP-122](https://google.aip.dev/122) ) parent: ParentMetaProjectRef = Field( ..., description="Reference to the the meta-project that created this iteration" ) + url: HttpUrl = Field(..., description="self reference") + + +class ProjectIterationItem(_BaseModelGet): + iteration_index: IterationID = Field( + ..., + x_mark_resource_id_segment=True, # [AIP-122](https://google.aip.dev/122) + ) + workcopy_project_id: ProjectID = Field( ..., description="ID to this iteration's working copy." "A working copy is a real project where this iteration is run", ) - -class ProjectIterationItem(BaseMetaProjectIteration): - workcopy_project_url: HttpUrl = Field( ..., description="reference to a working copy project" ) - url: HttpUrl = Field(..., description="self reference") -class ProjectIterationResultItem(BaseMetaProjectIteration): +class ProjectIterationResultItem(ProjectIterationItem): results: ExtractedResults - url: HttpUrl = Field(..., description="self reference") # ROUTES ------------------------------------------------------------ @@ -184,61 +225,58 @@ async def _list_meta_project_iterations_handler(request: web.Request) -> web.Res # SEE https://github.com/ITISFoundation/osparc-simcore/issues/2735 # parse and validate request ---- + q = parse_query_parameters(request) + meta_project_uuid = q.project_uuid + meta_project_commit_id = q.ref_id + url_for = create_url_for_function(request) vc_repo = VersionControlForMetaModeling(request) - _project_uuid = ProjectID(request.match_info["project_uuid"]) - _ref_id = request.match_info["ref_id"] - - _limit = int(request.query.get("limit", DEFAULT_NUMBER_OF_ITEMS_PER_PAGE)) - _offset = int(request.query.get("offset", 0)) - - try: - commit_id = CommitID(_ref_id) - except ValueError as err: - # e.g. HEAD - raise NotImplementedError( - "cannot convert ref (e.g. HEAD) -> commit id" - ) from err - # core function ---- - iterations = await _get_project_iterations_range( - vc_repo, _project_uuid, commit_id, offset=_offset, limit=_limit + iterations_range = await _get_project_iterations_range( + vc_repo, + meta_project_uuid, + meta_project_commit_id, + offset=q.offset, + limit=q.limit, ) - if iterations.total_count == 0: + if iterations_range.total_count == 0: raise web.HTTPNotFound( - reason=f"No iterations found for project {_project_uuid=}/{commit_id=}" + reason=f"No iterations found for project {meta_project_uuid=}/{meta_project_commit_id=}" ) - assert len(iterations.items) <= _limit # nosec + assert len(iterations_range.items) <= q.limit # nosec # parse and validate response ---- page_items = [ ProjectIterationItem( - name=f"projects/{_project_uuid}/checkpoint/{commit_id}/iterations/{iter_id}", - parent=ParentMetaProjectRef(project_id=_project_uuid, ref_id=commit_id), - workcopy_project_id=wcp_id, + name=f"projects/{meta_project_uuid}/checkpoint/{meta_project_commit_id}/iterations/{item.iteration_index}", + parent=ParentMetaProjectRef( + project_id=meta_project_uuid, ref_id=meta_project_commit_id + ), + iteration_index=item.iteration_index, + workcopy_project_id=item.project_id, workcopy_project_url=url_for( "get_project", - project_id=wcp_id, + project_id=item.project_id, ), url=url_for( f"{__name__}._list_meta_project_iterations_handler", - project_uuid=_project_uuid, - ref_id=commit_id, + project_uuid=meta_project_uuid, + ref_id=meta_project_commit_id, ), ) - for wcp_id, iter_id in iterations.items + for item in iterations_range.items ] page = Page[ProjectIterationItem].parse_obj( paginate_data( chunk=page_items, request_url=request.url, - total=iterations.total_count, - limit=_limit, - offset=_offset, + total=iterations_range.total_count, + limit=q.limit, + offset=q.offset, ) ) return web.Response( @@ -259,41 +297,38 @@ async def _create_meta_project_iterations_handler(request: web.Request) -> web.R # TODO: check access to non owned projects user_id = request[RQT_USERID_KEY] # SEE https://github.com/ITISFoundation/osparc-simcore/issues/2735 + q = parse_query_parameters(request) + meta_project_uuid = q.project_uuid + meta_project_commit_id = q.ref_id + url_for = create_url_for_function(request) vc_repo = VersionControlForMetaModeling(request) - _project_uuid = ProjectID(request.match_info["project_uuid"]) - _ref_id = request.match_info["ref_id"] - try: - commit_id = CommitID(_ref_id) - except ValueError as err: - # e.g. HEAD - raise NotImplementedError( - "cannot convert ref (e.g. HEAD) -> commit id" - ) from err - # core function ---- project_iterations = await create_or_get_project_iterations( - vc_repo, _project_uuid, commit_id + vc_repo, meta_project_uuid, meta_project_commit_id ) # parse and validate response ---- iterations_items = [ ProjectIterationItem( - name=f"projects/{_project_uuid}/checkpoint/{commit_id}/iterations/{iter_id}", - parent=ParentMetaProjectRef(project_id=_project_uuid, ref_id=commit_id), - workcopy_project_id=wcp_id, + name=f"projects/{meta_project_uuid}/checkpoint/{meta_project_commit_id}/iterations/{item.iteration_index}", + parent=ParentMetaProjectRef( + project_id=meta_project_uuid, ref_id=meta_project_commit_id + ), + iteration_index=item.iteration_index, + workcopy_project_id=item.project_id, workcopy_project_url=url_for( "get_project", - project_id=wcp_id, + project_id=item.project_id, ), url=url_for( f"{__name__}._create_meta_project_iterations_handler", - project_uuid=_project_uuid, - ref_id=commit_id, + project_uuid=meta_project_uuid, + ref_id=meta_project_commit_id, ), ) - for wcp_id, iter_id in project_iterations + for item in project_iterations ] return envelope_json_response(iterations_items, web.HTTPCreated) @@ -317,73 +352,68 @@ async def _list_meta_project_iterations_results_handler( request: web.Request, ) -> web.Response: # parse and validate request ---- + q = parse_query_parameters(request) + meta_project_uuid = q.project_uuid + meta_project_commit_id = q.ref_id + url_for = create_url_for_function(request) vc_repo = VersionControlForMetaModeling(request) - _project_uuid = ProjectID(request.match_info["project_uuid"]) - _ref_id = request.match_info["ref_id"] - - _limit = int(request.query.get("limit", DEFAULT_NUMBER_OF_ITEMS_PER_PAGE)) - _offset = int(request.query.get("offset", 0)) - - try: - commit_id = CommitID(_ref_id) - except ValueError as err: - # e.g. HEAD - raise NotImplementedError( - "cannot convert ref (e.g. HEAD) -> commit id" - ) from err - # core function ---- - iterations = await _get_project_iterations_range( - vc_repo, _project_uuid, commit_id, offset=_offset, limit=_limit + iterations_range = await _get_project_iterations_range( + vc_repo, + meta_project_uuid, + meta_project_commit_id, + offset=q.offset, + limit=q.limit, ) - if iterations.total_count == 0: + if iterations_range.total_count == 0: raise web.HTTPNotFound( - reason=f"No iterations found for project {_project_uuid=}/{commit_id=}" + reason=f"No iterations found for projects/{meta_project_uuid}/checkpoint/{meta_project_commit_id}" ) - assert len(iterations.items) <= _limit # nosec + assert len(iterations_range.items) <= q.limit # nosec # get every project from the database and extract results - # TODO: fetch ALL project iterations at once. Otherwise they will have different results - projects_data: List[Dict[str, Any]] = [] - for project_id, commit_id in iterations.items: - prj = await vc_repo.get_project( - f"{project_id}", include=["uuid", "name", "workbench"] - ) - projects_data.append(prj) + _prj_data = {} + for item in iterations_range.items: + # TODO: fetch ALL project iterations at once. Otherwise they will have different results + # TODO: if raises? + prj = await vc_repo.get_project(f"{item.project_id}", include=["workbench"]) + _prj_data[item.project_id] = prj["workbench"] - results: List[ExtractedResults] = [] - for prj in projects_data: + def _get_project_results(project_id) -> ExtractedResults: # TODO: if raises? - res = extract_project_results(prj) - results.append(res) + results = extract_project_results(_prj_data[project_id]) + return results # parse and validate response ---- page_items = [ ProjectIterationResultItem( - name=f"projects/{_project_uuid}/checkpoint/{commit_id}/iterations/{iter_id}/results", - parent=ParentMetaProjectRef(project_id=_project_uuid, ref_id=commit_id), - workcopy_project_id=wcp_id, - results=res, + name=f"projects/{meta_project_uuid}/checkpoint/{meta_project_commit_id}/iterations/{item.iteration_index}/results", + parent=ParentMetaProjectRef( + project_id=meta_project_uuid, ref_id=meta_project_commit_id + ), + iteration_index=item.iteration_index, + workcopy_project_id=item.project_id, + results=_get_project_results(item.project_id), url=url_for( f"{__name__}._list_meta_project_iterations_results_handler", - project_uuid=_project_uuid, - ref_id=commit_id, + project_uuid=meta_project_uuid, + ref_id=meta_project_commit_id, ), ) - for (wcp_id, iter_id), res in zip(iterations.items, results) + for item in iterations_range.items ] page = Page[ProjectIterationResultItem].parse_obj( paginate_data( chunk=page_items, request_url=request.url, - total=iterations.total_count, - limit=_limit, - offset=_offset, + total=iterations_range.total_count, + limit=q.limit, + offset=q.offset, ) ) return web.Response( From 3eafebf86404bca87946e431c0368f45bb26e2a7 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Feb 2022 19:45:32 +0100 Subject: [PATCH 06/19] minor fix on constraint --- .../src/simcore_service_webserver/meta_modeling_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling_results.py b/services/web/server/src/simcore_service_webserver/meta_modeling_results.py index 0714e29be24..b26eedf8b6d 100644 --- a/services/web/server/src/simcore_service_webserver/meta_modeling_results.py +++ b/services/web/server/src/simcore_service_webserver/meta_modeling_results.py @@ -17,7 +17,7 @@ class ExtractedResults(BaseModel): - progress: Dict[NodeIDStr, conint(ge=0, lt=100)] = Field( + progress: Dict[NodeIDStr, conint(ge=0, le=100)] = Field( ..., description="Progress in each computational node" ) labels: Dict[NodeIDStr, str] = Field( From 9d32b2b1d6028050b72477ad31afe283a848745b Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Feb 2022 19:56:37 +0100 Subject: [PATCH 07/19] updates OAS --- .../webserver/openapi-meta-projects.yaml | 96 +++++++++++++++++-- api/specs/webserver/openapi.yaml | 3 + .../api/v0/openapi.yaml | 81 +++++++++++++++- 3 files changed, 172 insertions(+), 8 deletions(-) diff --git a/api/specs/webserver/openapi-meta-projects.yaml b/api/specs/webserver/openapi-meta-projects.yaml index a723501cfa0..90b108468a7 100644 --- a/api/specs/webserver/openapi-meta-projects.yaml +++ b/api/specs/webserver/openapi-meta-projects.yaml @@ -52,7 +52,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/Page_IterationAsItem_" + $ref: "#/components/schemas/Page_IterationItem_" "404": description: This project has no iterations.Only meta-project have iterations @@ -63,6 +63,72 @@ paths: application/json: schema: $ref: "#/components/schemas/HTTPValidationError" + + /projects/{project_uuid}/checkpoint/{ref_id}/iterations/-/results: + get: + tags: + - meta-projects + summary: List Project Iterations Results + description: Lists current project's iterations results table + operationId: "simcore_service_webserver.meta_modeling_handlers._list_meta_project_iterations_results_handler" + parameters: + - description: Project unique identifier + required: true + schema: + title: Project Uuid + type: string + description: Project unique identifier + format: uuid + name: project_uuid + in: path + - required: true + schema: + title: Ref Id + anyOf: + - type: integer + - type: string + name: ref_id + in: path + - description: index to the first item to return (pagination) + required: false + schema: + title: Offset + exclusiveMinimum: false + type: integer + description: index to the first item to return (pagination) + default: 0 + minimum: 0 + name: offset + in: query + - description: maximum number of items to return (pagination) + required: false + schema: + title: Limit + maximum: 50.0 + minimum: 1.0 + type: integer + description: maximum number of items to return (pagination) + default: 20 + name: limit + in: query + responses: + "200": + description: Successful Response + content: + application/json: + schema: + $ref: "#/components/schemas/Page_IterationResultItem_" + "404": + description: + This project has no iterations.Only meta-project have iterations + and they must be explicitly created. + "422": + description: Validation Error + content: + application/json: + schema: + $ref: "#/components/schemas/HTTPValidationError" + components: schemas: HTTPValidationError: @@ -74,8 +140,8 @@ components: type: array items: $ref: "#/components/schemas/ValidationError" - IterationAsItem: - title: IterationAsItem + IterationItem: + title: IterationItem required: - name - parent @@ -177,8 +243,8 @@ components: title: Count minimum: 0.0 type: integer - Page_IterationAsItem_: - title: Page[IterationAsItem] + Page_IterationItem_: + title: Page[IterationItem] required: - _meta - _links @@ -193,7 +259,7 @@ components: title: Data type: array items: - $ref: "#/components/schemas/IterationAsItem" + $ref: "#/components/schemas/IterationItem" ParentMetaProjectRef: title: ParentMetaProjectRef required: @@ -227,3 +293,21 @@ components: type: title: Error Type type: string + Page_IterationResultItem_: + title: Page[IterationResultItem] + required: + - _meta + - _links + - data + type: object + properties: + _meta: + $ref: "#/components/schemas/PageMetaInfoLimitOffset" + _links: + $ref: "#/components/schemas/PageLinks" + data: + title: Data + type: array + items: + $ref: "#/components/schemas/IterationItem" + # NOTE: intentionally wrong. Will be deprecated diff --git a/api/specs/webserver/openapi.yaml b/api/specs/webserver/openapi.yaml index 6b1a5b97119..75870f98367 100644 --- a/api/specs/webserver/openapi.yaml +++ b/api/specs/webserver/openapi.yaml @@ -223,6 +223,9 @@ paths: /projects/{project_uuid}/checkpoint/{ref_id}/iterations: $ref: "./openapi-meta-projects.yaml#/paths/~1projects~1{project_uuid}~1checkpoint~1{ref_id}~1iterations" + /projects/{project_uuid}/checkpoint/{ref_id}/iterations/-/results: + $ref: "./openapi-meta-projects.yaml#/paths/~1projects~1{project_uuid}~1checkpoint~1{ref_id}~1iterations~1-~1results" + # REPOSITORY ------------------------------------------------------------------------- /repos/projects: $ref: "./openapi-version-control.yaml#/paths/~1repos~1projects" diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index d6334bfde49..553a0bbee80 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2943,7 +2943,7 @@ paths: content: application/json: schema: - title: 'Page[IterationAsItem]' + title: 'Page[IterationItem]' required: - _meta - _links @@ -3019,7 +3019,7 @@ paths: title: Data type: array items: - title: IterationAsItem + title: IterationItem required: - name - parent @@ -3098,6 +3098,83 @@ paths: type: title: Error Type type: string + '/projects/{project_uuid}/checkpoint/{ref_id}/iterations/-/results': + get: + tags: + - meta-projects + summary: List Project Iterations Results + description: Lists current project's iterations results table + operationId: simcore_service_webserver.meta_modeling_handlers._list_meta_project_iterations_results_handler + parameters: + - description: Project unique identifier + required: true + schema: + title: Project Uuid + type: string + description: Project unique identifier + format: uuid + name: project_uuid + in: path + - required: true + schema: + title: Ref Id + anyOf: + - type: integer + - type: string + name: ref_id + in: path + - description: index to the first item to return (pagination) + required: false + schema: + title: Offset + exclusiveMinimum: false + type: integer + description: index to the first item to return (pagination) + default: 0 + minimum: 0 + name: offset + in: query + - description: maximum number of items to return (pagination) + required: false + schema: + title: Limit + maximum: 50 + minimum: 1 + type: integer + description: maximum number of items to return (pagination) + default: 20 + name: limit + in: query + responses: + '200': + description: Successful Response + content: + application/json: + schema: + title: 'Page[IterationResultItem]' + required: + - _meta + - _links + - data + type: object + properties: + _meta: + $ref: '#/paths/~1projects~1%7Bproject_uuid%7D~1checkpoint~1%7Bref_id%7D~1iterations/get/responses/200/content/application~1json/schema/properties/_meta' + _links: + $ref: '#/paths/~1projects~1%7Bproject_uuid%7D~1checkpoint~1%7Bref_id%7D~1iterations/get/responses/200/content/application~1json/schema/properties/_links' + data: + title: Data + type: array + items: + $ref: '#/paths/~1projects~1%7Bproject_uuid%7D~1checkpoint~1%7Bref_id%7D~1iterations/get/responses/200/content/application~1json/schema/properties/data/items' + '404': + description: This project has no iterations.Only meta-project have iterations and they must be explicitly created. + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/paths/~1projects~1%7Bproject_uuid%7D~1checkpoint~1%7Bref_id%7D~1iterations/get/responses/422/content/application~1json/schema' /repos/projects: get: tags: From 846b6fd8f4c954ab9f5f11e5c40adb1c1ff58684 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 1 Mar 2022 10:29:39 +0100 Subject: [PATCH 08/19] fixes iteration model validation --- .../meta_modeling_iterations.py | 26 +++++++++---------- .../test_meta_modeling_iterations.py | 1 + 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling_iterations.py b/services/web/server/src/simcore_service_webserver/meta_modeling_iterations.py index 245a2ad56a7..82c1f20967d 100644 --- a/services/web/server/src/simcore_service_webserver/meta_modeling_iterations.py +++ b/services/web/server/src/simcore_service_webserver/meta_modeling_iterations.py @@ -168,7 +168,7 @@ def to_tag_name(self) -> str: """Composes unique tag name for this iteration""" return compose_iteration_tag_name( repo_commit_id=self.repo_commit_id, - iter_index=self.iteration_index, + iteration_index=self.iteration_index, total_count=self.total_count, parameters_checksum=self.parameters_checksum, ) @@ -177,22 +177,20 @@ def to_tag_name(self) -> str: # NOTE: compose_/parse_ functions are basically serialization functions for ProjectIteration # into/from string tags. An alternative approach would be simply using json.dump/load # but we should guarantee backwards compatibilty with old tags -# +# TODO: change this by json-serialization def compose_iteration_tag_name( repo_commit_id: CommitID, - iter_index: int, + iteration_index: IterationID, total_count: Union[int, str], parameters_checksum: SHA1Str, ) -> str: """Composes unique tag name for iter_index-th iteration of repo_commit_id out of total_count""" - return ( - f"iteration:{repo_commit_id}/{iter_index}/{total_count}/{parameters_checksum}" - ) + return f"iteration:{repo_commit_id}/{iteration_index}/{total_count}/{parameters_checksum}" def parse_iteration_tag_name(name: str) -> Dict[str, Any]: if m := re.match( - r"^iteration:(?P\d+)/(?P\d+)/(?P-*\d+)/(?P.*)$", + r"^iteration:(?P\d+)/(?P\d+)/(?P-*\d+)/(?P.*)$", name, ): return m.groupdict() @@ -270,14 +268,16 @@ async def get_or_create_runnable_projects( total_count = len(iterations) original_name = project["name"] - for iter_index, (parameters, updated_nodes) in enumerate(iterations): + # FIXME: in an optimization, iteration_index should start with LAST iterated index + for iteration_index, (parameters, updated_nodes) in enumerate(iterations, start=1): log.debug( - "Creating snapshot of project %s with parameters=%s", - project_uuid, - parameters, + "Creating snapshot of project %s with parameters=%s [%s]", + f"{project_uuid=}", + f"{parameters=}", + f"{updated_nodes=}", ) - project["name"] = f"{original_name}/{iter_index}" + project["name"] = f"{original_name}/{iteration_index}" project["workbench"].update( { # converts model in dict patching first thumbnail @@ -291,7 +291,7 @@ async def get_or_create_runnable_projects( project_iteration = ProjectIteration( repo_id=repo_id, repo_commit_id=main_commit_id, - iteration_index=iter_index, + iteration_index=iteration_index, total_count=total_count, parameters_checksum=_compute_params_checksum(parameters), ) diff --git a/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_iterations.py b/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_iterations.py index 55913cf5eeb..64f9f62818c 100644 --- a/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_iterations.py +++ b/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_iterations.py @@ -225,6 +225,7 @@ async def _mock_catalog_get(app, user_id, product_name, only_key_versions): f"/v0/projects/{project_uuid}/checkpoint/{head_ref_id}/iterations?offset=0" ) body = await resp.json() + assert resp.status == 200, f"{body=}" # nosec second_iterlist = Page[ProjectIterationItem].parse_obj(body).data assert len(second_iterlist) == 4 From c24b965e32582bf30410e67f20980e54ef7e97bd Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 1 Mar 2022 11:30:37 +0100 Subject: [PATCH 09/19] added factories to simplify creating models, login required, etc --- .../meta_modeling_handlers.py | 159 +++++++++++------- .../test_meta_modeling_iterations.py | 18 +- 2 files changed, 117 insertions(+), 60 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py b/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py index b7db6e72a61..9eec59d76bf 100644 --- a/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py +++ b/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py @@ -2,7 +2,7 @@ """ import logging -from typing import List, NamedTuple, Optional +from typing import Callable, List, NamedTuple, Optional from aiohttp import web from models_library.projects import ProjectID @@ -13,6 +13,7 @@ from pydantic.networks import HttpUrl from ._meta import api_version_prefix as VTAG +from .login.decorators import login_required from .meta_modeling_iterations import IterationID, ProjectIteration from .meta_modeling_results import ExtractedResults, extract_project_results from .meta_modeling_version_control import VersionControlForMetaModeling @@ -204,13 +205,70 @@ class ProjectIterationItem(_BaseModelGet): ..., description="reference to a working copy project" ) + @classmethod + def create( + cls, + meta_project_uuid, + meta_project_commit_id, + iteration_index, + project_id, + url_for: Callable, + ): + return cls( + name=f"projects/{meta_project_uuid}/checkpoint/{meta_project_commit_id}/iterations/{iteration_index}", + parent=ParentMetaProjectRef( + project_id=meta_project_uuid, ref_id=meta_project_commit_id + ), + iteration_index=iteration_index, + workcopy_project_id=project_id, + workcopy_project_url=url_for( + "get_project", + project_id=project_id, + ), + url=url_for( + f"{__name__}._get_meta_project_iterations_handler", + project_uuid=meta_project_uuid, + ref_id=meta_project_commit_id, + iter_id=iteration_index, + ), + ) + class ProjectIterationResultItem(ProjectIterationItem): results: ExtractedResults + @classmethod + def create( + cls, + meta_project_uuid, + meta_project_commit_id, + iteration_index, + project_id, + results, + url_for: Callable, + ): + return cls( + name=f"projects/{meta_project_uuid}/checkpoint/{meta_project_commit_id}/iterations/{iteration_index}/results", + parent=ParentMetaProjectRef( + project_id=meta_project_uuid, ref_id=meta_project_commit_id + ), + iteration_index=iteration_index, + workcopy_project_id=project_id, + results=results, + workcopy_project_url=url_for( + "get_project", + project_id=project_id, + ), + url=url_for( + f"{__name__}._get_meta_project_iteration_results_handler", + project_uuid=meta_project_uuid, + ref_id=meta_project_commit_id, + iter_id=iteration_index, + ), + ) -# ROUTES ------------------------------------------------------------ +# ROUTES ------------------------------------------------------------ routes = web.RouteTableDef() @@ -219,6 +277,7 @@ class ProjectIterationResultItem(ProjectIterationItem): f"/{VTAG}/projects/{{project_uuid}}/checkpoint/{{ref_id}}/iterations", name=f"{__name__}._list_meta_project_iterations_handler", ) +@login_required @permission_required("project.snapshot.read") async def _list_meta_project_iterations_handler(request: web.Request) -> web.Response: # TODO: check access to non owned projects user_id = request[RQT_USERID_KEY] @@ -250,22 +309,12 @@ async def _list_meta_project_iterations_handler(request: web.Request) -> web.Res # parse and validate response ---- page_items = [ - ProjectIterationItem( - name=f"projects/{meta_project_uuid}/checkpoint/{meta_project_commit_id}/iterations/{item.iteration_index}", - parent=ParentMetaProjectRef( - project_id=meta_project_uuid, ref_id=meta_project_commit_id - ), - iteration_index=item.iteration_index, - workcopy_project_id=item.project_id, - workcopy_project_url=url_for( - "get_project", - project_id=item.project_id, - ), - url=url_for( - f"{__name__}._list_meta_project_iterations_handler", - project_uuid=meta_project_uuid, - ref_id=meta_project_commit_id, - ), + ProjectIterationItem.create( + meta_project_uuid, + meta_project_commit_id, + item.iteration_index, + item.project_id, + url_for, ) for item in iterations_range.items ] @@ -311,22 +360,12 @@ async def _create_meta_project_iterations_handler(request: web.Request) -> web.R # parse and validate response ---- iterations_items = [ - ProjectIterationItem( - name=f"projects/{meta_project_uuid}/checkpoint/{meta_project_commit_id}/iterations/{item.iteration_index}", - parent=ParentMetaProjectRef( - project_id=meta_project_uuid, ref_id=meta_project_commit_id - ), - iteration_index=item.iteration_index, - workcopy_project_id=item.project_id, - workcopy_project_url=url_for( - "get_project", - project_id=item.project_id, - ), - url=url_for( - f"{__name__}._create_meta_project_iterations_handler", - project_uuid=meta_project_uuid, - ref_id=meta_project_commit_id, - ), + ProjectIterationItem.create( + meta_project_uuid, + meta_project_commit_id, + item.iteration_index, + item.project_id, + url_for, ) for item in project_iterations ] @@ -335,19 +374,24 @@ async def _create_meta_project_iterations_handler(request: web.Request) -> web.R # TODO: registry as route when implemented. Currently iteration is retrieved via GET /projects/{workcopy_project_id} -# @routes.get( -# f"/{VTAG}/projects/{{project_uuid}}/checkpoint/{{ref_id}}/iterations/{{iter_id}}", -# name=f"{__name__}._get_meta_project_iterations_handler", -# ) -# SEE https://github.com/ITISFoundation/osparc-simcore/issues/2735 +@routes.get( + f"/{VTAG}/projects/{{project_uuid}}/checkpoint/{{ref_id}}/iterations/{{iter_id}}", + name=f"{__name__}._get_meta_project_iterations_handler", +) +@login_required +@permission_required("project.snapshot.read") async def _get_meta_project_iterations_handler(request: web.Request) -> web.Response: - raise NotImplementedError + raise NotImplementedError( + "SEE https://github.com/ITISFoundation/osparc-simcore/issues/2735" + ) @routes.get( f"/{VTAG}/projects/{{project_uuid}}/checkpoint/{{ref_id}}/iterations/-/results", name=f"{__name__}._list_meta_project_iterations_results_handler", ) +@login_required +@permission_required("project.snapshot.read") async def _list_meta_project_iterations_results_handler( request: web.Request, ) -> web.Response: @@ -358,6 +402,7 @@ async def _list_meta_project_iterations_results_handler( url_for = create_url_for_function(request) vc_repo = VersionControlForMetaModeling(request) + assert vc_repo._user_id # nosec # core function ---- iterations_range = await _get_project_iterations_range( @@ -390,19 +435,13 @@ def _get_project_results(project_id) -> ExtractedResults: # parse and validate response ---- page_items = [ - ProjectIterationResultItem( - name=f"projects/{meta_project_uuid}/checkpoint/{meta_project_commit_id}/iterations/{item.iteration_index}/results", - parent=ParentMetaProjectRef( - project_id=meta_project_uuid, ref_id=meta_project_commit_id - ), - iteration_index=item.iteration_index, - workcopy_project_id=item.project_id, - results=_get_project_results(item.project_id), - url=url_for( - f"{__name__}._list_meta_project_iterations_results_handler", - project_uuid=meta_project_uuid, - ref_id=meta_project_commit_id, - ), + ProjectIterationResultItem.create( + meta_project_uuid, + meta_project_commit_id, + item.iteration_index, + item.project_id, + _get_project_results(item.project_id), + url_for, ) for item in iterations_range.items ] @@ -422,11 +461,15 @@ def _get_project_results(project_id) -> ExtractedResults: ) -# @routes.get( -# f"/{VTAG}/projects/{{project_uuid}}/checkpoint/{{ref_id}}/iterations/{{iter_id}}/results", -# name=f"{__name__}._get_meta_project_iterations_handler", -# ) +@routes.get( + f"/{VTAG}/projects/{{project_uuid}}/checkpoint/{{ref_id}}/iterations/{{iter_id}}/results", + name=f"{__name__}._get_meta_project_iteration_results_handler", +) +@login_required +@permission_required("project.snapshot.read") async def _get_meta_project_iteration_results_handler( request: web.Request, ) -> web.Response: - raise NotImplementedError + raise NotImplementedError( + "SEE https://github.com/ITISFoundation/osparc-simcore/issues/2735" + ) diff --git a/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_iterations.py b/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_iterations.py index 64f9f62818c..e48f9fe4ac5 100644 --- a/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_iterations.py +++ b/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_iterations.py @@ -20,7 +20,11 @@ from simcore_postgres_database.models.projects import projects from simcore_service_webserver._constants import APP_DB_ENGINE_KEY from simcore_service_webserver.director_v2_api import get_project_run_policy -from simcore_service_webserver.meta_modeling_handlers import Page, ProjectIterationItem +from simcore_service_webserver.meta_modeling_handlers import ( + Page, + ProjectIterationItem, + ProjectIterationResultItem, +) from simcore_service_webserver.meta_modeling_projects import ( meta_project_policy, projects_redirection_middleware, @@ -169,6 +173,16 @@ async def _mock_catalog_get(app, user_id, product_name, only_key_versions): # ---------------------------------------------- + # GET results of all iterations + # /projects/{project_uuid}/checkpoint/{ref_id}/iterations/-/results + resp = await client.get( + f"/v0/projects/{project_uuid}/checkpoint/{head_ref_id}/iterations/-/results" + ) + assert resp.status == HTTPStatus.OK, await resp.text() + body = await resp.json() + + results = Page[ProjectIterationResultItem].parse_obj(body).data + # GET project and MODIFY iterator values---------------------------------------------- # - Change iterations from 0:4 -> HEAD+1 resp = await client.get(f"/v0/projects/{project_uuid}") @@ -225,7 +239,7 @@ async def _mock_catalog_get(app, user_id, product_name, only_key_versions): f"/v0/projects/{project_uuid}/checkpoint/{head_ref_id}/iterations?offset=0" ) body = await resp.json() - assert resp.status == 200, f"{body=}" # nosec + assert resp.status == HTTPStatus.OK, f"{body=}" # nosec second_iterlist = Page[ProjectIterationItem].parse_obj(body).data assert len(second_iterlist) == 4 From 400a0ee6b10885b68b5ddcc3dc19a0757c46f5d4 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 1 Mar 2022 14:36:13 +0100 Subject: [PATCH 10/19] fixes tests --- .../webserver/openapi-meta-projects.yaml | 69 +++++++++++++++++++ api/specs/webserver/openapi.yaml | 6 ++ .../api/v0/openapi.yaml | 66 ++++++++++++++++++ .../meta_modeling_handlers.py | 3 +- 4 files changed, 142 insertions(+), 2 deletions(-) diff --git a/api/specs/webserver/openapi-meta-projects.yaml b/api/specs/webserver/openapi-meta-projects.yaml index 90b108468a7..ab3adcc5fe7 100644 --- a/api/specs/webserver/openapi-meta-projects.yaml +++ b/api/specs/webserver/openapi-meta-projects.yaml @@ -64,6 +64,40 @@ paths: schema: $ref: "#/components/schemas/HTTPValidationError" + /projects/{project_uuid}/checkpoint/{ref_id}/iterations/{iter_id}: + get: + tags: + - meta-projects + summary: Get Project Iterations + description: Get current project's iterations + operationId: "simcore_service_webserver.meta_modeling_handlers._get_meta_project_iterations_handler" + parameters: + - description: Project unique identifier + required: true + schema: + title: Project Uuid + type: string + description: Project unique identifier + format: uuid + name: project_uuid + in: path + - required: true + schema: + title: Ref Id + anyOf: + - type: integer + - type: string + name: ref_id + in: path + - required: true + name: iter_id + schema: + type: integer + in: path + responses: + "200": + description: Successful Response + /projects/{project_uuid}/checkpoint/{ref_id}/iterations/-/results: get: tags: @@ -129,6 +163,41 @@ paths: schema: $ref: "#/components/schemas/HTTPValidationError" + /projects/{project_uuid}/checkpoint/{ref_id}/iterations/{iter_id}/results: + get: + tags: + - meta-projects + summary: Get Project Iteration Results + description: Get current project's iterations + operationId: "simcore_service_webserver.meta_modeling_handlers._get_meta_project_iteration_results_handler" + parameters: + - description: Project unique identifier + required: true + schema: + title: Project Uuid + type: string + description: Project unique identifier + format: uuid + name: project_uuid + in: path + - required: true + schema: + title: Ref Id + anyOf: + - type: integer + - type: string + name: ref_id + in: path + - required: true + schema: + type: integer + name: iter_id + in: path + + responses: + "200": + description: Successful Response + components: schemas: HTTPValidationError: diff --git a/api/specs/webserver/openapi.yaml b/api/specs/webserver/openapi.yaml index 75870f98367..931a60feab0 100644 --- a/api/specs/webserver/openapi.yaml +++ b/api/specs/webserver/openapi.yaml @@ -223,9 +223,15 @@ paths: /projects/{project_uuid}/checkpoint/{ref_id}/iterations: $ref: "./openapi-meta-projects.yaml#/paths/~1projects~1{project_uuid}~1checkpoint~1{ref_id}~1iterations" + /projects/{project_uuid}/checkpoint/{ref_id}/iterations/{iter_id}: + $ref: "./openapi-meta-projects.yaml#/paths/~1projects~1{project_uuid}~1checkpoint~1{ref_id}~1iterations~1{iter_id}" + /projects/{project_uuid}/checkpoint/{ref_id}/iterations/-/results: $ref: "./openapi-meta-projects.yaml#/paths/~1projects~1{project_uuid}~1checkpoint~1{ref_id}~1iterations~1-~1results" + /projects/{project_uuid}/checkpoint/{ref_id}/iterations/{iter_id}/results: + $ref: "./openapi-meta-projects.yaml#/paths/~1projects~1{project_uuid}~1checkpoint~1{ref_id}~1iterations~1{iter_id}~1results" + # REPOSITORY ------------------------------------------------------------------------- /repos/projects: $ref: "./openapi-version-control.yaml#/paths/~1repos~1projects" diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 553a0bbee80..3c067716728 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -3098,6 +3098,39 @@ paths: type: title: Error Type type: string + '/projects/{project_uuid}/checkpoint/{ref_id}/iterations/{iter_id}': + get: + tags: + - meta-projects + summary: Get Project Iterations + description: Get current project's iterations + operationId: simcore_service_webserver.meta_modeling_handlers._get_meta_project_iterations_handler + parameters: + - description: Project unique identifier + required: true + schema: + title: Project Uuid + type: string + description: Project unique identifier + format: uuid + name: project_uuid + in: path + - required: true + schema: + title: Ref Id + anyOf: + - type: integer + - type: string + name: ref_id + in: path + - required: true + name: iter_id + schema: + type: integer + in: path + responses: + '200': + description: Successful Response '/projects/{project_uuid}/checkpoint/{ref_id}/iterations/-/results': get: tags: @@ -3175,6 +3208,39 @@ paths: application/json: schema: $ref: '#/paths/~1projects~1%7Bproject_uuid%7D~1checkpoint~1%7Bref_id%7D~1iterations/get/responses/422/content/application~1json/schema' + '/projects/{project_uuid}/checkpoint/{ref_id}/iterations/{iter_id}/results': + get: + tags: + - meta-projects + summary: Get Project Iteration Results + description: Get current project's iterations + operationId: simcore_service_webserver.meta_modeling_handlers._get_meta_project_iteration_results_handler + parameters: + - description: Project unique identifier + required: true + schema: + title: Project Uuid + type: string + description: Project unique identifier + format: uuid + name: project_uuid + in: path + - required: true + schema: + title: Ref Id + anyOf: + - type: integer + - type: string + name: ref_id + in: path + - required: true + schema: + type: integer + name: iter_id + in: path + responses: + '200': + description: Successful Response /repos/projects: get: tags: diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py b/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py index 9eec59d76bf..0a0399daa10 100644 --- a/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py +++ b/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py @@ -238,7 +238,7 @@ class ProjectIterationResultItem(ProjectIterationItem): results: ExtractedResults @classmethod - def create( + def create( # pylint: disable=arguments-differ cls, meta_project_uuid, meta_project_commit_id, @@ -402,7 +402,6 @@ async def _list_meta_project_iterations_results_handler( url_for = create_url_for_function(request) vc_repo = VersionControlForMetaModeling(request) - assert vc_repo._user_id # nosec # core function ---- iterations_range = await _get_project_iterations_range( From ca7c2246880127744303277a4b0137e037895c99 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 1 Mar 2022 17:49:53 +0100 Subject: [PATCH 11/19] enables dev/doc --- services/docker-compose.devel.yml | 1 + services/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/services/docker-compose.devel.yml b/services/docker-compose.devel.yml index eb3fbb5bb3b..e5b48293b70 100644 --- a/services/docker-compose.devel.yml +++ b/services/docker-compose.devel.yml @@ -69,6 +69,7 @@ services: - SC_BOOT_MODE=debug-ptvsd - WEBSERVER_RESOURCES_DELETION_TIMEOUT_SECONDS=15 - WEBSERVER_LOGLEVEL=${LOG_LEVEL:-DEBUG} + - REST_SWAGGER_API_DOC_ENABLED=1 dask-sidecar: volumes: diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 806f5fa6c43..8031f0db346 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -228,7 +228,7 @@ services: - traefik.http.services.${SWARM_STACK_NAME}_webserver.loadbalancer.sticky.cookie.secure=true - traefik.http.middlewares.${SWARM_STACK_NAME}_webserver_retry.retry.attempts=2 - traefik.http.routers.${SWARM_STACK_NAME}_webserver.service=${SWARM_STACK_NAME}_webserver - - traefik.http.routers.${SWARM_STACK_NAME}_webserver.rule=hostregexp(`{host:.+}`) && (Path(`/`, `/v0`,`/socket.io/`,`/static-frontend-data.json`, `/study/{study_uuid:\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b}`, `/view`, `/#/view`, `/#/error`) || PathPrefix(`/v0/`)) + - traefik.http.routers.${SWARM_STACK_NAME}_webserver.rule=hostregexp(`{host:.+}`) && (Path(`/`, `/v0`,`/socket.io/`,`/static-frontend-data.json`, `/study/{study_uuid:\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b}`, `/view`, `/#/view`, `/#/error`) || PathPrefix(`/v0/`) || PathPrefix(`/dev/`)) - traefik.http.routers.${SWARM_STACK_NAME}_webserver.entrypoints=http - traefik.http.routers.${SWARM_STACK_NAME}_webserver.priority=2 - traefik.http.routers.${SWARM_STACK_NAME}_webserver.middlewares=${SWARM_STACK_NAME}_gzip@docker, ${SWARM_STACK_NAME_NO_HYPHEN}_sslheader@docker, ${SWARM_STACK_NAME}_webserver_retry From afdb139708122f1eeefa0942547acddec5384c2d Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 2 Mar 2022 11:51:22 +0100 Subject: [PATCH 12/19] @GitHK review: adds model example and test --- .../meta_modeling_results.py | 26 +++++++++++++++++++ .../test_meta_modeling_results.py | 21 +++++++++++++-- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling_results.py b/services/web/server/src/simcore_service_webserver/meta_modeling_results.py index b26eedf8b6d..6a7ffd5698c 100644 --- a/services/web/server/src/simcore_service_webserver/meta_modeling_results.py +++ b/services/web/server/src/simcore_service_webserver/meta_modeling_results.py @@ -27,6 +27,32 @@ class ExtractedResults(BaseModel): ..., description="Captured outputs per node" ) + class Config: + schema_extra = { + "example": { + # sample with 2 computational services, 2 data sources (iterator+parameter) and 2 observers (probes) + "progress": { + "4c08265a-427b-4ac3-9eab-1d11c822ada4": 0, + "e33c6880-1b1d-4419-82d7-270197738aa9": 100, + }, + "labels": { + "0f1e38c9-dcb7-443c-a745-91b97ac28ccc": "Integer iterator", + "2d0ce8b9-c9c3-43ce-ad2f-ad493898de37": "Probe Sensor - Integer", + "445b44d1-59b3-425c-ac48-7c13e0f2ea5b": "Probe Sensor - Integer_2", + "d76fca06-f050-4790-88a8-0aac10c87b39": "Boolean Parameter", + }, + "values": { + "0f1e38c9-dcb7-443c-a745-91b97ac28ccc": { + "out_1": 1, + "out_2": [3, 4], + }, + "2d0ce8b9-c9c3-43ce-ad2f-ad493898de37": {"in_1": 7}, + "445b44d1-59b3-425c-ac48-7c13e0f2ea5b": {"in_1": 1}, + "d76fca06-f050-4790-88a8-0aac10c87b39": {"out_1": True}, + }, + } + } + def extract_project_results(workbench: Dict[str, Any]) -> ExtractedResults: # table : Name, Pogress, Labels.... , diff --git a/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_results.py b/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_results.py index ee39096bc66..f1e78358e24 100644 --- a/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_results.py +++ b/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_results.py @@ -4,10 +4,14 @@ import json -from typing import Any, Dict +from typing import Any, Dict, Type import pytest -from simcore_service_webserver.meta_modeling_results import extract_project_results +from pydantic import BaseModel +from simcore_service_webserver.meta_modeling_results import ( + ExtractedResults, + extract_project_results, +) @pytest.fixture @@ -119,3 +123,16 @@ def test_extract_project_results(fake_workbench: Dict[str, Any]): "445b44d1-59b3-425c-ac48-7c13e0f2ea5b": {"in_1": 1}, "d76fca06-f050-4790-88a8-0aac10c87b39": {"out_1": True}, } + + +@pytest.mark.parametrize( + "model_cls", + (ExtractedResults,), +) +def test_models_examples( + model_cls: Type[BaseModel], model_cls_examples: Dict[str, Any] +): + for name, example in model_cls_examples.items(): + print(name, ":", json.dumps(example, indent=1)) + model_instance = model_cls(**example) + assert model_instance, f"Failed with {name}" From cc32d511eaf0e9b640152addb6ee7db0a18d84ff Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 2 Mar 2022 11:59:03 +0100 Subject: [PATCH 13/19] GitHK review: doc --- .../meta_modeling_results.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling_results.py b/services/web/server/src/simcore_service_webserver/meta_modeling_results.py index 6a7ffd5698c..4191686fa8b 100644 --- a/services/web/server/src/simcore_service_webserver/meta_modeling_results.py +++ b/services/web/server/src/simcore_service_webserver/meta_modeling_results.py @@ -55,11 +55,19 @@ class Config: def extract_project_results(workbench: Dict[str, Any]) -> ExtractedResults: - # table : Name, Pogress, Labels.... , - # all projects are guaranted to be topologically identical (i.e. same node uuids ) + """Extracting results from a project's workbench section (i.e. pipeline). Specifically: + - data sources (e.g. outputs from iterators, paramters) + - progress of evaluators (e.g. a computational service) + - data observers (basically inputs from probes) + + NOTE: all projects produces from iterations preserve the same node uuids so + running this extraction on all projects from a iterations allows to create a + row for a table of results + """ + # nodeid -> % progress progress = {} - # nodeid -> label # NOTE labels are not uniqu + # nodeid -> label (this map is necessary because cannot guaratee labels to be unique) labels = {} # nodeid -> { port: value , ...} # results have two levels deep: node/port results = {} From 50add356d8c5ea6681f35f7e292b48e3266f4570 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 2 Mar 2022 13:19:51 +0100 Subject: [PATCH 14/19] @GitHK review: rm x-marks until completed. Left as TODO comments --- .../src/simcore_service_webserver/meta_modeling_handlers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py b/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py index 0a0399daa10..fcd93e1ecd3 100644 --- a/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py +++ b/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py @@ -180,7 +180,7 @@ class _BaseModelGet(BaseModel): name: str = Field( ..., description="Iteration's resource API name", - x_mark_resouce_name=True, # [AIP-122](https://google.aip.dev/122) + # TODO: PC x_mark_resouce_name=True, # [AIP-122](https://google.aip.dev/122) ) parent: ParentMetaProjectRef = Field( ..., description="Reference to the the meta-project that created this iteration" @@ -192,7 +192,7 @@ class _BaseModelGet(BaseModel): class ProjectIterationItem(_BaseModelGet): iteration_index: IterationID = Field( ..., - x_mark_resource_id_segment=True, # [AIP-122](https://google.aip.dev/122) + # TODO: PC x_mark_resource_id_segment=True, # [AIP-122](https://google.aip.dev/122) ) workcopy_project_id: ProjectID = Field( From bafbec2b35ea99c86924f266d7b40317d6141967 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 2 Mar 2022 19:38:09 +0100 Subject: [PATCH 15/19] adds address for public api --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ca530b619b2..8164594481f 100644 --- a/Makefile +++ b/Makefile @@ -232,7 +232,8 @@ TableWidth=140;\ printf "%24s | %90s | %12s | %12s\n" Name Endpoint User Password;\ printf "%.$${TableWidth}s\n" "$$separator";\ printf "$$rows" 'oSparc platform' 'http://$(get_my_ip).nip.io:9081';\ -printf "$$rows" 'oSparc platform web API' 'http://$(get_my_ip).nip.io:9081/dev/doc';\ +printf "$$rows" 'oSparc web API doc' 'http://$(get_my_ip).nip.io:9081/dev/doc';\ +printf "$$rows" 'oSparc public API doc' 'http://$(get_my_ip).nip.io:8006/dev/doc';\ printf "$$rows" 'Postgres DB' 'http://$(get_my_ip).nip.io:18080/?pgsql=postgres&username='$${POSTGRES_USER}'&db='$${POSTGRES_DB}'&ns=public' $${POSTGRES_USER} $${POSTGRES_PASSWORD};\ printf "$$rows" Portainer 'http://$(get_my_ip).nip.io:9000' admin adminadmin;\ printf "$$rows" Redis 'http://$(get_my_ip).nip.io:18081';\ From abfd5fff8f62743630d89299c5d21dbacdf3d6fe Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 2 Mar 2022 19:39:27 +0100 Subject: [PATCH 16/19] @Surfict review: traefik label --- services/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 8031f0db346..d6c741d64b0 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -228,7 +228,7 @@ services: - traefik.http.services.${SWARM_STACK_NAME}_webserver.loadbalancer.sticky.cookie.secure=true - traefik.http.middlewares.${SWARM_STACK_NAME}_webserver_retry.retry.attempts=2 - traefik.http.routers.${SWARM_STACK_NAME}_webserver.service=${SWARM_STACK_NAME}_webserver - - traefik.http.routers.${SWARM_STACK_NAME}_webserver.rule=hostregexp(`{host:.+}`) && (Path(`/`, `/v0`,`/socket.io/`,`/static-frontend-data.json`, `/study/{study_uuid:\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b}`, `/view`, `/#/view`, `/#/error`) || PathPrefix(`/v0/`) || PathPrefix(`/dev/`)) + - traefik.http.routers.${SWARM_STACK_NAME}_webserver.rule=hostregexp(`{host:.+}`) && (Path(`/`, `/v0`,`/socket.io/`,`/static-frontend-data.json`, `/study/{study_uuid:\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b}`, `/view`, `/#/view`, `/#/error`) || PathPrefix(`/v0/`, `/dev/`)) - traefik.http.routers.${SWARM_STACK_NAME}_webserver.entrypoints=http - traefik.http.routers.${SWARM_STACK_NAME}_webserver.priority=2 - traefik.http.routers.${SWARM_STACK_NAME}_webserver.middlewares=${SWARM_STACK_NAME}_gzip@docker, ${SWARM_STACK_NAME_NO_HYPHEN}_sslheader@docker, ${SWARM_STACK_NAME}_webserver_retry From 6ad7c677ad7c6202cfb36e96c3a5f46ad6d7d000 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 3 Mar 2022 11:07:36 +0100 Subject: [PATCH 17/19] adds in local --- services/docker-compose.local.yml | 8 ++++++-- services/docker-compose.yml | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/services/docker-compose.local.yml b/services/docker-compose.local.yml index 045061e4d0f..c6a9f2910a5 100644 --- a/services/docker-compose.local.yml +++ b/services/docker-compose.local.yml @@ -64,6 +64,11 @@ services: deploy: labels: - traefik.http.services.${SWARM_STACK_NAME}_webserver.loadbalancer.sticky.cookie.secure=false + - traefik.http.routers.${SWARM_STACK_NAME}_webserver_local.service=${SWARM_STACK_NAME}_webserver + - traefik.http.routers.${SWARM_STACK_NAME}_webserver_local.entrypoints=http + - traefik.http.routers.${SWARM_STACK_NAME}_webserver_local.rule= hostregexp(`{host:.+}`) && PathPrefix(`/dev/`) + - traefik.http.routers.${SWARM_STACK_NAME}_webserver_local.priority=3 + - traefik.http.routers.${SWARM_STACK_NAME}_webserver_local.middlewares=${SWARM_STACK_NAME}_gzip@docker, ${SWARM_STACK_NAME_NO_HYPHEN}_sslheader@docker, ${SWARM_STACK_NAME}_webserver_retry dask-sidecar: environment: @@ -134,8 +139,7 @@ services: - io.simcore.zone=${TRAEFIK_SIMCORE_ZONE} - traefik.enable=true - traefik.http.routers.${SWARM_STACK_NAME}_api_internal.service=api@internal - - traefik.http.routers.${SWARM_STACK_NAME}_api_internal.rule=PathPrefix(`/dashboard`) - || PathPrefix(`/api`) + - traefik.http.routers.${SWARM_STACK_NAME}_api_internal.rule=PathPrefix(`/dashboard`) || PathPrefix(`/api`) - traefik.http.routers.${SWARM_STACK_NAME}_api_internal.entrypoints=traefik_monitor - traefik.http.routers.${SWARM_STACK_NAME}_api_internal.middlewares=${SWARM_STACK_NAME}_gzip@docker - traefik.http.services.${SWARM_STACK_NAME}_api_internal.loadbalancer.server.port=8080 diff --git a/services/docker-compose.yml b/services/docker-compose.yml index d6c741d64b0..806f5fa6c43 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -228,7 +228,7 @@ services: - traefik.http.services.${SWARM_STACK_NAME}_webserver.loadbalancer.sticky.cookie.secure=true - traefik.http.middlewares.${SWARM_STACK_NAME}_webserver_retry.retry.attempts=2 - traefik.http.routers.${SWARM_STACK_NAME}_webserver.service=${SWARM_STACK_NAME}_webserver - - traefik.http.routers.${SWARM_STACK_NAME}_webserver.rule=hostregexp(`{host:.+}`) && (Path(`/`, `/v0`,`/socket.io/`,`/static-frontend-data.json`, `/study/{study_uuid:\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b}`, `/view`, `/#/view`, `/#/error`) || PathPrefix(`/v0/`, `/dev/`)) + - traefik.http.routers.${SWARM_STACK_NAME}_webserver.rule=hostregexp(`{host:.+}`) && (Path(`/`, `/v0`,`/socket.io/`,`/static-frontend-data.json`, `/study/{study_uuid:\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b}`, `/view`, `/#/view`, `/#/error`) || PathPrefix(`/v0/`)) - traefik.http.routers.${SWARM_STACK_NAME}_webserver.entrypoints=http - traefik.http.routers.${SWARM_STACK_NAME}_webserver.priority=2 - traefik.http.routers.${SWARM_STACK_NAME}_webserver.middlewares=${SWARM_STACK_NAME}_gzip@docker, ${SWARM_STACK_NAME_NO_HYPHEN}_sslheader@docker, ${SWARM_STACK_NAME}_webserver_retry From 71c879822435e7b0f3645e0362cd4a4dd6475a71 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 3 Mar 2022 14:44:47 +0100 Subject: [PATCH 18/19] fixes makefile missing quotes --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 8164594481f..449a0d291c6 100644 --- a/Makefile +++ b/Makefile @@ -237,10 +237,10 @@ printf "$$rows" 'oSparc public API doc' 'http://$(get_my_ip).nip.io:8006/dev/doc printf "$$rows" 'Postgres DB' 'http://$(get_my_ip).nip.io:18080/?pgsql=postgres&username='$${POSTGRES_USER}'&db='$${POSTGRES_DB}'&ns=public' $${POSTGRES_USER} $${POSTGRES_PASSWORD};\ printf "$$rows" Portainer 'http://$(get_my_ip).nip.io:9000' admin adminadmin;\ printf "$$rows" Redis 'http://$(get_my_ip).nip.io:18081';\ -printf "$$rows" 'Docker Registry' $${REGISTRY_URL} $${REGISTRY_USER} $${REGISTRY_PW};\ -printf "$$rows" "Dask Dashboard" "http://$(get_my_ip).nip.io:8787"; -printf "$$rows" "Traefik Dashboard" "http://$(get_my_ip).nip.io:8080/dashboard/"; -printf "$$rows" "Rabbit Dashboard" "http://$(get_my_ip).nip.io:15762" admin adminadmin; +printf "$$rows" "Dask Dashboard" "http://$(get_my_ip).nip.io:8787";\ +printf "$$rows" "Docker Registry" "$${REGISTRY_URL}" $${REGISTRY_USER} $${REGISTRY_PW};\ +printf "$$rows" "Rabbit Dashboard" "http://$(get_my_ip).nip.io:15762" admin adminadmin;\ +printf "$$rows" "Traefik Dashboard" "http://$(get_my_ip).nip.io:8080/dashboard/";\ printf "\n%s\n" "⚠️ if a DNS is not used (as displayed above), the interactive services started via dynamic-sidecar";\ echo "⚠️ will not be shown. The frontend accesses them via the uuid.services.YOUR_IP.nip.io:9081"; endef From 319acd2f0986c1a489890da2fb27fd4f078974b7 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 7 Mar 2022 13:11:54 +0100 Subject: [PATCH 19/19] missing variable --- services/docker-compose.devel.yml | 1 - services/docker-compose.local.yml | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/docker-compose.devel.yml b/services/docker-compose.devel.yml index e5b48293b70..eb3fbb5bb3b 100644 --- a/services/docker-compose.devel.yml +++ b/services/docker-compose.devel.yml @@ -69,7 +69,6 @@ services: - SC_BOOT_MODE=debug-ptvsd - WEBSERVER_RESOURCES_DELETION_TIMEOUT_SECONDS=15 - WEBSERVER_LOGLEVEL=${LOG_LEVEL:-DEBUG} - - REST_SWAGGER_API_DOC_ENABLED=1 dask-sidecar: volumes: diff --git a/services/docker-compose.local.yml b/services/docker-compose.local.yml index c6a9f2910a5..dca011007f9 100644 --- a/services/docker-compose.local.yml +++ b/services/docker-compose.local.yml @@ -58,6 +58,7 @@ services: webserver: environment: - SC_BOOT_MODE=${SC_BOOT_MODE:-default} + - REST_SWAGGER_API_DOC_ENABLED=1 ports: - "8080" - "3001:3000" @@ -66,7 +67,7 @@ services: - traefik.http.services.${SWARM_STACK_NAME}_webserver.loadbalancer.sticky.cookie.secure=false - traefik.http.routers.${SWARM_STACK_NAME}_webserver_local.service=${SWARM_STACK_NAME}_webserver - traefik.http.routers.${SWARM_STACK_NAME}_webserver_local.entrypoints=http - - traefik.http.routers.${SWARM_STACK_NAME}_webserver_local.rule= hostregexp(`{host:.+}`) && PathPrefix(`/dev/`) + - traefik.http.routers.${SWARM_STACK_NAME}_webserver_local.rule=hostregexp(`{host:.+}`) && PathPrefix(`/dev/`) - traefik.http.routers.${SWARM_STACK_NAME}_webserver_local.priority=3 - traefik.http.routers.${SWARM_STACK_NAME}_webserver_local.middlewares=${SWARM_STACK_NAME}_gzip@docker, ${SWARM_STACK_NAME_NO_HYPHEN}_sslheader@docker, ${SWARM_STACK_NAME}_webserver_retry