From 3d68a7a18fb18d088f77a8ead274af4cb69b8669 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Jun 2023 17:18:02 +0200 Subject: [PATCH 01/53] projects metadata table --- .../models/projects_metadata.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py new file mode 100644 index 00000000000..40fbc5369fe --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py @@ -0,0 +1,53 @@ +""" + These tables were designed to be controled by projects-plugin in + the webserver's service +""" + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import JSONB + +from ._common import ( + column_created_datetime, + column_modified_datetime, + register_modified_datetime_auto_update_trigger, +) +from .base import metadata +from .projects import projects + +projects_metadata = sa.Table( + "projects_metadata", + # + # Holds runtime metadata on a project. + # + # Things like 'stars', 'quality', 'classifiers' etc (or any kind of stats) + # should be moved here. + # + metadata, + sa.Column( + "project_uuid", + sa.String, + sa.ForeignKey( + projects.c.uuid, + onupdate="CASCADE", + ondelete="CASCADE", + name="fk_projects_metadata_project_uuid", + ), + nullable=False, + primary_key=True, + doc="The project unique identifier is also used to identify the associated job", + ), + sa.Column( + "user_metadata", + JSONB, + nullable=False, + server_default=sa.text("'{}'::jsonb"), + doc="Free json for user to store her metadata", + ), + # TIME STAMPS ---- + column_created_datetime(timezone=True), + column_modified_datetime(timezone=True), + sa.PrimaryKeyConstraint("project_uuid"), +) + + +register_modified_datetime_auto_update_trigger(projects_metadata) From 0c99a7732fcf7201aeff60afb75da8c3343f5547 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Jun 2023 17:18:16 +0200 Subject: [PATCH 02/53] projects to jobs table --- .../models/projects_to_jobs.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py new file mode 100644 index 00000000000..23781e2e520 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py @@ -0,0 +1,52 @@ +""" + These tables were designed to be controled by projects-plugin in + the webserver's service +""" + +import sqlalchemy as sa + +from ._common import ( + column_created_datetime, + column_modified_datetime, + register_modified_datetime_auto_update_trigger, +) +from .base import metadata +from .projects import projects + +projects_to_jobs = sa.Table( + "projects_to_jobs", + # + # Every job is mapped to a project and has an ancestor (see job_parent_name) + # but not every project is associated to a job. + # + # This table holds all projects associated to jobs + # + metadata, + sa.Column( + "project_uuid", + sa.String, + sa.ForeignKey( + projects.c.uuid, + onupdate="CASCADE", + ondelete="CASCADE", + name="fk_projects_to_jobs_project_uuid", + ), + nullable=False, + primary_key=True, + doc="The project unique identifier is also used to identify the associated job", + ), + sa.Column( + "job_parent_name", + sa.String, + nullable=False, + doc="Project's ancestor when create as a job. A project can be created as a" + " - solver job: solver name (e.g. /v0/solvers/{id}/releases/{version})" + " - study job: study name (e.g. /v0/studies/{id})", + ), + # TIME STAMPS ---- + column_created_datetime(timezone=True), + column_modified_datetime(timezone=True), + sa.PrimaryKeyConstraint("project_uuid"), +) + +register_modified_datetime_auto_update_trigger(projects_to_jobs) From a44fed12572372072132432970698cb06789733a Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Jun 2023 20:24:12 +0200 Subject: [PATCH 03/53] WIP --- .../tests/test_projects_metadata.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 packages/postgres-database/tests/test_projects_metadata.py diff --git a/packages/postgres-database/tests/test_projects_metadata.py b/packages/postgres-database/tests/test_projects_metadata.py new file mode 100644 index 00000000000..8ddbfffbac1 --- /dev/null +++ b/packages/postgres-database/tests/test_projects_metadata.py @@ -0,0 +1,40 @@ +# test create a job +# - from a study +# - from a solver +# - search jobs from a study, from a solver, etc +# - list studies -> projects uuids that are not jobs +# - list study jobs -> projects uuids that are Jodbs + +from simcore_postgres_database.models.projects import projects +from simcore_postgres_database.models.projects_metadata import projects_metadata +from simcore_postgres_database.models.projects_to_jobs import projects_to_jobs + + +def test_paginate_solver_jobs(): + # filter + assert projects_to_jobs + + +def test_create_job(): + assert projects + + # list jobs of a study + # list all jobs of a user + # list projects that are non-jobs + # + + +def test_create_job_metadata(): + assert projects_metadata + + +def test_read_job_metadata(): + ... + + +def test_update_job_metadata(): + ... + + +def test_delete_job_metadata(): + ... From d59b57ab70ad35e3e059ddf0d94d0ff2dedb30b2 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 27 Jun 2023 12:20:38 +0200 Subject: [PATCH 04/53] adds jobs_metadata to pg tables and renames table to metadata_jobs --- ...s_to_jobs.py => projects_metadata_jobs.py} | 26 ++++++++++++------- .../tests/test_projects_metadata.py | 2 +- 2 files changed, 17 insertions(+), 11 deletions(-) rename packages/postgres-database/src/simcore_postgres_database/models/{projects_to_jobs.py => projects_metadata_jobs.py} (66%) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata_jobs.py similarity index 66% rename from packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py rename to packages/postgres-database/src/simcore_postgres_database/models/projects_metadata_jobs.py index 23781e2e520..9d4b7904d8c 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata_jobs.py @@ -1,9 +1,5 @@ -""" - These tables were designed to be controled by projects-plugin in - the webserver's service -""" - import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import JSONB from ._common import ( column_created_datetime, @@ -13,13 +9,15 @@ from .base import metadata from .projects import projects -projects_to_jobs = sa.Table( - "projects_to_jobs", +projects_metadata_jobs = sa.Table( + "projects_metadata_jobs", # # Every job is mapped to a project and has an ancestor (see job_parent_name) # but not every project is associated to a job. # - # This table holds all projects associated to jobs + # This table + # - holds all projects associated to jobs + # - stores jobs ancestry relations and metadata # metadata, sa.Column( @@ -29,7 +27,7 @@ projects.c.uuid, onupdate="CASCADE", ondelete="CASCADE", - name="fk_projects_to_jobs_project_uuid", + name="fk_projects_metadata_jobs_project_uuid", ), nullable=False, primary_key=True, @@ -43,10 +41,18 @@ " - solver job: solver name (e.g. /v0/solvers/{id}/releases/{version})" " - study job: study name (e.g. /v0/studies/{id})", ), + sa.Column( + "job_metadata", + JSONB, + nullable=True, + server_default=sa.text("'{}'::jsonb"), + doc="Job can store here metadata. " + "Preserves class information during serialization/deserialization", + ), # TIME STAMPS ---- column_created_datetime(timezone=True), column_modified_datetime(timezone=True), sa.PrimaryKeyConstraint("project_uuid"), ) -register_modified_datetime_auto_update_trigger(projects_to_jobs) +register_modified_datetime_auto_update_trigger(projects_metadata_jobs) diff --git a/packages/postgres-database/tests/test_projects_metadata.py b/packages/postgres-database/tests/test_projects_metadata.py index 8ddbfffbac1..bd6858ae873 100644 --- a/packages/postgres-database/tests/test_projects_metadata.py +++ b/packages/postgres-database/tests/test_projects_metadata.py @@ -7,7 +7,7 @@ from simcore_postgres_database.models.projects import projects from simcore_postgres_database.models.projects_metadata import projects_metadata -from simcore_postgres_database.models.projects_to_jobs import projects_to_jobs +from simcore_postgres_database.models.projects_metadata_jobs import projects_to_jobs def test_paginate_solver_jobs(): From f064d2952ad735eab5feed1eef31fdffc613a728 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 27 Jun 2023 19:08:44 +0200 Subject: [PATCH 05/53] all tables together --- .../models/projects_metadata.py | 49 ++++++++++++++++ .../models/projects_metadata_jobs.py | 58 ------------------- 2 files changed, 49 insertions(+), 58 deletions(-) delete mode 100644 packages/postgres-database/src/simcore_postgres_database/models/projects_metadata_jobs.py diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py index 40fbc5369fe..f70909a468b 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py @@ -51,3 +51,52 @@ register_modified_datetime_auto_update_trigger(projects_metadata) + + +projects_jobs_metadata = sa.Table( + "projects_jobs_metadata", + # + # Every job is mapped to a project and has an ancestor (see job_parent_name) + # but not every project is associated to a job. + # + # This table + # - holds all projects associated to jobs + # - stores jobs ancestry relations and metadata + # + metadata, + sa.Column( + "project_uuid", + sa.String, + sa.ForeignKey( + projects.c.uuid, + onupdate="CASCADE", + ondelete="CASCADE", + name="fk_projects_jobs_metadata_project_uuid", + ), + nullable=False, + primary_key=True, + doc="The project unique identifier is also used to identify the associated job", + ), + sa.Column( + "parent_name", + sa.String, + nullable=False, + doc="Project's ancestor when create as a job. A project can be created as a" + " - solver job: solver name (e.g. /v0/solvers/{id}/releases/{version})" + " - study job: study name (e.g. /v0/studies/{id})", + ), + sa.Column( + "job_metadata", + JSONB, + nullable=True, + server_default=sa.text("'{}'::jsonb"), + doc="Job can store here metadata. " + "Preserves class information during serialization/deserialization", + ), + # TIME STAMPS ---- + column_created_datetime(timezone=True), + column_modified_datetime(timezone=True), + sa.PrimaryKeyConstraint("project_uuid"), +) + +register_modified_datetime_auto_update_trigger(projects_jobs_metadata) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata_jobs.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata_jobs.py deleted file mode 100644 index 9d4b7904d8c..00000000000 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata_jobs.py +++ /dev/null @@ -1,58 +0,0 @@ -import sqlalchemy as sa -from sqlalchemy.dialects.postgresql import JSONB - -from ._common import ( - column_created_datetime, - column_modified_datetime, - register_modified_datetime_auto_update_trigger, -) -from .base import metadata -from .projects import projects - -projects_metadata_jobs = sa.Table( - "projects_metadata_jobs", - # - # Every job is mapped to a project and has an ancestor (see job_parent_name) - # but not every project is associated to a job. - # - # This table - # - holds all projects associated to jobs - # - stores jobs ancestry relations and metadata - # - metadata, - sa.Column( - "project_uuid", - sa.String, - sa.ForeignKey( - projects.c.uuid, - onupdate="CASCADE", - ondelete="CASCADE", - name="fk_projects_metadata_jobs_project_uuid", - ), - nullable=False, - primary_key=True, - doc="The project unique identifier is also used to identify the associated job", - ), - sa.Column( - "job_parent_name", - sa.String, - nullable=False, - doc="Project's ancestor when create as a job. A project can be created as a" - " - solver job: solver name (e.g. /v0/solvers/{id}/releases/{version})" - " - study job: study name (e.g. /v0/studies/{id})", - ), - sa.Column( - "job_metadata", - JSONB, - nullable=True, - server_default=sa.text("'{}'::jsonb"), - doc="Job can store here metadata. " - "Preserves class information during serialization/deserialization", - ), - # TIME STAMPS ---- - column_created_datetime(timezone=True), - column_modified_datetime(timezone=True), - sa.PrimaryKeyConstraint("project_uuid"), -) - -register_modified_datetime_auto_update_trigger(projects_metadata_jobs) From 0dec77276fdcc4f4d9505f3732f735a5e0141b4a Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 27 Jun 2023 19:08:59 +0200 Subject: [PATCH 06/53] doc --- .../api/routes/solvers_jobs.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py index bbbef8f0610..81519f25738 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py @@ -83,6 +83,26 @@ async def list_jobs( SEE get_jobs_page for paginated version of this function """ + # ```mermaid + # sequenceDiagram + # participant API + # participant AS + # participant CS + # participant WS + # link API: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/api-server/openapi.json + # link CS: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/catalog/openapi.json + # link WS: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml + # + # API->>+AS: list_job + # AS->>+CS: get_service + # CS-->>-AS: ServiceGet + # AS->>+WS: list_projects(page*) + # WS-->>-AS: list[ProjectGet] + # AS-->>-API: list[Job] | Page[Job]* + # ``` + # SEE https://mermaid.live/ + # * = still not implemented + solver = await catalog_client.get_service( user_id=user_id, name=solver_key, @@ -171,6 +191,31 @@ async def create_job( NOTE: This operation does **not** start the job """ + # ```mermaid + # sequenceDiagram + # participant API + # participant AS + # participant CS + # participant WS + # link API: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/api-server/openapi.json + # link CS: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/catalog/openapi.json + # link WS: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml + # + # API->>+AS: create_job + # AS->>+CS: get_service + # CS-->>-AS: ServiceGet + # AS->>+CS: get_service_ports* + # CS-->>-AS: ServicePortGet* + # Note right of AS: Validate SolverInputs* + # AS->>+WS: create_project + # WS-->>-AS: ProjectGet + # AS-->>-API: Job + # ``` + # SEE https://mermaid.live/ + # * = still not implemented + + # -> catalog + # TODO: validate inputs against solver input schema # ensures user has access to solver solver = await catalog_client.get_service( user_id=user_id, From 56b0b132cbdfadc8f19020200bd3ef1e9449155b Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 27 Jun 2023 19:27:30 +0200 Subject: [PATCH 07/53] doc --- .../api/routes/solvers_jobs.py | 66 ++++++++++--------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py index 81519f25738..23dbe9784be 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py @@ -84,21 +84,22 @@ async def list_jobs( """ # ```mermaid - # sequenceDiagram - # participant API - # participant AS - # participant CS - # participant WS - # link API: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/api-server/openapi.json - # link CS: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/catalog/openapi.json - # link WS: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml + # sequenceDiagram + # participant API + # participant AS as api-server + # participant CS as catalog + # participant WS as web-server # - # API->>+AS: list_job - # AS->>+CS: get_service - # CS-->>-AS: ServiceGet - # AS->>+WS: list_projects(page*) - # WS-->>-AS: list[ProjectGet] - # AS-->>-API: list[Job] | Page[Job]* + # link API: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/api-server/openapi.json + # link CS: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/catalog/openapi.json + # link WS: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/web/server/src/simcore_service_webserver/api/v0/openapi + # + # API->>+AS: list_job + # AS->>+CS: get_service + # CS-->>-AS: ServiceGet + # AS->>+WS: list_projects(page*) + # WS-->>-AS: list[ProjectGet] + # AS-->>-API: list[Job] | Page[Job]* # ``` # SEE https://mermaid.live/ # * = still not implemented @@ -192,24 +193,27 @@ async def create_job( """ # ```mermaid - # sequenceDiagram - # participant API - # participant AS - # participant CS - # participant WS - # link API: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/api-server/openapi.json - # link CS: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/catalog/openapi.json - # link WS: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml + # sequenceDiagram + # participant API + # participant AS as api-server + # participant CS as catalog + # participant WS as web-server + # + # link API: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/api-server/openapi.json + # link CS: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/catalog/openapi.json + # link WS: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/web/server/src/simcore_service_webserver/api/v0/openapi # - # API->>+AS: create_job - # AS->>+CS: get_service - # CS-->>-AS: ServiceGet - # AS->>+CS: get_service_ports* - # CS-->>-AS: ServicePortGet* - # Note right of AS: Validate SolverInputs* - # AS->>+WS: create_project - # WS-->>-AS: ProjectGet - # AS-->>-API: Job + # API->>+AS: create_job + # AS->>+CS: get_service + # CS-->>-AS: ServiceGet + # AS->>+CS: get_service_ports* + # CS-->>-AS: ServicePortGet* + # Note right of AS: Validate SolverInputs* + # AS->>+WS: create_project + # WS-->>-AS: ProjectGet + # AS->>+WS: set_job_metadata* + # WS-->>-AS: ProjectJobMetaGet* + # AS-->>-API: Job # ``` # SEE https://mermaid.live/ # * = still not implemented From 0c66bff28c3d844587121557038e5fa134af5846 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 27 Jun 2023 20:08:01 +0200 Subject: [PATCH 08/53] drafts tests --- .../models/projects_metadata.py | 6 +- .../tests/test_projects_metadata.py | 104 +++++++++++++++--- 2 files changed, 91 insertions(+), 19 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py index f70909a468b..97b18d149a7 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py @@ -17,7 +17,7 @@ projects_metadata = sa.Table( "projects_metadata", # - # Holds runtime metadata on a project. + # Holds **runtime** metadata on a project # # Things like 'stars', 'quality', 'classifiers' etc (or any kind of stats) # should be moved here. @@ -37,7 +37,7 @@ doc="The project unique identifier is also used to identify the associated job", ), sa.Column( - "user_metadata", + "custom_metadata", JSONB, nullable=False, server_default=sa.text("'{}'::jsonb"), @@ -59,7 +59,7 @@ # Every job is mapped to a project and has an ancestor (see job_parent_name) # but not every project is associated to a job. # - # This table + # This table contains specific metadata on projects/jobs # - holds all projects associated to jobs # - stores jobs ancestry relations and metadata # diff --git a/packages/postgres-database/tests/test_projects_metadata.py b/packages/postgres-database/tests/test_projects_metadata.py index bd6858ae873..289678d2939 100644 --- a/packages/postgres-database/tests/test_projects_metadata.py +++ b/packages/postgres-database/tests/test_projects_metadata.py @@ -1,31 +1,103 @@ -# test create a job -# - from a study -# - from a solver -# - search jobs from a study, from a solver, etc -# - list studies -> projects uuids that are not jobs -# - list study jobs -> projects uuids that are Jodbs +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments + +from typing import Any, Awaitable, Callable +from uuid import UUID +import sqlalchemy as sa +from aiopg.sa.connection import SAConnection +from aiopg.sa.result import ResultProxy, RowProxy from simcore_postgres_database.models.projects import projects -from simcore_postgres_database.models.projects_metadata import projects_metadata -from simcore_postgres_database.models.projects_metadata_jobs import projects_to_jobs +from simcore_postgres_database.models.projects_metadata import ( + projects_jobs_metadata, + projects_metadata, +) +from sqlalchemy.dialects.postgresql import insert as pg_insert -def test_paginate_solver_jobs(): - # filter - assert projects_to_jobs +async def test_create_job( + connection: SAConnection, + create_fake_project: Callable[..., Awaitable[RowProxy]], + create_fake_user: Callable[..., Awaitable[RowProxy]], +): + user: RowProxy = await create_fake_user(connection) + + async def _create_solver_job(service_key: str, service_version: str, n: int): + parent_name = f"/v0/solvers/{service_key}/releases/{service_version}" + project: RowProxy = await create_fake_project(connection, user, hidden=True) + + query = projects_jobs_metadata.insert().values( + project_uuid=project.uuid, + parent_name=parent_name, + jobs_metadata={ + "__type__": "JobMeta", + "inputs_checksum": f"{n}2bfd4885aa1daf5c16fdd39b9118f652c4977c4021c900794dc125cf123718e", + "created_at": f"2022-06-01T15:{n}:56.807441", + }, + ) + result: ResultProxy = await connection.execute(query) + assert result + return UUID(project.uuid) + + # some project from the UI + project_study = await create_fake_project(connection, user, hidden=True) + + # some solver-job projects + created_jobs: list[UUID] = [ + await _create_solver_job( + service_key="simcore/comp/itis/sleeper", service_version="2.0.0", n=n + ) + for n in range(3) + ] + + assert project_study.uuid not in set(created_jobs) + # list jobs of a solver + j = projects.join( + projects_jobs_metadata, + (projects.c.uuid == projects_jobs_metadata.c.project_uuid), + ) + query = sa.select(projects_jobs_metadata, projects.c.hidden).select_from(j) + got_jobs = await (await connection.execute(query)).fetchall() + assert got_jobs -def test_create_job(): - assert projects + assert {j.project_uuid for j in got_jobs} == set(created_jobs) + assert all(j.hidden for j in got_jobs) # list jobs of a study # list all jobs of a user # list projects that are non-jobs - # + async def _upsert_custom_metadata(project_uuid, metadata: dict[str, Any]): + params = dict( + project_uuid=f"{project_uuid}", + custom_metadata=metadata, + ) + insert_stmt = pg_insert(projects_metadata).values(**params) + on_update_stmt = insert_stmt.on_conflict_do_update( + index_elements=[ + projects_metadata.c.project_uuid, + ], + set_=params, + ) + await connection.execute(on_update_stmt) -def test_create_job_metadata(): - assert projects_metadata + await _upsert_custom_metadata(project_study.uuid, metadata={"my data": "foo"}) + await _upsert_custom_metadata(got_jobs[0].uuid, metadata={"jobs data": "bar"}) + + +# test create a job +# - from a study +# - from a solver +# - search jobs from a study, from a solver, etc +# - list studies -> projects uuids that are not jobs +# - list study jobs -> projects uuids that are Jodbs + + +def test_paginate_solver_jobs(): + ... def test_read_job_metadata(): From 727a7abd236a2516e538ee43092fe399a97facbb Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 27 Jun 2023 20:25:15 +0200 Subject: [PATCH 09/53] migration projects-metadata --- ...647bd56403a_new_projects_metadata_table.py | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/6647bd56403a_new_projects_metadata_table.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/6647bd56403a_new_projects_metadata_table.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/6647bd56403a_new_projects_metadata_table.py new file mode 100644 index 00000000000..3c343979b92 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/6647bd56403a_new_projects_metadata_table.py @@ -0,0 +1,97 @@ +"""new projects_metadata table + +Revision ID: 6647bd56403a +Revises: 417f9eb848ce +Create Date: 2023-06-27 18:23:19.613468+00:00 + +""" +from typing import Final + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "6647bd56403a" +down_revision = "417f9eb848ce" +branch_labels = None +depends_on = None + + +# auto-update modified +# TRIGGERS ------------------------ +_TABLE_NAME: Final[str] = "projects_metadata" +_TRIGGER_NAME: Final[str] = "trigger_auto_update" # NOTE: scoped on table +_PROCEDURE_NAME: Final[ + str +] = f"{_TABLE_NAME}_auto_update_modified()" # NOTE: scoped on database +modified_timestamp_trigger = sa.DDL( + f""" +DROP TRIGGER IF EXISTS {_TRIGGER_NAME} on {_TABLE_NAME}; +CREATE TRIGGER {_TRIGGER_NAME} +BEFORE INSERT OR UPDATE ON {_TABLE_NAME} +FOR EACH ROW EXECUTE PROCEDURE {_PROCEDURE_NAME}; + """ +) + +# PROCEDURES ------------------------ +update_modified_timestamp_procedure = sa.DDL( + f""" +CREATE OR REPLACE FUNCTION {_PROCEDURE_NAME} +RETURNS TRIGGER AS $$ +BEGIN + NEW.modified := current_timestamp; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + """ +) + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + _TABLE_NAME, + sa.Column("project_uuid", sa.String(), nullable=False), + sa.Column( + "custom_metadata", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + sa.Column( + "created", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["project_uuid"], + ["projects.uuid"], + name="fk_projects_metadata_project_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("project_uuid"), + ) + # ### end Alembic commands ### + + # custom + update_modified_timestamp_procedure.execute(bind=op.get_context().bind) + modified_timestamp_trigger.execute(bind=op.get_context().bind) + + +def downgrade(): + # custom + op.execute(f"DROP TRIGGER IF EXISTS {_TRIGGER_NAME} on {_TABLE_NAME};") + op.execute(f"DROP FUNCTION {_PROCEDURE_NAME};") + + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table(_TABLE_NAME) + # ### end Alembic commands ### From e06046a9e05243240ff2c4b89e5f4488829f242a Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 27 Jun 2023 20:28:00 +0200 Subject: [PATCH 10/53] migration projects-jobs-metadata --- ...7b0985_new_projects_jobs_metadata_table.py | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/c02d277b0985_new_projects_jobs_metadata_table.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/c02d277b0985_new_projects_jobs_metadata_table.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/c02d277b0985_new_projects_jobs_metadata_table.py new file mode 100644 index 00000000000..1d6ce2596d7 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/c02d277b0985_new_projects_jobs_metadata_table.py @@ -0,0 +1,98 @@ +"""new projects_jobs_metadata table + +Revision ID: c02d277b0985 +Revises: 6647bd56403a +Create Date: 2023-06-27 18:26:36.326140+00:00 + +""" +from typing import Final + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "c02d277b0985" +down_revision = "6647bd56403a" +branch_labels = None +depends_on = None + +# auto-update modified +# TRIGGERS ------------------------ +_TABLE_NAME: Final[str] = "projects_jobs_metadata" +_TRIGGER_NAME: Final[str] = "trigger_auto_update" # NOTE: scoped on table +_PROCEDURE_NAME: Final[ + str +] = f"{_TABLE_NAME}_auto_update_modified()" # NOTE: scoped on database +modified_timestamp_trigger = sa.DDL( + f""" +DROP TRIGGER IF EXISTS {_TRIGGER_NAME} on {_TABLE_NAME}; +CREATE TRIGGER {_TRIGGER_NAME} +BEFORE INSERT OR UPDATE ON {_TABLE_NAME} +FOR EACH ROW EXECUTE PROCEDURE {_PROCEDURE_NAME}; + """ +) + +# PROCEDURES ------------------------ +update_modified_timestamp_procedure = sa.DDL( + f""" +CREATE OR REPLACE FUNCTION {_PROCEDURE_NAME} +RETURNS TRIGGER AS $$ +BEGIN + NEW.modified := current_timestamp; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + """ +) + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + _TABLE_NAME, + sa.Column("project_uuid", sa.String(), nullable=False), + sa.Column("parent_name", sa.String(), nullable=False), + sa.Column( + "job_metadata", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=True, + ), + sa.Column( + "created", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["project_uuid"], + ["projects.uuid"], + name="fk_projects_jobs_metadata_project_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("project_uuid"), + ) + # ### end Alembic commands ### + + # custom + update_modified_timestamp_procedure.execute(bind=op.get_context().bind) + modified_timestamp_trigger.execute(bind=op.get_context().bind) + + +def downgrade(): + # custom + op.execute(f"DROP TRIGGER IF EXISTS {_TRIGGER_NAME} on {_TABLE_NAME};") + op.execute(f"DROP FUNCTION {_PROCEDURE_NAME};") + + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table(_TABLE_NAME) + # ### end Alembic commands ### + # ### end Alembic commands ### From 1c468eebdc60dbb0b9f000704f12c403abfd406c Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 27 Jun 2023 20:40:55 +0200 Subject: [PATCH 11/53] updates read --- .../tests/test_projects_metadata.py | 59 ++++++++++--------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/packages/postgres-database/tests/test_projects_metadata.py b/packages/postgres-database/tests/test_projects_metadata.py index 289678d2939..fbe3a7c4dac 100644 --- a/packages/postgres-database/tests/test_projects_metadata.py +++ b/packages/postgres-database/tests/test_projects_metadata.py @@ -17,7 +17,7 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert -async def test_create_job( +async def test_jobs_workflow( connection: SAConnection, create_fake_project: Callable[..., Awaitable[RowProxy]], create_fake_user: Callable[..., Awaitable[RowProxy]], @@ -31,7 +31,7 @@ async def _create_solver_job(service_key: str, service_version: str, n: int): query = projects_jobs_metadata.insert().values( project_uuid=project.uuid, parent_name=parent_name, - jobs_metadata={ + job_metadata={ "__type__": "JobMeta", "inputs_checksum": f"{n}2bfd4885aa1daf5c16fdd39b9118f652c4977c4021c900794dc125cf123718e", "created_at": f"2022-06-01T15:{n}:56.807441", @@ -39,11 +39,12 @@ async def _create_solver_job(service_key: str, service_version: str, n: int): ) result: ResultProxy = await connection.execute(query) assert result - return UUID(project.uuid) + return project.uuid # some project from the UI project_study = await create_fake_project(connection, user, hidden=True) + # CREATE # some solver-job projects created_jobs: list[UUID] = [ await _create_solver_job( @@ -54,15 +55,27 @@ async def _create_solver_job(service_key: str, service_version: str, n: int): assert project_study.uuid not in set(created_jobs) - # list jobs of a solver - j = projects.join( - projects_jobs_metadata, - (projects.c.uuid == projects_jobs_metadata.c.project_uuid), - ) - query = sa.select(projects_jobs_metadata, projects.c.hidden).select_from(j) - got_jobs = await (await connection.execute(query)).fetchall() - assert got_jobs + # READ + async def _list_solver_jobs(service_key: str, service_version: str): + # list jobs of a solver + parent_name = f"/v0/solvers/{service_key}/releases/{service_version}" + + j = projects.join( + projects_jobs_metadata, + (projects.c.uuid == projects_jobs_metadata.c.project_uuid), + ) + query = ( + sa.select(projects_jobs_metadata, projects.c.hidden) + .select_from(j) + .where(projects_jobs_metadata.c.parent_name == parent_name) + ) + jobs = await (await connection.execute(query)).fetchall() + assert jobs + return jobs + got_jobs = await _list_solver_jobs( + service_key="simcore/comp/itis/sleeper", service_version="2.0.0" + ) assert {j.project_uuid for j in got_jobs} == set(created_jobs) assert all(j.hidden for j in got_jobs) @@ -84,29 +97,19 @@ async def _upsert_custom_metadata(project_uuid, metadata: dict[str, Any]): ) await connection.execute(on_update_stmt) + # UPDATE custom - meta await _upsert_custom_metadata(project_study.uuid, metadata={"my data": "foo"}) - await _upsert_custom_metadata(got_jobs[0].uuid, metadata={"jobs data": "bar"}) + await _upsert_custom_metadata( + got_jobs[0].project_uuid, metadata={"jobs data": "bar"} + ) + + # DELETE job by deleting project # test create a job +# # - from a study # - from a solver # - search jobs from a study, from a solver, etc # - list studies -> projects uuids that are not jobs # - list study jobs -> projects uuids that are Jodbs - - -def test_paginate_solver_jobs(): - ... - - -def test_read_job_metadata(): - ... - - -def test_update_job_metadata(): - ... - - -def test_delete_job_metadata(): - ... From 9a7a388b36d5f7ff13666fda1a7d24b7590eb0ec Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 28 Jun 2023 18:17:08 +0200 Subject: [PATCH 12/53] drafts job-metadata repo --- .../utils_projects_metadata.py | 154 ++++++++++++++++++ .../tests/test_projects_metadata.py | 19 +++ 2 files changed, 173 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py new file mode 100644 index 00000000000..b5b4358242a --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py @@ -0,0 +1,154 @@ +import datetime +import uuid +from dataclasses import dataclass +from typing import Any + +import sqlalchemy as sa +from aiopg.sa.connection import SAConnection +from aiopg.sa.results import ResultProxy +from simcore_postgres_database.models.projects_metadata import projects_jobs_metadata + +from .errors import ForeignKeyViolation +from .utils_models import FromRowMixin + +# +# Errors +# + + +class ProjectNotFound(Exception): + code = "projects.not_found" + + +class BaseProjectJobMetadataError(Exception): + ... + + +class ProjectJobMetadataNotFound(BaseProjectJobMetadataError): + code = "projects.job_metadata.not_found" + + +# +# Data +# + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ProjectJobMetadata(FromRowMixin): + project_uuid: uuid.UUID + parent_name: str + job_metadata: dict[str, Any] = {} + created: datetime.datetime + modified: datetime.datetime + + +# +# Repos +# + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ProjectJobMetadataRepo: + api_vtag: str = "v0" + + def _get_parent_name( + self, + service_key: str, + service_version: str, + ): + return f"/{self.api_vtag}/solvers/{service_key}/releases/{service_version}" + + async def create_solver_job( + self, + connection: SAConnection, + project_uuid: uuid.UUID, # pk + service_key: str, + service_version: str, + job_metadata: dict[str, Any] | None = None, + ) -> ProjectJobMetadata: + + values: dict[str, Any] = dict( + project_uuid=project_uuid, + parent_name=self._get_parent_name(service_key, service_version), + ) + if job_metadata: + values["job_metadata"] = job_metadata + + try: + insert_stmt = ( + projects_jobs_metadata.insert() + .values(**values) + .returning(*[projects_jobs_metadata.columns.keys()]) + ) + result: ResultProxy = await connection.execute(insert_stmt) + row = await result.first() + + return ProjectJobMetadata.from_row(row) + + except ForeignKeyViolation as exc: + raise ProjectNotFound( + f"Cannot create metadata without a valid project {project_uuid=}" + ) from exc + + async def list_solver_jobs( + self, + connection: SAConnection, + service_key: str, + service_version: str, + limit: int, + offset: int, + ) -> list[ProjectJobMetadata]: + assert limit > 0 # nosec + assert offset >= 0 # nosec + + parent_name = self._get_parent_name(service_key, service_version) + list_stmt = ( + sa.select(projects_jobs_metadata) + .where(projects_jobs_metadata.c.parent_name == parent_name) + .order_by(projects_jobs_metadata.c.created_at) + .offset(offset) + .limit(limit) + ) + result: ResultProxy = await connection.execute(list_stmt) + rows = await result.fetchall() + return [ProjectJobMetadata.from_row(row) for row in rows] + + async def get( + self, connection: SAConnection, project_uuid: uuid.UUID + ) -> ProjectJobMetadata: + get_stmt = sa.select(projects_jobs_metadata).where( + projects_jobs_metadata.c.project_uuid == f"{project_uuid}" + ) + result: ResultProxy = await connection.execute(get_stmt) + if row := await result.first(): + return ProjectJobMetadata.from_row(row) + raise ProjectJobMetadataNotFound + + async def update( + self, + connection: SAConnection, + *, + project_uuid: uuid.UUID, + job_metadata: dict[str, Any], + ) -> ProjectJobMetadata: + + update_stmt = ( + projects_jobs_metadata.update() + .where(projects_jobs_metadata.c.project_uuid == project_uuid) + .values(projects_jobs_metadata.c.job_metadata == job_metadata) + .returning(*list(projects_jobs_metadata.columns)) + ) + result = await connection.execute(update_stmt) + if row := await result.first(): + return ProjectJobMetadata.from_row(row) + raise ProjectJobMetadataNotFound + + async def delete(self, connection: SAConnection, project_uuid: uuid.UUID) -> None: + delete_stmt = sa.delete(projects_jobs_metadata).where( + projects_jobs_metadata.c.project_uuid == f"{project_uuid}" + ) + result = await connection.execute(delete_stmt) + if result.rowcount: + raise ProjectJobMetadataNotFound( + f"Could not delete non-existing metadata of {project_uuid=}" + ) diff --git a/packages/postgres-database/tests/test_projects_metadata.py b/packages/postgres-database/tests/test_projects_metadata.py index fbe3a7c4dac..749f6405570 100644 --- a/packages/postgres-database/tests/test_projects_metadata.py +++ b/packages/postgres-database/tests/test_projects_metadata.py @@ -6,6 +6,7 @@ from typing import Any, Awaitable, Callable from uuid import UUID +import pytest import sqlalchemy as sa from aiopg.sa.connection import SAConnection from aiopg.sa.result import ResultProxy, RowProxy @@ -17,6 +18,24 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert +@pytest.fixture +async def fake_user( + connection: SAConnection, + create_fake_user: Callable[..., Awaitable[RowProxy]], +) -> RowProxy: + user: RowProxy = await create_fake_user(connection, name=f"user.{__name__}") + return user + + +@pytest.fixture +async def fake_project( + connection: SAConnection, + fake_user: RowProxy, + create_fake_project: Callable[..., Awaitable[RowProxy]], +): + project: RowProxy = await create_fake_project(connection, fake_user, hidden=True) + + async def test_jobs_workflow( connection: SAConnection, create_fake_project: Callable[..., Awaitable[RowProxy]], From 1bf09b30c92139b0c45285b00f505e356e17b8e6 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 28 Jun 2023 18:39:40 +0200 Subject: [PATCH 13/53] drafts API --- .../projects_metadata.py | 0 .../projects/_metadata_api.py | 0 .../projects/_metadata_db.py | 18 ++++ .../projects/_metadata_handlers.py | 82 +++++++++++++++++++ 4 files changed, 100 insertions(+) create mode 100644 packages/models-library/src/models_library/api_schemas_webserver/projects_metadata.py create mode 100644 services/web/server/src/simcore_service_webserver/projects/_metadata_api.py create mode 100644 services/web/server/src/simcore_service_webserver/projects/_metadata_db.py create mode 100644 services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects_metadata.py b/packages/models-library/src/models_library/api_schemas_webserver/projects_metadata.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py new file mode 100644 index 00000000000..50a7af2c399 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py @@ -0,0 +1,18 @@ +from simcore_postgres_database.utils_projects_metadata import ( + ProjectJobMetadata, + ProjectJobMetadataNotFound, + ProjectJobMetadataRepo, + ProjectNotFound, +) + +assert ProjectJobMetadata # nosec +assert ProjectJobMetadataRepo # nosec +assert ProjectJobMetadataNotFound # nosec +assert ProjectNotFound + + +__all__: tuple[str, ...] = ( + "ProjectJobMetadata", + "ProjectJobMetadataRepo", + "ProjectJobMetadataNotFound", +) diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py new file mode 100644 index 00000000000..a91dd29612f --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py @@ -0,0 +1,82 @@ +from aiohttp import web + +from .._meta import api_version_prefix as VTAG +from ..login.decorators import login_required +from ..security.decorators import permission_required + +routes = web.RouteTableDef() + + +# +# projects/*/job-metadata +# + + +@routes.post( + f"/{VTAG}/projects/{{project_id}}/job-metadata", name="create_project_job_metadata" +) +@login_required +@permission_required("project.create") +async def create_project_job_metadata(request: web.Request) -> web.Response: + raise NotImplementedError + + +@routes.get( + f"/{VTAG}/projects/{{project_id}}/job-metadata", name="get_project_job_metadata" +) +@login_required +@permission_required("project.read") +async def get_project_job_metadata(request: web.Request) -> web.Response: + raise NotImplementedError + + +@routes.get(f"/{VTAG}/projects/-/job-metadata", name="list_projects_job_metadata") +@login_required +@permission_required("project.read") +async def list_projects_job_metadata(request: web.Request) -> web.Response: + raise NotImplementedError + + +# +# projects/*/custom-metadata +# + + +@routes.post( + f"/{VTAG}/projects/{{project_id}}/custom-metadata", + name="create_project_custom_metadata", +) +@login_required +@permission_required("project.create") +async def create_project_custom_metadata(request: web.Request) -> web.Response: + raise NotImplementedError + + +@routes.get( + f"/{VTAG}/projects/{{project_id}}/custom-metadata", + name="get_project_custom_metadata", +) +@login_required +@permission_required("project.read") +async def get_project_custom_metadata(request: web.Request) -> web.Response: + raise NotImplementedError + + +@routes.patch( + f"/{VTAG}/projects/{{project_id}}/custom-metadata", + name="update_project_custom_metadata", +) +@login_required +@permission_required("project.update") +async def update_project_custom_metadata(request: web.Request) -> web.Response: + raise NotImplementedError + + +@routes.delete( + f"/{VTAG}/projects/{{project_id}}/custom-metadata", + name="update_project_custom_metadata", +) +@login_required +@permission_required("project.delete") +async def delete_project_custom_metadata(request: web.Request) -> web.Response: + raise NotImplementedError From 6cd0a395f34ef79eda6a9a7b1172b6f2a309a4d4 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 29 Jun 2023 10:55:48 +0200 Subject: [PATCH 14/53] minor --- .../utils_projects_metadata.py | 26 +++++++++---------- .../projects/_metadata_db.py | 10 +++---- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py index b5b4358242a..61414a7488b 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py @@ -16,7 +16,7 @@ # -class ProjectNotFound(Exception): +class ProjectNotFoundError(Exception): code = "projects.not_found" @@ -24,7 +24,7 @@ class BaseProjectJobMetadataError(Exception): ... -class ProjectJobMetadataNotFound(BaseProjectJobMetadataError): +class ProjectJobMetadataNotFoundError(BaseProjectJobMetadataError): code = "projects.job_metadata.not_found" @@ -67,10 +67,10 @@ async def create_solver_job( job_metadata: dict[str, Any] | None = None, ) -> ProjectJobMetadata: - values: dict[str, Any] = dict( - project_uuid=project_uuid, - parent_name=self._get_parent_name(service_key, service_version), - ) + values: dict[str, Any] = { + "project_uuid": project_uuid, + "parent_name": self._get_parent_name(service_key, service_version), + } if job_metadata: values["job_metadata"] = job_metadata @@ -86,9 +86,8 @@ async def create_solver_job( return ProjectJobMetadata.from_row(row) except ForeignKeyViolation as exc: - raise ProjectNotFound( - f"Cannot create metadata without a valid project {project_uuid=}" - ) from exc + msg = f"Cannot create metadata without a valid project {project_uuid=}" + raise ProjectNotFoundError(msg) from exc async def list_solver_jobs( self, @@ -122,7 +121,7 @@ async def get( result: ResultProxy = await connection.execute(get_stmt) if row := await result.first(): return ProjectJobMetadata.from_row(row) - raise ProjectJobMetadataNotFound + raise ProjectJobMetadataNotFoundError async def update( self, @@ -141,7 +140,7 @@ async def update( result = await connection.execute(update_stmt) if row := await result.first(): return ProjectJobMetadata.from_row(row) - raise ProjectJobMetadataNotFound + raise ProjectJobMetadataNotFoundError async def delete(self, connection: SAConnection, project_uuid: uuid.UUID) -> None: delete_stmt = sa.delete(projects_jobs_metadata).where( @@ -149,6 +148,5 @@ async def delete(self, connection: SAConnection, project_uuid: uuid.UUID) -> Non ) result = await connection.execute(delete_stmt) if result.rowcount: - raise ProjectJobMetadataNotFound( - f"Could not delete non-existing metadata of {project_uuid=}" - ) + msg = f"Could not delete non-existing metadata of project_uuid={project_uuid!r}" + raise ProjectJobMetadataNotFoundError(msg) diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py index 50a7af2c399..bed3a9f09e6 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py @@ -1,18 +1,18 @@ from simcore_postgres_database.utils_projects_metadata import ( ProjectJobMetadata, - ProjectJobMetadataNotFound, + ProjectJobMetadataNotFoundError, ProjectJobMetadataRepo, - ProjectNotFound, + ProjectNotFoundError, ) assert ProjectJobMetadata # nosec assert ProjectJobMetadataRepo # nosec -assert ProjectJobMetadataNotFound # nosec -assert ProjectNotFound +assert ProjectJobMetadataNotFoundError # nosec +assert ProjectNotFoundError __all__: tuple[str, ...] = ( "ProjectJobMetadata", "ProjectJobMetadataRepo", - "ProjectJobMetadataNotFound", + "ProjectJobMetadataNotFoundError", ) From 014f3bdb8ec0453200027a7b7df9b803f7bbe3cf Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 4 Jul 2023 16:24:32 +0200 Subject: [PATCH 15/53] layout apis --- .../api/routes/solvers_jobs.py | 49 ------------------ .../projects/_metadata_handlers.py | 51 +++++++------------ 2 files changed, 18 insertions(+), 82 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py index 23dbe9784be..bbbef8f0610 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py @@ -83,27 +83,6 @@ async def list_jobs( SEE get_jobs_page for paginated version of this function """ - # ```mermaid - # sequenceDiagram - # participant API - # participant AS as api-server - # participant CS as catalog - # participant WS as web-server - # - # link API: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/api-server/openapi.json - # link CS: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/catalog/openapi.json - # link WS: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/web/server/src/simcore_service_webserver/api/v0/openapi - # - # API->>+AS: list_job - # AS->>+CS: get_service - # CS-->>-AS: ServiceGet - # AS->>+WS: list_projects(page*) - # WS-->>-AS: list[ProjectGet] - # AS-->>-API: list[Job] | Page[Job]* - # ``` - # SEE https://mermaid.live/ - # * = still not implemented - solver = await catalog_client.get_service( user_id=user_id, name=solver_key, @@ -192,34 +171,6 @@ async def create_job( NOTE: This operation does **not** start the job """ - # ```mermaid - # sequenceDiagram - # participant API - # participant AS as api-server - # participant CS as catalog - # participant WS as web-server - # - # link API: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/api-server/openapi.json - # link CS: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/catalog/openapi.json - # link WS: OAS @ https://editor.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/services/web/server/src/simcore_service_webserver/api/v0/openapi - # - # API->>+AS: create_job - # AS->>+CS: get_service - # CS-->>-AS: ServiceGet - # AS->>+CS: get_service_ports* - # CS-->>-AS: ServicePortGet* - # Note right of AS: Validate SolverInputs* - # AS->>+WS: create_project - # WS-->>-AS: ProjectGet - # AS->>+WS: set_job_metadata* - # WS-->>-AS: ProjectJobMetaGet* - # AS-->>-API: Job - # ``` - # SEE https://mermaid.live/ - # * = still not implemented - - # -> catalog - # TODO: validate inputs against solver input schema # ensures user has access to solver solver = await catalog_client.get_service( user_id=user_id, diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py index a91dd29612f..03ca1533595 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py @@ -1,3 +1,15 @@ +""" + +Design rationale: + +- Resource metadata/labels: https://cloud.google.com/apis/design/design_patterns#resource_labels + - named `metadata` instead of labels + - limit number of entries and depth? dict[str, st] ?? +- Singleton https://cloud.google.com/apis/design/design_patterns#singleton_resources + - the singleton is implicitly created or deleted when its parent is created or deleted + - Get and Update methods only +""" + from aiohttp import web from .._meta import api_version_prefix as VTAG @@ -12,15 +24,6 @@ # -@routes.post( - f"/{VTAG}/projects/{{project_id}}/job-metadata", name="create_project_job_metadata" -) -@login_required -@permission_required("project.create") -async def create_project_job_metadata(request: web.Request) -> web.Response: - raise NotImplementedError - - @routes.get( f"/{VTAG}/projects/{{project_id}}/job-metadata", name="get_project_job_metadata" ) @@ -30,10 +33,12 @@ async def get_project_job_metadata(request: web.Request) -> web.Response: raise NotImplementedError -@routes.get(f"/{VTAG}/projects/-/job-metadata", name="list_projects_job_metadata") +@routes.patch( + f"/{VTAG}/projects/{{project_id}}/job-metadata", name="create_project_job_metadata" +) @login_required -@permission_required("project.read") -async def list_projects_job_metadata(request: web.Request) -> web.Response: +@permission_required("project.create") +async def update_project_job_metadata(request: web.Request) -> web.Response: raise NotImplementedError @@ -42,16 +47,6 @@ async def list_projects_job_metadata(request: web.Request) -> web.Response: # -@routes.post( - f"/{VTAG}/projects/{{project_id}}/custom-metadata", - name="create_project_custom_metadata", -) -@login_required -@permission_required("project.create") -async def create_project_custom_metadata(request: web.Request) -> web.Response: - raise NotImplementedError - - @routes.get( f"/{VTAG}/projects/{{project_id}}/custom-metadata", name="get_project_custom_metadata", @@ -62,7 +57,7 @@ async def get_project_custom_metadata(request: web.Request) -> web.Response: raise NotImplementedError -@routes.patch( +@routes.put( f"/{VTAG}/projects/{{project_id}}/custom-metadata", name="update_project_custom_metadata", ) @@ -70,13 +65,3 @@ async def get_project_custom_metadata(request: web.Request) -> web.Response: @permission_required("project.update") async def update_project_custom_metadata(request: web.Request) -> web.Response: raise NotImplementedError - - -@routes.delete( - f"/{VTAG}/projects/{{project_id}}/custom-metadata", - name="update_project_custom_metadata", -) -@login_required -@permission_required("project.delete") -async def delete_project_custom_metadata(request: web.Request) -> web.Response: - raise NotImplementedError From 9ca523bec9bef66e88f12a9f3ec97f6f243777b0 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 4 Jul 2023 17:44:19 +0200 Subject: [PATCH 16/53] cleanup --- .../models/projects_metadata.py | 2 +- ...ata.py => test_utils_projects_metadata.py} | 6 +- .../projects/_metadata_db.py | 2 +- .../projects/_metadata_handlers.py | 10 +-- .../02/test_projects_metadata_handlers.py | 75 +++++++++++++++++++ 5 files changed, 86 insertions(+), 9 deletions(-) rename packages/postgres-database/tests/{test_projects_metadata.py => test_utils_projects_metadata.py} (97%) create mode 100644 services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py index 97b18d149a7..6b8ea27abbd 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py @@ -41,7 +41,7 @@ JSONB, nullable=False, server_default=sa.text("'{}'::jsonb"), - doc="Free json for user to store her metadata", + doc="Free json for user to store her metadata.", ), # TIME STAMPS ---- column_created_datetime(timezone=True), diff --git a/packages/postgres-database/tests/test_projects_metadata.py b/packages/postgres-database/tests/test_utils_projects_metadata.py similarity index 97% rename from packages/postgres-database/tests/test_projects_metadata.py rename to packages/postgres-database/tests/test_utils_projects_metadata.py index 749f6405570..5965b79b239 100644 --- a/packages/postgres-database/tests/test_projects_metadata.py +++ b/packages/postgres-database/tests/test_utils_projects_metadata.py @@ -3,7 +3,8 @@ # pylint: disable=unused-variable # pylint: disable=too-many-arguments -from typing import Any, Awaitable, Callable +from collections.abc import Awaitable, Callable +from typing import Any from uuid import UUID import pytest @@ -32,8 +33,9 @@ async def fake_project( connection: SAConnection, fake_user: RowProxy, create_fake_project: Callable[..., Awaitable[RowProxy]], -): +) -> RowProxy: project: RowProxy = await create_fake_project(connection, fake_user, hidden=True) + return project async def test_jobs_workflow( diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py index bed3a9f09e6..608800a1cc8 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py @@ -8,7 +8,7 @@ assert ProjectJobMetadata # nosec assert ProjectJobMetadataRepo # nosec assert ProjectJobMetadataNotFoundError # nosec -assert ProjectNotFoundError +assert ProjectNotFoundError # nosec __all__: tuple[str, ...] = ( diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py index 03ca1533595..7476259dff8 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py @@ -33,12 +33,12 @@ async def get_project_job_metadata(request: web.Request) -> web.Response: raise NotImplementedError -@routes.patch( - f"/{VTAG}/projects/{{project_id}}/job-metadata", name="create_project_job_metadata" +@routes.put( + f"/{VTAG}/projects/{{project_id}}/job-metadata", name="replace_project_job_metadata" ) @login_required @permission_required("project.create") -async def update_project_job_metadata(request: web.Request) -> web.Response: +async def replace_project_job_metadata(request: web.Request) -> web.Response: raise NotImplementedError @@ -59,9 +59,9 @@ async def get_project_custom_metadata(request: web.Request) -> web.Response: @routes.put( f"/{VTAG}/projects/{{project_id}}/custom-metadata", - name="update_project_custom_metadata", + name="replace_project_custom_metadata", ) @login_required @permission_required("project.update") -async def update_project_custom_metadata(request: web.Request) -> web.Response: +async def replace_project_custom_metadata(request: web.Request) -> web.Response: raise NotImplementedError diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py new file mode 100644 index 00000000000..477e74556b4 --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py @@ -0,0 +1,75 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments + +import json + +import pytest +from aiohttp import web +from aiohttp.test_utils import TestClient +from faker import Faker +from pytest_simcore.helpers.utils_assert import assert_status +from pytest_simcore.helpers.utils_login import UserInfoDict +from simcore_postgres_database.models.users import UserRole +from simcore_service_webserver.projects.models import ProjectDict + + +@pytest.mark.parametrize( + "user_role", + [ + UserRole.USER, + ], +) +async def test_custom_metadata_handlers( + client: TestClient, + faker: Faker, + logged_user: UserInfoDict, + user_project: ProjectDict, +): + # + # metadata is a singleton subresource of a project + # a singleton is implicitly created or deleted when its parent is created or deleted + # + assert client.app + + # get metadata of a non-existing project -> Not found + url = client.app.router["get_project_custom_metadata"].url_for( + project_id=faker.uuid4() + ) + response = await client.get(f"{url}") + + await assert_status(response, expected_cls=web.HTTPNotFound) + + # get metadata of an existing project the first time -> empty {} + url = client.app.router["get_project_custom_metadata"].url_for( + project_id=user_project["uuid"] + ) + response = await client.get(f"{url}") + data, _ = await assert_status(response, expected_cls=web.HTTPOk) + assert data["metadata"] == {} + + # replace metadata + custom_metadata = {"number": 3.14, "string": "str", "boolean": False} + custom_metadata["other"] = json.dumps(custom_metadata) + + url = client.app.router["replace_project_custom_metadata"].url_for( + project_id=user_project["uuid"] + ) + response = await client.put(f"{url}", json=custom_metadata) + + data, _ = await assert_status(response, expected_cls=web.HTTPOk) + assert data["metadata"] == custom_metadata + + # delete project + url = client.app.router["delete_project"].url_for(project_id=user_project["uuid"]) + response = await client.delete(f"{url}", json=custom_metadata) + await assert_status(response, expected_cls=web.HTTPNoContent) + + # no metadata -> project not foun d + url = client.app.router["get_project_custom_metadata"].url_for( + project_id=user_project["uuid"] + ) + response = await client.get(f"{url}") + await assert_status(response, expected_cls=web.HTTPNotFound) + await assert_status(response, expected_cls=web.HTTPNotFound) From a960d7fb9bb0d06279873e2b4b311179cfa2b2ef Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 4 Jul 2023 18:50:48 +0200 Subject: [PATCH 17/53] updates code --- packages/postgres-database/scripts/erd/cli.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/postgres-database/scripts/erd/cli.py b/packages/postgres-database/scripts/erd/cli.py index 19b188e7ea2..368c9fe2e51 100644 --- a/packages/postgres-database/scripts/erd/cli.py +++ b/packages/postgres-database/scripts/erd/cli.py @@ -19,6 +19,13 @@ import simcore_postgres_database.models from simcore_postgres_database.models.base import metadata +from simcore_postgres_database.models.projects import projects +from simcore_postgres_database.models.projects_comments import projects_comments +from simcore_postgres_database.models.projects_metadata import ( + projects_jobs_metadata, + projects_metadata, +) +from simcore_postgres_database.models.projects_nodes import projects_nodes from sqlalchemy_schemadisplay import create_schema_graph models_folder = Path(simcore_postgres_database.models.__file__).parent From 8c50887110a0804675cdf4868860ec0057ee6506 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 4 Jul 2023 18:56:19 +0200 Subject: [PATCH 18/53] rm migration --- ...647bd56403a_new_projects_metadata_table.py | 97 ------------------ ...7b0985_new_projects_jobs_metadata_table.py | 98 ------------------- 2 files changed, 195 deletions(-) delete mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/6647bd56403a_new_projects_metadata_table.py delete mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/c02d277b0985_new_projects_jobs_metadata_table.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/6647bd56403a_new_projects_metadata_table.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/6647bd56403a_new_projects_metadata_table.py deleted file mode 100644 index 3c343979b92..00000000000 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/6647bd56403a_new_projects_metadata_table.py +++ /dev/null @@ -1,97 +0,0 @@ -"""new projects_metadata table - -Revision ID: 6647bd56403a -Revises: 417f9eb848ce -Create Date: 2023-06-27 18:23:19.613468+00:00 - -""" -from typing import Final - -import sqlalchemy as sa -from alembic import op -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision = "6647bd56403a" -down_revision = "417f9eb848ce" -branch_labels = None -depends_on = None - - -# auto-update modified -# TRIGGERS ------------------------ -_TABLE_NAME: Final[str] = "projects_metadata" -_TRIGGER_NAME: Final[str] = "trigger_auto_update" # NOTE: scoped on table -_PROCEDURE_NAME: Final[ - str -] = f"{_TABLE_NAME}_auto_update_modified()" # NOTE: scoped on database -modified_timestamp_trigger = sa.DDL( - f""" -DROP TRIGGER IF EXISTS {_TRIGGER_NAME} on {_TABLE_NAME}; -CREATE TRIGGER {_TRIGGER_NAME} -BEFORE INSERT OR UPDATE ON {_TABLE_NAME} -FOR EACH ROW EXECUTE PROCEDURE {_PROCEDURE_NAME}; - """ -) - -# PROCEDURES ------------------------ -update_modified_timestamp_procedure = sa.DDL( - f""" -CREATE OR REPLACE FUNCTION {_PROCEDURE_NAME} -RETURNS TRIGGER AS $$ -BEGIN - NEW.modified := current_timestamp; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - """ -) - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - _TABLE_NAME, - sa.Column("project_uuid", sa.String(), nullable=False), - sa.Column( - "custom_metadata", - postgresql.JSONB(astext_type=sa.Text()), - server_default=sa.text("'{}'::jsonb"), - nullable=False, - ), - sa.Column( - "created", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.Column( - "modified", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.ForeignKeyConstraint( - ["project_uuid"], - ["projects.uuid"], - name="fk_projects_metadata_project_uuid", - onupdate="CASCADE", - ondelete="CASCADE", - ), - sa.PrimaryKeyConstraint("project_uuid"), - ) - # ### end Alembic commands ### - - # custom - update_modified_timestamp_procedure.execute(bind=op.get_context().bind) - modified_timestamp_trigger.execute(bind=op.get_context().bind) - - -def downgrade(): - # custom - op.execute(f"DROP TRIGGER IF EXISTS {_TRIGGER_NAME} on {_TABLE_NAME};") - op.execute(f"DROP FUNCTION {_PROCEDURE_NAME};") - - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table(_TABLE_NAME) - # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/c02d277b0985_new_projects_jobs_metadata_table.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/c02d277b0985_new_projects_jobs_metadata_table.py deleted file mode 100644 index 1d6ce2596d7..00000000000 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/c02d277b0985_new_projects_jobs_metadata_table.py +++ /dev/null @@ -1,98 +0,0 @@ -"""new projects_jobs_metadata table - -Revision ID: c02d277b0985 -Revises: 6647bd56403a -Create Date: 2023-06-27 18:26:36.326140+00:00 - -""" -from typing import Final - -import sqlalchemy as sa -from alembic import op -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision = "c02d277b0985" -down_revision = "6647bd56403a" -branch_labels = None -depends_on = None - -# auto-update modified -# TRIGGERS ------------------------ -_TABLE_NAME: Final[str] = "projects_jobs_metadata" -_TRIGGER_NAME: Final[str] = "trigger_auto_update" # NOTE: scoped on table -_PROCEDURE_NAME: Final[ - str -] = f"{_TABLE_NAME}_auto_update_modified()" # NOTE: scoped on database -modified_timestamp_trigger = sa.DDL( - f""" -DROP TRIGGER IF EXISTS {_TRIGGER_NAME} on {_TABLE_NAME}; -CREATE TRIGGER {_TRIGGER_NAME} -BEFORE INSERT OR UPDATE ON {_TABLE_NAME} -FOR EACH ROW EXECUTE PROCEDURE {_PROCEDURE_NAME}; - """ -) - -# PROCEDURES ------------------------ -update_modified_timestamp_procedure = sa.DDL( - f""" -CREATE OR REPLACE FUNCTION {_PROCEDURE_NAME} -RETURNS TRIGGER AS $$ -BEGIN - NEW.modified := current_timestamp; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - """ -) - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - _TABLE_NAME, - sa.Column("project_uuid", sa.String(), nullable=False), - sa.Column("parent_name", sa.String(), nullable=False), - sa.Column( - "job_metadata", - postgresql.JSONB(astext_type=sa.Text()), - server_default=sa.text("'{}'::jsonb"), - nullable=True, - ), - sa.Column( - "created", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.Column( - "modified", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.ForeignKeyConstraint( - ["project_uuid"], - ["projects.uuid"], - name="fk_projects_jobs_metadata_project_uuid", - onupdate="CASCADE", - ondelete="CASCADE", - ), - sa.PrimaryKeyConstraint("project_uuid"), - ) - # ### end Alembic commands ### - - # custom - update_modified_timestamp_procedure.execute(bind=op.get_context().bind) - modified_timestamp_trigger.execute(bind=op.get_context().bind) - - -def downgrade(): - # custom - op.execute(f"DROP TRIGGER IF EXISTS {_TRIGGER_NAME} on {_TABLE_NAME};") - op.execute(f"DROP FUNCTION {_PROCEDURE_NAME};") - - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table(_TABLE_NAME) - # ### end Alembic commands ### - # ### end Alembic commands ### From 7dab44675fe44629e5d69e1825484e42ed1a0c87 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 4 Jul 2023 21:07:10 +0200 Subject: [PATCH 19/53] fixes erd --- packages/postgres-database/scripts/erd/Dockerfile | 2 +- packages/postgres-database/scripts/erd/cli.py | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/packages/postgres-database/scripts/erd/Dockerfile b/packages/postgres-database/scripts/erd/Dockerfile index 4067d4f1d91..f419efc3f1c 100644 --- a/packages/postgres-database/scripts/erd/Dockerfile +++ b/packages/postgres-database/scripts/erd/Dockerfile @@ -16,7 +16,7 @@ RUN apt-get update \ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install --upgrade \ + pip --no-cache-dir install --upgrade \ pip~=23.1 \ wheel \ setuptools diff --git a/packages/postgres-database/scripts/erd/cli.py b/packages/postgres-database/scripts/erd/cli.py index 368c9fe2e51..19b188e7ea2 100644 --- a/packages/postgres-database/scripts/erd/cli.py +++ b/packages/postgres-database/scripts/erd/cli.py @@ -19,13 +19,6 @@ import simcore_postgres_database.models from simcore_postgres_database.models.base import metadata -from simcore_postgres_database.models.projects import projects -from simcore_postgres_database.models.projects_comments import projects_comments -from simcore_postgres_database.models.projects_metadata import ( - projects_jobs_metadata, - projects_metadata, -) -from simcore_postgres_database.models.projects_nodes import projects_nodes from sqlalchemy_schemadisplay import create_schema_graph models_folder = Path(simcore_postgres_database.models.__file__).parent From 0a25f51fe60fe9dbd90035a2a19eb4ee16960522 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 4 Jul 2023 21:11:29 +0200 Subject: [PATCH 20/53] only one table --- .../models/projects_metadata.py | 63 +++---------------- 1 file changed, 9 insertions(+), 54 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py index 6b8ea27abbd..c322b75a3b9 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py @@ -17,10 +17,14 @@ projects_metadata = sa.Table( "projects_metadata", # - # Holds **runtime** metadata on a project + # Keeps "third-party" metadata attached to a project # - # Things like 'stars', 'quality', 'classifiers' etc (or any kind of stats) - # should be moved here. + # These SHOULD NOT be actual properties of the project (e.g. uuid, name etc) + # but rather information attached by third-parties that "decorate" or qualify + # a project resource + # + # Things like 'stars', 'quality', 'classifiers', 'dev', etc (or any kind of stats) + # should be moved here # metadata, sa.Column( @@ -41,9 +45,9 @@ JSONB, nullable=False, server_default=sa.text("'{}'::jsonb"), - doc="Free json for user to store her metadata.", + doc="Unstructured free json for user to store metadata", ), - # TIME STAMPS ---- + # TIME STAMPS ----ß column_created_datetime(timezone=True), column_modified_datetime(timezone=True), sa.PrimaryKeyConstraint("project_uuid"), @@ -51,52 +55,3 @@ register_modified_datetime_auto_update_trigger(projects_metadata) - - -projects_jobs_metadata = sa.Table( - "projects_jobs_metadata", - # - # Every job is mapped to a project and has an ancestor (see job_parent_name) - # but not every project is associated to a job. - # - # This table contains specific metadata on projects/jobs - # - holds all projects associated to jobs - # - stores jobs ancestry relations and metadata - # - metadata, - sa.Column( - "project_uuid", - sa.String, - sa.ForeignKey( - projects.c.uuid, - onupdate="CASCADE", - ondelete="CASCADE", - name="fk_projects_jobs_metadata_project_uuid", - ), - nullable=False, - primary_key=True, - doc="The project unique identifier is also used to identify the associated job", - ), - sa.Column( - "parent_name", - sa.String, - nullable=False, - doc="Project's ancestor when create as a job. A project can be created as a" - " - solver job: solver name (e.g. /v0/solvers/{id}/releases/{version})" - " - study job: study name (e.g. /v0/studies/{id})", - ), - sa.Column( - "job_metadata", - JSONB, - nullable=True, - server_default=sa.text("'{}'::jsonb"), - doc="Job can store here metadata. " - "Preserves class information during serialization/deserialization", - ), - # TIME STAMPS ---- - column_created_datetime(timezone=True), - column_modified_datetime(timezone=True), - sa.PrimaryKeyConstraint("project_uuid"), -) - -register_modified_datetime_auto_update_trigger(projects_jobs_metadata) From 98e226074954a274dcf85c462c2d8161ad7ea435 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 4 Jul 2023 21:29:09 +0200 Subject: [PATCH 21/53] common models --- .../projects/_common_models.py | 25 ++++++++++++++++ .../projects/_handlers_crud.py | 30 +++++-------------- 2 files changed, 32 insertions(+), 23 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/projects/_common_models.py diff --git a/services/web/server/src/simcore_service_webserver/projects/_common_models.py b/services/web/server/src/simcore_service_webserver/projects/_common_models.py new file mode 100644 index 00000000000..3a79cead257 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/projects/_common_models.py @@ -0,0 +1,25 @@ +""" Handlers for STANDARD methods on /projects colletions + +Standard methods or CRUD that states for Create+Read(Get&List)+Update+Delete + +""" + +from models_library.projects import ProjectID +from models_library.users import UserID +from pydantic import BaseModel, Extra, Field +from servicelib.request_keys import RQT_USERID_KEY + +from .._constants import RQ_PRODUCT_KEY + + +class RequestContext(BaseModel): + user_id: UserID = Field(..., alias=RQT_USERID_KEY) # type: ignore[pydantic-alias] + product_name: str = Field(..., alias=RQ_PRODUCT_KEY) # type: ignore[pydantic-alias] + + +class ProjectPathParams(BaseModel): + project_id: ProjectID + + class Config: + allow_population_by_field_name = True + extra = Extra.forbid diff --git a/services/web/server/src/simcore_service_webserver/projects/_handlers_crud.py b/services/web/server/src/simcore_service_webserver/projects/_handlers_crud.py index 588e77762e3..11e0659983d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_handlers_crud.py +++ b/services/web/server/src/simcore_service_webserver/projects/_handlers_crud.py @@ -23,7 +23,6 @@ Page, ) from models_library.rest_pagination_utils import paginate_data -from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder from pydantic import BaseModel, Extra, Field, NonNegativeInt, validator from servicelib.aiohttp.long_running_tasks.server import start_long_running_task @@ -38,10 +37,8 @@ ) from servicelib.json_serialization import json_dumps from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON -from servicelib.request_keys import RQT_USERID_KEY from servicelib.rest_constants import RESPONSE_MODEL_POLICY -from .._constants import RQ_PRODUCT_KEY from .._meta import api_version_prefix as VTAG from ..catalog.client import get_services_for_user_in_product from ..director_v2 import api @@ -51,6 +48,7 @@ from ..security.decorators import permission_required from ..users.api import get_user_name from . import _crud_create_utils, _crud_read_utils, projects_api +from ._common_models import ProjectPathParams, RequestContext from ._crud_read_utils import OrderDirection, ProjectListFilters, ProjectOrderBy from ._permalink_api import update_or_pop_permalink_in_project from .db import ProjectDBAPI @@ -76,24 +74,11 @@ RQ_REQUESTED_REPO_PROJECT_UUID_KEY = f"{__name__}.RQT_REQUESTED_REPO_PROJECT_UUID_KEY" -log = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) routes = web.RouteTableDef() -class RequestContext(BaseModel): - user_id: UserID = Field(..., alias=RQT_USERID_KEY) # type: ignore[pydantic-alias] - product_name: str = Field(..., alias=RQ_PRODUCT_KEY) # type: ignore[pydantic-alias] - - -class ProjectPathParams(BaseModel): - project_id: ProjectID - - class Config: - allow_population_by_field_name = True - extra = Extra.forbid - - # # - Create https://google.aip.dev/133 # @@ -105,15 +90,15 @@ class _ProjectCreateParams(BaseModel): description="Option to create a project from existing template or study: from_study={study_uuid}", ) as_template: bool = Field( - False, + default=False, description="Option to create a template from existing project: as_template=true", ) copy_data: bool = Field( - True, + default=True, description="Option to copy data when creating from an existing template or as a template, defaults to True", ) hidden: bool = Field( - False, + default=False, description="Enables/disables hidden flag. Hidden projects are by default unlisted", ) @@ -227,9 +212,8 @@ def sort_by_should_have_special_format(cls, v): if field_info[1] == OrderDirection.DESC.value: direction = OrderDirection.DESC else: - raise ValueError( - "Field direction in the order_by parameter must contain either 'desc' direction or empty value for 'asc' direction." - ) + msg = "Field direction in the order_by parameter must contain either 'desc' direction or empty value for 'asc' direction." + raise ValueError(msg) parse_fields_with_direction.append( ProjectOrderBy(field=field_name, direction=direction) From 9a55e4e2006b1de47c956bf677d5239a1b003ef1 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 5 Jul 2023 11:55:15 +0200 Subject: [PATCH 22/53] buildling controler and service --- .../projects/_metadata_api.py | 16 +++++ .../projects/_metadata_handlers.py | 65 ++++++++++++------- 2 files changed, 56 insertions(+), 25 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py index e69de29bb2d..a8e4291a6e9 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py @@ -0,0 +1,16 @@ +from aiohttp import web +from models_library.api_schemas_webserver.projects_metadata import MetadataDict +from models_library.projects import ProjectID +from models_library.users import UserID + + +async def get_project_custom_metadata( + app: web.Application, user_id: UserID, project_id: ProjectID +) -> MetadataDict: + ... + + +async def set_project_custom_metadata( + app: web.Application, user_id: UserID, project_id: ProjectID, value: MetadataDict +) -> MetadataDict: + ... diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py index 7476259dff8..beab83487f6 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py @@ -10,38 +10,27 @@ - Get and Update methods only """ + from aiohttp import web +from models_library.api_schemas_webserver.projects_metadata import ( + ProjectCustomMetadataGet, + ProjectCustomMetadataReplace, +) +from servicelib.aiohttp.requests_validation import ( + parse_request_body_as, + parse_request_path_parameters_as, +) +from simcore_service_webserver.utils_aiohttp import envelope_json_response from .._meta import api_version_prefix as VTAG from ..login.decorators import login_required from ..security.decorators import permission_required +from . import _metadata_api +from ._common_models import ProjectPathParams, RequestContext routes = web.RouteTableDef() -# -# projects/*/job-metadata -# - - -@routes.get( - f"/{VTAG}/projects/{{project_id}}/job-metadata", name="get_project_job_metadata" -) -@login_required -@permission_required("project.read") -async def get_project_job_metadata(request: web.Request) -> web.Response: - raise NotImplementedError - - -@routes.put( - f"/{VTAG}/projects/{{project_id}}/job-metadata", name="replace_project_job_metadata" -) -@login_required -@permission_required("project.create") -async def replace_project_job_metadata(request: web.Request) -> web.Response: - raise NotImplementedError - - # # projects/*/custom-metadata # @@ -54,7 +43,18 @@ async def replace_project_job_metadata(request: web.Request) -> web.Response: @login_required @permission_required("project.read") async def get_project_custom_metadata(request: web.Request) -> web.Response: - raise NotImplementedError + req_ctx = RequestContext.parse_obj(request) + path_params = parse_request_path_parameters_as(ProjectPathParams, request) + + custom_metadata = await _metadata_api.get_project_custom_metadata( + request.app, user_id=req_ctx.user_id, project_id=path_params.project_id + ) + + return envelope_json_response( + ProjectCustomMetadataGet( + project_uuid=path_params.project_id, metadata=custom_metadata + ) + ) @routes.put( @@ -64,4 +64,19 @@ async def get_project_custom_metadata(request: web.Request) -> web.Response: @login_required @permission_required("project.update") async def replace_project_custom_metadata(request: web.Request) -> web.Response: - raise NotImplementedError + req_ctx = RequestContext.parse_obj(request) + path_params = parse_request_path_parameters_as(ProjectPathParams, request) + new_metadata = await parse_request_body_as(ProjectCustomMetadataReplace, request) + + custom_metadata = await _metadata_api.set_project_custom_metadata( + request.app, + user_id=req_ctx.user_id, + project_id=path_params.project_id, + value=new_metadata, + ) + + return envelope_json_response( + ProjectCustomMetadataGet( + project_uuid=path_params.project_id, metadata=custom_metadata + ) + ) From 903bbf133a4496dd106fd730be640a31fa4d7601 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 5 Jul 2023 14:17:33 +0200 Subject: [PATCH 23/53] building service and model --- .../projects_metadata.py | 21 ++++++++++++++ .../projects/_metadata_api.py | 7 +++-- .../projects/_metadata_db.py | 28 +++++++++---------- .../02/test_projects_metadata_handlers.py | 3 +- 4 files changed, 41 insertions(+), 18 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects_metadata.py b/packages/models-library/src/models_library/api_schemas_webserver/projects_metadata.py index e69de29bb2d..3dd33e4bfa4 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects_metadata.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects_metadata.py @@ -0,0 +1,21 @@ +from typing import TypeAlias + +from pydantic import Field, StrictBool, StrictFloat, StrictInt + +from ..projects import ProjectID +from ._base import InputSchema, OutputSchema + +# Limits metadata values +MetaValueType: TypeAlias = StrictBool | StrictInt | StrictFloat | str +MetadataDict: TypeAlias = dict[str, MetaValueType] + + +class ProjectCustomMetadataGet(OutputSchema): + project_uuid: ProjectID + metadata: MetadataDict = Field( + default_factory=dict, description="Custom key-value map" + ) + + +class ProjectCustomMetadataReplace(InputSchema): + metadata: MetadataDict diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py index a8e4291a6e9..cdacf8b4948 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py @@ -7,10 +7,13 @@ async def get_project_custom_metadata( app: web.Application, user_id: UserID, project_id: ProjectID ) -> MetadataDict: - ... + # check if user_id has access to get + # check if project_id exists + # get metadata + raise NotImplementedError async def set_project_custom_metadata( app: web.Application, user_id: UserID, project_id: ProjectID, value: MetadataDict ) -> MetadataDict: - ... + raise NotImplementedError diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py index 608800a1cc8..92490e8d248 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py @@ -1,18 +1,16 @@ -from simcore_postgres_database.utils_projects_metadata import ( - ProjectJobMetadata, - ProjectJobMetadataNotFoundError, - ProjectJobMetadataRepo, - ProjectNotFoundError, -) +from aiopg.sa.engine import Engine +from models_library.api_schemas_webserver.projects_metadata import MetadataDict +from models_library.projects import ProjectID +from models_library.users import UserID -assert ProjectJobMetadata # nosec -assert ProjectJobMetadataRepo # nosec -assert ProjectJobMetadataNotFoundError # nosec -assert ProjectNotFoundError # nosec +async def get_project_metadata( + engine: Engine, user_id: UserID, project_uuid: ProjectID +) -> MetadataDict: + raise NotImplementedError -__all__: tuple[str, ...] = ( - "ProjectJobMetadata", - "ProjectJobMetadataRepo", - "ProjectJobMetadataNotFoundError", -) + +async def upsert_project_metadata( + engine: Engine, user_id: UserID, project_id: ProjectID, metadata: MetadataDict +) -> MetadataDict: + raise NotImplementedError diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py index 477e74556b4..53693e285f5 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py @@ -39,7 +39,8 @@ async def test_custom_metadata_handlers( ) response = await client.get(f"{url}") - await assert_status(response, expected_cls=web.HTTPNotFound) + _, error = await assert_status(response, expected_cls=web.HTTPNotFound) + assert "project" in error["errors"]["message"] # get metadata of an existing project the first time -> empty {} url = client.app.router["get_project_custom_metadata"].url_for( From bc74f95d5c0dea10147cbdb46f5293f995845b18 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 5 Jul 2023 17:44:55 +0200 Subject: [PATCH 24/53] migrations --- ...3285aff5e84_new_projects_metadata_table.py | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/f3285aff5e84_new_projects_metadata_table.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/f3285aff5e84_new_projects_metadata_table.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/f3285aff5e84_new_projects_metadata_table.py new file mode 100644 index 00000000000..9faa3198f0a --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/f3285aff5e84_new_projects_metadata_table.py @@ -0,0 +1,98 @@ +"""new projects_metadata table + +Revision ID: f3285aff5e84 +Revises: 58b24613c3f7 +Create Date: 2023-07-05 15:06:56.003418+00:00 + +""" +from typing import Final + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "f3285aff5e84" +down_revision = "58b24613c3f7" +branch_labels = None +depends_on = None + + +# auto-update modified +# TRIGGERS ------------------------ +_TABLE_NAME: Final[str] = "projects_metadata" +_TRIGGER_NAME: Final[str] = "trigger_auto_update" # NOTE: scoped on table +_PROCEDURE_NAME: Final[ + str +] = f"{_TABLE_NAME}_auto_update_modified()" # NOTE: scoped on database +modified_timestamp_trigger = sa.DDL( + f""" +DROP TRIGGER IF EXISTS {_TRIGGER_NAME} on {_TABLE_NAME}; +CREATE TRIGGER {_TRIGGER_NAME} +BEFORE INSERT OR UPDATE ON {_TABLE_NAME} +FOR EACH ROW EXECUTE PROCEDURE {_PROCEDURE_NAME}; + """ +) + +# PROCEDURES ------------------------ +update_modified_timestamp_procedure = sa.DDL( + f""" +CREATE OR REPLACE FUNCTION {_PROCEDURE_NAME} +RETURNS TRIGGER AS $$ +BEGIN + NEW.modified := current_timestamp; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + """ +) + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "projects_metadata", + sa.Column("project_uuid", sa.String(), nullable=False), + sa.Column( + "custom_metadata", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + sa.Column( + "created", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["project_uuid"], + ["projects.uuid"], + name="fk_projects_metadata_project_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("project_uuid"), + ) + # ### end Alembic commands ### + + # custom + update_modified_timestamp_procedure.execute(bind=op.get_context().bind) + modified_timestamp_trigger.execute(bind=op.get_context().bind) + + +def downgrade(): + + # custom + op.execute(f"DROP TRIGGER IF EXISTS {_TRIGGER_NAME} on {_TABLE_NAME};") + op.execute(f"DROP FUNCTION {_PROCEDURE_NAME};") + + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("projects_metadata") + # ### end Alembic commands ### From f7266dad249cbb96c7785f1424011a2d6d073b50 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 5 Jul 2023 18:37:45 +0200 Subject: [PATCH 25/53] project metadata repo ready --- .../utils_projects_metadata.py | 169 ++++++------------ .../tests/test_utils_projects_metadata.py | 123 ++++--------- 2 files changed, 91 insertions(+), 201 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py index 61414a7488b..4da303994f2 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py @@ -5,10 +5,12 @@ import sqlalchemy as sa from aiopg.sa.connection import SAConnection -from aiopg.sa.results import ResultProxy -from simcore_postgres_database.models.projects_metadata import projects_jobs_metadata +from aiopg.sa.result import ResultProxy, RowProxy +from simcore_postgres_database.models.projects_metadata import projects_metadata +from sqlalchemy.dialects.postgresql import insert as pg_insert from .errors import ForeignKeyViolation +from .models.projects import projects from .utils_models import FromRowMixin # @@ -20,26 +22,16 @@ class ProjectNotFoundError(Exception): code = "projects.not_found" -class BaseProjectJobMetadataError(Exception): - ... - - -class ProjectJobMetadataNotFoundError(BaseProjectJobMetadataError): - code = "projects.job_metadata.not_found" - - # # Data # @dataclass(frozen=True, slots=True, kw_only=True) -class ProjectJobMetadata(FromRowMixin): - project_uuid: uuid.UUID - parent_name: str - job_metadata: dict[str, Any] = {} - created: datetime.datetime - modified: datetime.datetime +class ProjectMetadata(FromRowMixin): + custom_metadata: dict[str, Any] | None + created: datetime.datetime | None + modified: datetime.datetime | None # @@ -47,106 +39,57 @@ class ProjectJobMetadata(FromRowMixin): # -@dataclass(frozen=True, slots=True, kw_only=True) -class ProjectJobMetadataRepo: - api_vtag: str = "v0" - - def _get_parent_name( - self, - service_key: str, - service_version: str, - ): - return f"/{self.api_vtag}/solvers/{service_key}/releases/{service_version}" - - async def create_solver_job( - self, - connection: SAConnection, - project_uuid: uuid.UUID, # pk - service_key: str, - service_version: str, - job_metadata: dict[str, Any] | None = None, - ) -> ProjectJobMetadata: - - values: dict[str, Any] = { - "project_uuid": project_uuid, - "parent_name": self._get_parent_name(service_key, service_version), - } - if job_metadata: - values["job_metadata"] = job_metadata - - try: - insert_stmt = ( - projects_jobs_metadata.insert() - .values(**values) - .returning(*[projects_jobs_metadata.columns.keys()]) +class ProjectMetadataRepo: + @staticmethod + async def get(connection: SAConnection, project_uuid: uuid.UUID) -> ProjectMetadata: + # JOIN LEFT OUTER + get_stmt = ( + sa.select( + projects.c.uuid, + projects_metadata.c.custom_metadata, + projects_metadata.c.created, + projects_metadata.c.modified, ) - result: ResultProxy = await connection.execute(insert_stmt) - row = await result.first() - - return ProjectJobMetadata.from_row(row) - - except ForeignKeyViolation as exc: - msg = f"Cannot create metadata without a valid project {project_uuid=}" - raise ProjectNotFoundError(msg) from exc - - async def list_solver_jobs( - self, - connection: SAConnection, - service_key: str, - service_version: str, - limit: int, - offset: int, - ) -> list[ProjectJobMetadata]: - assert limit > 0 # nosec - assert offset >= 0 # nosec - - parent_name = self._get_parent_name(service_key, service_version) - list_stmt = ( - sa.select(projects_jobs_metadata) - .where(projects_jobs_metadata.c.parent_name == parent_name) - .order_by(projects_jobs_metadata.c.created_at) - .offset(offset) - .limit(limit) - ) - result: ResultProxy = await connection.execute(list_stmt) - rows = await result.fetchall() - return [ProjectJobMetadata.from_row(row) for row in rows] - - async def get( - self, connection: SAConnection, project_uuid: uuid.UUID - ) -> ProjectJobMetadata: - get_stmt = sa.select(projects_jobs_metadata).where( - projects_jobs_metadata.c.project_uuid == f"{project_uuid}" + .select_from( + sa.join( + projects, + projects_metadata, + projects.c.uuid == projects_metadata.c.project_uuid, + isouter=True, + ) + ) + .where(projects.c.uuid == f"{project_uuid}") ) result: ResultProxy = await connection.execute(get_stmt) - if row := await result.first(): - return ProjectJobMetadata.from_row(row) - raise ProjectJobMetadataNotFoundError - - async def update( - self, + row: RowProxy | None = await result.first() + if row is None: + msg = f"Project project_uuid={project_uuid!r} not found" + raise ProjectNotFoundError(msg) + return ProjectMetadata.from_row(row) + + @staticmethod + async def upsert( connection: SAConnection, *, project_uuid: uuid.UUID, - job_metadata: dict[str, Any], - ) -> ProjectJobMetadata: - - update_stmt = ( - projects_jobs_metadata.update() - .where(projects_jobs_metadata.c.project_uuid == project_uuid) - .values(projects_jobs_metadata.c.job_metadata == job_metadata) - .returning(*list(projects_jobs_metadata.columns)) - ) - result = await connection.execute(update_stmt) - if row := await result.first(): - return ProjectJobMetadata.from_row(row) - raise ProjectJobMetadataNotFoundError - - async def delete(self, connection: SAConnection, project_uuid: uuid.UUID) -> None: - delete_stmt = sa.delete(projects_jobs_metadata).where( - projects_jobs_metadata.c.project_uuid == f"{project_uuid}" - ) - result = await connection.execute(delete_stmt) - if result.rowcount: - msg = f"Could not delete non-existing metadata of project_uuid={project_uuid!r}" - raise ProjectJobMetadataNotFoundError(msg) + custom_metadata: dict[str, Any], + ) -> ProjectMetadata: + data = { + "project_uuid": f"{project_uuid}", + "custom_metadata": custom_metadata, + } + + try: + insert_stmt = pg_insert(projects_metadata).values(**data) + upsert_stmt = insert_stmt.on_conflict_do_update( + index_elements=[projects_metadata.c.project_uuid], + set_=data, + ).returning(sa.literal_column("*")) + + result: ResultProxy = await connection.execute(upsert_stmt) + row: RowProxy | None = await result.first() + assert row # nosec + return ProjectMetadata.from_row(row) + + except ForeignKeyViolation as err: + raise ProjectNotFoundError(project_uuid) from err diff --git a/packages/postgres-database/tests/test_utils_projects_metadata.py b/packages/postgres-database/tests/test_utils_projects_metadata.py index 5965b79b239..9a9c827962b 100644 --- a/packages/postgres-database/tests/test_utils_projects_metadata.py +++ b/packages/postgres-database/tests/test_utils_projects_metadata.py @@ -4,19 +4,15 @@ # pylint: disable=too-many-arguments from collections.abc import Awaitable, Callable -from typing import Any -from uuid import UUID import pytest -import sqlalchemy as sa from aiopg.sa.connection import SAConnection -from aiopg.sa.result import ResultProxy, RowProxy -from simcore_postgres_database.models.projects import projects -from simcore_postgres_database.models.projects_metadata import ( - projects_jobs_metadata, - projects_metadata, +from aiopg.sa.result import RowProxy +from faker import Faker +from simcore_postgres_database.utils_projects_metadata import ( + ProjectMetadataRepo, + ProjectNotFoundError, ) -from sqlalchemy.dialects.postgresql import insert as pg_insert @pytest.fixture @@ -38,99 +34,50 @@ async def fake_project( return project -async def test_jobs_workflow( +async def test_projects_metadata_repository( connection: SAConnection, - create_fake_project: Callable[..., Awaitable[RowProxy]], create_fake_user: Callable[..., Awaitable[RowProxy]], + create_fake_project: Callable[..., Awaitable[RowProxy]], + faker: Faker, ): user: RowProxy = await create_fake_user(connection) + project: RowProxy = await create_fake_project(connection, user, hidden=True) - async def _create_solver_job(service_key: str, service_version: str, n: int): - parent_name = f"/v0/solvers/{service_key}/releases/{service_version}" - project: RowProxy = await create_fake_project(connection, user, hidden=True) - - query = projects_jobs_metadata.insert().values( - project_uuid=project.uuid, - parent_name=parent_name, - job_metadata={ - "__type__": "JobMeta", - "inputs_checksum": f"{n}2bfd4885aa1daf5c16fdd39b9118f652c4977c4021c900794dc125cf123718e", - "created_at": f"2022-06-01T15:{n}:56.807441", - }, - ) - result: ResultProxy = await connection.execute(query) - assert result - return project.uuid - - # some project from the UI - project_study = await create_fake_project(connection, user, hidden=True) - - # CREATE - # some solver-job projects - created_jobs: list[UUID] = [ - await _create_solver_job( - service_key="simcore/comp/itis/sleeper", service_version="2.0.0", n=n + # subresource is attached to parent + user_metadata = {"float": 3.14, "int": 42, "string": "foo", "bool": True} + + with pytest.raises(ProjectNotFoundError): + await ProjectMetadataRepo.get(connection, project_uuid=faker.uuid4()) + + with pytest.raises(ProjectNotFoundError): + await ProjectMetadataRepo.upsert( + connection, project_uuid=faker.uuid4(), custom_metadata=user_metadata ) - for n in range(3) - ] - assert project_study.uuid not in set(created_jobs) + pm = await ProjectMetadataRepo.get(connection, project_uuid=project["uuid"]) + assert pm is not None + assert pm.custom_metadata is None - # READ - async def _list_solver_jobs(service_key: str, service_version: str): - # list jobs of a solver - parent_name = f"/v0/solvers/{service_key}/releases/{service_version}" + got = await ProjectMetadataRepo.upsert( + connection, project_uuid=project["uuid"], custom_metadata=user_metadata + ) + assert got.custom_metadata + assert user_metadata == got.custom_metadata - j = projects.join( - projects_jobs_metadata, - (projects.c.uuid == projects_jobs_metadata.c.project_uuid), - ) - query = ( - sa.select(projects_jobs_metadata, projects.c.hidden) - .select_from(j) - .where(projects_jobs_metadata.c.parent_name == parent_name) - ) - jobs = await (await connection.execute(query)).fetchall() - assert jobs - return jobs + pm = await ProjectMetadataRepo.get(connection, project_uuid=project["uuid"]) + assert pm is not None + assert pm == got - got_jobs = await _list_solver_jobs( - service_key="simcore/comp/itis/sleeper", service_version="2.0.0" + got2 = await ProjectMetadataRepo.upsert( + connection, project_uuid=project["uuid"], custom_metadata={} ) - assert {j.project_uuid for j in got_jobs} == set(created_jobs) - assert all(j.hidden for j in got_jobs) + assert got2.custom_metadata == {} + assert got.modified + assert got2.modified + assert got.modified < got2.modified # list jobs of a study # list all jobs of a user # list projects that are non-jobs - async def _upsert_custom_metadata(project_uuid, metadata: dict[str, Any]): - params = dict( - project_uuid=f"{project_uuid}", - custom_metadata=metadata, - ) - insert_stmt = pg_insert(projects_metadata).values(**params) - on_update_stmt = insert_stmt.on_conflict_do_update( - index_elements=[ - projects_metadata.c.project_uuid, - ], - set_=params, - ) - await connection.execute(on_update_stmt) - - # UPDATE custom - meta - await _upsert_custom_metadata(project_study.uuid, metadata={"my data": "foo"}) - await _upsert_custom_metadata( - got_jobs[0].project_uuid, metadata={"jobs data": "bar"} - ) - # DELETE job by deleting project - - -# test create a job -# -# - from a study -# - from a solver -# - search jobs from a study, from a solver, etc -# - list studies -> projects uuids that are not jobs -# - list study jobs -> projects uuids that are Jodbs From eb49d72f51f77ce399aa32fb0c7c28fb3a5683c3 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 6 Jul 2023 10:13:46 +0200 Subject: [PATCH 26/53] db --- .../simcore_postgres_database/utils_projects_metadata.py | 6 +++--- .../postgres-database/tests/test_utils_projects_metadata.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py index 4da303994f2..72e0d2f0dd4 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py @@ -18,7 +18,7 @@ # -class ProjectNotFoundError(Exception): +class DBProjectNotFoundError(Exception): code = "projects.not_found" @@ -64,7 +64,7 @@ async def get(connection: SAConnection, project_uuid: uuid.UUID) -> ProjectMetad row: RowProxy | None = await result.first() if row is None: msg = f"Project project_uuid={project_uuid!r} not found" - raise ProjectNotFoundError(msg) + raise DBProjectNotFoundError(msg) return ProjectMetadata.from_row(row) @staticmethod @@ -92,4 +92,4 @@ async def upsert( return ProjectMetadata.from_row(row) except ForeignKeyViolation as err: - raise ProjectNotFoundError(project_uuid) from err + raise DBProjectNotFoundError(project_uuid) from err diff --git a/packages/postgres-database/tests/test_utils_projects_metadata.py b/packages/postgres-database/tests/test_utils_projects_metadata.py index 9a9c827962b..fec9d4e1d86 100644 --- a/packages/postgres-database/tests/test_utils_projects_metadata.py +++ b/packages/postgres-database/tests/test_utils_projects_metadata.py @@ -10,8 +10,8 @@ from aiopg.sa.result import RowProxy from faker import Faker from simcore_postgres_database.utils_projects_metadata import ( + DBProjectNotFoundError, ProjectMetadataRepo, - ProjectNotFoundError, ) @@ -46,10 +46,10 @@ async def test_projects_metadata_repository( # subresource is attached to parent user_metadata = {"float": 3.14, "int": 42, "string": "foo", "bool": True} - with pytest.raises(ProjectNotFoundError): + with pytest.raises(DBProjectNotFoundError): await ProjectMetadataRepo.get(connection, project_uuid=faker.uuid4()) - with pytest.raises(ProjectNotFoundError): + with pytest.raises(DBProjectNotFoundError): await ProjectMetadataRepo.upsert( connection, project_uuid=faker.uuid4(), custom_metadata=user_metadata ) From 7769d4ba6da80f59c5cf1ea619bbce162c3d9dd9 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 6 Jul 2023 10:16:23 +0200 Subject: [PATCH 27/53] access rights --- .../projects/_access_rights_api.py | 15 +++++++++ .../projects/_access_rights_db.py | 19 +++++++++++ .../projects/_metadata_api.py | 25 +++++++++++---- .../projects/_metadata_db.py | 32 +++++++++++++++---- 4 files changed, 77 insertions(+), 14 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/projects/_access_rights_api.py create mode 100644 services/web/server/src/simcore_service_webserver/projects/_access_rights_db.py diff --git a/services/web/server/src/simcore_service_webserver/projects/_access_rights_api.py b/services/web/server/src/simcore_service_webserver/projects/_access_rights_api.py new file mode 100644 index 00000000000..5d920c649b3 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/projects/_access_rights_api.py @@ -0,0 +1,15 @@ +from aiohttp import web +from models_library.projects import ProjectID +from models_library.users import UserID + +from ..db.plugin import get_database_engine +from ._access_rights_db import get_project_owner +from .exceptions import ProjectInvalidRightsError + + +async def check_project_ownership( + app: web.Application, user_id: UserID, project_uuid: ProjectID +): + async with get_database_engine(app).acquire() as conn: + if await get_project_owner(conn, project_uuid=project_uuid) != user_id: + raise ProjectInvalidRightsError(user_id=user_id, project_uuid=project_uuid) diff --git a/services/web/server/src/simcore_service_webserver/projects/_access_rights_db.py b/services/web/server/src/simcore_service_webserver/projects/_access_rights_db.py new file mode 100644 index 00000000000..e49ec0613cf --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/projects/_access_rights_db.py @@ -0,0 +1,19 @@ +import sqlalchemy +from aiopg.sa.connection import SAConnection +from models_library.projects import ProjectID +from models_library.users import UserID +from simcore_postgres_database.models.projects import projects +from simcore_postgres_database.utils_projects_metadata import DBProjectNotFoundError + + +async def get_project_owner( + connection: SAConnection, project_uuid: ProjectID +) -> UserID: + stmt = sqlalchemy.select(projects.c.prj_owner).where( + projects.c.uuid == f"{project_uuid}" + ) + + owner_id = await connection.scalar(stmt) + if owner_id is None: + raise DBProjectNotFoundError(project_uuid) + return owner_id diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py index cdacf8b4948..87fa4ebfaa9 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py @@ -3,17 +3,28 @@ from models_library.projects import ProjectID from models_library.users import UserID +from ..db.plugin import get_database_engine +from ._access_rights_api import check_project_ownership +from ._metadata_db import get_project_metadata, upsert_project_metadata + async def get_project_custom_metadata( - app: web.Application, user_id: UserID, project_id: ProjectID + app: web.Application, user_id: UserID, project_uuid: ProjectID ) -> MetadataDict: - # check if user_id has access to get - # check if project_id exists - # get metadata - raise NotImplementedError + await check_project_ownership(app, user_id=user_id, project_uuid=project_uuid) + + return await get_project_metadata( + engine=get_database_engine(app), project_uuid=project_uuid + ) async def set_project_custom_metadata( - app: web.Application, user_id: UserID, project_id: ProjectID, value: MetadataDict + app: web.Application, user_id: UserID, project_uuid: ProjectID, value: MetadataDict ) -> MetadataDict: - raise NotImplementedError + await check_project_ownership(app, user_id=user_id, project_uuid=project_uuid) + + return await upsert_project_metadata( + engine=get_database_engine(app), + project_uuid=project_uuid, + custom_metadata=value, + ) diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py index 92490e8d248..59d96295da6 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py @@ -1,16 +1,34 @@ from aiopg.sa.engine import Engine from models_library.api_schemas_webserver.projects_metadata import MetadataDict from models_library.projects import ProjectID -from models_library.users import UserID +from pydantic import parse_obj_as +from simcore_postgres_database.utils_projects_metadata import ( + DBProjectNotFoundError, + ProjectMetadataRepo, +) -async def get_project_metadata( - engine: Engine, user_id: UserID, project_uuid: ProjectID -) -> MetadataDict: - raise NotImplementedError +async def get_project_metadata(engine: Engine, project_uuid: ProjectID) -> MetadataDict: + """ + Raises: + ProjectNotFoundError + """ + async with engine.acquire() as connection: + pm = await ProjectMetadataRepo.get(connection, project_uuid=project_uuid) + return parse_obj_as(MetadataDict, pm.custom_metadata) async def upsert_project_metadata( - engine: Engine, user_id: UserID, project_id: ProjectID, metadata: MetadataDict + engine: Engine, + project_uuid: ProjectID, + custom_metadata: MetadataDict, ) -> MetadataDict: - raise NotImplementedError + async with engine.acquire() as connection: + project_metadata = await ProjectMetadataRepo.upsert( + connection, project_uuid=project_uuid, custom_metadata=custom_metadata + ) + return parse_obj_as(MetadataDict, project_metadata.custom_metadata) + + +assert DBProjectNotFoundError # nosec +__all__: tuple[str, ...] = ("DBProjectNotFoundError",) From b9397c626f990d77b626fb6a60dc5bff3b694166 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 6 Jul 2023 14:06:58 +0200 Subject: [PATCH 28/53] error handling --- .../projects/_access_rights_api.py | 14 ++++++--- .../projects/_access_rights_db.py | 21 +++++++------ .../projects/_metadata_api.py | 6 ++-- .../projects/_metadata_db.py | 22 ++++++++++---- .../projects/_metadata_handlers.py | 30 ++++++++++++++++--- .../projects/plugin.py | 6 ++-- 6 files changed, 68 insertions(+), 31 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_access_rights_api.py b/services/web/server/src/simcore_service_webserver/projects/_access_rights_api.py index 5d920c649b3..f8b8b5010b6 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_access_rights_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_access_rights_api.py @@ -7,9 +7,15 @@ from .exceptions import ProjectInvalidRightsError -async def check_project_ownership( +async def validate_project_ownership( app: web.Application, user_id: UserID, project_uuid: ProjectID ): - async with get_database_engine(app).acquire() as conn: - if await get_project_owner(conn, project_uuid=project_uuid) != user_id: - raise ProjectInvalidRightsError(user_id=user_id, project_uuid=project_uuid) + """ + Raises: + ProjectInvalidRightsError: if not project is not owned by user_id + """ + if ( + await get_project_owner(get_database_engine(app), project_uuid=project_uuid) + != user_id + ): + raise ProjectInvalidRightsError(user_id=user_id, project_uuid=project_uuid) diff --git a/services/web/server/src/simcore_service_webserver/projects/_access_rights_db.py b/services/web/server/src/simcore_service_webserver/projects/_access_rights_db.py index e49ec0613cf..65be43edd77 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_access_rights_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_access_rights_db.py @@ -1,19 +1,18 @@ import sqlalchemy -from aiopg.sa.connection import SAConnection +from aiopg.sa.engine import Engine from models_library.projects import ProjectID from models_library.users import UserID from simcore_postgres_database.models.projects import projects from simcore_postgres_database.utils_projects_metadata import DBProjectNotFoundError -async def get_project_owner( - connection: SAConnection, project_uuid: ProjectID -) -> UserID: - stmt = sqlalchemy.select(projects.c.prj_owner).where( - projects.c.uuid == f"{project_uuid}" - ) +async def get_project_owner(engine: Engine, project_uuid: ProjectID) -> UserID: + async with engine.acquire() as connection: + stmt = sqlalchemy.select(projects.c.prj_owner).where( + projects.c.uuid == f"{project_uuid}" + ) - owner_id = await connection.scalar(stmt) - if owner_id is None: - raise DBProjectNotFoundError(project_uuid) - return owner_id + owner_id = await connection.scalar(stmt) + if owner_id is None: + raise DBProjectNotFoundError(project_uuid) + return owner_id diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py index 87fa4ebfaa9..a9e0a95e2b3 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py @@ -4,14 +4,14 @@ from models_library.users import UserID from ..db.plugin import get_database_engine -from ._access_rights_api import check_project_ownership +from ._access_rights_api import validate_project_ownership from ._metadata_db import get_project_metadata, upsert_project_metadata async def get_project_custom_metadata( app: web.Application, user_id: UserID, project_uuid: ProjectID ) -> MetadataDict: - await check_project_ownership(app, user_id=user_id, project_uuid=project_uuid) + await validate_project_ownership(app, user_id=user_id, project_uuid=project_uuid) return await get_project_metadata( engine=get_database_engine(app), project_uuid=project_uuid @@ -21,7 +21,7 @@ async def get_project_custom_metadata( async def set_project_custom_metadata( app: web.Application, user_id: UserID, project_uuid: ProjectID, value: MetadataDict ) -> MetadataDict: - await check_project_ownership(app, user_id=user_id, project_uuid=project_uuid) + await validate_project_ownership(app, user_id=user_id, project_uuid=project_uuid) return await upsert_project_metadata( engine=get_database_engine(app), diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py index 59d96295da6..8d09c39330e 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py @@ -7,6 +7,8 @@ ProjectMetadataRepo, ) +from .exceptions import ProjectNotFoundError + async def get_project_metadata(engine: Engine, project_uuid: ProjectID) -> MetadataDict: """ @@ -14,8 +16,12 @@ async def get_project_metadata(engine: Engine, project_uuid: ProjectID) -> Metad ProjectNotFoundError """ async with engine.acquire() as connection: - pm = await ProjectMetadataRepo.get(connection, project_uuid=project_uuid) - return parse_obj_as(MetadataDict, pm.custom_metadata) + try: + pm = await ProjectMetadataRepo.get(connection, project_uuid=project_uuid) + return parse_obj_as(MetadataDict, pm.custom_metadata) + + except DBProjectNotFoundError as err: + raise ProjectNotFoundError(project_uuid=project_uuid) from err async def upsert_project_metadata( @@ -24,10 +30,14 @@ async def upsert_project_metadata( custom_metadata: MetadataDict, ) -> MetadataDict: async with engine.acquire() as connection: - project_metadata = await ProjectMetadataRepo.upsert( - connection, project_uuid=project_uuid, custom_metadata=custom_metadata - ) - return parse_obj_as(MetadataDict, project_metadata.custom_metadata) + try: + pm = await ProjectMetadataRepo.upsert( + connection, project_uuid=project_uuid, custom_metadata=custom_metadata + ) + return parse_obj_as(MetadataDict, pm.custom_metadata) + + except DBProjectNotFoundError as err: + raise ProjectNotFoundError(project_uuid=project_uuid) from err assert DBProjectNotFoundError # nosec diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py index beab83487f6..1d6f2db0547 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py @@ -11,6 +11,8 @@ """ +import functools + from aiohttp import web from models_library.api_schemas_webserver.projects_metadata import ( ProjectCustomMetadataGet, @@ -20,6 +22,7 @@ parse_request_body_as, parse_request_path_parameters_as, ) +from servicelib.aiohttp.typing_extension import Handler from simcore_service_webserver.utils_aiohttp import envelope_json_response from .._meta import api_version_prefix as VTAG @@ -27,10 +30,27 @@ from ..security.decorators import permission_required from . import _metadata_api from ._common_models import ProjectPathParams, RequestContext +from .exceptions import ProjectInvalidRightsError, ProjectNotFoundError routes = web.RouteTableDef() +def _handle_project_exceptions(handler: Handler): + """Transforms project errors -> http errors""" + + @functools.wraps(handler) + async def wrapper(request: web.Request) -> web.StreamResponse: + try: + return await handler(request) + + except ProjectNotFoundError as exc: + raise web.HTTPNotFound(reason=f"{exc}") from exc + except ProjectInvalidRightsError as exc: + raise web.HTTPUnauthorized(reason=f"{exc}") from exc + + return wrapper + + # # projects/*/custom-metadata # @@ -42,12 +62,13 @@ ) @login_required @permission_required("project.read") +@_handle_project_exceptions async def get_project_custom_metadata(request: web.Request) -> web.Response: req_ctx = RequestContext.parse_obj(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) custom_metadata = await _metadata_api.get_project_custom_metadata( - request.app, user_id=req_ctx.user_id, project_id=path_params.project_id + request.app, user_id=req_ctx.user_id, project_uuid=path_params.project_id ) return envelope_json_response( @@ -63,16 +84,17 @@ async def get_project_custom_metadata(request: web.Request) -> web.Response: ) @login_required @permission_required("project.update") +@_handle_project_exceptions async def replace_project_custom_metadata(request: web.Request) -> web.Response: req_ctx = RequestContext.parse_obj(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) - new_metadata = await parse_request_body_as(ProjectCustomMetadataReplace, request) + replace = await parse_request_body_as(ProjectCustomMetadataReplace, request) custom_metadata = await _metadata_api.set_project_custom_metadata( request.app, user_id=req_ctx.user_id, - project_id=path_params.project_id, - value=new_metadata, + project_uuid=path_params.project_id, + value=replace.metadata, ) return envelope_json_response( diff --git a/services/web/server/src/simcore_service_webserver/projects/plugin.py b/services/web/server/src/simcore_service_webserver/projects/plugin.py index 93f6dcdb389..182de9fc0b1 100644 --- a/services/web/server/src/simcore_service_webserver/projects/plugin.py +++ b/services/web/server/src/simcore_service_webserver/projects/plugin.py @@ -18,6 +18,7 @@ _comments_handlers, _handlers, _handlers_crud, + _metadata_handlers, _nodes_handlers, _ports_handlers, _tags_handlers, @@ -34,7 +35,7 @@ def _create_routes(tag, specs, *handlers_module): for mod in handlers_module: handlers.update(get_handlers_from_namespace(mod)) - routes = map_handlers_with_operations( + return map_handlers_with_operations( handlers, filter( lambda o: tag in o.tags and "snapshot" not in o.path, @@ -43,8 +44,6 @@ def _create_routes(tag, specs, *handlers_module): strict=False, ) - return routes - @app_module_setup( "simcore_service_webserver.projects", @@ -71,6 +70,7 @@ def setup_projects(app: web.Application) -> bool: app.router.add_routes(_handlers.routes) app.router.add_routes(_handlers_crud.routes) app.router.add_routes(_comments_handlers.routes) + app.router.add_routes(_metadata_handlers.routes) app.router.add_routes(_ports_handlers.routes) app.router.add_routes( From 6e8af4b4a5eae8d14658eb978a8f46512b9785a0 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 6 Jul 2023 14:08:28 +0200 Subject: [PATCH 29/53] rename error --- .../exporter/_formatter/_sds.py | 4 ++-- .../projects/_handlers.py | 8 +++---- .../projects/_nodes_api.py | 4 ++-- .../projects/_nodes_handlers.py | 4 ++-- .../projects/exceptions.py | 24 +++++++++---------- .../projects/projects_api.py | 10 ++++---- 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py b/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py index d89e65e8221..b7f6e2f5fbd 100644 --- a/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py +++ b/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py @@ -8,7 +8,7 @@ from servicelib.pools import non_blocking_process_pool_executor from ...catalog.client import get_service -from ...projects.exceptions import ProjectsException +from ...projects.exceptions import BaseProjectError from ...projects.projects_api import get_project_for_user from ...scicrunch.db import ResearchResourceRepository from ..exceptions import SDSException @@ -56,7 +56,7 @@ async def create_sds_directory( user_id=user_id, include_state=True, ) - except ProjectsException as e: + except BaseProjectError as e: raise SDSException(f"Could not find project {project_id}") from e _logger.debug("Project data: %s", project_data) diff --git a/services/web/server/src/simcore_service_webserver/projects/_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_handlers.py index e840574a8ba..ce2324f04ed 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_handlers.py @@ -37,8 +37,8 @@ from .exceptions import ( ProjectInvalidRightsError, ProjectNotFoundError, - ProjectStartsTooManyDynamicNodes, - ProjectTooManyProjectOpened, + ProjectStartsTooManyDynamicNodesError, + ProjectTooManyProjectOpenedError, ) log = logging.getLogger(__name__) @@ -105,7 +105,7 @@ async def open_project(request: web.Request) -> web.Response: # user id opened project uuid if not query_params.disable_service_auto_start: - with contextlib.suppress(ProjectStartsTooManyDynamicNodes): + with contextlib.suppress(ProjectStartsTooManyDynamicNodesError): # NOTE: this method raises that exception when the number of dynamic # services in the project is highter than the maximum allowed per project # the project shall still open though. @@ -148,7 +148,7 @@ async def open_project(request: web.Request) -> web.Response: raise web.HTTPServiceUnavailable( reason="Unexpected error while starting services." ) from exc - except ProjectTooManyProjectOpened as exc: + except ProjectTooManyProjectOpenedError as exc: raise web.HTTPConflict(reason=f"{exc}") from exc except ProjectInvalidRightsError as exc: raise web.HTTPForbidden( diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_api.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_api.py index 169e9c3f790..15b04017c06 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_api.py @@ -24,7 +24,7 @@ from .._constants import APP_SETTINGS_KEY, RQT_USERID_KEY from ..application_settings import get_settings from ..storage.api import get_download_link -from .exceptions import ProjectStartsTooManyDynamicNodes +from .exceptions import ProjectStartsTooManyDynamicNodesError _logger = logging.getLogger(__name__) _NODE_START_INTERVAL_S: Final[datetime.timedelta] = datetime.timedelta(seconds=15) @@ -48,7 +48,7 @@ def check_num_service_per_projects_limit( if project_settings.PROJECTS_MAX_NUM_RUNNING_DYNAMIC_NODES > 0 and ( number_of_services >= project_settings.PROJECTS_MAX_NUM_RUNNING_DYNAMIC_NODES ): - raise ProjectStartsTooManyDynamicNodes( + raise ProjectStartsTooManyDynamicNodesError( user_id=user_id, project_uuid=project_uuid ) diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py index f2591d96546..4c6681ea153 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py @@ -55,7 +55,7 @@ ProjectNodeResourcesInsufficientRightsError, ProjectNodeResourcesInvalidError, ProjectNotFoundError, - ProjectStartsTooManyDynamicNodes, + ProjectStartsTooManyDynamicNodesError, ) log = logging.getLogger(__name__) @@ -244,7 +244,7 @@ async def start_node(request: web.Request) -> web.Response: raise web.HTTPNoContent(content_type=MIMETYPE_APPLICATION_JSON) - except ProjectStartsTooManyDynamicNodes as exc: + except ProjectStartsTooManyDynamicNodesError as exc: raise web.HTTPConflict(reason=f"{exc}") from exc diff --git a/services/web/server/src/simcore_service_webserver/projects/exceptions.py b/services/web/server/src/simcore_service_webserver/projects/exceptions.py index a45fe87893c..83d7b47f24d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/exceptions.py +++ b/services/web/server/src/simcore_service_webserver/projects/exceptions.py @@ -6,7 +6,7 @@ from models_library.users import UserID -class ProjectsException(Exception): +class BaseProjectError(Exception): """Basic exception for errors raised in projects""" def __init__(self, msg=None): @@ -17,7 +17,7 @@ def detailed_message(self): return f"{type(self)}: {self}" -class ProjectInvalidRightsError(ProjectsException): +class ProjectInvalidRightsError(BaseProjectError): """Invalid rights to access project""" def __init__(self, user_id, project_uuid): @@ -28,7 +28,7 @@ def __init__(self, user_id, project_uuid): self.project_uuid = project_uuid -class ProjectOwnerNotFoundError(ProjectsException): +class ProjectOwnerNotFoundError(BaseProjectError): """Project owner was not found""" def __init__(self, project_uuid): @@ -36,7 +36,7 @@ def __init__(self, project_uuid): self.project_uuid = project_uuid -class ProjectNotFoundError(ProjectsException): +class ProjectNotFoundError(BaseProjectError): """Project was not found in DB""" def __init__(self, project_uuid, *, search_context: Any | None = None): @@ -52,13 +52,13 @@ def detailed_message(self): return msg -class ProjectDeleteError(ProjectsException): +class ProjectDeleteError(BaseProjectError): def __init__(self, project_uuid, reason): super().__init__(f"Failed to complete deletion of {project_uuid=}: {reason}") self.project_uuid = project_uuid -class NodeNotFoundError(ProjectsException): +class NodeNotFoundError(BaseProjectError): """Node was not found in project""" def __init__(self, project_uuid: str, node_uuid: str): @@ -70,7 +70,7 @@ def __init__(self, project_uuid: str, node_uuid: str): ProjectLockError = redis.exceptions.LockError -class ProjectStartsTooManyDynamicNodes(ProjectsException): +class ProjectStartsTooManyDynamicNodesError(BaseProjectError): """user tried to start too many nodes concurrently""" def __init__(self, user_id: UserID, project_uuid: ProjectID): @@ -81,24 +81,24 @@ def __init__(self, user_id: UserID, project_uuid: ProjectID): self.project_uuid = project_uuid -class ProjectTooManyProjectOpened(ProjectsException): +class ProjectTooManyProjectOpenedError(BaseProjectError): def __init__(self, max_num_projects: int): super().__init__( f"You cannot open more than {max_num_projects} stud{'y' if max_num_projects == 1 else 'ies'} at once. Please close another study and retry." ) -class PermalinkNotAllowedError(ProjectsException): +class PermalinkNotAllowedError(BaseProjectError): ... -class PermalinkFactoryError(ProjectsException): +class PermalinkFactoryError(BaseProjectError): ... -class ProjectNodeResourcesInvalidError(ProjectsException): +class ProjectNodeResourcesInvalidError(BaseProjectError): ... -class ProjectNodeResourcesInsufficientRightsError(ProjectsException): +class ProjectNodeResourcesInsufficientRightsError(BaseProjectError): ... diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_api.py b/services/web/server/src/simcore_service_webserver/projects/projects_api.py index 802874ea279..b0505732d8c 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_api.py @@ -76,8 +76,8 @@ from .exceptions import ( NodeNotFoundError, ProjectLockError, - ProjectStartsTooManyDynamicNodes, - ProjectTooManyProjectOpened, + ProjectStartsTooManyDynamicNodesError, + ProjectTooManyProjectOpenedError, ) from .lock import get_project_locked_state, is_project_locked, lock_project from .models import ProjectDict @@ -324,7 +324,7 @@ async def add_project_node( ) if _is_node_dynamic(service_key): - with suppress(ProjectStartsTooManyDynamicNodes): + with suppress(ProjectStartsTooManyDynamicNodesError): # NOTE: we do not start the service if there are already too many await _start_dynamic_service( request, @@ -624,7 +624,7 @@ async def try_open_project_for_user( ) >= max_number_of_studies_per_user ): - raise ProjectTooManyProjectOpened( + raise ProjectTooManyProjectOpenedError( max_num_projects=max_number_of_studies_per_user ) @@ -991,7 +991,7 @@ async def run_project_dynamic_services( > project_settings.PROJECTS_MAX_NUM_RUNNING_DYNAMIC_NODES ): # we cannot start so many services so we are done - raise ProjectStartsTooManyDynamicNodes( + raise ProjectStartsTooManyDynamicNodesError( user_id=user_id, project_uuid=ProjectID(project["uuid"]) ) From 3dd1f291843fa7e8c8943e06506898cd91d7f102 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 6 Jul 2023 14:30:24 +0200 Subject: [PATCH 30/53] fixes test --- .../projects/_access_rights_db.py | 5 +- .../projects/_metadata_db.py | 46 +++++++++++-------- .../02/test_projects_metadata_handlers.py | 22 ++++++--- 3 files changed, 45 insertions(+), 28 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_access_rights_db.py b/services/web/server/src/simcore_service_webserver/projects/_access_rights_db.py index 65be43edd77..c6b5b45099d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_access_rights_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_access_rights_db.py @@ -3,7 +3,8 @@ from models_library.projects import ProjectID from models_library.users import UserID from simcore_postgres_database.models.projects import projects -from simcore_postgres_database.utils_projects_metadata import DBProjectNotFoundError + +from .exceptions import ProjectNotFoundError async def get_project_owner(engine: Engine, project_uuid: ProjectID) -> UserID: @@ -14,5 +15,5 @@ async def get_project_owner(engine: Engine, project_uuid: ProjectID) -> UserID: owner_id = await connection.scalar(stmt) if owner_id is None: - raise DBProjectNotFoundError(project_uuid) + raise ProjectNotFoundError(project_uuid=project_uuid) return owner_id diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py index 8d09c39330e..7d93573cc4c 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py @@ -1,3 +1,7 @@ +from contextlib import asynccontextmanager +from typing import AsyncIterator + +from aiopg.sa.connection import SAConnection from aiopg.sa.engine import Engine from models_library.api_schemas_webserver.projects_metadata import MetadataDict from models_library.projects import ProjectID @@ -10,18 +14,28 @@ from .exceptions import ProjectNotFoundError +@asynccontextmanager +async def _acquire_and_handle( + engine: Engine, project_uuid: ProjectID +) -> AsyncIterator[SAConnection]: + try: + async with engine.acquire() as connection: + + yield connection + + except DBProjectNotFoundError as err: + raise ProjectNotFoundError(project_uuid=project_uuid) from err + + async def get_project_metadata(engine: Engine, project_uuid: ProjectID) -> MetadataDict: """ Raises: ProjectNotFoundError """ - async with engine.acquire() as connection: - try: - pm = await ProjectMetadataRepo.get(connection, project_uuid=project_uuid) - return parse_obj_as(MetadataDict, pm.custom_metadata) - - except DBProjectNotFoundError as err: - raise ProjectNotFoundError(project_uuid=project_uuid) from err + async with _acquire_and_handle(engine, project_uuid) as connection: + pm = await ProjectMetadataRepo.get(connection, project_uuid=project_uuid) + # NOTE: if no metadata in table, it returns None -- which converts here to --> {} + return parse_obj_as(MetadataDict, pm.custom_metadata or {}) async def upsert_project_metadata( @@ -29,16 +43,8 @@ async def upsert_project_metadata( project_uuid: ProjectID, custom_metadata: MetadataDict, ) -> MetadataDict: - async with engine.acquire() as connection: - try: - pm = await ProjectMetadataRepo.upsert( - connection, project_uuid=project_uuid, custom_metadata=custom_metadata - ) - return parse_obj_as(MetadataDict, pm.custom_metadata) - - except DBProjectNotFoundError as err: - raise ProjectNotFoundError(project_uuid=project_uuid) from err - - -assert DBProjectNotFoundError # nosec -__all__: tuple[str, ...] = ("DBProjectNotFoundError",) + async with _acquire_and_handle(engine, project_uuid) as connection: + pm = await ProjectMetadataRepo.upsert( + connection, project_uuid=project_uuid, custom_metadata=custom_metadata + ) + return parse_obj_as(MetadataDict, pm.custom_metadata) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py index 53693e285f5..9a77d5ddd43 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py @@ -9,6 +9,11 @@ from aiohttp import web from aiohttp.test_utils import TestClient from faker import Faker +from models_library.api_schemas_webserver.projects_metadata import ( + ProjectCustomMetadataGet, + ProjectCustomMetadataReplace, +) +from pydantic import parse_obj_as from pytest_simcore.helpers.utils_assert import assert_status from pytest_simcore.helpers.utils_login import UserInfoDict from simcore_postgres_database.models.users import UserRole @@ -34,13 +39,16 @@ async def test_custom_metadata_handlers( assert client.app # get metadata of a non-existing project -> Not found + invalid_project_id = faker.uuid4() url = client.app.router["get_project_custom_metadata"].url_for( - project_id=faker.uuid4() + project_id=invalid_project_id ) response = await client.get(f"{url}") _, error = await assert_status(response, expected_cls=web.HTTPNotFound) - assert "project" in error["errors"]["message"] + error_message = error["errors"][0]["message"] + assert invalid_project_id in error_message + assert "project" in error_message.lower() # get metadata of an existing project the first time -> empty {} url = client.app.router["get_project_custom_metadata"].url_for( @@ -57,14 +65,17 @@ async def test_custom_metadata_handlers( url = client.app.router["replace_project_custom_metadata"].url_for( project_id=user_project["uuid"] ) - response = await client.put(f"{url}", json=custom_metadata) + response = await client.put( + f"{url}", json=ProjectCustomMetadataReplace(metadata=custom_metadata).dict() + ) data, _ = await assert_status(response, expected_cls=web.HTTPOk) - assert data["metadata"] == custom_metadata + + assert parse_obj_as(ProjectCustomMetadataGet, data).metadata == custom_metadata # delete project url = client.app.router["delete_project"].url_for(project_id=user_project["uuid"]) - response = await client.delete(f"{url}", json=custom_metadata) + response = await client.delete(f"{url}") await assert_status(response, expected_cls=web.HTTPNoContent) # no metadata -> project not foun d @@ -73,4 +84,3 @@ async def test_custom_metadata_handlers( ) response = await client.get(f"{url}") await assert_status(response, expected_cls=web.HTTPNotFound) - await assert_status(response, expected_cls=web.HTTPNotFound) From 19dd82670ecbecc9970cac571ff1e4b7573d48cc Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 6 Jul 2023 14:50:05 +0200 Subject: [PATCH 31/53] oas --- .../webserver/openapi-projects-metadata.yaml | 93 +++++++++++++++++++ api/specs/webserver/openapi.yaml | 3 + .../scripts/openapi_projects_metadata.py | 63 +++++++++++++ .../projects/_metadata_handlers.py | 4 +- 4 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 api/specs/webserver/openapi-projects-metadata.yaml create mode 100644 api/specs/webserver/scripts/openapi_projects_metadata.py diff --git a/api/specs/webserver/openapi-projects-metadata.yaml b/api/specs/webserver/openapi-projects-metadata.yaml new file mode 100644 index 00000000000..aea82cac40c --- /dev/null +++ b/api/specs/webserver/openapi-projects-metadata.yaml @@ -0,0 +1,93 @@ +paths: + /projects/{project_id}/metadata/custom: + get: + tags: + - project + summary: Get Project Custom Metadata + operationId: get_project_custom_metadata + parameters: + - required: true + schema: + type: string + format: uuid + title: Project Id + name: project_id + in: path + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Envelope_ProjectCustomMetadataGet_' + put: + tags: + - project + summary: Replace Project Custom Metadata + operationId: replace_project_custom_metadata + parameters: + - required: true + schema: + type: string + format: uuid + title: Project Id + name: project_id + in: path + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/ProjectCustomMetadataReplace' + required: true + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Envelope_ProjectCustomMetadataGet_' +components: + schemas: + Envelope_ProjectCustomMetadataGet_: + properties: + data: + $ref: '#/components/schemas/ProjectCustomMetadataGet' + error: + title: Error + type: object + title: Envelope[ProjectCustomMetadataGet] + ProjectCustomMetadataGet: + properties: + projectUuid: + type: string + format: uuid + title: Projectuuid + metadata: + additionalProperties: + anyOf: + - type: boolean + - type: integer + - type: number + - type: string + type: object + title: Metadata + description: Custom key-value map + type: object + required: + - projectUuid + title: ProjectCustomMetadataGet + ProjectCustomMetadataReplace: + properties: + metadata: + additionalProperties: + anyOf: + - type: boolean + - type: integer + - type: number + - type: string + type: object + title: Metadata + type: object + required: + - metadata + title: ProjectCustomMetadataReplace diff --git a/api/specs/webserver/openapi.yaml b/api/specs/webserver/openapi.yaml index 0500acd5432..43c5977bfca 100644 --- a/api/specs/webserver/openapi.yaml +++ b/api/specs/webserver/openapi.yaml @@ -256,6 +256,9 @@ paths: /projects/{project_id}/outputs: $ref: "./openapi-projects-ports.yaml#/paths/~1projects~1{project_id}~1outputs" + /projects/{project_id}/metadata/custom: + $ref: "./openapi-projects-metadata.yaml#/paths/~1projects~1{project_id}~1metadata~1custom" + /projects/{project_id}/metadata/ports: $ref: "./openapi-projects-ports.yaml#/paths/~1projects~1{project_id}~1metadata~1ports" diff --git a/api/specs/webserver/scripts/openapi_projects_metadata.py b/api/specs/webserver/scripts/openapi_projects_metadata.py new file mode 100644 index 00000000000..8416ec69e3a --- /dev/null +++ b/api/specs/webserver/scripts/openapi_projects_metadata.py @@ -0,0 +1,63 @@ +""" Helper script to automatically generate OAS + +This OAS are the source of truth +""" + +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments + + +from enum import Enum +from typing import Annotated + +from _common import CURRENT_DIR, create_openapi_specs +from fastapi import Depends, FastAPI, status +from models_library.api_schemas_webserver.projects_metadata import ( + ProjectCustomMetadataGet, + ProjectCustomMetadataReplace, +) +from models_library.generics import Envelope +from simcore_service_webserver.projects._metadata_handlers import ( + ProjectCustomMetadataGet, + ProjectPathParams, +) + +app = FastAPI(redoc_url=None) + +TAGS: list[str | Enum] = ["project"] + + +# +# API entrypoints +# + + +@app.get( + "/projects/{project_id}/metadata/custom", + response_model=Envelope[ProjectCustomMetadataGet], + tags=TAGS, + operation_id="get_project_custom_metadata", + status_code=status.HTTP_200_OK, +) +async def get_project_custom_metadata(params: Annotated[ProjectPathParams, Depends()]): + ... + + +@app.put( + "/projects/{project_id}/metadata/custom", + response_model=Envelope[ProjectCustomMetadataGet], + tags=TAGS, + operation_id="replace_project_custom_metadata", + status_code=status.HTTP_200_OK, +) +async def replace_project_custom_metadata( + params: Annotated[ProjectPathParams, Depends()], body: ProjectCustomMetadataReplace +): + ... + + +if __name__ == "__main__": + + create_openapi_specs(app, CURRENT_DIR.parent / "openapi-projects-metadata.yaml") diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py index 1d6f2db0547..9b2a6ee5e84 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py @@ -57,7 +57,7 @@ async def wrapper(request: web.Request) -> web.StreamResponse: @routes.get( - f"/{VTAG}/projects/{{project_id}}/custom-metadata", + f"/{VTAG}/projects/{{project_id}}/metadata/custom", name="get_project_custom_metadata", ) @login_required @@ -79,7 +79,7 @@ async def get_project_custom_metadata(request: web.Request) -> web.Response: @routes.put( - f"/{VTAG}/projects/{{project_id}}/custom-metadata", + f"/{VTAG}/projects/{{project_id}}/metadata/custom", name="replace_project_custom_metadata", ) @login_required From 00f1f387cf3b0fe594008d0ae62146dcb9641c2e Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 6 Jul 2023 14:50:23 +0200 Subject: [PATCH 32/53] =?UTF-8?q?services/webserver=20api=20version:=200.2?= =?UTF-8?q?3.0=20=E2=86=92=200.24.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/specs/webserver/openapi.yaml | 2 +- services/web/server/VERSION | 2 +- services/web/server/setup.cfg | 2 +- .../server/src/simcore_service_webserver/api/v0/openapi.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/specs/webserver/openapi.yaml b/api/specs/webserver/openapi.yaml index 43c5977bfca..1cd331fd2b8 100644 --- a/api/specs/webserver/openapi.yaml +++ b/api/specs/webserver/openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.0.0 info: title: "osparc-simcore web API" - version: 0.23.0 + version: 0.24.0 description: "API designed for the front-end app" contact: name: IT'IS Foundation diff --git a/services/web/server/VERSION b/services/web/server/VERSION index ca222b7cf39..2094a100ca8 100644 --- a/services/web/server/VERSION +++ b/services/web/server/VERSION @@ -1 +1 @@ -0.23.0 +0.24.0 diff --git a/services/web/server/setup.cfg b/services/web/server/setup.cfg index e54fab08a88..80bfd8bb744 100644 --- a/services/web/server/setup.cfg +++ b/services/web/server/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.23.0 +current_version = 0.24.0 commit = True message = services/webserver api version: {current_version} → {new_version} tag = False diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 5b714a0f061..ed0a1f7d44b 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.0.0 info: title: osparc-simcore web API - version: 0.23.0 + version: 0.24.0 description: API designed for the front-end app contact: name: IT'IS Foundation From d13cbc0b648d9d939aa6e3708383c5306a9f25f3 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 6 Jul 2023 14:51:37 +0200 Subject: [PATCH 33/53] updates OAS --- .../api/v0/openapi.yaml | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index ed0a1f7d44b..fbfb65defa5 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -6310,6 +6310,90 @@ paths: type: string error: title: Error + '/projects/{project_id}/metadata/custom': + get: + tags: + - project + summary: Get Project Custom Metadata + operationId: get_project_custom_metadata + parameters: + - required: true + schema: + type: string + format: uuid + title: Project Id + name: project_id + in: path + responses: + '200': + description: Successful Response + content: + application/json: + schema: + properties: + data: + properties: + projectUuid: + type: string + format: uuid + title: Projectuuid + metadata: + additionalProperties: + anyOf: + - type: boolean + - type: integer + - type: number + - type: string + type: object + title: Metadata + description: Custom key-value map + type: object + required: + - projectUuid + title: ProjectCustomMetadataGet + error: + title: Error + type: object + title: 'Envelope[ProjectCustomMetadataGet]' + put: + tags: + - project + summary: Replace Project Custom Metadata + operationId: replace_project_custom_metadata + parameters: + - required: true + schema: + type: string + format: uuid + title: Project Id + name: project_id + in: path + requestBody: + content: + application/json: + schema: + properties: + metadata: + additionalProperties: + anyOf: + - type: boolean + - type: integer + - type: number + - type: string + type: object + title: Metadata + type: object + required: + - metadata + title: ProjectCustomMetadataReplace + required: true + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/paths/~1projects~1%7Bproject_id%7D~1metadata~1custom/get/responses/200/content/application~1json/schema' '/projects/{project_id}/metadata/ports': get: tags: From 501685357824226e79863b8123ba8f7d6d663815 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 6 Jul 2023 15:10:47 +0200 Subject: [PATCH 34/53] fixes --- .../versions/f3285aff5e84_new_projects_metadata_table.py | 5 ++--- .../projects/_crud_create_utils.py | 6 +++--- .../unit/with_dbs/02/test_projects_metadata_handlers.py | 3 ++- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/f3285aff5e84_new_projects_metadata_table.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/f3285aff5e84_new_projects_metadata_table.py index 9faa3198f0a..3bfe63eb8db 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/f3285aff5e84_new_projects_metadata_table.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/f3285aff5e84_new_projects_metadata_table.py @@ -83,12 +83,11 @@ def upgrade(): # ### end Alembic commands ### # custom - update_modified_timestamp_procedure.execute(bind=op.get_context().bind) - modified_timestamp_trigger.execute(bind=op.get_context().bind) + op.execute(update_modified_timestamp_procedure) + op.execute(modified_timestamp_trigger) def downgrade(): - # custom op.execute(f"DROP TRIGGER IF EXISTS {_TRIGGER_NAME} on {_TABLE_NAME};") op.execute(f"DROP FUNCTION {_PROCEDURE_NAME};") diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_create_utils.py b/services/web/server/src/simcore_service_webserver/projects/_crud_create_utils.py index ce6caeeb3f4..d006e430ffb 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_create_utils.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_create_utils.py @@ -1,8 +1,9 @@ import asyncio import logging +from collections.abc import Coroutine from contextlib import AsyncExitStack from dataclasses import asdict -from typing import Any, Coroutine, TypeAlias +from typing import Any, TypeAlias from aiohttp import web from jsonschema import ValidationError as JsonSchemaValidationError @@ -120,7 +121,7 @@ async def _copy_project_nodes_from_source_project( def _mapped_node_id(node: ProjectNode) -> NodeID: return NodeID(nodes_map[NodeIDStr(f"{node.node_id}")]) - dst_project_node_creates = { + return { _mapped_node_id(node): ProjectNodeCreate( node_id=_mapped_node_id(node), **{ @@ -131,7 +132,6 @@ def _mapped_node_id(node: ProjectNode) -> NodeID: ) for node in await db.list_project_nodes(ProjectID(source_project["uuid"])) } - return dst_project_node_creates async def _copy_files_from_source_project( diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py index 9a77d5ddd43..61b4aa1f792 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py @@ -27,6 +27,7 @@ ], ) async def test_custom_metadata_handlers( + mocked_director_v2_api: None, # for deletion client: TestClient, faker: Faker, logged_user: UserInfoDict, @@ -78,7 +79,7 @@ async def test_custom_metadata_handlers( response = await client.delete(f"{url}") await assert_status(response, expected_cls=web.HTTPNoContent) - # no metadata -> project not foun d + # no metadata -> project not found url = client.app.router["get_project_custom_metadata"].url_for( project_id=user_project["uuid"] ) From f2d8c2ccdfa2670b1a389576da7d4759c207b127 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 6 Jul 2023 15:21:54 +0200 Subject: [PATCH 35/53] fixes deletion in test --- .../with_dbs/02/test_projects_handlers__delete.py | 9 +++++---- .../02/test_projects_metadata_handlers.py | 15 ++++++++++++++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_handlers__delete.py b/services/web/server/tests/unit/with_dbs/02/test_projects_handlers__delete.py index 046471c4f51..8a0d7489226 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_handlers__delete.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_handlers__delete.py @@ -4,7 +4,8 @@ # pylint: disable=too-many-arguments -from typing import Any, Callable, Iterator +from collections.abc import Callable, Iterator +from typing import Any from unittest import mock from unittest.mock import MagicMock, call @@ -129,11 +130,11 @@ async def test_delete_multiple_opened_project_forbidden( redis_client, ): # service in project - service = await create_dynamic_service_mock(logged_user["id"], user_project["uuid"]) + await create_dynamic_service_mock(logged_user["id"], user_project["uuid"]) # open project in tab1 client_session_id1 = client_session_id_factory() try: - sio1 = await socketio_client_factory(client_session_id1) + await socketio_client_factory(client_session_id1) except SocketConnectionError: if user_role != UserRole.ANONYMOUS: pytest.fail("socket io connection should not fail") @@ -151,7 +152,7 @@ async def test_delete_multiple_opened_project_forbidden( # delete project in tab2 client_session_id2 = client_session_id_factory() try: - sio2 = await socketio_client_factory(client_session_id2) + await socketio_client_factory(client_session_id2) except SocketConnectionError: if user_role != UserRole.ANONYMOUS: pytest.fail("socket io connection should not fail") diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py index 61b4aa1f792..9a283412817 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py @@ -16,7 +16,9 @@ from pydantic import parse_obj_as from pytest_simcore.helpers.utils_assert import assert_status from pytest_simcore.helpers.utils_login import UserInfoDict +from pytest_simcore.helpers.utils_webserver_unit_with_db import MockedStorageSubsystem from simcore_postgres_database.models.users import UserRole +from simcore_service_webserver.projects import _crud_delete_utils from simcore_service_webserver.projects.models import ProjectDict @@ -27,7 +29,10 @@ ], ) async def test_custom_metadata_handlers( - mocked_director_v2_api: None, # for deletion + # for deletion + mocked_director_v2_api: None, + storage_subsystem_mock: MockedStorageSubsystem, + # client: TestClient, faker: Faker, logged_user: UserInfoDict, @@ -79,6 +84,14 @@ async def test_custom_metadata_handlers( response = await client.delete(f"{url}") await assert_status(response, expected_cls=web.HTTPNoContent) + async def _wait_until_deleted(): + tasks = _crud_delete_utils.get_scheduled_tasks( + project_uuid=user_project["uuid"], user_id=logged_user["id"] + ) + await tasks[0] + + await _wait_until_deleted() + # no metadata -> project not found url = client.app.router["get_project_custom_metadata"].url_for( project_id=user_project["uuid"] From 7cfdf8add903bb154de55ca15476c9044445c6e6 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 6 Jul 2023 17:44:31 +0200 Subject: [PATCH 36/53] fix merge --- packages/postgres-database/scripts/erd/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/postgres-database/scripts/erd/Dockerfile b/packages/postgres-database/scripts/erd/Dockerfile index f419efc3f1c..4067d4f1d91 100644 --- a/packages/postgres-database/scripts/erd/Dockerfile +++ b/packages/postgres-database/scripts/erd/Dockerfile @@ -16,7 +16,7 @@ RUN apt-get update \ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip --no-cache-dir install --upgrade \ + pip install --upgrade \ pip~=23.1 \ wheel \ setuptools From 8688df17d416f6370f13580eb0f338a32e04ad37 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 6 Jul 2023 17:49:12 +0200 Subject: [PATCH 37/53] minor --- .../src/simcore_postgres_database/utils_projects_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py index 72e0d2f0dd4..4e0368ed8c1 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py @@ -19,7 +19,7 @@ class DBProjectNotFoundError(Exception): - code = "projects.not_found" + ... # From a1c71f7b6946deed25ccaec71b7fad5c10e7c581 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 6 Jul 2023 17:50:22 +0200 Subject: [PATCH 38/53] cleanup doc --- .../postgres-database/tests/test_utils_projects_metadata.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/postgres-database/tests/test_utils_projects_metadata.py b/packages/postgres-database/tests/test_utils_projects_metadata.py index fec9d4e1d86..6652353f7fe 100644 --- a/packages/postgres-database/tests/test_utils_projects_metadata.py +++ b/packages/postgres-database/tests/test_utils_projects_metadata.py @@ -75,9 +75,3 @@ async def test_projects_metadata_repository( assert got.modified assert got2.modified assert got.modified < got2.modified - - # list jobs of a study - # list all jobs of a user - # list projects that are non-jobs - - # DELETE job by deleting project From fb387e7d508b45a3682ccecfc23f1d29cff3704f Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 6 Jul 2023 17:52:03 +0200 Subject: [PATCH 39/53] minor --- .../simcore_service_webserver/projects/_access_rights_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_access_rights_api.py b/services/web/server/src/simcore_service_webserver/projects/_access_rights_api.py index f8b8b5010b6..40e1b850233 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_access_rights_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_access_rights_api.py @@ -12,7 +12,7 @@ async def validate_project_ownership( ): """ Raises: - ProjectInvalidRightsError: if not project is not owned by user_id + ProjectInvalidRightsError: if 'user_id' does not own 'project_uuid' """ if ( await get_project_owner(get_database_engine(app), project_uuid=project_uuid) From bb11dd0e5f1a15ed781e072e6f79015a48059e3e Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 6 Jul 2023 17:59:36 +0200 Subject: [PATCH 40/53] cleanup --- .../simcore_service_webserver/projects/_access_rights_api.py | 3 +++ .../simcore_service_webserver/projects/_access_rights_db.py | 3 +++ .../tests/unit/with_dbs/02/test_projects_metadata_handlers.py | 3 +++ 3 files changed, 9 insertions(+) diff --git a/services/web/server/src/simcore_service_webserver/projects/_access_rights_api.py b/services/web/server/src/simcore_service_webserver/projects/_access_rights_api.py index 40e1b850233..999c4ac0c6a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_access_rights_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_access_rights_api.py @@ -19,3 +19,6 @@ async def validate_project_ownership( != user_id ): raise ProjectInvalidRightsError(user_id=user_id, project_uuid=project_uuid) + + +# NOTE: Move here async def validate_project_permissions(app: web.Application, user_id: UserID, project_uuid: str, permission: PermissionStr diff --git a/services/web/server/src/simcore_service_webserver/projects/_access_rights_db.py b/services/web/server/src/simcore_service_webserver/projects/_access_rights_db.py index c6b5b45099d..4d0406c3faa 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_access_rights_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_access_rights_db.py @@ -17,3 +17,6 @@ async def get_project_owner(engine: Engine, project_uuid: ProjectID) -> UserID: if owner_id is None: raise ProjectNotFoundError(project_uuid=project_uuid) return owner_id + + +# NOTE: async def get_access_rights_and_user_groups(engine: Engine, user_id: UserID, project_uuid: ProjectID): diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py index 9a283412817..9428f614817 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py @@ -22,6 +22,9 @@ from simcore_service_webserver.projects.models import ProjectDict +@pytest.mark.acceptance_test( + "For https://github.com/ITISFoundation/osparc-simcore/issues/4313" +) @pytest.mark.parametrize( "user_role", [ From 94944efd4c1b6a7cddafab6496d5f18172dbdc89 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 6 Jul 2023 18:00:34 +0200 Subject: [PATCH 41/53] minor fix from last PR --- .../service-library/src/servicelib/aiohttp/incidents.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/service-library/src/servicelib/aiohttp/incidents.py b/packages/service-library/src/servicelib/aiohttp/incidents.py index 763b50c8e52..4d978ae0b48 100644 --- a/packages/service-library/src/servicelib/aiohttp/incidents.py +++ b/packages/service-library/src/servicelib/aiohttp/incidents.py @@ -1,9 +1,8 @@ -from typing import Any, Callable, Generic, TypeVar +from collections.abc import Callable +from typing import Any, Generic, TypeVar import attr -# UTILS --- - ItemT = TypeVar("ItemT") @@ -54,7 +53,9 @@ def append(self, item: ItemT): self._hits += 1 # sort is based on the __lt__ defined in ItemT - self._items = sorted(self._items, key=self.order_by, reverse=True) + if self.order_by is not None: + self._items = sorted(self._items, key=self.order_by, reverse=True) + if len(self._items) > self.max_size: self._items.pop() # min is dropped From a3cafe734e08c9ac5ff97b640e94ea570a038339 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 6 Jul 2023 18:04:26 +0200 Subject: [PATCH 42/53] minor --- .../postgres-database/tests/test_utils_projects_metadata.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/postgres-database/tests/test_utils_projects_metadata.py b/packages/postgres-database/tests/test_utils_projects_metadata.py index 6652353f7fe..79557421239 100644 --- a/packages/postgres-database/tests/test_utils_projects_metadata.py +++ b/packages/postgres-database/tests/test_utils_projects_metadata.py @@ -34,6 +34,9 @@ async def fake_project( return project +@pytest.mark.acceptance_test( + "For https://github.com/ITISFoundation/osparc-simcore/issues/4313" +) async def test_projects_metadata_repository( connection: SAConnection, create_fake_user: Callable[..., Awaitable[RowProxy]], From ddb22214960530b0be484ba9741a07d6e1389d5f Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 6 Jul 2023 19:00:45 +0200 Subject: [PATCH 43/53] Revert "minor fix from last PR" This reverts commit 94944efd4c1b6a7cddafab6496d5f18172dbdc89. --- .../service-library/src/servicelib/aiohttp/incidents.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/service-library/src/servicelib/aiohttp/incidents.py b/packages/service-library/src/servicelib/aiohttp/incidents.py index 4d978ae0b48..763b50c8e52 100644 --- a/packages/service-library/src/servicelib/aiohttp/incidents.py +++ b/packages/service-library/src/servicelib/aiohttp/incidents.py @@ -1,8 +1,9 @@ -from collections.abc import Callable -from typing import Any, Generic, TypeVar +from typing import Any, Callable, Generic, TypeVar import attr +# UTILS --- + ItemT = TypeVar("ItemT") @@ -53,9 +54,7 @@ def append(self, item: ItemT): self._hits += 1 # sort is based on the __lt__ defined in ItemT - if self.order_by is not None: - self._items = sorted(self._items, key=self.order_by, reverse=True) - + self._items = sorted(self._items, key=self.order_by, reverse=True) if len(self._items) > self.max_size: self._items.pop() # min is dropped From f96a7db1a3201387c5f311b500b4a372cda41e3b Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 7 Jul 2023 14:40:48 +0200 Subject: [PATCH 44/53] fixes route --- .../src/simcore_service_api_server/api/routes/solvers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py index 57380050d7a..d671058d6c6 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py @@ -93,7 +93,7 @@ async def list_solvers_releases( @router.get( - "releases/page", + "/releases/page", response_model=LimitOffsetPage[Solver], include_in_schema=API_SERVER_DEV_FEATURES_ENABLED, ) From 19c18d889093baa3bdcee3faf524ac07a112e3b4 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 7 Jul 2023 14:52:09 +0200 Subject: [PATCH 45/53] @sandereg review: todos in disguise --- .../simcore_service_webserver/projects/_access_rights_api.py | 3 --- .../simcore_service_webserver/projects/_access_rights_db.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_access_rights_api.py b/services/web/server/src/simcore_service_webserver/projects/_access_rights_api.py index 999c4ac0c6a..40e1b850233 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_access_rights_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_access_rights_api.py @@ -19,6 +19,3 @@ async def validate_project_ownership( != user_id ): raise ProjectInvalidRightsError(user_id=user_id, project_uuid=project_uuid) - - -# NOTE: Move here async def validate_project_permissions(app: web.Application, user_id: UserID, project_uuid: str, permission: PermissionStr diff --git a/services/web/server/src/simcore_service_webserver/projects/_access_rights_db.py b/services/web/server/src/simcore_service_webserver/projects/_access_rights_db.py index 4d0406c3faa..c6b5b45099d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_access_rights_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_access_rights_db.py @@ -17,6 +17,3 @@ async def get_project_owner(engine: Engine, project_uuid: ProjectID) -> UserID: if owner_id is None: raise ProjectNotFoundError(project_uuid=project_uuid) return owner_id - - -# NOTE: async def get_access_rights_and_user_groups(engine: Engine, user_id: UserID, project_uuid: ProjectID): From ce0c5a35228c39c14ff8d0bbf96467e6eed8beec Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 7 Jul 2023 15:26:02 +0200 Subject: [PATCH 46/53] @sanderegg review: put->patch --- .../scripts/openapi_projects_metadata.py | 14 +++++++------- .../api_schemas_webserver/projects_metadata.py | 2 +- .../projects/_metadata_handlers.py | 16 ++++++++-------- .../02/test_projects_metadata_handlers.py | 8 ++++---- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/api/specs/webserver/scripts/openapi_projects_metadata.py b/api/specs/webserver/scripts/openapi_projects_metadata.py index 8416ec69e3a..c05a0fa8bef 100644 --- a/api/specs/webserver/scripts/openapi_projects_metadata.py +++ b/api/specs/webserver/scripts/openapi_projects_metadata.py @@ -16,7 +16,7 @@ from fastapi import Depends, FastAPI, status from models_library.api_schemas_webserver.projects_metadata import ( ProjectCustomMetadataGet, - ProjectCustomMetadataReplace, + ProjectCustomMetadataUpdate, ) from models_library.generics import Envelope from simcore_service_webserver.projects._metadata_handlers import ( @@ -35,7 +35,7 @@ @app.get( - "/projects/{project_id}/metadata/custom", + "/projects/{project_id}/metadata", response_model=Envelope[ProjectCustomMetadataGet], tags=TAGS, operation_id="get_project_custom_metadata", @@ -45,15 +45,15 @@ async def get_project_custom_metadata(params: Annotated[ProjectPathParams, Depen ... -@app.put( - "/projects/{project_id}/metadata/custom", +@app.patch( + "/projects/{project_id}/metadata", response_model=Envelope[ProjectCustomMetadataGet], tags=TAGS, - operation_id="replace_project_custom_metadata", + operation_id="update_project_custom_metadata", status_code=status.HTTP_200_OK, ) -async def replace_project_custom_metadata( - params: Annotated[ProjectPathParams, Depends()], body: ProjectCustomMetadataReplace +async def update_project_custom_metadata( + params_: Annotated[ProjectPathParams, Depends()], body_: ProjectCustomMetadataUpdate ): ... diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects_metadata.py b/packages/models-library/src/models_library/api_schemas_webserver/projects_metadata.py index 3dd33e4bfa4..5bc54d4a04b 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects_metadata.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects_metadata.py @@ -17,5 +17,5 @@ class ProjectCustomMetadataGet(OutputSchema): ) -class ProjectCustomMetadataReplace(InputSchema): +class ProjectCustomMetadataUpdate(InputSchema): metadata: MetadataDict diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py index 9b2a6ee5e84..ad23dd554fc 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py @@ -16,7 +16,7 @@ from aiohttp import web from models_library.api_schemas_webserver.projects_metadata import ( ProjectCustomMetadataGet, - ProjectCustomMetadataReplace, + ProjectCustomMetadataUpdate, ) from servicelib.aiohttp.requests_validation import ( parse_request_body_as, @@ -57,7 +57,7 @@ async def wrapper(request: web.Request) -> web.StreamResponse: @routes.get( - f"/{VTAG}/projects/{{project_id}}/metadata/custom", + f"/{VTAG}/projects/{{project_id}}/metadata", name="get_project_custom_metadata", ) @login_required @@ -78,23 +78,23 @@ async def get_project_custom_metadata(request: web.Request) -> web.Response: ) -@routes.put( - f"/{VTAG}/projects/{{project_id}}/metadata/custom", - name="replace_project_custom_metadata", +@routes.patch( + f"/{VTAG}/projects/{{project_id}}/metadata", + name="update_project_custom_metadata", ) @login_required @permission_required("project.update") @_handle_project_exceptions -async def replace_project_custom_metadata(request: web.Request) -> web.Response: +async def update_project_custom_metadata(request: web.Request) -> web.Response: req_ctx = RequestContext.parse_obj(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) - replace = await parse_request_body_as(ProjectCustomMetadataReplace, request) + update = await parse_request_body_as(ProjectCustomMetadataUpdate, request) custom_metadata = await _metadata_api.set_project_custom_metadata( request.app, user_id=req_ctx.user_id, project_uuid=path_params.project_id, - value=replace.metadata, + value=update.metadata, ) return envelope_json_response( diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py index 9428f614817..957082570c1 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py @@ -11,7 +11,7 @@ from faker import Faker from models_library.api_schemas_webserver.projects_metadata import ( ProjectCustomMetadataGet, - ProjectCustomMetadataReplace, + ProjectCustomMetadataUpdate, ) from pydantic import parse_obj_as from pytest_simcore.helpers.utils_assert import assert_status @@ -71,11 +71,11 @@ async def test_custom_metadata_handlers( custom_metadata = {"number": 3.14, "string": "str", "boolean": False} custom_metadata["other"] = json.dumps(custom_metadata) - url = client.app.router["replace_project_custom_metadata"].url_for( + url = client.app.router["update_project_custom_metadata"].url_for( project_id=user_project["uuid"] ) - response = await client.put( - f"{url}", json=ProjectCustomMetadataReplace(metadata=custom_metadata).dict() + response = await client.patch( + f"{url}", json=ProjectCustomMetadataUpdate(metadata=custom_metadata).dict() ) data, _ = await assert_status(response, expected_cls=web.HTTPOk) From c489d202f74f0bdd3187636bf5f7484c509658ff Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 7 Jul 2023 15:44:49 +0200 Subject: [PATCH 47/53] @sanderegg review: renamed all as metadata --- .../scripts/openapi_projects_metadata.py | 20 ++++++------ .../projects_metadata.py | 8 ++--- ...3285aff5e84_new_projects_metadata_table.py | 2 +- .../models/projects_metadata.py | 4 +-- .../utils_projects_metadata.py | 4 +-- .../tests/test_utils_projects_metadata.py | 28 +++++++++------- .../projects/_metadata_api.py | 2 +- .../projects/_metadata_db.py | 4 +-- .../projects/_metadata_handlers.py | 32 ++++++++----------- .../02/test_projects_metadata_handlers.py | 16 +++++----- 10 files changed, 60 insertions(+), 60 deletions(-) diff --git a/api/specs/webserver/scripts/openapi_projects_metadata.py b/api/specs/webserver/scripts/openapi_projects_metadata.py index c05a0fa8bef..18469a627d4 100644 --- a/api/specs/webserver/scripts/openapi_projects_metadata.py +++ b/api/specs/webserver/scripts/openapi_projects_metadata.py @@ -15,12 +15,12 @@ from _common import CURRENT_DIR, create_openapi_specs from fastapi import Depends, FastAPI, status from models_library.api_schemas_webserver.projects_metadata import ( - ProjectCustomMetadataGet, - ProjectCustomMetadataUpdate, + ProjectMetadataGet, + ProjectMetadataUpdate, ) from models_library.generics import Envelope from simcore_service_webserver.projects._metadata_handlers import ( - ProjectCustomMetadataGet, + ProjectMetadataGet, ProjectPathParams, ) @@ -36,24 +36,24 @@ @app.get( "/projects/{project_id}/metadata", - response_model=Envelope[ProjectCustomMetadataGet], + response_model=Envelope[ProjectMetadataGet], tags=TAGS, - operation_id="get_project_custom_metadata", + operation_id="get_project_metadata", status_code=status.HTTP_200_OK, ) -async def get_project_custom_metadata(params: Annotated[ProjectPathParams, Depends()]): +async def get_project_metadata(_params: Annotated[ProjectPathParams, Depends()]): ... @app.patch( "/projects/{project_id}/metadata", - response_model=Envelope[ProjectCustomMetadataGet], + response_model=Envelope[ProjectMetadataGet], tags=TAGS, - operation_id="update_project_custom_metadata", + operation_id="update_project_metadata", status_code=status.HTTP_200_OK, ) -async def update_project_custom_metadata( - params_: Annotated[ProjectPathParams, Depends()], body_: ProjectCustomMetadataUpdate +async def update_project_metadata( + _params: Annotated[ProjectPathParams, Depends()], _body: ProjectMetadataUpdate ): ... diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects_metadata.py b/packages/models-library/src/models_library/api_schemas_webserver/projects_metadata.py index 5bc54d4a04b..c108dcd2fc2 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects_metadata.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects_metadata.py @@ -10,12 +10,12 @@ MetadataDict: TypeAlias = dict[str, MetaValueType] -class ProjectCustomMetadataGet(OutputSchema): +class ProjectMetadataGet(OutputSchema): project_uuid: ProjectID - metadata: MetadataDict = Field( + custom: MetadataDict = Field( default_factory=dict, description="Custom key-value map" ) -class ProjectCustomMetadataUpdate(InputSchema): - metadata: MetadataDict +class ProjectMetadataUpdate(InputSchema): + custom: MetadataDict diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/f3285aff5e84_new_projects_metadata_table.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/f3285aff5e84_new_projects_metadata_table.py index 3bfe63eb8db..0e4dc98204e 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/f3285aff5e84_new_projects_metadata_table.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/f3285aff5e84_new_projects_metadata_table.py @@ -54,7 +54,7 @@ def upgrade(): "projects_metadata", sa.Column("project_uuid", sa.String(), nullable=False), sa.Column( - "custom_metadata", + "custom", postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False, diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py index c322b75a3b9..2a450e5a80c 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py @@ -41,11 +41,11 @@ doc="The project unique identifier is also used to identify the associated job", ), sa.Column( - "custom_metadata", + "custom", JSONB, nullable=False, server_default=sa.text("'{}'::jsonb"), - doc="Unstructured free json for user to store metadata", + doc="Reserved for the user to store custom metadata", ), # TIME STAMPS ----ß column_created_datetime(timezone=True), diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py index 4e0368ed8c1..964f0549bf4 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py @@ -29,7 +29,7 @@ class DBProjectNotFoundError(Exception): @dataclass(frozen=True, slots=True, kw_only=True) class ProjectMetadata(FromRowMixin): - custom_metadata: dict[str, Any] | None + custom: dict[str, Any] | None created: datetime.datetime | None modified: datetime.datetime | None @@ -76,7 +76,7 @@ async def upsert( ) -> ProjectMetadata: data = { "project_uuid": f"{project_uuid}", - "custom_metadata": custom_metadata, + "custom": custom_metadata, } try: diff --git a/packages/postgres-database/tests/test_utils_projects_metadata.py b/packages/postgres-database/tests/test_utils_projects_metadata.py index 79557421239..1cc6a4a0571 100644 --- a/packages/postgres-database/tests/test_utils_projects_metadata.py +++ b/packages/postgres-database/tests/test_utils_projects_metadata.py @@ -57,24 +57,28 @@ async def test_projects_metadata_repository( connection, project_uuid=faker.uuid4(), custom_metadata=user_metadata ) - pm = await ProjectMetadataRepo.get(connection, project_uuid=project["uuid"]) - assert pm is not None - assert pm.custom_metadata is None + project_metadata = await ProjectMetadataRepo.get( + connection, project_uuid=project["uuid"] + ) + assert project_metadata is not None + assert project_metadata.custom is None got = await ProjectMetadataRepo.upsert( connection, project_uuid=project["uuid"], custom_metadata=user_metadata ) - assert got.custom_metadata - assert user_metadata == got.custom_metadata + assert got.custom + assert user_metadata == got.custom - pm = await ProjectMetadataRepo.get(connection, project_uuid=project["uuid"]) - assert pm is not None - assert pm == got + project_metadata = await ProjectMetadataRepo.get( + connection, project_uuid=project["uuid"] + ) + assert project_metadata is not None + assert project_metadata == got - got2 = await ProjectMetadataRepo.upsert( + got_after_update = await ProjectMetadataRepo.upsert( connection, project_uuid=project["uuid"], custom_metadata={} ) - assert got2.custom_metadata == {} + assert got_after_update.custom == {} assert got.modified - assert got2.modified - assert got.modified < got2.modified + assert got_after_update.modified + assert got.modified < got_after_update.modified diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py index a9e0a95e2b3..99ace0696cc 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py @@ -8,7 +8,7 @@ from ._metadata_db import get_project_metadata, upsert_project_metadata -async def get_project_custom_metadata( +async def get_project_metadata( app: web.Application, user_id: UserID, project_uuid: ProjectID ) -> MetadataDict: await validate_project_ownership(app, user_id=user_id, project_uuid=project_uuid) diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py index 7d93573cc4c..76f99234446 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py @@ -35,7 +35,7 @@ async def get_project_metadata(engine: Engine, project_uuid: ProjectID) -> Metad async with _acquire_and_handle(engine, project_uuid) as connection: pm = await ProjectMetadataRepo.get(connection, project_uuid=project_uuid) # NOTE: if no metadata in table, it returns None -- which converts here to --> {} - return parse_obj_as(MetadataDict, pm.custom_metadata or {}) + return parse_obj_as(MetadataDict, pm.custom or {}) async def upsert_project_metadata( @@ -47,4 +47,4 @@ async def upsert_project_metadata( pm = await ProjectMetadataRepo.upsert( connection, project_uuid=project_uuid, custom_metadata=custom_metadata ) - return parse_obj_as(MetadataDict, pm.custom_metadata) + return parse_obj_as(MetadataDict, pm.custom) diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py index ad23dd554fc..ff5eaf24e14 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py @@ -15,8 +15,8 @@ from aiohttp import web from models_library.api_schemas_webserver.projects_metadata import ( - ProjectCustomMetadataGet, - ProjectCustomMetadataUpdate, + ProjectMetadataGet, + ProjectMetadataUpdate, ) from servicelib.aiohttp.requests_validation import ( parse_request_body_as, @@ -25,7 +25,7 @@ from servicelib.aiohttp.typing_extension import Handler from simcore_service_webserver.utils_aiohttp import envelope_json_response -from .._meta import api_version_prefix as VTAG +from .._meta import api_version_prefix from ..login.decorators import login_required from ..security.decorators import permission_required from . import _metadata_api @@ -57,48 +57,44 @@ async def wrapper(request: web.Request) -> web.StreamResponse: @routes.get( - f"/{VTAG}/projects/{{project_id}}/metadata", - name="get_project_custom_metadata", + f"/{api_version_prefix}/projects/{{project_id}}/metadata", + name="get_project_metadata", ) @login_required @permission_required("project.read") @_handle_project_exceptions -async def get_project_custom_metadata(request: web.Request) -> web.Response: +async def get_project_metadata(request: web.Request) -> web.Response: req_ctx = RequestContext.parse_obj(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) - custom_metadata = await _metadata_api.get_project_custom_metadata( + custom_metadata = await _metadata_api.get_project_metadata( request.app, user_id=req_ctx.user_id, project_uuid=path_params.project_id ) return envelope_json_response( - ProjectCustomMetadataGet( - project_uuid=path_params.project_id, metadata=custom_metadata - ) + ProjectMetadataGet(project_uuid=path_params.project_id, custom=custom_metadata) ) @routes.patch( - f"/{VTAG}/projects/{{project_id}}/metadata", - name="update_project_custom_metadata", + f"/{api_version_prefix}/projects/{{project_id}}/metadata", + name="update_project_metadata", ) @login_required @permission_required("project.update") @_handle_project_exceptions -async def update_project_custom_metadata(request: web.Request) -> web.Response: +async def update_project_metadata(request: web.Request) -> web.Response: req_ctx = RequestContext.parse_obj(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) - update = await parse_request_body_as(ProjectCustomMetadataUpdate, request) + update = await parse_request_body_as(ProjectMetadataUpdate, request) custom_metadata = await _metadata_api.set_project_custom_metadata( request.app, user_id=req_ctx.user_id, project_uuid=path_params.project_id, - value=update.metadata, + value=update.custom, ) return envelope_json_response( - ProjectCustomMetadataGet( - project_uuid=path_params.project_id, metadata=custom_metadata - ) + ProjectMetadataGet(project_uuid=path_params.project_id, custom=custom_metadata) ) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py index 957082570c1..fec017955ac 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py @@ -10,8 +10,8 @@ from aiohttp.test_utils import TestClient from faker import Faker from models_library.api_schemas_webserver.projects_metadata import ( - ProjectCustomMetadataGet, - ProjectCustomMetadataUpdate, + ProjectMetadataGet, + ProjectMetadataUpdate, ) from pydantic import parse_obj_as from pytest_simcore.helpers.utils_assert import assert_status @@ -49,7 +49,7 @@ async def test_custom_metadata_handlers( # get metadata of a non-existing project -> Not found invalid_project_id = faker.uuid4() - url = client.app.router["get_project_custom_metadata"].url_for( + url = client.app.router["get_project_metadata"].url_for( project_id=invalid_project_id ) response = await client.get(f"{url}") @@ -60,7 +60,7 @@ async def test_custom_metadata_handlers( assert "project" in error_message.lower() # get metadata of an existing project the first time -> empty {} - url = client.app.router["get_project_custom_metadata"].url_for( + url = client.app.router["get_project_metadata"].url_for( project_id=user_project["uuid"] ) response = await client.get(f"{url}") @@ -71,16 +71,16 @@ async def test_custom_metadata_handlers( custom_metadata = {"number": 3.14, "string": "str", "boolean": False} custom_metadata["other"] = json.dumps(custom_metadata) - url = client.app.router["update_project_custom_metadata"].url_for( + url = client.app.router["update_project_metadata"].url_for( project_id=user_project["uuid"] ) response = await client.patch( - f"{url}", json=ProjectCustomMetadataUpdate(metadata=custom_metadata).dict() + f"{url}", json=ProjectMetadataUpdate(custom=custom_metadata).dict() ) data, _ = await assert_status(response, expected_cls=web.HTTPOk) - assert parse_obj_as(ProjectCustomMetadataGet, data).metadata == custom_metadata + assert parse_obj_as(ProjectMetadataGet, data).custom == custom_metadata # delete project url = client.app.router["delete_project"].url_for(project_id=user_project["uuid"]) @@ -96,7 +96,7 @@ async def _wait_until_deleted(): await _wait_until_deleted() # no metadata -> project not found - url = client.app.router["get_project_custom_metadata"].url_for( + url = client.app.router["get_project_metadata"].url_for( project_id=user_project["uuid"] ) response = await client.get(f"{url}") From 66ece812907385c498d41c6f889ace7ddd83b1f6 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 7 Jul 2023 16:11:02 +0200 Subject: [PATCH 48/53] @sandergg review: renamed functions --- .../utils_projects_metadata.py | 98 +++++++++---------- .../tests/test_utils_projects_metadata.py | 18 ++-- .../projects/_metadata_api.py | 6 +- .../projects/_metadata_db.py | 26 +++-- 4 files changed, 75 insertions(+), 73 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py index 964f0549bf4..d57babc1492 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py @@ -35,61 +35,59 @@ class ProjectMetadata(FromRowMixin): # -# Repos +# Helpers # -class ProjectMetadataRepo: - @staticmethod - async def get(connection: SAConnection, project_uuid: uuid.UUID) -> ProjectMetadata: - # JOIN LEFT OUTER - get_stmt = ( - sa.select( - projects.c.uuid, - projects_metadata.c.custom_metadata, - projects_metadata.c.created, - projects_metadata.c.modified, - ) - .select_from( - sa.join( - projects, - projects_metadata, - projects.c.uuid == projects_metadata.c.project_uuid, - isouter=True, - ) +async def get(connection: SAConnection, project_uuid: uuid.UUID) -> ProjectMetadata: + # JOIN LEFT OUTER + get_stmt = ( + sa.select( + projects.c.uuid, + projects_metadata.c.custom_metadata, + projects_metadata.c.created, + projects_metadata.c.modified, + ) + .select_from( + sa.join( + projects, + projects_metadata, + projects.c.uuid == projects_metadata.c.project_uuid, + isouter=True, ) - .where(projects.c.uuid == f"{project_uuid}") ) - result: ResultProxy = await connection.execute(get_stmt) + .where(projects.c.uuid == f"{project_uuid}") + ) + result: ResultProxy = await connection.execute(get_stmt) + row: RowProxy | None = await result.first() + if row is None: + msg = f"Project project_uuid={project_uuid!r} not found" + raise DBProjectNotFoundError(msg) + return ProjectMetadata.from_row(row) + + +async def upsert( + connection: SAConnection, + *, + project_uuid: uuid.UUID, + custom_metadata: dict[str, Any], +) -> ProjectMetadata: + data = { + "project_uuid": f"{project_uuid}", + "custom": custom_metadata, + } + + try: + insert_stmt = pg_insert(projects_metadata).values(**data) + upsert_stmt = insert_stmt.on_conflict_do_update( + index_elements=[projects_metadata.c.project_uuid], + set_=data, + ).returning(sa.literal_column("*")) + + result: ResultProxy = await connection.execute(upsert_stmt) row: RowProxy | None = await result.first() - if row is None: - msg = f"Project project_uuid={project_uuid!r} not found" - raise DBProjectNotFoundError(msg) + assert row # nosec return ProjectMetadata.from_row(row) - @staticmethod - async def upsert( - connection: SAConnection, - *, - project_uuid: uuid.UUID, - custom_metadata: dict[str, Any], - ) -> ProjectMetadata: - data = { - "project_uuid": f"{project_uuid}", - "custom": custom_metadata, - } - - try: - insert_stmt = pg_insert(projects_metadata).values(**data) - upsert_stmt = insert_stmt.on_conflict_do_update( - index_elements=[projects_metadata.c.project_uuid], - set_=data, - ).returning(sa.literal_column("*")) - - result: ResultProxy = await connection.execute(upsert_stmt) - row: RowProxy | None = await result.first() - assert row # nosec - return ProjectMetadata.from_row(row) - - except ForeignKeyViolation as err: - raise DBProjectNotFoundError(project_uuid) from err + except ForeignKeyViolation as err: + raise DBProjectNotFoundError(project_uuid) from err diff --git a/packages/postgres-database/tests/test_utils_projects_metadata.py b/packages/postgres-database/tests/test_utils_projects_metadata.py index 1cc6a4a0571..1cfce5e2e2a 100644 --- a/packages/postgres-database/tests/test_utils_projects_metadata.py +++ b/packages/postgres-database/tests/test_utils_projects_metadata.py @@ -9,10 +9,8 @@ from aiopg.sa.connection import SAConnection from aiopg.sa.result import RowProxy from faker import Faker -from simcore_postgres_database.utils_projects_metadata import ( - DBProjectNotFoundError, - ProjectMetadataRepo, -) +from simcore_postgres_database import utils_projects_metadata +from simcore_postgres_database.utils_projects_metadata import DBProjectNotFoundError @pytest.fixture @@ -50,32 +48,32 @@ async def test_projects_metadata_repository( user_metadata = {"float": 3.14, "int": 42, "string": "foo", "bool": True} with pytest.raises(DBProjectNotFoundError): - await ProjectMetadataRepo.get(connection, project_uuid=faker.uuid4()) + await utils_projects_metadata.get(connection, project_uuid=faker.uuid4()) with pytest.raises(DBProjectNotFoundError): - await ProjectMetadataRepo.upsert( + await utils_projects_metadata.upsert( connection, project_uuid=faker.uuid4(), custom_metadata=user_metadata ) - project_metadata = await ProjectMetadataRepo.get( + project_metadata = await utils_projects_metadata.get( connection, project_uuid=project["uuid"] ) assert project_metadata is not None assert project_metadata.custom is None - got = await ProjectMetadataRepo.upsert( + got = await utils_projects_metadata.upsert( connection, project_uuid=project["uuid"], custom_metadata=user_metadata ) assert got.custom assert user_metadata == got.custom - project_metadata = await ProjectMetadataRepo.get( + project_metadata = await utils_projects_metadata.get( connection, project_uuid=project["uuid"] ) assert project_metadata is not None assert project_metadata == got - got_after_update = await ProjectMetadataRepo.upsert( + got_after_update = await utils_projects_metadata.upsert( connection, project_uuid=project["uuid"], custom_metadata={} ) assert got_after_update.custom == {} diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py index 99ace0696cc..433a863540c 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py @@ -4,8 +4,8 @@ from models_library.users import UserID from ..db.plugin import get_database_engine +from . import _metadata_db from ._access_rights_api import validate_project_ownership -from ._metadata_db import get_project_metadata, upsert_project_metadata async def get_project_metadata( @@ -13,7 +13,7 @@ async def get_project_metadata( ) -> MetadataDict: await validate_project_ownership(app, user_id=user_id, project_uuid=project_uuid) - return await get_project_metadata( + return await _metadata_db.get_project_metadata( engine=get_database_engine(app), project_uuid=project_uuid ) @@ -23,7 +23,7 @@ async def set_project_custom_metadata( ) -> MetadataDict: await validate_project_ownership(app, user_id=user_id, project_uuid=project_uuid) - return await upsert_project_metadata( + return await _metadata_db.set_project_metadata( engine=get_database_engine(app), project_uuid=project_uuid, custom_metadata=value, diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py index 76f99234446..1ce0b1cd24c 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py @@ -1,15 +1,13 @@ +from collections.abc import AsyncIterator from contextlib import asynccontextmanager -from typing import AsyncIterator from aiopg.sa.connection import SAConnection from aiopg.sa.engine import Engine from models_library.api_schemas_webserver.projects_metadata import MetadataDict from models_library.projects import ProjectID from pydantic import parse_obj_as -from simcore_postgres_database.utils_projects_metadata import ( - DBProjectNotFoundError, - ProjectMetadataRepo, -) +from simcore_postgres_database import utils_projects_metadata +from simcore_postgres_database.utils_projects_metadata import DBProjectNotFoundError from .exceptions import ProjectNotFoundError @@ -31,20 +29,28 @@ async def get_project_metadata(engine: Engine, project_uuid: ProjectID) -> Metad """ Raises: ProjectNotFoundError + ValidationError: illegal metadata format in the database """ async with _acquire_and_handle(engine, project_uuid) as connection: - pm = await ProjectMetadataRepo.get(connection, project_uuid=project_uuid) + metadata = await utils_projects_metadata.get( + connection, project_uuid=project_uuid + ) # NOTE: if no metadata in table, it returns None -- which converts here to --> {} - return parse_obj_as(MetadataDict, pm.custom or {}) + return parse_obj_as(MetadataDict, metadata.custom or {}) -async def upsert_project_metadata( +async def set_project_metadata( engine: Engine, project_uuid: ProjectID, custom_metadata: MetadataDict, ) -> MetadataDict: + """ + Raises: + ProjectNotFoundError + ValidationError: illegal metadata format in the database + """ async with _acquire_and_handle(engine, project_uuid) as connection: - pm = await ProjectMetadataRepo.upsert( + metadata = await utils_projects_metadata.upsert( connection, project_uuid=project_uuid, custom_metadata=custom_metadata ) - return parse_obj_as(MetadataDict, pm.custom) + return parse_obj_as(MetadataDict, metadata.custom) From 79daf7d443b5f546661d68bbab47ac9988568367 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 7 Jul 2023 16:40:27 +0200 Subject: [PATCH 49/53] OAS --- .../webserver/openapi-projects-metadata.yaml | 42 +++++++++---------- api/specs/webserver/openapi.yaml | 4 +- .../api/v0/openapi.yaml | 30 ++++++------- 3 files changed, 38 insertions(+), 38 deletions(-) diff --git a/api/specs/webserver/openapi-projects-metadata.yaml b/api/specs/webserver/openapi-projects-metadata.yaml index aea82cac40c..44d154b9798 100644 --- a/api/specs/webserver/openapi-projects-metadata.yaml +++ b/api/specs/webserver/openapi-projects-metadata.yaml @@ -1,10 +1,10 @@ paths: - /projects/{project_id}/metadata/custom: + /projects/{project_id}/metadata: get: tags: - project - summary: Get Project Custom Metadata - operationId: get_project_custom_metadata + summary: Get Project Metadata + operationId: get_project_metadata parameters: - required: true schema: @@ -19,12 +19,12 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/Envelope_ProjectCustomMetadataGet_' - put: + $ref: '#/components/schemas/Envelope_ProjectMetadataGet_' + patch: tags: - project - summary: Replace Project Custom Metadata - operationId: replace_project_custom_metadata + summary: Update Project Metadata + operationId: update_project_metadata parameters: - required: true schema: @@ -37,7 +37,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ProjectCustomMetadataReplace' + $ref: '#/components/schemas/ProjectMetadataUpdate' required: true responses: '200': @@ -45,24 +45,24 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/Envelope_ProjectCustomMetadataGet_' + $ref: '#/components/schemas/Envelope_ProjectMetadataGet_' components: schemas: - Envelope_ProjectCustomMetadataGet_: + Envelope_ProjectMetadataGet_: properties: data: - $ref: '#/components/schemas/ProjectCustomMetadataGet' + $ref: '#/components/schemas/ProjectMetadataGet' error: title: Error type: object - title: Envelope[ProjectCustomMetadataGet] - ProjectCustomMetadataGet: + title: Envelope[ProjectMetadataGet] + ProjectMetadataGet: properties: projectUuid: type: string format: uuid title: Projectuuid - metadata: + custom: additionalProperties: anyOf: - type: boolean @@ -70,15 +70,15 @@ components: - type: number - type: string type: object - title: Metadata + title: Custom description: Custom key-value map type: object required: - projectUuid - title: ProjectCustomMetadataGet - ProjectCustomMetadataReplace: + title: ProjectMetadataGet + ProjectMetadataUpdate: properties: - metadata: + custom: additionalProperties: anyOf: - type: boolean @@ -86,8 +86,8 @@ components: - type: number - type: string type: object - title: Metadata + title: Custom type: object required: - - metadata - title: ProjectCustomMetadataReplace + - custom + title: ProjectMetadataUpdate diff --git a/api/specs/webserver/openapi.yaml b/api/specs/webserver/openapi.yaml index 1cd331fd2b8..68669071337 100644 --- a/api/specs/webserver/openapi.yaml +++ b/api/specs/webserver/openapi.yaml @@ -256,8 +256,8 @@ paths: /projects/{project_id}/outputs: $ref: "./openapi-projects-ports.yaml#/paths/~1projects~1{project_id}~1outputs" - /projects/{project_id}/metadata/custom: - $ref: "./openapi-projects-metadata.yaml#/paths/~1projects~1{project_id}~1metadata~1custom" + /projects/{project_id}/metadata: + $ref: "./openapi-projects-metadata.yaml#/paths/~1projects~1{project_id}~1metadata" /projects/{project_id}/metadata/ports: $ref: "./openapi-projects-ports.yaml#/paths/~1projects~1{project_id}~1metadata~1ports" diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index fbfb65defa5..9f91c15b7ff 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -6310,12 +6310,12 @@ paths: type: string error: title: Error - '/projects/{project_id}/metadata/custom': + '/projects/{project_id}/metadata': get: tags: - project - summary: Get Project Custom Metadata - operationId: get_project_custom_metadata + summary: Get Project Metadata + operationId: get_project_metadata parameters: - required: true schema: @@ -6337,7 +6337,7 @@ paths: type: string format: uuid title: Projectuuid - metadata: + custom: additionalProperties: anyOf: - type: boolean @@ -6345,21 +6345,21 @@ paths: - type: number - type: string type: object - title: Metadata + title: Custom description: Custom key-value map type: object required: - projectUuid - title: ProjectCustomMetadataGet + title: ProjectMetadataGet error: title: Error type: object - title: 'Envelope[ProjectCustomMetadataGet]' - put: + title: 'Envelope[ProjectMetadataGet]' + patch: tags: - project - summary: Replace Project Custom Metadata - operationId: replace_project_custom_metadata + summary: Update Project Metadata + operationId: update_project_metadata parameters: - required: true schema: @@ -6373,7 +6373,7 @@ paths: application/json: schema: properties: - metadata: + custom: additionalProperties: anyOf: - type: boolean @@ -6381,11 +6381,11 @@ paths: - type: number - type: string type: object - title: Metadata + title: Custom type: object required: - - metadata - title: ProjectCustomMetadataReplace + - custom + title: ProjectMetadataUpdate required: true responses: '200': @@ -6393,7 +6393,7 @@ paths: content: application/json: schema: - $ref: '#/paths/~1projects~1%7Bproject_id%7D~1metadata~1custom/get/responses/200/content/application~1json/schema' + $ref: '#/paths/~1projects~1%7Bproject_id%7D~1metadata/get/responses/200/content/application~1json/schema' '/projects/{project_id}/metadata/ports': get: tags: From 794e167f7d2c72b45f286bd76020d0059375ca74 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 7 Jul 2023 16:51:59 +0200 Subject: [PATCH 50/53] fix --- .../src/simcore_postgres_database/utils_projects_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py index d57babc1492..5f86f6c4999 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py @@ -44,7 +44,7 @@ async def get(connection: SAConnection, project_uuid: uuid.UUID) -> ProjectMetad get_stmt = ( sa.select( projects.c.uuid, - projects_metadata.c.custom_metadata, + projects_metadata.c.metadata, projects_metadata.c.created, projects_metadata.c.modified, ) From fe2ecea97b12d39254f078a968eea501b72cf4e4 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 7 Jul 2023 16:56:32 +0200 Subject: [PATCH 51/53] cleanup --- .../utils_projects_metadata.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py index 5f86f6c4999..b65276c515d 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py @@ -76,14 +76,13 @@ async def upsert( "project_uuid": f"{project_uuid}", "custom": custom_metadata, } + insert_stmt = pg_insert(projects_metadata).values(**data) + upsert_stmt = insert_stmt.on_conflict_do_update( + index_elements=[projects_metadata.c.project_uuid], + set_=data, + ).returning(sa.literal_column("*")) try: - insert_stmt = pg_insert(projects_metadata).values(**data) - upsert_stmt = insert_stmt.on_conflict_do_update( - index_elements=[projects_metadata.c.project_uuid], - set_=data, - ).returning(sa.literal_column("*")) - result: ResultProxy = await connection.execute(upsert_stmt) row: RowProxy | None = await result.first() assert row # nosec From efca20ee73344925cbc27e83e0fcd15538fc6ad4 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 7 Jul 2023 17:03:20 +0200 Subject: [PATCH 52/53] fix --- .../src/simcore_postgres_database/utils_projects_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py index b65276c515d..dafca9ffdad 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py @@ -44,7 +44,7 @@ async def get(connection: SAConnection, project_uuid: uuid.UUID) -> ProjectMetad get_stmt = ( sa.select( projects.c.uuid, - projects_metadata.c.metadata, + projects_metadata.c.custom, projects_metadata.c.created, projects_metadata.c.modified, ) From 9d263d81bf3148dfdc3b7042ae976b80f5342145 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 7 Jul 2023 18:11:24 +0200 Subject: [PATCH 53/53] fixes --- .../tests/unit/with_dbs/02/test_projects_metadata_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py index fec017955ac..65e9c4b93e6 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py @@ -65,7 +65,7 @@ async def test_custom_metadata_handlers( ) response = await client.get(f"{url}") data, _ = await assert_status(response, expected_cls=web.HTTPOk) - assert data["metadata"] == {} + assert data["custom"] == {} # replace metadata custom_metadata = {"number": 3.14, "string": "str", "boolean": False}