✨ Enhance compatibility-check for catalog service's ports including u…

…nits (#2913)
ITISFoundation · Mar 24, 2022 · f401ad2 · f401ad2
1 parent a920852
commit f401ad2
Show file tree

Hide file tree

Showing 15 changed files with 643 additions and 303 deletions.
diff --git a/packages/models-library/src/models_library/function_service/demo_units.py b/packages/models-library/src/models_library/function_service/demo_units.py
@@ -5,7 +5,7 @@
 
 def build_input(schema):
     description = schema.pop("description", schema["title"])
-    
+
     return {
         "label": schema["title"],
         "description": description,
@@ -15,11 +15,14 @@ def build_input(schema):
 
 
 # SEE https://github.com/hgrecco/pint/blob/master/pint/default_en.txt
+
 META = ServiceDockerData.parse_obj(
     {
         "integration-version": LATEST_INTEGRATION_VERSION,
         "key": f"{FUNCTION_SERVICE_KEY_PREFIX}/data-iterator/demo-units",
-        "version": "0.1.0",
+        "version": "0.2.0",
+        # CHANGELOG
+        # - 0.2.0: reverted order of first 5 outputs
         "type": ServiceType.BACKEND,
         "name": "Demo Units",
         "description": "Demo that takes base units as inputs and transform them in the outputs",
@@ -100,19 +103,18 @@ def build_input(schema):
             ),
         },
         "outputs": {
-            "length": build_input(
+            "mass": build_input(
                 {
-                    "title": "Distance",
-                    "description": "Distance value converted",
-                    "x_unit": "milli-meter",
+                    "title": "Mass",
+                    "minimum": 0,
+                    "x_unit": "kilo-gram",
                     "type": "number",
                 }
             ),
-            "time": build_input(
+            "luminosity": build_input(
                 {
-                    "title": "Time",
-                    "minimum": 0,
-                    "x_unit": "minute",
+                    "title": "Luminosity",
+                    "x_unit": "candela",
                     "type": "number",
                 }
             ),
@@ -123,18 +125,19 @@ def build_input(schema):
                     "type": "number",
                 }
             ),
-            "luminosity": build_input(
+            "time": build_input(
                 {
-                    "title": "Luminosity",
-                    "x_unit": "candela",
+                    "title": "Time",
+                    "minimum": 0,
+                    "x_unit": "minute",
                     "type": "number",
                 }
             ),
-            "mass": build_input(
+            "length": build_input(
                 {
-                    "title": "Mass",
-                    "minimum": 0,
-                    "x_unit": "kilo-gram",
+                    "title": "Distance",
+                    "description": "Distance value converted",
+                    "x_unit": "milli-meter",
                     "type": "number",
                 }
             ),

diff --git a/packages/models-library/src/models_library/projects_nodes_io.py b/packages/models-library/src/models_library/projects_nodes_io.py
@@ -1,5 +1,9 @@
 """
-    Models used as I/O of Nodes
+    Link models used at i/o port nodes:
+        - Link to files:
+            - Generic: DownloadLink
+            - At Custom Service: SimCoreFileLink, DatCoreFileLink
+        - Link to another port: PortLink
 """
 
 from pathlib import Path
@@ -26,30 +30,46 @@ class PortLink(BaseModel):
     node_uuid: NodeID = Field(
         ...,
         description="The node to get the port output from",
-        example=["da5068e0-8a8d-4fb9-9516-56e5ddaef15b"],
         alias="nodeUuid",
     )
     output: str = Field(
         ...,
         description="The port key in the node given by nodeUuid",
         regex=PROPERTY_KEY_RE,
-        example=["out_2"],
     )
 
     class Config:
         extra = Extra.forbid
+        schema_extra = {
+            "examples": [
+                # minimal
+                {
+                    "nodeUuid": "da5068e0-8a8d-4fb9-9516-56e5ddaef15b",
+                    "output": "out_2",
+                }
+            ],
+        }
 
 
 class DownloadLink(BaseModel):
     """I/O port type to hold a generic download link to a file (e.g. S3 pre-signed link, etc)"""
 
     download_link: AnyUrl = Field(..., alias="downloadLink")
-    label: Optional[str]
+    label: Optional[str] = None
 
     class Config:
         extra = Extra.forbid
+        schema_extra = {
+            "examples": [
+                # minimal
+                {
+                    "downloadLink": "https://fakeimg.pl/250x100/",
+                }
+            ],
+        }
 
 
+## CUSTOM STORAGE SERVICES -----------
 class BaseFileLink(BaseModel):
     """Base class for I/O port types with links to storage services"""
 
@@ -59,23 +79,17 @@ class BaseFileLink(BaseModel):
     store: Union[str, int] = Field(
         ...,
         description="The store identifier: '0' or 0 for simcore S3, '1' or 1 for datcore",
-        examples=["0", 1],
     )
 
     path: str = Field(
         ...,
         regex=r"^.+$",
         description="The path to the file in the storage provider domain",
-        examples=[
-            "N:package:b05739ef-260c-4038-b47d-0240d04b0599",
-            "94453a6a-c8d4-52b3-a22d-ccbf81f8d636/d4442ca4-23fd-5b6b-ba6d-0b75f711c109/y_1D.txt",
-        ],
     )
 
     label: Optional[str] = Field(
         None,
         description="The real file name",
-        examples=["MyFile.txt"],
     )
 
     e_tag: Optional[str] = Field(
@@ -112,9 +126,10 @@ def pre_fill_label_with_filename_ext(cls, v, values):
     class Config:
         extra = Extra.forbid
         schema_extra = {
+            # a project file
             "example": {
                 "store": "0",
-                "path": "api/0a3b2c56-dbcd-4871-b93b-d454b7883f9f/input.txt",
+                "path": "94453a6a-c8d4-52b3-a22d-ccbf81f8d636/0a3b2c56-dbcd-4871-b93b-d454b7883f9f/input.txt",
                 "eTag": "859fda0cb82fc4acb4686510a172d9a9-1",
                 "label": "input.txt",
             },
@@ -123,7 +138,12 @@ class Config:
                 {
                     "store": "0",
                     "path": "api/0a3b2c56-dbcd-4871-b93b-d454b7883f9f/input.txt",
-                }
+                },
+                # w/ store id as int
+                {
+                    "store": 0,
+                    "path": "94453a6a-c8d4-52b3-a22d-ccbf81f8d636/d4442ca4-23fd-5b6b-ba6d-0b75f711c109/y_1D.txt",
+                },
             ],
         }
 
@@ -134,13 +154,11 @@ class DatCoreFileLink(BaseFileLink):
     label: str = Field(
         ...,
         description="The real file name",
-        examples=["MyFile.txt"],
     )
 
     dataset: str = Field(
         ...,
         description="Unique identifier to access the dataset on datcore (REQUIRED for datcore)",
-        example=["N:dataset:f9f5ac51-33ea-4861-8e08-5b4faf655041"],
     )
 
     @validator("store", always=True)
@@ -162,4 +180,17 @@ class Config:
                 "path": "N:package:32df09ba-e8d6-46da-bd54-f696157de6ce",
                 "label": "initial_WTstates",
             },
+            "examples": [
+                # with store id as str
+                {
+                    "store": "1",
+                    "dataset": "N:dataset:ea2325d8-46d7-4fbd-a644-30f6433070b4",
+                    "path": "N:package:32df09ba-e8d6-46da-bd54-f696157de6ce",
+                    "label": "initial_WTstates",
+                },
+            ],
         }
+
+
+# Bundles all model links to a file vs PortLink
+LinkToFileTypes = Union[SimCoreFileLink, DatCoreFileLink, DownloadLink]
diff --git a/packages/models-library/tests/test_projects_pipeline.py b/packages/models-library/tests/test_projects_pipeline.py
@@ -3,16 +3,20 @@
 # pylint:disable=redefined-outer-name
 
 from pprint import pformat
+from typing import Dict, Type
 
 import pytest
 from models_library.projects_pipeline import ComputationTask
+from pydantic import BaseModel
 
 
 @pytest.mark.parametrize(
     "model_cls",
     (ComputationTask,),
 )
-def test_computation_task_model_examples(model_cls, model_cls_examples):
+def test_computation_task_model_examples(
+    model_cls: Type[BaseModel], model_cls_examples: Dict[str, Dict]
+):
     for name, example in model_cls_examples.items():
         print(name, ":", pformat(example))
         model_instance = model_cls(**example)

diff --git a/services/web/server/docker/boot.sh b/services/web/server/docker/boot.sh
@@ -29,8 +29,10 @@ elif [ "${SC_BUILD_TARGET}" = "production" ]; then
   APP_CONFIG=server-docker-prod.yaml
 fi
 
+APP_LOG_LEVEL=${WEBSERVER_LOGLEVEL:-${LOG_LEVEL:-${LOGLEVEL}}}
+
 # RUNNING application ----------------------------------------
-echo "$INFO" "Selected config $APP_CONFIG"
+echo "$INFO" "Selected config $APP_CONFIG w/ log-level $APP_LOG_LEVEL"
 
 # NOTE: the number of workers ```(2 x $num_cores) + 1``` is
 # the official recommendation [https://docs.gunicorn.org/en/latest/design.html#how-many-workers]
@@ -44,7 +46,7 @@ if [ "${SC_BOOT_MODE}" = "debug-ptvsd" ]; then
     --worker-class aiohttp.GunicornWebWorker \
     --workers="${WEBSERVER_GUNICORN_WORKERS:-1}" \
     --name="webserver_$(hostname)_$(date +'%Y-%m-%d_%T')_$$" \
-    --log-level="${LOG_LEVEL:-info}" \
+    --log-level="${APP_LOG_LEVEL:-info}" \
     --access-logfile='-' \
     --access-logformat='%a %t "%r" %s %b [%Dus] "%{Referer}i" "%{User-Agent}i"' \
     --reload
@@ -55,7 +57,7 @@ else
     --worker-class aiohttp.GunicornWebWorker \
     --workers="${WEBSERVER_GUNICORN_WORKERS:-1}" \
     --name="webserver_$(hostname)_$(date +'%Y-%m-%d_%T')_$$" \
-    --log-level="${LOG_LEVEL:-warning}" \
+    --log-level="${APP_LOG_LEVEL:-warning}" \
     --access-logfile='-' \
     --access-logformat='%a %t "%r" %s %b [%Dus] "%{Referer}i" "%{User-Agent}i"'
 fi
diff --git a/services/web/server/requirements/_base.in b/services/web/server/requirements/_base.in
@@ -17,67 +17,42 @@
 --requirement ../../../../packages/service-library/requirements/_aiohttp.in
 
 
-# aiohttp + extensions
-aiohttp
-aiohttp_jinja2
-aiohttp_session[secure]
-aiohttp_security
-aiohttp-swagger[performance]
-gunicorn[setproctitle]
-
-
 # web-sockets
 # From 5.0.0, https://github.com/miguelgrinberg/python-socketio/blob/main/CHANGES.md
 # test_resource_manager.py::test_websocket_resource_management fails because
 # socket_id saved in redis does not correspond to client's sio
 python-socketio~=4.6.1
 
-# postgres db
-aiopg[sa]
-asyncpg
-
-# i/o
-aiofiles
-
-# redis
-aioredis
-aioredlock
-
-# RabbitMQ client
-aio-pika
-
-# email
-aiosmtplib
-jinja_app_loader
-
-
-# data models
-pydantic[email]
-
-# security
-cryptography
-passlib
-
-# json
-orjson
-jsondiff
-json2html
-
-# asyncio debug
-aiodebug
+# SEE services/web/server/tests/unit/isolated/test_utils.py::test_yarl_url_compose_changed_with_latest_release
+yarl<1.6
 
 
-# misc
+aio-pika # RabbitMQ client
+aiodebug # asyncio debug
+aiofiles # i/o
+aiohttp
+aiohttp_jinja2
+aiohttp_security
+aiohttp_session[secure]
+aiohttp-swagger[performance]
+aiopg[sa] # db
+aioredis # redis
+aioredlock # redis
+aiosmtplib # email
+asyncpg # db
 change_case
+cryptography # security
 expiringdict
+gunicorn[setproctitle]
+jinja_app_loader # email
+json2html
+jsondiff
+openpyxl # excel
+orjson  # json
+parfive  # excel
+passlib
+pint # units
+pydantic[email] # models
+python-magic # excel
 semantic_version
 tenacity
-
-# import/export excel
-parfive
-openpyxl
-python-magic
-
-
-# SEE services/web/server/tests/unit/isolated/test_utils.py::test_yarl_url_compose_changed_with_latest_release
-yarl<1.6
diff --git a/services/web/server/requirements/_base.txt b/services/web/server/requirements/_base.txt
@@ -191,12 +191,16 @@ openpyxl==3.0.7
     # via -r requirements/_base.in
 orjson==3.5.3
     # via -r requirements/_base.in
+packaging==21.3
+    # via pint
 pamqp==2.3.0
     # via aiormq
 parfive==1.5.1
     # via -r requirements/_base.in
 passlib==1.7.4
     # via -r requirements/_base.in
+pint==0.18
+    # via -r requirements/_base.in
 prometheus-client==0.12.0
     # via -r requirements/../../../../packages/service-library/requirements/_aiohttp.in
 psycopg2-binary==2.9.1
@@ -225,6 +229,8 @@ pyinstrument==3.4.2
     #   -r requirements/../../../../packages/service-library/requirements/_base.in
 pyinstrument-cext==0.2.4
     # via pyinstrument
+pyparsing==3.0.7
+    # via packaging
 pyrsistent==0.18.0
     # via jsonschema
 python-engineio==3.14.2