Merge branch 'master' into maintenance/mypy-webserver-8

.github/workflows/ci-testing-deploy.yml

@@ -18,6 +18,10 @@ on:
   pull_request:
     branches:
       - "*"
+  # https://github.blog/changelog/2023-02-08-pull-request-merge-queue-public-beta/
+  merge_group:
+    branches:
+      - "master"
 
   workflow_dispatch:
     inputs:

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py

@@ -253,6 +253,9 @@ def get_dynamic_sidecar_spec(
                 "Hosts": [],
                 "Image": dynamic_sidecar_settings.DYNAMIC_SIDECAR_IMAGE,
                 "Init": True,
+                "CapabilityAdd": [
+                    "CAP_LINUX_IMMUTABLE",
+                ],
                 "Labels": {
                     # NOTE: these labels get on the tasks and that is also useful to trace
                     "user_id": f"{scheduler_data.user_id}",

services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py

@@ -217,6 +217,7 @@ def expected_dynamic_sidecar_spec(
                     "STORAGE_HOST": "storage",
                     "STORAGE_PORT": "8080",
                 },
+                "CapabilityAdd": ["CAP_LINUX_IMMUTABLE"],
                 "Hosts": [],
                 "Image": "local/dynamic-sidecar:MOCK",
                 "Init": True,

services/dynamic-sidecar/Dockerfile

@@ -19,6 +19,7 @@ RUN  --mount=type=cache,id=basecache,target=/var/cache/apt,mode=0755,sharing=loc
   apt-get install -y --no-install-recommends\
   curl \
   gosu \
+  libcap2-bin \
   ca-certificates \
   # required by python-magic
   libmagic1 \

services/dynamic-sidecar/docker/entrypoint.sh

@@ -110,4 +110,12 @@ echo "  $SC_USER_NAME rights    : $(id "$SC_USER_NAME")"
 echo "  local dir : $(ls -al)"
 echo "  volumes dir : $(ls -al "${DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR}")"
 
+echo "$INFO" "Available permissions"
+capsh --print
+
+PYTHON_BINARY=$(readlink --canonicalize $(which python))
+echo "$INFO" "Granting ${PYTHON_BINARY} CAP_LINUX_IMMUTABLE"
+setcap 'cap_linux_immutable+ep' "${PYTHON_BINARY}"
+getcap "${PYTHON_BINARY}"
+
 exec gosu "$SC_USER_NAME" "$@"

services/web/server/src/simcore_service_webserver/application.py

@@ -20,7 +20,7 @@
 from .email.plugin import setup_email
 from .exporter.plugin import setup_exporter
 from .garbage_collector import setup_garbage_collector
-from .groups import setup_groups
+from .groups.plugin import setup_groups
 from .invitations.plugin import setup_invitations
 from .login.plugin import setup_login
 from .long_running_tasks import setup_long_running_tasks
@@ -46,7 +46,7 @@
 from .users import setup_users
 from .version_control.plugin import setup_version_control
 
-logger = logging.getLogger(__name__)
+_logger = logging.getLogger(__name__)
 
 
 def create_application() -> web.Application:
@@ -128,7 +128,7 @@ async def finished_banner(_app: web.Application):
     app.on_startup.append(welcome_banner)
     app.on_shutdown.append(finished_banner)
 
-    logger.debug("Routes in app: \n %s", pformat(app.router.named_resources()))
+    _logger.debug("Routes in app: \n %s", pformat(app.router.named_resources()))
 
     return app
 

services/web/server/src/simcore_service_webserver/garbage_collector_utils.py

@@ -8,7 +8,7 @@
 
 from . import users_exceptions
 from .db_models import GroupType
-from .groups_api import get_group_from_gid
+from .groups.api import get_group_from_gid
 from .projects.projects_db import APP_PROJECT_DBAPI, ProjectAccessRights
 from .projects.projects_exceptions import ProjectNotFoundError
 from .users_api import get_user, get_user_id_from_gid

services/web/server/src/simcore_service_webserver/groups_classifiers.py → services/web/server/src/simcore_service_webserver/groups/_classifiers.py

@@ -9,33 +9,45 @@
 """
 
 import logging
-from typing import Any, Dict, Optional
+import re
+from typing import Any, Final, Literal
 
 import sqlalchemy as sa
 from aiohttp import web
 from aiopg.sa.result import RowProxy
-from pydantic import BaseModel, Field, HttpUrl, ValidationError, constr, validator
+from pydantic import (
+    BaseModel,
+    ConstrainedStr,
+    Field,
+    HttpUrl,
+    ValidationError,
+    parse_obj_as,
+    validator,
+)
 from simcore_postgres_database.models.classifiers import group_classifiers
 
-from ._constants import APP_DB_ENGINE_KEY
-from .scicrunch.db import ResearchResourceRepository
-from .scicrunch.service_client import SciCrunch
+from ..db import get_database_engine
+from ..scicrunch.db import ResearchResourceRepository
+from ..scicrunch.service_client import SciCrunch
+
+_logger = logging.getLogger(__name__)
+MAX_SIZE_SHORT_MSG: Final[int] = 100
 
-logger = logging.getLogger(__name__)
 
 # DOMAIN MODELS ---
 
-TreePath = constr(regex=r"[\w:]+")  # Examples 'a::b::c
-MAX_SIZE_SHORT_MSG = 100
+
+class TreePath(ConstrainedStr):
+    regex = re.compile(r"[\w:]+")  # Examples 'a::b::c
 
 
 class ClassifierItem(BaseModel):
     classifier: str = Field(
         ..., description="Unique identifier used to tag studies or services"
     )
     display_name: str
-    short_description: Optional[str]
-    url: Optional[HttpUrl] = Field(
+    short_description: str | None
+    url: HttpUrl | None = Field(
         None,
         description="Link to more information",
         example="https://scicrunch.org/resources/Any/search?q=osparc&l=osparc",
@@ -51,42 +63,37 @@ def truncate_to_short(cls, v):
 
 class Classifiers(BaseModel):
     # meta
-    vcs_url: Optional[str]
-    vcs_ref: Optional[str]
+    vcs_url: str | None
+    vcs_ref: str | None
 
     # data
-    classifiers: Dict[TreePath, ClassifierItem]
+    classifiers: dict[TreePath, ClassifierItem]
 
 
 # DATABASE --------
 
 
 class GroupClassifierRepository:
-    #
-    # TODO: a repo: retrieves engine from app
-    # TODO: a repo: some members acquire and retrieve connection
-    # TODO: a repo: any validation error in a repo is due to corrupt data in db!
-
     def __init__(self, app: web.Application):
-        self.engine = app[APP_DB_ENGINE_KEY]
+        self.engine = get_database_engine(app)
 
-    async def _get_bundle(self, gid: int) -> Optional[RowProxy]:
+    async def _get_bundle(self, gid: int) -> RowProxy | None:
         async with self.engine.acquire() as conn:
-            bundle: Optional[RowProxy] = await conn.scalar(
+            bundle: RowProxy | None = await conn.scalar(
                 sa.select([group_classifiers.c.bundle]).where(
                     group_classifiers.c.gid == gid
                 )
             )
             return bundle
 
-    async def get_classifiers_from_bundle(self, gid: int) -> Dict[str, Any]:
+    async def get_classifiers_from_bundle(self, gid: int) -> dict[str, Any]:
         bundle = await self._get_bundle(gid)
         if bundle:
             try:
                 # truncate bundle to what is needed and drop the rest
                 return Classifiers(**bundle).dict(exclude_unset=True, exclude_none=True)
             except ValidationError as err:
-                logger.error(
+                _logger.error(
                     "DB corrupt data in 'groups_classifiers' table. "
                     "Invalid classifier for gid=%d: %s. "
                     "Returning empty bundle.",
@@ -97,7 +104,7 @@ async def get_classifiers_from_bundle(self, gid: int) -> Dict[str, Any]:
 
     async def group_uses_scicrunch(self, gid: int) -> bool:
         async with self.engine.acquire() as conn:
-            value: Optional[RowProxy] = await conn.scalar(
+            value: RowProxy | None = await conn.scalar(
                 sa.select([group_classifiers.c.uses_scicrunch]).where(
                     group_classifiers.c.gid == gid
                 )
@@ -108,7 +115,9 @@ async def group_uses_scicrunch(self, gid: int) -> bool:
 # HELPERS FOR API HANDLERS --------------
 
 
-async def build_rrids_tree_view(app, tree_view_mode="std") -> Dict[str, Any]:
+async def build_rrids_tree_view(
+    app: web.Application, tree_view_mode: Literal["std"] = "std"
+) -> dict[str, Any]:
     if tree_view_mode != "std":
         raise web.HTTPNotImplemented(
             reason="Currently only 'std' option for the classifiers tree view is implemented"
@@ -117,7 +126,7 @@ async def build_rrids_tree_view(app, tree_view_mode="std") -> Dict[str, Any]:
     scicrunch = SciCrunch.get_instance(app)
     repo = ResearchResourceRepository(app)
 
-    flat_tree_view = {}
+    flat_tree_view: dict[TreePath, ClassifierItem] = {}
     for resource in await repo.list_resources():
         try:
             validated_item = ClassifierItem(
@@ -127,11 +136,11 @@ async def build_rrids_tree_view(app, tree_view_mode="std") -> Dict[str, Any]:
                 url=scicrunch.get_resolver_web_url(resource.rrid),
             )
 
-            node = validated_item.display_name.replace(":", " ")
+            node = parse_obj_as(TreePath, validated_item.display_name.replace(":", " "))
             flat_tree_view[node] = validated_item
 
         except ValidationError as err:
-            logger.warning(
+            _logger.warning(
                 "Cannot convert RRID into a classifier item. Skipping. Details: %s", err
             )