Merge branch 'master' into mypy/webserver-security-plugin

services/web/server/src/simcore_service_webserver/application.py

@@ -38,7 +38,7 @@
 from .security.plugin import setup_security
 from .session import setup_session
 from .socketio.plugin import setup_socketio
-from .statics import setup_statics
+from .statics.plugin import setup_statics
 from .storage import setup_storage
 from .studies_dispatcher.plugin import setup_studies_dispatcher
 from .tags import setup_tags

services/web/server/src/simcore_service_webserver/application_settings.py

@@ -40,7 +40,7 @@
 from .rest_settings import RestSettings
 from .scicrunch.settings import SciCrunchSettings
 from .session_settings import SessionSettings
-from .statics_settings import FrontEndAppSettings, StaticWebserverModuleSettings
+from .statics.settings import FrontEndAppSettings, StaticWebserverModuleSettings
 from .storage_settings import StorageSettings
 from .studies_dispatcher.settings import StudiesDispatcherSettings
 

services/web/server/src/simcore_service_webserver/products/_events.py

@@ -12,7 +12,7 @@
 )
 
 from .._constants import APP_DB_ENGINE_KEY, APP_PRODUCTS_KEY
-from ..statics_constants import FRONTEND_APP_DEFAULT, FRONTEND_APPS_AVAILABLE
+from ..statics._constants import FRONTEND_APP_DEFAULT, FRONTEND_APPS_AVAILABLE
 from ._db import iter_products
 from ._model import Product
 

services/web/server/src/simcore_service_webserver/products/_model.py

@@ -21,7 +21,7 @@
 )
 
 from ..db_models import products
-from ..statics_constants import FRONTEND_APPS_AVAILABLE
+from ..statics._constants import FRONTEND_APPS_AVAILABLE
 
 log = logging.getLogger(__name__)
 

services/web/server/src/simcore_service_webserver/statics_events.py → services/web/server/src/simcore_service_webserver/statics/_events.py

@@ -13,20 +13,20 @@
 from tenacity.wait import wait_fixed
 from yarl import URL
 
-from ._constants import APP_SETTINGS_KEY
-from .products.plugin import APP_PRODUCTS_KEY, Product
-from .statics_constants import (
+from .._constants import APP_PRODUCTS_KEY, APP_SETTINGS_KEY
+from ..products.plugin import Product
+from ._constants import (
     APP_FRONTEND_CACHED_INDEXES_KEY,
     APP_FRONTEND_CACHED_STATICS_JSON_KEY,
     FRONTEND_APPS_AVAILABLE,
 )
-from .statics_settings import (
+from .settings import (
     FrontEndAppSettings,
     StaticWebserverModuleSettings,
     get_plugin_settings,
 )
 
-log = logging.getLogger(__name__)
+_logger = logging.getLogger(__name__)
 
 
 # This retry policy aims to overcome the inconvenient fact that the swarm
@@ -39,7 +39,7 @@
 _STATIC_WEBSERVER_RETRY_ON_STARTUP_POLICY = dict(
     stop=stop_after_attempt(5),
     wait=wait_fixed(1.5),
-    before=before_log(log, logging.WARNING),
+    before=before_log(_logger, logging.WARNING),
     retry=retry_if_exception_type(ClientConnectionError),
     reraise=True,
 )
@@ -60,7 +60,7 @@ async def create_cached_indexes(app: web.Application) -> None:
 
     for frontend_name in FRONTEND_APPS_AVAILABLE:
         url = URL(settings.STATIC_WEBSERVER_URL) / frontend_name
-        log.info("Fetching index from %s", url)
+        _logger.info("Fetching index from %s", url)
         try:
             body = ""
             # web-static server might still not be up
@@ -72,7 +72,7 @@ async def create_cached_indexes(app: web.Application) -> None:
                     body = await response.text()
 
         except ClientError as err:
-            log.error("Could not fetch index from static server: %s", err)
+            _logger.error("Could not fetch index from static server: %s", err)
 
             # ANE: Yes this is supposed to fail the boot process
             raise RuntimeError(
@@ -83,7 +83,7 @@ async def create_cached_indexes(app: web.Application) -> None:
         body = body.replace(f"../resource/{frontend_name}", f"resource/{frontend_name}")
         body = body.replace("boot.js", f"{frontend_name}/boot.js")
 
-        log.info("Storing index for %s", url)
+        _logger.info("Storing index for %s", url)
         cached_indexes[frontend_name] = body
 
     app[APP_FRONTEND_CACHED_INDEXES_KEY] = cached_indexes
@@ -111,11 +111,11 @@ async def create_statics_json(app: web.Application) -> None:
     for product in products.values():
         data = deepcopy(common)
 
-        log.debug("Product %s", product.name)
+        _logger.debug("Product %s", product.name)
         data.update(product.to_statics())
 
         data_json = json_dumps(data)
-        log.debug("Front-end statics.json: %s", data_json)
+        _logger.debug("Front-end statics.json: %s", data_json)
 
         # cache computed statics.json
         app[APP_FRONTEND_CACHED_STATICS_JSON_KEY][product.name] = data_json

services/web/server/src/simcore_service_webserver/statics_handlers.py → services/web/server/src/simcore_service_webserver/statics/_handlers.py

@@ -2,14 +2,14 @@
 
 from aiohttp import web
 
-from .products.plugin import get_product_name
-from .statics_constants import (
+from ..products.plugin import get_product_name
+from ._constants import (
     APP_FRONTEND_CACHED_INDEXES_KEY,
     APP_FRONTEND_CACHED_STATICS_JSON_KEY,
     FRONTEND_APPS_AVAILABLE,
 )
 
-log = logging.getLogger(__name__)
+_logger = logging.getLogger(__name__)
 
 
 async def get_cached_frontend_index(request: web.Request):

services/web/server/src/simcore_service_webserver/statics.py → services/web/server/src/simcore_service_webserver/statics/plugin.py

@@ -11,17 +11,17 @@
 from aiohttp import web
 from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup
 
-from ._constants import INDEX_RESOURCE_NAME
-from .products.plugin import setup_products
-from .statics_events import create_cached_indexes, create_statics_json
-from .statics_handlers import get_cached_frontend_index, get_statics_json
-from .statics_settings import StaticWebserverModuleSettings, get_plugin_settings
+from .._constants import INDEX_RESOURCE_NAME
+from ..products.plugin import setup_products
+from ._events import create_cached_indexes, create_statics_json
+from ._handlers import get_cached_frontend_index, get_statics_json
+from .settings import StaticWebserverModuleSettings, get_plugin_settings
 
-log = logging.getLogger(__name__)
+_logger = logging.getLogger(__name__)
 
 
 @app_module_setup(
-    __name__, ModuleCategory.ADDON, settings_name="WEBSERVER_STATICWEB", logger=log
+    __name__, ModuleCategory.ADDON, settings_name="WEBSERVER_STATICWEB", logger=_logger
 )
 def setup_statics(app: web.Application) -> None:
 
@@ -33,11 +33,13 @@ def setup_statics(app: web.Application) -> None:
     # serves information composed by making 4 http requests (once for each product)
     # to the index.html in each of the 4 product directories /osparc, /s4l, /s4llite and /tis
     app.router.add_get("/", get_cached_frontend_index, name=INDEX_RESOURCE_NAME)
+
     # statics.json is computed here and contains information used
     # by the frontend to properly render the client
     app.router.add_get("/static-frontend-data.json", get_statics_json)
 
     # compute statics.json content
     app.on_startup.append(create_statics_json)
+
     # fetch all index.html for various frontends
     app.on_startup.append(create_cached_indexes)

services/web/server/src/simcore_service_webserver/statics_settings.py → services/web/server/src/simcore_service_webserver/statics/settings.py

@@ -2,59 +2,67 @@
 
     Typically dumped in statics.json
 """
-from typing import Any
+from typing import Any, TypedDict
 
 from aiohttp import web
 from models_library.utils.change_case import snake_to_camel
-from pydantic import AnyHttpUrl, Field
+from pydantic import AnyHttpUrl, Field, parse_obj_as
 from settings_library.base import BaseCustomSettings
 
-from ._constants import APP_SETTINGS_KEY
+from .._constants import APP_SETTINGS_KEY
+
+
+class ThirdPartyInfoDict(TypedDict):
+    name: str
+    version: str
+    url: str
+    thumbnail: str
+
 
 THIRD_PARTY_REFERENCES = [
-    dict(
+    ThirdPartyInfoDict(
         name="adminer",
         version="4.8.0",
         url="https://www.adminer.org/",
         thumbnail="https://www.adminer.org/static/images/logo.png",
     ),
-    dict(
+    ThirdPartyInfoDict(
         name="dask",
         version="-",
         url="https://docs.dask.org/en/latest/scheduler-overview.html",
         thumbnail="https://docs.dask.org/en/stable/_static/images/dask-horizontal-white.svg",
     ),
-    dict(
+    ThirdPartyInfoDict(
         name="docker",
         version="-",
         url="https://www.docker.com/",
         thumbnail="https://upload.wikimedia.org/wikipedia/en/thumb/f/f4/Docker_logo.svg/120px-Docker_logo.svg.png",
     ),
-    dict(
+    ThirdPartyInfoDict(
         name="github",
         version="-",
         url="https://github.com/",
         thumbnail="https://upload.wikimedia.org/wikipedia/commons/thumb/9/91/Octicons-mark-github.svg/2048px-Octicons-mark-github.svg.png",
     ),
-    dict(
+    ThirdPartyInfoDict(
         name="minio",
         version="-",
         url="https://min.io/",
         thumbnail="https://min.io/resources/img/logo.svg",
     ),
-    dict(
+    ThirdPartyInfoDict(
         name="portainer",
         version="-",
         url="https://www.portainer.io/",
         thumbnail="https://www.portainer.io/hubfs/Brand%20Assets/Logos/Portainer%20Logo%20Solid%20All%20-%20Blue%20no%20padding.svg",
     ),
-    dict(
+    ThirdPartyInfoDict(
         name="postgres",
         version="10.11",
         url="https://www.postgresql.org/",
         thumbnail="https://upload.wikimedia.org/wikipedia/commons/thumb/2/29/Postgresql_elephant.svg/120px-Postgresql_elephant.svg.png",
     ),
-    dict(
+    ThirdPartyInfoDict(
         name="redis",
         version="-",
         url="https://redis.io/",
@@ -84,25 +92,13 @@ def to_statics(self) -> dict[str, Any]:
 
 
 class StaticWebserverModuleSettings(BaseCustomSettings):
-    # TODO: remove
-    STATIC_WEBSERVER_ENABLED: bool = Field(
-        True,
-        description=(
-            "if enabled it will try to fetch and cache the 3 product index webpages"
-        ),
-        env=["STATIC_WEBSERVER_ENABLED", "WEBSERVER_STATIC_MODULE_ENABLED"],  # legacy
-    )
-
-    # TODO: move WEBSERVER_FRONTEND here??
-
-    # TODO: host/port
     STATIC_WEBSERVER_URL: AnyHttpUrl = Field(
-        "http://static-webserver:8000",  # NOSONAR
+        default=parse_obj_as(AnyHttpUrl, "http://static-webserver:8000"),  # NOSONAR
         description="url fort static content",
         env=[
             "STATIC_WEBSERVER_URL",
-            "WEBSERVER_STATIC_MODULE_STATIC_WEB_SERVER_URL",
-        ],  # legacy
+            "WEBSERVER_STATIC_MODULE_STATIC_WEB_SERVER_URL",  # legacy
+        ],
     )
 
 

services/web/server/tests/integration/01/test_exporter.py

@@ -70,7 +70,7 @@
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from yarl import URL
 
-log = logging.getLogger(__name__)
+_logger = logging.getLogger(__name__)
 
 pytest_simcore_core_services_selection = [
     "catalog",
@@ -532,7 +532,7 @@ def _get_header_params(header):
         with assemble_tmp_file_path(file_to_download_name) as downloaded_file_path:
             async with aiofiles.open(downloaded_file_path, mode="wb") as f:
                 await f.write(await export_response.read())
-                log.info("output_path %s", downloaded_file_path)
+                _logger.info("output_path %s", downloaded_file_path)
 
             reimported_project_uuid = await import_study_from_file(
                 client, downloaded_file_path, headers=headers

services/web/server/tests/unit/isolated/test_products_middlewares.py

@@ -13,7 +13,7 @@
 from simcore_service_webserver.products._middlewares import discover_product_middleware
 from simcore_service_webserver.products._model import Product
 from simcore_service_webserver.products.plugin import get_product_name
-from simcore_service_webserver.statics_constants import FRONTEND_APP_DEFAULT
+from simcore_service_webserver.statics._constants import FRONTEND_APP_DEFAULT
 from yarl import URL
 
 

services/web/server/tests/unit/isolated/test_statics.py

@@ -11,7 +11,7 @@
 from pathlib import Path
 
 import pytest
-from simcore_service_webserver.statics_constants import (
+from simcore_service_webserver.statics._constants import (
     FRONTEND_APP_DEFAULT,
     FRONTEND_APPS_AVAILABLE,
 )

services/web/server/tests/unit/isolated/test_statics_settings.py

@@ -3,10 +3,9 @@
 # pylint: disable=unused-variable
 
 import json
-from typing import Optional
 
 from pydantic import AnyHttpUrl, BaseModel, parse_obj_as
-from simcore_service_webserver.statics_settings import (
+from simcore_service_webserver.statics.settings import (
     THIRD_PARTY_REFERENCES,
     FrontEndAppSettings,
     StaticWebserverModuleSettings,
@@ -20,7 +19,7 @@ class OsparcDependency(BaseModel):
     name: str
     version: str
     url: AnyHttpUrl
-    thumbnail: Optional[AnyHttpUrl] = None
+    thumbnail: AnyHttpUrl | None = None
 
 
 def test_valid_osparc_dependencies():

services/web/server/tests/unit/with_dbs/03/login/test_login_utils_emails.py

@@ -25,7 +25,7 @@
     send_email_from_template,
     themed,
 )
-from simcore_service_webserver.statics_constants import FRONTEND_APPS_AVAILABLE
+from simcore_service_webserver.statics._constants import FRONTEND_APPS_AVAILABLE
 
 
 @pytest.fixture