ITISFoundation · matusdrobuliak66 · May 11, 2023 · May 7, 2023 · May 7, 2023 · May 8, 2023
@@ -8,12 +8,11 @@
 
 
 from enum import Enum
-from typing import Union
 
 from fastapi import FastAPI, status
 from models_library.generics import Envelope
 from pydantic import NonNegativeInt
-from simcore_service_webserver.storage_schemas import (
+from simcore_service_webserver.storage.schemas import (
     CompleteUpload,
     DatasetMetaData,
     FileLocation,
@@ -27,7 +26,7 @@
 
 app = FastAPI(redoc_url=None)
 
-TAGS: list[Union[str, Enum]] = [
+TAGS: list[str | Enum] = [
     "storage",
 ]
 

@@ -39,7 +39,7 @@
 from .session import setup_session
 from .socketio.plugin import setup_socketio
 from .statics.plugin import setup_statics
-from .storage import setup_storage
+from .storage.plugin import setup_storage
 from .studies_dispatcher.plugin import setup_studies_dispatcher
 from .tags import setup_tags
 from .tracing import setup_app_tracing

@@ -41,7 +41,7 @@
 from .scicrunch.settings import SciCrunchSettings
 from .session_settings import SessionSettings
 from .statics.settings import FrontEndAppSettings, StaticWebserverModuleSettings
-from .storage_settings import StorageSettings
+from .storage.settings import StorageSettings
 from .studies_dispatcher.settings import StudiesDispatcherSettings
 
 log = logging.getLogger(__name__)

@@ -11,11 +11,12 @@
 from servicelib.aiohttp.client_session import get_client_session
 from servicelib.utils import logged_gather
 
-from .. import catalog_client, db, storage_api
+from .. import catalog_client, db
 from .._meta import API_VERSION, APP_NAME, api_version_prefix
 from ..director_v2 import api as director_v2_api
 from ..login.decorators import login_required
 from ..security.decorators import permission_required
+from ..storage import api as storage_api
 from ..utils import get_task_info, get_tracemalloc_info
 from ..utils_aiohttp import envelope_json_response
 

@@ -98,7 +98,10 @@ async def get_confirmation(self, filter_dict) -> ConfirmationTokenDict | None:
             filter_dict["user_id"] = filter_dict.pop("user")["id"]
         async with self.pool.acquire() as conn:
             confirmation = await _sql.find_one(conn, self.confirm_tbl, filter_dict)
-            return ConfirmationTokenDict(**confirmation) if confirmation else None
+            confirmation_token: ConfirmationTokenDict | None = (
+                ConfirmationTokenDict(**confirmation) if confirmation else None  # type: ignore[misc]
+            )
+            return confirmation_token
 
     async def delete_confirmation(self, confirmation: ConfirmationTokenDict):
         async with self.pool.acquire() as conn:

@@ -17,7 +17,7 @@
 
 from ..application_settings import get_settings
 from ..director_v2 import api
-from ..storage_api import (
+from ..storage.api import (
     copy_data_folders_from_project,
     get_project_total_size_simcore_s3,
 )

@@ -14,7 +14,7 @@
 from models_library.users import UserID
 
 from ..director_v2 import api
-from ..storage_api import delete_data_folders_of_project
+from ..storage.api import delete_data_folders_of_project
 from ..users_api import UserNameDict
 from ..users_exceptions import UserNotFoundError
 from .projects_db import ProjectDBAPI

@@ -46,7 +46,7 @@
 from servicelib.utils import fire_and_forget_task, logged_gather
 from simcore_postgres_database.webserver_models import ProjectType
 
-from .. import catalog_client, storage_api
+from .. import catalog_client
 from ..director_v2 import api as director_v2_api
 from ..products.plugin import get_product_name
 from ..redis import get_redis_lock_manager_client_sdk
@@ -62,6 +62,7 @@
     send_group_messages,
     send_messages,
 )
+from ..storage import api as storage_api
 from ..users_api import UserRole, get_user_name, get_user_role
 from ..users_exceptions import UserNotFoundError
 from . import _delete_utils, _nodes_utils

@@ -19,9 +19,9 @@
 from servicelib.request_keys import RQT_USERID_KEY
 from yarl import URL
 
-from .login.decorators import login_required
-from .security.decorators import permission_required
-from .storage_settings import StorageSettings, get_plugin_settings
+from ..login.decorators import login_required
+from ..security.decorators import permission_required
+from .settings import StorageSettings, get_plugin_settings
 
 log = logging.getLogger(__name__)
 
@@ -35,7 +35,8 @@ def _get_base_storage_url(app: web.Application) -> URL:
 
 def _get_storage_vtag(app: web.Application) -> str:
     settings: StorageSettings = get_plugin_settings(app)
-    return settings.STORAGE_VTAG
+    storage_vtag: str = settings.STORAGE_VTAG
+    return storage_vtag
 
 
 def _resolve_storage_url(request: web.Request) -> URL:
@@ -51,7 +52,6 @@ def _resolve_storage_url(request: web.Request) -> URL:
     #    ('asdf', '')
     suffix = "/".join(request.url.raw_parts[BASEPATH_INDEX:])
 
-    # TODO: check request.query to storage! unsafe!?
     url = (endpoint / suffix).with_query(request.query).update_query(user_id=userid)
     return url
 
@@ -68,7 +68,6 @@ async def _request_storage(
         await extract_and_validate(request)
 
     url = _resolve_storage_url(request)
-    # _token_data, _token_secret = _get_token_key_and_secret(request)
 
     body = None
     if request.can_read_body:
@@ -88,7 +87,8 @@ def _unresolve_storage_url(request: web.Request, storage_url: AnyUrl) -> AnyUrl:
     converted_url = request.url.with_path(
         f"/v0/storage{storage_url.path.removeprefix(prefix)}"
     ).with_scheme(request.headers.get(X_FORWARDED_PROTO, request.url.scheme))
-    return parse_obj_as(AnyUrl, f"{converted_url}")
+    converted_url_: AnyUrl = parse_obj_as(AnyUrl, f"{converted_url}")
+    return converted_url_
 
 
 async def safe_unwrap(
@@ -109,7 +109,7 @@ def extract_link(data: dict | None) -> str:
     if data is None or "link" not in data:
         raise web.HTTPException(reason=f"No url found in response: '{data}'")
 
-    return data["link"]
+    return f"{data['link']}"
 
 
 # ---------------------------------------------------------------------

@@ -8,100 +8,100 @@
 from aiohttp import web
 from servicelib.aiohttp import openapi
 
-from . import storage_handlers
+from . import _handlers
 
 log = logging.getLogger(__name__)
 
 
 def create(specs: openapi.Spec) -> list[web.RouteDef]:
-    # TODO: consider the case in which server creates routes for both v0 and v1!!!
-    # TODO: should this be taken from servers instead?
+    # NOTE: consider the case in which server creates routes for both v0 and v1!!!
+    # NOTE: should this be taken from servers instead?
     BASEPATH = "/v" + specs.info.version.split(".")[0]
 
     log.debug("creating %s ", __name__)
     routes = []
 
-    # TODO: routing will be done automatically using operation_id/tags, etc...
+    # NOTE: routing will be done automatically using operation_id/tags, etc...
 
     # storage --
-    path, handler = "/storage/locations", storage_handlers.get_storage_locations
+    path, handler = "/storage/locations", _handlers.get_storage_locations
     operation_id = specs.paths[path].operations["get"].operation_id
     routes.append(web.get(BASEPATH + path, handler, name=operation_id))
 
     path, handler = (
         "/storage/locations/{location_id}:sync",
-        storage_handlers.synchronise_meta_data_table,
+        _handlers.synchronise_meta_data_table,
     )
     operation_id = specs.paths[path].operations["post"].operation_id
     routes.append(web.post(BASEPATH + path, handler, name=operation_id))
 
     path, handler = (
         "/storage/locations/{location_id}/datasets",
-        storage_handlers.get_datasets_metadata,
+        _handlers.get_datasets_metadata,
     )
     operation_id = specs.paths[path].operations["get"].operation_id
     routes.append(web.get(BASEPATH + path, handler, name=operation_id))
 
     path, handle = (
         "/storage/locations/{location_id}/files/metadata",
-        storage_handlers.get_files_metadata,
+        _handlers.get_files_metadata,
     )
     operation_id = specs.paths[path].operations["get"].operation_id
     routes.append(web.get(BASEPATH + path, handle, name=operation_id))
 
     path, handle = (
         "/storage/locations/{location_id}/datasets/{dataset_id}/metadata",
-        storage_handlers.get_files_metadata_dataset,
+        _handlers.get_files_metadata_dataset,
     )
     operation_id = specs.paths[path].operations["get"].operation_id
     routes.append(web.get(BASEPATH + path, handle, name=operation_id))
 
     path, handle = (
         "/storage/locations/{location_id}/files/{file_id}/metadata",
-        storage_handlers.get_file_metadata,
+        _handlers.get_file_metadata,
     )
     operation_id = specs.paths[path].operations["get"].operation_id
     routes.append(web.get(BASEPATH + path, handle, name=operation_id))
 
     _FILE_PATH = "/storage/locations/{location_id}/files/{file_id}"
     path, handle = (
         _FILE_PATH,
-        storage_handlers.download_file,
+        _handlers.download_file,
     )
     operation_id = specs.paths[path].operations["get"].operation_id
     routes.append(web.get(BASEPATH + path, handle, name=operation_id))
 
     path, handle = (
         _FILE_PATH,
-        storage_handlers.delete_file,
+        _handlers.delete_file,
     )
     operation_id = specs.paths[path].operations["delete"].operation_id
     routes.append(web.delete(BASEPATH + path, handle, name=operation_id))
 
     path, handle = (
         _FILE_PATH,
-        storage_handlers.upload_file,
+        _handlers.upload_file,
     )
     operation_id = specs.paths[path].operations["put"].operation_id
     routes.append(web.put(BASEPATH + path, handle, name=operation_id))
 
     path, handle = (
         f"{_FILE_PATH}:complete",
-        storage_handlers.complete_upload_file,
+        _handlers.complete_upload_file,
     )
     operation_id = specs.paths[path].operations["post"].operation_id
     routes.append(web.post(BASEPATH + path, handle, name=operation_id))
 
     path, handle = (
         f"{_FILE_PATH}:abort",
-        storage_handlers.abort_upload_file,
+        _handlers.abort_upload_file,
     )
     operation_id = specs.paths[path].operations["post"].operation_id
     routes.append(web.post(BASEPATH + path, handle, name=operation_id))
 
     path, handle = (
         f"{_FILE_PATH}:complete/futures/{{future_id}}",
-        storage_handlers.is_completed_upload_file,
+        _handlers.is_completed_upload_file,
     )
     operation_id = specs.paths[path].operations["post"].operation_id
     routes.append(web.post(BASEPATH + path, handle, name=operation_id))

@@ -25,9 +25,9 @@
 from servicelib.logging_utils import get_log_record_extra, log_context
 from yarl import URL
 
-from .projects.project_models import ProjectDict
-from .projects.projects_utils import NodesMap
-from .storage_settings import StorageSettings, get_plugin_settings
+from ..projects.project_models import ProjectDict
+from ..projects.projects_utils import NodesMap
+from .settings import StorageSettings, get_plugin_settings
 
 log = logging.getLogger(__name__)
 

@@ -8,7 +8,7 @@
 from servicelib.aiohttp.application_keys import APP_OPENAPI_SPECS_KEY
 from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup
 
-from . import storage_routes
+from . import _routes
 
 log = logging.getLogger(__name__)
 
@@ -20,5 +20,5 @@ def setup_storage(app: web.Application):
 
     specs = app[APP_OPENAPI_SPECS_KEY]  # validated openapi specs
 
-    routes = storage_routes.create(specs)
+    routes = _routes.create(specs)
     app.router.add_routes(routes)