From 97027194436b4b2fa43e74f318e7ce63af5e7cda Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Mon, 27 May 2024 11:04:35 +0200 Subject: [PATCH 01/31] add EFS mount option to sidecar --- .../core/dynamic_services_settings/sidecar.py | 15 ++++++ .../docker_service_specs/sidecar.py | 22 +++++++- .../modules/dynamic_sidecar/volumes.py | 53 ++++++++++++++++++- 3 files changed, 88 insertions(+), 2 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py index 547a81484ce..5aef568b8d8 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py +++ b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py @@ -5,6 +5,7 @@ from models_library.basic_types import BootModeEnum, PortInt from models_library.docker import DockerPlacementConstraint +from models_library.users import UserID from models_library.utils.common_validators import ( ensure_unique_dict_values_validator, ensure_unique_list_values_validator, @@ -53,6 +54,18 @@ def enforce_r_clone_requirement(cls, v: int, values) -> PositiveInt: return v +class EfsSettings(SettingsLibraryRCloneSettings): + EFS_DNS_NAME: str = Field( + description="time to cache directory entries for", + example="fs-xxx.efs.us-east-1.amazonaws.com", + ) + EFS_ENABLED_FOR_USERS: list[UserID] = Field( + default_factory=list, + example='["1", "2"]', + ) + EFS_BASE_DIRECTORY: str = Field(default="project-specific-data") + + class PlacementSettings(BaseCustomSettings): # This is just a service placement constraint, see # https://docs.docker.com/engine/swarm/services/#control-service-placement. @@ -124,6 +137,8 @@ class DynamicSidecarSettings(BaseCustomSettings, MixinLoggingSettings): DYNAMIC_SIDECAR_R_CLONE_SETTINGS: RCloneSettings = Field(auto_default_from_env=True) + DYNAMIC_SIDECAR_EFS_SETTINGS: EfsSettings | None = Field(auto_default_from_env=True) + DYNAMIC_SIDECAR_PLACEMENT_SETTINGS: PlacementSettings = Field( auto_default_from_env=True ) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py index 82b63e85af8..9b3152472b1 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py @@ -251,10 +251,30 @@ def _get_mounts( volume_size_limit=volume_size_limits.get(f"{path_to_mount}"), ) ) + + # We check whether user has access to EFS feature + _mount_efs_enabled = False + efs_settings = dynamic_sidecar_settings.DYNAMIC_SIDECAR_EFS_SETTINGS + if efs_settings and scheduler_data.user_id in efs_settings.EFS_ENABLED_FOR_USERS: + _mount_efs_enabled = True + # state paths now get mounted via different driver and are synced to s3 automatically for path_to_mount in scheduler_data.paths_mapping.state_paths: + if _mount_efs_enabled: + assert dynamic_sidecar_settings.DYNAMIC_SIDECAR_EFS_SETTINGS # nosec + mounts.append( + DynamicSidecarVolumesPathsResolver.mount_efs( + swarm_stack_name=dynamic_services_scheduler_settings.SWARM_STACK_NAME, + path=path_to_mount, + node_uuid=scheduler_data.node_uuid, + run_id=scheduler_data.run_id, + project_id=scheduler_data.project_id, + user_id=scheduler_data.user_id, + efs_settings=dynamic_sidecar_settings.DYNAMIC_SIDECAR_EFS_SETTINGS, + ) + ) # for now only enable this with dev features enabled - if app_settings.DIRECTOR_V2_DEV_FEATURE_R_CLONE_MOUNTS_ENABLED: + elif app_settings.DIRECTOR_V2_DEV_FEATURE_R_CLONE_MOUNTS_ENABLED: mounts.append( DynamicSidecarVolumesPathsResolver.mount_r_clone( swarm_stack_name=dynamic_services_scheduler_settings.SWARM_STACK_NAME, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py index 986f71e1bb1..2cbf8346849 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py @@ -9,7 +9,7 @@ from servicelib.docker_constants import PREFIX_DYNAMIC_SIDECAR_VOLUMES from settings_library.r_clone import S3Provider -from ...core.dynamic_services_settings.sidecar import RCloneSettings +from ...core.dynamic_services_settings.sidecar import EfsSettings, RCloneSettings from .errors import DynamicSidecarError DY_SIDECAR_SHARED_STORE_PATH = Path("/shared-store") @@ -75,6 +75,24 @@ def _get_s3_volume_driver_config( return driver_config +def _get_efs_volume_driver_config( + efs_settings: EfsSettings, + project_id: ProjectID, + node_uuid: NodeID, + storage_directory_name: str, +) -> dict[str, Any]: + assert "/" not in storage_directory_name # nosec + driver_config: dict[str, Any] = { + "Name": "nfs_data", + "Options": { + "type": "nfs", + "o": f"addr={efs_settings.EFS_DNS_NAME},rw,nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport", + "device": f":/{efs_settings.EFS_BASE_DIRECTORY}/{project_id}/{node_uuid}/{storage_directory_name}", + }, + } + return driver_config + + class DynamicSidecarVolumesPathsResolver: BASE_PATH: Path = Path("/dy-volumes") @@ -221,3 +239,36 @@ def mount_r_clone( ), }, } + + @classmethod + def mount_efs( + cls, + swarm_stack_name: str, + path: Path, + node_uuid: NodeID, + run_id: RunID, + project_id: ProjectID, + user_id: UserID, + efs_settings: EfsSettings, + ) -> dict[str, Any]: + return { + "Source": cls.source(path, node_uuid, run_id), + "Target": cls.target(path), + "Type": "volume", + "VolumeOptions": { + "Labels": { + "source": cls.source(path, node_uuid, run_id), + "run_id": f"{run_id}", + "node_uuid": f"{node_uuid}", + "study_id": f"{project_id}", + "user_id": f"{user_id}", + "swarm_stack_name": swarm_stack_name, + }, + "DriverConfig": _get_efs_volume_driver_config( + efs_settings=efs_settings, + project_id=project_id, + node_uuid=node_uuid, + storage_directory_name=cls._volume_name(path).strip("_"), + ), + }, + } From 99e9d87fb982d4c3373fa8735da73e19e571bcc3 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Mon, 27 May 2024 11:07:20 +0200 Subject: [PATCH 02/31] add EFS mount option to sidecar --- .../core/dynamic_services_settings/sidecar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py index 5aef568b8d8..82dbea438c4 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py +++ b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py @@ -56,7 +56,7 @@ def enforce_r_clone_requirement(cls, v: int, values) -> PositiveInt: class EfsSettings(SettingsLibraryRCloneSettings): EFS_DNS_NAME: str = Field( - description="time to cache directory entries for", + description="AWS Elastic File System DNS name", example="fs-xxx.efs.us-east-1.amazonaws.com", ) EFS_ENABLED_FOR_USERS: list[UserID] = Field( From 4109b7b9abbf71c8e8f235e05e59b2313bb37188 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Mon, 27 May 2024 18:36:03 +0200 Subject: [PATCH 03/31] fix --- .../core/dynamic_services_settings/sidecar.py | 2 +- services/docker-compose.yml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py index 82dbea438c4..b8ef19d76b9 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py +++ b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py @@ -54,7 +54,7 @@ def enforce_r_clone_requirement(cls, v: int, values) -> PositiveInt: return v -class EfsSettings(SettingsLibraryRCloneSettings): +class EfsSettings(BaseCustomSettings): EFS_DNS_NAME: str = Field( description="AWS Elastic File System DNS name", example="fs-xxx.efs.us-east-1.amazonaws.com", diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 3ef810a2a3d..70c13600010 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -300,6 +300,9 @@ services: R_CLONE_OPTION_TRANSFERS: ${R_CLONE_OPTION_TRANSFERS} R_CLONE_PROVIDER: ${R_CLONE_PROVIDER} + EFS_DNS_NAME: ${EFS_DNS_NAME} + EFS_ENABLED_FOR_USERS: ${EFS_ENABLED_FOR_USERS} + RABBIT_HOST: ${RABBIT_HOST} RABBIT_PASSWORD: ${RABBIT_PASSWORD} RABBIT_PORT: ${RABBIT_PORT} From 1004f7b729d0506d3731f1df81b430553319b2c7 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 28 May 2024 12:30:14 +0000 Subject: [PATCH 04/31] fix efs volume driver config: --- .../modules/dynamic_sidecar/volumes.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py index 2cbf8346849..35c008456c9 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py @@ -83,11 +83,10 @@ def _get_efs_volume_driver_config( ) -> dict[str, Any]: assert "/" not in storage_directory_name # nosec driver_config: dict[str, Any] = { - "Name": "nfs_data", "Options": { "type": "nfs", "o": f"addr={efs_settings.EFS_DNS_NAME},rw,nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport", - "device": f":/{efs_settings.EFS_BASE_DIRECTORY}/{project_id}/{node_uuid}/{storage_directory_name}", + "device": f":/{efs_settings.EFS_BASE_DIRECTORY}", # /{project_id}/{node_uuid}/{storage_directory_name} }, } return driver_config From a4bdbe529b94d1ede0226ff4b8576e2a2fe6d5e6 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Tue, 4 Jun 2024 16:51:04 +0200 Subject: [PATCH 05/31] daily work --- .../api_schemas_efs_guardian/__init__.py | 9 +++ .../rpc_interfaces/efs_guardian/__init__.py | 0 .../efs_guardian/efs_manager.py | 32 +++++++++++ .../api/rpc/_efs_guardian.py | 25 +++++++++ .../api/rpc/{rpc_routes.py => routes.py} | 13 ++++- .../core/application.py | 8 ++- .../core/settings.py | 5 +- .../services/efs_manager.py | 41 ++++++++++++++ .../services/efs_manager_setup.py | 54 ++++++++++++++++++ .../services/modules/__init__.py | 0 .../services/modules/rabbitmq.py | 56 +++++++++++++++++++ 11 files changed, 239 insertions(+), 4 deletions(-) create mode 100644 packages/models-library/src/models_library/api_schemas_efs_guardian/__init__.py create mode 100644 packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/efs_guardian/__init__.py create mode 100644 packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/efs_guardian/efs_manager.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/api/rpc/_efs_guardian.py rename services/efs-guardian/src/simcore_service_efs_guardian/api/rpc/{rpc_routes.py => routes.py} (53%) create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager_setup.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/services/modules/__init__.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/services/modules/rabbitmq.py diff --git a/packages/models-library/src/models_library/api_schemas_efs_guardian/__init__.py b/packages/models-library/src/models_library/api_schemas_efs_guardian/__init__.py new file mode 100644 index 00000000000..50793febaf9 --- /dev/null +++ b/packages/models-library/src/models_library/api_schemas_efs_guardian/__init__.py @@ -0,0 +1,9 @@ +from typing import Final + +from pydantic import parse_obj_as + +from ..rabbitmq_basic_types import RPCNamespace + +EFS_GUARDIAN_RPC_NAMESPACE: Final[RPCNamespace] = parse_obj_as( + RPCNamespace, "efs-guardian" +) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/efs_guardian/__init__.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/efs_guardian/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/efs_guardian/efs_manager.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/efs_guardian/efs_manager.py new file mode 100644 index 00000000000..709a179b03a --- /dev/null +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/efs_guardian/efs_manager.py @@ -0,0 +1,32 @@ +import logging +from typing import Final + +from models_library.api_schemas_efs_guardian import EFS_GUARDIAN_RPC_NAMESPACE +from models_library.projects import ProjectID +from models_library.projects_nodes_io import NodeID +from models_library.rabbitmq_basic_types import RPCMethodName +from pydantic import NonNegativeInt, parse_obj_as + +from ....logging_utils import log_decorator +from ....rabbitmq import RabbitMQRPCClient + +_logger = logging.getLogger(__name__) + + +_DEFAULT_TIMEOUT_S: Final[NonNegativeInt] = 20 + + +@log_decorator(_logger, level=logging.DEBUG) +async def get_pricing_plan( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + project_id: ProjectID, + node_id: NodeID, +) -> None: + await rabbitmq_rpc_client.request( + EFS_GUARDIAN_RPC_NAMESPACE, + parse_obj_as(RPCMethodName, "create_project_specific_data_dir"), + project_id=project_id, + node_id=node_id, + timeout_s=_DEFAULT_TIMEOUT_S, + ) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/api/rpc/_efs_guardian.py b/services/efs-guardian/src/simcore_service_efs_guardian/api/rpc/_efs_guardian.py new file mode 100644 index 00000000000..61b5e588b83 --- /dev/null +++ b/services/efs-guardian/src/simcore_service_efs_guardian/api/rpc/_efs_guardian.py @@ -0,0 +1,25 @@ +from pathlib import Path + +from fastapi import FastAPI +from models_library.projects import ProjectID +from models_library.projects_nodes_io import NodeID +from servicelib.rabbitmq import RPCRouter + +from ...services.efs_manager_setup import get_efs_manager + +router = RPCRouter() + + +@router.expose(reraise_if_error_type=()) +async def create_project_specific_data_dir( + app: FastAPI, + *, + project_id: ProjectID, + node_id: NodeID, +) -> Path: + _efs_manager = get_efs_manager(app) + + return await _efs_manager.create_project_specific_data_dir( + project_id=project_id, + node_id=node_id, + ) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/api/rpc/rpc_routes.py b/services/efs-guardian/src/simcore_service_efs_guardian/api/rpc/routes.py similarity index 53% rename from services/efs-guardian/src/simcore_service_efs_guardian/api/rpc/rpc_routes.py rename to services/efs-guardian/src/simcore_service_efs_guardian/api/rpc/routes.py index c79ed1f7ed3..9a1f349fa29 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/api/rpc/rpc_routes.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/api/rpc/routes.py @@ -1,11 +1,22 @@ from collections.abc import Awaitable, Callable from fastapi import FastAPI +from models_library.api_schemas_efs_guardian import EFS_GUARDIAN_RPC_NAMESPACE +from servicelib.rabbitmq import RPCRouter + +from ...services.modules.rabbitmq import get_rabbitmq_rpc_server +from . import _efs_guardian + +ROUTERS: list[RPCRouter] = [ + _efs_guardian.router, +] def on_app_startup(app: FastAPI) -> Callable[[], Awaitable[None]]: async def _start() -> None: - assert app # nosec + rpc_server = get_rabbitmq_rpc_server(app) + for router in ROUTERS: + await rpc_server.register_router(router, EFS_GUARDIAN_RPC_NAMESPACE, app) return _start diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py b/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py index da0d9deb0d2..88c20f25ea3 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py @@ -11,7 +11,9 @@ APP_STARTED_DISABLED_BANNER_MSG, ) from ..api.rest.routes import setup_api_routes -from ..api.rpc.rpc_routes import setup_rpc_routes +from ..api.rpc.routes import setup_rpc_routes +from ..services.efs_manager_setup import setup as setup_efs_manager +from ..services.modules.rabbitmq import setup as setup_rabbitmq from .settings import ApplicationSettings logger = logging.getLogger(__name__) @@ -34,10 +36,12 @@ def create_app(settings: ApplicationSettings) -> FastAPI: assert app.state.settings.API_VERSION == API_VERSION # nosec # PLUGINS SETUP + setup_rabbitmq(app) + setup_api_routes(app) setup_rpc_routes(app) - # ERROR HANDLERS + setup_efs_manager(app) # EVENTS async def _on_startup() -> None: diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py index aedbca71f0c..607220b9c8a 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py @@ -10,6 +10,7 @@ ) from pydantic import Field, PositiveInt, validator from settings_library.base import BaseCustomSettings +from settings_library.rabbit import RabbitSettings from settings_library.utils_logging import MixinLoggingSettings from .._meta import API_VERSION, API_VTAG, APP_NAME @@ -22,7 +23,8 @@ class AwsEfsSettings(BaseCustomSettings): description="AWS Elastic File System DNS name", example="fs-xxx.efs.us-east-1.amazonaws.com", ) - EFS_BASE_DIRECTORY: str = Field(default="project-specific-data") + EFS_PROJECT_SPECIFIC_DATA_DIRECTORY: str = Field(default="project-specific-data") + EFS_MOUNTED_PATH: str = Field(default="/data/efs") class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): @@ -69,6 +71,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): EFS_GUARDIAN_AWS_EFS_SETTINGS: AwsEfsSettings | None = Field( auto_default_from_env=True ) + EFS_GUARDIAN_RABBITMQ: RabbitSettings | None = Field(auto_default_from_env=True) @cached_property def LOG_LEVEL(self) -> LogLevel: # noqa: N802 diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager.py new file mode 100644 index 00000000000..aa5ee44382e --- /dev/null +++ b/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager.py @@ -0,0 +1,41 @@ +from dataclasses import dataclass +from pathlib import Path + +from fastapi import FastAPI +from models_library.projects import ProjectID +from models_library.projects_nodes_io import NodeID + + +@dataclass(frozen=True) +class EfsManager: + app: FastAPI + + _efs_mounted_path: str + _project_specific_data_base_directory: str + + @classmethod + async def create( + cls, + app: FastAPI, + efs_mounted_path: str, + project_specific_data_base_directory: str, + ): + return cls(app, efs_mounted_path, project_specific_data_base_directory) + + async def initialize_directories(self): + _dir_path = ( + Path(self._efs_mounted_path) / self._project_specific_data_base_directory + ) + Path.mkdir(_dir_path, parents=True, exist_ok=True) + + async def create_project_specific_data_dir( + self, project_id: ProjectID, node_id: NodeID + ) -> Path: + _dir_path = ( + Path(self._efs_mounted_path) + / self._project_specific_data_base_directory + / f"{project_id}" + / f"{node_id}" + ) + Path.mkdir(_dir_path, parents=True, exist_ok=True) + return _dir_path diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager_setup.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager_setup.py new file mode 100644 index 00000000000..9f0ded69552 --- /dev/null +++ b/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager_setup.py @@ -0,0 +1,54 @@ +import logging +from typing import cast + +from fastapi import FastAPI +from simcore_service_efs_guardian.core.settings import AwsEfsSettings +from tenacity import ( + AsyncRetrying, + before_sleep_log, + stop_after_delay, + wait_random_exponential, +) + +from ..exceptions.custom_errors import ApplicationSetupError +from .efs_manager import EfsManager + +_logger = logging.getLogger(__name__) + + +def setup(app: FastAPI) -> None: + async def on_startup() -> None: + aws_efs_settings: AwsEfsSettings = ( + app.state.settings.EFS_GUARDIAN_AWS_EFS_SETTINGS + ) + + app.state.efs_manager = None + app.state.efs_manager = efs_manager = await EfsManager.create( + app, + aws_efs_settings.EFS_MOUNTED_PATH, + aws_efs_settings.EFS_PROJECT_SPECIFIC_DATA_DIRECTORY, + ) + + async for attempt in AsyncRetrying( + reraise=True, + stop=stop_after_delay(120), + wait=wait_random_exponential(max=30), + before_sleep=before_sleep_log(_logger, logging.WARNING), + ): + with attempt: + await efs_manager.initialize_directories() + + async def on_shutdown() -> None: + if app.state.efs_manager: + ... + + app.add_event_handler("startup", on_startup) + app.add_event_handler("shutdown", on_shutdown) + + +def get_efs_manager(app: FastAPI) -> EfsManager: + if not app.state.efs_manager: + raise ApplicationSetupError( + msg="Efs Manager is not available. Please check the configuration." + ) + return cast(EfsManager, app.state.efs_manager) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/modules/__init__.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/modules/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/modules/rabbitmq.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/modules/rabbitmq.py new file mode 100644 index 00000000000..82ef1aae84c --- /dev/null +++ b/services/efs-guardian/src/simcore_service_efs_guardian/services/modules/rabbitmq.py @@ -0,0 +1,56 @@ +import logging +from typing import cast + +from fastapi import FastAPI +from servicelib.rabbitmq import ( + RabbitMQClient, + RabbitMQRPCClient, + wait_till_rabbitmq_responsive, +) +from settings_library.rabbit import RabbitSettings + +from ...exceptions.custom_errors import ApplicationSetupError + +logger = logging.getLogger(__name__) + + +def setup(app: FastAPI) -> None: + async def on_startup() -> None: + app.state.rabbitmq_client = None + settings: RabbitSettings | None = app.state.settings.EFS_GUARDIAN_RABBITMQ + if not settings: + raise ApplicationSetupError( + msg="Rabbit MQ client is de-activated in the settings" + ) + await wait_till_rabbitmq_responsive(settings.dsn) + app.state.rabbitmq_client = RabbitMQClient( + client_name="efs-guardian", settings=settings + ) + app.state.rabbitmq_rpc_server = await RabbitMQRPCClient.create( + client_name="efs_guardian_rpc_server", settings=settings + ) + + async def on_shutdown() -> None: + if app.state.rabbitmq_client: + await app.state.rabbitmq_client.close() + if app.state.rabbitmq_rpc_server: + await app.state.rabbitmq_rpc_server.close() + + app.add_event_handler("startup", on_startup) + app.add_event_handler("shutdown", on_shutdown) + + +def get_rabbitmq_client(app: FastAPI) -> RabbitMQClient: + if not app.state.rabbitmq_client: + raise ApplicationSetupError( + msg="RabbitMQ client is not available. Please check the configuration." + ) + return cast(RabbitMQClient, app.state.rabbitmq_client) + + +def get_rabbitmq_rpc_server(app: FastAPI) -> RabbitMQRPCClient: + assert app.state.rabbitmq_rpc_server # nosec + return cast(RabbitMQRPCClient, app.state.rabbitmq_rpc_server) + + +__all__ = ("RabbitMQClient",) From 417d9d956ab76cf41ebc595de7d31d64e44a937b Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 5 Jun 2024 16:14:31 +0200 Subject: [PATCH 06/31] adding tests --- .../efs_guardian/efs_manager.py | 8 +++-- services/efs-guardian/tests/unit/conftest.py | 26 +++++++++++++- .../tests/unit/test_api_health.py | 6 +++- .../tests/unit/test_efs_manager.py | 34 +++++++++++++++++++ 4 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 services/efs-guardian/tests/unit/test_efs_manager.py diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/efs_guardian/efs_manager.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/efs_guardian/efs_manager.py index 709a179b03a..b5e8aa219e9 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/efs_guardian/efs_manager.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/efs_guardian/efs_manager.py @@ -1,4 +1,5 @@ import logging +from pathlib import Path from typing import Final from models_library.api_schemas_efs_guardian import EFS_GUARDIAN_RPC_NAMESPACE @@ -17,16 +18,17 @@ @log_decorator(_logger, level=logging.DEBUG) -async def get_pricing_plan( +async def create_project_specific_data_dir( rabbitmq_rpc_client: RabbitMQRPCClient, *, project_id: ProjectID, node_id: NodeID, -) -> None: - await rabbitmq_rpc_client.request( +) -> Path: + output: Path = await rabbitmq_rpc_client.request( EFS_GUARDIAN_RPC_NAMESPACE, parse_obj_as(RPCMethodName, "create_project_specific_data_dir"), project_id=project_id, node_id=node_id, timeout_s=_DEFAULT_TIMEOUT_S, ) + return output diff --git a/services/efs-guardian/tests/unit/conftest.py b/services/efs-guardian/tests/unit/conftest.py index 9c53ab29a3f..7017b13c222 100644 --- a/services/efs-guardian/tests/unit/conftest.py +++ b/services/efs-guardian/tests/unit/conftest.py @@ -3,8 +3,9 @@ # pylint:disable=redefined-outer-name import re -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Callable from pathlib import Path +from typing import Awaitable import httpx import pytest @@ -14,13 +15,24 @@ from fastapi import FastAPI from httpx import ASGITransport from pytest_simcore.helpers.utils_envs import EnvVarsDict, setenvs_from_dict +from servicelib.rabbitmq import RabbitMQRPCClient +from settings_library.rabbit import RabbitSettings from simcore_service_efs_guardian.core.application import create_app from simcore_service_efs_guardian.core.settings import ApplicationSettings pytest_plugins = [ "pytest_simcore.cli_runner", + "pytest_simcore.docker_compose", + "pytest_simcore.docker_registry", + "pytest_simcore.docker_swarm", "pytest_simcore.environment_configs", + "pytest_simcore.pydantic_models", + "pytest_simcore.pytest_global_environs", + "pytest_simcore.rabbit_service", "pytest_simcore.repository_paths", + "pytest_simcore.tmp_path_extra", + "pytest_simcore.aws_s3_service", + "pytest_simcore.aws_server", ] @@ -83,6 +95,9 @@ def app_environment( monkeypatch, { **docker_compose_service_efs_guardian_env_vars, + "EFS_DNS_NAME": "fs-xxx.efs.us-east-1.amazonaws.com", + "EFS_MOUNTED_PATH": "/tmp/efs", + "EFS_PROJECT_SPECIFIC_DATA_DIRECTORY": "project-specific-data", }, ) @@ -115,3 +130,12 @@ async def client(app: FastAPI) -> AsyncIterator[httpx.AsyncClient]: client._transport, ASGITransport # pylint: disable=protected-access ) yield client + + +@pytest.fixture +async def rpc_client( + rabbit_service: RabbitSettings, + app: FastAPI, + rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], +) -> RabbitMQRPCClient: + return await rabbitmq_rpc_client("client") diff --git a/services/efs-guardian/tests/unit/test_api_health.py b/services/efs-guardian/tests/unit/test_api_health.py index 791fb2bee26..476df2003cd 100644 --- a/services/efs-guardian/tests/unit/test_api_health.py +++ b/services/efs-guardian/tests/unit/test_api_health.py @@ -3,10 +3,14 @@ # pylint:disable=redefined-outer-name import httpx +from settings_library.rabbit import RabbitSettings from starlette import status +pytest_simcore_core_services_selection = ["rabbit"] +pytest_simcore_ops_services_selection = [] -async def test_healthcheck(client: httpx.AsyncClient): + +async def test_healthcheck(rabbit_service: RabbitSettings, client: httpx.AsyncClient): response = await client.get("/") response.raise_for_status() assert response.status_code == status.HTTP_200_OK diff --git a/services/efs-guardian/tests/unit/test_efs_manager.py b/services/efs-guardian/tests/unit/test_efs_manager.py new file mode 100644 index 00000000000..ce16894efeb --- /dev/null +++ b/services/efs-guardian/tests/unit/test_efs_manager.py @@ -0,0 +1,34 @@ +from pathlib import Path + +from faker import Faker +from fastapi import FastAPI +from servicelib.rabbitmq import RabbitMQRPCClient +from servicelib.rabbitmq.rpc_interfaces.efs_guardian import efs_manager +from simcore_service_efs_guardian.core.settings import AwsEfsSettings + +pytest_simcore_core_services_selection = ["rabbit"] +pytest_simcore_ops_services_selection = [] + + +async def test_rpc_pricing_plans_workflow( + rpc_client: RabbitMQRPCClient, + faker: Faker, + app: FastAPI, +): + aws_efs_settings: AwsEfsSettings = app.state.settings.EFS_GUARDIAN_AWS_EFS_SETTINGS + + _project_id = faker.uuid4() + _node_id = faker.uuid4() + + result = await efs_manager.create_project_specific_data_dir( + rpc_client, project_id=_project_id, node_id=_node_id + ) + assert isinstance(result, Path) + _expected_path = ( + Path(aws_efs_settings.EFS_MOUNTED_PATH) + / aws_efs_settings.EFS_PROJECT_SPECIFIC_DATA_DIRECTORY + / _project_id + / _node_id + ) + assert _expected_path == result + assert _expected_path.exists From 353848590f28d1a8149db3f5d029a004ff3ae84f Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 5 Jun 2024 16:16:41 +0200 Subject: [PATCH 07/31] adding tests --- services/efs-guardian/tests/unit/test_efs_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/efs-guardian/tests/unit/test_efs_manager.py b/services/efs-guardian/tests/unit/test_efs_manager.py index ce16894efeb..be138ba9ab0 100644 --- a/services/efs-guardian/tests/unit/test_efs_manager.py +++ b/services/efs-guardian/tests/unit/test_efs_manager.py @@ -10,7 +10,7 @@ pytest_simcore_ops_services_selection = [] -async def test_rpc_pricing_plans_workflow( +async def test_rpc_create_project_specific_data_dir( rpc_client: RabbitMQRPCClient, faker: Faker, app: FastAPI, From 0590ddfed4ffea8aedb646f7767105f466a7925b Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 09:22:21 +0200 Subject: [PATCH 08/31] review @sanderegg --- .../src/simcore_service_efs_guardian/core/settings.py | 3 ++- .../services/efs_manager.py | 10 ++++------ services/efs-guardian/tests/unit/test_efs_manager.py | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py index 607220b9c8a..96363ec9e81 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py @@ -1,4 +1,5 @@ from functools import cached_property +from pathlib import Path from typing import Final, cast from fastapi import FastAPI @@ -24,7 +25,7 @@ class AwsEfsSettings(BaseCustomSettings): example="fs-xxx.efs.us-east-1.amazonaws.com", ) EFS_PROJECT_SPECIFIC_DATA_DIRECTORY: str = Field(default="project-specific-data") - EFS_MOUNTED_PATH: str = Field(default="/data/efs") + EFS_MOUNTED_PATH: Path = Field(default="/data/efs") class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager.py index aa5ee44382e..4e249188c47 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager.py @@ -10,29 +10,27 @@ class EfsManager: app: FastAPI - _efs_mounted_path: str + _efs_mounted_path: Path _project_specific_data_base_directory: str @classmethod async def create( cls, app: FastAPI, - efs_mounted_path: str, + efs_mounted_path: Path, project_specific_data_base_directory: str, ): return cls(app, efs_mounted_path, project_specific_data_base_directory) async def initialize_directories(self): - _dir_path = ( - Path(self._efs_mounted_path) / self._project_specific_data_base_directory - ) + _dir_path = self._efs_mounted_path / self._project_specific_data_base_directory Path.mkdir(_dir_path, parents=True, exist_ok=True) async def create_project_specific_data_dir( self, project_id: ProjectID, node_id: NodeID ) -> Path: _dir_path = ( - Path(self._efs_mounted_path) + self._efs_mounted_path / self._project_specific_data_base_directory / f"{project_id}" / f"{node_id}" diff --git a/services/efs-guardian/tests/unit/test_efs_manager.py b/services/efs-guardian/tests/unit/test_efs_manager.py index be138ba9ab0..3f224074f84 100644 --- a/services/efs-guardian/tests/unit/test_efs_manager.py +++ b/services/efs-guardian/tests/unit/test_efs_manager.py @@ -25,7 +25,7 @@ async def test_rpc_create_project_specific_data_dir( ) assert isinstance(result, Path) _expected_path = ( - Path(aws_efs_settings.EFS_MOUNTED_PATH) + aws_efs_settings.EFS_MOUNTED_PATH / aws_efs_settings.EFS_PROJECT_SPECIFIC_DATA_DIRECTORY / _project_id / _node_id From bf39b37c3253dda31b77ecb716cc34f33ab94d2b Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 10:15:32 +0200 Subject: [PATCH 09/31] review @sanderegg --- .env-devel | 2 ++ services/docker-compose.yml | 6 ++++++ .../core/settings.py | 6 ++---- services/efs-guardian/tests/unit/conftest.py | 14 ++++++++++++++ .../efs-guardian/tests/unit/test_api_health.py | 18 ++++++++++++++++++ .../tests/unit/test_efs_manager.py | 18 ++++++++++++++++++ 6 files changed, 60 insertions(+), 4 deletions(-) diff --git a/.env-devel b/.env-devel index 6d9f9cf3f5a..5d9562f0a3c 100644 --- a/.env-devel +++ b/.env-devel @@ -63,6 +63,8 @@ DIRECTOR_PORT=8080 DIRECTOR_REGISTRY_CACHING_TTL=900 DIRECTOR_REGISTRY_CACHING=True +EFS_DNS_NAME=fs-xxx.efs.us-east-1.amazonaws.com + # DIRECTOR_V2 ---- COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"/home/scu/.dask/dask-crt.pem","tls_client_cert":"/home/scu/.dask/dask-crt.pem","tls_client_key":"/home/scu/.dask/dask-key.pem"}' COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_FILE_LINK_TYPE=S3 diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 1c94c000275..9055c0f6723 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -351,6 +351,12 @@ services: - default environment: LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + RABBIT_HOST: ${RABBIT_HOST} + RABBIT_PASSWORD: ${RABBIT_PASSWORD} + RABBIT_PORT: ${RABBIT_PORT} + RABBIT_SECURE: ${RABBIT_SECURE} + RABBIT_USER: ${RABBIT_USER} + EFS_DNS_NAME: ${EFS_DNS_NAME} invitations: image: ${DOCKER_REGISTRY:-itisfoundation}/invitations:${DOCKER_IMAGE_TAG:-latest} diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py index 96363ec9e81..6dfd4a730f7 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py @@ -69,10 +69,8 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) - EFS_GUARDIAN_AWS_EFS_SETTINGS: AwsEfsSettings | None = Field( - auto_default_from_env=True - ) - EFS_GUARDIAN_RABBITMQ: RabbitSettings | None = Field(auto_default_from_env=True) + EFS_GUARDIAN_AWS_EFS_SETTINGS: AwsEfsSettings = Field(auto_default_from_env=True) + EFS_GUARDIAN_RABBITMQ: RabbitSettings = Field(auto_default_from_env=True) @cached_property def LOG_LEVEL(self) -> LogLevel: # noqa: N802 diff --git a/services/efs-guardian/tests/unit/conftest.py b/services/efs-guardian/tests/unit/conftest.py index 7017b13c222..1030c433bd9 100644 --- a/services/efs-guardian/tests/unit/conftest.py +++ b/services/efs-guardian/tests/unit/conftest.py @@ -139,3 +139,17 @@ async def rpc_client( rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], ) -> RabbitMQRPCClient: return await rabbitmq_rpc_client("client") + + +# @pytest.fixture +# def mocked_setup_rabbitmq(mocker: MockerFixture): +# return ( +# mocker.patch( +# "simcore_service_efs_guardian.core.application.setup_rabbitmq", +# autospec=True, +# ), +# mocker.patch( +# "simcore_service_efs_guardian.core.application.setup_rpc_routes", +# autospec=True, +# ), +# ) diff --git a/services/efs-guardian/tests/unit/test_api_health.py b/services/efs-guardian/tests/unit/test_api_health.py index 476df2003cd..22bc3377a8f 100644 --- a/services/efs-guardian/tests/unit/test_api_health.py +++ b/services/efs-guardian/tests/unit/test_api_health.py @@ -3,6 +3,9 @@ # pylint:disable=redefined-outer-name import httpx +import pytest +from pytest_simcore.helpers.typing_env import EnvVarsDict +from pytest_simcore.helpers.utils_envs import setenvs_from_dict from settings_library.rabbit import RabbitSettings from starlette import status @@ -10,6 +13,21 @@ pytest_simcore_ops_services_selection = [] +@pytest.fixture +def app_environment( + monkeypatch: pytest.MonkeyPatch, + app_environment: EnvVarsDict, + rabbit_env_vars_dict: EnvVarsDict, # rabbitMQ settings from 'rabbit' service +) -> EnvVarsDict: + return setenvs_from_dict( + monkeypatch, + { + **app_environment, + **rabbit_env_vars_dict, + }, + ) + + async def test_healthcheck(rabbit_service: RabbitSettings, client: httpx.AsyncClient): response = await client.get("/") response.raise_for_status() diff --git a/services/efs-guardian/tests/unit/test_efs_manager.py b/services/efs-guardian/tests/unit/test_efs_manager.py index 3f224074f84..ed5491f0cac 100644 --- a/services/efs-guardian/tests/unit/test_efs_manager.py +++ b/services/efs-guardian/tests/unit/test_efs_manager.py @@ -1,7 +1,10 @@ from pathlib import Path +import pytest from faker import Faker from fastapi import FastAPI +from pytest_simcore.helpers.typing_env import EnvVarsDict +from pytest_simcore.helpers.utils_envs import setenvs_from_dict from servicelib.rabbitmq import RabbitMQRPCClient from servicelib.rabbitmq.rpc_interfaces.efs_guardian import efs_manager from simcore_service_efs_guardian.core.settings import AwsEfsSettings @@ -10,6 +13,21 @@ pytest_simcore_ops_services_selection = [] +@pytest.fixture +def app_environment( + monkeypatch: pytest.MonkeyPatch, + app_environment: EnvVarsDict, + rabbit_env_vars_dict: EnvVarsDict, # rabbitMQ settings from 'rabbit' service +) -> EnvVarsDict: + return setenvs_from_dict( + monkeypatch, + { + **app_environment, + **rabbit_env_vars_dict, + }, + ) + + async def test_rpc_create_project_specific_data_dir( rpc_client: RabbitMQRPCClient, faker: Faker, From 2fa1f4a0b02a1c87782408b3441f764ce43c779d Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 11:00:12 +0200 Subject: [PATCH 10/31] fix --- .../src/simcore_service_efs_guardian/core/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py index 6dfd4a730f7..f9ea4936269 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py @@ -25,7 +25,7 @@ class AwsEfsSettings(BaseCustomSettings): example="fs-xxx.efs.us-east-1.amazonaws.com", ) EFS_PROJECT_SPECIFIC_DATA_DIRECTORY: str = Field(default="project-specific-data") - EFS_MOUNTED_PATH: Path = Field(default="/data/efs") + EFS_MOUNTED_PATH: Path = Field(default=Path("/data/efs")) class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): From a11d65d5de992c072479ffbb964bb2600d95a347 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 11:06:21 +0200 Subject: [PATCH 11/31] fix --- .env-devel | 1 + services/docker-compose.yml | 1 + .../src/simcore_service_efs_guardian/core/settings.py | 5 ++++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.env-devel b/.env-devel index c0ea57e596b..b93c1ad9b9b 100644 --- a/.env-devel +++ b/.env-devel @@ -65,6 +65,7 @@ DIRECTOR_REGISTRY_CACHING_TTL=900 DIRECTOR_REGISTRY_CACHING=True EFS_DNS_NAME=fs-xxx.efs.us-east-1.amazonaws.com +EFS_MOUNTED_PATH=/tmp/efs # DIRECTOR_V2 ---- COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"/home/scu/.dask/dask-crt.pem","tls_client_cert":"/home/scu/.dask/dask-crt.pem","tls_client_key":"/home/scu/.dask/dask-key.pem"}' diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 9d8747ce023..194e76769e8 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -357,6 +357,7 @@ services: RABBIT_SECURE: ${RABBIT_SECURE} RABBIT_USER: ${RABBIT_USER} EFS_DNS_NAME: ${EFS_DNS_NAME} + EFS_MOUNTED_PATH: ${EFS_MOUNTED_PATH} invitations: image: ${DOCKER_REGISTRY:-itisfoundation}/invitations:${DOCKER_IMAGE_TAG:-latest} diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py index f9ea4936269..e35bcd64d0a 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py @@ -25,7 +25,10 @@ class AwsEfsSettings(BaseCustomSettings): example="fs-xxx.efs.us-east-1.amazonaws.com", ) EFS_PROJECT_SPECIFIC_DATA_DIRECTORY: str = Field(default="project-specific-data") - EFS_MOUNTED_PATH: Path = Field(default=Path("/data/efs")) + EFS_MOUNTED_PATH: Path = Field( + default=Path("/data/efs"), + description="This is the path where EFS is mounted to the EC2 machine", + ) class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): From 16ad846994c31823d721118d571b2e18b07e07ce Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 11:25:16 +0200 Subject: [PATCH 12/31] modifying settings --- .env-devel | 1 + .../src/settings_library/efs.py | 22 +++++++++++++++++++ services/docker-compose.yml | 2 ++ .../core/settings.py | 14 +----------- 4 files changed, 26 insertions(+), 13 deletions(-) create mode 100644 packages/settings-library/src/settings_library/efs.py diff --git a/.env-devel b/.env-devel index b93c1ad9b9b..aaf3e14befe 100644 --- a/.env-devel +++ b/.env-devel @@ -66,6 +66,7 @@ DIRECTOR_REGISTRY_CACHING=True EFS_DNS_NAME=fs-xxx.efs.us-east-1.amazonaws.com EFS_MOUNTED_PATH=/tmp/efs +EFS_ENABLED_FOR_USERS=[] # DIRECTOR_V2 ---- COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"/home/scu/.dask/dask-crt.pem","tls_client_cert":"/home/scu/.dask/dask-crt.pem","tls_client_key":"/home/scu/.dask/dask-key.pem"}' diff --git a/packages/settings-library/src/settings_library/efs.py b/packages/settings-library/src/settings_library/efs.py new file mode 100644 index 00000000000..86166072dba --- /dev/null +++ b/packages/settings-library/src/settings_library/efs.py @@ -0,0 +1,22 @@ +from pathlib import Path + +from models_library.users import UserID +from pydantic import Field + +from .base import BaseCustomSettings + + +class AwsEfsSettings(BaseCustomSettings): + EFS_DNS_NAME: str = Field( + description="AWS Elastic File System DNS name", + example="fs-xxx.efs.us-east-1.amazonaws.com", + ) + EFS_PROJECT_SPECIFIC_DATA_DIRECTORY: str = Field(default="project-specific-data") + EFS_MOUNTED_PATH: Path = Field( + default=Path("/data/efs"), + description="This is the path where EFS is mounted to the EC2 machine", + ) + EFS_ENABLED_FOR_USERS: list[UserID] = Field( + description="This is temporary solution so we can enable it for specific users for testing purpose", + example=[1], + ) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 911da8d0013..02243853817 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -301,6 +301,7 @@ services: R_CLONE_PROVIDER: ${R_CLONE_PROVIDER} EFS_DNS_NAME: ${EFS_DNS_NAME} + EFS_MOUNTED_PATH: ${EFS_MOUNTED_PATH} EFS_ENABLED_FOR_USERS: ${EFS_ENABLED_FOR_USERS} RABBIT_HOST: ${RABBIT_HOST} @@ -361,6 +362,7 @@ services: RABBIT_USER: ${RABBIT_USER} EFS_DNS_NAME: ${EFS_DNS_NAME} EFS_MOUNTED_PATH: ${EFS_MOUNTED_PATH} + EFS_ENABLED_FOR_USERS: ${EFS_ENABLED_FOR_USERS} invitations: image: ${DOCKER_REGISTRY:-itisfoundation}/invitations:${DOCKER_IMAGE_TAG:-latest} diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py index e35bcd64d0a..adf172a4b0c 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py @@ -1,5 +1,4 @@ from functools import cached_property -from pathlib import Path from typing import Final, cast from fastapi import FastAPI @@ -11,6 +10,7 @@ ) from pydantic import Field, PositiveInt, validator from settings_library.base import BaseCustomSettings +from settings_library.efs import AwsEfsSettings from settings_library.rabbit import RabbitSettings from settings_library.utils_logging import MixinLoggingSettings @@ -19,18 +19,6 @@ EFS_GUARDIAN_ENV_PREFIX: Final[str] = "EFS_GUARDIAN_" -class AwsEfsSettings(BaseCustomSettings): - EFS_DNS_NAME: str = Field( - description="AWS Elastic File System DNS name", - example="fs-xxx.efs.us-east-1.amazonaws.com", - ) - EFS_PROJECT_SPECIFIC_DATA_DIRECTORY: str = Field(default="project-specific-data") - EFS_MOUNTED_PATH: Path = Field( - default=Path("/data/efs"), - description="This is the path where EFS is mounted to the EC2 machine", - ) - - class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): # CODE STATICS --------------------------------------------------------- API_VERSION: str = API_VERSION From 1ceab311a2166d4d6abfce1553ca42ec29954216 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 11:27:02 +0200 Subject: [PATCH 13/31] fix test --- services/efs-guardian/tests/unit/test_efs_manager.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/services/efs-guardian/tests/unit/test_efs_manager.py b/services/efs-guardian/tests/unit/test_efs_manager.py index ed5491f0cac..acc72a9790f 100644 --- a/services/efs-guardian/tests/unit/test_efs_manager.py +++ b/services/efs-guardian/tests/unit/test_efs_manager.py @@ -1,3 +1,9 @@ +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name +# pylint: disable=too-many-arguments +# pylint: disable=unused-argument +# pylint: disable=unused-variable + from pathlib import Path import pytest From 2bd963dcc3355d6c74eb1a05c0a737a82c8868a0 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 12:26:23 +0200 Subject: [PATCH 14/31] merge master --- .../core/dynamic_services_settings/sidecar.py | 18 ++++-------------- .../docker_service_specs/sidecar.py | 17 ++++++++++++++--- .../scheduler/_core/_event_create_sidecars.py | 7 +++++-- ...les_dynamic_sidecar_docker_service_specs.py | 6 +++--- 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py index b8ef19d76b9..35d1e9e7afb 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py +++ b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py @@ -5,13 +5,13 @@ from models_library.basic_types import BootModeEnum, PortInt from models_library.docker import DockerPlacementConstraint -from models_library.users import UserID from models_library.utils.common_validators import ( ensure_unique_dict_values_validator, ensure_unique_list_values_validator, ) from pydantic import Field, PositiveInt, validator from settings_library.base import BaseCustomSettings +from settings_library.efs import AwsEfsSettings from settings_library.r_clone import RCloneSettings as SettingsLibraryRCloneSettings from settings_library.utils_logging import MixinLoggingSettings from settings_library.utils_service import DEFAULT_FASTAPI_PORT @@ -54,18 +54,6 @@ def enforce_r_clone_requirement(cls, v: int, values) -> PositiveInt: return v -class EfsSettings(BaseCustomSettings): - EFS_DNS_NAME: str = Field( - description="AWS Elastic File System DNS name", - example="fs-xxx.efs.us-east-1.amazonaws.com", - ) - EFS_ENABLED_FOR_USERS: list[UserID] = Field( - default_factory=list, - example='["1", "2"]', - ) - EFS_BASE_DIRECTORY: str = Field(default="project-specific-data") - - class PlacementSettings(BaseCustomSettings): # This is just a service placement constraint, see # https://docs.docker.com/engine/swarm/services/#control-service-placement. @@ -137,7 +125,9 @@ class DynamicSidecarSettings(BaseCustomSettings, MixinLoggingSettings): DYNAMIC_SIDECAR_R_CLONE_SETTINGS: RCloneSettings = Field(auto_default_from_env=True) - DYNAMIC_SIDECAR_EFS_SETTINGS: EfsSettings | None = Field(auto_default_from_env=True) + DYNAMIC_SIDECAR_EFS_SETTINGS: AwsEfsSettings | None = Field( + auto_default_from_env=True + ) DYNAMIC_SIDECAR_PLACEMENT_SETTINGS: PlacementSettings = Field( auto_default_from_env=True diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py index 83fbaae93f8..4a02c2ab9ac 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py @@ -16,6 +16,8 @@ from models_library.service_settings_labels import SimcoreServiceSettingsLabel from models_library.utils.json_serialization import json_dumps from pydantic import ByteSize, parse_obj_as +from servicelib.rabbitmq import RabbitMQRPCClient +from servicelib.rabbitmq.rpc_interfaces.efs_guardian import efs_manager from servicelib.utils import unused_port from settings_library.node_ports import StorageAuthSettings @@ -196,13 +198,14 @@ def get_prometheus_monitoring_networks( ) -def _get_mounts( +async def _get_mounts( *, scheduler_data: SchedulerData, dynamic_sidecar_settings: DynamicSidecarSettings, dynamic_services_scheduler_settings: DynamicServicesSchedulerSettings, app_settings: AppSettings, has_quota_support: bool, + rpc_client: RabbitMQRPCClient, ) -> list[dict[str, Any]]: mounts: list[dict[str, Any]] = [ # docker socket needed to use the docker api @@ -257,6 +260,12 @@ def _get_mounts( _mount_efs_enabled = False efs_settings = dynamic_sidecar_settings.DYNAMIC_SIDECAR_EFS_SETTINGS if efs_settings and scheduler_data.user_id in efs_settings.EFS_ENABLED_FOR_USERS: + # Ask guardian to create directory + await efs_manager.create_project_specific_data_dir( + rpc_client, + project_id=scheduler_data.project_id, + node_id=scheduler_data.node_uuid, + ) _mount_efs_enabled = True # state paths now get mounted via different driver and are synced to s3 automatically @@ -371,7 +380,7 @@ def _get_ports( return ports -def get_dynamic_sidecar_spec( # pylint:disable=too-many-arguments# noqa: PLR0913 +async def get_dynamic_sidecar_spec( # pylint:disable=too-many-arguments# noqa: PLR0913 scheduler_data: SchedulerData, dynamic_sidecar_settings: DynamicSidecarSettings, dynamic_services_scheduler_settings: DynamicServicesSchedulerSettings, @@ -384,6 +393,7 @@ def get_dynamic_sidecar_spec( # pylint:disable=too-many-arguments# noqa: PLR091 hardware_info: HardwareInfo | None, metrics_collection_allowed: bool, telemetry_enabled: bool, + rpc_client: RabbitMQRPCClient, ) -> AioDockerServiceSpec: """ The dynamic-sidecar is responsible for managing the lifecycle @@ -395,12 +405,13 @@ def get_dynamic_sidecar_spec( # pylint:disable=too-many-arguments# noqa: PLR091 """ compose_namespace = get_compose_namespace(scheduler_data.node_uuid) - mounts = _get_mounts( + mounts = await _get_mounts( scheduler_data=scheduler_data, dynamic_services_scheduler_settings=dynamic_services_scheduler_settings, dynamic_sidecar_settings=dynamic_sidecar_settings, app_settings=app_settings, has_quota_support=has_quota_support, + rpc_client=rpc_client, ) ports = _get_ports( diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_event_create_sidecars.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_event_create_sidecars.py index 572017c8567..6943f7a0852 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_event_create_sidecars.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_event_create_sidecars.py @@ -18,7 +18,7 @@ from models_library.service_settings_labels import SimcoreServiceSettingsLabel from models_library.services import RunID from models_library.utils.json_serialization import json_dumps -from servicelib.rabbitmq import RabbitMQClient +from servicelib.rabbitmq import RabbitMQClient, RabbitMQRPCClient from simcore_postgres_database.models.comp_tasks import NodeClass from .....core.dynamic_services_settings import DynamicServicesSettings @@ -222,9 +222,11 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None: user_id=scheduler_data.user_id, product_name=scheduler_data.product_name ) + rpc_client: RabbitMQRPCClient = app.state.rabbitmq_rpc_client + # WARNING: do NOT log, this structure has secrets in the open # If you want to log, please use an obfuscator - dynamic_sidecar_service_spec_base: AioDockerServiceSpec = get_dynamic_sidecar_spec( + dynamic_sidecar_service_spec_base: AioDockerServiceSpec = await get_dynamic_sidecar_spec( scheduler_data=scheduler_data, dynamic_sidecar_settings=dynamic_sidecar_settings, dynamic_services_scheduler_settings=dynamic_services_scheduler_settings, @@ -236,6 +238,7 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None: allow_internet_access=allow_internet_access, metrics_collection_allowed=metrics_collection_allowed, telemetry_enabled=is_telemetry_enabled, + rpc_client=rpc_client, ) catalog_client = CatalogClient.instance(app) diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py index 9a0371e2d80..784f956c243 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py @@ -408,7 +408,7 @@ def expected_dynamic_sidecar_spec( } -def test_get_dynamic_proxy_spec( +async def test_get_dynamic_proxy_spec( mocked_catalog_service_api: respx.MockRouter, minimal_app: FastAPI, scheduler_data: SchedulerData, @@ -436,7 +436,7 @@ def test_get_dynamic_proxy_spec( for count in range(1, 11): # loop to check it does not repeat copies print(f"{count:*^50}") - dynamic_sidecar_spec: AioDockerServiceSpec = get_dynamic_sidecar_spec( + dynamic_sidecar_spec: AioDockerServiceSpec = await get_dynamic_sidecar_spec( scheduler_data=scheduler_data, dynamic_sidecar_settings=dynamic_sidecar_settings, dynamic_services_scheduler_settings=dynamic_services_scheduler_settings, @@ -530,7 +530,7 @@ async def test_merge_dynamic_sidecar_specs_with_user_specific_specs( hardware_info: HardwareInfo, fake_service_specifications: dict[str, Any], ): - dynamic_sidecar_spec: AioDockerServiceSpec = get_dynamic_sidecar_spec( + dynamic_sidecar_spec: AioDockerServiceSpec = await get_dynamic_sidecar_spec( scheduler_data=scheduler_data, dynamic_sidecar_settings=dynamic_sidecar_settings, dynamic_services_scheduler_settings=dynamic_services_scheduler_settings, From f204a1cea315a0010baeb3781119eba668c2a0d3 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 13:14:11 +0200 Subject: [PATCH 15/31] fix --- .../modules/dynamic_sidecar/docker_service_specs/sidecar.py | 1 - .../modules/dynamic_sidecar/volumes.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py index 4a02c2ab9ac..91688a0205b 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py @@ -260,7 +260,6 @@ async def _get_mounts( _mount_efs_enabled = False efs_settings = dynamic_sidecar_settings.DYNAMIC_SIDECAR_EFS_SETTINGS if efs_settings and scheduler_data.user_id in efs_settings.EFS_ENABLED_FOR_USERS: - # Ask guardian to create directory await efs_manager.create_project_specific_data_dir( rpc_client, project_id=scheduler_data.project_id, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py index 35c008456c9..9da1ddeeeb6 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py @@ -86,7 +86,7 @@ def _get_efs_volume_driver_config( "Options": { "type": "nfs", "o": f"addr={efs_settings.EFS_DNS_NAME},rw,nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport", - "device": f":/{efs_settings.EFS_BASE_DIRECTORY}", # /{project_id}/{node_uuid}/{storage_directory_name} + "device": f":/{efs_settings.EFS_BASE_DIRECTORY}/{project_id}/{node_uuid}/{storage_directory_name}", }, } return driver_config From 6bd4b84348cb8107040b7b0eb90a5430ba1259ac Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 13:20:30 +0200 Subject: [PATCH 16/31] fix --- .../modules/dynamic_sidecar/volumes.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py index 9da1ddeeeb6..47f35fdff4b 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py @@ -7,9 +7,10 @@ from models_library.services import RunID from models_library.users import UserID from servicelib.docker_constants import PREFIX_DYNAMIC_SIDECAR_VOLUMES +from settings_library.efs import AwsEfsSettings from settings_library.r_clone import S3Provider -from ...core.dynamic_services_settings.sidecar import EfsSettings, RCloneSettings +from ...core.dynamic_services_settings.sidecar import RCloneSettings from .errors import DynamicSidecarError DY_SIDECAR_SHARED_STORE_PATH = Path("/shared-store") @@ -76,7 +77,7 @@ def _get_s3_volume_driver_config( def _get_efs_volume_driver_config( - efs_settings: EfsSettings, + efs_settings: AwsEfsSettings, project_id: ProjectID, node_uuid: NodeID, storage_directory_name: str, @@ -86,7 +87,7 @@ def _get_efs_volume_driver_config( "Options": { "type": "nfs", "o": f"addr={efs_settings.EFS_DNS_NAME},rw,nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport", - "device": f":/{efs_settings.EFS_BASE_DIRECTORY}/{project_id}/{node_uuid}/{storage_directory_name}", + "device": f":/{efs_settings.EFS_PROJECT_SPECIFIC_DATA_DIRECTORY}/{project_id}/{node_uuid}/{storage_directory_name}", }, } return driver_config @@ -248,7 +249,7 @@ def mount_efs( run_id: RunID, project_id: ProjectID, user_id: UserID, - efs_settings: EfsSettings, + efs_settings: AwsEfsSettings, ) -> dict[str, Any]: return { "Source": cls.source(path, node_uuid, run_id), From 22340aaf5d62c047a9d1f0e0e69e897c3cb16a12 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 13:39:25 +0200 Subject: [PATCH 17/31] fix --- packages/settings-library/src/settings_library/efs.py | 3 +-- .../src/simcore_service_efs_guardian/core/settings.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/settings-library/src/settings_library/efs.py b/packages/settings-library/src/settings_library/efs.py index 86166072dba..92db4882b19 100644 --- a/packages/settings-library/src/settings_library/efs.py +++ b/packages/settings-library/src/settings_library/efs.py @@ -1,6 +1,5 @@ from pathlib import Path -from models_library.users import UserID from pydantic import Field from .base import BaseCustomSettings @@ -16,7 +15,7 @@ class AwsEfsSettings(BaseCustomSettings): default=Path("/data/efs"), description="This is the path where EFS is mounted to the EC2 machine", ) - EFS_ENABLED_FOR_USERS: list[UserID] = Field( + EFS_ENABLED_FOR_USERS: list[int] = Field( description="This is temporary solution so we can enable it for specific users for testing purpose", example=[1], ) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py index bb52d21b8d3..adf172a4b0c 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py @@ -1,5 +1,4 @@ from functools import cached_property -from pathlib import Path from typing import Final, cast from fastapi import FastAPI @@ -19,7 +18,7 @@ EFS_GUARDIAN_ENV_PREFIX: Final[str] = "EFS_GUARDIAN_" - + class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): # CODE STATICS --------------------------------------------------------- API_VERSION: str = API_VERSION From 2c33266c09c8289b713fbe8573540fddbffe911c Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 13:48:23 +0200 Subject: [PATCH 18/31] fix test --- ...modules_dynamic_sidecar_docker_service_specs.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py index 784f956c243..72088873882 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py @@ -6,6 +6,7 @@ import json from collections.abc import Mapping from typing import Any, cast +from unittest.mock import Mock import pytest import respx @@ -26,6 +27,7 @@ from models_library.services import RunID, ServiceKeyVersion from models_library.utils.json_serialization import json_dumps from models_library.wallets import WalletInfo +from pytest_mock import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.utils_envs import setenvs_from_dict from settings_library.s3 import S3Settings @@ -120,6 +122,16 @@ def hardware_info() -> HardwareInfo: return HardwareInfo.parse_obj(HardwareInfo.Config.schema_extra["examples"][0]) +@pytest.fixture +def mocked_setup_rabbitmq(mocker: MockerFixture): + return ( + mocker.patch( + "simcore_service_resource_usage_tracker.core.application.setup_rabbitmq", + autospec=True, + ), + ) + + @pytest.fixture def expected_dynamic_sidecar_spec( run_id: RunID, @@ -448,6 +460,7 @@ async def test_get_dynamic_proxy_spec( allow_internet_access=False, metrics_collection_allowed=True, telemetry_enabled=True, + rpc_client=Mock(), ) exclude_keys: Mapping[int | str, Any] = { @@ -542,6 +555,7 @@ async def test_merge_dynamic_sidecar_specs_with_user_specific_specs( allow_internet_access=False, metrics_collection_allowed=True, telemetry_enabled=True, + rpc_client=Mock(), ) assert dynamic_sidecar_spec dynamic_sidecar_spec_dict = dynamic_sidecar_spec.dict() From bcd16adead96d3572b1cc1f209b6b44ceeedfbab Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 13:52:17 +0200 Subject: [PATCH 19/31] fix test --- ...st_modules_dynamic_sidecar_docker_service_specs.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py index 72088873882..ee5f4a4f15a 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py @@ -27,7 +27,6 @@ from models_library.services import RunID, ServiceKeyVersion from models_library.utils.json_serialization import json_dumps from models_library.wallets import WalletInfo -from pytest_mock import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.utils_envs import setenvs_from_dict from settings_library.s3 import S3Settings @@ -122,16 +121,6 @@ def hardware_info() -> HardwareInfo: return HardwareInfo.parse_obj(HardwareInfo.Config.schema_extra["examples"][0]) -@pytest.fixture -def mocked_setup_rabbitmq(mocker: MockerFixture): - return ( - mocker.patch( - "simcore_service_resource_usage_tracker.core.application.setup_rabbitmq", - autospec=True, - ), - ) - - @pytest.fixture def expected_dynamic_sidecar_spec( run_id: RunID, From 5dae0250f9365783f116cf87ac13aaab05c489a4 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 14:06:01 +0200 Subject: [PATCH 20/31] fix import --- .../simcore_service_efs_guardian/services/efs_manager_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager_setup.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager_setup.py index 9f0ded69552..e418d27cc59 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager_setup.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager_setup.py @@ -2,7 +2,7 @@ from typing import cast from fastapi import FastAPI -from simcore_service_efs_guardian.core.settings import AwsEfsSettings +from settings_library.efs import AwsEfsSettings from tenacity import ( AsyncRetrying, before_sleep_log, From 61c8359ce0de30fdbbef133d2ba5995a49583d9b Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 14:17:39 +0200 Subject: [PATCH 21/31] fix path --- .../modules/dynamic_sidecar/volumes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py index 47f35fdff4b..710d87ad2f5 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py @@ -87,7 +87,7 @@ def _get_efs_volume_driver_config( "Options": { "type": "nfs", "o": f"addr={efs_settings.EFS_DNS_NAME},rw,nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport", - "device": f":/{efs_settings.EFS_PROJECT_SPECIFIC_DATA_DIRECTORY}/{project_id}/{node_uuid}/{storage_directory_name}", + "device": f":/{efs_settings.EFS_PROJECT_SPECIFIC_DATA_DIRECTORY}/{project_id}/{node_uuid}", }, } return driver_config From 0d4d9a787cea85c3fb2fb99ffbb5830cb39f84cf Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 14:40:41 +0200 Subject: [PATCH 22/31] fix path --- .../rpc_interfaces/efs_guardian/efs_manager.py | 1 + .../docker_service_specs/sidecar.py | 16 +++++++++++----- .../modules/dynamic_sidecar/volumes.py | 11 ++++++----- .../api/rpc/_efs_guardian.py | 6 ++---- .../services/efs_manager.py | 3 ++- .../efs-guardian/tests/unit/test_efs_manager.py | 7 ++++++- 6 files changed, 28 insertions(+), 16 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/efs_guardian/efs_manager.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/efs_guardian/efs_manager.py index b5e8aa219e9..80f02702d40 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/efs_guardian/efs_manager.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/efs_guardian/efs_manager.py @@ -23,6 +23,7 @@ async def create_project_specific_data_dir( *, project_id: ProjectID, node_id: NodeID, + storage_directory_name: str, ) -> Path: output: Path = await rabbitmq_rpc_client.request( EFS_GUARDIAN_RPC_NAMESPACE, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py index 91688a0205b..a23be796dce 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py @@ -260,17 +260,22 @@ async def _get_mounts( _mount_efs_enabled = False efs_settings = dynamic_sidecar_settings.DYNAMIC_SIDECAR_EFS_SETTINGS if efs_settings and scheduler_data.user_id in efs_settings.EFS_ENABLED_FOR_USERS: - await efs_manager.create_project_specific_data_dir( - rpc_client, - project_id=scheduler_data.project_id, - node_id=scheduler_data.node_uuid, - ) _mount_efs_enabled = True # state paths now get mounted via different driver and are synced to s3 automatically for path_to_mount in scheduler_data.paths_mapping.state_paths: if _mount_efs_enabled: assert dynamic_sidecar_settings.DYNAMIC_SIDECAR_EFS_SETTINGS # nosec + + _storage_directory_name = DynamicSidecarVolumesPathsResolver.volume_name( + path_to_mount + ).strip("_") + await efs_manager.create_project_specific_data_dir( + rpc_client, + project_id=scheduler_data.project_id, + node_id=scheduler_data.node_uuid, + storage_directory_name=_storage_directory_name, + ) mounts.append( DynamicSidecarVolumesPathsResolver.mount_efs( swarm_stack_name=dynamic_services_scheduler_settings.SWARM_STACK_NAME, @@ -280,6 +285,7 @@ async def _get_mounts( project_id=scheduler_data.project_id, user_id=scheduler_data.user_id, efs_settings=dynamic_sidecar_settings.DYNAMIC_SIDECAR_EFS_SETTINGS, + storage_directory_name=_storage_directory_name, ) ) # for now only enable this with dev features enabled diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py index 710d87ad2f5..cc3edfc0e8f 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py @@ -87,7 +87,7 @@ def _get_efs_volume_driver_config( "Options": { "type": "nfs", "o": f"addr={efs_settings.EFS_DNS_NAME},rw,nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport", - "device": f":/{efs_settings.EFS_PROJECT_SPECIFIC_DATA_DIRECTORY}/{project_id}/{node_uuid}", + "device": f":/{efs_settings.EFS_PROJECT_SPECIFIC_DATA_DIRECTORY}/{project_id}/{node_uuid}/{storage_directory_name}", }, } return driver_config @@ -103,7 +103,7 @@ def target(cls, path: Path) -> str: return f"{target_path}" @classmethod - def _volume_name(cls, path: Path) -> str: + def volume_name(cls, path: Path) -> str: return f"{path}".replace(os.sep, "_") @classmethod @@ -122,7 +122,7 @@ def source(cls, path: Path, node_uuid: NodeID, run_id: RunID) -> str: # NOTE: issues can occur when the paths of the mounted outputs, inputs # and state folders are very long and share the same subdirectory path. # Reversing volume name to prevent these issues from happening. - reversed_volume_name = cls._volume_name(path)[::-1] + reversed_volume_name = cls.volume_name(path)[::-1] unique_name = f"{PREFIX_DYNAMIC_SIDECAR_VOLUMES}_{run_id}_{node_uuid}_{reversed_volume_name}" return unique_name[:255] @@ -235,7 +235,7 @@ def mount_r_clone( r_clone_settings=r_clone_settings, project_id=project_id, node_uuid=node_uuid, - storage_directory_name=cls._volume_name(path).strip("_"), + storage_directory_name=cls.volume_name(path).strip("_"), ), }, } @@ -250,6 +250,7 @@ def mount_efs( project_id: ProjectID, user_id: UserID, efs_settings: AwsEfsSettings, + storage_directory_name: str, ) -> dict[str, Any]: return { "Source": cls.source(path, node_uuid, run_id), @@ -268,7 +269,7 @@ def mount_efs( efs_settings=efs_settings, project_id=project_id, node_uuid=node_uuid, - storage_directory_name=cls._volume_name(path).strip("_"), + storage_directory_name=storage_directory_name, ), }, } diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/api/rpc/_efs_guardian.py b/services/efs-guardian/src/simcore_service_efs_guardian/api/rpc/_efs_guardian.py index 61b5e588b83..9fe0978471d 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/api/rpc/_efs_guardian.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/api/rpc/_efs_guardian.py @@ -12,14 +12,12 @@ @router.expose(reraise_if_error_type=()) async def create_project_specific_data_dir( - app: FastAPI, - *, - project_id: ProjectID, - node_id: NodeID, + app: FastAPI, *, project_id: ProjectID, node_id: NodeID, storage_directory_name: str ) -> Path: _efs_manager = get_efs_manager(app) return await _efs_manager.create_project_specific_data_dir( project_id=project_id, node_id=node_id, + storage_directory_name=storage_directory_name, ) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager.py index 4e249188c47..d3c2d0802b5 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager.py @@ -27,13 +27,14 @@ async def initialize_directories(self): Path.mkdir(_dir_path, parents=True, exist_ok=True) async def create_project_specific_data_dir( - self, project_id: ProjectID, node_id: NodeID + self, project_id: ProjectID, node_id: NodeID, storage_directory_name: str ) -> Path: _dir_path = ( self._efs_mounted_path / self._project_specific_data_base_directory / f"{project_id}" / f"{node_id}" + / f"{storage_directory_name}" ) Path.mkdir(_dir_path, parents=True, exist_ok=True) return _dir_path diff --git a/services/efs-guardian/tests/unit/test_efs_manager.py b/services/efs-guardian/tests/unit/test_efs_manager.py index acc72a9790f..ede34e1824c 100644 --- a/services/efs-guardian/tests/unit/test_efs_manager.py +++ b/services/efs-guardian/tests/unit/test_efs_manager.py @@ -43,9 +43,13 @@ async def test_rpc_create_project_specific_data_dir( _project_id = faker.uuid4() _node_id = faker.uuid4() + _storage_directory_name = faker.name() result = await efs_manager.create_project_specific_data_dir( - rpc_client, project_id=_project_id, node_id=_node_id + rpc_client, + project_id=_project_id, + node_id=_node_id, + storage_directory_name=_storage_directory_name, ) assert isinstance(result, Path) _expected_path = ( @@ -53,6 +57,7 @@ async def test_rpc_create_project_specific_data_dir( / aws_efs_settings.EFS_PROJECT_SPECIFIC_DATA_DIRECTORY / _project_id / _node_id + / _storage_directory_name ) assert _expected_path == result assert _expected_path.exists From a7e564a8f7e7e3858f1624293a13477307345f84 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 14:49:45 +0200 Subject: [PATCH 23/31] review @GitHK --- .../modules/dynamic_sidecar/docker_service_specs/sidecar.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py index a23be796dce..84fcc5f0060 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py @@ -257,14 +257,14 @@ async def _get_mounts( ) # We check whether user has access to EFS feature - _mount_efs_enabled = False + use_efs = False efs_settings = dynamic_sidecar_settings.DYNAMIC_SIDECAR_EFS_SETTINGS if efs_settings and scheduler_data.user_id in efs_settings.EFS_ENABLED_FOR_USERS: - _mount_efs_enabled = True + use_efs = True # state paths now get mounted via different driver and are synced to s3 automatically for path_to_mount in scheduler_data.paths_mapping.state_paths: - if _mount_efs_enabled: + if use_efs: assert dynamic_sidecar_settings.DYNAMIC_SIDECAR_EFS_SETTINGS # nosec _storage_directory_name = DynamicSidecarVolumesPathsResolver.volume_name( From f0ab1d939345c149b249819e33431cbebba41867 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 15:09:34 +0200 Subject: [PATCH 24/31] review @GitHK --- .../src/settings_library/efs.py | 27 +++++++++++++++++++ .../modules/dynamic_sidecar/volumes.py | 13 +++++++-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/packages/settings-library/src/settings_library/efs.py b/packages/settings-library/src/settings_library/efs.py index 92db4882b19..5096c595c4f 100644 --- a/packages/settings-library/src/settings_library/efs.py +++ b/packages/settings-library/src/settings_library/efs.py @@ -19,3 +19,30 @@ class AwsEfsSettings(BaseCustomSettings): description="This is temporary solution so we can enable it for specific users for testing purpose", example=[1], ) + + +NFS_PROTOCOL = "4.1" +READ_SIZE = "1048576" +WRITE_SIZE = "1048576" +RECOVERY_MODE = "hard" +NFS_REQUEST_TIMEOUT = "600" +NUMBER_OF_RETRANSMISSIONS = "2" +PORT_MODE = "noresvport" + +""" +`sudo mount -t nfs -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport` + +Explanation: + +nfsvers=4.1: Specifies the NFS protocol version to use; here, it is version 4.1, which supports improved security features and performance optimizations over earlier versions. + +rsize=1048576 and wsize=1048576: Set the read and write buffer sizes in bytes, respectively. Here, both are set to 1,048,576 bytes (1 MB). Larger buffer sizes can improve performance over high-latency networks by allowing more data to be transferred with each read or write request. + +hard: Specifies the recovery behavior of the NFS client. If the NFS server becomes unreachable, the NFS client will retry the request until the server becomes available again. The alternative is soft, where the NFS client gives up after a certain number of retries, potentially leading to data corruption or loss. + +timeo=600: Sets the timeout value for NFS requests in deciseconds (tenths of a second). Here, 600 deciseconds means 60 seconds. This is how long the NFS client will wait for a response from the server before retrying or failing. + +retrans=2: Sets the number of retransmissions for each NFS request if a response is not received before the timeout. Here, it will retry each request twice. + +noresvport: Normally, NFS uses a reserved port (number below 1024) for communicating, which requires root privileges on the client side. noresvport allows using non-reserved ports, which can be helpful in environments where clients don't have root privileges. +""" diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py index cc3edfc0e8f..8b1f6ca72f7 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py @@ -7,7 +7,16 @@ from models_library.services import RunID from models_library.users import UserID from servicelib.docker_constants import PREFIX_DYNAMIC_SIDECAR_VOLUMES -from settings_library.efs import AwsEfsSettings +from settings_library.efs import ( + NFS_PROTOCOL, + NFS_REQUEST_TIMEOUT, + NUMBER_OF_RETRANSMISSIONS, + PORT_MODE, + READ_SIZE, + RECOVERY_MODE, + WRITE_SIZE, + AwsEfsSettings, +) from settings_library.r_clone import S3Provider from ...core.dynamic_services_settings.sidecar import RCloneSettings @@ -86,7 +95,7 @@ def _get_efs_volume_driver_config( driver_config: dict[str, Any] = { "Options": { "type": "nfs", - "o": f"addr={efs_settings.EFS_DNS_NAME},rw,nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport", + "o": f"addr={efs_settings.EFS_DNS_NAME},rw,nfsvers={NFS_PROTOCOL},rsize={READ_SIZE},wsize={WRITE_SIZE},{RECOVERY_MODE},timeo={NFS_REQUEST_TIMEOUT},retrans={NUMBER_OF_RETRANSMISSIONS},{PORT_MODE}", "device": f":/{efs_settings.EFS_PROJECT_SPECIFIC_DATA_DIRECTORY}/{project_id}/{node_uuid}/{storage_directory_name}", }, } From cc3f1b1a445b3436f7df9cc43b62bfdfc5ce05fc Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 15:10:40 +0200 Subject: [PATCH 25/31] review @GitHK --- packages/settings-library/src/settings_library/efs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/settings-library/src/settings_library/efs.py b/packages/settings-library/src/settings_library/efs.py index 5096c595c4f..b75f9a7d3a5 100644 --- a/packages/settings-library/src/settings_library/efs.py +++ b/packages/settings-library/src/settings_library/efs.py @@ -15,9 +15,9 @@ class AwsEfsSettings(BaseCustomSettings): default=Path("/data/efs"), description="This is the path where EFS is mounted to the EC2 machine", ) - EFS_ENABLED_FOR_USERS: list[int] = Field( + EFS_ENABLED_FOR_USERS: set[int] = Field( description="This is temporary solution so we can enable it for specific users for testing purpose", - example=[1], + example={1}, ) From 37dd243fdfdbf1f2cfcd01010ee5a43028d1b695 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 15:11:52 +0200 Subject: [PATCH 26/31] fix --- .env-devel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env-devel b/.env-devel index aaf3e14befe..fc62c2ca9bc 100644 --- a/.env-devel +++ b/.env-devel @@ -66,7 +66,7 @@ DIRECTOR_REGISTRY_CACHING=True EFS_DNS_NAME=fs-xxx.efs.us-east-1.amazonaws.com EFS_MOUNTED_PATH=/tmp/efs -EFS_ENABLED_FOR_USERS=[] +EFS_ENABLED_FOR_USERS={} # DIRECTOR_V2 ---- COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"/home/scu/.dask/dask-crt.pem","tls_client_cert":"/home/scu/.dask/dask-crt.pem","tls_client_key":"/home/scu/.dask/dask-key.pem"}' From d1baa6f6dd0abd23b7d832f49464bb4e67c641fa Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 15:42:29 +0200 Subject: [PATCH 27/31] review @mrnicegyu11 --- .env-devel | 2 +- packages/settings-library/src/settings_library/efs.py | 2 +- .../modules/dynamic_sidecar/docker_service_specs/sidecar.py | 5 ++++- services/docker-compose.yml | 4 ++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.env-devel b/.env-devel index fc62c2ca9bc..5cc3bcb157c 100644 --- a/.env-devel +++ b/.env-devel @@ -66,7 +66,7 @@ DIRECTOR_REGISTRY_CACHING=True EFS_DNS_NAME=fs-xxx.efs.us-east-1.amazonaws.com EFS_MOUNTED_PATH=/tmp/efs -EFS_ENABLED_FOR_USERS={} +EFS_ONLY_ENABLED_FOR_USERIDS={} # DIRECTOR_V2 ---- COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"/home/scu/.dask/dask-crt.pem","tls_client_cert":"/home/scu/.dask/dask-crt.pem","tls_client_key":"/home/scu/.dask/dask-key.pem"}' diff --git a/packages/settings-library/src/settings_library/efs.py b/packages/settings-library/src/settings_library/efs.py index b75f9a7d3a5..1c5ac36e59b 100644 --- a/packages/settings-library/src/settings_library/efs.py +++ b/packages/settings-library/src/settings_library/efs.py @@ -15,7 +15,7 @@ class AwsEfsSettings(BaseCustomSettings): default=Path("/data/efs"), description="This is the path where EFS is mounted to the EC2 machine", ) - EFS_ENABLED_FOR_USERS: set[int] = Field( + EFS_ONLY_ENABLED_FOR_USERIDS: set[int] = Field( description="This is temporary solution so we can enable it for specific users for testing purpose", example={1}, ) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py index 84fcc5f0060..b26576068d8 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py @@ -259,7 +259,10 @@ async def _get_mounts( # We check whether user has access to EFS feature use_efs = False efs_settings = dynamic_sidecar_settings.DYNAMIC_SIDECAR_EFS_SETTINGS - if efs_settings and scheduler_data.user_id in efs_settings.EFS_ENABLED_FOR_USERS: + if ( + efs_settings + and scheduler_data.user_id in efs_settings.EFS_ONLY_ENABLED_FOR_USERIDS + ): use_efs = True # state paths now get mounted via different driver and are synced to s3 automatically diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 02243853817..07709547396 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -302,7 +302,7 @@ services: EFS_DNS_NAME: ${EFS_DNS_NAME} EFS_MOUNTED_PATH: ${EFS_MOUNTED_PATH} - EFS_ENABLED_FOR_USERS: ${EFS_ENABLED_FOR_USERS} + EFS_ONLY_ENABLED_FOR_USERIDS: ${EFS_ONLY_ENABLED_FOR_USERIDS} RABBIT_HOST: ${RABBIT_HOST} RABBIT_PASSWORD: ${RABBIT_PASSWORD} @@ -362,7 +362,7 @@ services: RABBIT_USER: ${RABBIT_USER} EFS_DNS_NAME: ${EFS_DNS_NAME} EFS_MOUNTED_PATH: ${EFS_MOUNTED_PATH} - EFS_ENABLED_FOR_USERS: ${EFS_ENABLED_FOR_USERS} + EFS_ONLY_ENABLED_FOR_USERIDS: ${EFS_ONLY_ENABLED_FOR_USERIDS} invitations: image: ${DOCKER_REGISTRY:-itisfoundation}/invitations:${DOCKER_IMAGE_TAG:-latest} From 218d906776db48da30de77dec632f69b3e5f823a Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 16:15:26 +0200 Subject: [PATCH 28/31] revert back to list for now: --- .env-devel | 2 +- packages/settings-library/src/settings_library/efs.py | 4 ++-- .../modules/dynamic_sidecar/volumes.py | 1 + services/efs-guardian/tests/unit/conftest.py | 1 + 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.env-devel b/.env-devel index 5cc3bcb157c..36a884224dd 100644 --- a/.env-devel +++ b/.env-devel @@ -66,7 +66,7 @@ DIRECTOR_REGISTRY_CACHING=True EFS_DNS_NAME=fs-xxx.efs.us-east-1.amazonaws.com EFS_MOUNTED_PATH=/tmp/efs -EFS_ONLY_ENABLED_FOR_USERIDS={} +EFS_ONLY_ENABLED_FOR_USERIDS=[] # DIRECTOR_V2 ---- COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"/home/scu/.dask/dask-crt.pem","tls_client_cert":"/home/scu/.dask/dask-crt.pem","tls_client_key":"/home/scu/.dask/dask-key.pem"}' diff --git a/packages/settings-library/src/settings_library/efs.py b/packages/settings-library/src/settings_library/efs.py index 1c5ac36e59b..98bf3a31bd6 100644 --- a/packages/settings-library/src/settings_library/efs.py +++ b/packages/settings-library/src/settings_library/efs.py @@ -15,9 +15,9 @@ class AwsEfsSettings(BaseCustomSettings): default=Path("/data/efs"), description="This is the path where EFS is mounted to the EC2 machine", ) - EFS_ONLY_ENABLED_FOR_USERIDS: set[int] = Field( + EFS_ONLY_ENABLED_FOR_USERIDS: list[int] = Field( description="This is temporary solution so we can enable it for specific users for testing purpose", - example={1}, + example=[1], ) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py index 8b1f6ca72f7..8a6d85c906b 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py @@ -113,6 +113,7 @@ def target(cls, path: Path) -> str: @classmethod def volume_name(cls, path: Path) -> str: + """Returns a volume name created from path. There is not possibility to go back to the original path from the volume name""" return f"{path}".replace(os.sep, "_") @classmethod diff --git a/services/efs-guardian/tests/unit/conftest.py b/services/efs-guardian/tests/unit/conftest.py index 1030c433bd9..25a4333f945 100644 --- a/services/efs-guardian/tests/unit/conftest.py +++ b/services/efs-guardian/tests/unit/conftest.py @@ -98,6 +98,7 @@ def app_environment( "EFS_DNS_NAME": "fs-xxx.efs.us-east-1.amazonaws.com", "EFS_MOUNTED_PATH": "/tmp/efs", "EFS_PROJECT_SPECIFIC_DATA_DIRECTORY": "project-specific-data", + "EFS_ONLY_ENABLED_FOR_USERIDS": "[]", }, ) From 1e46207c855bd30769a74af175d9a6bfbae0f1e3 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 16:22:42 +0200 Subject: [PATCH 29/31] remove all defaults review @mrnicegyu11 --- .../rpc_interfaces/efs_guardian/efs_manager.py | 1 + .../settings-library/src/settings_library/efs.py | 3 +-- services/efs-guardian/tests/unit/conftest.py | 14 -------------- 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/efs_guardian/efs_manager.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/efs_guardian/efs_manager.py index 80f02702d40..592959eb08c 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/efs_guardian/efs_manager.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/efs_guardian/efs_manager.py @@ -30,6 +30,7 @@ async def create_project_specific_data_dir( parse_obj_as(RPCMethodName, "create_project_specific_data_dir"), project_id=project_id, node_id=node_id, + storage_directory_name=storage_directory_name, timeout_s=_DEFAULT_TIMEOUT_S, ) return output diff --git a/packages/settings-library/src/settings_library/efs.py b/packages/settings-library/src/settings_library/efs.py index 98bf3a31bd6..d09b8abb20f 100644 --- a/packages/settings-library/src/settings_library/efs.py +++ b/packages/settings-library/src/settings_library/efs.py @@ -10,9 +10,8 @@ class AwsEfsSettings(BaseCustomSettings): description="AWS Elastic File System DNS name", example="fs-xxx.efs.us-east-1.amazonaws.com", ) - EFS_PROJECT_SPECIFIC_DATA_DIRECTORY: str = Field(default="project-specific-data") + EFS_PROJECT_SPECIFIC_DATA_DIRECTORY: str EFS_MOUNTED_PATH: Path = Field( - default=Path("/data/efs"), description="This is the path where EFS is mounted to the EC2 machine", ) EFS_ONLY_ENABLED_FOR_USERIDS: list[int] = Field( diff --git a/services/efs-guardian/tests/unit/conftest.py b/services/efs-guardian/tests/unit/conftest.py index 25a4333f945..0b974e13645 100644 --- a/services/efs-guardian/tests/unit/conftest.py +++ b/services/efs-guardian/tests/unit/conftest.py @@ -140,17 +140,3 @@ async def rpc_client( rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], ) -> RabbitMQRPCClient: return await rabbitmq_rpc_client("client") - - -# @pytest.fixture -# def mocked_setup_rabbitmq(mocker: MockerFixture): -# return ( -# mocker.patch( -# "simcore_service_efs_guardian.core.application.setup_rabbitmq", -# autospec=True, -# ), -# mocker.patch( -# "simcore_service_efs_guardian.core.application.setup_rpc_routes", -# autospec=True, -# ), -# ) From 72a918de666493978813f1d5b5be5f5658c40545 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 16:59:25 +0200 Subject: [PATCH 30/31] add env --- .env-devel | 1 + 1 file changed, 1 insertion(+) diff --git a/.env-devel b/.env-devel index 36a884224dd..d50e0af58b5 100644 --- a/.env-devel +++ b/.env-devel @@ -66,6 +66,7 @@ DIRECTOR_REGISTRY_CACHING=True EFS_DNS_NAME=fs-xxx.efs.us-east-1.amazonaws.com EFS_MOUNTED_PATH=/tmp/efs +EFS_PROJECT_SPECIFIC_DATA_DIRECTORY=project-specific-data EFS_ONLY_ENABLED_FOR_USERIDS=[] # DIRECTOR_V2 ---- From 92d1ae3baf005cfce19f09ca121c969ac30b8ef5 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 6 Jun 2024 17:29:46 +0200 Subject: [PATCH 31/31] fix --- services/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 07709547396..0d4f2fe390e 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -303,6 +303,7 @@ services: EFS_DNS_NAME: ${EFS_DNS_NAME} EFS_MOUNTED_PATH: ${EFS_MOUNTED_PATH} EFS_ONLY_ENABLED_FOR_USERIDS: ${EFS_ONLY_ENABLED_FOR_USERIDS} + EFS_PROJECT_SPECIFIC_DATA_DIRECTORY: ${EFS_PROJECT_SPECIFIC_DATA_DIRECTORY} RABBIT_HOST: ${RABBIT_HOST} RABBIT_PASSWORD: ${RABBIT_PASSWORD} @@ -363,6 +364,7 @@ services: EFS_DNS_NAME: ${EFS_DNS_NAME} EFS_MOUNTED_PATH: ${EFS_MOUNTED_PATH} EFS_ONLY_ENABLED_FOR_USERIDS: ${EFS_ONLY_ENABLED_FOR_USERIDS} + EFS_PROJECT_SPECIFIC_DATA_DIRECTORY: ${EFS_PROJECT_SPECIFIC_DATA_DIRECTORY} invitations: image: ${DOCKER_REGISTRY:-itisfoundation}/invitations:${DOCKER_IMAGE_TAG:-latest}