✨Comp-backend override resources with selected hardware

packages/models-library/src/models_library/rabbitmq_basic_types.py

@@ -19,3 +19,9 @@ def from_entries(cls, entries: dict[str, str]) -> "RPCNamespace":
         """
         composed_string = "-".join(f"{k}_{v}" for k, v in sorted(entries.items()))
         return parse_obj_as(cls, composed_string)
+
+
+class RPCMethodName(ConstrainedStr):
+    min_length: int = 1
+    max_length: int = 252
+    regex: re.Pattern[str] | None = re.compile(REGEX_RABBIT_QUEUE_ALLOWED_SYMBOLS)

packages/models-library/src/models_library/services_resources.py

@@ -56,6 +56,9 @@ def ensure_limits_are_equal_or_above_reservations(cls, values):
     def set_reservation_same_as_limit(self) -> None:
         self.reservation = self.limit
 
+    def set_value(self, value: StrictInt | StrictFloat | str) -> None:
+        self.limit = self.reservation = value
+
     class Config:
         validate_assignment = True
 

packages/service-library/src/servicelib/rabbitmq/__init__.py

@@ -8,7 +8,6 @@
     RPCNotInitializedError,
     RPCServerError,
 )
-from ._models import RPCMethodName
 from ._rpc_router import RPCRouter
 from ._utils import wait_till_rabbitmq_responsive
 
@@ -17,7 +16,6 @@
     "RabbitMQClient",
     "RabbitMQRPCClient",
     "RemoteMethodNotRegisteredError",
-    "RPCMethodName",
     "RPCNamespace",
     "RPCNotInitializedError",
     "RPCRouter",

packages/service-library/src/servicelib/rabbitmq/_client_rpc.py

@@ -6,14 +6,14 @@
 from typing import Any
 
 import aio_pika
-from models_library.rabbitmq_basic_types import RPCNamespace
+from models_library.rabbitmq_basic_types import RPCMethodName, RPCNamespace
 from pydantic import PositiveInt
 from settings_library.rabbit import RabbitSettings
 
 from ..logging_utils import log_context
 from ._client_base import RabbitMQClientBase
 from ._errors import RemoteMethodNotRegisteredError, RPCNotInitializedError
-from ._models import RPCMethodName, RPCNamespacedMethodName
+from ._models import RPCNamespacedMethodName
 from ._rpc_router import RPCRouter
 from ._utils import get_rabbitmq_client_unique_name
 

packages/service-library/src/servicelib/rabbitmq/_constants.py

@@ -1,3 +1,4 @@
 from typing import Final
 
 BIND_TO_ALL_TOPICS: Final[str] = "#"
+RPC_REMOTE_METHOD_TIMEOUT_S: Final[int] = 30

packages/service-library/src/servicelib/rabbitmq/_models.py

@@ -4,6 +4,7 @@
 
 from models_library.rabbitmq_basic_types import (
     REGEX_RABBIT_QUEUE_ALLOWED_SYMBOLS,
+    RPCMethodName,
     RPCNamespace,
 )
 from pydantic import ConstrainedStr, parse_obj_as
@@ -19,12 +20,6 @@ def routing_key(self) -> str | None:
         ...
 
 
-class RPCMethodName(ConstrainedStr):
-    min_length: int = 1
-    max_length: int = 252
-    regex: re.Pattern[str] | None = re.compile(REGEX_RABBIT_QUEUE_ALLOWED_SYMBOLS)
-
-
 class RPCNamespacedMethodName(ConstrainedStr):
     min_length: int = 1
     max_length: int = 255

packages/service-library/src/servicelib/rabbitmq/_rpc_router.py

@@ -5,11 +5,11 @@
 from dataclasses import dataclass, field
 from typing import Any, TypeVar
 
+from models_library.rabbitmq_basic_types import RPCMethodName
 from pydantic import SecretStr
 
 from ..logging_utils import log_context
 from ._errors import RPCServerError
-from ._models import RPCMethodName
 
 DecoratedCallable = TypeVar("DecoratedCallable", bound=Callable[..., Any])
 

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/clusters_keeper/clusters.py

@@ -0,0 +1,26 @@
+from models_library.api_schemas_clusters_keeper import CLUSTERS_KEEPER_RPC_NAMESPACE
+from models_library.api_schemas_clusters_keeper.clusters import OnDemandCluster
+from models_library.rabbitmq_basic_types import RPCMethodName
+from models_library.users import UserID
+from models_library.wallets import WalletID
+
+from ..._client_rpc import RabbitMQRPCClient
+from ..._constants import RPC_REMOTE_METHOD_TIMEOUT_S
+
+
+async def get_or_create_cluster(
+    client: RabbitMQRPCClient, *, user_id: UserID, wallet_id: WalletID | None
+) -> OnDemandCluster:
+    """**Remote method**
+
+    Raises:
+        RPCServerError -- if anything happens remotely
+    """
+    on_demand_cluster: OnDemandCluster = await client.request(
+        CLUSTERS_KEEPER_RPC_NAMESPACE,
+        RPCMethodName("get_or_create_cluster"),
+        timeout_s=RPC_REMOTE_METHOD_TIMEOUT_S,
+        user_id=user_id,
+        wallet_id=wallet_id,
+    )
+    return on_demand_cluster

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/clusters_keeper/ec2_instances.py

@@ -0,0 +1,24 @@
+from models_library.api_schemas_clusters_keeper import CLUSTERS_KEEPER_RPC_NAMESPACE
+from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceType
+from models_library.rabbitmq_basic_types import RPCMethodName
+
+from ..._client_rpc import RabbitMQRPCClient
+from ..._constants import RPC_REMOTE_METHOD_TIMEOUT_S
+
+
+async def get_instance_type_details(
+    client: RabbitMQRPCClient, *, instance_type_names: set[str]
+) -> list[EC2InstanceType]:
+    """**Remote method**
+
+    Raises:
+        RPCServerError -- if anything happens remotely
+
+    """
+    instance_types: list[EC2InstanceType] = await client.request(
+        CLUSTERS_KEEPER_RPC_NAMESPACE,
+        RPCMethodName("get_instance_type_details"),
+        timeout_s=RPC_REMOTE_METHOD_TIMEOUT_S,
+        instance_type_names=instance_type_names,
+    )
+    return instance_types

packages/service-library/tests/rabbitmq/test_rabbitmq_rpc.py

@@ -6,11 +6,11 @@
 from typing import Any, Final
 
 import pytest
+from models_library.rabbitmq_basic_types import RPCMethodName
 from pydantic import NonNegativeInt, ValidationError
 from servicelib.rabbitmq import (
     RabbitMQRPCClient,
     RemoteMethodNotRegisteredError,
-    RPCMethodName,
     RPCNamespace,
     RPCNotInitializedError,
 )

packages/service-library/tests/rabbitmq/test_rabbitmq_rpc_router.py

@@ -5,12 +5,8 @@
 
 import pytest
 from faker import Faker
-from servicelib.rabbitmq import (
-    RabbitMQRPCClient,
-    RPCMethodName,
-    RPCNamespace,
-    RPCRouter,
-)
+from models_library.rabbitmq_basic_types import RPCMethodName
+from servicelib.rabbitmq import RabbitMQRPCClient, RPCNamespace, RPCRouter
 
 pytest_simcore_core_services_selection = [
     "rabbit",

services/clusters-keeper/src/simcore_service_clusters_keeper/modules/ec2.py

@@ -74,7 +74,16 @@ async def get_ec2_instance_capabilities(
         self,
         instance_type_names: set[InstanceTypeType],
     ) -> list[EC2InstanceType]:
-        """instance_type_names must be a set of unique values"""
+        """returns the ec2 instance types from a list of instance type names
+            NOTE: the order might differ!
+        Arguments:
+            instance_type_names -- the types to filter with
+
+        Raises:
+            Ec2InstanceTypeInvalidError: some invalid types were used as filter
+            ClustersKeeperRuntimeError: unexpected error communicating with EC2
+
+        """
         try:
             instance_types = await self.client.describe_instance_types(
                 InstanceTypes=list(instance_type_names)

services/clusters-keeper/tests/unit/test_rpc_clusters.py

@@ -11,12 +11,14 @@
 import pytest
 from faker import Faker
 from fastapi import FastAPI
-from models_library.api_schemas_clusters_keeper import CLUSTERS_KEEPER_RPC_NAMESPACE
 from models_library.api_schemas_clusters_keeper.clusters import OnDemandCluster
 from models_library.users import UserID
 from models_library.wallets import WalletID
 from pytest_mock.plugin import MockerFixture
-from servicelib.rabbitmq import RabbitMQRPCClient, RPCMethodName
+from servicelib.rabbitmq import RabbitMQRPCClient
+from servicelib.rabbitmq.rpc_interfaces.clusters_keeper.clusters import (
+    get_or_create_cluster,
+)
 from simcore_service_clusters_keeper.utils.ec2 import HEARTBEAT_TAG_KEY
 from types_aiobotocore_ec2 import EC2Client
 
@@ -105,9 +107,9 @@ async def test_get_or_create_cluster(
     mocked_dask_ping_scheduler: MockedDaskModule,
 ):
     # send rabbitmq rpc to create_cluster
-    rpc_response = await clusters_keeper_rabbitmq_rpc_client.request(
-        CLUSTERS_KEEPER_RPC_NAMESPACE,
-        RPCMethodName("get_or_create_cluster"),
+
+    rpc_response = await get_or_create_cluster(
+        clusters_keeper_rabbitmq_rpc_client,
         user_id=user_id,
         wallet_id=wallet_id if use_wallet_id else None,
     )
@@ -121,9 +123,8 @@ async def test_get_or_create_cluster(
     mocked_dask_ping_scheduler.ping_scheduler.reset_mock()
 
     # calling it again returns the existing cluster
-    rpc_response = await clusters_keeper_rabbitmq_rpc_client.request(
-        CLUSTERS_KEEPER_RPC_NAMESPACE,
-        RPCMethodName("get_or_create_cluster"),
+    rpc_response = await get_or_create_cluster(
+        clusters_keeper_rabbitmq_rpc_client,
         user_id=user_id,
         wallet_id=wallet_id if use_wallet_id else None,
     )

services/clusters-keeper/tests/unit/test_rpc_ec2_instances.py

@@ -4,9 +4,11 @@
 
 import pytest
 from fastapi import FastAPI
-from models_library.api_schemas_clusters_keeper import CLUSTERS_KEEPER_RPC_NAMESPACE
 from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceType
-from servicelib.rabbitmq import RabbitMQRPCClient, RPCMethodName
+from servicelib.rabbitmq import RabbitMQRPCClient, RPCServerError
+from servicelib.rabbitmq.rpc_interfaces.clusters_keeper.ec2_instances import (
+    get_instance_type_details,
+)
 
 pytest_simcore_core_services_selection = [
     "rabbit",
@@ -34,10 +36,9 @@ async def test_get_instance_type_details_all_options(
     clusters_keeper_rabbitmq_rpc_client: RabbitMQRPCClient,
 ):
     # an empty set returns all options
-    rpc_response = await clusters_keeper_rabbitmq_rpc_client.request(
-        CLUSTERS_KEEPER_RPC_NAMESPACE,
-        RPCMethodName("get_instance_type_details"),
-        instance_type_names=set(),
+
+    rpc_response = await get_instance_type_details(
+        clusters_keeper_rabbitmq_rpc_client, instance_type_names=[]
     )
     assert rpc_response
     assert isinstance(rpc_response, list)
@@ -48,14 +49,23 @@ async def test_get_instance_type_details_specific_type_names(
     _base_configuration: None,
     clusters_keeper_rabbitmq_rpc_client: RabbitMQRPCClient,
 ):
-    # an empty set returns all options
-    rpc_response = await clusters_keeper_rabbitmq_rpc_client.request(
-        CLUSTERS_KEEPER_RPC_NAMESPACE,
-        RPCMethodName("get_instance_type_details"),
+    rpc_response = await get_instance_type_details(
+        clusters_keeper_rabbitmq_rpc_client,
         instance_type_names={"t2.micro", "g4dn.xlarge"},
     )
     assert rpc_response
     assert isinstance(rpc_response, list)
     assert len(rpc_response) == 2
-    assert rpc_response[0].name == "g4dn.xlarge"
     assert rpc_response[1].name == "t2.micro"
+    assert rpc_response[0].name == "g4dn.xlarge"
+
+
+async def test_get_instance_type_details_with_invalid_type_names(
+    _base_configuration: None,
+    clusters_keeper_rabbitmq_rpc_client: RabbitMQRPCClient,
+):
+    with pytest.raises(RPCServerError):
+        await get_instance_type_details(
+            clusters_keeper_rabbitmq_rpc_client,
+            instance_type_names={"t2.micro", "g4dn.xlarge", "invalid.name"},
+        )

services/director-v2/src/simcore_service_director_v2/api/dependencies/rabbitmq.py

@@ -1,7 +1,13 @@
 from fastapi import Request
-from servicelib.rabbitmq import RabbitMQClient
+from servicelib.rabbitmq import RabbitMQClient, RabbitMQRPCClient
+
+from ...modules.rabbitmq import get_rabbitmq_rpc_client
 
 
 def get_rabbitmq_client(request: Request) -> RabbitMQClient:
     assert type(request.app.state.rabbitmq_client) == RabbitMQClient  # nosec
     return request.app.state.rabbitmq_client
+
+
+def rabbitmq_rpc_client(request: Request) -> RabbitMQRPCClient:
+    return get_rabbitmq_rpc_client(request.app)