♻️/✨use placement constraints instead of generic resources ⚠️

.env-devel

@@ -47,6 +47,7 @@ DASK_SCHEDULER_PORT=8786
 
 DIRECTOR_REGISTRY_CACHING_TTL=900
 DIRECTOR_REGISTRY_CACHING=True
+DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}'
 
 COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_URL=tcp://dask-scheduler:8786
 COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_FILE_LINK_TYPE=S3
@@ -202,6 +203,7 @@ DIRECTOR_V2_HOST=director-v2
 DIRECTOR_V2_PORT=8000
 DIRECTOR_V2_DYNAMIC_SCHEDULER_IGNORE_SERVICES_SHUTDOWN_WHEN_CREDITS_LIMIT_REACHED=1
 DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS=[]
+DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}'
 DYNAMIC_SIDECAR_ENABLE_VOLUME_LIMITS=False
 
 DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=3600

packages/models-library/src/models_library/service_settings_labels.py

@@ -81,6 +81,18 @@ class SimcoreServiceSettingLabelEntry(BaseModel):
         description="The value of the service setting (shall follow Docker REST API scheme for services",
     )
 
+    def set_destination_containers(self, value: list[str]) -> None:
+        # NOTE: private attributes cannot be transformed into properties
+        # since it conflicts with pydantic's internals which treats them
+        # as fields
+        self._destination_containers = value
+
+    def get_destination_containers(self) -> list[str]:
+        # NOTE: private attributes cannot be transformed into properties
+        # since it conflicts with pydantic's internals which treats them
+        # as fields
+        return self._destination_containers
+
     @validator("setting_type", pre=True)
     @classmethod
     def ensure_backwards_compatible_setting_type(cls, v):

packages/models-library/tests/test_service_settings_labels.py

@@ -86,10 +86,7 @@ def test_service_settings():
     # ensure private attribute assignment
     for service_setting in simcore_settings_settings_label:
         # pylint: disable=protected-access
-        service_setting._destination_containers = [  # noqa: SLF001
-            "random_value1",
-            "random_value2",
-        ]
+        service_setting.set_destination_containers(["random_value1", "random_value2"])
 
 
 @pytest.mark.parametrize("model_cls", [SimcoreServiceLabels])

services/director-v2/.env-devel

@@ -22,6 +22,8 @@ DIRECTOR_V2_SELF_SIGNED_SSL_SECRET_ID=1234
 DIRECTOR_V2_SELF_SIGNED_SSL_SECRET_NAME=1234
 DIRECTOR_V2_SELF_SIGNED_SSL_FILENAME=filename
 
+DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}'
+
 LOG_LEVEL=DEBUG
 
 POSTGRES_ENDPOINT=${POSTGRES_ENDPOINT}

services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/__init__.py

@@ -4,7 +4,7 @@
 from .egress_proxy import EgressProxySettings
 from .proxy import DynamicSidecarProxySettings
 from .scheduler import DynamicServicesSchedulerSettings
-from .sidecar import DynamicSidecarSettings
+from .sidecar import DynamicSidecarSettings, PlacementSettings
 
 
 class DynamicServicesSettings(BaseCustomSettings):
@@ -25,3 +25,7 @@ class DynamicServicesSettings(BaseCustomSettings):
     DYNAMIC_SIDECAR_EGRESS_PROXY_SETTINGS: EgressProxySettings = Field(
         auto_default_from_env=True
     )
+
+    DYNAMIC_SIDECAR_PLACEMENT_SETTINGS: PlacementSettings = Field(
+        auto_default_from_env=True
+    )

services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py

@@ -1,10 +1,11 @@
 import logging
+import re
+import warnings
 from enum import Enum
 from pathlib import Path
-from typing import Final
 
 from models_library.basic_types import BootModeEnum, PortInt
-from pydantic import Field, NonNegativeInt, PositiveInt, validator
+from pydantic import ConstrainedStr, Field, PositiveInt, validator
 from settings_library.base import BaseCustomSettings
 from settings_library.r_clone import RCloneSettings as SettingsLibraryRCloneSettings
 from settings_library.utils_logging import MixinLoggingSettings
@@ -14,8 +15,6 @@
 
 _logger = logging.getLogger(__name__)
 
-_MINUTE: Final[NonNegativeInt] = 60
-
 
 class VFSCacheMode(str, Enum):
     __slots__ = ()
@@ -50,6 +49,46 @@ def enforce_r_clone_requirement(cls, v: int, values) -> PositiveInt:
         return v
 
 
+class PlacementConstraintStr(ConstrainedStr):
+    strip_whitespace = True
+    regex = re.compile(
+        r"^(?!-)(?![.])(?!.*--)(?!.*[.][.])[a-zA-Z0-9.-]*(?<!-)(?<![.])(!=|==)[a-zA-Z0-9_. -]*$"
+    )
+
+
+class PlacementSettings(BaseCustomSettings):
+    # This is just a service placement constraint, see
+    # https://docs.docker.com/engine/swarm/services/#control-service-placement.
+    DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS: list[PlacementConstraintStr] = Field(
+        default_factory=list,
+        example='["node.labels.region==east", "one!=yes"]',
+    )
+
+    DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS: dict[
+        str, PlacementConstraintStr
+    ] = Field(
+        default_factory=dict,
+        description=(
+            "Use placement constraints in place of generic resources, for details "
+            "see https://github.com/ITISFoundation/osparc-simcore/issues/5250 "
+            "When `None` (default), uses generic resources"
+        ),
+        example='{"AIRAM": "node.labels.custom==true"}',
+    )
+
+    @validator("DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS")
+    @classmethod
+    def warn_if_any_values_provided(cls, value: dict) -> dict:
+        if len(value) > 0:
+            warnings.warn(  # noqa: B028
+                "Generic resources will be replaced by the following "
+                f"placement constraints {value}. This is a workaround "
+                "for https://github.com/moby/swarmkit/pull/3162",
+                UserWarning,
+            )
+        return value
+
+
 class DynamicSidecarSettings(BaseCustomSettings, MixinLoggingSettings):
     DYNAMIC_SIDECAR_SC_BOOT_MODE: BootModeEnum = Field(
         ...,
@@ -73,6 +112,10 @@ class DynamicSidecarSettings(BaseCustomSettings, MixinLoggingSettings):
 
     DYNAMIC_SIDECAR_R_CLONE_SETTINGS: RCloneSettings = Field(auto_default_from_env=True)
 
+    DYNAMIC_SIDECAR_PLACEMENT_SETTINGS: PlacementSettings = Field(
+        auto_default_from_env=True
+    )
+
     #
     # DEVELOPMENT ONLY config
     #

services/director-v2/src/simcore_service_director_v2/core/settings.py

@@ -3,7 +3,6 @@
 
 
 import datetime
-import re
 from functools import cached_property
 
 from models_library.basic_types import (
@@ -19,15 +18,7 @@
     ClusterAuthentication,
     NoAuthentication,
 )
-from pydantic import (
-    AnyHttpUrl,
-    AnyUrl,
-    ConstrainedStr,
-    Field,
-    NonNegativeInt,
-    parse_obj_as,
-    validator,
-)
+from pydantic import AnyHttpUrl, AnyUrl, Field, NonNegativeInt, parse_obj_as, validator
 from settings_library.base import BaseCustomSettings
 from settings_library.catalog import CatalogSettings
 from settings_library.docker_registry import RegistrySettings
@@ -50,13 +41,6 @@
 from .dynamic_services_settings import DynamicServicesSettings
 
 
-class PlacementConstraintStr(ConstrainedStr):
-    strip_whitespace = True
-    regex = re.compile(
-        r"^(?!-)(?![.])(?!.*--)(?!.*[.][.])[a-zA-Z0-9.-]*(?<!-)(?<![.])(!=|==)[a-zA-Z0-9_. -]*$"
-    )
-
-
 class DirectorV0Settings(BaseCustomSettings):
     DIRECTOR_V0_ENABLED: bool = True
 
@@ -225,13 +209,6 @@ class AppSettings(BaseCustomSettings, MixinLoggingSettings):
         description="resource usage tracker service client's plugin",
     )
 
-    # This is just a service placement constraint, see
-    # https://docs.docker.com/engine/swarm/services/#control-service-placement.
-    DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS: list[PlacementConstraintStr] = Field(
-        default_factory=list,
-        example='["node.labels.region==east", "one!=yes"]',
-    )
-
     @validator("LOG_LEVEL", pre=True)
     @classmethod
     def _validate_loglevel(cls, value: str) -> str:

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/settings.py

@@ -26,6 +26,7 @@
     MATCH_SERVICE_VERSION,
 )
 
+from ....core.dynamic_services_settings.sidecar import PlacementConstraintStr
 from ....modules.director_v0 import DirectorV0Client
 from ..errors import DynamicSidecarError
 
@@ -280,8 +281,7 @@ def _add_compose_destination_containers_to_settings_entries(
     def _inject_destination_container(
         item: SimcoreServiceSettingLabelEntry,
     ) -> SimcoreServiceSettingLabelEntry:
-        # pylint: disable=protected-access
-        item._destination_containers = destination_containers
+        item.set_destination_containers(destination_containers)
         return item
 
     return [_inject_destination_container(x) for x in settings]
@@ -290,6 +290,8 @@ def _inject_destination_container(
 def _merge_resources_in_settings(
     settings: deque[SimcoreServiceSettingLabelEntry],
     service_resources: ServiceResourcesDict,
+    *,
+    placement_substitutions: dict[str, PlacementConstraintStr],
 ) -> deque[SimcoreServiceSettingLabelEntry]:
     """All oSPARC services which have defined resource requirements will be added"""
     log.debug("MERGING\n%s\nAND\n%s", f"{settings=}", f"{service_resources}")
@@ -338,6 +340,9 @@ def _merge_resources_in_settings(
                     "MemoryBytes"
                 ] += resource_value.reservation
             else:  # generic resources
+                if resource_name in placement_substitutions:
+                    # NOTE: placement constraint will be used in favour of this generic resource
+                    continue
                 generic_resource = {
                     "DiscreteResourceSpec": {
                         "Kind": resource_name,
@@ -382,8 +387,7 @@ def _format_env_var(env_var: str, destination_container: list[str]) -> str:
             # process entry
             list_of_env_vars = entry.value if entry.value else []
 
-            # pylint: disable=protected-access
-            destination_containers: list[str] = entry._destination_containers
+            destination_containers: list[str] = entry.get_destination_containers()
 
             # transforms settings defined environment variables
             # from `ENV_VAR=PAYLOAD`
@@ -459,10 +463,12 @@ async def get_labels_for_involved_services(
 
 async def merge_settings_before_use(
     director_v0_client: DirectorV0Client,
+    *,
     service_key: str,
     service_tag: str,
     service_user_selection_boot_options: dict[EnvVarKey, str],
     service_resources: ServiceResourcesDict,
+    placement_substitutions: dict[str, PlacementConstraintStr],
 ) -> SimcoreServiceSettingsLabel:
     labels_for_involved_services = await get_labels_for_involved_services(
         director_v0_client=director_v0_client,
@@ -501,7 +507,9 @@ async def merge_settings_before_use(
             )
         )
 
-    settings = _merge_resources_in_settings(settings, service_resources)
+    settings = _merge_resources_in_settings(
+        settings, service_resources, placement_substitutions=placement_substitutions
+    )
     settings = _patch_target_service_into_env_vars(settings)
 
     return SimcoreServiceSettingsLabel.parse_obj(settings)

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py

@@ -20,8 +20,11 @@
 from ....core.dynamic_services_settings.scheduler import (
     DynamicServicesSchedulerSettings,
 )
-from ....core.dynamic_services_settings.sidecar import DynamicSidecarSettings
-from ....core.settings import AppSettings, PlacementConstraintStr
+from ....core.dynamic_services_settings.sidecar import (
+    DynamicSidecarSettings,
+    PlacementConstraintStr,
+)
+from ....core.settings import AppSettings
 from ....models.dynamic_services_scheduler import SchedulerData
 from .._namespace import get_compose_namespace
 from ..volumes import DynamicSidecarVolumesPathsResolver
@@ -361,8 +364,11 @@ def get_dynamic_sidecar_spec(
         | standard_simcore_docker_labels
     )
 
+    placement_settings = (
+        app_settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR.DYNAMIC_SIDECAR_PLACEMENT_SETTINGS
+    )
     placement_constraints = deepcopy(
-        app_settings.DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS
+        placement_settings.DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS
     )
     # if service has a pricing plan apply constraints for autoscaling
     if hardware_info and len(hardware_info.aws_ec2_instances) == 1:
@@ -374,6 +380,16 @@ def get_dynamic_sidecar_spec(
             )
         )
 
+    placement_substitutions: dict[
+        str, PlacementConstraintStr
+    ] = (
+        placement_settings.DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS
+    )
+    for image_resources in scheduler_data.service_resources.values():
+        for resource_name in image_resources.resources:
+            if resource_name in placement_substitutions:
+                placement_constraints.append(placement_substitutions[resource_name])
+
     #  -----------
     create_service_params = {
         "endpoint_spec": {"Ports": ports} if ports else {},

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py

@@ -26,7 +26,10 @@
 from .....core.dynamic_services_settings.scheduler import (
     DynamicServicesSchedulerSettings,
 )
-from .....core.dynamic_services_settings.sidecar import DynamicSidecarSettings
+from .....core.dynamic_services_settings.sidecar import (
+    DynamicSidecarSettings,
+    PlacementSettings,
+)
 from .....models.dynamic_services_scheduler import (
     DockerContainerInspect,
     DockerStatus,
@@ -125,6 +128,10 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None:
         dynamic_services_scheduler_settings: DynamicServicesSchedulerSettings = (
             app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER
         )
+        dynamic_services_placement_settings: PlacementSettings = (
+            app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR.DYNAMIC_SIDECAR_PLACEMENT_SETTINGS
+        )
+
         # the dynamic-sidecar should merge all the settings, especially:
         # resources and placement derived from all the images in
         # the provided docker-compose spec
@@ -152,6 +159,7 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None:
             service_tag=scheduler_data.version,
             service_user_selection_boot_options=boot_options,
             service_resources=scheduler_data.service_resources,
+            placement_substitutions=dynamic_services_placement_settings.DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS,
         )
 
         groups_extra_properties = get_repository(app, GroupsExtraPropertiesRepository)