diff --git a/.env-devel b/.env-devel index c8cd92b07b7..eb29840418e 100644 --- a/.env-devel +++ b/.env-devel @@ -47,6 +47,7 @@ DASK_SCHEDULER_PORT=8786 DIRECTOR_REGISTRY_CACHING_TTL=900 DIRECTOR_REGISTRY_CACHING=True +DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}' COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_URL=tcp://dask-scheduler:8786 COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_FILE_LINK_TYPE=S3 @@ -202,6 +203,7 @@ DIRECTOR_V2_HOST=director-v2 DIRECTOR_V2_PORT=8000 DIRECTOR_V2_DYNAMIC_SCHEDULER_IGNORE_SERVICES_SHUTDOWN_WHEN_CREDITS_LIMIT_REACHED=1 DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS=[] +DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}' DYNAMIC_SIDECAR_ENABLE_VOLUME_LIMITS=False DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=3600 diff --git a/packages/models-library/src/models_library/docker.py b/packages/models-library/src/models_library/docker.py index e120f715b5a..79438c4f941 100644 --- a/packages/models-library/src/models_library/docker.py +++ b/packages/models-library/src/models_library/docker.py @@ -35,6 +35,13 @@ class DockerGenericTag(ConstrainedStr): regex: re.Pattern[str] | None = DOCKER_GENERIC_TAG_KEY_RE +class DockerPlacementConstraint(ConstrainedStr): + strip_whitespace = True + regex = re.compile( + r"^(?!-)(?![.])(?!.*--)(?!.*[.][.])[a-zA-Z0-9.-]*(? None: + # NOTE: private attributes cannot be transformed into properties + # since it conflicts with pydantic's internals which treats them + # as fields + self._destination_containers = value + + def get_destination_containers(self) -> list[str]: + # NOTE: private attributes cannot be transformed into properties + # since it conflicts with pydantic's internals which treats them + # as fields + return self._destination_containers + @validator("setting_type", pre=True) @classmethod def ensure_backwards_compatible_setting_type(cls, v): diff --git a/packages/models-library/src/models_library/utils/common_validators.py b/packages/models-library/src/models_library/utils/common_validators.py index 09f3a241d38..09e3d7138b8 100644 --- a/packages/models-library/src/models_library/utils/common_validators.py +++ b/packages/models-library/src/models_library/utils/common_validators.py @@ -42,3 +42,17 @@ def _validator(value: Any): return value return _validator + + +def ensure_unique_list_values_validator(list_data: list) -> list: + if len(list_data) != len(set(list_data)): + msg = f"List values must be unique, provided: {list_data}" + raise ValueError(msg) + return list_data + + +def ensure_unique_dict_values_validator(dict_data: dict) -> dict: + if len(dict_data) != len(set(dict_data.values())): + msg = f"Dictionary values must be unique, provided: {dict_data}" + raise ValueError(msg) + return dict_data diff --git a/packages/models-library/tests/test_service_settings_labels.py b/packages/models-library/tests/test_service_settings_labels.py index f23efa73d13..3ce18fe3fcf 100644 --- a/packages/models-library/tests/test_service_settings_labels.py +++ b/packages/models-library/tests/test_service_settings_labels.py @@ -86,10 +86,7 @@ def test_service_settings(): # ensure private attribute assignment for service_setting in simcore_settings_settings_label: # pylint: disable=protected-access - service_setting._destination_containers = [ # noqa: SLF001 - "random_value1", - "random_value2", - ] + service_setting.set_destination_containers(["random_value1", "random_value2"]) @pytest.mark.parametrize("model_cls", [SimcoreServiceLabels]) diff --git a/services/director-v2/.env-devel b/services/director-v2/.env-devel index 342d368443d..c76d8413a28 100644 --- a/services/director-v2/.env-devel +++ b/services/director-v2/.env-devel @@ -22,6 +22,8 @@ DIRECTOR_V2_SELF_SIGNED_SSL_SECRET_ID=1234 DIRECTOR_V2_SELF_SIGNED_SSL_SECRET_NAME=1234 DIRECTOR_V2_SELF_SIGNED_SSL_FILENAME=filename +DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}' + LOG_LEVEL=DEBUG POSTGRES_ENDPOINT=${POSTGRES_ENDPOINT} diff --git a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/__init__.py b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/__init__.py index 8e58d0d9019..276e21b40c3 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/__init__.py +++ b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/__init__.py @@ -4,7 +4,7 @@ from .egress_proxy import EgressProxySettings from .proxy import DynamicSidecarProxySettings from .scheduler import DynamicServicesSchedulerSettings -from .sidecar import DynamicSidecarSettings +from .sidecar import DynamicSidecarSettings, PlacementSettings class DynamicServicesSettings(BaseCustomSettings): @@ -25,3 +25,7 @@ class DynamicServicesSettings(BaseCustomSettings): DYNAMIC_SIDECAR_EGRESS_PROXY_SETTINGS: EgressProxySettings = Field( auto_default_from_env=True ) + + DYNAMIC_SIDECAR_PLACEMENT_SETTINGS: PlacementSettings = Field( + auto_default_from_env=True + ) diff --git a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py index d47d6bd0764..8a37164015e 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py +++ b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py @@ -1,10 +1,15 @@ import logging +import warnings from enum import Enum from pathlib import Path -from typing import Final from models_library.basic_types import BootModeEnum, PortInt -from pydantic import Field, NonNegativeInt, PositiveInt, validator +from models_library.docker import DockerPlacementConstraint +from models_library.utils.common_validators import ( + ensure_unique_dict_values_validator, + ensure_unique_list_values_validator, +) +from pydantic import Field, PositiveInt, validator from settings_library.base import BaseCustomSettings from settings_library.r_clone import RCloneSettings as SettingsLibraryRCloneSettings from settings_library.utils_logging import MixinLoggingSettings @@ -14,8 +19,6 @@ _logger = logging.getLogger(__name__) -_MINUTE: Final[NonNegativeInt] = 60 - class VFSCacheMode(str, Enum): __slots__ = () @@ -50,6 +53,49 @@ def enforce_r_clone_requirement(cls, v: int, values) -> PositiveInt: return v +class PlacementSettings(BaseCustomSettings): + # This is just a service placement constraint, see + # https://docs.docker.com/engine/swarm/services/#control-service-placement. + DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS: list[DockerPlacementConstraint] = Field( + default_factory=list, + example='["node.labels.region==east", "one!=yes"]', + ) + + DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS: dict[ + str, DockerPlacementConstraint + ] = Field( + default_factory=dict, + description=( + "Use placement constraints in place of generic resources, for details " + "see https://github.com/ITISFoundation/osparc-simcore/issues/5250 " + "When `None` (default), uses generic resources" + ), + example='{"AIRAM": "node.labels.custom==true"}', + ) + + _unique_custom_constraints = validator( + "DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS", + allow_reuse=True, + )(ensure_unique_list_values_validator) + + _unique_resource_placement_constraints_substitutions = validator( + "DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS", + allow_reuse=True, + )(ensure_unique_dict_values_validator) + + @validator("DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS") + @classmethod + def warn_if_any_values_provided(cls, value: dict) -> dict: + if len(value) > 0: + warnings.warn( # noqa: B028 + "Generic resources will be replaced by the following " + f"placement constraints {value}. This is a workaround " + "for https://github.com/moby/swarmkit/pull/3162", + UserWarning, + ) + return value + + class DynamicSidecarSettings(BaseCustomSettings, MixinLoggingSettings): DYNAMIC_SIDECAR_SC_BOOT_MODE: BootModeEnum = Field( ..., @@ -73,6 +119,10 @@ class DynamicSidecarSettings(BaseCustomSettings, MixinLoggingSettings): DYNAMIC_SIDECAR_R_CLONE_SETTINGS: RCloneSettings = Field(auto_default_from_env=True) + DYNAMIC_SIDECAR_PLACEMENT_SETTINGS: PlacementSettings = Field( + auto_default_from_env=True + ) + # # DEVELOPMENT ONLY config # diff --git a/services/director-v2/src/simcore_service_director_v2/core/settings.py b/services/director-v2/src/simcore_service_director_v2/core/settings.py index e42a92dba49..18b44284ccd 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/settings.py +++ b/services/director-v2/src/simcore_service_director_v2/core/settings.py @@ -3,7 +3,6 @@ import datetime -import re from functools import cached_property from models_library.basic_types import ( @@ -19,15 +18,7 @@ ClusterAuthentication, NoAuthentication, ) -from pydantic import ( - AnyHttpUrl, - AnyUrl, - ConstrainedStr, - Field, - NonNegativeInt, - parse_obj_as, - validator, -) +from pydantic import AnyHttpUrl, AnyUrl, Field, NonNegativeInt, parse_obj_as, validator from settings_library.base import BaseCustomSettings from settings_library.catalog import CatalogSettings from settings_library.docker_registry import RegistrySettings @@ -50,13 +41,6 @@ from .dynamic_services_settings import DynamicServicesSettings -class PlacementConstraintStr(ConstrainedStr): - strip_whitespace = True - regex = re.compile( - r"^(?!-)(?![.])(?!.*--)(?!.*[.][.])[a-zA-Z0-9.-]*(? str: diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/settings.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/settings.py index a961175d213..46a2034f50f 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/settings.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/settings.py @@ -5,7 +5,10 @@ from models_library.basic_types import PortInt from models_library.boot_options import BootOption, EnvVarKey -from models_library.docker import to_simcore_runtime_docker_label_key +from models_library.docker import ( + DockerPlacementConstraint, + to_simcore_runtime_docker_label_key, +) from models_library.service_settings_labels import ( ComposeSpecLabelDict, SimcoreServiceLabels, @@ -280,8 +283,7 @@ def _add_compose_destination_containers_to_settings_entries( def _inject_destination_container( item: SimcoreServiceSettingLabelEntry, ) -> SimcoreServiceSettingLabelEntry: - # pylint: disable=protected-access - item._destination_containers = destination_containers + item.set_destination_containers(destination_containers) return item return [_inject_destination_container(x) for x in settings] @@ -290,6 +292,8 @@ def _inject_destination_container( def _merge_resources_in_settings( settings: deque[SimcoreServiceSettingLabelEntry], service_resources: ServiceResourcesDict, + *, + placement_substitutions: dict[str, DockerPlacementConstraint], ) -> deque[SimcoreServiceSettingLabelEntry]: """All oSPARC services which have defined resource requirements will be added""" log.debug("MERGING\n%s\nAND\n%s", f"{settings=}", f"{service_resources}") @@ -338,6 +342,9 @@ def _merge_resources_in_settings( "MemoryBytes" ] += resource_value.reservation else: # generic resources + if resource_name in placement_substitutions: + # NOTE: placement constraint will be used in favour of this generic resource + continue generic_resource = { "DiscreteResourceSpec": { "Kind": resource_name, @@ -382,8 +389,7 @@ def _format_env_var(env_var: str, destination_container: list[str]) -> str: # process entry list_of_env_vars = entry.value if entry.value else [] - # pylint: disable=protected-access - destination_containers: list[str] = entry._destination_containers + destination_containers: list[str] = entry.get_destination_containers() # transforms settings defined environment variables # from `ENV_VAR=PAYLOAD` @@ -459,10 +465,12 @@ async def get_labels_for_involved_services( async def merge_settings_before_use( director_v0_client: DirectorV0Client, + *, service_key: str, service_tag: str, service_user_selection_boot_options: dict[EnvVarKey, str], service_resources: ServiceResourcesDict, + placement_substitutions: dict[str, DockerPlacementConstraint], ) -> SimcoreServiceSettingsLabel: labels_for_involved_services = await get_labels_for_involved_services( director_v0_client=director_v0_client, @@ -501,7 +509,9 @@ async def merge_settings_before_use( ) ) - settings = _merge_resources_in_settings(settings, service_resources) + settings = _merge_resources_in_settings( + settings, service_resources, placement_substitutions=placement_substitutions + ) settings = _patch_target_service_into_env_vars(settings) return SimcoreServiceSettingsLabel.parse_obj(settings) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py index 6ce83b7c4e3..1ccd656b006 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py @@ -8,6 +8,7 @@ from models_library.docker import ( DOCKER_TASK_EC2_INSTANCE_TYPE_PLACEMENT_CONSTRAINT_KEY, DockerLabelKey, + DockerPlacementConstraint, StandardSimcoreDockerLabels, to_simcore_runtime_docker_label_key, ) @@ -21,7 +22,7 @@ DynamicServicesSchedulerSettings, ) from ....core.dynamic_services_settings.sidecar import DynamicSidecarSettings -from ....core.settings import AppSettings, PlacementConstraintStr +from ....core.settings import AppSettings from ....models.dynamic_services_scheduler import SchedulerData from .._namespace import get_compose_namespace from ..volumes import DynamicSidecarVolumesPathsResolver @@ -361,19 +362,32 @@ def get_dynamic_sidecar_spec( | standard_simcore_docker_labels ) + placement_settings = ( + app_settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR.DYNAMIC_SIDECAR_PLACEMENT_SETTINGS + ) placement_constraints = deepcopy( - app_settings.DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS + placement_settings.DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS ) # if service has a pricing plan apply constraints for autoscaling if hardware_info and len(hardware_info.aws_ec2_instances) == 1: ec2_instance_type: str = hardware_info.aws_ec2_instances[0] placement_constraints.append( parse_obj_as( - PlacementConstraintStr, + DockerPlacementConstraint, f"node.labels.{DOCKER_TASK_EC2_INSTANCE_TYPE_PLACEMENT_CONSTRAINT_KEY}=={ec2_instance_type}", ) ) + placement_substitutions: dict[ + str, DockerPlacementConstraint + ] = ( + placement_settings.DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS + ) + for image_resources in scheduler_data.service_resources.values(): + for resource_name in image_resources.resources: + if resource_name in placement_substitutions: + placement_constraints.append(placement_substitutions[resource_name]) + # ----------- create_service_params = { "endpoint_spec": {"Ports": ports} if ports else {}, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py index b493d607871..28bcaa97e8e 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py @@ -26,7 +26,10 @@ from .....core.dynamic_services_settings.scheduler import ( DynamicServicesSchedulerSettings, ) -from .....core.dynamic_services_settings.sidecar import DynamicSidecarSettings +from .....core.dynamic_services_settings.sidecar import ( + DynamicSidecarSettings, + PlacementSettings, +) from .....models.dynamic_services_scheduler import ( DockerContainerInspect, DockerStatus, @@ -125,6 +128,10 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None: dynamic_services_scheduler_settings: DynamicServicesSchedulerSettings = ( app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER ) + dynamic_services_placement_settings: PlacementSettings = ( + app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR.DYNAMIC_SIDECAR_PLACEMENT_SETTINGS + ) + # the dynamic-sidecar should merge all the settings, especially: # resources and placement derived from all the images in # the provided docker-compose spec @@ -152,6 +159,7 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None: service_tag=scheduler_data.version, service_user_selection_boot_options=boot_options, service_resources=scheduler_data.service_resources, + placement_substitutions=dynamic_services_placement_settings.DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS, ) groups_extra_properties = get_repository(app, GroupsExtraPropertiesRepository) diff --git a/services/director-v2/tests/unit/test_core_settings.py b/services/director-v2/tests/unit/test_core_settings.py index e192abfed0e..c36d37e5792 100644 --- a/services/director-v2/tests/unit/test_core_settings.py +++ b/services/director-v2/tests/unit/test_core_settings.py @@ -14,6 +14,7 @@ ) from simcore_service_director_v2.core.dynamic_services_settings.sidecar import ( DynamicSidecarSettings, + PlacementSettings, RCloneSettings, ) from simcore_service_director_v2.core.settings import AppSettings, BootModeEnum @@ -173,8 +174,11 @@ def test_services_custom_constraints( ) -> None: monkeypatch.setenv("DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS", custom_constraints) settings = AppSettings.create_from_envs() - assert isinstance(settings.DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS, list) - assert expected == settings.DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS + custom_constraints = ( + settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR_PLACEMENT_SETTINGS.DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS + ) + assert isinstance(custom_constraints, list) + assert expected == custom_constraints @pytest.mark.parametrize( @@ -212,7 +216,10 @@ def test_services_custom_constraints_default_empty_list( project_env_devel_environment: EnvVarsDict, ) -> None: settings = AppSettings.create_from_envs() - assert [] == settings.DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS + assert ( + [] + == settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR_PLACEMENT_SETTINGS.DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS + ) def test_class_dynamicsidecarsettings_in_development( @@ -258,3 +265,16 @@ def test_class_dynamicsidecarsettings_in_production( def test_envoy_log_level(): for enum in (EnvoyLogLevel("WARNING"), EnvoyLogLevel.WARNING): assert enum.to_log_level() == "warning" + + +def test_placement_settings(monkeypatch: pytest.MonkeyPatch): + assert PlacementSettings() + + monkeypatch.setenv( + "DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS", "{}" + ) + placement_settings = PlacementSettings() + assert ( + placement_settings.DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS + == {} + ) diff --git a/services/director/src/simcore_service_director/config.py b/services/director/src/simcore_service_director/config.py index 82c765fe34d..9bccaa0a539 100644 --- a/services/director/src/simcore_service_director/config.py +++ b/services/director/src/simcore_service_director/config.py @@ -1,8 +1,10 @@ """Director service configuration """ +import json import logging import os +import warnings from distutils.util import strtobool from typing import Dict, Optional @@ -60,6 +62,31 @@ def _from_env_with_default(env: str, python_type, default): "DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS", "" ) + +def _parse_placement_substitutions() -> Dict[str, str]: + str_env_var: str = os.environ.get( + "DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS", "{}" + ) + result: Dict[str, str] = json.loads(str_env_var) + + if len(result) > 0: + warnings.warn( # noqa: B028 + "Generic resources will be replaced by the following " + f"placement constraints {result}. This is a workaround " + "for https://github.com/moby/swarmkit/pull/3162", + UserWarning, + ) + if len(result) != len(set(result.values())): + msg = f"Dictionary values must be unique, provided: {result}" + raise ValueError(msg) + + return result + + +DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS: Dict[ + str, str +] = _parse_placement_substitutions() + # for passing self-signed certificate to spawned services DIRECTOR_SELF_SIGNED_SSL_SECRET_ID: str = os.environ.get( "DIRECTOR_SELF_SIGNED_SSL_SECRET_ID", "" @@ -139,7 +166,7 @@ def _from_env_with_default(env: str, python_type, default): # tracing TRACING_ENABLED: bool = strtobool(os.environ.get("TRACING_ENABLED", "True")) TRACING_ZIPKIN_ENDPOINT: str = os.environ.get( - "TRACING_ZIPKIN_ENDPOINT", "http://jaeger:9411" # NOSONAR + "TRACING_ZIPKIN_ENDPOINT", "http://jaeger:9411" # NOSONAR ) # resources: not taken from servicelib.resources since the director uses a fixed hash of that library diff --git a/services/director/src/simcore_service_director/producer.py b/services/director/src/simcore_service_director/producer.py index 9d3db4d03a8..1c6047d183f 100644 --- a/services/director/src/simcore_service_director/producer.py +++ b/services/director/src/simcore_service_director/producer.py @@ -7,7 +7,7 @@ from enum import Enum from http import HTTPStatus from pprint import pformat -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Set, Tuple import aiodocker import aiohttp @@ -292,6 +292,11 @@ async def _create_docker_service_params( f"traefik.http.routers.{service_name}.middlewares" ] += f", {service_name}_stripprefixregex" + placement_constraints_to_substitute: List[str] = [] + placement_substitutions: Dict[ + str, str + ] = config.DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS + for param in service_parameters_labels: _check_setting_correctness(param) # replace %service_uuid% by the given uuid @@ -319,6 +324,32 @@ async def _create_docker_service_params( "NanoCPUs" ] = param["value"]["cpu_reservation"] # REST-API compatible + if ( + placement_substitutions + and "Reservations" in param["value"] + and "GenericResources" in param["value"]["Reservations"] + ): + # Use placement constraints in place of generic resources, for details + # see https://github.com/ITISFoundation/osparc-simcore/issues/5250 + # removing them form here + generic_resources: list = param["value"]["Reservations"][ + "GenericResources" + ] + + to_remove: Set[str] = set() + for generic_resource in generic_resources: + kind = generic_resource["DiscreteResourceSpec"]["Kind"] + if kind in placement_substitutions: + placement_constraints_to_substitute.append(kind) + to_remove.add(kind) + + # only include generic resources which must not be substituted + param["value"]["Reservations"]["GenericResources"] = [ + x + for x in generic_resources + if x["DiscreteResourceSpec"]["Kind"] not in to_remove + ] + if "Limits" in param["value"] or "Reservations" in param["value"]: docker_params["task_template"]["Resources"].update(param["value"]) @@ -377,6 +408,12 @@ async def _create_docker_service_params( mount_settings ) + # add placement constraints based on what was found + for generic_resource_kind in placement_constraints_to_substitute: + docker_params["task_template"]["Placement"]["Constraints"] += [ + placement_substitutions[generic_resource_kind] + ] + # attach the service to the swarm network dedicated to services try: swarm_network = await _get_swarm_network(client) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 84c417bf5f8..3acddae3065 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -192,6 +192,7 @@ services: - DIRECTOR_SELF_SIGNED_SSL_SECRET_ID=${DIRECTOR_SELF_SIGNED_SSL_SECRET_ID} - DIRECTOR_SELF_SIGNED_SSL_SECRET_NAME=${DIRECTOR_SELF_SIGNED_SSL_SECRET_NAME} - DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS=${DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS} + - DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS=${DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS} - EXTRA_HOSTS_SUFFIX=${EXTRA_HOSTS_SUFFIX:-undefined} - LOGLEVEL=${LOG_LEVEL:-WARNING} - MONITORING_ENABLED=${MONITORING_ENABLED:-True} @@ -247,6 +248,7 @@ services: - DIRECTOR_SELF_SIGNED_SSL_SECRET_ID=${DIRECTOR_SELF_SIGNED_SSL_SECRET_ID} - DIRECTOR_SELF_SIGNED_SSL_SECRET_NAME=${DIRECTOR_SELF_SIGNED_SSL_SECRET_NAME} - DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS=${DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS} + - DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS=${DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS} - DIRECTOR_V2_DEV_FEATURES_ENABLED=${DIRECTOR_V2_DEV_FEATURES_ENABLED} - DIRECTOR_V2_DYNAMIC_SCHEDULER_IGNORE_SERVICES_SHUTDOWN_WHEN_CREDITS_LIMIT_REACHED=${DIRECTOR_V2_DYNAMIC_SCHEDULER_IGNORE_SERVICES_SHUTDOWN_WHEN_CREDITS_LIMIT_REACHED}