✨Saving volume stats inside the shared store volume (#4267)

Co-authored-by: Andrei Neagu <[email protected]>

packages/models-library/src/models_library/aiodocker_api.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, validator
 
 from .generated_models.docker_rest_api import (
@@ -13,7 +11,7 @@
 
 
 class AioDockerContainerSpec(ContainerSpec):
-    Env: Optional[dict[str, Optional[str]]] = Field(
+    Env: dict[str, str | None] | None = Field(
         None,
         description="aiodocker expects here a dictionary and re-convert it back internally`.\n",
     )
@@ -35,7 +33,7 @@ def convert_list_to_dict(cls, v):
 class AioDockerResources1(Resources1):
     # NOTE: The Docker REST API documentation is wrong!!!
     # Do not set that back to singular Reservation.
-    Reservation: Optional[ResourceObject] = Field(
+    Reservation: ResourceObject | None = Field(
         None, description="Define resources reservation.", alias="Reservations"
     )
 
@@ -44,19 +42,18 @@ class Config(Resources1.Config):
 
 
 class AioDockerTaskSpec(TaskSpec):
-    ContainerSpec: Optional[AioDockerContainerSpec] = Field(
+    ContainerSpec: AioDockerContainerSpec | None = Field(
         None,
     )
 
-    Resources: Optional[AioDockerResources1] = Field(
+    Resources: AioDockerResources1 | None = Field(
         None,
         description="Resource requirements which apply to each individual container created\nas part of the service.\n",
     )
 
 
 class AioDockerServiceSpec(ServiceSpec):
-
-    TaskTemplate: Optional[AioDockerTaskSpec] = None
+    TaskTemplate: AioDockerTaskSpec | None = None
 
     class Config(ServiceSpec.Config):
         alias_generator = to_snake_case

packages/models-library/src/models_library/clusters.py

@@ -1,4 +1,4 @@
-from typing import Dict, Final, Literal, Optional, Union
+from typing import Final, Literal, Union
 
 from pydantic import AnyUrl, BaseModel, Extra, Field, HttpUrl, SecretStr, root_validator
 from pydantic.types import NonNegativeInt
@@ -48,6 +48,7 @@ class Config(BaseAuthentication.Config):
 
 class KerberosAuthentication(BaseAuthentication):
     type: Literal["kerberos"] = "kerberos"
+
     # NOTE: the entries here still need to be defined
     class Config(BaseAuthentication.Config):
         schema_extra = {
@@ -87,10 +88,10 @@ class NoAuthentication(BaseAuthentication):
 
 class BaseCluster(BaseModel):
     name: str = Field(..., description="The human readable name of the cluster")
-    description: Optional[str] = None
+    description: str | None = None
     type: ClusterType
     owner: GroupID
-    thumbnail: Optional[HttpUrl] = Field(
+    thumbnail: HttpUrl | None = Field(
         None,
         description="url to the image describing this cluster",
         examples=["https://placeimg.com/171/96/tech/grayscale/?0.jpg"],
@@ -99,7 +100,7 @@ class BaseCluster(BaseModel):
     authentication: ClusterAuthentication = Field(
         ..., description="Dask gateway authentication"
     )
-    access_rights: Dict[GroupID, ClusterAccessRights] = Field(default_factory=dict)
+    access_rights: dict[GroupID, ClusterAccessRights] = Field(default_factory=dict)
 
     class Config:
         extra = Extra.forbid

packages/models-library/src/models_library/function_services_catalog/services/iter_sensitivity.py

@@ -1,5 +1,5 @@
 from copy import deepcopy
-from typing import Any, Dict, Iterator, List, Tuple
+from typing import Any, Iterator
 
 from pydantic import schema_of
 
@@ -8,7 +8,7 @@
 from .._key_labels import FUNCTION_SERVICE_KEY_PREFIX
 from .._utils import EN, OM, FunctionServices, create_fake_thumbnail_url
 
-LIST_NUMBERS_SCHEMA: Dict[str, Any] = schema_of(List[float], title="list[number]")
+LIST_NUMBERS_SCHEMA: dict[str, Any] = schema_of(list[float], title="list[number]")
 
 
 META = ServiceDockerData.parse_obj(
@@ -66,11 +66,10 @@
 
 def eval_sensitivity(
     *,
-    paramrefs: List[float],
-    paramdiff: List[float],
+    paramrefs: list[float],
+    paramdiff: list[float],
     diff_or_fact: bool,
-) -> Iterator[Tuple[int, List[float], List[float]]]:
-
+) -> Iterator[tuple[int, list[float], list[float]]]:
     # This code runs in the backend
     assert len(paramrefs) == len(paramdiff)  # nosec
 
@@ -95,7 +94,7 @@ def eval_sensitivity(
 
 
 def _sensitivity_generator(
-    paramrefs: List[float], paramdiff: List[float], diff_or_fact: bool
+    paramrefs: list[float], paramdiff: list[float], diff_or_fact: bool
 ) -> Iterator[OutputsDict]:
     for i, paramtestplus, paramtestminus in eval_sensitivity(
         paramrefs=paramrefs, paramdiff=paramdiff, diff_or_fact=diff_or_fact

packages/models-library/src/models_library/services.py

@@ -70,6 +70,7 @@
 
 LATEST_INTEGRATION_VERSION = "1.0.0"
 
+
 # CONSTRAINT TYPES -------------------------------------------
 class ServicePortKey(ConstrainedStr):
     regex = re.compile(PROPERTY_KEY_RE)

packages/models-library/src/models_library/sidecar_volumes.py

@@ -0,0 +1,55 @@
+from datetime import datetime
+from enum import auto
+
+import arrow
+from pydantic import BaseModel, Field
+
+from .utils.enums import StrAutoEnum
+
+
+class VolumeCategory(StrAutoEnum):
+    """
+    These uniquely identify volumes which are mounted by
+    the dynamic-sidecar and user services.
+
+    This is primarily used to keep track of the status of
+    each individual volume on the volumes.
+
+    The status is ingested by the agent and processed
+    when the volume is removed.
+    """
+
+    # contains data relative to output ports
+    OUTPUTS = auto()
+
+    # contains data relative to input ports
+    INPUTS = auto()
+
+    # contains files which represent the state of the service
+    # usually the user's workspace
+    STATES = auto()
+
+    # contains dynamic-sidecar data required to maintain state
+    # between restarts
+    SHARED_STORE = auto()
+
+
+class VolumeStatus(StrAutoEnum):
+    """
+    Used by the agent to figure out what to do with the data
+    present on the volume.
+    """
+
+    CONTENT_NEEDS_TO_BE_SAVED = auto()
+    CONTENT_WAS_SAVED = auto()
+    CONTENT_NO_SAVE_REQUIRED = auto()
+
+
+class VolumeState(BaseModel):
+    status: VolumeStatus
+    last_changed: datetime = Field(default_factory=lambda: arrow.utcnow().datetime)
+
+    def __eq__(self, other: object) -> bool:
+        # only include status for equality last_changed is not important
+        is_equal: bool = self.status == getattr(other, "status", None)
+        return is_equal

packages/models-library/src/models_library/utils/database_models_factory.py

@@ -6,7 +6,7 @@
 import json
 import warnings
 from datetime import datetime
-from typing import Any, Callable, Container, Optional
+from typing import Any, Callable, Container
 from uuid import UUID
 
 import sqlalchemy as sa
@@ -42,8 +42,8 @@ def _eval_defaults(
     parsing both the client and the server (if include_server_defaults==True) defaults
     in the sa model.
     """
-    default: Optional[Any] = None
-    default_factory: Optional[Callable] = None
+    default: Any | None = None
+    default_factory: Callable | None = None
 
     if (
         column.default is None
@@ -104,11 +104,10 @@ def create_pydantic_model_from_sa_table(
     table: sa.Table,
     *,
     config: type = OrmConfig,
-    exclude: Optional[Container[str]] = None,
+    exclude: Container[str] | None = None,
     include_server_defaults: bool = False,
-    extra_policies: Optional[list[PolicyCallable]] = None,
+    extra_policies: list[PolicyCallable] | None = None,
 ) -> type[BaseModel]:
-
     fields = {}
     exclude = exclude or []
     extra_policies = extra_policies or DEFAULT_EXTRA_POLICIES
@@ -126,7 +125,7 @@ def create_pydantic_model_from_sa_table(
             name = f"{table.name.lower()}_{name}"
 
         # type ---
-        pydantic_type: Optional[type] = None
+        pydantic_type: type | None = None
         if hasattr(column.type, "impl"):
             if hasattr(column.type.impl, "python_type"):
                 pydantic_type = column.type.impl.python_type

packages/models-library/src/models_library/utils/fastapi_encoders.py

@@ -10,7 +10,6 @@
     from fastapi.encoders import jsonable_encoder
 
 except ImportError:  # for aiohttp-only services
-
     # Taken 'as is' from https://github.com/tiangolo/fastapi/blob/master/fastapi/encoders.py
     # to be used in aiohttp-based services w/o having to install fastapi
     #

packages/models-library/tests/test_basic_types.py

@@ -7,7 +7,6 @@
 
 @pytest.mark.skip(reason="DEV: testing parse_obj_as")
 def test_parse_uuid_as_a_string(faker: Faker):
-
     expected_uuid = faker.uuid4()
     got_uuid = parse_obj_as(UUIDStr, expected_uuid)
 

packages/models-library/tests/test_emails.py

@@ -12,4 +12,3 @@ class Profile(BaseModel):
 
     data = Profile(email=email_input)
     assert data.email == "[email protected]"
-

packages/models-library/tests/test_errors.py

@@ -3,15 +3,13 @@
 # pylint: disable=unused-variable
 
 
-from typing import List
-
 import pytest
 from models_library.errors import ErrorDict
 from pydantic import BaseModel, ValidationError, conint
 
 
 class B(BaseModel):
-    y: List[int]
+    y: list[int]
 
 
 class A(BaseModel):
@@ -20,14 +18,13 @@ class A(BaseModel):
 
 
 def test_pydantic_error_dict():
-
     with pytest.raises(ValidationError) as exc_info:
         A(x=-1, b={"y": [0, "wrong"]})
 
     assert isinstance(exc_info.value, ValidationError)
 
     # demos ValidationError.errors() work
-    errors: List[ErrorDict] = exc_info.value.errors()
+    errors: list[ErrorDict] = exc_info.value.errors()
     assert len(errors) == 2
 
     # checks ErrorDict interface

packages/models-library/tests/test_service_settings_labels.py

@@ -56,17 +56,15 @@
     [(x.example, x.items, x.uses_dynamic_sidecar) for x in SIMCORE_SERVICE_EXAMPLES],
     ids=[x.id for x in SIMCORE_SERVICE_EXAMPLES],
 )
-def test_simcore_service_labels(
-    example: dict, items: int, uses_dynamic_sidecar: bool
-) -> None:
+def test_simcore_service_labels(example: dict, items: int, uses_dynamic_sidecar: bool):
     simcore_service_labels = SimcoreServiceLabels.parse_obj(example)
 
     assert simcore_service_labels
     assert len(simcore_service_labels.dict(exclude_unset=True)) == items
     assert simcore_service_labels.needs_dynamic_sidecar == uses_dynamic_sidecar
 
 
-def test_service_settings() -> None:
+def test_service_settings():
     simcore_settings_settings_label = SimcoreServiceSettingsLabel.parse_obj(
         SimcoreServiceSettingLabelEntry.Config.schema_extra["examples"]
     )
@@ -92,7 +90,7 @@ def test_service_settings() -> None:
 )
 def test_service_settings_model_examples(
     model_cls: type[BaseModel], model_cls_examples: dict[str, dict[str, Any]]
-) -> None:
+):
     for name, example in model_cls_examples.items():
         print(name, ":", pformat(example))
         model_instance = model_cls(**example)
@@ -105,7 +103,7 @@ def test_service_settings_model_examples(
 )
 def test_correctly_detect_dynamic_sidecar_boot(
     model_cls: type[BaseModel], model_cls_examples: dict[str, dict[str, Any]]
-) -> None:
+):
     for name, example in model_cls_examples.items():
         print(name, ":", pformat(example))
         model_instance = model_cls(**example)
@@ -114,7 +112,7 @@ def test_correctly_detect_dynamic_sidecar_boot(
         )
 
 
-def test_raises_error_if_http_entrypoint_is_missing() -> None:
+def test_raises_error_if_http_entrypoint_is_missing():
     simcore_service_labels: dict[str, Any] = deepcopy(
         SimcoreServiceLabels.Config.schema_extra["examples"][2]
     )
@@ -124,21 +122,21 @@ def test_raises_error_if_http_entrypoint_is_missing() -> None:
         SimcoreServiceLabels(**simcore_service_labels)
 
 
-def test_path_mappings_none_state_paths() -> None:
+def test_path_mappings_none_state_paths():
     sample_data = deepcopy(PathMappingsLabel.Config.schema_extra["examples"][0])
     sample_data["state_paths"] = None
     with pytest.raises(ValidationError):
         PathMappingsLabel(**sample_data)
 
 
-def test_path_mappings_json_encoding() -> None:
+def test_path_mappings_json_encoding():
     for example in PathMappingsLabel.Config.schema_extra["examples"]:
         path_mappings = PathMappingsLabel.parse_obj(example)
         print(path_mappings)
         assert PathMappingsLabel.parse_raw(path_mappings.json()) == path_mappings
 
 
-def test_simcore_services_labels_compose_spec_null_container_http_entry_provided() -> None:
+def test_simcore_services_labels_compose_spec_null_container_http_entry_provided():
     sample_data = deepcopy(SimcoreServiceLabels.Config.schema_extra["examples"][2])
     assert sample_data["simcore.service.container-http-entrypoint"]
 
@@ -147,7 +145,7 @@ def test_simcore_services_labels_compose_spec_null_container_http_entry_provided
         SimcoreServiceLabels(**sample_data)
 
 
-def test_raises_error_wrong_restart_policy() -> None:
+def test_raises_error_wrong_restart_policy():
     simcore_service_labels: dict[str, Any] = deepcopy(
         SimcoreServiceLabels.Config.schema_extra["examples"][2]
     )

packages/models-library/tests/test_services.py

@@ -198,6 +198,7 @@ def test_same_regex_patterns_in_jsonschema_and_python(
 ):
     # read file in
     json_schema_config = json_schema_dict(json_schema_file_name)
+
     # go to keys
     def _find_pattern_entry(obj: dict[str, Any], key: str) -> Any:
         if key in obj:

packages/models-library/tests/test_services_io.py

@@ -32,7 +32,6 @@ def test_service_port_units(project_tests_dir: Path):
 
 
 def test_build_input_ports_from_json_schemas():
-
     # builds ServiceInput using json-schema
     port_meta = ServiceInput.from_json_schema(
         port_schema={

packages/models-library/tests/test_sidecar_volumes.py

@@ -0,0 +1,18 @@
+# pylint: disable=redefined-outer-name
+
+import pytest
+from models_library.sidecar_volumes import VolumeState, VolumeStatus
+from pytest import FixtureRequest
+
+
+@pytest.fixture(params=VolumeStatus)
+def status(request: FixtureRequest) -> VolumeStatus:
+    return request.param
+
+
+def test_volume_state_equality_does_not_use_last_changed(status: VolumeStatus):
+    # NOTE: `last_changed` is initialized with the utc datetime
+    #  at the moment of the creation of the object.
+    assert VolumeState(status=status) == VolumeState(status=status)
+    schema_property_count = len(VolumeState.schema()["properties"])
+    assert len(VolumeState(status=status).dict()) == schema_property_count