Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

✨Saving volume stats inside the shared store volume #4267

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 5 additions & 8 deletions packages/models-library/src/models_library/aiodocker_api.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
from typing import Optional

from pydantic import Field, validator

from .generated_models.docker_rest_api import (
Expand All @@ -13,7 +11,7 @@


class AioDockerContainerSpec(ContainerSpec):
Env: Optional[dict[str, Optional[str]]] = Field(
Env: dict[str, str | None] | None = Field(
None,
description="aiodocker expects here a dictionary and re-convert it back internally`.\n",
)
Expand All @@ -35,7 +33,7 @@ def convert_list_to_dict(cls, v):
class AioDockerResources1(Resources1):
# NOTE: The Docker REST API documentation is wrong!!!
# Do not set that back to singular Reservation.
Reservation: Optional[ResourceObject] = Field(
Reservation: ResourceObject | None = Field(
None, description="Define resources reservation.", alias="Reservations"
)

Expand All @@ -44,19 +42,18 @@ class Config(Resources1.Config):


class AioDockerTaskSpec(TaskSpec):
ContainerSpec: Optional[AioDockerContainerSpec] = Field(
ContainerSpec: AioDockerContainerSpec | None = Field(
None,
)

Resources: Optional[AioDockerResources1] = Field(
Resources: AioDockerResources1 | None = Field(
None,
description="Resource requirements which apply to each individual container created\nas part of the service.\n",
)


class AioDockerServiceSpec(ServiceSpec):

TaskTemplate: Optional[AioDockerTaskSpec] = None
TaskTemplate: AioDockerTaskSpec | None = None

class Config(ServiceSpec.Config):
alias_generator = to_snake_case
Expand Down
9 changes: 5 additions & 4 deletions packages/models-library/src/models_library/clusters.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from typing import Dict, Final, Literal, Optional, Union
from typing import Final, Literal, Union

from pydantic import AnyUrl, BaseModel, Extra, Field, HttpUrl, SecretStr, root_validator
from pydantic.types import NonNegativeInt
Expand Down Expand Up @@ -48,6 +48,7 @@ class Config(BaseAuthentication.Config):

class KerberosAuthentication(BaseAuthentication):
type: Literal["kerberos"] = "kerberos"

# NOTE: the entries here still need to be defined
class Config(BaseAuthentication.Config):
schema_extra = {
Expand Down Expand Up @@ -87,10 +88,10 @@ class NoAuthentication(BaseAuthentication):

class BaseCluster(BaseModel):
name: str = Field(..., description="The human readable name of the cluster")
description: Optional[str] = None
description: str | None = None
type: ClusterType
owner: GroupID
thumbnail: Optional[HttpUrl] = Field(
thumbnail: HttpUrl | None = Field(
None,
description="url to the image describing this cluster",
examples=["https://placeimg.com/171/96/tech/grayscale/?0.jpg"],
Expand All @@ -99,7 +100,7 @@ class BaseCluster(BaseModel):
authentication: ClusterAuthentication = Field(
..., description="Dask gateway authentication"
)
access_rights: Dict[GroupID, ClusterAccessRights] = Field(default_factory=dict)
access_rights: dict[GroupID, ClusterAccessRights] = Field(default_factory=dict)

class Config:
extra = Extra.forbid
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from copy import deepcopy
from typing import Any, Dict, Iterator, List, Tuple
from typing import Any, Iterator

from pydantic import schema_of

Expand All @@ -8,7 +8,7 @@
from .._key_labels import FUNCTION_SERVICE_KEY_PREFIX
from .._utils import EN, OM, FunctionServices, create_fake_thumbnail_url

LIST_NUMBERS_SCHEMA: Dict[str, Any] = schema_of(List[float], title="list[number]")
LIST_NUMBERS_SCHEMA: dict[str, Any] = schema_of(list[float], title="list[number]")


META = ServiceDockerData.parse_obj(
Expand Down Expand Up @@ -66,11 +66,10 @@

def eval_sensitivity(
*,
paramrefs: List[float],
paramdiff: List[float],
paramrefs: list[float],
paramdiff: list[float],
diff_or_fact: bool,
) -> Iterator[Tuple[int, List[float], List[float]]]:

) -> Iterator[tuple[int, list[float], list[float]]]:
# This code runs in the backend
assert len(paramrefs) == len(paramdiff) # nosec

Expand All @@ -95,7 +94,7 @@ def eval_sensitivity(


def _sensitivity_generator(
paramrefs: List[float], paramdiff: List[float], diff_or_fact: bool
paramrefs: list[float], paramdiff: list[float], diff_or_fact: bool
) -> Iterator[OutputsDict]:
for i, paramtestplus, paramtestminus in eval_sensitivity(
paramrefs=paramrefs, paramdiff=paramdiff, diff_or_fact=diff_or_fact
Expand Down
1 change: 1 addition & 0 deletions packages/models-library/src/models_library/services.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@

LATEST_INTEGRATION_VERSION = "1.0.0"


# CONSTRAINT TYPES -------------------------------------------
class ServicePortKey(ConstrainedStr):
regex = re.compile(PROPERTY_KEY_RE)
Expand Down
55 changes: 55 additions & 0 deletions packages/models-library/src/models_library/sidecar_volumes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
from datetime import datetime
from enum import auto

import arrow
from pydantic import BaseModel, Field

from .utils.enums import StrAutoEnum


class VolumeCategory(StrAutoEnum):
"""
These uniquely identify volumes which are mounted by
the dynamic-sidecar and user services.

This is primarily used to keep track of the status of
each individual volume on the volumes.

The status is ingested by the agent and processed
when the volume is removed.
"""

# contains data relative to output ports
OUTPUTS = auto()

# contains data relative to input ports
INPUTS = auto()

# contains files which represent the state of the service
# usually the user's workspace
STATES = auto()

# contains dynamic-sidecar data required to maintain state
# between restarts
SHARED_STORE = auto()


class VolumeStatus(StrAutoEnum):
"""
Used by the agent to figure out what to do with the data
present on the volume.
"""

CONTENT_NEEDS_TO_BE_SAVED = auto()
CONTENT_WAS_SAVED = auto()
CONTENT_NO_SAVE_REQUIRED = auto()


class VolumeState(BaseModel):
status: VolumeStatus
last_changed: datetime = Field(default_factory=lambda: arrow.utcnow().datetime)

def __eq__(self, other: object) -> bool:
# only include status for equality last_changed is not important
is_equal: bool = self.status == getattr(other, "status", None)
return is_equal
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import json
import warnings
from datetime import datetime
from typing import Any, Callable, Container, Optional
from typing import Any, Callable, Container
from uuid import UUID

import sqlalchemy as sa
Expand Down Expand Up @@ -42,8 +42,8 @@ def _eval_defaults(
parsing both the client and the server (if include_server_defaults==True) defaults
in the sa model.
"""
default: Optional[Any] = None
default_factory: Optional[Callable] = None
default: Any | None = None
default_factory: Callable | None = None

if (
column.default is None
Expand Down Expand Up @@ -104,11 +104,10 @@ def create_pydantic_model_from_sa_table(
table: sa.Table,
*,
config: type = OrmConfig,
exclude: Optional[Container[str]] = None,
exclude: Container[str] | None = None,
include_server_defaults: bool = False,
extra_policies: Optional[list[PolicyCallable]] = None,
extra_policies: list[PolicyCallable] | None = None,
) -> type[BaseModel]:

fields = {}
exclude = exclude or []
extra_policies = extra_policies or DEFAULT_EXTRA_POLICIES
Expand All @@ -126,7 +125,7 @@ def create_pydantic_model_from_sa_table(
name = f"{table.name.lower()}_{name}"

# type ---
pydantic_type: Optional[type] = None
pydantic_type: type | None = None
if hasattr(column.type, "impl"):
if hasattr(column.type.impl, "python_type"):
pydantic_type = column.type.impl.python_type
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
from fastapi.encoders import jsonable_encoder

except ImportError: # for aiohttp-only services

# Taken 'as is' from https://github.com/tiangolo/fastapi/blob/master/fastapi/encoders.py
# to be used in aiohttp-based services w/o having to install fastapi
#
Expand Down
1 change: 0 additions & 1 deletion packages/models-library/tests/test_basic_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@

@pytest.mark.skip(reason="DEV: testing parse_obj_as")
def test_parse_uuid_as_a_string(faker: Faker):

expected_uuid = faker.uuid4()
got_uuid = parse_obj_as(UUIDStr, expected_uuid)

Expand Down
1 change: 0 additions & 1 deletion packages/models-library/tests/test_emails.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,3 @@ class Profile(BaseModel):

data = Profile(email=email_input)
assert data.email == "[email protected]"

7 changes: 2 additions & 5 deletions packages/models-library/tests/test_errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,13 @@
# pylint: disable=unused-variable


from typing import List

import pytest
from models_library.errors import ErrorDict
from pydantic import BaseModel, ValidationError, conint


class B(BaseModel):
y: List[int]
y: list[int]


class A(BaseModel):
Expand All @@ -20,14 +18,13 @@ class A(BaseModel):


def test_pydantic_error_dict():

with pytest.raises(ValidationError) as exc_info:
A(x=-1, b={"y": [0, "wrong"]})

assert isinstance(exc_info.value, ValidationError)

# demos ValidationError.errors() work
errors: List[ErrorDict] = exc_info.value.errors()
errors: list[ErrorDict] = exc_info.value.errors()
assert len(errors) == 2

# checks ErrorDict interface
Expand Down
20 changes: 9 additions & 11 deletions packages/models-library/tests/test_service_settings_labels.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,17 +56,15 @@
[(x.example, x.items, x.uses_dynamic_sidecar) for x in SIMCORE_SERVICE_EXAMPLES],
ids=[x.id for x in SIMCORE_SERVICE_EXAMPLES],
)
def test_simcore_service_labels(
example: dict, items: int, uses_dynamic_sidecar: bool
) -> None:
def test_simcore_service_labels(example: dict, items: int, uses_dynamic_sidecar: bool):
simcore_service_labels = SimcoreServiceLabels.parse_obj(example)

assert simcore_service_labels
assert len(simcore_service_labels.dict(exclude_unset=True)) == items
assert simcore_service_labels.needs_dynamic_sidecar == uses_dynamic_sidecar


def test_service_settings() -> None:
def test_service_settings():
simcore_settings_settings_label = SimcoreServiceSettingsLabel.parse_obj(
SimcoreServiceSettingLabelEntry.Config.schema_extra["examples"]
)
Expand All @@ -92,7 +90,7 @@ def test_service_settings() -> None:
)
def test_service_settings_model_examples(
model_cls: type[BaseModel], model_cls_examples: dict[str, dict[str, Any]]
) -> None:
):
for name, example in model_cls_examples.items():
print(name, ":", pformat(example))
model_instance = model_cls(**example)
Expand All @@ -105,7 +103,7 @@ def test_service_settings_model_examples(
)
def test_correctly_detect_dynamic_sidecar_boot(
model_cls: type[BaseModel], model_cls_examples: dict[str, dict[str, Any]]
) -> None:
):
for name, example in model_cls_examples.items():
print(name, ":", pformat(example))
model_instance = model_cls(**example)
Expand All @@ -114,7 +112,7 @@ def test_correctly_detect_dynamic_sidecar_boot(
)


def test_raises_error_if_http_entrypoint_is_missing() -> None:
def test_raises_error_if_http_entrypoint_is_missing():
simcore_service_labels: dict[str, Any] = deepcopy(
SimcoreServiceLabels.Config.schema_extra["examples"][2]
)
Expand All @@ -124,21 +122,21 @@ def test_raises_error_if_http_entrypoint_is_missing() -> None:
SimcoreServiceLabels(**simcore_service_labels)


def test_path_mappings_none_state_paths() -> None:
def test_path_mappings_none_state_paths():
sample_data = deepcopy(PathMappingsLabel.Config.schema_extra["examples"][0])
sample_data["state_paths"] = None
with pytest.raises(ValidationError):
PathMappingsLabel(**sample_data)


def test_path_mappings_json_encoding() -> None:
def test_path_mappings_json_encoding():
for example in PathMappingsLabel.Config.schema_extra["examples"]:
path_mappings = PathMappingsLabel.parse_obj(example)
print(path_mappings)
assert PathMappingsLabel.parse_raw(path_mappings.json()) == path_mappings


def test_simcore_services_labels_compose_spec_null_container_http_entry_provided() -> None:
def test_simcore_services_labels_compose_spec_null_container_http_entry_provided():
sample_data = deepcopy(SimcoreServiceLabels.Config.schema_extra["examples"][2])
assert sample_data["simcore.service.container-http-entrypoint"]

Expand All @@ -147,7 +145,7 @@ def test_simcore_services_labels_compose_spec_null_container_http_entry_provided
SimcoreServiceLabels(**sample_data)


def test_raises_error_wrong_restart_policy() -> None:
def test_raises_error_wrong_restart_policy():
simcore_service_labels: dict[str, Any] = deepcopy(
SimcoreServiceLabels.Config.schema_extra["examples"][2]
)
Expand Down
1 change: 1 addition & 0 deletions packages/models-library/tests/test_services.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,7 @@ def test_same_regex_patterns_in_jsonschema_and_python(
):
# read file in
json_schema_config = json_schema_dict(json_schema_file_name)

# go to keys
def _find_pattern_entry(obj: dict[str, Any], key: str) -> Any:
if key in obj:
Expand Down
1 change: 0 additions & 1 deletion packages/models-library/tests/test_services_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ def test_service_port_units(project_tests_dir: Path):


def test_build_input_ports_from_json_schemas():

# builds ServiceInput using json-schema
port_meta = ServiceInput.from_json_schema(
port_schema={
Expand Down
18 changes: 18 additions & 0 deletions packages/models-library/tests/test_sidecar_volumes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# pylint: disable=redefined-outer-name

import pytest
from models_library.sidecar_volumes import VolumeState, VolumeStatus
from pytest import FixtureRequest


@pytest.fixture(params=VolumeStatus)
def status(request: FixtureRequest) -> VolumeStatus:
return request.param


def test_volume_state_equality_does_not_use_last_changed(status: VolumeStatus):
# NOTE: `last_changed` is initialized with the utc datetime
# at the moment of the creation of the object.
assert VolumeState(status=status) == VolumeState(status=status)
schema_property_count = len(VolumeState.schema()["properties"])
assert len(VolumeState(status=status).dict()) == schema_property_count
Loading