Revert "♻️ Volumes keep track if the data requires saving 🚨 (#3974)"

packages/models-library/src/models_library/users.py

@@ -1,6 +1,4 @@
-from typing import TypeAlias
-
 from pydantic import PositiveInt
 
-UserID: TypeAlias = PositiveInt
-GroupID: TypeAlias = PositiveInt
+UserID = PositiveInt
+GroupID = PositiveInt

packages/service-library/requirements/_base.in

@@ -4,14 +4,13 @@
 --constraint ../../../requirements/constraints.txt
 --constraint ./constraints.txt
 
-
-aio-pika
 aiodebug
 aiofiles
+aio-pika
 arrow # date/time
+redis
 pydantic
 pyinstrument
 pyyaml
-redis
 tenacity
 tqdm

services/agent/src/simcore_service_agent/core/settings.py

@@ -2,7 +2,6 @@
 
 from models_library.basic_types import BootModeEnum, LogLevel
 from pydantic import Field, NonNegativeInt, validator
-from servicelib.file_constants import AGENT_FILE_NAME, HIDDEN_FILE_NAME
 from settings_library.base import BaseCustomSettings
 from settings_library.r_clone import S3Provider
 from settings_library.utils_logging import MixinLoggingSettings
@@ -33,7 +32,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings):
         5, description="parallel transfers to s3"
     )
     AGENT_VOLUMES_CLEANUP_EXCLUDE_FILES: list[str] = Field(
-        [AGENT_FILE_NAME, HIDDEN_FILE_NAME, "key_values.json"],
+        [".hidden_do_not_remove", "key_values.json"],
         description="Files to ignore when syncing to s3",
     )
     AGENT_VOLUMES_CLEANUP_INTERVAL_S: NonNegativeInt = Field(

services/agent/tests/unit/test_modules_volumes_cleanup_s3.py

@@ -11,11 +11,6 @@
 from aiodocker.volumes import DockerVolume
 from pydantic import HttpUrl
 from pytest import LogCaptureFixture
-from servicelib.file_constants import (
-    AGENT_FILE_NAME,
-    HIDDEN_FILE_NAME,
-    KEY_VALUE_FILE_NAME,
-)
 from simcore_service_agent.core.settings import ApplicationSettings
 from simcore_service_agent.modules.volumes_cleanup._s3 import (
     S3Provider,
@@ -80,9 +75,8 @@ async def _download_files_from_bucket(
 
 def _create_data(folder: Path) -> None:
     for file in {  # pylint:disable=use-sequence-for-iteration
-        AGENT_FILE_NAME,
-        HIDDEN_FILE_NAME,
-        KEY_VALUE_FILE_NAME,
+        ".hidden_do_not_remove",
+        "key_values.json",
         "f1.txt",
         "f2.txt",
         "f3.txt",

services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_services.py

@@ -142,7 +142,6 @@ async def create_dynamic_service(
             request_dns=x_dynamic_sidecar_request_dns,
             request_scheme=x_dynamic_sidecar_request_scheme,
             request_simcore_user_agent=x_simcore_user_agent,
-            can_save=service.can_save,
         )
 
     return cast(DynamicServiceGet, await scheduler.get_stack_status(service.node_uuid))

services/director-v2/src/simcore_service_director_v2/models/domains/dynamic_services.py

@@ -38,9 +38,6 @@ class DynamicServiceCreate(ServiceDetails):
     service_resources: ServiceResourcesDict
 
     product_name: str = Field(..., description="Current product name")
-    can_save: bool = Field(
-        ..., description="the service data must be saved when closing"
-    )
 
     class Config:
         schema_extra = {
@@ -52,7 +49,6 @@ class Config:
                 "node_uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa",
                 "basepath": "/x/75c7f3f4-18f9-4678-8610-54a2ade78eaa",
                 "product_name": "osparc",
-                "can_save": True,
                 "service_resources": ServiceResourcesDictHelpers.Config.schema_extra[
                     "examples"
                 ][0],

services/director-v2/src/simcore_service_director_v2/models/schemas/dynamic_services/scheduler.py

@@ -438,13 +438,12 @@ def from_http_request(
         request_dns: str,
         request_scheme: str,
         request_simcore_user_agent: str,
-        can_save: bool,
         run_id: UUID | None = None,
     ) -> "SchedulerData":
         # This constructor method sets current product
         names_helper = DynamicSidecarNamesHelper.make(service.node_uuid)
 
-        obj_dict: dict[str, Any] = dict(
+        obj_dict = dict(
             service_name=names_helper.service_name_dynamic_sidecar,
             hostname=names_helper.service_name_dynamic_sidecar,
             port=port,
@@ -465,7 +464,7 @@ def from_http_request(
             request_scheme=request_scheme,
             proxy_service_name=names_helper.proxy_service_name,
             request_simcore_user_agent=request_simcore_user_agent,
-            dynamic_sidecar={"service_removal_state": {"can_save": can_save}},
+            dynamic_sidecar={},
         )
         if run_id:
             obj_dict["run_id"] = run_id

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py

@@ -8,7 +8,6 @@
 from models_library.projects import ProjectID
 from models_library.projects_networks import DockerNetworkAlias
 from models_library.projects_nodes_io import NodeID
-from models_library.volumes import VolumeCategory
 from pydantic import AnyHttpUrl, PositiveFloat
 from servicelib.fastapi.long_running_tasks.client import (
     Client,
@@ -20,7 +19,6 @@
 )
 from servicelib.logging_utils import log_context
 from servicelib.utils import logged_gather
-from servicelib.volumes_utils import VolumeStatus
 from simcore_service_director_v2.core.settings import DynamicSidecarSettings
 
 from ....models.schemas.dynamic_services import SchedulerData
@@ -403,19 +401,6 @@ async def restart_containers(self, dynamic_sidecar_endpoint: AnyHttpUrl) -> None
             _debug_progress_callback,
         )
 
-    @log_decorator(logger=logger)
-    async def update_volume_state(
-        self,
-        dynamic_sidecar_endpoint: AnyHttpUrl,
-        volume_category: VolumeCategory,
-        volume_status: VolumeStatus,
-    ) -> None:
-        await self._thin_client.put_volumes(
-            dynamic_sidecar_endpoint,
-            volume_category=volume_category,
-            volume_status=volume_status,
-        )
-
 
 async def setup(app: FastAPI) -> None:
     with log_context(logger, logging.DEBUG, "dynamic-sidecar api client setup"):

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_thin.py

@@ -4,10 +4,8 @@
 
 from fastapi import FastAPI, status
 from httpx import Response, Timeout
-from models_library.volumes import VolumeCategory
 from pydantic import AnyHttpUrl
 from servicelib.docker_constants import SUFFIX_EGRESS_PROXY_NAME
-from servicelib.volumes_utils import VolumeStatus
 
 from ....core.settings import DynamicSidecarSettings
 from ._base import BaseThinClient, expect_status, retry_on_errors
@@ -227,15 +225,3 @@ async def post_containers_tasks_restart(
     ) -> Response:
         url = self._get_url(dynamic_sidecar_endpoint, "/containers:restart")
         return await self.client.post(url)
-
-    @retry_on_errors
-    @expect_status(status.HTTP_204_NO_CONTENT)
-    async def put_volumes(
-        self,
-        dynamic_sidecar_endpoint: AnyHttpUrl,
-        volume_category: VolumeCategory,
-        volume_status: VolumeStatus,
-    ) -> Response:
-        url = self._get_url(dynamic_sidecar_endpoint, f"/volumes/{volume_category}")
-
-        return await self.client.put(url, json={"status": volume_status})

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_abc.py

@@ -79,7 +79,6 @@ async def add_service(
         request_dns: str,
         request_scheme: str,
         request_simcore_user_agent: str,
-        can_save: bool,
     ) -> None:
         """
         Adds a new service.

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py

@@ -12,14 +12,12 @@
 from models_library.rabbitmq_messages import InstrumentationRabbitMessage
 from models_library.service_settings_labels import SimcoreServiceLabels
 from models_library.services import ServiceKeyVersion
-from models_library.volumes import VolumeCategory
 from servicelib.fastapi.long_running_tasks.client import (
     ProgressCallback,
     TaskClientResultError,
 )
 from servicelib.fastapi.long_running_tasks.server import TaskProgress
 from servicelib.utils import logged_gather
-from servicelib.volumes_utils import VolumeStatus
 from simcore_postgres_database.models.comp_tasks import NodeClass
 from tenacity import TryAgain
 from tenacity._asyncio import AsyncRetrying
@@ -139,11 +137,6 @@ async def service_save_state(
     await dynamic_sidecar_client.save_service_state(
         scheduler_data.endpoint, progress_callback=progress_callback
     )
-    await dynamic_sidecar_client.update_volume_state(
-        scheduler_data.endpoint,
-        volume_category=VolumeCategory.STATES,
-        volume_status=VolumeStatus.CONTENT_WAS_SAVED,
-    )
 
 
 async def service_push_outputs(
@@ -156,11 +149,6 @@ async def service_push_outputs(
     await dynamic_sidecar_client.push_service_output_ports(
         scheduler_data.endpoint, progress_callback=progress_callback
     )
-    await dynamic_sidecar_client.update_volume_state(
-        scheduler_data.endpoint,
-        volume_category=VolumeCategory.OUTPUTS,
-        volume_status=VolumeStatus.CONTENT_WAS_SAVED,
-    )
 
 
 async def service_remove_sidecar_proxy_docker_networks_and_volumes(
@@ -420,32 +408,6 @@ async def prepare_services_environment(
     dynamic_sidecar_client = get_dynamic_sidecar_client(app, scheduler_data.node_uuid)
     dynamic_sidecar_endpoint = scheduler_data.endpoint
 
-    # update if volume requires saving
-    def _get_state_params(can_save: bool | None) -> dict[str, VolumeStatus]:
-        return (
-            {"volume_status": VolumeStatus.CONTENT_WAS_SAVED}
-            if can_save
-            else {"volume_status": VolumeStatus.CONTENT_NO_SAVE_REQUIRED}
-        )
-
-    update_volume_state_params = _get_state_params(
-        scheduler_data.dynamic_sidecar.service_removal_state.can_save
-    )
-    await logged_gather(
-        *(
-            dynamic_sidecar_client.update_volume_state(
-                scheduler_data.endpoint,
-                volume_category=VolumeCategory.STATES,
-                **update_volume_state_params,
-            ),
-            dynamic_sidecar_client.update_volume_state(
-                scheduler_data.endpoint,
-                volume_category=VolumeCategory.OUTPUTS,
-                **update_volume_state_params,
-            ),
-        )
-    )
-
     async def _pull_outputs_and_state():
         tasks = [
             dynamic_sidecar_client.pull_service_output_ports(dynamic_sidecar_endpoint)

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py

@@ -149,7 +149,6 @@ async def add_service(
         request_dns: str,
         request_scheme: str,
         request_simcore_user_agent: str,
-        can_save: bool,
     ) -> None:
         """Invoked before the service is started"""
         scheduler_data = SchedulerData.from_http_request(
@@ -159,7 +158,6 @@ async def add_service(
             request_dns=request_dns,
             request_scheme=request_scheme,
             request_simcore_user_agent=request_simcore_user_agent,
-            can_save=can_save,
         )
         await self._add_service(scheduler_data)