🐛 do not send bps metrics where they are less than 1 (#6634)

Co-authored-by: Andrei Neagu <[email protected]>
ITISFoundation · Nov 7, 2024 · e11f17c · e11f17c
1 parent d8f1720
commit e11f17c
Show file tree

Hide file tree

Showing 5 changed files with 32 additions and 21 deletions.
diff --git a/.../src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py b/.../src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py
@@ -173,9 +173,10 @@ async def service_save_state(
         size = await sidecars_client.save_service_state(
             scheduler_data.endpoint, progress_callback=progress_callback
         )
-    get_instrumentation(app).dynamic_sidecar_metrics.push_service_state_rate.labels(
-        **get_metrics_labels(scheduler_data)
-    ).observe(get_rate(size, duration.to_flaot()))
+    if size and size > 0:
+        get_instrumentation(app).dynamic_sidecar_metrics.push_service_state_rate.labels(
+            **get_metrics_labels(scheduler_data)
+        ).observe(get_rate(size, duration.to_float()))
 
     await sidecars_client.update_volume_state(
         scheduler_data.endpoint,
@@ -479,10 +480,14 @@ async def _pull_output_ports_with_metrics() -> None:
             size: int = await sidecars_client.pull_service_output_ports(
                 dynamic_sidecar_endpoint
             )
-
-        get_instrumentation(app).dynamic_sidecar_metrics.output_ports_pull_rate.labels(
-            **get_metrics_labels(scheduler_data)
-        ).observe(get_rate(size, duration.to_flaot()))
+        if size and size > 0:
+            get_instrumentation(
+                app
+            ).dynamic_sidecar_metrics.output_ports_pull_rate.labels(
+                **get_metrics_labels(scheduler_data)
+            ).observe(
+                get_rate(size, duration.to_float())
+            )
 
     async def _pull_user_services_images_with_metrics() -> None:
         with track_duration() as duration:
@@ -493,16 +498,21 @@ async def _pull_user_services_images_with_metrics() -> None:
         ).dynamic_sidecar_metrics.pull_user_services_images_duration.labels(
             **get_metrics_labels(scheduler_data)
         ).observe(
-            duration.to_flaot()
+            duration.to_float()
         )
 
     async def _restore_service_state_with_metrics() -> None:
         with track_duration() as duration:
             size = await sidecars_client.restore_service_state(dynamic_sidecar_endpoint)
 
-        get_instrumentation(app).dynamic_sidecar_metrics.pull_service_state_rate.labels(
-            **get_metrics_labels(scheduler_data)
-        ).observe(get_rate(size, duration.to_flaot()))
+        if size and size > 0:
+            get_instrumentation(
+                app
+            ).dynamic_sidecar_metrics.pull_service_state_rate.labels(
+                **get_metrics_labels(scheduler_data)
+            ).observe(
+                get_rate(size, duration.to_float())
+            )
 
     tasks = [
         _pull_user_services_images_with_metrics(),

diff --git a/...-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py b/...-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py
@@ -464,13 +464,14 @@ async def retrieve_service_inputs(
         )
         duration = time.time() - started
 
-        get_instrumentation(
-            self.app
-        ).dynamic_sidecar_metrics.input_ports_pull_rate.labels(
-            **get_metrics_labels(scheduler_data)
-        ).observe(
-            get_rate(transferred_bytes, duration)
-        )
+        if transferred_bytes and transferred_bytes > 0:
+            get_instrumentation(
+                self.app
+            ).dynamic_sidecar_metrics.input_ports_pull_rate.labels(
+                **get_metrics_labels(scheduler_data)
+            ).observe(
+                get_rate(transferred_bytes, duration)
+            )
 
         if scheduler_data.restart_policy == RestartPolicy.ON_INPUTS_DOWNLOADED:
             logger.info("Will restart containers")

diff --git a/services/director-v2/src/simcore_service_director_v2/modules/instrumentation/_models.py b/services/director-v2/src/simcore_service_director_v2/modules/instrumentation/_models.py
@@ -88,7 +88,7 @@ def __post_init__(self) -> None:
             "time to pull docker images",
             labelnames=_INSTRUMENTATION_LABELS,
             namespace=_METRICS_NAMESPACE,
-            buckets=_RATE_BPS_BUCKETS,
+            buckets=_BUCKETS_TIME_S,
             subsystem=self.subsystem,
             registry=self.registry,
         )

diff --git a/services/director-v2/src/simcore_service_director_v2/modules/instrumentation/_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/instrumentation/_utils.py
@@ -42,7 +42,7 @@ def set_value(self, value):
 
         self._value = float(value)
 
-    def to_flaot(self) -> float:
+    def to_float(self) -> float:
         if not isinstance(self._value, float):
             msg = "Value must be a float or an int."
             raise TypeError(msg)

diff --git a/services/director-v2/tests/unit/test_modules_instrumentation__utils.py b/services/director-v2/tests/unit/test_modules_instrumentation__utils.py
@@ -7,4 +7,4 @@ def test_track_duration():
     with track_duration() as duration:
         time.sleep(0.1)
 
-    assert duration.to_flaot() > 0.1
+    assert duration.to_float() > 0.1