Optimize validation layout updates

changelog.d/20241205_161129_mzhiltso_update_honeypot_selection.md

@@ -0,0 +1,5 @@
+### Changed
+
+- Improved uniformity of validation frames distribution in honeypot tasks and
+  random honeypot rerolls
+  (<https://github.com/cvat-ai/cvat/pull/8776>)

cvat/apps/dataset_manager/bindings.py

@@ -50,7 +50,7 @@ class Attribute(NamedTuple):
         value: Any
 
     @classmethod
-    def add_prefetch_info(cls, queryset: QuerySet):
+    def add_prefetch_info(cls, queryset: QuerySet[Label]) -> QuerySet[Label]:
         assert issubclass(queryset.model, Label)
 
         return add_prefetch_fields(queryset, [

cvat/apps/dataset_manager/task.py

@@ -84,7 +84,7 @@ def merge_table_rows(rows, keys_for_merge, field_id):
 
 class JobAnnotation:
     @classmethod
-    def add_prefetch_info(cls, queryset: QuerySet, prefetch_images: bool = True):
+    def add_prefetch_info(cls, queryset: QuerySet[models.Job], prefetch_images: bool = True) -> QuerySet[models.Job]:
         assert issubclass(queryset.model, models.Job)
 
         label_qs = add_prefetch_fields(models.Label.objects.all(), [

cvat/apps/engine/cache.py

@@ -65,6 +65,7 @@
 from cvat.apps.engine.rq_job_handler import RQJobMetaField
 from cvat.apps.engine.utils import (
     CvatChunkTimestampMismatchError,
+    format_list,
     get_rq_lock_for_job,
     load_image,
     md5_hash,
@@ -275,6 +276,15 @@ def _delete_cache_item(self, key: str):
         except pickle.UnpicklingError:
             slogger.glob.error(f"Failed to remove item from the cache: key {key}", exc_info=True)
 
+    def _bulk_delete_cache_items(self, keys: Sequence[str]):
+        try:
+            self._cache().delete_many(keys)
+            slogger.glob.info(f"Removed chunks from the cache: keys {format_list(keys)}")
+        except pickle.UnpicklingError:
+            slogger.glob.error(
+                f"Failed to remove items from the cache: keys {format_list(keys)}", exc_info=True
+            )
+
     def _get_cache_item(self, key: str) -> Optional[_CacheItem]:
         try:
             item = self._cache().get(key)
@@ -468,6 +478,15 @@ def remove_segment_chunk(
             self._make_chunk_key(db_segment, chunk_number=chunk_number, quality=quality)
         )
 
+    def remove_segment_chunks(self, params: Sequence[dict[str, Any]]) -> None:
+        # TODO: maybe add a more generic version
+        keys_to_remove = []
+        for item_params in params:
+            db_obj = item_params.pop("db_segment")
+            keys_to_remove.append(self._make_chunk_key(db_obj, **item_params))
+
+        self._bulk_delete_cache_items(keys_to_remove)
+
     def get_cloud_preview(self, db_storage: models.CloudStorage) -> Optional[DataWithMime]:
         return self._to_data_with_mime(self._get_cache_item(self._make_preview_key(db_storage)))
 

cvat/apps/engine/quality_control.py

@@ -0,0 +1,47 @@
+# Copyright (C) 2024 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from typing import Generic, Mapping, Sequence, TypeVar
+
+import numpy as np
+
+_T = TypeVar("_T")
+
+
+class HoneypotFrameSelector(Generic[_T]):
+    def __init__(
+        self, validation_frame_counts: Mapping[_T, int], *, rng: np.random.Generator | None = None
+    ):
+        self.validation_frame_counts = validation_frame_counts
+
+        if not rng:
+            rng = np.random.default_rng()
+
+        self.rng = rng
+
+    def select_next_frames(self, count: int) -> Sequence[_T]:
+        # This approach guarantees that:
+        # - every GT frame is used
+        # - GT frames are used uniformly (at most min count + 1)
+        # - GT frames are not repeated in jobs
+        # - honeypot sets are different in jobs
+        # - honeypot sets are random
+        # if possible (if the job and GT counts allow this).
+        pick = []
+
+        for random_number in self.rng.random(count):
+            least_count = min(c for f, c in self.validation_frame_counts.items() if f not in pick)
+            least_used_frames = tuple(
+                f
+                for f, c in self.validation_frame_counts.items()
+                if f not in pick
+                if c == least_count
+            )
+
+            selected_item = int(random_number * len(least_used_frames))
+            selected_frame = least_used_frames[selected_item]
+            pick.append(selected_frame)
+            self.validation_frame_counts[selected_frame] += 1
+
+        return pick