Merge branch 'main' into expired-token

scripts/monitor/rano_monitor/assets/shared.tcss

@@ -1,3 +1,11 @@
 .warning {
     border: tall $warning;
 }
+
+.tumor-status {
+    color: $success;
+}
+
+.brain-status {
+    color: $warning;
+}

scripts/monitor/rano_monitor/assets/tarball-browser.tcss

@@ -16,14 +16,6 @@ Button {
     background: $accent;
 }
 
-.tumor-status {
-    color: $success;
-}
-
-.brain-status {
-    color: $warning;
-}
-
 #package-btn {
     min-width: 50%;
 }

scripts/monitor/rano_monitor/dataset_browser.py

@@ -5,6 +5,7 @@
 import yaml
 from rano_monitor.messages import InvalidSubjectsUpdated
 from rano_monitor.messages import ReportUpdated
+from rano_monitor.messages import AnnotationsLoaded
 from rano_monitor.utils import generate_full_report
 from rano_monitor.widgets.subject_details import SubjectDetails
 from rano_monitor.widgets.subject_list_view import SubjectListView
@@ -19,6 +20,7 @@
     Header,
     ListView,
     Static,
+    Input,
 )
 
 
@@ -31,6 +33,7 @@ class DatasetBrowser(App):
         Binding("y", "respond('y')", "Yes", show=False),
         Binding("n", "respond('n')", "No", show=False),
     ]
+    AUTO_FOCUS = ""  # Don't focus automatically to search bar
 
     subjects = var([])
     report = reactive({})
@@ -64,6 +67,7 @@ def compose(self) -> ComposeResult:
         yield Header()
         with Container():
             with Container(id="list-container"):
+                yield Input(placeholder="Search", id="subjects-search")
                 yield SubjectListView(id="subjects-list")
             with VerticalScroll():
                 yield Summary(id="summary")
@@ -107,6 +111,10 @@ def on_mount(self):
         subject_details.set_invalid_path(self.invalid_path)
         subject_details.review_cmd = self.review_cmd
 
+        # Set dataset path to listview
+        listview = self.query_one("#subjects-list", ListView)
+        listview.dset_path = self.dset_data_path
+
         # Execute handlers
         self.prompt_watchdog.manual_execute()
         self.invalid_watchdog.manual_execute()
@@ -146,6 +154,17 @@ def on_button_pressed(self, event: Button.Pressed) -> None:
         elif event.control == n_button:
             self.action_respond("n")
 
+    def on_input_changed(self, event: Input.Changed) -> None:
+        search_input = self.query_one("#subjects-search")
+        subjects_list = self.query_one("#subjects-list")
+        if event.control == search_input:
+            search_term = search_input.value
+            subjects_list.update_list(search_term)
+
+    def on_annotations_loaded(self, message: AnnotationsLoaded):
+        subjects_list = self.query_one("#subjects-list")
+        subjects_list.update_list()
+
     def update_prompt(self, prompt: str):
         self.prompt = prompt
         show_prompt = bool(len(prompt))

scripts/monitor/rano_monitor/messages/__init__.py

@@ -1,4 +1,5 @@
 from .invalid_subject_updated import InvalidSubjectsUpdated
 from .report_updated import ReportUpdated
+from .annotations_loaded import AnnotationsLoaded
 
-__all__ = [InvalidSubjectsUpdated, ReportUpdated]
+__all__ = [InvalidSubjectsUpdated, ReportUpdated, AnnotationsLoaded]

scripts/monitor/rano_monitor/messages/annotations_loaded.py

@@ -0,0 +1,5 @@
+from textual.message import Message
+
+
+class AnnotationsLoaded(Message):
+    pass

scripts/monitor/rano_monitor/utils.py

@@ -375,12 +375,6 @@ def unpackage_reviews(file, app, dset_data_path):
         identified_masks
     )
 
-    if len(identified_reviewed):
-        app.notify("Reviewed cases identified")
-
-    if len(identified_brainmasks):
-        app.notify("Brain masks identified")
-
     extracts = get_identified_extract_paths(
         identified_reviewed,
         identified_under_review,
@@ -398,3 +392,58 @@ def unpackage_reviews(file, app, dset_data_path):
             if os.path.exists(target_file):
                 delete(target_file, dset_data_path)
             tar.extract(member, dest)
+
+
+def brain_has_been_reviewed(brainpath, backup_brainpath):
+    if not os.path.exists(backup_brainpath):
+        return False
+
+    brain_hash = get_hash(brainpath)
+    backup_hash = get_hash(backup_brainpath)
+    return brain_hash != backup_hash
+
+
+def tumor_has_been_finalized(finalized_tumor_path):
+    finalized_files = os.listdir(finalized_tumor_path)
+    finalized_files = [file for file in finalized_files if not file.startswith(".")]
+
+    return len(finalized_files) > 0
+
+
+def can_review(subject):
+    return MANUAL_REVIEW_STAGE <= abs(subject["status"]) < DONE_STAGE
+
+
+def get_finalized_tumor_path(subject: str, dset_path: str) -> str:
+    """Get's the path to the finalized tumor path based solely on the
+    subject identifier and data path. Works regardless of wether the subject is in
+    that stage or the folder being pointed to exists or not.
+
+    Args:
+        subject (str): subject identified, written as {subject}|{timepoint}
+
+    Returns:
+        str: _description_
+    """
+    id, tp = subject.split("|")
+    return os.path.join(
+        dset_path,
+        "tumor_extracted",
+        "DataForQC",
+        id,
+        tp,
+        "TumorMasksForQC",
+        "finalized",
+    )
+
+
+def get_brainmask_path(subject: str, dset_path: str) -> str:
+    id, tp = subject.split("|")
+    return os.path.join(
+        dset_path,
+        "tumor_extracted",
+        "DataForQC",
+        id,
+        tp,
+        BRAINMASK,
+    )

scripts/monitor/rano_monitor/widgets/subject_details.py

@@ -3,7 +3,6 @@
 import pandas as pd
 from rano_monitor.constants import (
     DEFAULT_SEGMENTATION,
-    DONE_STAGE,
     MANUAL_REVIEW_STAGE,
 )
 from rano_monitor.messages import InvalidSubjectsUpdated
@@ -14,6 +13,7 @@
     review_brain,
     review_tumor,
     to_local_path,
+    can_review,
 )
 from rano_monitor.widgets.copyable_item import CopyableItem
 from textual.app import ComposeResult
@@ -116,8 +116,7 @@ def update_subject(self):
         # This SHOULD NOT be here for general data prep monitoring.
         # Additional configuration must be set
         # to make this kind of features generic
-        can_review = MANUAL_REVIEW_STAGE <= abs(subject["status"]) < DONE_STAGE
-        buttons_container.display = "block" if can_review else "none"
+        buttons_container.display = "block" if can_review(subject) else "none"
 
         # Only display finalize button for the manual review
         can_finalize = abs(subject["status"]) == MANUAL_REVIEW_STAGE

scripts/monitor/rano_monitor/widgets/subject_list_view.py

@@ -1,13 +1,21 @@
+import os
 import pandas as pd
 from rano_monitor.messages import InvalidSubjectsUpdated
 from rano_monitor.messages.report_updated import ReportUpdated
 from textual.widgets import Label, ListItem, ListView
+from rano_monitor.utils import (
+    get_hash,
+    tumor_has_been_finalized,
+    get_finalized_tumor_path,
+    get_brainmask_path,
+)
 
 
 class SubjectListView(ListView):
     report = {}
     highlight = set()
     invalid_subjects = set()
+    dset_path = ""
 
     def on_report_updated(self, message: ReportUpdated) -> None:
         self.report = message.report
@@ -16,14 +24,11 @@ def on_report_updated(self, message: ReportUpdated) -> None:
         if len(self.report) > 0:
             self.update_list()
 
-    def on_invalid_subjects_updated(
-            self,
-            message: InvalidSubjectsUpdated
-    ) -> None:
+    def on_invalid_subjects_updated(self, message: InvalidSubjectsUpdated) -> None:
         self.invalid_subjects = message.invalid_subjects
         self.update_list()
 
-    def update_list(self):
+    def update_list(self, search_term=""):
         # Check for content differences with old report
         # apply alert class to listitem
         report = self.report
@@ -40,12 +45,40 @@ def update_list(self):
                 status = status.capitalize().replace("_", " ")
                 if subject in self.invalid_subjects:
                     status = "Invalidated"
-                widget = ListItem(
+
+                list_contents = [
                     Label(subject),
                     Label(status, classes="subtitle"),
-                )
+                ]
+
+                tumor_path = get_finalized_tumor_path(subject, self.dset_path)
+                if os.path.exists(tumor_path) and tumor_has_been_finalized(tumor_path):
+                    list_contents.append(
+                        Label("Tumor finalized", classes="tumor-status")
+                    )
+
+                brain_path = get_brainmask_path(subject, self.dset_path)
+                exp_hash = report_df.loc[subject]["brain_mask_hash"]
+                if os.path.exists(brain_path) and get_hash(brain_path) != exp_hash:
+                    list_contents.append(
+                        Label("Brain Mask Modified", classes="brain-status")
+                    )
+
+                widget = ListItem(*list_contents)
+
             if subject in self.highlight:
                 widget.set_class(True, "highlight")
+
+            should_display = True
+            if search_term != "":
+                should_display = (
+                    subject == "SUMMARY"
+                    or search_term.lower() in subject.lower()
+                    or search_term.lower() in status.lower()
+                )
+
+            if not should_display:
+                continue
             widgets.append(widget)
 
         current_idx = self.index

scripts/monitor/rano_monitor/widgets/summary.py

@@ -3,6 +3,7 @@
 from rano_monitor.constants import REVIEW_FILENAME, REVIEWED_FILENAME
 from rano_monitor.messages import InvalidSubjectsUpdated
 from rano_monitor.messages import ReportUpdated
+from rano_monitor.messages import AnnotationsLoaded
 from rano_monitor.utils import package_review_cases, unpackage_reviews
 from textual.app import ComposeResult
 from textual.containers import Center
@@ -23,6 +24,11 @@ class Summary(Static):
 
     def compose(self) -> ComposeResult:
         yield Static("Report Status")
+        yield Static(
+            "HINT: To move forward with processing and finalized annotations, ensure the preparation pipeline is running.",
+            id="hint-msg",
+            classes="warning",
+        )
         yield Center(id="summary-content")
         with Center(id="package-btns"):
             yield Button(
@@ -64,7 +70,10 @@ def update_summary(self):
 
         widgets = []
         for name, val in status_percents.items():
-            wname = Label(name.capitalize().replace("_", " "))
+            count = status_counts[name] if name in status_counts else 0
+            wname = Label(
+                f'{name.capitalize().replace("_", " ")} ({count}/{len(report_df)})'
+            )
             wpbar = ProgressBar(total=1, show_eta=False)
             wpbar.advance(val)
             widget = Center(wname, wpbar, classes="pbar")
@@ -77,17 +86,39 @@ def update_summary(self):
 
         content.mount(*widgets)
 
-    def on_button_pressed(self, event: Button.Pressed) -> None:
+    async def _package_review_cases(self):
+        pkg_btn = self.query_one("#package-btn", Button)
+        label = pkg_btn.label
+        pkg_btn.disabled = True
+        pkg_btn.label = "Creating package..."
+        self.notify("Packaging review cases. This may take a while")
+        package_review_cases(self.report, self.dset_path)
+        self.notify(f"{REVIEW_FILENAME} was created on the working directory")
+        pkg_btn.label = label
+        pkg_btn.disabled = False
+
+    async def _unpackage_reviews(self):
+        unpkg_btn = self.query_one("#unpackage-btn", Button)
+        label = unpkg_btn.label
+        unpkg_btn.disabled = True
+        unpkg_btn.label = "Loading annotations..."
+        self.notify("Loading annotations. This may take a while")
+        unpackage_reviews(REVIEWED_FILENAME, self, self.dset_path)
+        self.notify("Annotations have been loaded")
+        unpkg_btn.label = label
+        unpkg_btn.disabled = False
+        self.post_message(AnnotationsLoaded())
+
+    async def on_button_pressed(self, event: Button.Pressed) -> None:
         event.stop()
         pkg_btn = self.query_one("#package-btn", Button)
         unpkg_btn = self.query_one("#unpackage-btn", Button)
 
         if event.control == pkg_btn:
-            package_review_cases(self.report, self.dset_path)
-            self.notify(f"{REVIEW_FILENAME} was created on the working directory")
+            self.run_worker(self._package_review_cases(), exclusive=True, thread=True)
         elif event.control == unpkg_btn:
             if REVIEWED_FILENAME not in os.listdir("."):
                 self.notify(f"{REVIEWED_FILENAME} not found in {os.path.abspath('.')}")
                 return
 
-            unpackage_reviews(REVIEWED_FILENAME, self, self.dset_path)
+            self.run_worker(self._unpackage_reviews(), exclusive=True, thread=True)

scripts/monitor/rano_monitor/widgets/tarball_subject_view.py

@@ -3,10 +3,11 @@
 from rano_monitor.constants import BRAINMASK, BRAINMASK_BAK, DEFAULT_SEGMENTATION
 from rano_monitor.utils import (
     finalize,
-    get_hash,
     is_editor_installed,
     review_brain,
     review_tumor,
+    brain_has_been_reviewed,
+    tumor_has_been_finalized,
 )
 from textual.app import ComposeResult
 from textual.containers import Container, Horizontal
@@ -49,9 +50,15 @@ def on_mount(self):
     def update_status(self):
         tumor_status = self.query_one(".tumor-status", Static)
         brain_status = self.query_one(".brain-status", Static)
-        if self.__tumor_has_been_finalized():
+
+        id, tp = self.subject.split("|")
+        finalized_tumor_path = os.path.join(self.contents_path, id, tp, "finalized")
+        brainpath = os.path.join(self.contents_path, id, tp, BRAINMASK)
+        backup_brainpath = os.path.join(self.contents_path, id, tp, BRAINMASK_BAK)
+
+        if tumor_has_been_finalized(finalized_tumor_path):
             tumor_status.display = "block"
-        if self.__brain_has_been_reviewed():
+        if brain_has_been_reviewed(brainpath, backup_brainpath):
             brain_status.display = "block"
 
     def __update_buttons(self):

scripts/monitor/setup.py

@@ -9,7 +9,7 @@
 
 setup(
     name="rano-monitor",
-    version="0.0.1",
+    version="0.0.2",
     description="TUI for monitoring medperf datasets",
     url="https://github.com/mlcommons/medperf",
     author="MLCommons",