diff --git a/scripts/monitor/rano_monitor/assets/shared.tcss b/scripts/monitor/rano_monitor/assets/shared.tcss index 8bc76bd06..c0b25b565 100644 --- a/scripts/monitor/rano_monitor/assets/shared.tcss +++ b/scripts/monitor/rano_monitor/assets/shared.tcss @@ -1,3 +1,11 @@ .warning { border: tall $warning; } + +.tumor-status { + color: $success; +} + +.brain-status { + color: $warning; +} diff --git a/scripts/monitor/rano_monitor/assets/tarball-browser.tcss b/scripts/monitor/rano_monitor/assets/tarball-browser.tcss index 55b21a827..709698a7f 100644 --- a/scripts/monitor/rano_monitor/assets/tarball-browser.tcss +++ b/scripts/monitor/rano_monitor/assets/tarball-browser.tcss @@ -16,14 +16,6 @@ Button { background: $accent; } -.tumor-status { - color: $success; -} - -.brain-status { - color: $warning; -} - #package-btn { min-width: 50%; } diff --git a/scripts/monitor/rano_monitor/dataset_browser.py b/scripts/monitor/rano_monitor/dataset_browser.py index 21bbf8741..47bd12b54 100644 --- a/scripts/monitor/rano_monitor/dataset_browser.py +++ b/scripts/monitor/rano_monitor/dataset_browser.py @@ -5,6 +5,7 @@ import yaml from rano_monitor.messages import InvalidSubjectsUpdated from rano_monitor.messages import ReportUpdated +from rano_monitor.messages import AnnotationsLoaded from rano_monitor.utils import generate_full_report from rano_monitor.widgets.subject_details import SubjectDetails from rano_monitor.widgets.subject_list_view import SubjectListView @@ -19,6 +20,7 @@ Header, ListView, Static, + Input, ) @@ -31,6 +33,7 @@ class DatasetBrowser(App): Binding("y", "respond('y')", "Yes", show=False), Binding("n", "respond('n')", "No", show=False), ] + AUTO_FOCUS = "" # Don't focus automatically to search bar subjects = var([]) report = reactive({}) @@ -64,6 +67,7 @@ def compose(self) -> ComposeResult: yield Header() with Container(): with Container(id="list-container"): + yield Input(placeholder="Search", id="subjects-search") yield SubjectListView(id="subjects-list") with VerticalScroll(): yield Summary(id="summary") @@ -107,6 +111,10 @@ def on_mount(self): subject_details.set_invalid_path(self.invalid_path) subject_details.review_cmd = self.review_cmd + # Set dataset path to listview + listview = self.query_one("#subjects-list", ListView) + listview.dset_path = self.dset_data_path + # Execute handlers self.prompt_watchdog.manual_execute() self.invalid_watchdog.manual_execute() @@ -146,6 +154,17 @@ def on_button_pressed(self, event: Button.Pressed) -> None: elif event.control == n_button: self.action_respond("n") + def on_input_changed(self, event: Input.Changed) -> None: + search_input = self.query_one("#subjects-search") + subjects_list = self.query_one("#subjects-list") + if event.control == search_input: + search_term = search_input.value + subjects_list.update_list(search_term) + + def on_annotations_loaded(self, message: AnnotationsLoaded): + subjects_list = self.query_one("#subjects-list") + subjects_list.update_list() + def update_prompt(self, prompt: str): self.prompt = prompt show_prompt = bool(len(prompt)) diff --git a/scripts/monitor/rano_monitor/messages/__init__.py b/scripts/monitor/rano_monitor/messages/__init__.py index 2abb1e07f..5d88cbf2d 100644 --- a/scripts/monitor/rano_monitor/messages/__init__.py +++ b/scripts/monitor/rano_monitor/messages/__init__.py @@ -1,4 +1,5 @@ from .invalid_subject_updated import InvalidSubjectsUpdated from .report_updated import ReportUpdated +from .annotations_loaded import AnnotationsLoaded -__all__ = [InvalidSubjectsUpdated, ReportUpdated] +__all__ = [InvalidSubjectsUpdated, ReportUpdated, AnnotationsLoaded] diff --git a/scripts/monitor/rano_monitor/messages/annotations_loaded.py b/scripts/monitor/rano_monitor/messages/annotations_loaded.py new file mode 100644 index 000000000..e5d080f6d --- /dev/null +++ b/scripts/monitor/rano_monitor/messages/annotations_loaded.py @@ -0,0 +1,5 @@ +from textual.message import Message + + +class AnnotationsLoaded(Message): + pass diff --git a/scripts/monitor/rano_monitor/utils.py b/scripts/monitor/rano_monitor/utils.py index fac4b552a..1c61ea97f 100644 --- a/scripts/monitor/rano_monitor/utils.py +++ b/scripts/monitor/rano_monitor/utils.py @@ -375,12 +375,6 @@ def unpackage_reviews(file, app, dset_data_path): identified_masks ) - if len(identified_reviewed): - app.notify("Reviewed cases identified") - - if len(identified_brainmasks): - app.notify("Brain masks identified") - extracts = get_identified_extract_paths( identified_reviewed, identified_under_review, @@ -398,3 +392,58 @@ def unpackage_reviews(file, app, dset_data_path): if os.path.exists(target_file): delete(target_file, dset_data_path) tar.extract(member, dest) + + +def brain_has_been_reviewed(brainpath, backup_brainpath): + if not os.path.exists(backup_brainpath): + return False + + brain_hash = get_hash(brainpath) + backup_hash = get_hash(backup_brainpath) + return brain_hash != backup_hash + + +def tumor_has_been_finalized(finalized_tumor_path): + finalized_files = os.listdir(finalized_tumor_path) + finalized_files = [file for file in finalized_files if not file.startswith(".")] + + return len(finalized_files) > 0 + + +def can_review(subject): + return MANUAL_REVIEW_STAGE <= abs(subject["status"]) < DONE_STAGE + + +def get_finalized_tumor_path(subject: str, dset_path: str) -> str: + """Get's the path to the finalized tumor path based solely on the + subject identifier and data path. Works regardless of wether the subject is in + that stage or the folder being pointed to exists or not. + + Args: + subject (str): subject identified, written as {subject}|{timepoint} + + Returns: + str: _description_ + """ + id, tp = subject.split("|") + return os.path.join( + dset_path, + "tumor_extracted", + "DataForQC", + id, + tp, + "TumorMasksForQC", + "finalized", + ) + + +def get_brainmask_path(subject: str, dset_path: str) -> str: + id, tp = subject.split("|") + return os.path.join( + dset_path, + "tumor_extracted", + "DataForQC", + id, + tp, + BRAINMASK, + ) diff --git a/scripts/monitor/rano_monitor/widgets/subject_details.py b/scripts/monitor/rano_monitor/widgets/subject_details.py index ab0467d80..eea9b2695 100644 --- a/scripts/monitor/rano_monitor/widgets/subject_details.py +++ b/scripts/monitor/rano_monitor/widgets/subject_details.py @@ -3,7 +3,6 @@ import pandas as pd from rano_monitor.constants import ( DEFAULT_SEGMENTATION, - DONE_STAGE, MANUAL_REVIEW_STAGE, ) from rano_monitor.messages import InvalidSubjectsUpdated @@ -14,6 +13,7 @@ review_brain, review_tumor, to_local_path, + can_review, ) from rano_monitor.widgets.copyable_item import CopyableItem from textual.app import ComposeResult @@ -116,8 +116,7 @@ def update_subject(self): # This SHOULD NOT be here for general data prep monitoring. # Additional configuration must be set # to make this kind of features generic - can_review = MANUAL_REVIEW_STAGE <= abs(subject["status"]) < DONE_STAGE - buttons_container.display = "block" if can_review else "none" + buttons_container.display = "block" if can_review(subject) else "none" # Only display finalize button for the manual review can_finalize = abs(subject["status"]) == MANUAL_REVIEW_STAGE diff --git a/scripts/monitor/rano_monitor/widgets/subject_list_view.py b/scripts/monitor/rano_monitor/widgets/subject_list_view.py index 57e599808..2b8759c48 100644 --- a/scripts/monitor/rano_monitor/widgets/subject_list_view.py +++ b/scripts/monitor/rano_monitor/widgets/subject_list_view.py @@ -1,13 +1,21 @@ +import os import pandas as pd from rano_monitor.messages import InvalidSubjectsUpdated from rano_monitor.messages.report_updated import ReportUpdated from textual.widgets import Label, ListItem, ListView +from rano_monitor.utils import ( + get_hash, + tumor_has_been_finalized, + get_finalized_tumor_path, + get_brainmask_path, +) class SubjectListView(ListView): report = {} highlight = set() invalid_subjects = set() + dset_path = "" def on_report_updated(self, message: ReportUpdated) -> None: self.report = message.report @@ -16,14 +24,11 @@ def on_report_updated(self, message: ReportUpdated) -> None: if len(self.report) > 0: self.update_list() - def on_invalid_subjects_updated( - self, - message: InvalidSubjectsUpdated - ) -> None: + def on_invalid_subjects_updated(self, message: InvalidSubjectsUpdated) -> None: self.invalid_subjects = message.invalid_subjects self.update_list() - def update_list(self): + def update_list(self, search_term=""): # Check for content differences with old report # apply alert class to listitem report = self.report @@ -40,12 +45,40 @@ def update_list(self): status = status.capitalize().replace("_", " ") if subject in self.invalid_subjects: status = "Invalidated" - widget = ListItem( + + list_contents = [ Label(subject), Label(status, classes="subtitle"), - ) + ] + + tumor_path = get_finalized_tumor_path(subject, self.dset_path) + if os.path.exists(tumor_path) and tumor_has_been_finalized(tumor_path): + list_contents.append( + Label("Tumor finalized", classes="tumor-status") + ) + + brain_path = get_brainmask_path(subject, self.dset_path) + exp_hash = report_df.loc[subject]["brain_mask_hash"] + if os.path.exists(brain_path) and get_hash(brain_path) != exp_hash: + list_contents.append( + Label("Brain Mask Modified", classes="brain-status") + ) + + widget = ListItem(*list_contents) + if subject in self.highlight: widget.set_class(True, "highlight") + + should_display = True + if search_term != "": + should_display = ( + subject == "SUMMARY" + or search_term.lower() in subject.lower() + or search_term.lower() in status.lower() + ) + + if not should_display: + continue widgets.append(widget) current_idx = self.index diff --git a/scripts/monitor/rano_monitor/widgets/summary.py b/scripts/monitor/rano_monitor/widgets/summary.py index 9f3f2e7b1..570b0088f 100644 --- a/scripts/monitor/rano_monitor/widgets/summary.py +++ b/scripts/monitor/rano_monitor/widgets/summary.py @@ -3,6 +3,7 @@ from rano_monitor.constants import REVIEW_FILENAME, REVIEWED_FILENAME from rano_monitor.messages import InvalidSubjectsUpdated from rano_monitor.messages import ReportUpdated +from rano_monitor.messages import AnnotationsLoaded from rano_monitor.utils import package_review_cases, unpackage_reviews from textual.app import ComposeResult from textual.containers import Center @@ -23,6 +24,11 @@ class Summary(Static): def compose(self) -> ComposeResult: yield Static("Report Status") + yield Static( + "HINT: To move forward with processing and finalized annotations, ensure the preparation pipeline is running.", + id="hint-msg", + classes="warning", + ) yield Center(id="summary-content") with Center(id="package-btns"): yield Button( @@ -64,7 +70,10 @@ def update_summary(self): widgets = [] for name, val in status_percents.items(): - wname = Label(name.capitalize().replace("_", " ")) + count = status_counts[name] if name in status_counts else 0 + wname = Label( + f'{name.capitalize().replace("_", " ")} ({count}/{len(report_df)})' + ) wpbar = ProgressBar(total=1, show_eta=False) wpbar.advance(val) widget = Center(wname, wpbar, classes="pbar") @@ -77,17 +86,39 @@ def update_summary(self): content.mount(*widgets) - def on_button_pressed(self, event: Button.Pressed) -> None: + async def _package_review_cases(self): + pkg_btn = self.query_one("#package-btn", Button) + label = pkg_btn.label + pkg_btn.disabled = True + pkg_btn.label = "Creating package..." + self.notify("Packaging review cases. This may take a while") + package_review_cases(self.report, self.dset_path) + self.notify(f"{REVIEW_FILENAME} was created on the working directory") + pkg_btn.label = label + pkg_btn.disabled = False + + async def _unpackage_reviews(self): + unpkg_btn = self.query_one("#unpackage-btn", Button) + label = unpkg_btn.label + unpkg_btn.disabled = True + unpkg_btn.label = "Loading annotations..." + self.notify("Loading annotations. This may take a while") + unpackage_reviews(REVIEWED_FILENAME, self, self.dset_path) + self.notify("Annotations have been loaded") + unpkg_btn.label = label + unpkg_btn.disabled = False + self.post_message(AnnotationsLoaded()) + + async def on_button_pressed(self, event: Button.Pressed) -> None: event.stop() pkg_btn = self.query_one("#package-btn", Button) unpkg_btn = self.query_one("#unpackage-btn", Button) if event.control == pkg_btn: - package_review_cases(self.report, self.dset_path) - self.notify(f"{REVIEW_FILENAME} was created on the working directory") + self.run_worker(self._package_review_cases(), exclusive=True, thread=True) elif event.control == unpkg_btn: if REVIEWED_FILENAME not in os.listdir("."): self.notify(f"{REVIEWED_FILENAME} not found in {os.path.abspath('.')}") return - unpackage_reviews(REVIEWED_FILENAME, self, self.dset_path) + self.run_worker(self._unpackage_reviews(), exclusive=True, thread=True) diff --git a/scripts/monitor/rano_monitor/widgets/tarball_subject_view.py b/scripts/monitor/rano_monitor/widgets/tarball_subject_view.py index 50fcafc20..d61f8cf5f 100644 --- a/scripts/monitor/rano_monitor/widgets/tarball_subject_view.py +++ b/scripts/monitor/rano_monitor/widgets/tarball_subject_view.py @@ -3,10 +3,11 @@ from rano_monitor.constants import BRAINMASK, BRAINMASK_BAK, DEFAULT_SEGMENTATION from rano_monitor.utils import ( finalize, - get_hash, is_editor_installed, review_brain, review_tumor, + brain_has_been_reviewed, + tumor_has_been_finalized, ) from textual.app import ComposeResult from textual.containers import Container, Horizontal @@ -49,9 +50,15 @@ def on_mount(self): def update_status(self): tumor_status = self.query_one(".tumor-status", Static) brain_status = self.query_one(".brain-status", Static) - if self.__tumor_has_been_finalized(): + + id, tp = self.subject.split("|") + finalized_tumor_path = os.path.join(self.contents_path, id, tp, "finalized") + brainpath = os.path.join(self.contents_path, id, tp, BRAINMASK) + backup_brainpath = os.path.join(self.contents_path, id, tp, BRAINMASK_BAK) + + if tumor_has_been_finalized(finalized_tumor_path): tumor_status.display = "block" - if self.__brain_has_been_reviewed(): + if brain_has_been_reviewed(brainpath, backup_brainpath): brain_status.display = "block" def __update_buttons(self): diff --git a/scripts/monitor/setup.py b/scripts/monitor/setup.py index a70cc176f..8c31a159d 100644 --- a/scripts/monitor/setup.py +++ b/scripts/monitor/setup.py @@ -9,7 +9,7 @@ setup( name="rano-monitor", - version="0.0.1", + version="0.0.2", description="TUI for monitoring medperf datasets", url="https://github.com/mlcommons/medperf", author="MLCommons",