diff --git a/docs/source/gui.rst b/docs/source/gui.rst index 5ff544f..d85e22c 100644 --- a/docs/source/gui.rst +++ b/docs/source/gui.rst @@ -61,6 +61,10 @@ The panel-stack (on the left below) contains data-views such as videos, scatter .. image:: ../media/use_case2.gif :align: center + +.. note:: + When shift+drag or command/control+drag are performed over the annotator widget, they will not affect selections, but rather be used to edit the current annotations. + | Color the scatter plot @@ -75,3 +79,12 @@ Another way to probe the scatter plot is through node coloring. .. image:: ../media/use_case4.gif :align: center + +Annotate videos +~~~~~~~~~~~~~~~ + +Annotation of videos, e.g., marking the intervals when one or more behaviors occur, can be performed using the annotator widget (see tutorial). The widget is divided into rows corresonding to each annotation label. + +* Use shift+drag within a row to add an interval. +* Use command/control+drag within a row to subtract an interval. +* Right click to import or export annotations. \ No newline at end of file diff --git a/docs/source/tutorials.rst b/docs/source/tutorials.rst index 916adeb..09bc0ee 100644 --- a/docs/source/tutorials.rst +++ b/docs/source/tutorials.rst @@ -261,4 +261,33 @@ Add a plot of mouse velocity linewidth=2) -Next: `How to use the interface `_. +Video Annotation +---------------- + +The code below shows how to set up a SNUB project for video annotation, e.g., for marking the intervals when one or more behaviors are occuring. + +Define inputs +~~~~~~~~~~~~~ + +.. code-block:: python + + import snub.io + + video_path = "path/to/my/video.mp4" + labels = ["run", "rear", "groom"] + project_directory = 'annotation_project' + + # create project directory + video_duration = snub.io.generate_video_timestamps(video_path).max() + + snub.io.create_project( + project_directory, + duration=video_duration, + layout_mode="rows" + ) + + # add video + snub.io.add_video(project_directory, video_path) + + # add annotation widget + snub.io.add_annotator(project_directory, "my_annotator", labels=labels) \ No newline at end of file diff --git a/snub/gui/help/annotator.md b/snub/gui/help/annotator.md new file mode 100644 index 0000000..942a038 --- /dev/null +++ b/snub/gui/help/annotator.md @@ -0,0 +1,14 @@ +## Annotation + +The annotator widget can be used to define intervals when particular events happen. The widget is initialized with set of labels and is divided into rows (one for each label). + +* Use shift+drag within a row to add an interval +* Use command/control+drag within a row to subtract an interval + +### Saving and loading annotations + +Annotations are stored as a list of intervals (start and end times in seconds) for each label. By default, the current state of the widget is automatically saved to a file within the SNUB project whenever a change is made. This file can be accessed directly, or a copy of the annotations can be exported in json format using the context menu. Annotations can also be imported from elsewhere in the same format. + +* Right-click on the annotator widget to open the context menu +* Click on the checkbox to toggle automatic saving +* Click Import or Export to load or save the current annotations \ No newline at end of file diff --git a/snub/gui/help/help_menu.py b/snub/gui/help/help_menu.py index a337da7..28b709a 100644 --- a/snub/gui/help/help_menu.py +++ b/snub/gui/help/help_menu.py @@ -63,6 +63,7 @@ class HelpMenu: "Scatter plots": "scatter_plots.md", "Selections": "selections.md", "Video player": "video.md", + "Annotation": "annotator.md", } def __init__(self, parent): diff --git a/snub/gui/main.py b/snub/gui/main.py index 65f7de1..c6e5163 100644 --- a/snub/gui/main.py +++ b/snub/gui/main.py @@ -103,6 +103,7 @@ def __init__(self, project_directory): panel.new_current_time.connect(self.update_current_time) panel.selection_change.connect(self.update_selected_intervals) for track in self.trackStack.tracks_flat(): + track.new_current_time.connect(self.update_current_time) if isinstance(track, TracePlot): if track.bound_rois is not None: track.bind_rois(self.panelStack.get_by_name(track.bound_rois)) @@ -479,8 +480,6 @@ def load_selection(self): options=options, ) - print(file_name) - if file_name: try: self.deselect_all() diff --git a/snub/gui/tracks/annotator.py b/snub/gui/tracks/annotator.py index dfd6b2a..3cf8426 100644 --- a/snub/gui/tracks/annotator.py +++ b/snub/gui/tracks/annotator.py @@ -76,6 +76,7 @@ def __init__( config, data_path=None, autosave=True, + update_time_on_drag=True, label_color=(255, 255, 255), off_color=(0, 0, 0), on_color=(255, 0, 0), @@ -91,6 +92,7 @@ def __init__( self.off_color = off_color self.on_color = on_color self.autosave = autosave + self.update_time_on_drag = update_time_on_drag self.bounds = config["bounds"] self.drag_mode = 0 # +1 for shift-click, -1 for command-click @@ -156,8 +158,6 @@ def mousePressEvent(self, event): self.drag_start(t, ix, -1) else: super(Annotator, self).mouseMoveEvent(event) - else: - super(Annotator, self).mouseMoveEvent(event) def mouseReleaseEvent(self, event): self.drag_end() @@ -167,6 +167,8 @@ def drag_start(self, t, label_ix, mode): self.drag_mode = mode self.drag_initial_time = t self.drag_label_ix = label_ix + if self.update_time_on_drag: + self.new_current_time.emit(t) def drag_end(self): self.drag_mode = 0 @@ -183,6 +185,8 @@ def drag_move(self, t, mode): elif mode == -1: self.annotation_intervals[self.drag_label_ix].remove_interval(s, e) self.update() + if self.update_time_on_drag: + self.new_current_time.emit(t) def _position_to_label_ix(self, y): return int(y / self.height() * len(self.labels)) @@ -243,6 +247,17 @@ def add_menu_item(name, slot, item_type="label"): else: checkbox.setChecked(False) + # toggle update_time_on_drag + checkbox = add_menu_item( + "Update time on drag", + self.toggle_update_time_on_drag, + item_type="checkbox", + ) + if self.update_time_on_drag: + checkbox.setChecked(True) + else: + checkbox.setChecked(False) + # import/export annotations add_menu_item("Export annotations", self.export_annotations) add_menu_item("Import annotations", self.import_annotations) @@ -262,6 +277,9 @@ def add_menu_item(name, slot, item_type="label"): def toggle_autosave(self, state): self.autosave = state + def toggle_update_time_on_drag(self, state): + self.update_time_on_drag = state + def export_annotations(self): options = QFileDialog.Options() options |= QFileDialog.DontUseNativeDialog diff --git a/snub/gui/tracks/base.py b/snub/gui/tracks/base.py index eb5dcd9..946b389 100644 --- a/snub/gui/tracks/base.py +++ b/snub/gui/tracks/base.py @@ -48,6 +48,8 @@ def accept_range(self): class Track(QWidget): + new_current_time = pyqtSignal(float) + def __init__(self, config, parent=None, height_ratio=1, order=0, **kwargs): super().__init__(parent=parent) self.layout_mode = config["layout_mode"] diff --git a/snub/io/project.py b/snub/io/project.py index db48230..7f77478 100644 --- a/snub/io/project.py +++ b/snub/io/project.py @@ -9,6 +9,8 @@ import scipy.sparse from vidio import VideoReader +from snub.io.video import generate_video_timestamps + def generate_intervals(start_time, binsize, num_intervals): """Generate an array of start/end times for non-overlapping @@ -404,7 +406,7 @@ def add_video( videopath, copy=False, name=None, - fps=30, + fps=None, start_time=0, timestamps=None, size_ratio=1, @@ -432,10 +434,10 @@ def add_video( The name of the video, which is displayed in SNUB and can be used to edit the config file. If no name is given, the video's filename will be used. - fps: float, default=30 + fps: float, default=None The video framerate. This parameter is used in conjunction with ``start_time`` to generate a timestamps file, unless an array of timestamps is directly - provided. + provided. If None, it is inferred from the video file. start_time: float, default=0 The start time of the video (in seconds). This parameter is used in conjunction @@ -480,13 +482,7 @@ def add_video( # load/create timestamps and save as .npy if timestamps is None: - video_length = len(VideoReader(videopath)) - timestamps = np.arange(video_length) / fps + start_time - print( - "Creating timestamps array with start_time={}, fps={}, and n_frames={}".format( - start_time, fps, video_length - ) - ) + timestamps = generate_video_timestamps(videopath, fps, start_time) elif isinstance(timestamps, str): if timestamps.endswith(".npy"): timestamps = np.load(timestamps) diff --git a/snub/io/video.py b/snub/io/video.py index 45b8076..b25b606 100644 --- a/snub/io/video.py +++ b/snub/io/video.py @@ -3,6 +3,7 @@ import scipy import tqdm import os +from vidio.read import VideoReader def azure_ir_transform(input_image): @@ -193,3 +194,14 @@ def fast_prct_filt(input_data, level=8, frames_window=3000): data -= tr_BL[padbefore:-padafter].T return data.squeeze() + + +def generate_video_timestamps(videopath, fps=None, start_time=0): + """ + Generate timestamps from a video file. + """ + reader = VideoReader(videopath) + if fps is None: + fps = reader.fps + timestamps = np.arange(len(reader)) / fps + start_time + return timestamps