Proofreading without skeletons (#6625)

* first PoC

* use activeCellId for proofreading

* prototype active segment pulsing

* Revert "prototype active segment pulsing"

This reverts commit b37134e.

* show cross hair for active segment position in proofreading mode and increase opacity of active segment

* renaming

* clean up and remove agglomerate skeletons from proofreading saga

* allow to select node in proofreading mode and dispatch proofreading action

* remove passive meshes, refactor cross hair shader

* fix superfluous passive flag

* fix that the cursor never changed again after activating the bounding box tool

* add shift and ctrl modifiers to proofreading tool for faster actions

* include shortcuts in context menu

* update changelog

* apply PR feedback and clean up at 0,0,0

CHANGELOG.unreleased.md

@@ -28,6 +28,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Re-phrased some backend (error) messages to improve clarity and provide helping hints. [#6616](https://github.com/scalableminds/webknossos/pull/6616)
 - The layer visibility is now encoded in the sharing link. The user opening the link will see the same layers that were visible when copying the link. [#6634](https://github.com/scalableminds/webknossos/pull/6634)
 - Voxelytics workflows can now be viewed by anyone with the link who is in the right organization. [#6622](https://github.com/scalableminds/webknossos/pull/6622)
+- Reworked the proofreading mode so that agglomerate skeletons are no longer needed (nor automatically loaded). Instead, segments can be selected by left-clicking onto them, indicated by a small white cross. To merge or split agglomerates, then either use the shortcuts `Shift + Leftclick`/`Ctrl + Leftclick` or use the context menu. [#6625](https://github.com/scalableminds/webknossos/pull/6625)
 
 ### Fixed
 - Fixed a bug in the dataset import view, where the layer name text field would lose focus after each key press. [#6615](https://github.com/scalableminds/webknossos/pull/6615)

frontend/javascripts/oxalis/controller/combinations/bounding_box_handlers.ts

@@ -227,7 +227,8 @@ export function getClosestHoveredBoundingBox(
 export const highlightAndSetCursorOnHoveredBoundingBox = _.throttle(
   (delta: Point2, position: Point2, planeId: OrthoView) => {
     const hoveredEdgesInfo = getClosestHoveredBoundingBox(position, planeId);
-    const inputCatcher = document.getElementById(`inputcatcher_${planeId}`);
+    // Access the parent element as that is where the cursor style property is set
+    const inputCatcher = document.getElementById(`inputcatcher_${planeId}`)?.parentElement;
 
     if (hoveredEdgesInfo != null && inputCatcher != null) {
       const [primaryHoveredEdge, secondaryHoveredEdge] = hoveredEdgesInfo;

frontend/javascripts/oxalis/controller/combinations/tool_controls.ts

@@ -40,7 +40,11 @@ import * as Utils from "libs/utils";
 import * as VolumeHandlers from "oxalis/controller/combinations/volume_handlers";
 import { document } from "libs/window";
 import api from "oxalis/api/internal_api";
-import { proofreadAtPosition } from "oxalis/model/actions/proofread_actions";
+import {
+  minCutAgglomerateWithPositionAction,
+  proofreadAtPosition,
+  proofreadMerge,
+} from "oxalis/model/actions/proofread_actions";
 import { calculateGlobalPos } from "oxalis/model/accessors/view_mode_accessor";
 import { V3 } from "libs/mjs";
 
@@ -690,13 +694,13 @@ export class QuickSelectTool {
         _delta: Point2,
         pos: Point2,
         _id: string | null | undefined,
-        _event: MouseEvent,
+        event: MouseEvent,
       ) => {
         if (!isDragging || startPos == null) {
           return;
         }
         const newCurrentPos = V3.floor(calculateGlobalPos(Store.getState(), pos));
-        if (_event.shiftKey) {
+        if (event.shiftKey) {
           // If shift is held, the rectangle is resized on topLeft and bottomRight
           // so that the center is constant.
           // We don't use the passed _delta variable, because that is given in pixel-space
@@ -751,37 +755,47 @@ export class ProofreadTool {
   }
 
   static onLeftClick(
-    planeView: PlaneView,
+    _planeView: PlaneView,
     pos: Point2,
     plane: OrthoView,
-    _event: MouseEvent,
-    isTouch: boolean,
+    event: MouseEvent,
+    _isTouch: boolean,
   ) {
-    const didSelectNode = SkeletonHandlers.handleSelectNode(planeView, pos, plane, isTouch);
-    if (didSelectNode) {
-      // Don't do anything else
-      return;
-    }
-
     if (plane === OrthoViews.TDView) {
       // The click position cannot be mapped to a 3D coordinate in the
-      // 3D viewport (unless a node was clicked which is already handled above).
+      // 3D viewport.
       return;
     }
 
     const globalPosition = calculateGlobalPos(Store.getState(), pos);
-    Store.dispatch(proofreadAtPosition(globalPosition));
+
+    if (event.shiftKey) {
+      Store.dispatch(proofreadMerge(globalPosition));
+    } else if (event.ctrlKey) {
+      Store.dispatch(minCutAgglomerateWithPositionAction(globalPosition));
+    } else {
+      Store.dispatch(proofreadAtPosition(globalPosition));
+      VolumeHandlers.handlePickCell(pos);
+    }
   }
 
   static getActionDescriptors(
     _activeTool: AnnotationTool,
     _useLegacyBindings: boolean,
-    _shiftKey: boolean,
-    _ctrlKey: boolean,
+    shiftKey: boolean,
+    ctrlKey: boolean,
     _altKey: boolean,
   ): ActionDescriptor {
+    let leftClick = "Select Segment to Proofread";
+
+    if (shiftKey) {
+      leftClick = "Merge with active Segment";
+    } else if (ctrlKey) {
+      leftClick = "Split from active Segment";
+    }
+
     return {
-      leftClick: "Select Segment to Proofread",
+      leftClick,
       rightClick: "Context Menu",
     };
   }

frontend/javascripts/oxalis/controller/scene_controller.ts

@@ -202,7 +202,7 @@ class SceneController {
     return color;
   }
 
-  constructIsosurfaceMesh(cellId: number, geometry: THREE.BufferGeometry, passive: boolean) {
+  constructIsosurfaceMesh(cellId: number, geometry: THREE.BufferGeometry) {
     const color = this.getColorObjectForSegment(cellId);
     const meshMaterial = new THREE.MeshLambertMaterial({
       color,
@@ -213,14 +213,13 @@ class SceneController {
     const mesh = new THREE.Mesh(geometry, meshMaterial);
     mesh.castShadow = true;
     mesh.receiveShadow = true;
-    mesh.renderOrder = passive ? 1 : 0;
     const tweenAnimation = new TWEEN.Tween({
       opacity: 0,
     });
     tweenAnimation
       .to(
         {
-          opacity: passive ? 0.4 : 1,
+          opacity: 1,
         },
         500,
       )
@@ -245,33 +244,25 @@ class SceneController {
 
     const meshNumber = _.size(this.stlMeshes);
 
-    const mesh = this.constructIsosurfaceMesh(meshNumber, geometry, false);
+    const mesh = this.constructIsosurfaceMesh(meshNumber, geometry);
     this.meshesRootGroup.add(mesh);
     this.stlMeshes[id] = mesh;
     this.updateMeshPostion(id, position);
   }
 
-  addIsosurfaceFromVertices(
-    vertices: Float32Array,
-    segmentationId: number,
-    // Passive isosurfaces are ignored during picking, are shown more transparently, and are rendered
-    // last so that all non-passive isosurfaces are rendered before them. This makes sure that non-passive
-    // isosurfaces are not skipped during rendering if they are overlapped by passive ones.
-    passive: boolean,
-  ): void {
+  addIsosurfaceFromVertices(vertices: Float32Array, segmentationId: number): void {
     let bufferGeometry = new THREE.BufferGeometry();
     bufferGeometry.setAttribute("position", new THREE.BufferAttribute(vertices, 3));
 
     bufferGeometry = mergeVertices(bufferGeometry);
     bufferGeometry.computeVertexNormals();
 
-    this.addIsosurfaceFromGeometry(bufferGeometry, segmentationId, passive);
+    this.addIsosurfaceFromGeometry(bufferGeometry, segmentationId);
   }
 
   addIsosurfaceFromGeometry(
     geometry: THREE.BufferGeometry,
     segmentationId: number,
-    passive: boolean = false,
     offset: Vector3 | null = null,
     scale: Vector3 | null = null,
   ): void {
@@ -281,13 +272,11 @@ class SceneController {
       this.isosurfacesRootGroup.add(newGroup);
       // @ts-ignore
       newGroup.cellId = segmentationId;
-      // @ts-ignore
-      newGroup.passive = passive;
       if (scale != null) {
         newGroup.scale.copy(new THREE.Vector3(...scale));
       }
     }
-    const mesh = this.constructIsosurfaceMesh(segmentationId, geometry, passive);
+    const mesh = this.constructIsosurfaceMesh(segmentationId, geometry);
     if (offset) {
       mesh.translateX(offset[0]);
       mesh.translateY(offset[1]);

frontend/javascripts/oxalis/geometries/helper_geometries.ts

@@ -88,6 +88,7 @@ export class QuickSelectGeometry {
       opacity: 0.5,
     });
     this.rectangle = new THREE.Mesh(geometry, material);
+    this.rectangle.visible = false;
 
     const centerGeometry = new THREE.PlaneGeometry(2, 2);
     const centerMaterial = new THREE.MeshBasicMaterial({
@@ -97,6 +98,7 @@ export class QuickSelectGeometry {
       opacity: 0.9,
     });
     this.centerMarker = new THREE.Mesh(centerGeometry, centerMaterial);
+    this.centerMarker.visible = false;
 
     this.meshGroup = new THREE.Group();
     this.meshGroup.add(this.rectangle);
@@ -144,6 +146,16 @@ export class QuickSelectGeometry {
 
     this.centerMarker.position.set(...centerPosition);
 
+    // Hide the objects if the rectangle has size zero, so whenever
+    // the quick select tool is not currently used to draw a rectangle.
+    if (V3.isEqual(endPosition, startPosition)) {
+      this.centerMarker.visible = false;
+      this.rectangle.visible = false;
+    } else {
+      this.centerMarker.visible = true;
+      this.rectangle.visible = true;
+    }
+
     app.vent.trigger("rerender");
   }
 

frontend/javascripts/oxalis/geometries/materials/plane_material_factory.ts

@@ -1,12 +1,19 @@
 import * as THREE from "three";
 import _ from "lodash";
 import type { OrthoView, Vector3 } from "oxalis/constants";
-import { ViewModeValues, OrthoViewValues, OrthoViews, MappingStatusEnum } from "oxalis/constants";
+import {
+  ViewModeValues,
+  OrthoViewValues,
+  OrthoViews,
+  MappingStatusEnum,
+  AnnotationToolEnum,
+} from "oxalis/constants";
 import { calculateGlobalPos } from "oxalis/model/accessors/view_mode_accessor";
 import { isBrushTool } from "oxalis/model/accessors/tool_accessor";
 import {
   getActiveCellId,
   getActiveSegmentationTracing,
+  getActiveSegmentPosition,
 } from "oxalis/model/accessors/volumetracing_accessor";
 import {
   getAddressSpaceDimensions,
@@ -148,6 +155,9 @@ class PlaneMaterialFactory {
       globalMousePosition: {
         value: new THREE.Vector3(0, 0, 0),
       },
+      activeSegmentPosition: {
+        value: new THREE.Vector3(-1, -1, -1),
+      },
       brushSizeInPixel: {
         value: 0,
       },
@@ -163,6 +173,9 @@ class PlaneMaterialFactory {
       showBrush: {
         value: false,
       },
+      isProofreading: {
+        value: false,
+      },
       viewMode: {
         value: 0,
       },
@@ -588,6 +601,20 @@ class PlaneMaterialFactory {
           (storeState) => storeState.uiInformation.activeTool,
           (annotationTool) => {
             this.uniforms.showBrush.value = isBrushTool(annotationTool);
+            this.uniforms.isProofreading.value = annotationTool === AnnotationToolEnum.PROOFREAD;
+          },
+          true,
+        ),
+      );
+      this.storePropertyUnsubscribers.push(
+        listenToStoreProperty(
+          (storeState) => getActiveSegmentPosition(storeState),
+          (activeSegmentPosition) => {
+            if (activeSegmentPosition != null) {
+              this.uniforms.activeSegmentPosition.value.set(...activeSegmentPosition);
+            } else {
+              this.uniforms.activeSegmentPosition.value.set(-1, -1, -1);
+            }
           },
           true,
         ),

frontend/javascripts/oxalis/model/accessors/tool_accessor.ts

@@ -237,7 +237,8 @@ export function adaptActiveToolToShortcuts(
     activeTool === AnnotationToolEnum.MOVE ||
     activeTool === AnnotationToolEnum.ERASE_BRUSH ||
     activeTool === AnnotationToolEnum.ERASE_TRACE ||
-    activeTool === AnnotationToolEnum.QUICK_SELECT
+    activeTool === AnnotationToolEnum.QUICK_SELECT ||
+    activeTool === AnnotationToolEnum.PROOFREAD
   ) {
     // These tools do not have any modifier-related behavior currently (except for ALT
     // which is already handled below)

frontend/javascripts/oxalis/model/accessors/volumetracing_accessor.ts

@@ -334,6 +334,20 @@ export function getVisibleSegments(state: OxalisState): SegmentMap | null | unde
   return state.localSegmentationData[layer.name].segments;
 }
 
+export function getActiveSegmentPosition(state: OxalisState): Vector3 | null | undefined {
+  const layer = getVisibleSegmentationLayer(state);
+  if (layer == null) return null;
+
+  const volumeTracing = getVolumeTracingByLayerName(state.tracing, layer.name);
+  if (volumeTracing == null) return null;
+
+  const activeCellId = getActiveCellId(volumeTracing);
+  if (activeCellId == null) return null;
+
+  const segments = getSegmentsForLayer(state, layer.name);
+  return segments.getNullable(activeCellId)?.somePosition;
+}
+
 /*
   This function returns the resolution and zoom step in which the given segmentation
   tracing layer is currently rendered (if it is rendered). These properties should be used

frontend/javascripts/oxalis/model/actions/proofread_actions.ts

@@ -34,9 +34,8 @@ export const minCutAgglomerateAction = (sourceNodeId: number, targetNodeId: numb
     targetNodeId,
   } as const);
 
-export const minCutAgglomerateWithPositionAction = (sourceNodeId: number, position: Vector3) =>
+export const minCutAgglomerateWithPositionAction = (position: Vector3) =>
   ({
     type: "MIN_CUT_AGGLOMERATE_WITH_POSITION",
-    sourceNodeId,
     position,
   } as const);

frontend/javascripts/oxalis/model/actions/segmentation_actions.ts

@@ -4,7 +4,6 @@ export type AdHocIsosurfaceInfo = {
   mappingName: string | null | undefined;
   mappingType: MappingType | null | undefined;
   useDataStore?: boolean | null | undefined;
-  passive?: boolean | null | undefined;
   preferredQuality?: number | null | undefined;
 };
 export type LoadAdHocMeshAction = ReturnType<typeof loadAdHocMeshAction>;

frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts

@@ -402,11 +402,7 @@ function* maybeLoadIsosurface(
         getSceneController().removeIsosurfaceById(segmentId);
       }
 
-      getSceneController().addIsosurfaceFromVertices(
-        vertices,
-        segmentId,
-        isosurfaceExtraInfo.passive || false,
-      );
+      getSceneController().addIsosurfaceFromVertices(vertices, segmentId);
       return neighbors.map((neighbor) => getNeighborPosition(clippedPosition, neighbor));
     } catch (exception) {
       retryCount++;
@@ -685,7 +681,6 @@ function* loadPrecomputedMeshForSegmentId(
             { context: sceneController, fn: sceneController.addIsosurfaceFromGeometry },
             geometry,
             id,
-            false,
             chunk.position,
             // Apply the scale from the segment info, which includes dataset scale and mag
             scale,
@@ -715,7 +710,6 @@ function* loadPrecomputedMeshForSegmentId(
             { context: sceneController, fn: sceneController.addIsosurfaceFromGeometry },
             geometry,
             id,
-            false,
           );
         }
       },