Improve flight mode tracing performance for large tracings (#3880)

* improve flight mode tracing performance for large tracings, decrease undo stack size

* update changelog

CHANGELOG.md

@@ -18,6 +18,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.md).
 - The mapping selection dropbown is now sorted alphabetically. [#3864](https://github.com/scalableminds/webknossos/pull/3864)
 
 ### Changed
+- Improved the flight mode performance for tracings with very large trees (>80.000 nodes). [#3880](https://github.com/scalableminds/webknossos/pull/3880)
 - Tweaked the highlighting of the active node. The inner node looks exactly as a non-active node and is not round, anymore. An active node is circled by a "halo". In arbitrary mode, the halo is hidden and the active node is round. [#3868](https://github.com/scalableminds/webknossos/pull/3868)
 - Brush size is independent of zoom value, now. This change simplifies volume annotations, as brush sizes can be adapted to certain structures (e.g., vesicles) and don't need to be changed when zooming. [#3868](https://github.com/scalableminds/webknossos/pull/3889)
 

frontend/javascripts/libs/diffable_map.js

@@ -1,6 +1,6 @@
 // @flow
 
-const defaultItemsPerBatch = 10000;
+const defaultItemsPerBatch = 1000;
 let idCounter = 0;
 const idSymbol = Symbol("id");
 
@@ -15,7 +15,6 @@ const idSymbol = Symbol("id");
 class DiffableMap<K: number, V> {
   chunks: Array<Map<K, V>>;
   entryCount: number;
-  existsCache: Map<K, boolean>;
   itemsPerBatch: number;
 
   constructor(optKeyValueArray?: ?Array<[K, V]>, itemsPerBatch?: ?number) {
@@ -27,7 +26,6 @@ class DiffableMap<K: number, V> {
       writable: true,
     });
     this.chunks = [];
-    this.existsCache = new Map();
     this.entryCount = 0;
     this.itemsPerBatch = itemsPerBatch != null ? itemsPerBatch : defaultItemsPerBatch;
 
@@ -77,67 +75,60 @@ class DiffableMap<K: number, V> {
   }
 
   set(key: K, value: V): DiffableMap<K, V> {
-    if (this.existsCache.has(key)) {
-      let idx = 0;
-      while (this.chunks[idx] != null) {
-        if (this.chunks[idx].has(key)) {
-          const newMap = shallowCopy(this);
-          newMap.chunks[idx] = new Map(this.chunks[idx]);
-          newMap.chunks[idx].set(key, value);
-          return newMap;
-        }
-        idx++;
+    let idx = 0;
+    while (this.chunks[idx] != null) {
+      if (this.chunks[idx].has(key)) {
+        const newMap = shallowCopy(this);
+        newMap.chunks[idx] = new Map(this.chunks[idx]);
+        newMap.chunks[idx].set(key, value);
+        return newMap;
       }
-      // Satisfy flow.
-      throw new Error("This code path should never be reached due to the above logic.");
+      idx++;
+    }
+
+    // The key was not found in the existing chunks
+    const isTooFull = this.entryCount / this.chunks.length > this.itemsPerBatch;
+    const nonFullMapIdx =
+      isTooFull || this.chunks.length === 0 ? -1 : Math.floor(Math.random() * this.chunks.length);
+
+    // Key didn't exist. Add it.
+    const newDiffableMap = shallowCopy(this);
+    newDiffableMap.entryCount = this.entryCount + 1;
+    if (nonFullMapIdx > -1) {
+      newDiffableMap.chunks[nonFullMapIdx] = new Map(this.chunks[nonFullMapIdx]);
+      newDiffableMap.chunks[nonFullMapIdx].set(key, value);
+      return newDiffableMap;
     } else {
-      const isTooFull = this.entryCount / this.chunks.length > this.itemsPerBatch;
-      const nonFullMapIdx =
-        isTooFull || this.chunks.length === 0 ? -1 : Math.floor(Math.random() * this.chunks.length);
-
-      // Key didn't exist. Add it.
-      const newDiffableMap = shallowCopy(this);
-      newDiffableMap.existsCache.set(key, true);
-      newDiffableMap.entryCount = this.entryCount + 1;
-      if (nonFullMapIdx > -1) {
-        newDiffableMap.chunks[nonFullMapIdx] = new Map(this.chunks[nonFullMapIdx]);
-        newDiffableMap.chunks[nonFullMapIdx].set(key, value);
-        return newDiffableMap;
-      } else {
-        const freshMap = new Map();
-        freshMap.set(key, value);
-        newDiffableMap.chunks.push(freshMap);
-        return newDiffableMap;
-      }
+      const freshMap = new Map();
+      freshMap.set(key, value);
+      newDiffableMap.chunks.push(freshMap);
+      return newDiffableMap;
     }
   }
 
   mutableSet(key: K, value: V): void {
-    if (this.existsCache.has(key)) {
-      let idx = 0;
-      while (this.chunks[idx] != null) {
-        if (this.chunks[idx].has(key)) {
-          this.chunks[idx].set(key, value);
-          return;
-        }
-        idx++;
+    let idx = 0;
+    while (this.chunks[idx] != null) {
+      if (this.chunks[idx].has(key)) {
+        this.chunks[idx].set(key, value);
+        return;
       }
+      idx++;
+    }
+
+    // The key was not found in the existing chunks
+    const isTooFull = this.entryCount / this.chunks.length > this.itemsPerBatch;
+    const nonFullMapIdx =
+      isTooFull || this.chunks.length === 0 ? -1 : Math.floor(Math.random() * this.chunks.length);
+
+    // Key didn't exist. Add it.
+    this.entryCount++;
+    if (nonFullMapIdx > -1) {
+      this.chunks[nonFullMapIdx].set(key, value);
     } else {
-      // let idx = 0;
-      const isTooFull = this.entryCount / this.chunks.length > this.itemsPerBatch;
-      const nonFullMapIdx =
-        isTooFull || this.chunks.length === 0 ? -1 : Math.floor(Math.random() * this.chunks.length);
-
-      // Key didn't exist. Add it.
-      this.existsCache.set(key, true);
-      this.entryCount++;
-      if (nonFullMapIdx > -1) {
-        this.chunks[nonFullMapIdx].set(key, value);
-      } else {
-        const freshMap = new Map();
-        freshMap.set(key, value);
-        this.chunks.push(freshMap);
-      }
+      const freshMap = new Map();
+      freshMap.set(key, value);
+      this.chunks.push(freshMap);
     }
   }
 
@@ -150,31 +141,26 @@ class DiffableMap<K: number, V> {
 
     // Clone other attributes
     newDiffableMap.setId(this.getId());
-    newDiffableMap.existsCache = new Map(this.existsCache);
     newDiffableMap.entryCount = this.entryCount;
     newDiffableMap.itemsPerBatch = this.itemsPerBatch;
 
     return newDiffableMap;
   }
 
   delete(key: K): DiffableMap<K, V> {
-    if (!this.existsCache.has(key)) {
-      return this;
-    }
     let idx = 0;
     while (this.chunks[idx] != null) {
       if (this.chunks[idx].has(key)) {
         const newMap = shallowCopy(this);
-        newMap.existsCache.delete(key);
         newMap.entryCount--;
         newMap.chunks[idx] = new Map(this.chunks[idx]);
         newMap.chunks[idx].delete(key);
         return newMap;
       }
       idx++;
     }
-    // Satisfy flow.
-    throw new Error("This code path should never be reached due to the above logic.");
+    // The key was not found in the existing chunks
+    return this;
   }
 
   map<T>(fn: (value: V) => T): Array<T> {
@@ -226,12 +212,10 @@ class DiffableMap<K: number, V> {
 // It creates a new DiffableMap on the basis of another one, while
 // shallowly copying the internal chunks.
 // When modifying a chunk, that chunk should be manually cloned.
-// The existsCache is safely cloned.
 function shallowCopy<K: number, V>(template: DiffableMap<K, V>): DiffableMap<K, V> {
   const newMap = new DiffableMap();
   newMap.setId(template.getId());
   newMap.chunks = template.chunks.slice();
-  newMap.existsCache = new Map(template.existsCache);
   newMap.entryCount = template.entryCount;
   newMap.itemsPerBatch = template.itemsPerBatch;
   return newMap;

frontend/javascripts/oxalis/model/edge_collection.js

@@ -54,7 +54,23 @@ export default class EdgeCollection {
   }
 
   addEdge(edge: Edge, mutate: boolean = false): EdgeCollection {
-    return this.addEdges([edge], mutate);
+    // This is a performance optimized version of addEdges for a single edge.
+    // In the single-edge case, it is faster to call .set on the diffable map which will
+    // clone only the affected chunk and not all chunks (as addEdges).
+    const newEdgeCount = this.edgeCount + 1;
+    const outgoingEdges = this.outMap.getNullable(edge.source) || [];
+    const ingoingEdges = this.inMap.getNullable(edge.target) || [];
+
+    if (mutate) {
+      this.outMap.mutableSet(edge.source, outgoingEdges.concat(edge));
+      this.inMap.mutableSet(edge.target, ingoingEdges.concat(edge));
+      this.edgeCount = newEdgeCount;
+      return this;
+    } else {
+      const newOutgoingEdges = this.outMap.set(edge.source, outgoingEdges.concat(edge));
+      const newIngoingEdges = this.inMap.set(edge.target, ingoingEdges.concat(edge));
+      return EdgeCollection.loadFromMaps(newOutgoingEdges, newIngoingEdges, newEdgeCount);
+    }
   }
 
   removeEdge(edge: Edge): EdgeCollection {

frontend/javascripts/oxalis/model/reducers/skeletontracing_reducer_helpers.js

@@ -131,15 +131,15 @@ export function createNode(
     };
 
     // Create a new edge
-    const newEdges = activeNodeMaybe
-      .map(activeNode => [
-        {
+    const edges = activeNodeMaybe
+      .map(activeNode => {
+        const newEdge = {
           source: activeNode.id,
           target: nextNewId,
-        },
-      ])
-      .getOrElse([]);
-    const edges = tree.edges.addEdges(newEdges);
+        };
+        return tree.edges.addEdge(newEdge);
+      })
+      .getOrElse(tree.edges);
 
     return Maybe.Just([node, edges]);
   }

frontend/javascripts/oxalis/model/sagas/save_saga.js

@@ -48,7 +48,7 @@ import { enforceSkeletonTracing } from "../accessors/skeletontracing_accessor";
 const PUSH_THROTTLE_TIME = 30000; // 30s
 const SAVE_RETRY_WAITING_TIME = 2000;
 const MAX_SAVE_RETRY_WAITING_TIME = 300000; // 5m
-const UNDO_HISTORY_SIZE = 100;
+const UNDO_HISTORY_SIZE = 20;
 
 export const maximumActionCountPerBatch = 5000;
 const maximumActionCountPerSave = 15000;

frontend/javascripts/test/libs/diffable_map.spec.js

@@ -50,9 +50,6 @@ test("DiffableMap should be clonable and mutable on clone/mutableSet", t => {
   t.is(map1.get(1), 1);
   t.false(map1.has(2));
 
-  // existsCache should not be shared
-  t.false(map1.existsCache === map2.existsCache);
-
   // Id should be the same since the internal structures look the same
   t.is(map1.getId(), map2.getId());
 

frontend/javascripts/test/model/edge_collection.spec.js

@@ -119,3 +119,35 @@ test("EdgeCollection diffing should work with smaller batch size when there is a
   t.deepEqual(onlyA.sort(edgeSort), [edges[0]]);
   t.deepEqual(onlyB.sort(edgeSort), edges.slice(8));
 });
+
+test("EdgeCollection addEdge should not mutate the original edge collection", t => {
+  const edgeA = { source: 0, target: 1 };
+  const edgeB = { source: 2, target: 1 };
+  const edgeC = { source: 3, target: 2 };
+  const edgeD = { source: 3, target: 4 };
+
+  const edgeCollectionA = new EdgeCollection().addEdges([edgeA, edgeB, edgeC]);
+
+  const edgeCollectionB = edgeCollectionA.addEdge(edgeD);
+
+  const { onlyA, onlyB } = diffEdgeCollections(edgeCollectionA, edgeCollectionB);
+
+  t.deepEqual(onlyA, []);
+  t.deepEqual(onlyB, [edgeD]);
+});
+
+test("EdgeCollection addEdge should mutate the original edge collection if specified", t => {
+  const edgeA = { source: 0, target: 1 };
+  const edgeB = { source: 2, target: 1 };
+  const edgeC = { source: 3, target: 2 };
+  const edgeD = { source: 3, target: 4 };
+
+  const edgeCollectionA = new EdgeCollection().addEdges([edgeA, edgeB, edgeC]);
+
+  const edgeCollectionB = edgeCollectionA.addEdge(edgeD, true);
+
+  const { onlyA, onlyB } = diffEdgeCollections(edgeCollectionA, edgeCollectionB);
+
+  t.deepEqual(onlyA, []);
+  t.deepEqual(onlyB, []);
+});