Merge pull request #2947 from scalableminds/chunk-save-actions

Send save actions in chunks to server

CHANGELOG.md

@@ -16,6 +16,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.md).
 - When a lot of changes need to be persisted to the server (e.g., after importing a large NML), the save button will show a percentage-based progress indicator.
 - Added the possibility to import multiple NML files into the active tracing. This can be done by dragging and dropping the files directly into the tracing view. [#2908](https://github.com/scalableminds/webknossos/pull/2908)            
 - During the import of multiple NML files, the user can select an option to automatically create a group per file so that the imported trees are organized in a hierarchy. [#2908](https://github.com/scalableminds/webknossos/pull/2908)
+
 ### Changed
 
 -
@@ -57,6 +58,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.md).
 - Added the shortcut to copy the currently hovered cell id (CTRL + I) to non-volume-tracings, too. [#2726](https://github.com/scalableminds/webknossos/pull/2726)
 - Added permission for team managers to refresh datasets. [#2688](https://github.com/scalableminds/webknossos/pull/2688)
 - Added backend-unit-test setup and a first test for NML validation. [#2829](https://github.com/scalableminds/webknossos/pull/2829)
+- Added progress indicators to the save button for cases where the saving takes some time (e.g., when importing a large NML). [#2947](https://github.com/scalableminds/webknossos/pull/2947)
 - Added the possibility to not sort comments by name. When clicking the sort button multiple times, sorting is switched to sort by IDs. [#2915](https://github.com/scalableminds/webknossos/pull/2915)
 - Added displayName for organizations. [#2869](https://github.com/scalableminds/webknossos/pull/2869)
 - Added onboarding flow for initial setup of WebKnossos. [#2859](https://github.com/scalableminds/webknossos/pull/2859)
@@ -69,6 +71,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.md).
 - When deleting the last node of a tree, that tree will not be removed automatically anymore. Instead, the tree will just be empty. To remove that active tree, the "delete" shortcut can be used again. [#2806](https://github.com/scalableminds/webknossos/pull/2806)
 - Changed the type of the initial node of new tasks to be a branchpoint (if not created via NML). [#2799](https://github.com/scalableminds/webknossos/pull/2799)
 - The dataset gallery got a redesign with mobile support. [#2761](https://github.com/scalableminds/webknossos/pull/2761)
+- Improved the performance of saving large changes to a tracing (e.g., when importing a large NML). [#2947](https://github.com/scalableminds/webknossos/pull/2947)
 - Improved loading speed of buckets. [#2724](https://github.com/scalableminds/webknossos/pull/2724)
 - Changed the task search, when filtered by user, to show all instead of just active tasks (except for canceled tasks). [#2774](https://github.com/scalableminds/webknossos/pull/2774)
 - Improved the import dialog for datasets. Important fields can now be edited via form inputs instead of having to change the JSON. The JSON is still changeable when enabling an "Advanced" mode. [#2881](https://github.com/scalableminds/webknossos/pull/2881)

app/assets/javascripts/oxalis/model/helpers/generate_dummy_trees.js

@@ -47,7 +47,7 @@ export default function generateDummyTrees(
       treeId: currentTreeId++,
       nodes,
       edges,
-      color: { r: Math.random(), g: Math.random(), b: Math.random() },
+      color: { r: 0, g: 0, b: 0 },
       branchPoints: [],
       comments: [],
       name: "explorative_2017-10-09_SCM_Boy_023",

app/assets/javascripts/oxalis/model/reducers/save_reducer.js

@@ -18,7 +18,8 @@ function SaveReducer(state: OxalisState, action: ActionType): OxalisState {
       const stats = _.some(action.items, ua => ua.name !== "updateTracing")
         ? Utils.toNullable(getStats(state.tracing))
         : null;
-      if (action.items.length > 0) {
+      const { items } = action;
+      if (items.length > 0) {
         return update(state, {
           save: {
             queue: {
@@ -27,29 +28,55 @@ function SaveReducer(state: OxalisState, action: ActionType): OxalisState {
                   // Placeholder, the version number will be updated before sending to the server
                   version: -1,
                   timestamp: Date.now(),
-                  actions: action.items,
+                  actions: items,
                   stats,
                 },
               ],
             },
+            progressInfo: {
+              totalActionCount: { $apply: count => count + items.length },
+            },
           },
         });
       }
       return state;
     }
 
     case "SHIFT_SAVE_QUEUE": {
-      if (action.count > 0) {
+      const { count } = action;
+      if (count > 0) {
+        const processedQueueActions = _.sumBy(
+          state.save.queue.slice(0, count),
+          batch => batch.actions.length,
+        );
+        const remainingQueue = state.save.queue.slice(count);
+        const resetCounter = remainingQueue.length === 0;
         return update(state, {
-          save: { queue: { $set: state.save.queue.slice(action.count) } },
+          save: {
+            queue: { $set: remainingQueue },
+            progressInfo: {
+              // Reset progress counters if the queue is empty. Otherwise,
+              // increase processedActionCount and leave totalActionCount as is
+              processedActionCount: {
+                $apply: oldCount => (resetCounter ? 0 : oldCount + processedQueueActions),
+              },
+              totalActionCount: { $apply: oldCount => (resetCounter ? 0 : oldCount) },
+            },
+          },
         });
       }
       return state;
     }
 
     case "DISCARD_SAVE_QUEUE": {
       return update(state, {
-        save: { queue: { $set: [] } },
+        save: {
+          queue: { $set: [] },
+          progressInfo: {
+            processedActionCount: { $set: 0 },
+            totalActionCount: { $set: 0 },
+          },
+        },
       });
     }
 

app/assets/javascripts/oxalis/model/sagas/save_saga.js

@@ -37,6 +37,9 @@ const PUSH_THROTTLE_TIME = 30000; // 30s
 const SAVE_RETRY_WAITING_TIME = 5000;
 const UNDO_HISTORY_SIZE = 100;
 
+export const maximumActionCountPerBatch = 5000;
+const maximumActionCountPerSave = 15000;
+
 export function* collectUndoStates(): Generator<*, *, *> {
   const undoStack = [];
   const redoStack = [];
@@ -88,6 +91,7 @@ export function* pushAnnotationAsync(): Generator<*, *, *> {
     // could have been triggered during the call to sendRequestToServer
     saveQueue = yield select(state => state.save.queue);
     if (saveQueue.length === 0) {
+      yield put(setSaveBusyAction(false));
       // Save queue is empty, wait for push event
       yield take("PUSH_SAVE_QUEUE");
     }
@@ -100,7 +104,6 @@ export function* pushAnnotationAsync(): Generator<*, *, *> {
     if (saveQueue.length > 0) {
       yield call(sendRequestToServer);
     }
-    yield put(setSaveBusyAction(false));
   }
 }
 
@@ -111,9 +114,30 @@ export function sendRequestWithToken(
   return doWithToken(token => Request.sendJSONReceiveJSON(`${urlWithoutToken}${token}`, data));
 }
 
+// This function returns the first n batches of the provided array, so that the count of
+// all actions in these n batches does not exceed maximumActionCountPerSave
+function sliceAppropriateBatchCount(batches: Array<SaveQueueEntryType>): Array<SaveQueueEntryType> {
+  const slicedBatches = [];
+  let actionCount = 0;
+
+  for (const batch of batches) {
+    const newActionCount = actionCount + batch.actions.length;
+    if (newActionCount <= maximumActionCountPerSave) {
+      actionCount = newActionCount;
+      slicedBatches.push(batch);
+    } else {
+      break;
+    }
+  }
+
+  return slicedBatches;
+}
+
 export function* sendRequestToServer(timestamp: number = Date.now()): Generator<*, *, *> {
-  const saveQueue = yield select(state => state.save.queue);
-  let compactedSaveQueue = compactUpdateActions(saveQueue);
+  const fullSaveQueue = yield select(state => state.save.queue);
+  const saveQueue = sliceAppropriateBatchCount(fullSaveQueue);
+
+  let compactedSaveQueue = compactSaveQueue(saveQueue);
   const { version, type, tracingId } = yield select(state => state.tracing);
   const dataStoreUrl = yield select(state => state.dataset.dataStore.url);
   compactedSaveQueue = addVersionNumbers(compactedSaveQueue, version);
@@ -307,19 +331,18 @@ function compactDeletedTrees(updateActions: Array<UpdateAction>) {
   );
 }
 
-export function compactUpdateActions(
+export function compactUpdateActions(updateActions: Array<UpdateAction>): Array<UpdateAction> {
+  return compactDeletedTrees(
+    compactMovedNodesAndEdges(removeUnrelevantUpdateActions(updateActions)),
+  );
+}
+
+export function compactSaveQueue(
   updateActionsBatches: Array<SaveQueueEntryType>,
 ): Array<SaveQueueEntryType> {
-  const result = updateActionsBatches
-    .map(updateActionsBatch =>
-      _.chain(updateActionsBatch)
-        .cloneDeep()
-        .update("actions", removeUnrelevantUpdateActions)
-        .update("actions", compactMovedNodesAndEdges)
-        .update("actions", compactDeletedTrees)
-        .value(),
-    )
-    .filter(updateActionsBatch => updateActionsBatch.actions.length > 0);
+  const result = updateActionsBatches.filter(
+    updateActionsBatch => updateActionsBatch.actions.length > 0,
+  );
 
   // This part of the code removes all entries from the save queue that consist only of
   // an updateTracing update action, except for the last one
@@ -371,9 +394,15 @@ export function* saveTracingAsync(): Generator<any, any, any> {
     }
     const tracing = yield select(state => state.tracing);
     const flycam = yield select(state => state.flycam);
-    const items = Array.from(yield call(performDiffTracing, prevTracing, tracing, flycam));
+    const items = compactUpdateActions(
+      Array.from(yield call(performDiffTracing, prevTracing, tracing, flycam)),
+    );
     if (items.length > 0) {
-      yield put(pushSaveQueueAction(items));
+      const updateActionChunks = _.chunk(items, maximumActionCountPerBatch);
+
+      for (const updateActionChunk of updateActionChunks) {
+        yield put(pushSaveQueueAction(updateActionChunk));
+      }
     }
     prevTracing = tracing;
   }

app/assets/javascripts/oxalis/store.js

@@ -244,10 +244,16 @@ export type SaveQueueEntryType = {
   actions: Array<UpdateAction>,
 };
 
+export type ProgressInfoType = {
+  +processedActionCount: number,
+  +totalActionCount: number,
+};
+
 export type SaveStateType = {
   +isBusy: boolean,
   +queue: Array<SaveQueueEntryType>,
   +lastSaveTimestamp: number,
+  +progressInfo: ProgressInfoType,
 };
 
 export type FlycamType = {
@@ -410,6 +416,10 @@ export const defaultState: OxalisState = {
     queue: [],
     isBusy: false,
     lastSaveTimestamp: 0,
+    progressInfo: {
+      processedActionCount: 0,
+      totalActionCount: 0,
+    },
   },
   flycam: {
     zoomStep: 1.3,

app/assets/javascripts/oxalis/view/action-bar/save_button.js

@@ -1,9 +1,17 @@
 // @flow
 import React from "react";
+import { connect } from "react-redux";
 import Model from "oxalis/model";
 import ButtonComponent from "oxalis/view/components/button_component";
+import type { OxalisState, ProgressInfoType } from "oxalis/store";
+
+type StateProps = {|
+  progressInfo: ProgressInfoType,
+  isBusy: boolean,
+|};
 
 type Props = {
+  ...StateProps,
   onClick: (SyntheticInputEvent<HTMLButtonElement>) => Promise<*>,
 };
 
@@ -29,29 +37,55 @@ class SaveButton extends React.PureComponent<Props, State> {
 
   savedPollingInterval: number = 0;
   _forceUpdate = () => {
-    this.setState({ isStateSaved: Model.stateSaved() });
+    const isStateSaved = Model.stateSaved();
+    this.setState({
+      isStateSaved,
+    });
   };
 
   getSaveButtonIcon() {
     if (this.state.isStateSaved) {
       return "check";
+    } else if (this.props.isBusy) {
+      return "loading";
     } else {
       return "hourglass";
     }
   }
 
+  shouldShowProgress(): boolean {
+    // For a low action count, the progress info would show only for a very short amount of time
+    return this.props.isBusy && this.props.progressInfo.totalActionCount > 5000;
+  }
+
   render() {
+    const { progressInfo } = this.props;
     return (
       <ButtonComponent
         key="save-button"
         type="primary"
         onClick={this.props.onClick}
         icon={this.getSaveButtonIcon()}
       >
-        Save
+        {this.shouldShowProgress() ? (
+          <React.Fragment>
+            {Math.floor((progressInfo.processedActionCount / progressInfo.totalActionCount) * 100)}{" "}
+            %
+          </React.Fragment>
+        ) : (
+          <React.Fragment>Save</React.Fragment>
+        )}
       </ButtonComponent>
     );
   }
 }
 
-export default SaveButton;
+function mapStateToProps(state: OxalisState): StateProps {
+  const { progressInfo, isBusy } = state.save;
+  return {
+    progressInfo,
+    isBusy,
+  };
+}
+
+export default connect(mapStateToProps)(SaveButton);

app/assets/javascripts/test/reducers/save_reducer.spec.js

@@ -32,6 +32,10 @@ const initialState = {
     isBusy: false,
     queue: [],
     lastSaveTimestamp: 0,
+    progressInfo: {
+      processedActionCount: 0,
+      totalActionCount: 0,
+    },
   },
 };