Batch nml task upload to avoid uploading too many nmls with one reque…

…st (#6216) * batch nml task upload into batch of max 100 * add changelog entry * Update CHANGELOG.unreleased.md Co-authored-by: Daniel <[email protected]> Co-authored-by: Daniel <[email protected]>
scalableminds · May 18, 2022 · 03db25f · 03db25f
1 parent 48cc2b9
commit 03db25f
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 7 deletions.
diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md
@@ -12,6 +12,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 
 ### Added
 - Added Volume Interpolation feature. When enabled, it suffices to only label every 2nd slice. The skipped slices will be filled automatically by interpolating between the labeled slices. This feature is disabled by default. Note that the feature is even forbidden for tasks by default, but can be enabled/recommended. [#6162](https://github.com/scalableminds/webknossos/pull/6162)
+- Added a batching mechanism to the task creation via NML to support uploading more than 100 NMLs at a time. [#6216](https://github.com/scalableminds/webknossos/pull/6216)
 - Added a long-running job that applies the merging done via a merger mode tracing to a new output dataset. The job is accessible via a button next to the merger mode button once the merger mode is active. [#6086](https://github.com/scalableminds/webknossos/pull/6086)
 - Added support to stream zarr files using the corresponding [zarr spec](https://zarr.readthedocs.io/en/stable/spec/v2.html#storage). [#6144](https://github.com/scalableminds/webknossos/pull/6144)
 

diff --git a/frontend/javascripts/admin/task/task_create_bulk_view.tsx b/frontend/javascripts/admin/task/task_create_bulk_view.tsx
@@ -11,7 +11,7 @@ import Messages from "messages";
 import Toast from "libs/toast";
 const FormItem = Form.Item;
 const { TextArea } = Input;
-const NUM_TASKS_PER_BATCH = 100;
+export const NUM_TASKS_PER_BATCH = 100;
 export type NewTask = {
   readonly boundingBox: BoundingBoxObject | null | undefined;
   readonly dataSet: string;

diff --git a/frontend/javascripts/admin/task/task_create_form_view.tsx b/frontend/javascripts/admin/task/task_create_form_view.tsx
@@ -27,7 +27,7 @@ import type {
   TaskCreationResponse,
   TaskCreationResponseContainer,
 } from "admin/task/task_create_bulk_view";
-import { normFile } from "admin/task/task_create_bulk_view";
+import { normFile, NUM_TASKS_PER_BATCH } from "admin/task/task_create_bulk_view";
 import { Vector3Input, Vector6Input } from "libs/vector_input";
 import type { Vector6 } from "oxalis/constants";
 import {
@@ -338,27 +338,38 @@ class TaskCreateFormView extends React.PureComponent<Props, State> {
         isUploading: true,
       });
       // or create a new one either from the form values or with an NML file
-      let response;
+      let taskResponses: Array<TaskCreationResponse> = [];
+      let warnings: Array<string> = [];
 
       try {
         if (this.state.specificationType === SpecificationEnum.Nml) {
           // Workaround: Antd replaces file objects in the formValues with a wrapper file
           // The original file object is contained in the originFileObj property
           // This is most likely not intentional and may change in a future Antd version
           // @ts-expect-error ts-migrate(7006) FIXME: Parameter 'wrapperFile' implicitly has an 'any' ty... Remove this comment to see the full error message
-          formValues.nmlFiles = formValues.nmlFiles.map((wrapperFile) => wrapperFile.originFileObj);
-          response = await createTaskFromNML(formValues);
+          const nmlFiles = formValues.nmlFiles.map((wrapperFile) => wrapperFile.originFileObj);
+          for (let i = 0; i < nmlFiles.length; i += NUM_TASKS_PER_BATCH) {
+            const batchOfNmls = nmlFiles.slice(i, i + NUM_TASKS_PER_BATCH);
+            formValues.nmlFiles = batchOfNmls;
+            // eslint-disable-next-line no-await-in-loop
+            const response = await createTaskFromNML(formValues);
+            taskResponses = taskResponses.concat(response.tasks);
+            warnings = warnings.concat(response.warnings);
+          }
         } else {
           if (this.state.specificationType !== SpecificationEnum.BaseAnnotation) {
             // Ensure that the base annotation field is null, if the specification mode
             // does not include that field.
             formValues.baseAnnotation = null;
           }
 
-          response = await createTasks([formValues]);
+          ({ tasks: taskResponses, warnings } = await createTasks([formValues]));
         }
 
-        handleTaskCreationResponse(response);
+        handleTaskCreationResponse({
+          tasks: taskResponses,
+          warnings: _.uniq(warnings),
+        });
       } finally {
         this.setState({
           isUploading: false,