Tiff Export Job

CHANGELOG.unreleased.md

@@ -16,6 +16,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Added the annotation option "center new nodes" to switch whether newly created nodes should be centered or not. [#4150](https://github.com/scalableminds/webknossos/pull/5112)
 - For webKnossos maintenance, superUsers can now join organizations without being listed as a user there. [#5151](https://github.com/scalableminds/webknossos/pull/5151)
 - Added the possibility to track events for analytics in the backend. [#5156](https://github.com/scalableminds/webknossos/pull/5156)
+- Added the possibility to export binary data as tiff (if long-runnings jobs are enabled). [#5195](https://github.com/scalableminds/webknossos/pull/5195)
 
 ### Changed
 - Change the font to [Titillium Web](http://nta.accademiadiurbino.it/titillium/). [#5161](https://github.com/scalableminds/webknossos/pull/5161)

app/controllers/JobsController.scala

@@ -1,5 +1,8 @@
 package controllers
 
+import java.nio.file.{Files, Paths}
+import java.util.Date
+
 import com.mohiva.play.silhouette.api.Silhouette
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.scalableminds.webknossos.datastore.rpc.{RPC, RPCRequest}
@@ -19,6 +22,7 @@ import slick.lifted.Rep
 import utils.{ObjectId, SQLClient, SQLDAO, WkConf}
 
 import scala.concurrent.{ExecutionContext, Future}
+import scala.util.Random
 
 case class Job(
     _id: ObjectId,
@@ -91,7 +95,7 @@ class JobService @Inject()(wkConf: WkConf, jobDAO: JobDAO, rpc: RPC, analyticsSe
     } else {
       val updateResult = for {
         _ <- Fox.successful(celeryInfosLastUpdated = System.currentTimeMillis())
-        celeryInfoJson <- flowerRpc("/api/tasks").getWithJsonResponse[JsObject]
+        celeryInfoJson <- flowerRpc("/api/tasks?offset=0").getWithJsonResponse[JsObject]
         celeryInfoMap <- celeryInfoJson
           .validate[Map[String, JsObject]] ?~> "Could not validate celery response as json map"
         _ <- Fox.serialCombined(celeryInfoMap.keys.toList)(jobId =>
@@ -171,4 +175,37 @@ class JobsController @Inject()(jobDAO: JobDAO,
       } yield Ok(js)
     }
 
+  def runTiffExportJob(organizationName: String,
+                       dataSetName: String,
+                       layerName: String,
+                       bbox: String): Action[AnyContent] =
+    sil.SecuredAction.async { implicit request =>
+      for {
+        organization <- organizationDAO.findOneByName(organizationName) ?~> Messages("organization.notFound",
+                                                                                     organizationName)
+        _ <- bool2Fox(request.identity._organization == organization._id) ~> FORBIDDEN
+        command = "export_tiff"
+        exportFileName = s"${formatDateForFilename(new Date())}__${dataSetName}__${layerName}__${Random.alphanumeric.take(12).mkString}.zip"
+        commandArgs = Json.obj(
+          "kwargs" -> Json.obj("organization_name" -> organizationName,
+                               "dataset_name" -> dataSetName,
+                               "layer_name" -> layerName,
+                               "bbox" -> bbox,
+                               "export_file_name" -> exportFileName))
+        job <- jobService.runJob(command, commandArgs, request.identity) ?~> "job.couldNotRunTiffExport"
+        js <- jobService.publicWrites(job)
+      } yield Ok(js)
+    }
+
+  def downloadExport(organizationName: String, exportFileName: String): Action[AnyContent] =
+    sil.SecuredAction.async { implicit request =>
+      for {
+        organization <- organizationDAO.findOneByName(organizationName) ?~> Messages("organization.notFound",
+                                                                                     organizationName)
+        _ <- bool2Fox(request.identity._organization == organization._id) ~> FORBIDDEN
+        filePath = Paths.get("binaryData", organizationName, ".export", exportFileName)
+        _ <- bool2Fox(Files.exists(filePath)) ?~> "job.export.fileNotFound"
+      } yield Ok.sendPath(filePath, inline = false)
+    }
+
 }

conf/messages

@@ -313,7 +313,9 @@ initialData.organizationsNotEmpty=There are already organizations present in the
 
 job.notFound = Job with id {0} could not be found.
 job.couldNotRunCubing = Failed to start WKW conversion job.
+job.couldNotRunTiffExport = Failed to start Tiff export job.
 job.disabled = Long-running jobs are not enabled for this webKnossos instance.
+job.export.fileNotFound = Exported file not found. The link may be expired.
 
 agglomerateSkeleton.failed=Could not generate agglomerate skeleton.
 

frontend/javascripts/admin/admin_rest_api.js

@@ -53,7 +53,7 @@ import type { DatasetConfiguration, Tracing } from "oxalis/store";
 import type { NewTask, TaskCreationResponseContainer } from "admin/task/task_create_bulk_view";
 import type { QueryObject } from "admin/task/task_search_form";
 import { V3 } from "libs/mjs";
-import type { Vector3 } from "oxalis/constants";
+import type { Vector3, Vector6 } from "oxalis/constants";
 import type { Versions } from "oxalis/view/version_view";
 import { parseProtoTracing } from "oxalis/model/helpers/proto_helpers";
 import DataLayer from "oxalis/model/data_layer";
@@ -794,18 +794,32 @@ export async function getJobs(): Promise<Array<APIJob>> {
     id: job.id,
     type: job.command,
     datasetName: job.commandArgs.kwargs.dataset_name,
+    exportFileName: job.commandArgs.kwargs.export_file_name,
     state: job.celeryInfo.state,
     createdAt: job.created,
   }));
 }
 
-export async function startJob(
-  jobName: string,
-  organization: string,
+export async function startCubingJob(
+  datasetName: string,
+  organizationName: string,
   scale: Vector3,
 ): Promise<Array<APIJob>> {
   return Request.receiveJSON(
-    `/api/jobs/run/cubing/${organization}/${jobName}?scale=${scale.toString()}`,
+    `/api/jobs/run/cubing/${organizationName}/${datasetName}?scale=${scale.toString()}`,
+  );
+}
+
+export async function startTiffExportJob(
+  datasetName: string,
+  organizationName: string,
+  layerName: string,
+  bbox: Vector6,
+): Promise<Array<APIJob>> {
+  return Request.receiveJSON(
+    `/api/jobs/run/tiffExport/${organizationName}/${datasetName}/${layerName}?bbox=${bbox.join(
+      ",",
+    )}`,
   );
 }
 

frontend/javascripts/admin/dataset/dataset_upload_view.js

@@ -7,7 +7,7 @@ import _ from "lodash";
 import { type RouterHistory, withRouter } from "react-router-dom";
 import type { APITeam, APIDataStore, APIUser, APIDatasetId } from "types/api_flow_types";
 import type { OxalisState } from "oxalis/store";
-import { finishDatasetUpload, createResumableUpload, startJob } from "admin/admin_rest_api";
+import { finishDatasetUpload, createResumableUpload, startCubingJob } from "admin/admin_rest_api";
 import Toast from "libs/toast";
 import * as Utils from "libs/utils";
 import messages from "messages";
@@ -132,7 +132,7 @@ class DatasetUploadView extends React.PureComponent<PropsWithFormAndRouter, Stat
             organization: datasetId.owningOrganization,
             name: datasetId.name,
             initialTeams: formValues.initialTeams.map(team => team.id),
-            needsConversion: formValues.needsConversion,
+            needsConversion: this.state.needsConversion,
           };
 
           finishDatasetUpload(formValues.datastore, uploadInfo).then(
@@ -141,7 +141,7 @@ class DatasetUploadView extends React.PureComponent<PropsWithFormAndRouter, Stat
               trackAction("Upload dataset");
               await Utils.sleep(3000); // wait for 3 seconds so the server can catch up / do its thing
               if (this.state.needsConversion) {
-                await startJob(formValues.name, activeUser.organization, formValues.scale);
+                await startCubingJob(formValues.name, activeUser.organization, formValues.scale);
                 Toast.info(
                   <React.Fragment>
                     The conversion for the uploaded dataset was started.

frontend/javascripts/admin/job/job_list_view.js

@@ -138,19 +138,37 @@ class JobListView extends React.PureComponent<Props, State> {
                 fixed="right"
                 width={150}
                 render={(__, job: APIJob) => (
-                  <span>
-                    {job.state === "SUCCESS" && job.datasetName && this.props.activeUser && (
-                      <Link
-                        to={`/datasets/${this.props.activeUser.organization}/${
-                          job.datasetName
-                        }/view`}
-                        title="View Dataset"
-                      >
-                        <Icon type="eye-o" />
-                        View
-                      </Link>
-                    )}
-                  </span>
+                  <div>
+                    <span>
+                      {job.state === "SUCCESS" &&
+                        job.datasetName &&
+                        !job.exportFileName &&
+                        this.props.activeUser && (
+                          <Link
+                            to={`/datasets/${this.props.activeUser.organization}/${
+                              job.datasetName
+                            }/view`}
+                            title="View Dataset"
+                          >
+                            <Icon type="eye-o" />
+                            View
+                          </Link>
+                        )}
+                    </span>
+                    <span>
+                      {job.state === "SUCCESS" && job.exportFileName && this.props.activeUser && (
+                        <a
+                          href={`/api/jobs/downloadExport/${this.props.activeUser.organization}/${
+                            job.exportFileName
+                          }`}
+                          title="Download"
+                        >
+                          <Icon type="download" />
+                          Download
+                        </a>
+                      )}
+                    </span>
+                  </div>
                 )}
               />
             </Table>

frontend/javascripts/oxalis/view/settings/export_bounding_box_modal.js

@@ -0,0 +1,97 @@
+// @flow
+import { Button, Modal } from "antd";
+import React, { useState } from "react";
+import type { BoundingBoxType } from "oxalis/constants";
+import type { APIDataset } from "types/api_flow_types";
+import { startTiffExportJob } from "admin/admin_rest_api";
+import * as Utils from "libs/utils";
+
+type Props = {
+  destroy: () => void,
+  dataset: APIDataset,
+  boundingBox: BoundingBoxType,
+};
+
+const ExportBoundingBoxModal = ({ destroy, dataset, boundingBox }: Props) => {
+  const [startedExports, setStartedExports] = useState([]);
+
+  const handleClose = () => {
+    destroy();
+  };
+
+  const handleStartExport = layerName => {
+    console.log("start export for", layerName);
+    startTiffExportJob(
+      dataset.name,
+      dataset.owningOrganization,
+      layerName,
+      Utils.computeArrayFromBoundingBox(boundingBox),
+    );
+    setStartedExports(startedExports.concat(layerName));
+  };
+
+  const layerNames = dataset.dataSource.dataLayers.map(layer => {
+    const nameIfColor = layer.category === "color" ? layer.name : null;
+    const nameIfVolume =
+      layer.category === "segmentation" && layer.fallbackLayerInfo && layer.fallbackLayerInfo.name
+        ? layer.fallbackLayerInfo.name
+        : null;
+    return nameIfColor || nameIfVolume;
+  });
+
+  const exportButtonsList = layerNames.map(layerName =>
+    layerName ? (
+      <p>
+        <Button
+          key={layerName}
+          onClick={() => handleStartExport(layerName)}
+          disabled={startedExports.includes(layerName)}
+        >
+          {layerName}
+          {startedExports.includes(layerName) ? " (started)" : null}
+        </Button>
+      </p>
+    ) : null,
+  );
+
+  const downloadHint =
+    startedExports.length > 0 ? (
+      <p>
+        Go to{" "}
+        <a href="/jobs" target="_blank">
+          Jobs Overview Page
+        </a>{" "}
+        for running exports and to download the results.
+      </p>
+    ) : null;
+
+  console.log(dataset.dataSource.dataLayers);
+
+  const bboxText = Utils.computeArrayFromBoundingBox(boundingBox).join(", ");
+
+  return (
+    <Modal
+      title="Export Bounding Box as Tiff Stack"
+      onCancel={handleClose}
+      visible
+      width={500}
+      footer={[
+        <Button key="close" type="primary" onClick={handleClose}>
+          close
+        </Button>,
+      ]}
+    >
+      <p>
+        Data from the selected bounding box at {bboxText} will be exported as tiff stack zip
+        archive.
+      </p>
+      <p>Please select a layer to export:</p>
+
+      {exportButtonsList}
+
+      {downloadHint}
+    </Modal>
+  );
+};
+
+export default ExportBoundingBoxModal;

frontend/javascripts/oxalis/view/settings/setting_input_views.js

@@ -6,6 +6,8 @@ import _ from "lodash";
 import type { Vector3, Vector6 } from "oxalis/constants";
 import * as Utils from "libs/utils";
 
+import features from "features";
+
 type NumberSliderSettingProps = {
   onChange: (value: number) => void,
   value: number,
@@ -254,6 +256,7 @@ type UserBoundingBoxInputProps = {
   tooltipTitle: string,
   onChange: UserBoundingBoxInputUpdate => void,
   onDelete: () => void,
+  onExport: () => void,
 };
 
 type State = {
@@ -338,20 +341,26 @@ export class UserBoundingBoxInput extends React.PureComponent<UserBoundingBoxInp
   render() {
     const { name } = this.state;
     const tooltipStyle = this.state.isValid ? null : { backgroundColor: "red" };
-    const { tooltipTitle, color, isVisible, onDelete } = this.props;
+    const { tooltipTitle, color, isVisible, onDelete, onExport } = this.props;
     const upscaledColor = ((color.map(colorPart => colorPart * 255): any): Vector3);
     const iconStyle = { margin: "auto 0px auto 6px" };
+    const exportButton = features().jobsEnabled ? (
+      <Tooltip title="Export data from this bouding box.">
+        <Icon type="download" onClick={onExport} style={iconStyle} />
+      </Tooltip>
+    ) : null;
     return (
       <React.Fragment>
         <Row style={{ marginBottom: 16 }}>
-          <Col span={22}>
+          <Col span={20}>
             <Switch
               size="small"
               onChange={this.handleVisibilityChange}
               checked={isVisible}
               style={{ margin: "auto 0px" }}
             />
           </Col>
+          <Col span={2}>{exportButton}</Col>
           <Col span={2}>
             <Tooltip title="Delete this bounding box.">
               <Icon type="delete" onClick={onDelete} style={iconStyle} />

frontend/javascripts/oxalis/view/settings/user_settings_view.js

@@ -47,6 +47,9 @@ import Constants, { type ControlMode, ControlModeEnum, type ViewMode } from "oxa
 import Toast from "libs/toast";
 import * as Utils from "libs/utils";
 
+import renderIndependently from "libs/render_independently";
+import ExportBoundingBoxModal from "oxalis/view/settings/export_bounding_box_modal";
+
 const { Panel } = Collapse;
 
 type UserSettingsViewProps = {
@@ -136,6 +139,20 @@ class UserSettingsView extends PureComponent<UserSettingsViewProps> {
     this.props.onChangeBoundingBoxes(updatedUserBoundingBoxes);
   };
 
+  handleExportUserBoundingBox = (id: number) => {
+    const { userBoundingBoxes } = getSomeTracing(this.props.tracing);
+    const selectedBoundingBox = userBoundingBoxes.find(boundingBox => boundingBox.id === id);
+    if (selectedBoundingBox) {
+      renderIndependently(destroy => (
+        <ExportBoundingBoxModal
+          dataset={this.props.dataset}
+          boundingBox={selectedBoundingBox.boundingBox}
+          destroy={destroy}
+        />
+      ));
+    }
+  };
+
   getViewportOptions = () => {
     switch (this.props.viewMode) {
       case Constants.MODE_PLANE_TRACING:
@@ -437,6 +454,7 @@ class UserSettingsView extends PureComponent<UserSettingsViewProps> {
                 isVisible={bb.isVisible}
                 onChange={_.partial(this.handleChangeUserBoundingBox, bb.id)}
                 onDelete={_.partial(this.handleDeleteUserBoundingBox, bb.id)}
+                onExport={_.partial(this.handleExportUserBoundingBox, bb.id)}
               />
             ))}
             <div style={{ display: "inline-block", width: "100%" }}>

frontend/javascripts/types/api_flow_types.js

@@ -545,6 +545,7 @@ export type APIFeatureToggles = {
 export type APIJob = {
   +id: string,
   +datasetName: ?string,
+  +exportFileName: ?string,
   +type: string,
   +state: string,
   +createdAt: number,