Export Volume Annotations

CHANGELOG.unreleased.md

@@ -16,6 +16,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - The time spent on a project is now displayed in the project list. [#5209](https://github.com/scalableminds/webknossos/pull/5209)
 - Added the possibility to export binary data as tiff (if long-runnings jobs are enabled). [#5195](https://github.com/scalableminds/webknossos/pull/5195)
 - Added a link to dataset view mode from annotation mode info tab. [#5262](https://github.com/scalableminds/webknossos/pull/5262)
+- Added the possibility to export also volume annotations as tiff (if long-runnings jobs are enabled). [#5246](https://github.com/scalableminds/webknossos/pull/5246)
 
 ### Changed
 - Measured distances will be shown in voxel space, too. [#5240](https://github.com/scalableminds/webknossos/pull/5240)

app/controllers/JobsController.scala

@@ -11,12 +11,13 @@ import com.scalableminds.webknossos.schema.Tables.{Jobs, JobsRow}
 import com.typesafe.scalalogging.LazyLogging
 import javax.inject.Inject
 import models.analytics.{AnalyticsService, RunJobEvent}
+import models.annotation.TracingStoreRpcClient
 import models.organization.OrganizationDAO
 import models.user.User
 import net.liftweb.common.{Failure, Full}
 import oxalis.security.WkEnv
 import play.api.i18n.Messages
-import play.api.libs.json.{JsObject, JsValue, Json}
+import play.api.libs.json._
 import play.api.mvc.{Action, AnyContent}
 import slick.jdbc.PostgresProfile.api._
 import slick.lifted.Rep
@@ -125,10 +126,12 @@ class JobService @Inject()(wkConf: WkConf, jobDAO: JobDAO, rpc: RPC, analyticsSe
   def runJob(command: String, commandArgs: JsObject, owner: User): Fox[Job] =
     for {
       _ <- bool2Fox(wkConf.Features.jobsEnabled) ?~> "jobs.disabled"
+      argsWrapped = Json.obj("kwargs" -> commandArgs)
       result <- flowerRpc(s"/api/task/async-apply/tasks.$command")
-        .postWithJsonResponse[JsValue, Map[String, JsValue]](commandArgs)
+        .postWithJsonResponse[JsValue, Map[String, JsValue]](argsWrapped)
       celeryJobId <- result("task-id").validate[String].toFox ?~> "Could not parse job submit answer"
-      job = Job(ObjectId.generate, owner._id, command, commandArgs, celeryJobId)
+      argsWithoutToken = Json.obj("kwargs" -> (commandArgs - "webknossos_token"))
+      job = Job(ObjectId.generate, owner._id, command, argsWithoutToken, celeryJobId)
       _ <- jobDAO.insertOne(job)
       _ = analyticsService.track(RunJobEvent(owner, command))
     } yield job
@@ -173,9 +176,7 @@ class JobsController @Inject()(jobDAO: JobDAO,
                                                                                      organizationName)
         _ <- bool2Fox(request.identity._organization == organization._id) ~> FORBIDDEN
         command = "tiff_cubing"
-        commandArgs = Json.obj(
-          "kwargs" -> Json
-            .obj("organization_name" -> organizationName, "dataset_name" -> dataSetName, "scale" -> scale))
+        commandArgs = Json.obj("organization_name" -> organizationName, "dataset_name" -> dataSetName, "scale" -> scale)
 
         job <- jobService.runJob(command, commandArgs, request.identity) ?~> "job.couldNotRunCubing"
         js <- jobService.publicWrites(job)
@@ -184,22 +185,28 @@ class JobsController @Inject()(jobDAO: JobDAO,
 
   def runTiffExportJob(organizationName: String,
                        dataSetName: String,
-                       layerName: String,
-                       bbox: String): Action[AnyContent] =
+                       bbox: String,
+                       layerName: Option[String],
+                       tracingId: Option[String],
+                       tracingVersion: Option[String]): Action[AnyContent] =
     sil.SecuredAction.async { implicit request =>
       for {
         organization <- organizationDAO.findOneByName(organizationName) ?~> Messages("organization.notFound",
                                                                                      organizationName)
         _ <- bool2Fox(request.identity._organization == organization._id) ?~> "job.export.notAllowed.organization" ~> FORBIDDEN
         _ <- jobService.assertTiffExportBoundingBoxLimits(bbox)
         command = "export_tiff"
-        exportFileName = s"${formatDateForFilename(new Date())}__${dataSetName}__${layerName}.zip"
+        exportFileName = s"${formatDateForFilename(new Date())}__${dataSetName}__${tracingId.map(_ => "volume").getOrElse(layerName.getOrElse(""))}.zip"
         commandArgs = Json.obj(
-          "kwargs" -> Json.obj("organization_name" -> organizationName,
-                               "dataset_name" -> dataSetName,
-                               "layer_name" -> layerName,
-                               "bbox" -> bbox,
-                               "export_file_name" -> exportFileName))
+          "organization_name" -> organizationName,
+          "dataset_name" -> dataSetName,
+          "bbox" -> bbox,
+          "webknossos_token" -> TracingStoreRpcClient.webKnossosToken,
+          "export_file_name" -> exportFileName,
+          "layer_name" -> layerName,
+          "volume_tracing_id" -> tracingId,
+          "volume_tracing_version" -> tracingVersion
+        )
         job <- jobService.runJob(command, commandArgs, request.identity) ?~> "job.couldNotRunTiffExport"
         js <- jobService.publicWrites(job)
       } yield Ok(js)

conf/webknossos.latest.routes

@@ -195,4 +195,4 @@ GET           /jobs                                                            c
 GET           /jobs/:id                                                        controllers.JobsController.get(id: String)
 GET           /jobs/:id/downloadExport/:exportFileName                         controllers.JobsController.downloadExport(id: String, exportFileName: String)
 GET           /jobs/run/cubing/:organizationName/:dataSetName                  controllers.JobsController.runCubingJob(organizationName: String, dataSetName: String, scale: String)
-GET           /jobs/run/tiffExport/:organizationName/:dataSetName/:layerName   controllers.JobsController.runTiffExportJob(organizationName: String, dataSetName: String, layerName: String, bbox: String)
+GET           /jobs/run/tiffExport/:organizationName/:dataSetName              controllers.JobsController.runTiffExportJob(organizationName: String, dataSetName: String, bbox: String, layerName: Option[String], tracingId: Option[String], tracingVersion: Option[String])

frontend/javascripts/admin/admin_rest_api.js

@@ -821,6 +821,7 @@ export async function getJobs(): Promise<Array<APIJob>> {
     layerName: job.commandArgs.kwargs.layer_name,
     boundingBox: job.commandArgs.kwargs.bbox,
     exportFileName: job.commandArgs.kwargs.export_file_name,
+    tracingId: job.commandArgs.kwargs.volume_tracing_id,
     state: job.celeryInfo.state || "UNKNOWN",
     createdAt: job.created,
   }));
@@ -839,13 +840,18 @@ export async function startCubingJob(
 export async function startTiffExportJob(
   datasetName: string,
   organizationName: string,
-  layerName: string,
   bbox: Vector6,
+  layerName: ?string,
+  tracingId: ?string,
+  tracingVersion: ?number = null,
 ): Promise<Array<APIJob>> {
+  const layerNameSuffix = layerName != null ? `&layerName=${layerName}` : "";
+  const tracingIdSuffix = tracingId != null ? `&tracingId=${tracingId}` : "";
+  const tracingVersionSuffix = tracingVersion != null ? `&tracingVersion=${tracingVersion}` : "";
   return Request.receiveJSON(
-    `/api/jobs/run/tiffExport/${organizationName}/${datasetName}/${layerName}?bbox=${bbox.join(
+    `/api/jobs/run/tiffExport/${organizationName}/${datasetName}?bbox=${bbox.join(
       ",",
-    )}`,
+    )}${layerNameSuffix}${tracingIdSuffix}${tracingVersionSuffix}`,
   );
 }
 

frontend/javascripts/admin/job/job_list_view.js

@@ -86,9 +86,10 @@ class JobListView extends React.PureComponent<Props, State> {
     if (job.type === "tiff_cubing" && job.datasetName) {
       return <span>{`Tiff to WKW conversion of ${job.datasetName}`}</span>;
     } else if (job.type === "export_tiff" && job.organizationName && job.datasetName) {
+      const layerLabel = job.tracingId != null ? "volume annotation" : job.layerName || "a";
       return (
         <span>
-          Tiff export from {job.layerName || "a"} layer of{" "}
+          Tiff export from {layerLabel} layer of{" "}
           <Link to={`/datasets/${job.organizationName}/${job.datasetName}/view`}>
             {job.datasetName}
           </Link>{" "}

frontend/javascripts/oxalis/view/settings/export_bounding_box_modal.js

@@ -2,54 +2,83 @@
 import { Button, Modal, Alert } from "antd";
 import React, { useState } from "react";
 import type { BoundingBoxType } from "oxalis/constants";
-import type { APIDataset } from "types/api_flow_types";
+import type { VolumeTracing } from "oxalis/store";
+import type { APIDataset, APIDataLayer } from "types/api_flow_types";
 import { startTiffExportJob } from "admin/admin_rest_api";
+import { getResolutionInfo } from "oxalis/model/accessors/dataset_accessor";
+import Model from "oxalis/model";
 import features from "features";
 import * as Utils from "libs/utils";
 
 type Props = {
   destroy: () => void,
+  volumeTracing: ?VolumeTracing,
   dataset: APIDataset,
-  hasVolumeTracing: boolean,
   boundingBox: BoundingBoxType,
 };
 
-const ExportBoundingBoxModal = ({ destroy, dataset, hasVolumeTracing, boundingBox }: Props) => {
+const ExportBoundingBoxModal = ({ destroy, dataset, boundingBox, volumeTracing }: Props) => {
   const [startedExports, setStartedExports] = useState([]);
 
   const handleClose = () => {
     destroy();
   };
 
-  const handleStartExport = layerName => {
-    startTiffExportJob(
+  const exportKey = layerInfos => (layerInfos.layerName || "") + (layerInfos.tracingId || "");
+
+  const handleStartExport = async layerInfos => {
+    setStartedExports(startedExports.concat(exportKey(layerInfos)));
+    if (layerInfos.tracingId) {
+      await Model.ensureSavedState();
+    }
+    await startTiffExportJob(
       dataset.name,
       dataset.owningOrganization,
-      layerName,
       Utils.computeArrayFromBoundingBox(boundingBox),
+      layerInfos.layerName,
+      layerInfos.tracingId,
     );
-    setStartedExports(startedExports.concat(layerName));
   };
 
-  const layerNames = dataset.dataSource.dataLayers.map(layer => {
-    const nameIfFromDataset = layer.category === "color" || !hasVolumeTracing ? layer.name : null;
-    const nameIfVolume =
-      hasVolumeTracing && layer.category === "segmentation" && layer.fallbackLayerInfo != null
-        ? layer.fallbackLayerInfo.name
-        : null;
-    return nameIfFromDataset || nameIfVolume;
+  const hasMag1 = (layer: APIDataLayer) => getResolutionInfo(layer.resolutions).hasIndex(0);
+
+  const allLayerInfos = dataset.dataSource.dataLayers.map(layer => {
+    if (layer.category === "color" || volumeTracing == null)
+      return {
+        displayName: layer.name,
+        layerName: layer.name,
+        tracingId: null,
+        tracingVersion: null,
+        hasMag1: hasMag1(layer),
+      };
+    if (layer.fallbackLayerInfo != null)
+      return {
+        displayName: "Volume annotation with fallback segmentation",
+        layerName: layer.fallbackLayerInfo.name,
+        tracingId: volumeTracing.tracingId,
+        tracingVersion: volumeTracing.version,
+        hasMag1: hasMag1(layer),
+      };
+    return {
+      displayName: "Volume annotation",
+      layerName: null,
+      tracingId: volumeTracing.tracingId,
+      tracingVersion: volumeTracing.version,
+      hasMag1: hasMag1(layer),
+    };
   });
 
-  const exportButtonsList = layerNames.map(layerName =>
-    layerName ? (
+  const exportButtonsList = allLayerInfos.map(layerInfos =>
+    layerInfos ? (
       <p>
         <Button
-          key={layerName}
-          onClick={() => handleStartExport(layerName)}
-          disabled={startedExports.includes(layerName)}
+          key={exportKey(layerInfos)}
+          onClick={() => handleStartExport(layerInfos)}
+          disabled={startedExports.includes(exportKey(layerInfos)) || !layerInfos.hasMag1}
         >
-          {layerName}
-          {startedExports.includes(layerName) ? " (started)" : null}
+          {layerInfos.displayName}
+          {!layerInfos.hasMag1 ? " (resolution 1 missing)" : ""}
+          {startedExports.includes(exportKey(layerInfos)) ? " (started)" : ""}
         </Button>
       </p>
     ) : null,

frontend/javascripts/oxalis/view/settings/user_settings_view.js

@@ -146,7 +146,7 @@ class UserSettingsView extends PureComponent<UserSettingsViewProps> {
       renderIndependently(destroy => (
         <ExportBoundingBoxModal
           dataset={this.props.dataset}
-          hasVolumeTracing={this.props.tracing.volume != null}
+          volumeTracing={this.props.tracing.volume}
           boundingBox={selectedBoundingBox.boundingBox}
           destroy={destroy}
         />

frontend/javascripts/types/api_flow_types.js

@@ -550,6 +550,7 @@ export type APIJob = {
   +datasetName: ?string,
   +exportFileName: ?string,
   +layerName: ?string,
+  +tracingId: ?string,
   +organizationName: ?string,
   +boundingBox: ?string,
   +type: string,

webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala

@@ -117,7 +117,7 @@ trait VolumeTracingBucketHelper
     with BucketKeys
     with VolumeBucketReversionHelper {
 
-  protected val cacheTimeout: FiniteDuration = 20 minutes
+  protected val cacheTimeout: FiniteDuration = 70 minutes
 
   implicit def volumeDataStore: FossilDBClient
   implicit def volumeDataCache: TemporaryVolumeDataStore