From 3d649450d8b2840e8c297e34d569517bb9cbf6c6 Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 8 Mar 2021 11:58:10 +0100 Subject: [PATCH 1/6] [WIP] export volume annotations --- app/controllers/JobsController.scala | 21 +++++++++++---------- conf/application.conf | 2 +- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/app/controllers/JobsController.scala b/app/controllers/JobsController.scala index 915459c70e..6e574e9dd1 100644 --- a/app/controllers/JobsController.scala +++ b/app/controllers/JobsController.scala @@ -11,12 +11,13 @@ import com.scalableminds.webknossos.schema.Tables.{Jobs, JobsRow} import com.typesafe.scalalogging.LazyLogging import javax.inject.Inject import models.analytics.{AnalyticsService, RunJobEvent} +import models.annotation.TracingStoreRpcClient import models.organization.OrganizationDAO import models.user.User import net.liftweb.common.{Failure, Full} import oxalis.security.WkEnv import play.api.i18n.Messages -import play.api.libs.json.{JsObject, JsValue, Json} +import play.api.libs.json._ import play.api.mvc.{Action, AnyContent} import slick.jdbc.PostgresProfile.api._ import slick.lifted.Rep @@ -125,10 +126,12 @@ class JobService @Inject()(wkConf: WkConf, jobDAO: JobDAO, rpc: RPC, analyticsSe def runJob(command: String, commandArgs: JsObject, owner: User): Fox[Job] = for { _ <- bool2Fox(wkConf.Features.jobsEnabled) ?~> "jobs.disabled" + argsWrapped = Json.obj("kwargs" -> commandArgs) + argsWithToken = Json.obj("kwargs" -> (commandArgs ++ Json.obj("webknossos_token" -> TracingStoreRpcClient.webKnossosToken))) result <- flowerRpc(s"/api/task/async-apply/tasks.$command") - .postWithJsonResponse[JsValue, Map[String, JsValue]](commandArgs) + .postWithJsonResponse[JsValue, Map[String, JsValue]](argsWithToken) celeryJobId <- result("task-id").validate[String].toFox ?~> "Could not parse job submit answer" - job = Job(ObjectId.generate, owner._id, command, commandArgs, celeryJobId) + job = Job(ObjectId.generate, owner._id, command, argsWrapped, celeryJobId) _ <- jobDAO.insertOne(job) _ = analyticsService.track(RunJobEvent(owner, command)) } yield job @@ -173,9 +176,7 @@ class JobsController @Inject()(jobDAO: JobDAO, organizationName) _ <- bool2Fox(request.identity._organization == organization._id) ~> FORBIDDEN command = "tiff_cubing" - commandArgs = Json.obj( - "kwargs" -> Json - .obj("organization_name" -> organizationName, "dataset_name" -> dataSetName, "scale" -> scale)) + commandArgs = Json.obj("organization_name" -> organizationName, "dataset_name" -> dataSetName, "scale" -> scale) job <- jobService.runJob(command, commandArgs, request.identity) ?~> "job.couldNotRunCubing" js <- jobService.publicWrites(job) @@ -193,13 +194,13 @@ class JobsController @Inject()(jobDAO: JobDAO, _ <- bool2Fox(request.identity._organization == organization._id) ~> FORBIDDEN _ <- jobService.assertTiffExportBoundingBoxLimits(bbox) command = "export_tiff" - exportFileName = s"${formatDateForFilename(new Date())}__${dataSetName}__${layerName}.zip" - commandArgs = Json.obj( - "kwargs" -> Json.obj("organization_name" -> organizationName, + exportFileName = s"${formatDateForFilename(new Date())}__${dataSetName}__$layerName.zip" + commandArgs = Json.obj("organization_name" -> organizationName, "dataset_name" -> dataSetName, "layer_name" -> layerName, "bbox" -> bbox, - "export_file_name" -> exportFileName)) + "export_file_name" -> exportFileName, + "volume_tracing_id" -> "c100dd44-c56b-486f-9e08-bf40d2ddd3a2") job <- jobService.runJob(command, commandArgs, request.identity) ?~> "job.couldNotRunTiffExport" js <- jobService.publicWrites(job) } yield Ok(js) diff --git a/conf/application.conf b/conf/application.conf index 54da3d30f3..38494cdcf1 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -81,7 +81,7 @@ features { autoBrushReadyDatasets = [] taskReopenAllowedInSeconds = 30 allowDeleteDatasets = true - jobsEnabled = false + jobsEnabled = true # For new users, the dashboard will show a banner which encourages the user to check out the following dataset. # If isDemoInstance == true, `/createExplorative/hybrid/true` is appended to the URL so that a new tracing is opened. # If isDemoInstance == false, `/view` is appended to the URL so that it's opened in view mode (since the user might not From 03a5608c61e7b815ac0866e23dab483192243608 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 9 Mar 2021 08:18:52 +0100 Subject: [PATCH 2/6] changelog --- CHANGELOG.unreleased.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index ef65948306..d1ebc59578 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -14,6 +14,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released - Added CTRL+Scroll for zooming, which enables pinch-to-zoom on some trackpads. [#5224](https://github.com/scalableminds/webknossos/pull/5224) - The time spent on a project is now displayed in the project list. [#5209](https://github.com/scalableminds/webknossos/pull/5209) - Added the possibility to export binary data as tiff (if long-runnings jobs are enabled). [#5195](https://github.com/scalableminds/webknossos/pull/5195) +- Added the possibility to export also volume annotations as tiff (if long-runnings jobs are enabled). [#5246](https://github.com/scalableminds/webknossos/pull/5246) ### Changed - From f9927ac89a7435be9f91f93b03fba44df0ec86c7 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 9 Mar 2021 13:25:01 +0100 Subject: [PATCH 3/6] adapt to exporting volume annotations with wk-worker --- app/controllers/JobsController.scala | 30 ++++--- conf/webknossos.latest.routes | 2 +- frontend/javascripts/admin/admin_rest_api.js | 12 ++- .../javascripts/admin/job/job_list_view.js | 3 +- .../settings/export_bounding_box_modal.js | 78 ++++++++++++++----- .../view/settings/user_settings_view.js | 1 + frontend/javascripts/types/api_flow_types.js | 1 + .../volume/VolumeTracingBucketHelper.scala | 2 +- 8 files changed, 92 insertions(+), 37 deletions(-) diff --git a/app/controllers/JobsController.scala b/app/controllers/JobsController.scala index 6e574e9dd1..ebb8141cb5 100644 --- a/app/controllers/JobsController.scala +++ b/app/controllers/JobsController.scala @@ -127,11 +127,11 @@ class JobService @Inject()(wkConf: WkConf, jobDAO: JobDAO, rpc: RPC, analyticsSe for { _ <- bool2Fox(wkConf.Features.jobsEnabled) ?~> "jobs.disabled" argsWrapped = Json.obj("kwargs" -> commandArgs) - argsWithToken = Json.obj("kwargs" -> (commandArgs ++ Json.obj("webknossos_token" -> TracingStoreRpcClient.webKnossosToken))) result <- flowerRpc(s"/api/task/async-apply/tasks.$command") - .postWithJsonResponse[JsValue, Map[String, JsValue]](argsWithToken) + .postWithJsonResponse[JsValue, Map[String, JsValue]](argsWrapped) celeryJobId <- result("task-id").validate[String].toFox ?~> "Could not parse job submit answer" - job = Job(ObjectId.generate, owner._id, command, argsWrapped, celeryJobId) + argsWithoutToken = Json.obj("kwargs" -> (commandArgs - "webknossos_token")) + job = Job(ObjectId.generate, owner._id, command, argsWithoutToken, celeryJobId) _ <- jobDAO.insertOne(job) _ = analyticsService.track(RunJobEvent(owner, command)) } yield job @@ -185,8 +185,10 @@ class JobsController @Inject()(jobDAO: JobDAO, def runTiffExportJob(organizationName: String, dataSetName: String, - layerName: String, - bbox: String): Action[AnyContent] = + bbox: String, + layerName: Option[String], + tracingId: Option[String], + tracingVersion: Option[String]): Action[AnyContent] = sil.SecuredAction.async { implicit request => for { organization <- organizationDAO.findOneByName(organizationName) ?~> Messages("organization.notFound", @@ -194,13 +196,17 @@ class JobsController @Inject()(jobDAO: JobDAO, _ <- bool2Fox(request.identity._organization == organization._id) ~> FORBIDDEN _ <- jobService.assertTiffExportBoundingBoxLimits(bbox) command = "export_tiff" - exportFileName = s"${formatDateForFilename(new Date())}__${dataSetName}__$layerName.zip" - commandArgs = Json.obj("organization_name" -> organizationName, - "dataset_name" -> dataSetName, - "layer_name" -> layerName, - "bbox" -> bbox, - "export_file_name" -> exportFileName, - "volume_tracing_id" -> "c100dd44-c56b-486f-9e08-bf40d2ddd3a2") + exportFileName = s"${formatDateForFilename(new Date())}__${dataSetName}__${tracingId.map(_ => "volume").getOrElse(layerName.getOrElse(""))}.zip" + commandArgs = Json.obj( + "organization_name" -> organizationName, + "dataset_name" -> dataSetName, + "bbox" -> bbox, + "webknossos_token" -> TracingStoreRpcClient.webKnossosToken, + "export_file_name" -> exportFileName, + "layer_name" -> layerName, + "volume_tracing_id" -> tracingId, + "volume_tracing_version" -> tracingVersion + ) job <- jobService.runJob(command, commandArgs, request.identity) ?~> "job.couldNotRunTiffExport" js <- jobService.publicWrites(job) } yield Ok(js) diff --git a/conf/webknossos.latest.routes b/conf/webknossos.latest.routes index 331f13d257..848978b1b5 100644 --- a/conf/webknossos.latest.routes +++ b/conf/webknossos.latest.routes @@ -197,4 +197,4 @@ GET /jobs c GET /jobs/:id controllers.JobsController.get(id: String) GET /jobs/:id/downloadExport/:exportFileName controllers.JobsController.downloadExport(id: String, exportFileName: String) GET /jobs/run/cubing/:organizationName/:dataSetName controllers.JobsController.runCubingJob(organizationName: String, dataSetName: String, scale: String) -GET /jobs/run/tiffExport/:organizationName/:dataSetName/:layerName controllers.JobsController.runTiffExportJob(organizationName: String, dataSetName: String, layerName: String, bbox: String) +GET /jobs/run/tiffExport/:organizationName/:dataSetName controllers.JobsController.runTiffExportJob(organizationName: String, dataSetName: String, bbox: String, layerName: Option[String], tracingId: Option[String], tracingVersion: Option[String]) diff --git a/frontend/javascripts/admin/admin_rest_api.js b/frontend/javascripts/admin/admin_rest_api.js index 65c3544adf..8f74db4bc7 100644 --- a/frontend/javascripts/admin/admin_rest_api.js +++ b/frontend/javascripts/admin/admin_rest_api.js @@ -797,6 +797,7 @@ export async function getJobs(): Promise> { organizationName: job.commandArgs.kwargs.organization_name, layerName: job.commandArgs.kwargs.layer_name, exportFileName: job.commandArgs.kwargs.export_file_name, + tracingId: job.commandArgs.kwargs.volume_tracing_id, state: job.celeryInfo.state || "UNKNOWN", createdAt: job.created, })); @@ -815,13 +816,18 @@ export async function startCubingJob( export async function startTiffExportJob( datasetName: string, organizationName: string, - layerName: string, bbox: Vector6, + layerName: ?string, + tracingId: ?string, + tracingVersion: ?number = null, ): Promise> { + const layerNameSuffix = layerName != null ? `&layerName=${layerName}` : ""; + const tracingIdSuffix = tracingId != null ? `&tracingId=${tracingId}` : ""; + const tracingVersionSuffix = tracingVersion != null ? `&tracingVersion=${tracingVersion}` : ""; return Request.receiveJSON( - `/api/jobs/run/tiffExport/${organizationName}/${datasetName}/${layerName}?bbox=${bbox.join( + `/api/jobs/run/tiffExport/${organizationName}/${datasetName}?bbox=${bbox.join( ",", - )}`, + )}${layerNameSuffix}${tracingIdSuffix}${tracingVersionSuffix}`, ); } diff --git a/frontend/javascripts/admin/job/job_list_view.js b/frontend/javascripts/admin/job/job_list_view.js index 42ae8e1320..75ed7e19de 100644 --- a/frontend/javascripts/admin/job/job_list_view.js +++ b/frontend/javascripts/admin/job/job_list_view.js @@ -86,9 +86,10 @@ class JobListView extends React.PureComponent { if (job.type === "tiff_cubing" && job.datasetName) { return {`Tiff to WKW conversion of ${job.datasetName}`}; } else if (job.type === "export_tiff" && job.organizationName && job.datasetName) { + const layerLabel = job.tracingId != null ? "volume annotation" : job.layerName || "a"; return ( - Tiff export from {job.layerName || "a"} layer of{" "} + Tiff export from {layerLabel} layer of{" "} {job.datasetName} diff --git a/frontend/javascripts/oxalis/view/settings/export_bounding_box_modal.js b/frontend/javascripts/oxalis/view/settings/export_bounding_box_modal.js index 39a71ec95f..6ea556d1a6 100644 --- a/frontend/javascripts/oxalis/view/settings/export_bounding_box_modal.js +++ b/frontend/javascripts/oxalis/view/settings/export_bounding_box_modal.js @@ -2,53 +2,93 @@ import { Button, Modal, Alert } from "antd"; import React, { useState } from "react"; import type { BoundingBoxType } from "oxalis/constants"; -import type { APIDataset } from "types/api_flow_types"; +import type { VolumeTracing } from "oxalis/store"; +import type { APIDataset, APIDataLayer } from "types/api_flow_types"; import { startTiffExportJob } from "admin/admin_rest_api"; +import Model from "oxalis/model"; import features from "features"; import * as Utils from "libs/utils"; type Props = { destroy: () => void, + volumeTracing: ?VolumeTracing, dataset: APIDataset, boundingBox: BoundingBoxType, }; -const ExportBoundingBoxModal = ({ destroy, dataset, boundingBox }: Props) => { +const ExportBoundingBoxModal = ({ destroy, dataset, boundingBox, volumeTracing }: Props) => { const [startedExports, setStartedExports] = useState([]); const handleClose = () => { destroy(); }; - const handleStartExport = layerName => { - startTiffExportJob( + const exportKey = layerInfos => (layerInfos.layerName || "") + (layerInfos.tracingId || ""); + + const handleStartExport = async layerInfos => { + setStartedExports(startedExports.concat(exportKey(layerInfos))); + if (layerInfos.tracingId) { + await Model.ensureSavedState(); + } + await startTiffExportJob( dataset.name, dataset.owningOrganization, - layerName, Utils.computeArrayFromBoundingBox(boundingBox), + layerInfos.layerName, + layerInfos.tracingId, ); - setStartedExports(startedExports.concat(layerName)); }; - const layerNames = dataset.dataSource.dataLayers.map(layer => { - const nameIfColor = layer.category === "color" ? layer.name : null; - const nameIfVolume = - layer.category === "segmentation" && layer.fallbackLayerInfo && layer.fallbackLayerInfo.name - ? layer.fallbackLayerInfo.name + const hasMag1 = (layer: APIDataLayer) => layer.resolutions.map(r => Math.max(...r)).includes(1); + + const allLayerInfos = dataset.dataSource.dataLayers.map(layer => { + const infosIfFromDataset = + layer.category === "color" || volumeTracing == null + ? { + displayName: layer.name, + layerName: layer.name, + tracingId: null, + tracingVersion: null, + hasMag1: hasMag1(layer), + } + : null; + const infosIfVolumeWithFallback = + layer.category === "segmentation" && + volumeTracing != null && + layer.fallbackLayerInfo && + layer.fallbackLayerInfo.name + ? { + displayName: "Volume Annotation with fallback segmentation", + layerName: layer.fallbackLayerInfo.name, + tracingId: volumeTracing.tracingId, + tracingVersion: volumeTracing.version, + hasMag1: hasMag1(layer), + } + : null; + const infosIfVolumeWithoutFallback = + layer.category === "segmentation" && volumeTracing != null && !layer.fallbackLayerInfo + ? { + displayName: "Volume Annotation", + layerName: null, + tracingId: volumeTracing.tracingId, + tracingVersion: volumeTracing.version, + hasMag1: hasMag1(layer), + } : null; - return nameIfColor || nameIfVolume; + return infosIfFromDataset || infosIfVolumeWithFallback || infosIfVolumeWithoutFallback; }); - const exportButtonsList = layerNames.map(layerName => - layerName ? ( + const exportButtonsList = allLayerInfos.map(layerInfos => + layerInfos ? (

) : null, diff --git a/frontend/javascripts/oxalis/view/settings/user_settings_view.js b/frontend/javascripts/oxalis/view/settings/user_settings_view.js index b813b2baca..11c5f15c49 100644 --- a/frontend/javascripts/oxalis/view/settings/user_settings_view.js +++ b/frontend/javascripts/oxalis/view/settings/user_settings_view.js @@ -146,6 +146,7 @@ class UserSettingsView extends PureComponent { renderIndependently(destroy => ( diff --git a/frontend/javascripts/types/api_flow_types.js b/frontend/javascripts/types/api_flow_types.js index 383c7c900c..ccfeb025e8 100644 --- a/frontend/javascripts/types/api_flow_types.js +++ b/frontend/javascripts/types/api_flow_types.js @@ -550,6 +550,7 @@ export type APIJob = { +datasetName: ?string, +exportFileName: ?string, +layerName: ?string, + +tracingId: ?string, +organizationName: ?string, +type: string, +state: string, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala index 8d9a2f2cbe..f3269479ce 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala @@ -117,7 +117,7 @@ trait VolumeTracingBucketHelper with BucketKeys with VolumeBucketReversionHelper { - protected val cacheTimeout: FiniteDuration = 20 minutes + protected val cacheTimeout: FiniteDuration = 70 minutes implicit def volumeDataStore: FossilDBClient implicit def volumeDataCache: TemporaryVolumeDataStore From 7bf26eafefb0f3bbe99b51d69752c6c909174391 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 9 Mar 2021 13:25:41 +0100 Subject: [PATCH 4/6] disable jobs by default --- conf/application.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/application.conf b/conf/application.conf index 38494cdcf1..54da3d30f3 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -81,7 +81,7 @@ features { autoBrushReadyDatasets = [] taskReopenAllowedInSeconds = 30 allowDeleteDatasets = true - jobsEnabled = true + jobsEnabled = false # For new users, the dashboard will show a banner which encourages the user to check out the following dataset. # If isDemoInstance == true, `/createExplorative/hybrid/true` is appended to the URL so that a new tracing is opened. # If isDemoInstance == false, `/view` is appended to the URL so that it's opened in view mode (since the user might not From 733a548657f37785a4ee6acea6a6cd5e7db5d5fc Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 11 Mar 2021 08:32:29 +0100 Subject: [PATCH 5/6] implement pr feedback --- .../oxalis/view/settings/export_bounding_box_modal.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/javascripts/oxalis/view/settings/export_bounding_box_modal.js b/frontend/javascripts/oxalis/view/settings/export_bounding_box_modal.js index 0c1c2f6449..ba4b224827 100644 --- a/frontend/javascripts/oxalis/view/settings/export_bounding_box_modal.js +++ b/frontend/javascripts/oxalis/view/settings/export_bounding_box_modal.js @@ -5,6 +5,7 @@ import type { BoundingBoxType } from "oxalis/constants"; import type { VolumeTracing } from "oxalis/store"; import type { APIDataset, APIDataLayer } from "types/api_flow_types"; import { startTiffExportJob } from "admin/admin_rest_api"; +import { getResolutionInfo } from "oxalis/model/accessors/dataset_accessor" import Model from "oxalis/model"; import features from "features"; import * as Utils from "libs/utils"; @@ -39,7 +40,7 @@ const ExportBoundingBoxModal = ({ destroy, dataset, boundingBox, volumeTracing } ); }; - const hasMag1 = (layer: APIDataLayer) => layer.resolutions.map(r => Math.max(...r)).includes(1); + const hasMag1 = (layer: APIDataLayer) => getResolutionInfo(layer.resolutions).hasIndex(0); const allLayerInfos = dataset.dataSource.dataLayers.map(layer => { if (layer.category === "color" || volumeTracing == null) From c3b78883107115aa28baa824c2e965b1ce8d2fd8 Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 11 Mar 2021 08:46:26 +0100 Subject: [PATCH 6/6] pretty --- .../oxalis/view/settings/export_bounding_box_modal.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/javascripts/oxalis/view/settings/export_bounding_box_modal.js b/frontend/javascripts/oxalis/view/settings/export_bounding_box_modal.js index ba4b224827..cec2bbb0b9 100644 --- a/frontend/javascripts/oxalis/view/settings/export_bounding_box_modal.js +++ b/frontend/javascripts/oxalis/view/settings/export_bounding_box_modal.js @@ -5,7 +5,7 @@ import type { BoundingBoxType } from "oxalis/constants"; import type { VolumeTracing } from "oxalis/store"; import type { APIDataset, APIDataLayer } from "types/api_flow_types"; import { startTiffExportJob } from "admin/admin_rest_api"; -import { getResolutionInfo } from "oxalis/model/accessors/dataset_accessor" +import { getResolutionInfo } from "oxalis/model/accessors/dataset_accessor"; import Model from "oxalis/model"; import features from "features"; import * as Utils from "libs/utils";