From e1b25a5100fc49259fe7f1c2713dc429682a1419 Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 30 Jan 2019 20:07:42 +0100 Subject: [PATCH 01/31] [WIP] volume tasks --- app/controllers/TaskController.scala | 50 +++++++++++++++---- app/models/annotation/AnnotationService.scala | 16 ++++-- conf/messages | 1 + 3 files changed, 53 insertions(+), 14 deletions(-) diff --git a/app/controllers/TaskController.scala b/app/controllers/TaskController.scala index ed71516a46..96ec871b59 100755 --- a/app/controllers/TaskController.scala +++ b/app/controllers/TaskController.scala @@ -3,7 +3,6 @@ package controllers import java.io.File import javax.inject.Inject - import com.scalableminds.util.geometry.{BoundingBox, Point3D, Vector3D} import com.scalableminds.util.mvc.ResultBox import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext} @@ -21,6 +20,7 @@ import net.liftweb.common.Box import oxalis.security.WkEnv import com.mohiva.play.silhouette.api.Silhouette import com.mohiva.play.silhouette.api.actions.{SecuredRequest, UserAwareRequest} +import com.scalableminds.webknossos.tracingstore.VolumeTracing.VolumeTracing import models.annotation.nml.NmlResults.NmlParseResult import play.api.libs.Files import play.api.i18n.{Messages, MessagesApi, MessagesProvider} @@ -41,7 +41,10 @@ case class TaskParameters( editPosition: Point3D, editRotation: Vector3D, creationInfo: Option[String], - description: Option[String] + description: Option[String], + includeSkeleton: Option[Boolean], + includeVolume: Option[Boolean], + volumeShowFallbackLayer: Option[Boolean], ) object TaskParameters { @@ -93,11 +96,26 @@ class TaskController @Inject()(annotationService: AnnotationService, for { _ <- bool2Fox(request.body.length <= 1000) ?~> "task.create.limitExceeded" result <- createTasks(request.body.map { params => - val tracing = annotationService.createTracingBase(params.dataSet, - params.boundingBox, - params.editPosition, - params.editRotation) - (params, tracing) + val skeletonTracingOpt: Option[SkeletonTracing] = + if (params.includeSkeleton.getOrElse(true)) { + Some(annotationService.createSkeletonTracingBase( + params.dataSet, + params.boundingBox, + params.editPosition, + params.editRotation + )) + } else None + val volumeTracingOpt: Option[VolumeTracing] = + if (params.includeVolume.getOrElse(false)) { + Some(annotationService.createVolumeTracingBase( + params.dataSet, + params.boundingBox, + params.editPosition, + params.editRotation, + params.volumeShowFallbackLayer.getOrElse(false) + )) + } else None + (params, skeletonTracingOpt, volumeTracingOpt) }) } yield result } @@ -121,7 +139,7 @@ class TaskController @Inject()(annotationService: AnnotationService, .parseResults skeletonSuccesses <- Fox.serialCombined(parseResults)(_.toSkeletonSuccessFox) ?~> "task.create.failed" result <- createTasks(skeletonSuccesses.map(s => - (buildFullParams(params, s.skeletonTracing.get, s.fileName, s.description), s.skeletonTracing.get))) + (buildFullParams(params, s.skeletonTracing.get, s.fileName, s.description), s.skeletonTracing, None))) } yield { result } @@ -144,12 +162,22 @@ class TaskController @Inject()(annotationService: AnnotationService, tracing.editPosition, tracing.editRotation, Some(fileName), - description + description, + None, /* task creation from file does not yet support volumes */ + None, + None ) } - def createTasks(requestedTasks: List[(TaskParameters, SkeletonTracing)])( + def createTasks(requestedTasks: List[(TaskParameters, Option[SkeletonTracing], Option[VolumeTracing])])( implicit request: SecuredRequest[WkEnv, _]): Fox[Result] = { + + def assertEachHasEitherSkeletonOrVolume: Fox[Boolean] = { + bool2Fox(requestedTasks.forall { + tuple => tuple._2.isDefined || tuple._3.isDefined + }) + } + def assertAllOnSameDataset(firstDatasetName: String): Fox[String] = { def allOnSameDatasetIter(requestedTasksRest: List[(TaskParameters, SkeletonTracing)], dataSetName: String): Boolean = @@ -172,12 +200,14 @@ class TaskController @Inject()(annotationService: AnnotationService, } yield js for { + _ <- assertEachHasEitherSkeletonOrVolume ?~> "task.create.needsEitherSkeletonOrVolume" firstDatasetName <- requestedTasks.headOption.map(_._1.dataSet).toFox _ <- assertAllOnSameDataset(firstDatasetName) dataSet <- dataSetDAO.findOneByNameAndOrganization(firstDatasetName, request.identity._organization) ?~> Messages( "dataSet.notFound", firstDatasetName) tracingStoreClient <- tracingStoreService.clientFor(dataSet) + // TODO skeletonTracingIds: List[Box[String]] <- tracingStoreClient.saveSkeletonTracings( SkeletonTracings(requestedTasks.map(_._2))) requestedTasksWithTracingIds = requestedTasks zip skeletonTracingIds diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index 9f5c3fa6f4..207fefe1b0 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -250,10 +250,10 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor } } - def createTracingBase(dataSetName: String, - boundingBox: Option[BoundingBox], - startPosition: Point3D, - startRotation: Vector3D) = { + def createSkeletonTracingBase(dataSetName: String, + boundingBox: Option[BoundingBox], + startPosition: Point3D, + startRotation: Vector3D): SkeletonTracing = { val initialNode = NodeDefaults.createInstance.withId(1).withPosition(startPosition).withRotation(startRotation) val initialTree = Tree( 1, @@ -277,6 +277,14 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor ) } + def createVolumeTracingBase(dataSetName: String, + boundingBox: Option[BoundingBox], + startPosition: Point3D, + startRotation: Vector3D, + volumeShowFallbackLayer: Boolean): VolumeTracing = { + + } + def abortInitializedAnnotationOnFailure(initializingAnnotationId: ObjectId, insertedAnnotationBox: Box[Annotation]) = insertedAnnotationBox match { case Full(_) => Fox.successful(()) diff --git a/conf/messages b/conf/messages index e1555a0c8b..39f557bf21 100644 --- a/conf/messages +++ b/conf/messages @@ -200,6 +200,7 @@ mesh.delete.failed=Failed to delete mesh task.create.success=Task successfully created task.create.failed=Failed to create Task task.create.limitExceeded=Cannot create more than 1000 tasks in one request. +task.create.needsEitherSkeletonOrVolume=Each task needs to either be skeleton or volume. task.finished=Task is finished task.assigned=You got a new task task.tooManyOpenOnes=You already have too many open tasks From 4df850fc32c118a6d53b7d4b1472bae164dab904 Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 31 Jan 2019 15:44:45 +0100 Subject: [PATCH 02/31] fix volume task creation + requesting. TODO: download --- .../admin/task/task_create_form_view.js | 2 +- app/controllers/AnnotationIOController.scala | 4 +- app/controllers/TaskController.scala | 90 +++++++++++-------- app/models/annotation/AnnotationService.scala | 70 ++++++++++----- .../annotation/TracingStoreRpcClient.scala | 14 ++- conf/messages | 1 + .../SkeletonTracingController.scala | 10 +-- .../controllers/TracingController.scala | 10 ++- .../controllers/VolumeTracingController.scala | 6 +- ...alableminds.webknossos.tracingstore.routes | 1 + .../proto/SkeletonTracing.proto | 6 +- .../proto/VolumeTracing.proto | 6 +- 12 files changed, 141 insertions(+), 79 deletions(-) diff --git a/app/assets/javascripts/admin/task/task_create_form_view.js b/app/assets/javascripts/admin/task/task_create_form_view.js index c327ce1cae..8da9f1b10a 100644 --- a/app/assets/javascripts/admin/task/task_create_form_view.js +++ b/app/assets/javascripts/admin/task/task_create_form_view.js @@ -163,7 +163,7 @@ class TaskCreateFormView extends React.PureComponent { response = await createTaskFromNML(formValues); } else { - response = await createTasks([formValues]); + response = await createTasks([{...formValues, "typ": "volume"}]); } handleTaskCreationResponse(response); } finally { diff --git a/app/controllers/AnnotationIOController.scala b/app/controllers/AnnotationIOController.scala index a6ba444218..8f6da57628 100755 --- a/app/controllers/AnnotationIOController.scala +++ b/app/controllers/AnnotationIOController.scala @@ -10,7 +10,7 @@ import javax.inject.Inject import com.scalableminds.util.io.{NamedEnumeratorStream, ZipIO} import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext} import com.scalableminds.util.tools.{Fox, FoxImplicits} -import com.scalableminds.webknossos.tracingstore.SkeletonTracing.{SkeletonTracing, SkeletonTracings} +import com.scalableminds.webknossos.tracingstore.SkeletonTracing.{SkeletonTracing, SkeletonTracingOpt, SkeletonTracings} import com.scalableminds.webknossos.tracingstore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.tracingstore.tracings.{ProtoGeometryImplicits, TracingType} import com.typesafe.scalalogging.LazyLogging @@ -145,7 +145,7 @@ class AnnotationIOController @Inject()(nmlWriter: NmlWriter, } yield savedTracingId } mergedSkeletonTracingIdOpt <- Fox.runOptional(skeletonTracings.headOption) { s => - tracingStoreClient.mergeSkeletonTracingsByContents(SkeletonTracings(skeletonTracings), + tracingStoreClient.mergeSkeletonTracingsByContents(SkeletonTracings(skeletonTracings.map(t => SkeletonTracingOpt(Some(t)))), persistTracing = true) } annotation <- annotationService.createFrom(request.identity, diff --git a/app/controllers/TaskController.scala b/app/controllers/TaskController.scala index 96ec871b59..ca377ecf92 100755 --- a/app/controllers/TaskController.scala +++ b/app/controllers/TaskController.scala @@ -7,8 +7,8 @@ import com.scalableminds.util.geometry.{BoundingBox, Point3D, Vector3D} import com.scalableminds.util.mvc.ResultBox import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext} import com.scalableminds.util.tools.{Fox, FoxImplicits, JsonHelper} -import com.scalableminds.webknossos.tracingstore.SkeletonTracing.{SkeletonTracing, SkeletonTracings} -import com.scalableminds.webknossos.tracingstore.tracings.ProtoGeometryImplicits +import com.scalableminds.webknossos.tracingstore.SkeletonTracing.{SkeletonTracing, SkeletonTracingOpt, SkeletonTracings} +import com.scalableminds.webknossos.tracingstore.tracings.{ProtoGeometryImplicits, TracingType} import models.annotation.nml.{NmlResults, NmlService} import models.annotation.{AnnotationService, TracingStoreService} import models.binary.{DataSetDAO, DataSetService} @@ -20,7 +20,7 @@ import net.liftweb.common.Box import oxalis.security.WkEnv import com.mohiva.play.silhouette.api.Silhouette import com.mohiva.play.silhouette.api.actions.{SecuredRequest, UserAwareRequest} -import com.scalableminds.webknossos.tracingstore.VolumeTracing.VolumeTracing +import com.scalableminds.webknossos.tracingstore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings} import models.annotation.nml.NmlResults.NmlParseResult import play.api.libs.Files import play.api.i18n.{Messages, MessagesApi, MessagesProvider} @@ -42,8 +42,7 @@ case class TaskParameters( editRotation: Vector3D, creationInfo: Option[String], description: Option[String], - includeSkeleton: Option[Boolean], - includeVolume: Option[Boolean], + typ: Option[String], volumeShowFallbackLayer: Option[Boolean], ) @@ -95,31 +94,43 @@ class TaskController @Inject()(annotationService: AnnotationService, def create = sil.SecuredAction.async(validateJson[List[TaskParameters]]) { implicit request => for { _ <- bool2Fox(request.body.length <= 1000) ?~> "task.create.limitExceeded" - result <- createTasks(request.body.map { params => - val skeletonTracingOpt: Option[SkeletonTracing] = - if (params.includeSkeleton.getOrElse(true)) { - Some(annotationService.createSkeletonTracingBase( - params.dataSet, - params.boundingBox, - params.editPosition, - params.editRotation - )) - } else None - val volumeTracingOpt: Option[VolumeTracing] = - if (params.includeVolume.getOrElse(false)) { - Some(annotationService.createVolumeTracingBase( - params.dataSet, - params.boundingBox, - params.editPosition, - params.editRotation, - params.volumeShowFallbackLayer.getOrElse(false) - )) - } else None - (params, skeletonTracingOpt, volumeTracingOpt) - }) + skeletonBaseOpts: List[Option[SkeletonTracing]] <- createTaskSkeletonTracingBases(request.body) + volumeBaseOpts: List[Option[VolumeTracing]] <- createTaskVolumeTracingBases(request.body, request.identity._organization) + result <- createTasks((request.body, skeletonBaseOpts, volumeBaseOpts).zipped.toList) } yield result } + def createTaskSkeletonTracingBases(paramsList: List[TaskParameters]): Fox[List[Option[SkeletonTracing]]] = + Fox.successful(paramsList.map { params => + val tracingType = params.typ.flatMap(TracingType.fromString).getOrElse(TracingType.skeleton) + if (tracingType == TracingType.skeleton || tracingType == TracingType.hybrid) { + Some(annotationService.createSkeletonTracingBase( + params.dataSet, + params.boundingBox, + params.editPosition, + params.editRotation + )) + } else None + }) + + def createTaskVolumeTracingBases(paramsList: List[TaskParameters], organizationId: ObjectId) + (implicit ctx: DBAccessContext, m: MessagesProvider): Fox[List[Option[VolumeTracing]]] = { + Fox.serialCombined(paramsList) { params => + val tracingType = params.typ.flatMap(TracingType.fromString).getOrElse(TracingType.skeleton) + if (tracingType == TracingType.volume || tracingType == TracingType.hybrid) { + annotationService.createVolumeTracingBase( + params.dataSet, + organizationId, + params.boundingBox, + params.editPosition, + params.editRotation, + params.volumeShowFallbackLayer.getOrElse(false) + ).map(Some(_)) + } else Fox.successful(None) + } + } + + //Note that from-files tasks do not support volume tracings yet @SuppressWarnings(Array("OptionGet")) //We surpress this warning because we know each skeleton exists due to `toSkeletonSuccessFox` def createFromFiles = sil.SecuredAction.async { implicit request => for { @@ -163,7 +174,6 @@ class TaskController @Inject()(annotationService: AnnotationService, tracing.editRotation, Some(fileName), description, - None, /* task creation from file does not yet support volumes */ None, None ) @@ -179,7 +189,7 @@ class TaskController @Inject()(annotationService: AnnotationService, } def assertAllOnSameDataset(firstDatasetName: String): Fox[String] = { - def allOnSameDatasetIter(requestedTasksRest: List[(TaskParameters, SkeletonTracing)], + def allOnSameDatasetIter(requestedTasksRest: List[(TaskParameters, Option[SkeletonTracing], Option[VolumeTracing])], dataSetName: String): Boolean = requestedTasksRest match { case List() => true @@ -207,19 +217,21 @@ class TaskController @Inject()(annotationService: AnnotationService, "dataSet.notFound", firstDatasetName) tracingStoreClient <- tracingStoreService.clientFor(dataSet) - // TODO - skeletonTracingIds: List[Box[String]] <- tracingStoreClient.saveSkeletonTracings( - SkeletonTracings(requestedTasks.map(_._2))) - requestedTasksWithTracingIds = requestedTasks zip skeletonTracingIds + skeletonTracingIds: List[Box[Option[String]]] <- tracingStoreClient.saveSkeletonTracings( + SkeletonTracings(requestedTasks.map(taskTuple => SkeletonTracingOpt(taskTuple._2)))) + volumeTracingIds: List[Box[Option[String]]] <- tracingStoreClient.saveVolumeTracings( + VolumeTracings(requestedTasks.map(taskTuple => VolumeTracingOpt(taskTuple._3)))) + requestedTasksWithTracingIds = (requestedTasks, skeletonTracingIds, volumeTracingIds).zipped.toList taskObjects: List[Fox[Task]] = requestedTasksWithTracingIds.map(r => - createTaskWithoutAnnotationBase(r._1._1, r._2)) - zipped = (requestedTasks, skeletonTracingIds, taskObjects).zipped.toList + createTaskWithoutAnnotationBase(r._1._1, r._2, r._3)) + zipped = (requestedTasks, skeletonTracingIds.zip(volumeTracingIds), taskObjects).zipped.toList annotationBases = zipped.map( tuple => annotationService.createAnnotationBase( taskFox = tuple._3, request.identity._id, - skeletonTracingIdBox = tuple._2, + skeletonTracingIdBox = tuple._2._1, + volumeTracingIdBox = tuple._2._2, dataSet._id, description = tuple._1._1.description )) @@ -244,10 +256,12 @@ class TaskController @Inject()(annotationService: AnnotationService, case _ => Fox.successful(()) } - private def createTaskWithoutAnnotationBase(params: TaskParameters, skeletonTracingIdBox: Box[String])( + private def createTaskWithoutAnnotationBase(params: TaskParameters, skeletonTracingIdBox: Box[Option[String]], volumeTracingIdBox: Box[Option[String]]) ( implicit request: SecuredRequest[WkEnv, _]): Fox[Task] = for { - _ <- skeletonTracingIdBox.toFox + skeletonIdOpt <- skeletonTracingIdBox.toFox + volumeIdOpt <- volumeTracingIdBox.toFox + _ <- bool2Fox(skeletonIdOpt.isDefined || volumeIdOpt.isDefined) ?~> "task.create.needsEitherSkeletonOrVolume" taskTypeIdValidated <- ObjectId.parse(params.taskTypeId) taskType <- taskTypeDAO.findOne(taskTypeIdValidated) ?~> "taskType.notFound" project <- projectDAO.findOneByName(params.projectName) ?~> Messages("project.notFound", params.projectName) diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index 207fefe1b0..35f84ff7b3 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -80,8 +80,14 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor } }).flatten - private def createVolumeTracing(dataSource: DataSource, withFallback: Boolean): VolumeTracing = { - val fallbackLayer = if (withFallback) { + private def createVolumeTracing( + dataSource: DataSource, + withFallback: Boolean, + boundingBox: Option[BoundingBox] = None, + startPosition: Option[Point3D] = None, + startRotation: Option[Vector3D] = None + ): VolumeTracing = { + val fallbackLayer: Option[SegmentationLayer] = if (withFallback) { dataSource.dataLayers.flatMap { case layer: SegmentationLayer => Some(layer) case _ => None @@ -90,12 +96,12 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor VolumeTracing( None, - dataSource.boundingBox, + boundingBoxToProto(boundingBox.getOrElse(dataSource.boundingBox)), System.currentTimeMillis(), dataSource.id.name, - dataSource.center, - VolumeTracingDefaults.editRotation, - fallbackLayer.map(layer => elementClassToProto(layer.elementClass)).getOrElse(VolumeTracingDefaults.elementClass), + point3DToProto(startPosition.getOrElse(dataSource.center)), + vector3DToProto(startRotation.getOrElse(vector3DFromProto(VolumeTracingDefaults.editRotation))), + elementClassToProto(fallbackLayer.map(layer => layer.elementClass).getOrElse(VolumeTracingDefaults.elementClass)), fallbackLayer.map(_.name), fallbackLayer.map(_.largestSegmentId).getOrElse(VolumeTracingDefaults.largestSegmentId), 0, @@ -212,13 +218,13 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor } yield result def tracingFromBase(annotationBase: Annotation, dataSet: DataSet)(implicit ctx: DBAccessContext, - m: MessagesProvider): Fox[String] = + m: MessagesProvider): Fox[(Option[String], Option[String])] = for { dataSource <- bool2Fox(dataSet.isUsable) ?~> Messages("dataSet.notImported", dataSet.name) tracingStoreClient <- tracingStoreService.clientFor(dataSet) - skeletonTracingId <- annotationBase.skeletonTracingId.toFox - newTracingId <- tracingStoreClient.duplicateSkeletonTracing(skeletonTracingId) - } yield newTracingId + newSkeletonId: Option[String] <- Fox.runOptional(annotationBase.skeletonTracingId)(skeletonId => tracingStoreClient.duplicateSkeletonTracing(skeletonId)) + newVolumeId: Option[String] <- Fox.runOptional(annotationBase.volumeTracingId)(volumeId => tracingStoreClient.duplicateVolumeTracing(volumeId)) + } yield (newSkeletonId, newVolumeId) def createAnnotationFor(user: User, task: Task, initializingAnnotationId: ObjectId)( implicit m: MessagesProvider, @@ -227,11 +233,12 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor for { dataSetName <- dataSetDAO.getNameById(annotation._dataSet)(GlobalAccessContext) ?~> "dataSet.notFound" dataSet <- dataSetDAO.findOne(annotation._dataSet) ?~> "dataSet.noAccess" - newTracingId <- tracingFromBase(annotation, dataSet) ?~> "Failed to use annotation base as template." + (newSkeletonId, newVolumeId) <- tracingFromBase(annotation, dataSet) ?~> "Failed to use annotation base as template." newAnnotation = annotation.copy( _id = initializingAnnotationId, _user = user._id, - skeletonTracingId = Some(newTracingId), + skeletonTracingId = newSkeletonId, + volumeTracingId = newVolumeId, state = Active, typ = AnnotationType.Task, created = System.currentTimeMillis, @@ -278,11 +285,25 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor } def createVolumeTracingBase(dataSetName: String, + organizationId: ObjectId, boundingBox: Option[BoundingBox], startPosition: Point3D, startRotation: Vector3D, - volumeShowFallbackLayer: Boolean): VolumeTracing = { - + volumeShowFallbackLayer: Boolean)(implicit ctx: DBAccessContext, + m: MessagesProvider): Fox[VolumeTracing] = { + for { + dataSet <- dataSetDAO.findOneByNameAndOrganization(dataSetName, organizationId) ?~> Messages("dataset.notFound", dataSetName) + dataSource <- dataSetService.dataSourceFor(dataSet).flatMap(_.toUsable) + volumeTracing = createVolumeTracing( + dataSource, + withFallback = volumeShowFallbackLayer, + boundingBox = boundingBox.flatMap { box => + if (box.isEmpty) None else Some(box) + }, + startPosition = Some(startPosition), + startRotation = Some(startRotation) + ) + } yield volumeTracing } def abortInitializedAnnotationOnFailure(initializingAnnotationId: ObjectId, insertedAnnotationBox: Box[Annotation]) = @@ -294,22 +315,25 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor def createAnnotationBase( taskFox: Fox[Task], userId: ObjectId, - skeletonTracingIdBox: Box[String], + skeletonTracingIdBox: Box[Option[String]], + volumeTracingIdBox: Box[Option[String]], dataSetId: ObjectId, description: Option[String] )(implicit ctx: DBAccessContext) = for { task <- taskFox + skeletonIdOpt <- skeletonTracingIdBox.toFox + volumeIdOpt <- volumeTracingIdBox.toFox + _ <- bool2Fox(skeletonIdOpt.isDefined || volumeIdOpt.isDefined) ?~> "annotation.needsAtleastOne" taskType <- taskTypeDAO.findOne(task._taskType)(GlobalAccessContext) - skeletonTracingId <- skeletonTracingIdBox.toFox project <- projectDAO.findOne(task._project) annotationBase = Annotation(ObjectId.generate, dataSetId, Some(task._id), project._team, userId, - Some(skeletonTracingId), - None, + skeletonIdOpt, + volumeIdOpt, description.getOrElse(""), typ = AnnotationType.TracingBase) _ <- annotationDAO.insertOne(annotationBase) @@ -380,12 +404,12 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor List[Option[Task]], List[String])]] = { - def getTracings(dataSetId: ObjectId, tracingIds: List[String]) = + def getTracings(dataSetId: ObjectId, tracingIds: List[String]): Fox[List[SkeletonTracing]] = for { dataSet <- dataSetDAO.findOne(dataSetId) tracingStoreClient <- tracingStoreService.clientFor(dataSet) tracingContainers <- Fox.serialCombined(tracingIds.grouped(1000).toList)(tracingStoreClient.getSkeletonTracings) - } yield tracingContainers.flatMap(_.tracings) + } yield tracingContainers.flatMap(_.tracings).flatMap(_.tracing) def getUsers(userIds: List[ObjectId]) = Fox.serialCombined(userIds)(userService.findOneById(_, useCache = true)) @@ -456,8 +480,10 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor task <- taskFor(annotation) annotationBase <- baseForTask(task._id) dataSet <- dataSetDAO.findOne(annotationBase._dataSet)(GlobalAccessContext) ?~> "dataSet.notFound" - newTracingId <- tracingFromBase(annotationBase, dataSet) - _ <- annotationDAO.updateSkeletonTracingId(annotation._id, newTracingId) + (newSkeletonIdOpt, newVolumeIdOpt) <- tracingFromBase(annotationBase, dataSet) + _ <- Fox.bool2Fox(newSkeletonIdOpt.isDefined || newVolumeIdOpt.isDefined) ?~> "annotation.needsEitherSkeletonOrVolume" + _ <- Fox.runOptional(newSkeletonIdOpt)(newSkeletonId => annotationDAO.updateSkeletonTracingId(annotation._id, newSkeletonId)) + _ <- Fox.runOptional(newVolumeIdOpt)(newVolumeId => annotationDAO.updateVolumeTracingId(annotation._id, newVolumeId)) } yield () case _ if !annotation.skeletonTracingId.isDefined => Fox.failure("annotation.revert.skeletonOnly") diff --git a/app/models/annotation/TracingStoreRpcClient.scala b/app/models/annotation/TracingStoreRpcClient.scala index ec823e3558..cb268907a6 100644 --- a/app/models/annotation/TracingStoreRpcClient.scala +++ b/app/models/annotation/TracingStoreRpcClient.scala @@ -5,10 +5,11 @@ import java.io.File import akka.stream.scaladsl.Source import akka.util.ByteString import com.scalableminds.webknossos.tracingstore.SkeletonTracing.{SkeletonTracing, SkeletonTracings} -import com.scalableminds.webknossos.tracingstore.VolumeTracing.VolumeTracing +import com.scalableminds.webknossos.tracingstore.VolumeTracing.{VolumeTracing, VolumeTracings} import com.scalableminds.webknossos.tracingstore.tracings.TracingSelector import com.scalableminds.webknossos.datastore.rpc.RPC import com.scalableminds.util.tools.JsonHelper.boxFormat +import com.scalableminds.util.tools.JsonHelper.optionFormat import com.scalableminds.util.tools.Fox import com.typesafe.scalalogging.LazyLogging import models.binary.{DataSet, DataStoreRpcClient} @@ -48,11 +49,18 @@ class TracingStoreRpcClient(tracingStore: TracingStore, dataSet: DataSet, rpc: R .postProtoWithJsonResponse[SkeletonTracing, String](tracing) } - def saveSkeletonTracings(tracings: SkeletonTracings): Fox[List[Box[String]]] = { + def saveSkeletonTracings(tracings: SkeletonTracings): Fox[List[Box[Option[String]]]] = { logger.debug("Called to save SkeletonTracings." + baseInfo) rpc(s"${tracingStore.url}/tracings/skeleton/saveMultiple") .addQueryString("token" -> TracingStoreRpcClient.webKnossosToken) - .postProtoWithJsonResponse[SkeletonTracings, List[Box[String]]](tracings) + .postProtoWithJsonResponse[SkeletonTracings, List[Box[Option[String]]]](tracings) + } + + def saveVolumeTracings(tracings: VolumeTracings): Fox[List[Box[Option[String]]]] = { + logger.debug("Called to save VolumeTracings." + baseInfo) + rpc(s"${tracingStore.url}/tracings/volume/saveMultiple") + .addQueryString("token" -> TracingStoreRpcClient.webKnossosToken) + .postProtoWithJsonResponse[VolumeTracings, List[Box[Option[String]]]](tracings) } def duplicateSkeletonTracing(skeletonTracingId: String, versionString: Option[String] = None): Fox[String] = { diff --git a/conf/messages b/conf/messages index 39f557bf21..7a8dcfe8c6 100644 --- a/conf/messages +++ b/conf/messages @@ -183,6 +183,7 @@ annotation.transferee.noDataSetAccess=Cannot transfer annotation to a user who h annotation.timelogging.read.failed=Time annotation.write.failed=Could not convert annotation to json annotation.noSkeleton=No skeleton tracing found for this annotation +annotation.needsEitherSkeletonOrVolume=Annotation needs at least one of skeleton or volume annotation.makeHybrid.explorationalsOnly=Could not convert annotation to hybrid annotation because it is only allowed for explorational annotations. annotation.makeHybrid.failed=Could not convert to hybrid. annotation.makeHybrid.alreadyHybrid=Could not convert annotation to hybrid annotation because it is already a hybrid annotation. diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala index 14dafb0075..b8b416c804 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala @@ -1,7 +1,7 @@ package com.scalableminds.webknossos.tracingstore.controllers import com.google.inject.Inject -import com.scalableminds.webknossos.tracingstore.SkeletonTracing.{SkeletonTracing, SkeletonTracings} +import com.scalableminds.webknossos.tracingstore.SkeletonTracing.{SkeletonTracing, SkeletonTracingOpt, SkeletonTracings} import com.scalableminds.webknossos.datastore.services.{AccessTokenService, UserAccessRequest} import com.scalableminds.webknossos.tracingstore.{TracingStoreAccessTokenService, TracingStoreWkRpcClient} import com.scalableminds.webknossos.tracingstore.tracings.TracingSelector @@ -21,16 +21,16 @@ class SkeletonTracingController @Inject()(val tracingService: SkeletonTracingSer implicit val tracingsCompanion = SkeletonTracings - implicit def packMultiple(tracings: List[SkeletonTracing]): SkeletonTracings = SkeletonTracings(tracings) + implicit def packMultiple(tracings: List[SkeletonTracing]): SkeletonTracings = SkeletonTracings(tracings.map(t => SkeletonTracingOpt(Some(t)))) - implicit def unpackMultiple(tracings: SkeletonTracings): List[SkeletonTracing] = tracings.tracings.toList + implicit def unpackMultiple(tracings: SkeletonTracings): List[Option[SkeletonTracing]] = tracings.tracings.toList.map(_.tracing) def mergedFromContents(persist: Boolean) = Action.async(validateProto[SkeletonTracings]) { implicit request => log { accessTokenService.validateAccess(UserAccessRequest.webknossos) { AllowRemoteOrigin { - val tracings = request.body.tracings - val mergedTracing = tracingService.merge(tracings) + val tracings: List[Option[SkeletonTracing]] = request.body + val mergedTracing = tracingService.merge(tracings.flatten) tracingService.save(mergedTracing, None, mergedTracing.version, toCache = !persist).map { newId => Ok(Json.toJson(newId)) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala index c51c4addd7..fb2b74913e 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala @@ -6,6 +6,7 @@ import com.scalableminds.webknossos.datastore.services.{AccessTokenService, User import com.scalableminds.webknossos.tracingstore.{TracingStoreAccessTokenService, TracingStoreWkRpcClient} import com.scalableminds.webknossos.tracingstore.tracings.{TracingSelector, TracingService, UpdateAction, UpdateActionGroup} import com.scalableminds.util.tools.JsonHelper.boxFormat +import com.scalableminds.util.tools.JsonHelper.optionFormat import net.liftweb.common.Failure import play.api.i18n.Messages import play.api.libs.json.{Json, Reads} @@ -28,7 +29,7 @@ trait TracingController[T <: GeneratedMessage with Message[T], Ts <: GeneratedMe implicit val tracingsCompanion: GeneratedMessageCompanion[Ts] - implicit def unpackMultiple(tracings: Ts): List[T] + implicit def unpackMultiple(tracings: Ts): List[Option[T]] implicit def packMultiple(tracings: List[T]): Ts @@ -55,8 +56,11 @@ trait TracingController[T <: GeneratedMessage with Message[T], Ts <: GeneratedMe log { accessTokenService.validateAccess(UserAccessRequest.webknossos) { AllowRemoteOrigin { - val savedIds = Fox.sequence(request.body.map { tracing => - tracingService.save(tracing, None, 0, toCache = false) + val savedIds = Fox.sequence(request.body.map { tracingOpt: Option[T] => + tracingOpt match { + case Some(tracing) => tracingService.save(tracing, None, 0, toCache = false).map(Some(_)) + case _ => Fox.successful(None) + } }) savedIds.map(id => Ok(Json.toJson(id))) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index d3edf5ab13..dc32f83ffd 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -3,7 +3,7 @@ package com.scalableminds.webknossos.tracingstore.controllers import akka.stream.scaladsl.Source import com.google.inject.Inject import com.scalableminds.webknossos.datastore.DataStoreConfig -import com.scalableminds.webknossos.tracingstore.VolumeTracing.{VolumeTracing, VolumeTracings} +import com.scalableminds.webknossos.tracingstore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings} import com.scalableminds.webknossos.datastore.models.WebKnossosDataRequest import com.scalableminds.webknossos.datastore.services.{AccessTokenService, UserAccessRequest} import com.scalableminds.webknossos.tracingstore.{TracingStoreAccessTokenService, TracingStoreConfig, TracingStoreWkRpcClient} @@ -28,9 +28,9 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService implicit val tracingsCompanion = VolumeTracings - implicit def packMultiple(tracings: List[VolumeTracing]): VolumeTracings = VolumeTracings(tracings) + implicit def packMultiple(tracings: List[VolumeTracing]): VolumeTracings = VolumeTracings(tracings.map(t => VolumeTracingOpt(Some(t)))) - implicit def unpackMultiple(tracings: VolumeTracings): List[VolumeTracing] = tracings.tracings.toList + implicit def unpackMultiple(tracings: VolumeTracings): List[Option[VolumeTracing]] = tracings.tracings.toList.map(_.tracing) override def freezeVersions = config.Tracingstore.freezeVolumeVersions diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index 2d74d529d1..95918038da 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -7,6 +7,7 @@ GET /health @com.scalablemi # Volume tracings POST /volume/save @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.save +POST /volume/saveMultiple @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.saveMultiple POST /volume/:tracingId/initialData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.initialData(tracingId: String) GET /volume/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.get(tracingId: String, version: Option[Long]) POST /volume/:tracingId/update @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.update(tracingId: String) diff --git a/webknossos-tracingstore/proto/SkeletonTracing.proto b/webknossos-tracingstore/proto/SkeletonTracing.proto index cc5ed1f32e..f83f3600fc 100644 --- a/webknossos-tracingstore/proto/SkeletonTracing.proto +++ b/webknossos-tracingstore/proto/SkeletonTracing.proto @@ -70,6 +70,10 @@ message SkeletonTracing { repeated TreeGroup treeGroups = 11; } +message SkeletonTracingOpt { + optional SkeletonTracing tracing = 1; +} + message SkeletonTracings { - repeated SkeletonTracing tracings = 1; + repeated SkeletonTracingOpt tracings = 1; } diff --git a/webknossos-tracingstore/proto/VolumeTracing.proto b/webknossos-tracingstore/proto/VolumeTracing.proto index 1516f32aef..ca40eaac7c 100644 --- a/webknossos-tracingstore/proto/VolumeTracing.proto +++ b/webknossos-tracingstore/proto/VolumeTracing.proto @@ -27,6 +27,10 @@ message VolumeTracing { optional BoundingBox userBoundingBox = 12; } +message VolumeTracingOpt { + optional VolumeTracing tracing = 1; +} + message VolumeTracings { - repeated VolumeTracing tracings = 1; + repeated VolumeTracingOpt tracings = 1; } From 6823756b2dc0bdb3c2cc26dee85a8f7def556978 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Thu, 31 Jan 2019 17:19:13 +0100 Subject: [PATCH 03/31] allow to choose volume tasks via UI for simple task creation --- .../admin/task/task_create_form_view.js | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/app/assets/javascripts/admin/task/task_create_form_view.js b/app/assets/javascripts/admin/task/task_create_form_view.js index 8da9f1b10a..32f6ff3eb6 100644 --- a/app/assets/javascripts/admin/task/task_create_form_view.js +++ b/app/assets/javascripts/admin/task/task_create_form_view.js @@ -160,10 +160,10 @@ class TaskCreateFormView extends React.PureComponent { formValues.nmlFiles = formValues.nmlFiles.map( wrapperFile => wrapperFile.originFileObj, ); - - response = await createTaskFromNML(formValues); + const { typ, ...formValuesWithoutTyp } = formValues; + response = await createTaskFromNML(formValuesWithoutTyp); } else { - response = await createTasks([{...formValues, "typ": "volume"}]); + response = await createTasks([formValues]); } handleTaskCreationResponse(response); } finally { @@ -348,6 +348,21 @@ class TaskCreateFormView extends React.PureComponent { )} + + {getFieldDecorator("typ", { + initialValue: "skeleton", + })( + + + Skeleton + + + Volume + + , + )} + + {getFieldDecorator("editPosition", { rules: [{ required: true }], From 71177348e6971804b9908c0e32e7cafdabcd7a83 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Fri, 1 Feb 2019 10:02:33 +0100 Subject: [PATCH 04/31] enforce page reload when switching from/to/between volume tasks; show tracing type in dashboard --- app/assets/javascripts/admin/api_flow_types.js | 1 + .../javascripts/admin/task/task_create_bulk_view.js | 1 + .../javascripts/dashboard/dashboard_task_list_view.js | 2 ++ app/assets/javascripts/oxalis/api/api_latest.js | 11 ++++++++--- app/assets/javascripts/oxalis/view/action_bar_view.js | 10 +++++++--- .../test/fixtures/tasktracing_server_objects.js | 1 + 6 files changed, 20 insertions(+), 6 deletions(-) diff --git a/app/assets/javascripts/admin/api_flow_types.js b/app/assets/javascripts/admin/api_flow_types.js index b6a22381f2..3d80c613c0 100644 --- a/app/assets/javascripts/admin/api_flow_types.js +++ b/app/assets/javascripts/admin/api_flow_types.js @@ -280,6 +280,7 @@ export type APITask = { +team: string, +tracingTime: ?number, +type: APITaskType, + +typ: "skeleton" | "volume", +directLinks?: Array, }; diff --git a/app/assets/javascripts/admin/task/task_create_bulk_view.js b/app/assets/javascripts/admin/task/task_create_bulk_view.js index 17e5482ca3..f29ba1680a 100644 --- a/app/assets/javascripts/admin/task/task_create_bulk_view.js +++ b/app/assets/javascripts/admin/task/task_create_bulk_view.js @@ -42,6 +42,7 @@ export type NewTask = { +taskTypeId: string, +csvFile?: File, +nmlFiles?: File, + +typ?: "skeleton" | "volume", }; export type TaskCreationResponse = { diff --git a/app/assets/javascripts/dashboard/dashboard_task_list_view.js b/app/assets/javascripts/dashboard/dashboard_task_list_view.js index 4f3eb7e8f8..0c2aba69cd 100644 --- a/app/assets/javascripts/dashboard/dashboard_task_list_view.js +++ b/app/assets/javascripts/dashboard/dashboard_task_list_view.js @@ -386,6 +386,8 @@ class DashboardTaskListView extends React.PureComponent {task.type.summary} () + {task.annotation.tracing.skeleton == null ? null : skeleton} + {task.annotation.tracing.volume == null ? null : volume} {task.type.settings.allowedModes.map(mode => ( {mode} ))} diff --git a/app/assets/javascripts/oxalis/api/api_latest.js b/app/assets/javascripts/oxalis/api/api_latest.js index ba7733f5ad..91a3ed9158 100644 --- a/app/assets/javascripts/oxalis/api/api_latest.js +++ b/app/assets/javascripts/oxalis/api/api_latest.js @@ -371,6 +371,10 @@ class TracingApi { const state = Store.getState(); const { annotationType, annotationId } = state.tracing; const { task } = state; + if (task == null) { + // Satisfy flow + throw new Error("Cannot find task to finish."); + } await Model.save(); await finishAnnotation(annotationId, annotationType); @@ -378,16 +382,17 @@ class TracingApi { const annotation = await requestTask(); const isDifferentDataset = state.dataset.name !== annotation.dataSetName; - const isDifferentTaskType = annotation.task.type.id !== Utils.__guard__(task, x => x.type.id); + const isDifferentTaskType = annotation.task.type.id !== task.type.id; + const involvesVolumeTask = state.tracing.volume != null || annotation.tracing.volume != null; - const currentScript = task != null && task.script != null ? task.script.gist : null; + const currentScript = task.script != null ? task.script.gist : null; const nextScript = annotation.task.script != null ? annotation.task.script.gist : null; const isDifferentScript = currentScript !== nextScript; const newTaskUrl = `/annotations/${annotation.typ}/${annotation.id}`; // In some cases the page needs to be reloaded, in others the tracing can be hot-swapped - if (isDifferentDataset || isDifferentTaskType || isDifferentScript) { + if (isDifferentDataset || isDifferentTaskType || isDifferentScript || involvesVolumeTask) { location.href = newTaskUrl; } else { await this.restart(annotation.typ, annotation.id, ControlModeEnum.TRACE); diff --git a/app/assets/javascripts/oxalis/view/action_bar_view.js b/app/assets/javascripts/oxalis/view/action_bar_view.js index 9cfc910f1a..f62930323d 100644 --- a/app/assets/javascripts/oxalis/view/action_bar_view.js +++ b/app/assets/javascripts/oxalis/view/action_bar_view.js @@ -41,6 +41,7 @@ type StateProps = {| hasVolume: boolean, hasSkeleton: boolean, showVersionRestore: boolean, + isReadOnly: boolean, |}; type OwnProps = {| layoutProps: LayoutProps, @@ -124,7 +125,7 @@ class ActionBarView extends React.PureComponent { /> ); - const readonlyDropdown = ( + const viewDropdown = ( {layoutMenu}}> @@ -138,11 +139,13 @@ class ActionBarView extends React.PureComponent { {isTraceMode && !this.props.showVersionRestore ? ( ) : ( - readonlyDropdown + viewDropdown )} {this.props.showVersionRestore ? VersionRestoreWarning : null} - {this.props.hasVolume && isVolumeSupported ? : null} + {!this.props.isReadOnly && this.props.hasVolume && isVolumeSupported ? ( + + ) : null} {this.props.hasSkeleton && isTraceMode ? : null} {isTraceMode ? null : this.renderStartTracingButton()} @@ -163,6 +166,7 @@ const mapStateToProps = (state: OxalisState): StateProps => ({ showVersionRestore: state.uiInformation.showVersionRestore, hasVolume: state.tracing.volume != null, hasSkeleton: state.tracing.skeleton != null, + isReadOnly: !state.tracing.restrictions.allowUpdate, }); export default connect(mapStateToProps)(ActionBarView); diff --git a/app/assets/javascripts/test/fixtures/tasktracing_server_objects.js b/app/assets/javascripts/test/fixtures/tasktracing_server_objects.js index 4804ce3c80..486355b7fe 100644 --- a/app/assets/javascripts/test/fixtures/tasktracing_server_objects.js +++ b/app/assets/javascripts/test/fixtures/tasktracing_server_objects.js @@ -49,6 +49,7 @@ export const annotation: APIAnnotation = { formattedHash: "9c67ec", projectName: "sampleProject", team: "Connectomics department", + typ: "skeleton", type: { id: "5b1e45faa000009d00abc2c6", summary: "sampleTaskType", From 5f8ea8ae8be3ad41c74cc12cf83e8222499c755f Mon Sep 17 00:00:00 2001 From: Florian M Date: Fri, 1 Feb 2019 13:22:35 +0100 Subject: [PATCH 05/31] add tracing type to task type --- app/controllers/TaskController.scala | 69 ++++++++++--------- app/controllers/TaskTypeController.scala | 7 +- app/models/annotation/AnnotationService.scala | 8 +-- app/models/task/TaskType.scala | 18 +++-- .../038-add-tasktype-tracingtype.sql | 13 ++++ .../038-add-tasktype-tracingtype.sql | 11 +++ conf/messages | 1 + tools/postgres/schema.sql | 4 +- 8 files changed, 84 insertions(+), 47 deletions(-) create mode 100644 conf/evolutions/038-add-tasktype-tracingtype.sql create mode 100644 conf/evolutions/reversions/038-add-tasktype-tracingtype.sql diff --git a/app/controllers/TaskController.scala b/app/controllers/TaskController.scala index ca377ecf92..6bffe2ccd9 100755 --- a/app/controllers/TaskController.scala +++ b/app/controllers/TaskController.scala @@ -41,9 +41,7 @@ case class TaskParameters( editPosition: Point3D, editRotation: Vector3D, creationInfo: Option[String], - description: Option[String], - typ: Option[String], - volumeShowFallbackLayer: Option[Boolean], + description: Option[String] ) object TaskParameters { @@ -100,35 +98,43 @@ class TaskController @Inject()(annotationService: AnnotationService, } yield result } - def createTaskSkeletonTracingBases(paramsList: List[TaskParameters]): Fox[List[Option[SkeletonTracing]]] = - Fox.successful(paramsList.map { params => - val tracingType = params.typ.flatMap(TracingType.fromString).getOrElse(TracingType.skeleton) - if (tracingType == TracingType.skeleton || tracingType == TracingType.hybrid) { - Some(annotationService.createSkeletonTracingBase( - params.dataSet, - params.boundingBox, - params.editPosition, - params.editRotation - )) - } else None - }) + def createTaskSkeletonTracingBases(paramsList: List[TaskParameters]) + (implicit ctx: DBAccessContext, m: MessagesProvider): Fox[List[Option[SkeletonTracing]]] = + Fox.serialCombined(paramsList) { params => + for { + taskTypeIdValidated <- ObjectId.parse(params.taskTypeId) + taskType <- taskTypeDAO.findOne(taskTypeIdValidated) ?~> "taskType.notFound" + skeletonTracingOpt <- + if (taskType.tracingType == TracingType.skeleton || taskType.tracingType == TracingType.hybrid) { + Fox.successful(Some(annotationService.createSkeletonTracingBase( + params.dataSet, + params.boundingBox, + params.editPosition, + params.editRotation + ))) + } else Fox.successful(None) + } yield skeletonTracingOpt + } def createTaskVolumeTracingBases(paramsList: List[TaskParameters], organizationId: ObjectId) - (implicit ctx: DBAccessContext, m: MessagesProvider): Fox[List[Option[VolumeTracing]]] = { + (implicit ctx: DBAccessContext, m: MessagesProvider): Fox[List[Option[VolumeTracing]]] = Fox.serialCombined(paramsList) { params => - val tracingType = params.typ.flatMap(TracingType.fromString).getOrElse(TracingType.skeleton) - if (tracingType == TracingType.volume || tracingType == TracingType.hybrid) { - annotationService.createVolumeTracingBase( - params.dataSet, - organizationId, - params.boundingBox, - params.editPosition, - params.editRotation, - params.volumeShowFallbackLayer.getOrElse(false) - ).map(Some(_)) - } else Fox.successful(None) + for { + taskTypeIdValidated <- ObjectId.parse(params.taskTypeId) + taskType <- taskTypeDAO.findOne(taskTypeIdValidated) ?~> "taskType.notFound" + volumeTracingOpt <- + if (taskType.tracingType == TracingType.volume || taskType.tracingType == TracingType.hybrid) { + annotationService.createVolumeTracingBase( + params.dataSet, + organizationId, + params.boundingBox, + params.editPosition, + params.editRotation, + false + ).map(Some(_)) + } else Fox.successful(None) + } yield volumeTracingOpt } - } //Note that from-files tasks do not support volume tracings yet @SuppressWarnings(Array("OptionGet")) //We surpress this warning because we know each skeleton exists due to `toSkeletonSuccessFox` @@ -173,9 +179,7 @@ class TaskController @Inject()(annotationService: AnnotationService, tracing.editPosition, tracing.editRotation, Some(fileName), - description, - None, - None + description ) } @@ -263,7 +267,6 @@ class TaskController @Inject()(annotationService: AnnotationService, volumeIdOpt <- volumeTracingIdBox.toFox _ <- bool2Fox(skeletonIdOpt.isDefined || volumeIdOpt.isDefined) ?~> "task.create.needsEitherSkeletonOrVolume" taskTypeIdValidated <- ObjectId.parse(params.taskTypeId) - taskType <- taskTypeDAO.findOne(taskTypeIdValidated) ?~> "taskType.notFound" project <- projectDAO.findOneByName(params.projectName) ?~> Messages("project.notFound", params.projectName) _ <- validateScript(params.scriptId) ?~> "script.invalid" _ <- Fox.assertTrue(userService.isTeamManagerOrAdminOf(request.identity, project._team)) @@ -271,7 +274,7 @@ class TaskController @Inject()(annotationService: AnnotationService, ObjectId.generate, project._id, params.scriptId.map(ObjectId(_)), - taskType._id, + taskTypeIdValidated, params.neededExperience, params.openInstances, //all instances are open at this time params.openInstances, diff --git a/app/controllers/TaskTypeController.scala b/app/controllers/TaskTypeController.scala index a4d3af583b..2dd2a1c9c3 100755 --- a/app/controllers/TaskTypeController.scala +++ b/app/controllers/TaskTypeController.scala @@ -3,6 +3,7 @@ package controllers import com.scalableminds.util.accesscontext.DBAccessContext import javax.inject.Inject import com.scalableminds.util.tools.{Fox, FoxImplicits} +import com.scalableminds.webknossos.tracingstore.tracings.TracingType import models.annotation.AnnotationSettings import models.task._ import models.user.UserService @@ -12,7 +13,7 @@ import com.mohiva.play.silhouette.api.actions.{SecuredRequest, UserAwareRequest} import play.api.i18n.{Messages, MessagesApi} import play.api.libs.functional.syntax._ import play.api.libs.json.Reads._ -import play.api.libs.json._ +import play.api.libs.json.{Reads, _} import utils.ObjectId import scala.concurrent.ExecutionContext @@ -30,7 +31,8 @@ class TaskTypeController @Inject()(taskTypeDAO: TaskTypeDAO, (__ \ 'description).read[String] and (__ \ 'teamId).read[String](ObjectId.stringObjectIdReads("teamId")) and (__ \ 'settings).read[AnnotationSettings] and - (__ \ 'recommendedConfiguration).readNullable[JsValue])(taskTypeService.fromForm _) + (__ \ 'recommendedConfiguration).readNullable[JsValue] and + (__ \ 'tracingType).read[String])(taskTypeService.fromForm _) def create = sil.SecuredAction.async(parse.json) { implicit request => withJsonBodyUsing(taskTypePublicReads) { taskType => @@ -63,6 +65,7 @@ class TaskTypeController @Inject()(taskTypeDAO: TaskTypeDAO, for { taskTypeIdValidated <- ObjectId.parse(taskTypeId) ?~> "taskType.id.invalid" taskType <- taskTypeDAO.findOne(taskTypeIdValidated) ?~> "taskType.notFound" + _ <- bool2Fox(taskTypeFromForm.tracingType == taskType.tracingType) ?~> "taskType.tracingTypeImmutable" updatedTaskType = taskTypeFromForm.copy(_id = taskType._id) _ <- Fox.assertTrue(userService.isTeamManagerOrAdminOf(request.identity, taskType._team)) _ <- Fox.assertTrue(userService.isTeamManagerOrAdminOf(request.identity, updatedTaskType._team)) diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index 35f84ff7b3..7de1e22cab 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -384,11 +384,11 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor l5: List[E], l6: List[F], l7: List[G]): List[(A, B, C, D, E, F, G)] = - (((((l1, l2, l3).zipped.toList, l4).zipped.toList, l5).zipped.toList, l6).zipped.toList, l7).zipped.toList.map { - tuple: (((((A, B, C), D), E), F), G) => - (tuple._1._1._1._1._1, + ((((((l1, l2).zipped.toList, l3).zipped.toList, l4).zipped.toList, l5).zipped.toList, l6).zipped.toList, l7).zipped.toList.map { + tuple: ((((((A, B), C), D), E), F), G) => + (tuple._1._1._1._1._1._1, + tuple._1._1._1._1._1._2, tuple._1._1._1._1._2, - tuple._1._1._1._1._3, tuple._1._1._1._2, tuple._1._1._2, tuple._1._2, diff --git a/app/models/task/TaskType.scala b/app/models/task/TaskType.scala index 5933caf72e..c98922cdaf 100755 --- a/app/models/task/TaskType.scala +++ b/app/models/task/TaskType.scala @@ -21,6 +21,7 @@ case class TaskType( description: String, settings: AnnotationSettings = AnnotationSettings.defaultFor(TracingType.skeleton), recommendedConfiguration: Option[JsValue] = None, + tracingType: TracingType.Value = TracingType.skeleton, created: Long = System.currentTimeMillis(), isDeleted: Boolean = false ) extends FoxImplicits {} @@ -32,11 +33,12 @@ class TaskTypeService @Inject()(teamDAO: TeamDAO)(implicit ec: ExecutionContext) description: String, team: String, settings: AnnotationSettings, - recommendedConfiguration: Option[JsValue] - ) = - TaskType(ObjectId.generate, ObjectId(team), summary, description, settings, recommendedConfiguration) + recommendedConfiguration: Option[JsValue], + tracingType: String + ): TaskType = + TaskType(ObjectId.generate, ObjectId(team), summary, description, settings, recommendedConfiguration, TracingType.fromString(tracingType).get) - def publicWrites(taskType: TaskType)(implicit ctx: DBAccessContext) = + def publicWrites(taskType: TaskType)(implicit ctx: DBAccessContext): Fox[JsObject] = for { team <- teamDAO.findOne(taskType._team)(GlobalAccessContext) ?~> "team.notFound" } yield @@ -47,7 +49,8 @@ class TaskTypeService @Inject()(teamDAO: TeamDAO)(implicit ec: ExecutionContext) "teamId" -> team._id.toString, "teamName" -> team.name, "settings" -> Json.toJson(taskType.settings), - "recommendedConfiguration" -> taskType.recommendedConfiguration + "recommendedConfiguration" -> taskType.recommendedConfiguration, + "tracingType" -> taskType.tracingType ) } @@ -72,6 +75,7 @@ class TaskTypeDAO @Inject()(sqlClient: SQLClient)(implicit ec: ExecutionContext) r.settingsSomaclickingallowed ), r.recommendedconfiguration.map(Json.parse), + TracingType.fromString(r.tracingtype).get, r.created.getTime, r.isdeleted )) @@ -108,11 +112,11 @@ class TaskTypeDAO @Inject()(sqlClient: SQLClient)(implicit ec: ExecutionContext) for { _ <- run( sqlu"""insert into webknossos.taskTypes(_id, _team, summary, description, settings_allowedModes, settings_preferredMode, - settings_branchPointsAllowed, settings_somaClickingAllowed, recommendedConfiguration, created, isDeleted) + settings_branchPointsAllowed, settings_somaClickingAllowed, recommendedConfiguration, tracingType, created, isDeleted) values(${t._id.id}, ${t._team.id}, ${t.summary}, ${t.description}, '#${sanitize( writeArrayTuple(t.settings.allowedModes))}', #${optionLiteral(t.settings.preferredMode.map(sanitize(_)))}, ${t.settings.branchPointsAllowed}, ${t.settings.somaClickingAllowed}, #${optionLiteral( - t.recommendedConfiguration.map(c => sanitize(Json.toJson(c).toString)))}, + t.recommendedConfiguration.map(c => sanitize(Json.toJson(c).toString)))}, '#${t.tracingType.toString}', ${new java.sql.Timestamp(t.created)}, ${t.isDeleted})""") } yield () } diff --git a/conf/evolutions/038-add-tasktype-tracingtype.sql b/conf/evolutions/038-add-tasktype-tracingtype.sql new file mode 100644 index 0000000000..6f740519dc --- /dev/null +++ b/conf/evolutions/038-add-tasktype-tracingtype.sql @@ -0,0 +1,13 @@ +-- https://github.com/scalableminds/webknossos/pull/3712 + +START TRANSACTION; + +DROP VIEW webknossos.taskTypes_; + +ALTER TABLE webknossos.taskTypes ADD COLUMN tracingType webknossos.TASKTYPE_TRACINGTYPES NOT NULL DEFAULT 'skeleton'; + +CREATE VIEW webknossos.taskTypes_ AS SELECT * FROM webknossos.taskTypes WHERE NOT isDeleted; + +UPDATE webknossos.releaseInformation SET schemaVersion = 38; + +COMMIT TRANSACTION; diff --git a/conf/evolutions/reversions/038-add-tasktype-tracingtype.sql b/conf/evolutions/reversions/038-add-tasktype-tracingtype.sql new file mode 100644 index 0000000000..8bbf7af9ef --- /dev/null +++ b/conf/evolutions/reversions/038-add-tasktype-tracingtype.sql @@ -0,0 +1,11 @@ +START TRANSACTION; + +DROP VIEW webknossos.taskTypes_; + +ALTER TABLE webknossos.taskTypes DROP COLUMN tracingType; + +CREATE VIEW webknossos.taskTypes_ AS SELECT * FROM webknossos.taskTypes WHERE NOT isDeleted; + +UPDATE webknossos.releaseInformation SET schemaVersion = 37; + +COMMIT TRANSACTION; diff --git a/conf/messages b/conf/messages index 7a8dcfe8c6..917701b424 100644 --- a/conf/messages +++ b/conf/messages @@ -224,6 +224,7 @@ taskType.deleteSuccess=Task type “{0}” successfully deleted taskType.deleteFailure=Task type “{0}” deletion failed taskType.noAnnotations=We couldn’t find finished annotations for this task type taskType.id.invalid=The provided task type id is invalid. +taskType.tracingTypeImmutable=Tracing types of task types are immutable. Consider creating a new task type. mail.welcome.subject = Welcome to ScalableMinds diff --git a/tools/postgres/schema.sql b/tools/postgres/schema.sql index 850b0e0580..c2b910b0b4 100644 --- a/tools/postgres/schema.sql +++ b/tools/postgres/schema.sql @@ -21,7 +21,7 @@ START TRANSACTION; CREATE TABLE webknossos.releaseInformation ( schemaVersion BIGINT NOT NULL ); -INSERT INTO webknossos.releaseInformation(schemaVersion) values(37); +INSERT INTO webknossos.releaseInformation(schemaVersion) values(38); COMMIT TRANSACTION; CREATE TABLE webknossos.analytics( @@ -173,6 +173,7 @@ CREATE TABLE webknossos.scripts( ); CREATE TYPE webknossos.TASKTYPE_MODES AS ENUM ('orthogonal', 'flight', 'oblique', 'volume'); +CREATE TYPE webknossos.TASKTYPE_TRACINGTYPES AS ENUM ('skeleton', 'volume', 'hybrid'); CREATE TABLE webknossos.taskTypes( _id CHAR(24) PRIMARY KEY DEFAULT '', _team CHAR(24) NOT NULL, @@ -183,6 +184,7 @@ CREATE TABLE webknossos.taskTypes( settings_branchPointsAllowed BOOLEAN NOT NULL, settings_somaClickingAllowed BOOLEAN NOT NULL, recommendedConfiguration JSONB, + tracingType webknossos.TASKTYPE_TRACINGTYPES NOT NULL DEFAULT 'skeleton', created TIMESTAMPTZ NOT NULL DEFAULT NOW(), isDeleted BOOLEAN NOT NULL DEFAULT false ); From 4153a141db7bcc3eaf6956f27b8e47a29f97e4aa Mon Sep 17 00:00:00 2001 From: Florian M Date: Fri, 1 Feb 2019 15:10:09 +0100 Subject: [PATCH 06/31] multi-task download: fetch volume data --- app/models/annotation/AnnotationMerger.scala | 2 +- app/models/annotation/AnnotationService.scala | 90 ++++++++++++------- .../annotation/TracingStoreRpcClient.scala | 28 ++++-- .../SkeletonTracingController.scala | 9 +- .../controllers/TracingController.scala | 7 +- .../controllers/VolumeTracingController.scala | 5 ++ .../tracings/TracingService.scala | 7 +- ...alableminds.webknossos.tracingstore.routes | 1 + 8 files changed, 106 insertions(+), 43 deletions(-) diff --git a/app/models/annotation/AnnotationMerger.scala b/app/models/annotation/AnnotationMerger.scala index 3086ea3a01..3598d569ef 100644 --- a/app/models/annotation/AnnotationMerger.scala +++ b/app/models/annotation/AnnotationMerger.scala @@ -65,7 +65,7 @@ class AnnotationMerger @Inject()(dataSetDAO: DataSetDAO, tracingStoreService: Tr for { dataSet <- dataSetDAO.findOne(dataSetId) tracingStoreClient <- tracingStoreService.clientFor(dataSet) - skeletonTracingIds <- Fox.combined(annotations.map(_.skeletonTracingId.toFox)) + skeletonTracingIds = annotations.map(_.skeletonTracingId) tracingReference <- tracingStoreClient.mergeSkeletonTracingsByIds(skeletonTracingIds, persistTracing) ?~> "Failed to merge skeleton tracings." } yield { tracingReference diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index 7de1e22cab..559c59a2fa 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -2,17 +2,16 @@ package models.annotation import java.io.{BufferedOutputStream, File, FileOutputStream} +import akka.stream.scaladsl.Source +import akka.util.ByteString import com.scalableminds.util.geometry.{BoundingBox, Point3D, Scale, Vector3D} import com.scalableminds.util.io.{NamedEnumeratorStream, ZipIO} import com.scalableminds.util.accesscontext.{AuthorizedAccessContext, DBAccessContext, GlobalAccessContext} import com.scalableminds.util.mvc.Formatter import com.scalableminds.util.tools.{BoxImplicits, Fox, FoxImplicits, TextUtils} import com.scalableminds.webknossos.tracingstore.SkeletonTracing._ -import com.scalableminds.webknossos.tracingstore.VolumeTracing.VolumeTracing -import com.scalableminds.webknossos.datastore.models.datasource.{ - DataSourceLike => DataSource, - SegmentationLayerLike => SegmentationLayer -} +import com.scalableminds.webknossos.tracingstore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings} +import com.scalableminds.webknossos.datastore.models.datasource.{DataSourceLike => DataSource, SegmentationLayerLike => SegmentationLayer} import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.webknossos.tracingstore.tracings.skeleton.{NodeDefaults, SkeletonTracingDefaults} import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeTracingDefaults @@ -368,25 +367,30 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor tracingsAndNamesFlattened = flattenTupledLists(tracingsNamesAndScalesAsTuples) nmlsAndNames = tracingsAndNamesFlattened.map( tuple => - (nmlWriter.toNmlStream(Some(tuple._1), None, Some(tuple._4), tuple._3, tuple._7, Some(tuple._5), tuple._6), - tuple._2)) + (nmlWriter.toNmlStream(tuple._1, tuple._2, Some(tuple._6), tuple._5, tuple._9, Some(tuple._7), tuple._8), + tuple._4, tuple._3)) zip <- createZip(nmlsAndNames, zipFileName) } yield zip - private def flattenTupledLists[A, B, C, D, E, F, G]( - tupledLists: List[(List[A], List[B], List[C], List[D], List[E], List[F], List[G])]) = - tupledLists.flatMap(tuple => zippedSevenLists(tuple._1, tuple._2, tuple._3, tuple._4, tuple._5, tuple._6, tuple._7)) - - private def zippedSevenLists[A, B, C, D, E, F, G](l1: List[A], - l2: List[B], - l3: List[C], - l4: List[D], - l5: List[E], - l6: List[F], - l7: List[G]): List[(A, B, C, D, E, F, G)] = - ((((((l1, l2).zipped.toList, l3).zipped.toList, l4).zipped.toList, l5).zipped.toList, l6).zipped.toList, l7).zipped.toList.map { - tuple: ((((((A, B), C), D), E), F), G) => - (tuple._1._1._1._1._1._1, + private def flattenTupledLists[A, B, C, D, E, F, G, H, I]( + tupledLists: List[(List[A], List[B], List[C], List[D], List[E], List[F], List[G], List[H], List[I])]) = + tupledLists.flatMap(tuple => zippedEightLists(tuple._1, tuple._2, tuple._3, tuple._4, tuple._5, tuple._6, tuple._7, tuple._8, tuple._9)) + + private def zippedEightLists[A, B, C, D, E, F, G, H, I] + (l1: List[A], + l2: List[B], + l3: List[C], + l4: List[D], + l5: List[E], + l6: List[F], + l7: List[G], + l8: List[H], + l9: List[I]): List[(A, B, C, D, E, F, G, H, I)] = + ((((((((l1,l2).zipped.toList, l3).zipped.toList, l4).zipped.toList, l5).zipped.toList, l6).zipped.toList, l7).zipped.toList, l8).zipped.toList, l9).zipped.toList.map { + tuple: ((((((((A, B), C), D), E), F), G), H), I) => + (tuple._1._1._1._1._1._1._1._1, + tuple._1._1._1._1._1._1._1._2, + tuple._1._1._1._1._1._1._2, tuple._1._1._1._1._1._2, tuple._1._1._1._1._2, tuple._1._1._1._2, @@ -396,7 +400,9 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor } private def getTracingsScalesAndNamesFor(annotations: List[Annotation])( - implicit ctx: DBAccessContext): Fox[List[(List[SkeletonTracing], + implicit ctx: DBAccessContext): Fox[List[(List[Option[SkeletonTracing]], + List[Option[VolumeTracing]], + List[Option[Source[ByteString, _]]], List[String], List[Option[Scale]], List[Annotation], @@ -404,12 +410,32 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor List[Option[Task]], List[String])]] = { - def getTracings(dataSetId: ObjectId, tracingIds: List[String]): Fox[List[SkeletonTracing]] = + def getSkeletonTracings(dataSetId: ObjectId, tracingIds: List[Option[String]]): Fox[List[Option[SkeletonTracing]]] = + for { + dataSet <- dataSetDAO.findOne(dataSetId) + tracingStoreClient <- tracingStoreService.clientFor(dataSet) + tracingContainers: List[SkeletonTracings] <- Fox.serialCombined(tracingIds.grouped(1000).toList)(tracingStoreClient.getSkeletonTracings) + tracingOpts: List[SkeletonTracingOpt] = tracingContainers.flatMap(_.tracings) + } yield tracingOpts.map(_.tracing) + + def getVolumeTracings(dataSetId: ObjectId, tracingIds: List[Option[String]]): Fox[List[Option[VolumeTracing]]] = for { dataSet <- dataSetDAO.findOne(dataSetId) tracingStoreClient <- tracingStoreService.clientFor(dataSet) - tracingContainers <- Fox.serialCombined(tracingIds.grouped(1000).toList)(tracingStoreClient.getSkeletonTracings) - } yield tracingContainers.flatMap(_.tracings).flatMap(_.tracing) + tracingContainers: List[VolumeTracings] <- Fox.serialCombined(tracingIds.grouped(1000).toList)(tracingStoreClient.getVolumeTracings) + tracingOpts: List[VolumeTracingOpt] = tracingContainers.flatMap(_.tracings) + } yield tracingOpts.map(_.tracing) + + + def getVolumeDataObjects(dataSetId: ObjectId, tracingIds: List[Option[String]]): Fox[List[Option[Source[ByteString, Any]]]] = + for { + dataSet <- dataSetDAO.findOne(dataSetId) + tracingStoreClient <- tracingStoreService.clientFor(dataSet) + tracingDataObjects: List[Option[Source[ByteString, Any]]] <- Fox.serialCombined(tracingIds) { + case Some(tracingId) => tracingStoreClient.getVolumeData(tracingId).map(Some(_)) + case None => Fox.successful(None) + } + } yield tracingDataObjects def getUsers(userIds: List[ObjectId]) = Fox.serialCombined(userIds)(userService.findOneById(_, useCache = true)) @@ -429,26 +455,28 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor Fox.combined(annotations.map(a => organizationDAO.findOrganizationNameForAnnotation(a._id))) val annotationsGrouped: Map[ObjectId, List[Annotation]] = annotations.groupBy(_._dataSet) - val tracings = annotationsGrouped.map { + val tracingsGrouped = annotationsGrouped.map { case (dataSetId, annotations) => { for { scale <- getDatasetScale(dataSetId) - tracings <- getTracings(dataSetId, annotations.flatMap(_.skeletonTracingId)) + skeletonTracings <- getSkeletonTracings(dataSetId, annotations.map(_.skeletonTracingId)) + volumeTracings <- getVolumeTracings(dataSetId, annotations.map(_.volumeTracingId)) + volumeDataObjects <- getVolumeDataObjects(dataSetId, annotations.map(_.volumeTracingId)) users <- getUsers(annotations.map(_._user)) ?~> "user.notFound" taskOpts <- getTasks(annotations.map(_._task)) ?~> "task.notFound" names <- getNames(annotations) organizationNames <- getOrganizationNames(annotations) - } yield (tracings, names, annotations.map(a => scale), annotations, users, taskOpts, organizationNames) + } yield (skeletonTracings, volumeTracings, volumeDataObjects, names, annotations.map(a => scale), annotations, users, taskOpts, organizationNames) } } - Fox.combined(tracings.toList) + Fox.combined(tracingsGrouped.toList) } - private def createZip(nmls: List[(Enumerator[Array[Byte]], String)], zipFileName: String): Future[TemporaryFile] = { + private def createZip(nmls: List[(Enumerator[Array[Byte]], String, Option[Source[ByteString, _]])], zipFileName: String): Future[TemporaryFile] = { val zipped = temporaryFileCreator.create(normalize(zipFileName), ".zip") val zipper = ZipIO.startZip(new BufferedOutputStream(new FileOutputStream(new File(zipped.path.toString)))) - def addToZip(nmls: List[(Enumerator[Array[Byte]], String)]): Future[Boolean] = + def addToZip(nmls: List[(Enumerator[Array[Byte]], String, Option[Source[ByteString, _]])]): Future[Boolean] = nmls match { case head :: tail => zipper.withFile(head._2 + ".nml")(NamedEnumeratorStream("", head._1).writeTo).flatMap(_ => addToZip(tail)) diff --git a/app/models/annotation/TracingStoreRpcClient.scala b/app/models/annotation/TracingStoreRpcClient.scala index cb268907a6..b96eb65795 100644 --- a/app/models/annotation/TracingStoreRpcClient.scala +++ b/app/models/annotation/TracingStoreRpcClient.scala @@ -34,12 +34,20 @@ class TracingStoreRpcClient(tracingStore: TracingStore, dataSet: DataSet, rpc: R .getWithProtoResponse[SkeletonTracing](SkeletonTracing) } - def getSkeletonTracings(tracingIds: List[String]): Fox[SkeletonTracings] = { + def getSkeletonTracings(tracingIds: List[Option[String]]): Fox[SkeletonTracings] = { logger.debug("Called to get multiple SkeletonTracings." + baseInfo) rpc(s"${tracingStore.url}/tracings/skeleton/getMultiple") .addQueryString("token" -> TracingStoreRpcClient.webKnossosToken) - .postJsonWithProtoResponse[List[TracingSelector], SkeletonTracings](tracingIds.map(TracingSelector(_)))( - SkeletonTracings) + .postJsonWithProtoResponse[List[Option[TracingSelector]], SkeletonTracings](tracingIds.map(id => id.map(TracingSelector(_))))( + SkeletonTracings) + } + + def getVolumeTracings(tracingIds: List[Option[String]]): Fox[VolumeTracings] = { + logger.debug("Called to get multiple VolumeTracings." + baseInfo) + rpc(s"${tracingStore.url}/tracings/volume/getMultiple") + .addQueryString("token" -> TracingStoreRpcClient.webKnossosToken) + .postJsonWithProtoResponse[List[Option[TracingSelector]], VolumeTracings](tracingIds.map(id => id.map(TracingSelector(_))))( + VolumeTracings) } def saveSkeletonTracing(tracing: SkeletonTracing): Fox[String] = { @@ -78,12 +86,12 @@ class TracingStoreRpcClient(tracingStore: TracingStore, dataSet: DataSet, rpc: R .getWithJsonResponse[String] } - def mergeSkeletonTracingsByIds(tracingIds: List[String], persistTracing: Boolean): Fox[String] = { + def mergeSkeletonTracingsByIds(tracingIds: List[Option[String]], persistTracing: Boolean): Fox[String] = { logger.debug("Called to merge SkeletonTracings by ids." + baseInfo) rpc(s"${tracingStore.url}/tracings/skeleton/mergedFromIds") .addQueryString("token" -> TracingStoreRpcClient.webKnossosToken) .addQueryString("persist" -> persistTracing.toString) - .postWithJsonResponse[List[TracingSelector], String](tracingIds.map(TracingSelector(_))) + .postWithJsonResponse[List[Option[TracingSelector]], String](tracingIds.map(id => id.map(TracingSelector(_)))) } def mergeSkeletonTracingsByContents(tracings: SkeletonTracings, persistTracing: Boolean): Fox[String] = { @@ -129,4 +137,14 @@ class TracingStoreRpcClient(tracingStore: TracingStore, dataSet: DataSet, rpc: R } } + def getVolumeData(tracingId: String, version: Option[Long] = None): Fox[Source[ByteString, _]] = { + logger.debug("Called to get volume data." + baseInfo) + for { + data <- rpc(s"${tracingStore.url}/tracings/volume/${tracingId}/allData") + .addQueryString("token" -> TracingStoreRpcClient.webKnossosToken) + .addQueryStringOptional("version", version.map(_.toString)) + .getStream + } yield data + } + } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala index b8b416c804..b224cbe046 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala @@ -3,9 +3,12 @@ package com.scalableminds.webknossos.tracingstore.controllers import com.google.inject.Inject import com.scalableminds.webknossos.tracingstore.SkeletonTracing.{SkeletonTracing, SkeletonTracingOpt, SkeletonTracings} import com.scalableminds.webknossos.datastore.services.{AccessTokenService, UserAccessRequest} +import com.scalableminds.webknossos.tracingstore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings} import com.scalableminds.webknossos.tracingstore.{TracingStoreAccessTokenService, TracingStoreWkRpcClient} import com.scalableminds.webknossos.tracingstore.tracings.TracingSelector import com.scalableminds.webknossos.tracingstore.tracings.skeleton._ +import com.scalableminds.util.tools.JsonHelper.boxFormat +import com.scalableminds.util.tools.JsonHelper.optionFormat import play.api.i18n.Messages import play.api.libs.json.Json import play.api.mvc.PlayBodyParsers @@ -23,6 +26,8 @@ class SkeletonTracingController @Inject()(val tracingService: SkeletonTracingSer implicit def packMultiple(tracings: List[SkeletonTracing]): SkeletonTracings = SkeletonTracings(tracings.map(t => SkeletonTracingOpt(Some(t)))) + implicit def packMultipleOpt(tracings: List[Option[SkeletonTracing]]): SkeletonTracings = SkeletonTracings(tracings.map(t => SkeletonTracingOpt(t))) + implicit def unpackMultiple(tracings: SkeletonTracings): List[Option[SkeletonTracing]] = tracings.tracings.toList.map(_.tracing) def mergedFromContents(persist: Boolean) = Action.async(validateProto[SkeletonTracings]) { implicit request => @@ -39,13 +44,13 @@ class SkeletonTracingController @Inject()(val tracingService: SkeletonTracingSer } } - def mergedFromIds(persist: Boolean) = Action.async(validateJson[List[TracingSelector]]) { implicit request => + def mergedFromIds(persist: Boolean) = Action.async(validateJson[List[Option[TracingSelector]]]) { implicit request => log { accessTokenService.validateAccess(UserAccessRequest.webknossos) { AllowRemoteOrigin { for { tracings <- tracingService.findMultiple(request.body, applyUpdates = true) ?~> Messages("tracing.notFound") - mergedTracing = tracingService.merge(tracings) + mergedTracing = tracingService.merge(tracings.flatten) newId <- tracingService.save(mergedTracing, None, mergedTracing.version, toCache = !persist) } yield { Ok(Json.toJson(newId)) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala index fb2b74913e..7ef3ed79cd 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala @@ -15,7 +15,8 @@ import scalapb.{GeneratedMessage, GeneratedMessageCompanion, Message} import scala.concurrent.ExecutionContext -trait TracingController[T <: GeneratedMessage with Message[T], Ts <: GeneratedMessage with Message[Ts]] extends Controller { +trait TracingController[T <: GeneratedMessage with Message[T], + Ts <: GeneratedMessage with Message[Ts]] extends Controller { def tracingService: TracingService[T] @@ -33,6 +34,8 @@ trait TracingController[T <: GeneratedMessage with Message[T], Ts <: GeneratedMe implicit def packMultiple(tracings: List[T]): Ts + implicit def packMultipleOpt(tracings: List[Option[T]]): Ts + implicit val updateActionReads: Reads[UpdateAction[T]] = tracingService.updateActionReads implicit val ec: ExecutionContext @@ -82,7 +85,7 @@ trait TracingController[T <: GeneratedMessage with Message[T], Ts <: GeneratedMe } } - def getMultiple = Action.async(validateJson[List[TracingSelector]]) { implicit request => + def getMultiple = Action.async(validateJson[List[Option[TracingSelector]]]) { implicit request => log { accessTokenService.validateAccess(UserAccessRequest.webknossos) { AllowRemoteOrigin { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index dc32f83ffd..6be18df000 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -6,9 +6,12 @@ import com.scalableminds.webknossos.datastore.DataStoreConfig import com.scalableminds.webknossos.tracingstore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings} import com.scalableminds.webknossos.datastore.models.WebKnossosDataRequest import com.scalableminds.webknossos.datastore.services.{AccessTokenService, UserAccessRequest} +import com.scalableminds.webknossos.tracingstore.SkeletonTracing.{SkeletonTracing, SkeletonTracingOpt} import com.scalableminds.webknossos.tracingstore.{TracingStoreAccessTokenService, TracingStoreConfig, TracingStoreWkRpcClient} import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeTracingService +import com.scalableminds.util.tools.JsonHelper.boxFormat +import com.scalableminds.util.tools.JsonHelper.optionFormat import play.api.i18n.Messages import play.api.libs.iteratee.Enumerator import play.api.libs.iteratee.streams.IterateeStreams @@ -30,6 +33,8 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService implicit def packMultiple(tracings: List[VolumeTracing]): VolumeTracings = VolumeTracings(tracings.map(t => VolumeTracingOpt(Some(t)))) + implicit def packMultipleOpt(tracings: List[Option[VolumeTracing]]): VolumeTracings = VolumeTracings(tracings.map(t => VolumeTracingOpt(t))) + implicit def unpackMultiple(tracings: VolumeTracings): List[Option[VolumeTracing]] = tracings.tracings.toList.map(_.tracing) override def freezeVersions = config.Tracingstore.freezeVolumeVersions diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala index aa07e64108..c27ce58be6 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala @@ -48,9 +48,12 @@ trait TracingService[T <: GeneratedMessage with Message[T]] extends KeyValueStor } } - def findMultiple(selectors: List[TracingSelector], useCache: Boolean = true, applyUpdates: Boolean = false): Fox[List[T]] = { + def findMultiple(selectors: List[Option[TracingSelector]], useCache: Boolean = true, applyUpdates: Boolean = false): Fox[List[Option[T]]] = { Fox.combined { - selectors.map(selector => find(selector.tracingId, selector.version, useCache, applyUpdates)) + selectors.map { + case Some(selector) => find(selector.tracingId, selector.version, useCache, applyUpdates).map(Some(_)) + case None => Fox.successful(None) + } } } diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index 95918038da..4102934782 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -15,6 +15,7 @@ GET /volume/:tracingId/allData @com.scalablemin POST /volume/:tracingId/data @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.data(tracingId: String) GET /volume/:tracingId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.duplicate(tracingId: String, version: Option[Long]) GET /volume/:tracingId/updateActionLog @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.updateActionLog(tracingId: String) +POST /volume/getMultiple @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getMultiple # Skeleton tracings POST /skeleton/save @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.save From b6ad9c01ad9ce8fd68873fc9e50af430b3ddcf05 Mon Sep 17 00:00:00 2001 From: Florian M Date: Fri, 1 Feb 2019 15:19:40 +0100 Subject: [PATCH 07/31] add enums to evolutions --- conf/evolutions/038-add-tasktype-tracingtype.sql | 2 ++ conf/evolutions/reversions/038-add-tasktype-tracingtype.sql | 2 ++ 2 files changed, 4 insertions(+) diff --git a/conf/evolutions/038-add-tasktype-tracingtype.sql b/conf/evolutions/038-add-tasktype-tracingtype.sql index 6f740519dc..aaa9e55c24 100644 --- a/conf/evolutions/038-add-tasktype-tracingtype.sql +++ b/conf/evolutions/038-add-tasktype-tracingtype.sql @@ -2,6 +2,8 @@ START TRANSACTION; +CREATE TYPE webknossos.TASKTYPE_TRACINGTYPES AS ENUM ('skeleton', 'volume', 'hybrid'); + DROP VIEW webknossos.taskTypes_; ALTER TABLE webknossos.taskTypes ADD COLUMN tracingType webknossos.TASKTYPE_TRACINGTYPES NOT NULL DEFAULT 'skeleton'; diff --git a/conf/evolutions/reversions/038-add-tasktype-tracingtype.sql b/conf/evolutions/reversions/038-add-tasktype-tracingtype.sql index 8bbf7af9ef..370551f2a1 100644 --- a/conf/evolutions/reversions/038-add-tasktype-tracingtype.sql +++ b/conf/evolutions/reversions/038-add-tasktype-tracingtype.sql @@ -6,6 +6,8 @@ ALTER TABLE webknossos.taskTypes DROP COLUMN tracingType; CREATE VIEW webknossos.taskTypes_ AS SELECT * FROM webknossos.taskTypes WHERE NOT isDeleted; +DROP TYPE webknossos.TASKTYPE_TRACINGTYPES; + UPDATE webknossos.releaseInformation SET schemaVersion = 37; COMMIT TRANSACTION; From 4365fa41c41b5ce71d2b75f44fca5dc7a7c9e493 Mon Sep 17 00:00:00 2001 From: Florian M Date: Fri, 1 Feb 2019 15:21:57 +0100 Subject: [PATCH 08/31] scalafmt --- app/controllers/AnnotationIOController.scala | 5 +- app/controllers/TaskController.scala | 61 ++++--- app/models/annotation/AnnotationService.scala | 158 ++++++++++-------- .../annotation/TracingStoreRpcClient.scala | 8 +- app/models/task/TaskType.scala | 8 +- 5 files changed, 138 insertions(+), 102 deletions(-) diff --git a/app/controllers/AnnotationIOController.scala b/app/controllers/AnnotationIOController.scala index 8f6da57628..a54e3e0ebc 100755 --- a/app/controllers/AnnotationIOController.scala +++ b/app/controllers/AnnotationIOController.scala @@ -145,8 +145,9 @@ class AnnotationIOController @Inject()(nmlWriter: NmlWriter, } yield savedTracingId } mergedSkeletonTracingIdOpt <- Fox.runOptional(skeletonTracings.headOption) { s => - tracingStoreClient.mergeSkeletonTracingsByContents(SkeletonTracings(skeletonTracings.map(t => SkeletonTracingOpt(Some(t)))), - persistTracing = true) + tracingStoreClient.mergeSkeletonTracingsByContents( + SkeletonTracings(skeletonTracings.map(t => SkeletonTracingOpt(Some(t)))), + persistTracing = true) } annotation <- annotationService.createFrom(request.identity, dataSet, diff --git a/app/controllers/TaskController.scala b/app/controllers/TaskController.scala index 6bffe2ccd9..66d9fd9c73 100755 --- a/app/controllers/TaskController.scala +++ b/app/controllers/TaskController.scala @@ -93,46 +93,51 @@ class TaskController @Inject()(annotationService: AnnotationService, for { _ <- bool2Fox(request.body.length <= 1000) ?~> "task.create.limitExceeded" skeletonBaseOpts: List[Option[SkeletonTracing]] <- createTaskSkeletonTracingBases(request.body) - volumeBaseOpts: List[Option[VolumeTracing]] <- createTaskVolumeTracingBases(request.body, request.identity._organization) + volumeBaseOpts: List[Option[VolumeTracing]] <- createTaskVolumeTracingBases(request.body, + request.identity._organization) result <- createTasks((request.body, skeletonBaseOpts, volumeBaseOpts).zipped.toList) } yield result } - def createTaskSkeletonTracingBases(paramsList: List[TaskParameters]) - (implicit ctx: DBAccessContext, m: MessagesProvider): Fox[List[Option[SkeletonTracing]]] = + def createTaskSkeletonTracingBases(paramsList: List[TaskParameters])( + implicit ctx: DBAccessContext, + m: MessagesProvider): Fox[List[Option[SkeletonTracing]]] = Fox.serialCombined(paramsList) { params => for { taskTypeIdValidated <- ObjectId.parse(params.taskTypeId) taskType <- taskTypeDAO.findOne(taskTypeIdValidated) ?~> "taskType.notFound" - skeletonTracingOpt <- - if (taskType.tracingType == TracingType.skeleton || taskType.tracingType == TracingType.hybrid) { - Fox.successful(Some(annotationService.createSkeletonTracingBase( - params.dataSet, - params.boundingBox, - params.editPosition, - params.editRotation - ))) - } else Fox.successful(None) + skeletonTracingOpt <- if (taskType.tracingType == TracingType.skeleton || taskType.tracingType == TracingType.hybrid) { + Fox.successful( + Some( + annotationService.createSkeletonTracingBase( + params.dataSet, + params.boundingBox, + params.editPosition, + params.editRotation + ))) + } else Fox.successful(None) } yield skeletonTracingOpt } - def createTaskVolumeTracingBases(paramsList: List[TaskParameters], organizationId: ObjectId) - (implicit ctx: DBAccessContext, m: MessagesProvider): Fox[List[Option[VolumeTracing]]] = + def createTaskVolumeTracingBases(paramsList: List[TaskParameters], organizationId: ObjectId)( + implicit ctx: DBAccessContext, + m: MessagesProvider): Fox[List[Option[VolumeTracing]]] = Fox.serialCombined(paramsList) { params => for { taskTypeIdValidated <- ObjectId.parse(params.taskTypeId) taskType <- taskTypeDAO.findOne(taskTypeIdValidated) ?~> "taskType.notFound" - volumeTracingOpt <- - if (taskType.tracingType == TracingType.volume || taskType.tracingType == TracingType.hybrid) { - annotationService.createVolumeTracingBase( + volumeTracingOpt <- if (taskType.tracingType == TracingType.volume || taskType.tracingType == TracingType.hybrid) { + annotationService + .createVolumeTracingBase( params.dataSet, organizationId, params.boundingBox, params.editPosition, params.editRotation, false - ).map(Some(_)) - } else Fox.successful(None) + ) + .map(Some(_)) + } else Fox.successful(None) } yield volumeTracingOpt } @@ -186,15 +191,15 @@ class TaskController @Inject()(annotationService: AnnotationService, def createTasks(requestedTasks: List[(TaskParameters, Option[SkeletonTracing], Option[VolumeTracing])])( implicit request: SecuredRequest[WkEnv, _]): Fox[Result] = { - def assertEachHasEitherSkeletonOrVolume: Fox[Boolean] = { - bool2Fox(requestedTasks.forall { - tuple => tuple._2.isDefined || tuple._3.isDefined + def assertEachHasEitherSkeletonOrVolume: Fox[Boolean] = + bool2Fox(requestedTasks.forall { tuple => + tuple._2.isDefined || tuple._3.isDefined }) - } def assertAllOnSameDataset(firstDatasetName: String): Fox[String] = { - def allOnSameDatasetIter(requestedTasksRest: List[(TaskParameters, Option[SkeletonTracing], Option[VolumeTracing])], - dataSetName: String): Boolean = + def allOnSameDatasetIter( + requestedTasksRest: List[(TaskParameters, Option[SkeletonTracing], Option[VolumeTracing])], + dataSetName: String): Boolean = requestedTasksRest match { case List() => true case head :: tail => head._1.dataSet == dataSetName && allOnSameDatasetIter(tail, dataSetName) @@ -260,8 +265,10 @@ class TaskController @Inject()(annotationService: AnnotationService, case _ => Fox.successful(()) } - private def createTaskWithoutAnnotationBase(params: TaskParameters, skeletonTracingIdBox: Box[Option[String]], volumeTracingIdBox: Box[Option[String]]) ( - implicit request: SecuredRequest[WkEnv, _]): Fox[Task] = + private def createTaskWithoutAnnotationBase( + params: TaskParameters, + skeletonTracingIdBox: Box[Option[String]], + volumeTracingIdBox: Box[Option[String]])(implicit request: SecuredRequest[WkEnv, _]): Fox[Task] = for { skeletonIdOpt <- skeletonTracingIdBox.toFox volumeIdOpt <- volumeTracingIdBox.toFox diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index 559c59a2fa..a4e5ea51a3 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -11,7 +11,10 @@ import com.scalableminds.util.mvc.Formatter import com.scalableminds.util.tools.{BoxImplicits, Fox, FoxImplicits, TextUtils} import com.scalableminds.webknossos.tracingstore.SkeletonTracing._ import com.scalableminds.webknossos.tracingstore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings} -import com.scalableminds.webknossos.datastore.models.datasource.{DataSourceLike => DataSource, SegmentationLayerLike => SegmentationLayer} +import com.scalableminds.webknossos.datastore.models.datasource.{ + DataSourceLike => DataSource, + SegmentationLayerLike => SegmentationLayer +} import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.webknossos.tracingstore.tracings.skeleton.{NodeDefaults, SkeletonTracingDefaults} import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeTracingDefaults @@ -80,12 +83,12 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor }).flatten private def createVolumeTracing( - dataSource: DataSource, - withFallback: Boolean, - boundingBox: Option[BoundingBox] = None, - startPosition: Option[Point3D] = None, - startRotation: Option[Vector3D] = None - ): VolumeTracing = { + dataSource: DataSource, + withFallback: Boolean, + boundingBox: Option[BoundingBox] = None, + startPosition: Option[Point3D] = None, + startRotation: Option[Vector3D] = None + ): VolumeTracing = { val fallbackLayer: Option[SegmentationLayer] = if (withFallback) { dataSource.dataLayers.flatMap { case layer: SegmentationLayer => Some(layer) @@ -216,13 +219,16 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor result <- annotationDAO.countActiveAnnotationsFor(user._id, AnnotationType.Task, teamManagerTeamIds) } yield result - def tracingFromBase(annotationBase: Annotation, dataSet: DataSet)(implicit ctx: DBAccessContext, - m: MessagesProvider): Fox[(Option[String], Option[String])] = + def tracingFromBase(annotationBase: Annotation, dataSet: DataSet)( + implicit ctx: DBAccessContext, + m: MessagesProvider): Fox[(Option[String], Option[String])] = for { dataSource <- bool2Fox(dataSet.isUsable) ?~> Messages("dataSet.notImported", dataSet.name) tracingStoreClient <- tracingStoreService.clientFor(dataSet) - newSkeletonId: Option[String] <- Fox.runOptional(annotationBase.skeletonTracingId)(skeletonId => tracingStoreClient.duplicateSkeletonTracing(skeletonId)) - newVolumeId: Option[String] <- Fox.runOptional(annotationBase.volumeTracingId)(volumeId => tracingStoreClient.duplicateVolumeTracing(volumeId)) + newSkeletonId: Option[String] <- Fox.runOptional(annotationBase.skeletonTracingId)(skeletonId => + tracingStoreClient.duplicateSkeletonTracing(skeletonId)) + newVolumeId: Option[String] <- Fox.runOptional(annotationBase.volumeTracingId)(volumeId => + tracingStoreClient.duplicateVolumeTracing(volumeId)) } yield (newSkeletonId, newVolumeId) def createAnnotationFor(user: User, task: Task, initializingAnnotationId: ObjectId)( @@ -283,27 +289,27 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor ) } - def createVolumeTracingBase(dataSetName: String, - organizationId: ObjectId, - boundingBox: Option[BoundingBox], - startPosition: Point3D, - startRotation: Vector3D, - volumeShowFallbackLayer: Boolean)(implicit ctx: DBAccessContext, - m: MessagesProvider): Fox[VolumeTracing] = { + def createVolumeTracingBase( + dataSetName: String, + organizationId: ObjectId, + boundingBox: Option[BoundingBox], + startPosition: Point3D, + startRotation: Vector3D, + volumeShowFallbackLayer: Boolean)(implicit ctx: DBAccessContext, m: MessagesProvider): Fox[VolumeTracing] = for { - dataSet <- dataSetDAO.findOneByNameAndOrganization(dataSetName, organizationId) ?~> Messages("dataset.notFound", dataSetName) + dataSet <- dataSetDAO.findOneByNameAndOrganization(dataSetName, organizationId) ?~> Messages("dataset.notFound", + dataSetName) dataSource <- dataSetService.dataSourceFor(dataSet).flatMap(_.toUsable) volumeTracing = createVolumeTracing( dataSource, withFallback = volumeShowFallbackLayer, boundingBox = boundingBox.flatMap { box => - if (box.isEmpty) None else Some(box) - }, + if (box.isEmpty) None else Some(box) + }, startPosition = Some(startPosition), startRotation = Some(startRotation) ) } yield volumeTracing - } def abortInitializedAnnotationOnFailure(initializingAnnotationId: ObjectId, insertedAnnotationBox: Box[Annotation]) = insertedAnnotationBox match { @@ -368,53 +374,56 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor nmlsAndNames = tracingsAndNamesFlattened.map( tuple => (nmlWriter.toNmlStream(tuple._1, tuple._2, Some(tuple._6), tuple._5, tuple._9, Some(tuple._7), tuple._8), - tuple._4, tuple._3)) + tuple._4, + tuple._3)) zip <- createZip(nmlsAndNames, zipFileName) } yield zip private def flattenTupledLists[A, B, C, D, E, F, G, H, I]( tupledLists: List[(List[A], List[B], List[C], List[D], List[E], List[F], List[G], List[H], List[I])]) = - tupledLists.flatMap(tuple => zippedEightLists(tuple._1, tuple._2, tuple._3, tuple._4, tuple._5, tuple._6, tuple._7, tuple._8, tuple._9)) - - private def zippedEightLists[A, B, C, D, E, F, G, H, I] - (l1: List[A], - l2: List[B], - l3: List[C], - l4: List[D], - l5: List[E], - l6: List[F], - l7: List[G], - l8: List[H], - l9: List[I]): List[(A, B, C, D, E, F, G, H, I)] = - ((((((((l1,l2).zipped.toList, l3).zipped.toList, l4).zipped.toList, l5).zipped.toList, l6).zipped.toList, l7).zipped.toList, l8).zipped.toList, l9).zipped.toList.map { - tuple: ((((((((A, B), C), D), E), F), G), H), I) => - (tuple._1._1._1._1._1._1._1._1, - tuple._1._1._1._1._1._1._1._2, - tuple._1._1._1._1._1._1._2, - tuple._1._1._1._1._1._2, - tuple._1._1._1._1._2, - tuple._1._1._1._2, - tuple._1._1._2, - tuple._1._2, - tuple._2) + tupledLists.flatMap(tuple => + zippedEightLists(tuple._1, tuple._2, tuple._3, tuple._4, tuple._5, tuple._6, tuple._7, tuple._8, tuple._9)) + + private def zippedEightLists[A, B, C, D, E, F, G, H, I](l1: List[A], + l2: List[B], + l3: List[C], + l4: List[D], + l5: List[E], + l6: List[F], + l7: List[G], + l8: List[H], + l9: List[I]): List[(A, B, C, D, E, F, G, H, I)] = + ((((((((l1, l2).zipped.toList, l3).zipped.toList, l4).zipped.toList, l5).zipped.toList, l6).zipped.toList, l7).zipped.toList, + l8).zipped.toList, + l9).zipped.toList.map { tuple: ((((((((A, B), C), D), E), F), G), H), I) => + (tuple._1._1._1._1._1._1._1._1, + tuple._1._1._1._1._1._1._1._2, + tuple._1._1._1._1._1._1._2, + tuple._1._1._1._1._1._2, + tuple._1._1._1._1._2, + tuple._1._1._1._2, + tuple._1._1._2, + tuple._1._2, + tuple._2) } - private def getTracingsScalesAndNamesFor(annotations: List[Annotation])( - implicit ctx: DBAccessContext): Fox[List[(List[Option[SkeletonTracing]], - List[Option[VolumeTracing]], - List[Option[Source[ByteString, _]]], - List[String], - List[Option[Scale]], - List[Annotation], - List[User], - List[Option[Task]], - List[String])]] = { + private def getTracingsScalesAndNamesFor(annotations: List[Annotation])(implicit ctx: DBAccessContext): Fox[ + List[(List[Option[SkeletonTracing]], + List[Option[VolumeTracing]], + List[Option[Source[ByteString, _]]], + List[String], + List[Option[Scale]], + List[Annotation], + List[User], + List[Option[Task]], + List[String])]] = { def getSkeletonTracings(dataSetId: ObjectId, tracingIds: List[Option[String]]): Fox[List[Option[SkeletonTracing]]] = for { dataSet <- dataSetDAO.findOne(dataSetId) tracingStoreClient <- tracingStoreService.clientFor(dataSet) - tracingContainers: List[SkeletonTracings] <- Fox.serialCombined(tracingIds.grouped(1000).toList)(tracingStoreClient.getSkeletonTracings) + tracingContainers: List[SkeletonTracings] <- Fox.serialCombined(tracingIds.grouped(1000).toList)( + tracingStoreClient.getSkeletonTracings) tracingOpts: List[SkeletonTracingOpt] = tracingContainers.flatMap(_.tracings) } yield tracingOpts.map(_.tracing) @@ -422,18 +431,19 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor for { dataSet <- dataSetDAO.findOne(dataSetId) tracingStoreClient <- tracingStoreService.clientFor(dataSet) - tracingContainers: List[VolumeTracings] <- Fox.serialCombined(tracingIds.grouped(1000).toList)(tracingStoreClient.getVolumeTracings) + tracingContainers: List[VolumeTracings] <- Fox.serialCombined(tracingIds.grouped(1000).toList)( + tracingStoreClient.getVolumeTracings) tracingOpts: List[VolumeTracingOpt] = tracingContainers.flatMap(_.tracings) } yield tracingOpts.map(_.tracing) - - def getVolumeDataObjects(dataSetId: ObjectId, tracingIds: List[Option[String]]): Fox[List[Option[Source[ByteString, Any]]]] = + def getVolumeDataObjects(dataSetId: ObjectId, + tracingIds: List[Option[String]]): Fox[List[Option[Source[ByteString, Any]]]] = for { dataSet <- dataSetDAO.findOne(dataSetId) tracingStoreClient <- tracingStoreService.clientFor(dataSet) tracingDataObjects: List[Option[Source[ByteString, Any]]] <- Fox.serialCombined(tracingIds) { case Some(tracingId) => tracingStoreClient.getVolumeData(tracingId).map(Some(_)) - case None => Fox.successful(None) + case None => Fox.successful(None) } } yield tracingDataObjects @@ -456,7 +466,7 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor val annotationsGrouped: Map[ObjectId, List[Annotation]] = annotations.groupBy(_._dataSet) val tracingsGrouped = annotationsGrouped.map { - case (dataSetId, annotations) => { + case (dataSetId, annotations) => for { scale <- getDatasetScale(dataSetId) skeletonTracings <- getSkeletonTracings(dataSetId, annotations.map(_.skeletonTracingId)) @@ -466,13 +476,22 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor taskOpts <- getTasks(annotations.map(_._task)) ?~> "task.notFound" names <- getNames(annotations) organizationNames <- getOrganizationNames(annotations) - } yield (skeletonTracings, volumeTracings, volumeDataObjects, names, annotations.map(a => scale), annotations, users, taskOpts, organizationNames) - } + } yield + (skeletonTracings, + volumeTracings, + volumeDataObjects, + names, + annotations.map(a => scale), + annotations, + users, + taskOpts, + organizationNames) } Fox.combined(tracingsGrouped.toList) } - private def createZip(nmls: List[(Enumerator[Array[Byte]], String, Option[Source[ByteString, _]])], zipFileName: String): Future[TemporaryFile] = { + private def createZip(nmls: List[(Enumerator[Array[Byte]], String, Option[Source[ByteString, _]])], + zipFileName: String): Future[TemporaryFile] = { val zipped = temporaryFileCreator.create(normalize(zipFileName), ".zip") val zipper = ZipIO.startZip(new BufferedOutputStream(new FileOutputStream(new File(zipped.path.toString)))) @@ -509,9 +528,12 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor annotationBase <- baseForTask(task._id) dataSet <- dataSetDAO.findOne(annotationBase._dataSet)(GlobalAccessContext) ?~> "dataSet.notFound" (newSkeletonIdOpt, newVolumeIdOpt) <- tracingFromBase(annotationBase, dataSet) - _ <- Fox.bool2Fox(newSkeletonIdOpt.isDefined || newVolumeIdOpt.isDefined) ?~> "annotation.needsEitherSkeletonOrVolume" - _ <- Fox.runOptional(newSkeletonIdOpt)(newSkeletonId => annotationDAO.updateSkeletonTracingId(annotation._id, newSkeletonId)) - _ <- Fox.runOptional(newVolumeIdOpt)(newVolumeId => annotationDAO.updateVolumeTracingId(annotation._id, newVolumeId)) + _ <- Fox + .bool2Fox(newSkeletonIdOpt.isDefined || newVolumeIdOpt.isDefined) ?~> "annotation.needsEitherSkeletonOrVolume" + _ <- Fox.runOptional(newSkeletonIdOpt)(newSkeletonId => + annotationDAO.updateSkeletonTracingId(annotation._id, newSkeletonId)) + _ <- Fox.runOptional(newVolumeIdOpt)(newVolumeId => + annotationDAO.updateVolumeTracingId(annotation._id, newVolumeId)) } yield () case _ if !annotation.skeletonTracingId.isDefined => Fox.failure("annotation.revert.skeletonOnly") diff --git a/app/models/annotation/TracingStoreRpcClient.scala b/app/models/annotation/TracingStoreRpcClient.scala index b96eb65795..2cf233d7e0 100644 --- a/app/models/annotation/TracingStoreRpcClient.scala +++ b/app/models/annotation/TracingStoreRpcClient.scala @@ -38,16 +38,16 @@ class TracingStoreRpcClient(tracingStore: TracingStore, dataSet: DataSet, rpc: R logger.debug("Called to get multiple SkeletonTracings." + baseInfo) rpc(s"${tracingStore.url}/tracings/skeleton/getMultiple") .addQueryString("token" -> TracingStoreRpcClient.webKnossosToken) - .postJsonWithProtoResponse[List[Option[TracingSelector]], SkeletonTracings](tracingIds.map(id => id.map(TracingSelector(_))))( - SkeletonTracings) + .postJsonWithProtoResponse[List[Option[TracingSelector]], SkeletonTracings](tracingIds.map(id => + id.map(TracingSelector(_))))(SkeletonTracings) } def getVolumeTracings(tracingIds: List[Option[String]]): Fox[VolumeTracings] = { logger.debug("Called to get multiple VolumeTracings." + baseInfo) rpc(s"${tracingStore.url}/tracings/volume/getMultiple") .addQueryString("token" -> TracingStoreRpcClient.webKnossosToken) - .postJsonWithProtoResponse[List[Option[TracingSelector]], VolumeTracings](tracingIds.map(id => id.map(TracingSelector(_))))( - VolumeTracings) + .postJsonWithProtoResponse[List[Option[TracingSelector]], VolumeTracings](tracingIds.map(id => + id.map(TracingSelector(_))))(VolumeTracings) } def saveSkeletonTracing(tracing: SkeletonTracing): Fox[String] = { diff --git a/app/models/task/TaskType.scala b/app/models/task/TaskType.scala index c98922cdaf..d695c68276 100755 --- a/app/models/task/TaskType.scala +++ b/app/models/task/TaskType.scala @@ -36,7 +36,13 @@ class TaskTypeService @Inject()(teamDAO: TeamDAO)(implicit ec: ExecutionContext) recommendedConfiguration: Option[JsValue], tracingType: String ): TaskType = - TaskType(ObjectId.generate, ObjectId(team), summary, description, settings, recommendedConfiguration, TracingType.fromString(tracingType).get) + TaskType(ObjectId.generate, + ObjectId(team), + summary, + description, + settings, + recommendedConfiguration, + TracingType.fromString(tracingType).get) def publicWrites(taskType: TaskType)(implicit ctx: DBAccessContext): Fox[JsObject] = for { From 288da2dfde6734a01741cb64e2a25c87cd53a7b3 Mon Sep 17 00:00:00 2001 From: Florian M Date: Fri, 1 Feb 2019 15:57:22 +0100 Subject: [PATCH 09/31] add volume data to multi-task zip downloads --- app/controllers/AnnotationIOController.scala | 2 ++ app/controllers/TaskTypeController.scala | 2 +- app/models/annotation/AnnotationService.scala | 26 +++++++++++++------ app/models/annotation/nml/NmlWriter.scala | 10 ++++--- app/models/task/TaskType.scala | 4 +-- conf/messages | 2 ++ test/backend/NMLUnitTestSuite.scala | 2 +- .../tracingstore/tracings/TracingType.scala | 2 +- 8 files changed, 33 insertions(+), 17 deletions(-) diff --git a/app/controllers/AnnotationIOController.scala b/app/controllers/AnnotationIOController.scala index a54e3e0ebc..906215e3c2 100755 --- a/app/controllers/AnnotationIOController.scala +++ b/app/controllers/AnnotationIOController.scala @@ -219,6 +219,7 @@ class AnnotationIOController @Inject()(nmlWriter: NmlWriter, None, Some(annotation), dataSet.scale, + None, organizationName, Some(user), taskOpt), @@ -248,6 +249,7 @@ class AnnotationIOController @Inject()(nmlWriter: NmlWriter, Some(volumeTracing), Some(annotation), dataSet.scale, + None, organizationName, Some(user), taskOpt)), diff --git a/app/controllers/TaskTypeController.scala b/app/controllers/TaskTypeController.scala index 2dd2a1c9c3..50e95dabf4 100755 --- a/app/controllers/TaskTypeController.scala +++ b/app/controllers/TaskTypeController.scala @@ -32,7 +32,7 @@ class TaskTypeController @Inject()(taskTypeDAO: TaskTypeDAO, (__ \ 'teamId).read[String](ObjectId.stringObjectIdReads("teamId")) and (__ \ 'settings).read[AnnotationSettings] and (__ \ 'recommendedConfiguration).readNullable[JsValue] and - (__ \ 'tracingType).read[String])(taskTypeService.fromForm _) + (__ \ 'tracingType).read[TracingType.Value])(taskTypeService.fromForm _) def create = sil.SecuredAction.async(parse.json) { implicit request => withJsonBodyUsing(taskTypePublicReads) { taskType => diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index a4e5ea51a3..8004186242 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -2,7 +2,9 @@ package models.annotation import java.io.{BufferedOutputStream, File, FileOutputStream} -import akka.stream.scaladsl.Source +import akka.actor.ActorSystem +import akka.stream.ActorMaterializer +import akka.stream.scaladsl.{Source, StreamConverters} import akka.util.ByteString import com.scalableminds.util.geometry.{BoundingBox, Point3D, Scale, Vector3D} import com.scalableminds.util.io.{NamedEnumeratorStream, ZipIO} @@ -11,10 +13,7 @@ import com.scalableminds.util.mvc.Formatter import com.scalableminds.util.tools.{BoxImplicits, Fox, FoxImplicits, TextUtils} import com.scalableminds.webknossos.tracingstore.SkeletonTracing._ import com.scalableminds.webknossos.tracingstore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings} -import com.scalableminds.webknossos.datastore.models.datasource.{ - DataSourceLike => DataSource, - SegmentationLayerLike => SegmentationLayer -} +import com.scalableminds.webknossos.datastore.models.datasource.{DataSourceLike => DataSource, SegmentationLayerLike => SegmentationLayer} import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.webknossos.tracingstore.tracings.skeleton.{NodeDefaults, SkeletonTracingDefaults} import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeTracingDefaults @@ -66,6 +65,9 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor with ProtoGeometryImplicits with LazyLogging { + implicit val actorSystem = ActorSystem() + implicit val materializer = ActorMaterializer() + private def selectSuitableTeam(user: User, dataSet: DataSet)(implicit ctx: DBAccessContext): Fox[ObjectId] = (for { userTeamIds <- userService.teamIdsFor(user._id) @@ -373,7 +375,7 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor tracingsAndNamesFlattened = flattenTupledLists(tracingsNamesAndScalesAsTuples) nmlsAndNames = tracingsAndNamesFlattened.map( tuple => - (nmlWriter.toNmlStream(tuple._1, tuple._2, Some(tuple._6), tuple._5, tuple._9, Some(tuple._7), tuple._8), + (nmlWriter.toNmlStream(tuple._1, tuple._2, Some(tuple._6), tuple._5, Some(tuple._4 + "_data.zip"), tuple._9, Some(tuple._7), tuple._8), tuple._4, tuple._3)) zip <- createZip(nmlsAndNames, zipFileName) @@ -497,8 +499,16 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor def addToZip(nmls: List[(Enumerator[Array[Byte]], String, Option[Source[ByteString, _]])]): Future[Boolean] = nmls match { - case head :: tail => - zipper.withFile(head._2 + ".nml")(NamedEnumeratorStream("", head._1).writeTo).flatMap(_ => addToZip(tail)) + case head :: tail => { + val dataEnumeratorOpt: Option[Enumerator[Array[Byte]]] = head._3.map(volumeData => Enumerator.fromStream(volumeData.runWith(StreamConverters.asInputStream()))) + val writeVolumeDataResult = dataEnumeratorOpt match { + case Some(dataEnumerator) => zipper.withFile(head._2 + "_data.zip")(NamedEnumeratorStream("", dataEnumerator).writeTo) + case None => Future.successful(()) + } + writeVolumeDataResult. + flatMap(_ => zipper.withFile(head._2 + ".nml")(NamedEnumeratorStream("", head._1).writeTo)) + .flatMap(_ => addToZip(tail)) + } case _ => Future.successful(true) } diff --git a/app/models/annotation/nml/NmlWriter.scala b/app/models/annotation/nml/NmlWriter.scala index 8bf945ef3b..03e84919b0 100644 --- a/app/models/annotation/nml/NmlWriter.scala +++ b/app/models/annotation/nml/NmlWriter.scala @@ -38,6 +38,7 @@ class NmlWriter @Inject()(implicit ec: ExecutionContext) extends FoxImplicits { volumeTracing: Option[VolumeTracing], annotation: Option[Annotation], scale: Option[Scale], + volumeFilename: Option[String], organizationName: String, annotationOwner: Option[User], annotationTask: Option[Task]): Enumerator[Array[Byte]] = Enumerator.outputStream { os => @@ -45,7 +46,7 @@ class NmlWriter @Inject()(implicit ec: ExecutionContext) extends FoxImplicits { new IndentingXMLStreamWriter(outputService.createXMLStreamWriter(os)) for { - nml <- toNml(skeletonTracing, volumeTracing, annotation, scale, organizationName, annotationOwner, annotationTask) + nml <- toNml(skeletonTracing, volumeTracing, annotation, scale, volumeFilename, organizationName, annotationOwner, annotationTask) _ = os.close() } yield nml } @@ -54,6 +55,7 @@ class NmlWriter @Inject()(implicit ec: ExecutionContext) extends FoxImplicits { volumeTracingOpt: Option[VolumeTracing], annotation: Option[Annotation], scale: Option[Scale], + volumeFilename: Option[String], organizationName: String, annotationOwner: Option[User], annotationTask: Option[Task])(implicit writer: XMLStreamWriter): Fox[Unit] = @@ -68,7 +70,7 @@ class NmlWriter @Inject()(implicit ec: ExecutionContext) extends FoxImplicits { scale).toFox _ = writeParameters(parameters) _ = skeletonTracingOpt.foreach(writeSkeletonThings) - _ = volumeTracingOpt.foreach(writeVolumeThings) + _ = volumeTracingOpt.foreach(writeVolumeThings(_, volumeFilename)) } yield () } _ = writer.writeEndDocument() @@ -173,10 +175,10 @@ class NmlWriter @Inject()(implicit ec: ExecutionContext) extends FoxImplicits { } } - def writeVolumeThings(volumeTracing: VolumeTracing)(implicit writer: XMLStreamWriter): Unit = + def writeVolumeThings(volumeTracing: VolumeTracing, volumeFilename: Option[String])(implicit writer: XMLStreamWriter): Unit = Xml.withinElementSync("volume") { writer.writeAttribute("id", "1") - writer.writeAttribute("location", "data.zip") + writer.writeAttribute("location", volumeFilename.getOrElse("data.zip")) volumeTracing.fallbackLayer.foreach(writer.writeAttribute("fallbackLayer", _)) } diff --git a/app/models/task/TaskType.scala b/app/models/task/TaskType.scala index d695c68276..22b622549d 100755 --- a/app/models/task/TaskType.scala +++ b/app/models/task/TaskType.scala @@ -34,7 +34,7 @@ class TaskTypeService @Inject()(teamDAO: TeamDAO)(implicit ec: ExecutionContext) team: String, settings: AnnotationSettings, recommendedConfiguration: Option[JsValue], - tracingType: String + tracingType: TracingType.Value ): TaskType = TaskType(ObjectId.generate, ObjectId(team), @@ -42,7 +42,7 @@ class TaskTypeService @Inject()(teamDAO: TeamDAO)(implicit ec: ExecutionContext) description, settings, recommendedConfiguration, - TracingType.fromString(tracingType).get) + tracingType) def publicWrites(taskType: TaskType)(implicit ctx: DBAccessContext): Fox[JsObject] = for { diff --git a/conf/messages b/conf/messages index 917701b424..feb409228d 100644 --- a/conf/messages +++ b/conf/messages @@ -226,6 +226,8 @@ taskType.noAnnotations=We couldn’t find finished annotations for this task typ taskType.id.invalid=The provided task type id is invalid. taskType.tracingTypeImmutable=Tracing types of task types are immutable. Consider creating a new task type. +tracingType.invalid=The provided tracing type is invalid. + mail.welcome.subject = Welcome to ScalableMinds script.notFound=Script couldn’t be found diff --git a/test/backend/NMLUnitTestSuite.scala b/test/backend/NMLUnitTestSuite.scala index 41916ab640..fa2644485b 100644 --- a/test/backend/NMLUnitTestSuite.scala +++ b/test/backend/NMLUnitTestSuite.scala @@ -21,7 +21,7 @@ class NMLUnitTestSuite extends FlatSpec { def getObjectId = ObjectId.generate def writeAndParseTracing(skeletonTracing: SkeletonTracing): Box[(Option[SkeletonTracing], Option[(VolumeTracing, String)], String, Option[String])] = { - val nmlEnumarator = new NmlWriter().toNmlStream(Some(skeletonTracing), None, None, None, "testOrganization", None, None) + val nmlEnumarator = new NmlWriter().toNmlStream(Some(skeletonTracing), None, None, None, None, "testOrganization", None, None) val arrayFuture = Iteratee.flatten(nmlEnumarator |>> Iteratee.consume[Array[Byte]]()).run val array = Await.result(arrayFuture, Duration.Inf) NmlParser.parse("", new ByteArrayInputStream(array)) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingType.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingType.scala index b86023b6d6..141dddc873 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingType.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingType.scala @@ -1,6 +1,6 @@ package com.scalableminds.webknossos.tracingstore.tracings -import play.api.libs.json.{Format, Json, Reads, Writes} +import play.api.libs.json._ object TracingType extends Enumeration { val skeleton, volume, hybrid = Value From 06958fff8af569136f46a225baefb76b06c5b13d Mon Sep 17 00:00:00 2001 From: Florian M Date: Fri, 1 Feb 2019 15:59:02 +0100 Subject: [PATCH 10/31] scalafmt --- app/models/annotation/AnnotationService.scala | 24 ++++++++++++++----- app/models/annotation/nml/NmlWriter.scala | 12 ++++++++-- app/models/task/TaskType.scala | 8 +------ 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index 8004186242..db30162ca7 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -13,7 +13,10 @@ import com.scalableminds.util.mvc.Formatter import com.scalableminds.util.tools.{BoxImplicits, Fox, FoxImplicits, TextUtils} import com.scalableminds.webknossos.tracingstore.SkeletonTracing._ import com.scalableminds.webknossos.tracingstore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings} -import com.scalableminds.webknossos.datastore.models.datasource.{DataSourceLike => DataSource, SegmentationLayerLike => SegmentationLayer} +import com.scalableminds.webknossos.datastore.models.datasource.{ + DataSourceLike => DataSource, + SegmentationLayerLike => SegmentationLayer +} import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.webknossos.tracingstore.tracings.skeleton.{NodeDefaults, SkeletonTracingDefaults} import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeTracingDefaults @@ -375,7 +378,14 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor tracingsAndNamesFlattened = flattenTupledLists(tracingsNamesAndScalesAsTuples) nmlsAndNames = tracingsAndNamesFlattened.map( tuple => - (nmlWriter.toNmlStream(tuple._1, tuple._2, Some(tuple._6), tuple._5, Some(tuple._4 + "_data.zip"), tuple._9, Some(tuple._7), tuple._8), + (nmlWriter.toNmlStream(tuple._1, + tuple._2, + Some(tuple._6), + tuple._5, + Some(tuple._4 + "_data.zip"), + tuple._9, + Some(tuple._7), + tuple._8), tuple._4, tuple._3)) zip <- createZip(nmlsAndNames, zipFileName) @@ -500,13 +510,15 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor def addToZip(nmls: List[(Enumerator[Array[Byte]], String, Option[Source[ByteString, _]])]): Future[Boolean] = nmls match { case head :: tail => { - val dataEnumeratorOpt: Option[Enumerator[Array[Byte]]] = head._3.map(volumeData => Enumerator.fromStream(volumeData.runWith(StreamConverters.asInputStream()))) + val dataEnumeratorOpt: Option[Enumerator[Array[Byte]]] = + head._3.map(volumeData => Enumerator.fromStream(volumeData.runWith(StreamConverters.asInputStream()))) val writeVolumeDataResult = dataEnumeratorOpt match { - case Some(dataEnumerator) => zipper.withFile(head._2 + "_data.zip")(NamedEnumeratorStream("", dataEnumerator).writeTo) + case Some(dataEnumerator) => + zipper.withFile(head._2 + "_data.zip")(NamedEnumeratorStream("", dataEnumerator).writeTo) case None => Future.successful(()) } - writeVolumeDataResult. - flatMap(_ => zipper.withFile(head._2 + ".nml")(NamedEnumeratorStream("", head._1).writeTo)) + writeVolumeDataResult + .flatMap(_ => zipper.withFile(head._2 + ".nml")(NamedEnumeratorStream("", head._1).writeTo)) .flatMap(_ => addToZip(tail)) } case _ => diff --git a/app/models/annotation/nml/NmlWriter.scala b/app/models/annotation/nml/NmlWriter.scala index 03e84919b0..b34e76e24a 100644 --- a/app/models/annotation/nml/NmlWriter.scala +++ b/app/models/annotation/nml/NmlWriter.scala @@ -46,7 +46,14 @@ class NmlWriter @Inject()(implicit ec: ExecutionContext) extends FoxImplicits { new IndentingXMLStreamWriter(outputService.createXMLStreamWriter(os)) for { - nml <- toNml(skeletonTracing, volumeTracing, annotation, scale, volumeFilename, organizationName, annotationOwner, annotationTask) + nml <- toNml(skeletonTracing, + volumeTracing, + annotation, + scale, + volumeFilename, + organizationName, + annotationOwner, + annotationTask) _ = os.close() } yield nml } @@ -175,7 +182,8 @@ class NmlWriter @Inject()(implicit ec: ExecutionContext) extends FoxImplicits { } } - def writeVolumeThings(volumeTracing: VolumeTracing, volumeFilename: Option[String])(implicit writer: XMLStreamWriter): Unit = + def writeVolumeThings(volumeTracing: VolumeTracing, volumeFilename: Option[String])( + implicit writer: XMLStreamWriter): Unit = Xml.withinElementSync("volume") { writer.writeAttribute("id", "1") writer.writeAttribute("location", volumeFilename.getOrElse("data.zip")) diff --git a/app/models/task/TaskType.scala b/app/models/task/TaskType.scala index 22b622549d..177d4e6d21 100755 --- a/app/models/task/TaskType.scala +++ b/app/models/task/TaskType.scala @@ -36,13 +36,7 @@ class TaskTypeService @Inject()(teamDAO: TeamDAO)(implicit ec: ExecutionContext) recommendedConfiguration: Option[JsValue], tracingType: TracingType.Value ): TaskType = - TaskType(ObjectId.generate, - ObjectId(team), - summary, - description, - settings, - recommendedConfiguration, - tracingType) + TaskType(ObjectId.generate, ObjectId(team), summary, description, settings, recommendedConfiguration, tracingType) def publicWrites(taskType: TaskType)(implicit ctx: DBAccessContext): Fox[JsObject] = for { From 05d41c387c15d5fb379cdf29753aa494ffc989eb Mon Sep 17 00:00:00 2001 From: Florian M Date: Fri, 1 Feb 2019 15:59:55 +0100 Subject: [PATCH 11/31] add tasktype tracing type to test db --- test/db/taskTypes.csv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/db/taskTypes.csv b/test/db/taskTypes.csv index 6c619d81ad..4dfbe5ed7d 100644 --- a/test/db/taskTypes.csv +++ b/test/db/taskTypes.csv @@ -1,3 +1,3 @@ -_id,_team,summary,description,settings_allowedModes,settings_preferredMode,settings_branchPointsAllowed,settings_somaClickingAllowed,created,isDeleted -'570b9f4c2a7c0e4c008da6ee','570b9f4b2a7c0e3b008da6ec','ek_0563_BipolarCells','Check those cells out!','{orthogonal,oblique,flight}',,t,t,,'2016-04-11T12:57:48.000Z',f -'570b9f4c2a7c0e4c008da6ff','69882b370d889b84020efd4f','ek_0674_BipolarCells','Check those cells out!','{orthogonal,oblique,flight}',,t,t,,'2016-04-11T12:57:48.000Z',f +_id,_team,summary,description,settings_allowedModes,settings_preferredMode,settings_branchPointsAllowed,settings_somaClickingAllowed,tracingType,created,isDeleted +'570b9f4c2a7c0e4c008da6ee','570b9f4b2a7c0e3b008da6ec','ek_0563_BipolarCells','Check those cells out!','{orthogonal,oblique,flight}',,t,t,,'skeleton','2016-04-11T12:57:48.000Z',f +'570b9f4c2a7c0e4c008da6ff','69882b370d889b84020efd4f','ek_0674_BipolarCells','Check those cells out!','{orthogonal,oblique,flight}',,t,t,,'skeleton','2016-04-11T12:57:48.000Z',f From 7b4e31273236a3a3be5381e54edf69bfb3b09559 Mon Sep 17 00:00:00 2001 From: Florian M Date: Fri, 1 Feb 2019 16:47:44 +0100 Subject: [PATCH 12/31] validate tracing type when reading task type from db --- app/models/task/TaskType.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/models/task/TaskType.scala b/app/models/task/TaskType.scala index 177d4e6d21..df806156cc 100755 --- a/app/models/task/TaskType.scala +++ b/app/models/task/TaskType.scala @@ -62,7 +62,9 @@ class TaskTypeDAO @Inject()(sqlClient: SQLClient)(implicit ec: ExecutionContext) def isDeletedColumn(x: Tasktypes): Rep[Boolean] = x.isdeleted def parse(r: TasktypesRow): Fox[TaskType] = - Some( + for { + tracingType <- TracingType.fromString(r.tracingtype) ?~> "failed to parse tracing type" + } yield TaskType( ObjectId(r._Id), ObjectId(r._Team), @@ -75,10 +77,10 @@ class TaskTypeDAO @Inject()(sqlClient: SQLClient)(implicit ec: ExecutionContext) r.settingsSomaclickingallowed ), r.recommendedconfiguration.map(Json.parse), - TracingType.fromString(r.tracingtype).get, + tracingType, r.created.getTime, r.isdeleted - )) + ) override def readAccessQ(requestingUserId: ObjectId) = s"""(_team in (select _team from webknossos.user_team_roles where _user = '${requestingUserId.id}') From 2a58ae9a1c55584ae49d95e567a72d2867c3b1c8 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Fri, 1 Feb 2019 17:17:25 +0100 Subject: [PATCH 13/31] adapt front-end to volume task types --- .../javascripts/admin/api_flow_types.js | 1 + .../admin/task/task_create_form_view.js | 18 +----- .../admin/tasktype/task_type_create_view.js | 55 ++++++++++++++----- .../admin/tasktype/task_type_list_view.js | 24 ++++++-- 4 files changed, 60 insertions(+), 38 deletions(-) diff --git a/app/assets/javascripts/admin/api_flow_types.js b/app/assets/javascripts/admin/api_flow_types.js index 3d80c613c0..b6e68c3898 100644 --- a/app/assets/javascripts/admin/api_flow_types.js +++ b/app/assets/javascripts/admin/api_flow_types.js @@ -210,6 +210,7 @@ export type APITaskType = { +teamName: string, +settings: APISettings, +recommendedConfiguration: ?string, + +tracingType: "skeleton" | "volume", }; export type TaskStatus = { +open: number, +active: number, +finished: number }; diff --git a/app/assets/javascripts/admin/task/task_create_form_view.js b/app/assets/javascripts/admin/task/task_create_form_view.js index 32f6ff3eb6..cf595b745a 100644 --- a/app/assets/javascripts/admin/task/task_create_form_view.js +++ b/app/assets/javascripts/admin/task/task_create_form_view.js @@ -160,8 +160,7 @@ class TaskCreateFormView extends React.PureComponent { formValues.nmlFiles = formValues.nmlFiles.map( wrapperFile => wrapperFile.originFileObj, ); - const { typ, ...formValuesWithoutTyp } = formValues; - response = await createTaskFromNML(formValuesWithoutTyp); + response = await createTaskFromNML(formValues); } else { response = await createTasks([formValues]); } @@ -348,21 +347,6 @@ class TaskCreateFormView extends React.PureComponent { )} - - {getFieldDecorator("typ", { - initialValue: "skeleton", - })( - - - Skeleton - - - Volume - - , - )} - - {getFieldDecorator("editPosition", { rules: [{ required: true }], diff --git a/app/assets/javascripts/admin/tasktype/task_type_create_view.js b/app/assets/javascripts/admin/tasktype/task_type_create_view.js index 626d190193..4b3045805f 100644 --- a/app/assets/javascripts/admin/tasktype/task_type_create_view.js +++ b/app/assets/javascripts/admin/tasktype/task_type_create_view.js @@ -1,5 +1,5 @@ // @flow -import { Form, Checkbox, Input, Select, Card, Button } from "antd"; +import { Button, Card, Checkbox, Form, Input, Radio, Select } from "antd"; import { type RouterHistory, withRouter } from "react-router-dom"; import React from "react"; import _ from "lodash"; @@ -16,6 +16,8 @@ import RecommendedConfigurationView, { DEFAULT_RECOMMENDED_CONFIGURATION, } from "admin/tasktype/recommended_configuration_view"; +const RadioGroup = Radio.Group; + const FormItem = Form.Item; const { Option } = Select; const { TextArea } = Input; @@ -94,10 +96,11 @@ class TaskTypeCreateView extends React.PureComponent { render() { const { getFieldDecorator } = this.props.form; - const titlePrefix = this.props.taskTypeId ? "Update " : "Create"; + const isEditingMode = this.props.taskTypeId != null; + const titlePrefix = isEditingMode ? "Update " : "Create"; return ( -
+
{titlePrefix} Task Type}>
@@ -152,6 +155,21 @@ class TaskTypeCreateView extends React.PureComponent { })(