From 6983a80cc4c5d3e2262e4d6bb3edd25cf9f44885 Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 14 Oct 2020 15:49:42 +0200 Subject: [PATCH 1/3] backend nml parser performance improvements --- app/Startup.scala | 4 +- app/controllers/AnnotationIOController.scala | 19 ++- app/models/annotation/nml/NmlParser.scala | 146 +++++++++++------- build.sbt | 3 + conf/application.conf | 4 +- tools/proxy/proxy.js | 2 +- .../services/DataSourceService.scala | 4 +- 7 files changed, 114 insertions(+), 68 deletions(-) diff --git a/app/Startup.scala b/app/Startup.scala index c300589f5e..5ee31d1f75 100755 --- a/app/Startup.scala +++ b/app/Startup.scala @@ -45,13 +45,13 @@ class Startup @Inject()(actorSystem: ActorSystem, annotationDAO.deleteOldInitializingAnnotations } - ensurePostgresDatabase.onComplete { _ => + /*ensurePostgresDatabase.onComplete { _ => initialDataService.insert.futureBox.map { case Full(_) => () case Failure(msg, _, _) => logger.info("No initial data inserted: " + msg) case _ => logger.warn("Error while inserting initial data") } - } + }*/ lifecycle.addStopHook { () => Future.successful { diff --git a/app/controllers/AnnotationIOController.scala b/app/controllers/AnnotationIOController.scala index 083aa8f2b8..56eb3d9de4 100755 --- a/app/controllers/AnnotationIOController.scala +++ b/app/controllers/AnnotationIOController.scala @@ -63,19 +63,23 @@ class AnnotationIOController @Inject()(nmlWriter: NmlWriter, def upload: Action[MultipartFormData[TemporaryFile]] = sil.SecuredAction.async(parse.multipartFormData) { implicit request => log { + logger.info("upload request received") val shouldCreateGroupForEachFile: Boolean = request.body.dataParts("createGroupForEachFile").headOption.contains("true") val overwritingDataSetName: Option[String] = request.body.dataParts.get("datasetName").flatMap(_.headOption) val attachedFiles = request.body.files.map(f => (new File(f.ref.path.toString), f.filename)) val parsedFiles = nmlService.extractFromFiles(attachedFiles, useZipName = true, overwritingDataSetName) + logger.info("wrap...") val tracingsProcessed = nmlService.wrapOrPrefixTrees(parsedFiles.parseResults, shouldCreateGroupForEachFile) + logger.info("wrap done") val parseSuccesses: List[NmlParseResult] = tracingsProcessed.filter(_.succeeded) if (parseSuccesses.isEmpty) { returnError(parsedFiles) } else { + logger.info("sort, extract name, description") val (skeletonTracings, volumeTracingsWithDataLocations) = extractTracings(parseSuccesses) val name = nameForUploaded(parseSuccesses.map(_.fileName)) val description = descriptionForNMLs(parseSuccesses.map(_.description)) @@ -97,11 +101,13 @@ class AnnotationIOController @Inject()(nmlWriter: NmlWriter, ) } yield mergedIdOpt } + _ <- Fox.successful(logger.info("send to tracingstore...")) mergedSkeletonTracingIdOpt <- Fox.runOptional(skeletonTracings.headOption) { _ => - tracingStoreClient.mergeSkeletonTracingsByContents( + time(silent=false, "send to tracingstore")(tracingStoreClient.mergeSkeletonTracingsByContents( SkeletonTracings(skeletonTracings.map(t => SkeletonTracingOpt(Some(t)))), - persistTracing = true) + persistTracing = true)) } + _ <- Fox.successful(logger.info("register to annotation db...")) annotation <- annotationService.createFrom(request.identity, dataSet, mergedSkeletonTracingIdOpt, @@ -118,6 +124,15 @@ class AnnotationIOController @Inject()(nmlWriter: NmlWriter, } } + def time[R](silent: Boolean, label: String)(block: => R): R = { + val t0 = System.nanoTime() + val result = block // call-by-name + val t1 = System.nanoTime() + val duration = (t1 - t0)/1000000 + if (!silent && t1 - t0 > 5000000) {println(f"$duration ms " + label)} + result + } + private def findDataSetForUploadedAnnotations( skeletonTracings: List[SkeletonTracing], volumeTracings: List[VolumeTracing], diff --git a/app/models/annotation/nml/NmlParser.scala b/app/models/annotation/nml/NmlParser.scala index d525f2af8d..82978eaa9a 100755 --- a/app/models/annotation/nml/NmlParser.scala +++ b/app/models/annotation/nml/NmlParser.scala @@ -7,20 +7,16 @@ import com.scalableminds.webknossos.tracingstore.SkeletonTracing._ import com.scalableminds.webknossos.tracingstore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.tracingstore.geometry.{Color, NamedBoundingBox} import com.scalableminds.webknossos.tracingstore.tracings.{ColorGenerator, ProtoGeometryImplicits} -import com.scalableminds.webknossos.tracingstore.tracings.skeleton.{ - MultiComponentTreeSplitter, - NodeDefaults, - SkeletonTracingDefaults, - TreeValidator -} +import com.scalableminds.webknossos.tracingstore.tracings.skeleton.{MultiComponentTreeSplitter, NodeDefaults, SkeletonTracingDefaults, TreeValidator} import com.scalableminds.webknossos.tracingstore.tracings.volume.Volume import com.scalableminds.util.geometry.{BoundingBox, Point3D, Scale, Vector3D} -import com.scalableminds.util.tools.ExtendedTypes.{ExtendedString, ExtendedDouble} +import com.scalableminds.util.tools.ExtendedTypes.{ExtendedDouble, ExtendedString} import com.typesafe.scalalogging.LazyLogging import net.liftweb.common.Box._ -import net.liftweb.common.{Box, Empty, Failure} +import net.liftweb.common.{Box, Empty, Failure, Full} import play.api.i18n.{Messages, MessagesProvider} +import scala.collection.mutable import scala.xml.{NodeSeq, XML, Node => XMLNode} object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGenerator { @@ -41,25 +37,34 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener val DEFAULT_TIMESTAMP = 0L + def time[R](silent: Boolean, label: String)(block: => R): R = { + val t0 = System.nanoTime() + val result = block // call-by-name + val t1 = System.nanoTime() + val duration = (t1 - t0)/1000000 + if (!silent && t1 - t0 > 5000000) {println(f"$duration ms " + label)} + result + } + @SuppressWarnings(Array("TraversableHead")) //We check if volumes are empty before accessing the head def parse(name: String, nmlInputStream: InputStream, overwritingDataSetName: Option[String], isTaskUpload: Boolean)( implicit m: MessagesProvider) : Box[(Option[SkeletonTracing], Option[(VolumeTracing, String)], String, Option[String])] = try { - val data = XML.load(nmlInputStream) + val data = time(silent=false, "load xml from input stream")(XML.load(nmlInputStream)) for { parameters <- (data \ "parameters").headOption ?~ Messages("nml.parameters.notFound") scale <- parseScale(parameters \ "scale") ?~ Messages("nml.scale.invalid") - time = parseTime(parameters \ "time") + timestamp = parseTime(parameters \ "time") comments <- parseComments(data \ "comments") - branchPoints <- parseBranchPoints(data \ "branchpoints", time) - trees <- parseTrees(data \ "thing", branchPoints, comments) + branchPoints <- parseBranchPoints(data \ "branchpoints", timestamp) + trees <- time(silent=false, "parse trees")(parseTrees(data \ "thing", buildBranchPointMap(branchPoints), buildCommentMap(comments))) treeGroups <- extractTreeGroups(data \ "groups") volumes = extractVolumes(data \ "volume") - treesAndGroupsAfterSplitting = MultiComponentTreeSplitter.splitMulticomponentTrees(trees, treeGroups) + treesAndGroupsAfterSplitting = time(silent=false, "splitMultiComponentTrees")(MultiComponentTreeSplitter.splitMulticomponentTrees(trees, treeGroups)) treesSplit = treesAndGroupsAfterSplitting._1 treeGroupsAfterSplit = treesAndGroupsAfterSplitting._2 - _ <- TreeValidator.validateTrees(treesSplit, treeGroupsAfterSplit, branchPoints, comments) + _ <- time(silent=false, "validateTrees")(TreeValidator.validateTrees(treesSplit, treeGroupsAfterSplit, branchPoints, comments)) } yield { val dataSetName = overwritingDataSetName.getOrElse(parseDataSetName(parameters \ "experiment")) val description = parseDescription(parameters \ "experiment") @@ -87,7 +92,7 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener (VolumeTracing( None, boundingBoxToProto(taskBoundingBox.getOrElse(BoundingBox.empty)), - time, + timestamp, dataSetName, editPosition, editRotation, @@ -108,7 +113,7 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener Some( SkeletonTracing(dataSetName, treesSplit, - time, + timestamp, taskBoundingBox, activeNodeId, editPosition, @@ -151,7 +156,7 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener def extractVolumes(volumeNodes: NodeSeq) = volumeNodes.map(node => Volume((node \ "@location").text, (node \ "@fallbackLayer").map(_.text).headOption)) - private def parseTrees(treeNodes: NodeSeq, branchPoints: Seq[BranchPoint], comments: Seq[Comment])( + private def parseTrees(treeNodes: NodeSeq, branchPoints: Map[Int, List[BranchPoint]], comments: Map[Int, List[Comment]])( implicit m: MessagesProvider) = treeNodes .map(treeNode => parseTree(treeNode, branchPoints, comments)) @@ -232,9 +237,9 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener }.toList.toSingleBox(Messages("nml.element.invalid", "branchpoints")) private def parsePoint3D(node: XMLNode) = { - val xText = (node \ "@x").text - val yText = (node \ "@y").text - val zText = (node \ "@z").text + val xText = getSingleAttribute(node, "x") + val yText = getSingleAttribute(node, "y") + val zText = getSingleAttribute(node, "z") for { x <- xText.toIntOpt.orElse(xText.toFloatOpt.map(math.round)) y <- yText.toIntOpt.orElse(yText.toFloatOpt.map(math.round)) @@ -251,9 +256,9 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener private def parseRotationForNode(node: XMLNode) = for { - rotX <- (node \ "@rotX").text.toDoubleOpt - rotY <- (node \ "@rotY").text.toDoubleOpt - rotZ <- (node \ "@rotZ").text.toDoubleOpt + rotX <- getSingleAttribute(node, "rotX").toDoubleOpt + rotY <- getSingleAttribute(node, "rotY").toDoubleOpt + rotZ <- getSingleAttribute(node, "rotZ").toDoubleOpt } yield Vector3D(rotX, rotY, rotZ) private def parseScale(nodes: NodeSeq) = @@ -289,68 +294,93 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener case None => color.map(c => !c.a.isNearZero) } - private def parseTree(tree: XMLNode, branchPoints: Seq[BranchPoint], comments: Seq[Comment])( - implicit m: MessagesProvider): Box[Tree] = + private def parseTree(tree: XMLNode, branchPoints: Map[Int, List[BranchPoint]], comments: Map[Int, List[Comment]])( + implicit m: MessagesProvider): Box[Tree] = { for { - id <- (tree \ "@id").text.toIntOpt ?~ Messages("nml.tree.id.invalid", (tree \ "@id").text) + id <- getSingleAttribute(tree, "id").toIntOpt ?~ Messages("nml.tree.id.invalid", (tree \ "@id").text) color = parseColor(tree) name = parseName(tree) groupId = parseGroupId(tree) isVisible = parseVisibility(tree, color) - nodes <- (tree \ "nodes" \ "node") - .map(parseNode) - .toList - .toSingleBox(Messages("nml.tree.elements.invalid", "nodes", id)) - edges <- (tree \ "edges" \ "edge") - .map(parseEdge) - .toList - .toSingleBox(Messages("nml.tree.elements.invalid", "edges", id)) + nodes <- (tree \ "nodes" \ "node").map(parseNode).toList.toSingleBox(Messages("nml.tree.elements.invalid", "nodes", id)) + edges <- (tree \ "edges" \ "edge").map(parseEdge).toList.toSingleBox(Messages("nml.tree.elements.invalid", "edges", id)) nodeIds = nodes.map(_.id) - treeBP = branchPoints.filter(bp => nodeIds.contains(bp.nodeId)).toList - treeComments = comments.filter(bp => nodeIds.contains(bp.nodeId)).toList - createdTimestamp = if (nodes.isEmpty) System.currentTimeMillis() - else nodes.minBy(_.createdTimestamp).createdTimestamp - } yield Tree(id, nodes, edges, color, treeBP, treeComments, name, createdTimestamp, groupId, isVisible) + treeBranchPoints = nodeIds.flatMap(nodeId => branchPoints.getOrElse(nodeId, List())) + treeComments = nodeIds.flatMap(nodeId => comments.getOrElse(nodeId, List())) + createdTimestamp = if (nodes.isEmpty) System.currentTimeMillis() else nodes.minBy(_.createdTimestamp).createdTimestamp + } yield Tree(id, nodes, edges, color, treeBranchPoints, treeComments, name, createdTimestamp, groupId, isVisible) + } - private def parseComments(comments: NodeSeq)(implicit m: MessagesProvider) = + private def parseComments(comments: NodeSeq)(implicit m: MessagesProvider): Box[List[Comment]] = (for { - comment <- comments \ "comment" + commentNode <- comments \ "comment" } yield { for { - nodeId <- (comment \ "@node").text.toIntOpt ?~ Messages("nml.comment.node.invalid", (comment \ "@content").text) + nodeId <- getSingleAttribute(commentNode, "node").toIntOpt ?~ Messages("nml.comment.node.invalid", getSingleAttribute(commentNode, "node")) } yield { - val content = (comment \ "@content").text + val content = getSingleAttribute(commentNode, "content") Comment(nodeId, content) } }).toList.toSingleBox(Messages("nml.element.invalid", "comments")) - private def parseEdge(edge: XMLNode)(implicit m: MessagesProvider): Box[Edge] = + private def buildCommentMap(comments: List[Comment]): Map[Int, List[Comment]] = { + val commentMap = new mutable.HashMap[Int, List[Comment]]() + comments.foreach { c => + if (commentMap.contains(c.nodeId)) { + commentMap(c.nodeId) = c :: commentMap(c.nodeId) + } else { + commentMap(c.nodeId) = List(c) + } + } + commentMap.toMap + } + + private def buildBranchPointMap(branchPoints: List[BranchPoint]): Map[Int, List[BranchPoint]] = { + val branchPointMap = new mutable.HashMap[Int, List[BranchPoint]]() + branchPoints.foreach { b => + if (branchPointMap.contains(b.nodeId)) { + branchPointMap(b.nodeId) = b :: branchPointMap(b.nodeId) + } else { + branchPointMap(b.nodeId) = List(b) + } + } + branchPointMap.toMap + } + + private def getSingleAttribute(xmlNode: XMLNode, attribute: String): String = { + xmlNode.attribute(attribute).flatMap(_.headOption.map(_.text)).getOrElse("") + } + + private def parseEdge(edge: XMLNode)(implicit m: MessagesProvider): Box[Edge] = { + val sourceStr = getSingleAttribute(edge, "source") + val targetStr = getSingleAttribute(edge, "target") for { - source <- (edge \ "@source").text.toIntOpt ?~ Messages("nml.edge.invalid", "source", (edge \ "@source").text) - target <- (edge \ "@target").text.toIntOpt ?~ Messages("nml.edge.invalid", "target", (edge \ "@target").text) + source <- sourceStr.toIntOpt ?~ Messages("nml.edge.invalid", sourceStr) + target <- targetStr.toIntOpt ?~ Messages("nml.edge.invalid", targetStr) } yield { Edge(source, target) } + } - private def parseViewport(node: NodeSeq) = - (node \ "@inVp").text.toIntOpt.getOrElse(DEFAULT_VIEWPORT) + private def parseViewport(node: XMLNode) = + getSingleAttribute(node, "inVp").toIntOpt.getOrElse(DEFAULT_VIEWPORT) - private def parseResolution(node: NodeSeq) = - (node \ "@inMag").text.toIntOpt.getOrElse(DEFAULT_RESOLUTION) + private def parseResolution(node: XMLNode) = + getSingleAttribute(node, "inMag").toIntOpt.getOrElse(DEFAULT_RESOLUTION) - private def parseBitDepth(node: NodeSeq) = - (node \ "@bitDepth").text.toIntOpt.getOrElse(DEFAULT_BITDEPTH) + private def parseBitDepth(node: XMLNode) = + getSingleAttribute(node, "bitDepth").toIntOpt.getOrElse(DEFAULT_BITDEPTH) - private def parseInterpolation(node: NodeSeq) = - (node \ "@interpolation").text.toBooleanOpt.getOrElse(DEFAULT_INTERPOLATION) + private def parseInterpolation(node: XMLNode) = + getSingleAttribute(node, "interpolation").toBooleanOpt.getOrElse(DEFAULT_INTERPOLATION) - private def parseTimestamp(node: NodeSeq) = - (node \ "@time").text.toLongOpt.getOrElse(DEFAULT_TIMESTAMP) + private def parseTimestamp(node: XMLNode) = + getSingleAttribute(node, "time").toLongOpt.getOrElse(DEFAULT_TIMESTAMP) private def parseNode(node: XMLNode)(implicit m: MessagesProvider): Box[Node] = for { - id <- (node \ "@id").text.toIntOpt ?~ Messages("nml.node.id.invalid", "", (node \ "@id").text) - radius <- (node \ "@radius").text.toFloatOpt ?~ Messages("nml.node.attribute.invalid", "radius", id) + id <- getSingleAttribute(node, "id").toIntOpt ?~ Messages("nml.node.id.invalid", "", (node \ "@id").text) + radius <- getSingleAttribute(node, "radius").toFloatOpt ?~ Messages("nml.node.attribute.invalid", "radius", id) position <- parsePoint3D(node) ?~ Messages("nml.node.attribute.invalid", "position", id) } yield { val viewport = parseViewport(node) diff --git a/build.sbt b/build.sbt index 7b72582517..2fc0506acc 100644 --- a/build.sbt +++ b/build.sbt @@ -15,6 +15,9 @@ ThisBuild / scalacOptions ++= Seq( "-language:postfixOps" ) +PlayKeys.devSettings := Seq("play.server.akka.requestTimeout" -> "10000s", + "play.server.http.idleTimeout" -> "10000s") + scapegoatIgnoredFiles := Seq(".*/Tables.scala", ".*/Routes.scala", ".*/ReverseRoutes.scala", diff --git a/conf/application.conf b/conf/application.conf index 3c79b145f4..8acb1da025 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -216,8 +216,8 @@ play.filters.headers.contentTypeOptions = null play.filters.headers.frameOptions = null # Note that these take effect only in production mode (timeouts are shorter in dev) -play.server.http.idleTimeout = 1000s -play.server.akka.requestTimeout = 1000s +play.server.http.idleTimeout = 10000s +play.server.akka.requestTimeout = 10000s # Avoid the creation of a pid file pidfile.path = "/dev/null" diff --git a/tools/proxy/proxy.js b/tools/proxy/proxy.js index 8a1824384e..d8e5aff7ef 100644 --- a/tools/proxy/proxy.js +++ b/tools/proxy/proxy.js @@ -57,7 +57,7 @@ function spawnIfNotSpecified(keyword, command, args, options) { function killAll() { for (const proc of Object.values(processes).filter(x => x)) { if (proc.connected) { - proc.kill(); + proc.kill("SIGKILL"); } } } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala index c03be0ec20..2b7a6a0c51 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala @@ -48,9 +48,7 @@ class DataSourceService @Inject()( var inboxCheckVerboseCounter = 0 def tick: Unit = { - checkInbox(verbose = inboxCheckVerboseCounter == 0) - inboxCheckVerboseCounter += 1 - if (inboxCheckVerboseCounter >= 10) inboxCheckVerboseCounter = 0 + () } private def skipTrash(path: Path) = !path.toString.contains(".trash") From bfedeab3134a057344f24d0d7b68210c9bc9bca6 Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 15 Oct 2020 13:14:51 +0200 Subject: [PATCH 2/3] always get single attributes directly, cleanup --- app/Startup.scala | 4 +- app/controllers/AnnotationIOController.scala | 19 +- app/models/annotation/nml/NmlParser.scala | 233 +++++++++--------- .../model/reducers/skeletontracing_reducer.js | 4 +- .../services/DataSourceService.scala | 3 +- 5 files changed, 123 insertions(+), 140 deletions(-) diff --git a/app/Startup.scala b/app/Startup.scala index 5ee31d1f75..c300589f5e 100755 --- a/app/Startup.scala +++ b/app/Startup.scala @@ -45,13 +45,13 @@ class Startup @Inject()(actorSystem: ActorSystem, annotationDAO.deleteOldInitializingAnnotations } - /*ensurePostgresDatabase.onComplete { _ => + ensurePostgresDatabase.onComplete { _ => initialDataService.insert.futureBox.map { case Full(_) => () case Failure(msg, _, _) => logger.info("No initial data inserted: " + msg) case _ => logger.warn("Error while inserting initial data") } - }*/ + } lifecycle.addStopHook { () => Future.successful { diff --git a/app/controllers/AnnotationIOController.scala b/app/controllers/AnnotationIOController.scala index 56eb3d9de4..083aa8f2b8 100755 --- a/app/controllers/AnnotationIOController.scala +++ b/app/controllers/AnnotationIOController.scala @@ -63,23 +63,19 @@ class AnnotationIOController @Inject()(nmlWriter: NmlWriter, def upload: Action[MultipartFormData[TemporaryFile]] = sil.SecuredAction.async(parse.multipartFormData) { implicit request => log { - logger.info("upload request received") val shouldCreateGroupForEachFile: Boolean = request.body.dataParts("createGroupForEachFile").headOption.contains("true") val overwritingDataSetName: Option[String] = request.body.dataParts.get("datasetName").flatMap(_.headOption) val attachedFiles = request.body.files.map(f => (new File(f.ref.path.toString), f.filename)) val parsedFiles = nmlService.extractFromFiles(attachedFiles, useZipName = true, overwritingDataSetName) - logger.info("wrap...") val tracingsProcessed = nmlService.wrapOrPrefixTrees(parsedFiles.parseResults, shouldCreateGroupForEachFile) - logger.info("wrap done") val parseSuccesses: List[NmlParseResult] = tracingsProcessed.filter(_.succeeded) if (parseSuccesses.isEmpty) { returnError(parsedFiles) } else { - logger.info("sort, extract name, description") val (skeletonTracings, volumeTracingsWithDataLocations) = extractTracings(parseSuccesses) val name = nameForUploaded(parseSuccesses.map(_.fileName)) val description = descriptionForNMLs(parseSuccesses.map(_.description)) @@ -101,13 +97,11 @@ class AnnotationIOController @Inject()(nmlWriter: NmlWriter, ) } yield mergedIdOpt } - _ <- Fox.successful(logger.info("send to tracingstore...")) mergedSkeletonTracingIdOpt <- Fox.runOptional(skeletonTracings.headOption) { _ => - time(silent=false, "send to tracingstore")(tracingStoreClient.mergeSkeletonTracingsByContents( + tracingStoreClient.mergeSkeletonTracingsByContents( SkeletonTracings(skeletonTracings.map(t => SkeletonTracingOpt(Some(t)))), - persistTracing = true)) + persistTracing = true) } - _ <- Fox.successful(logger.info("register to annotation db...")) annotation <- annotationService.createFrom(request.identity, dataSet, mergedSkeletonTracingIdOpt, @@ -124,15 +118,6 @@ class AnnotationIOController @Inject()(nmlWriter: NmlWriter, } } - def time[R](silent: Boolean, label: String)(block: => R): R = { - val t0 = System.nanoTime() - val result = block // call-by-name - val t1 = System.nanoTime() - val duration = (t1 - t0)/1000000 - if (!silent && t1 - t0 > 5000000) {println(f"$duration ms " + label)} - result - } - private def findDataSetForUploadedAnnotations( skeletonTracings: List[SkeletonTracing], volumeTracings: List[VolumeTracing], diff --git a/app/models/annotation/nml/NmlParser.scala b/app/models/annotation/nml/NmlParser.scala index 82978eaa9a..8c3a336798 100755 --- a/app/models/annotation/nml/NmlParser.scala +++ b/app/models/annotation/nml/NmlParser.scala @@ -7,64 +7,51 @@ import com.scalableminds.webknossos.tracingstore.SkeletonTracing._ import com.scalableminds.webknossos.tracingstore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.tracingstore.geometry.{Color, NamedBoundingBox} import com.scalableminds.webknossos.tracingstore.tracings.{ColorGenerator, ProtoGeometryImplicits} -import com.scalableminds.webknossos.tracingstore.tracings.skeleton.{MultiComponentTreeSplitter, NodeDefaults, SkeletonTracingDefaults, TreeValidator} +import com.scalableminds.webknossos.tracingstore.tracings.skeleton.{ + MultiComponentTreeSplitter, + NodeDefaults, + SkeletonTracingDefaults, + TreeValidator +} import com.scalableminds.webknossos.tracingstore.tracings.volume.Volume -import com.scalableminds.util.geometry.{BoundingBox, Point3D, Scale, Vector3D} +import com.scalableminds.util.geometry.{BoundingBox, Point3D, Vector3D} import com.scalableminds.util.tools.ExtendedTypes.{ExtendedDouble, ExtendedString} import com.typesafe.scalalogging.LazyLogging import net.liftweb.common.Box._ -import net.liftweb.common.{Box, Empty, Failure, Full} +import net.liftweb.common.{Box, Empty, Failure} import play.api.i18n.{Messages, MessagesProvider} -import scala.collection.mutable +import scala.collection.{immutable, mutable} import scala.xml.{NodeSeq, XML, Node => XMLNode} object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGenerator { - val DEFAULT_TIME = 0L - - val DEFAULT_ACTIVE_NODE_ID = 1 - - val DEFAULT_COLOR = Color(1, 0, 0, 0) - - val DEFAULT_VIEWPORT = 0 - - val DEFAULT_RESOLUTION = 0 - - val DEFAULT_BITDEPTH = 0 - - val DEFAULT_INTERPOLATION = false - - val DEFAULT_TIMESTAMP = 0L - - def time[R](silent: Boolean, label: String)(block: => R): R = { - val t0 = System.nanoTime() - val result = block // call-by-name - val t1 = System.nanoTime() - val duration = (t1 - t0)/1000000 - if (!silent && t1 - t0 > 5000000) {println(f"$duration ms " + label)} - result - } + private val DEFAULT_TIME = 0L + private val DEFAULT_VIEWPORT = 0 + private val DEFAULT_RESOLUTION = 0 + private val DEFAULT_BITDEPTH = 0 + private val DEFAULT_DESCRIPTION = "" + private val DEFAULT_INTERPOLATION = false + private val DEFAULT_TIMESTAMP = 0L @SuppressWarnings(Array("TraversableHead")) //We check if volumes are empty before accessing the head def parse(name: String, nmlInputStream: InputStream, overwritingDataSetName: Option[String], isTaskUpload: Boolean)( implicit m: MessagesProvider) : Box[(Option[SkeletonTracing], Option[(VolumeTracing, String)], String, Option[String])] = try { - val data = time(silent=false, "load xml from input stream")(XML.load(nmlInputStream)) + val data = XML.load(nmlInputStream) for { parameters <- (data \ "parameters").headOption ?~ Messages("nml.parameters.notFound") - scale <- parseScale(parameters \ "scale") ?~ Messages("nml.scale.invalid") timestamp = parseTime(parameters \ "time") comments <- parseComments(data \ "comments") branchPoints <- parseBranchPoints(data \ "branchpoints", timestamp) - trees <- time(silent=false, "parse trees")(parseTrees(data \ "thing", buildBranchPointMap(branchPoints), buildCommentMap(comments))) + trees <- parseTrees(data \ "thing", buildBranchPointMap(branchPoints), buildCommentMap(comments)) treeGroups <- extractTreeGroups(data \ "groups") volumes = extractVolumes(data \ "volume") - treesAndGroupsAfterSplitting = time(silent=false, "splitMultiComponentTrees")(MultiComponentTreeSplitter.splitMulticomponentTrees(trees, treeGroups)) + treesAndGroupsAfterSplitting = MultiComponentTreeSplitter.splitMulticomponentTrees(trees, treeGroups) treesSplit = treesAndGroupsAfterSplitting._1 treeGroupsAfterSplit = treesAndGroupsAfterSplitting._2 - _ <- time(silent=false, "validateTrees")(TreeValidator.validateTrees(treesSplit, treeGroupsAfterSplit, branchPoints, comments)) + _ <- TreeValidator.validateTrees(treesSplit, treeGroupsAfterSplit, branchPoints, comments) } yield { val dataSetName = overwritingDataSetName.getOrElse(parseDataSetName(parameters \ "experiment")) val description = parseDescription(parameters \ "experiment") @@ -111,18 +98,20 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener if (treesSplit.isEmpty) None else Some( - SkeletonTracing(dataSetName, - treesSplit, - timestamp, - taskBoundingBox, - activeNodeId, - editPosition, - editRotation, - zoomLevel, - version = 0, - None, - treeGroupsAfterSplit, - userBoundingBoxes) + SkeletonTracing( + dataSetName, + treesSplit, + timestamp, + taskBoundingBox, + activeNodeId, + editPosition, + editRotation, + zoomLevel, + version = 0, + None, + treeGroupsAfterSplit, + userBoundingBoxes + ) ) (skeletonTracing, volumeTracingWithDataLocation, description, organizationName) @@ -141,48 +130,55 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener Failure(s"Failed to parse NML '$name': " + e.toString) } - def extractTreeGroups(treeGroupContainerNodes: NodeSeq)(implicit m: MessagesProvider): Box[List[TreeGroup]] = { + private def extractTreeGroups(treeGroupContainerNodes: NodeSeq)( + implicit m: MessagesProvider): Box[List[TreeGroup]] = { val treeGroupNodes = treeGroupContainerNodes.flatMap(_ \ "group") treeGroupNodes.map(parseTreeGroup).toList.toSingleBox(Messages("nml.element.invalid", "tree groups")) } - def parseTreeGroup(node: XMLNode)(implicit m: MessagesProvider): Box[TreeGroup] = + private def parseTreeGroup(node: XMLNode)(implicit m: MessagesProvider): Box[TreeGroup] = { + val idText = getSingleAttribute(node, "id") for { - id <- (node \ "@id").text.toIntOpt ?~ Messages("nml.treegroup.id.invalid", (node \ "@id").text) + id <- idText.toIntOpt ?~ Messages("nml.treegroup.id.invalid", idText) children <- (node \ "group").map(parseTreeGroup).toList.toSingleBox("") - name = (node \ "@name").text + name = getSingleAttribute(node, "name") } yield TreeGroup(name, id, children) + } - def extractVolumes(volumeNodes: NodeSeq) = - volumeNodes.map(node => Volume((node \ "@location").text, (node \ "@fallbackLayer").map(_.text).headOption)) + private def extractVolumes(volumeNodes: NodeSeq): immutable.Seq[Volume] = + volumeNodes.map(node => Volume(getSingleAttribute(node, "location"), getSingleAttributeOpt(node, "fallbackLayer"))) - private def parseTrees(treeNodes: NodeSeq, branchPoints: Map[Int, List[BranchPoint]], comments: Map[Int, List[Comment]])( - implicit m: MessagesProvider) = + private def parseTrees(treeNodes: NodeSeq, + branchPoints: Map[Int, List[BranchPoint]], + comments: Map[Int, List[Comment]])(implicit m: MessagesProvider) = treeNodes .map(treeNode => parseTree(treeNode, branchPoints, comments)) .toList .toSingleBox(Messages("nml.element.invalid", "trees")) + @SuppressWarnings(Array("TraversableHead")) // We check that size == 1 before accessing head private def parseBoundingBoxes(boundingBoxNodes: NodeSeq)(implicit m: MessagesProvider): Seq[NamedBoundingBox] = - if (boundingBoxNodes.size == 1 && (boundingBoxNodes \ "@id").text.isEmpty) { - Seq.empty ++ parseBoundingBox(boundingBoxNodes).map(NamedBoundingBox(0, None, None, None, _)) + if (boundingBoxNodes.size == 1 && getSingleAttribute(boundingBoxNodes.head, "id").isEmpty) { + Seq.empty ++ parseBoundingBox(boundingBoxNodes.head).map(NamedBoundingBox(0, None, None, None, _)) } else { - boundingBoxNodes.flatMap(node => + boundingBoxNodes.flatMap(node => { + val idText = getSingleAttribute(node, "id") for { - id <- (node \ "@id").text.toIntOpt ?~ Messages("nml.boundingbox.id.invalid", (node \ "@id").text) - name = (node \ "@name").text - isVisible = (node \ "@isVisible").text.toBooleanOpt + id <- idText.toIntOpt ?~ Messages("nml.boundingbox.id.invalid", idText) + name = getSingleAttribute(node, "name") + isVisible = getSingleAttribute(node, "isVisible").toBooleanOpt color = parseColor(node) boundingBox <- parseBoundingBox(node) nameOpt = if (name.isEmpty) None else Some(name) - } yield NamedBoundingBox(id, nameOpt, isVisible, color, boundingBox)) + } yield NamedBoundingBox(id, nameOpt, isVisible, color, boundingBox) + }) } private def parseTaskBoundingBox( - node: NodeSeq, + nodes: NodeSeq, isTask: Boolean, userBoundingBoxes: Seq[NamedBoundingBox]): Option[Either[BoundingBox, NamedBoundingBox]] = - parseBoundingBox(node).map { bb => + nodes.headOption.flatMap(node => parseBoundingBox(node)).map { bb => if (isTask) { Left(bb) } else { @@ -191,49 +187,49 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener } } - private def parseBoundingBox(node: NodeSeq) = + private def parseBoundingBox(node: XMLNode) = for { - topLeftX <- (node \ "@topLeftX").text.toIntOpt - topLeftY <- (node \ "@topLeftY").text.toIntOpt - topLeftZ <- (node \ "@topLeftZ").text.toIntOpt - width <- (node \ "@width").text.toIntOpt - height <- (node \ "@height").text.toIntOpt - depth <- (node \ "@depth").text.toIntOpt + topLeftX <- getSingleAttribute(node, "topLeftX").toIntOpt + topLeftY <- getSingleAttribute(node, "topLeftY").toIntOpt + topLeftZ <- getSingleAttribute(node, "topLeftZ").toIntOpt + width <- getSingleAttribute(node, "width").toIntOpt + height <- getSingleAttribute(node, "height").toIntOpt + depth <- getSingleAttribute(node, "depth").toIntOpt } yield BoundingBox(Point3D(topLeftX, topLeftY, topLeftZ), width, height, depth) - private def parseDataSetName(node: NodeSeq) = - (node \ "@name").text + private def parseDataSetName(nodes: NodeSeq): String = + nodes.headOption.map(node => getSingleAttribute(node, "name")).getOrElse("") - private def parseDescription(node: NodeSeq) = - (node \ "@description").text + private def parseDescription(nodes: NodeSeq): String = + nodes.headOption.map(node => getSingleAttribute(node, "description")).getOrElse(DEFAULT_DESCRIPTION) - private def parseOrganizationName(node: NodeSeq) = - (node \ "@organization").headOption.map(_.text) + private def parseOrganizationName(nodes: NodeSeq): Option[String] = + nodes.headOption.flatMap(node => getSingleAttributeOpt(node, "organization")) - private def parseActiveNode(node: NodeSeq) = - (node \ "@id").text.toIntOpt + private def parseActiveNode(nodes: NodeSeq): Option[Int] = + nodes.headOption.flatMap(node => getSingleAttribute(node, "id").toIntOpt) - private def parseTime(node: NodeSeq) = - (node \ "@ms").text.toLongOpt.getOrElse(DEFAULT_TIME) + private def parseTime(nodes: NodeSeq): Long = + nodes.headOption.flatMap(node => getSingleAttribute(node, "ms").toLongOpt).getOrElse(DEFAULT_TIME) - private def parseEditPosition(node: NodeSeq) = - node.headOption.flatMap(parsePoint3D) + private def parseEditPosition(nodes: NodeSeq): Option[Point3D] = + nodes.headOption.flatMap(parsePoint3D) - private def parseEditRotation(node: NodeSeq) = - node.headOption.flatMap(parseRotationForParams) + private def parseEditRotation(nodes: NodeSeq): Option[Vector3D] = + nodes.headOption.flatMap(parseRotationForParams) - private def parseZoomLevel(node: NodeSeq) = - (node \ "@zoom").text.toDoubleOpt + private def parseZoomLevel(nodes: NodeSeq) = + nodes.headOption.flatMap(node => getSingleAttribute(node, "zoom").toDoubleOpt) private def parseBranchPoints(branchPoints: NodeSeq, defaultTimestamp: Long)( implicit m: MessagesProvider): Box[List[BranchPoint]] = (branchPoints \ "branchpoint").zipWithIndex.map { case (branchPoint, index) => - (branchPoint \ "@id").text.toIntOpt.map { nodeId => - val parsedTimestamp = (branchPoint \ "@time").text.toLongOpt + getSingleAttribute(branchPoint, "id").toIntOpt.map { nodeId => + val parsedTimestamp = getSingleAttribute(branchPoint, "time").toLongOpt val timestamp = parsedTimestamp.getOrElse(defaultTimestamp - index) BranchPoint(nodeId, timestamp) - } ?~ Messages("nml.node.id.invalid", "branchpoint", (branchPoint \ "@id").text) + } ?~ Messages("nml.node.id.invalid", "branchpoint", getSingleAttribute(branchPoint, "id")) }.toList.toSingleBox(Messages("nml.element.invalid", "branchpoints")) private def parsePoint3D(node: XMLNode) = { @@ -249,9 +245,9 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener private def parseRotationForParams(node: XMLNode) = for { - rotX <- (node \ "@xRot").text.toDoubleOpt - rotY <- (node \ "@yRot").text.toDoubleOpt - rotZ <- (node \ "@zRot").text.toDoubleOpt + rotX <- getSingleAttribute(node, "xRot").toDoubleOpt + rotY <- getSingleAttribute(node, "yRot").toDoubleOpt + rotZ <- getSingleAttribute(node, "zRot").toDoubleOpt } yield Vector3D(rotX, rotY, rotZ) private def parseRotationForNode(node: XMLNode) = @@ -261,20 +257,12 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener rotZ <- getSingleAttribute(node, "rotZ").toDoubleOpt } yield Vector3D(rotX, rotY, rotZ) - private def parseScale(nodes: NodeSeq) = - nodes.headOption.flatMap(node => - for { - x <- (node \ "@x").text.toFloatOpt - y <- (node \ "@y").text.toFloatOpt - z <- (node \ "@z").text.toFloatOpt - } yield Scale(x, y, z)) - private def parseColorOpt(node: XMLNode) = for { - colorRed <- (node \ "@color.r").text.toFloatOpt - colorBlue <- (node \ "@color.g").text.toFloatOpt - colorGreen <- (node \ "@color.b").text.toFloatOpt - colorAlpha <- (node \ "@color.a").text.toFloatOpt + colorRed <- getSingleAttribute(node, "color.r").toFloatOpt + colorBlue <- getSingleAttribute(node, "color.g").toFloatOpt + colorGreen <- getSingleAttribute(node, "color.b").toFloatOpt + colorAlpha <- getSingleAttribute(node, "color.a").toFloatOpt } yield { Color(colorRed, colorBlue, colorGreen, colorAlpha) } @@ -283,31 +271,39 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener parseColorOpt(node) private def parseName(node: XMLNode) = - (node \ "@name").text + getSingleAttribute(node, "name") private def parseGroupId(node: XMLNode) = - (node \ "@groupId").text.toIntOpt + getSingleAttribute(node, "groupId").toIntOpt private def parseVisibility(node: XMLNode, color: Option[Color]): Option[Boolean] = - (node \ "@isVisible").text.toBooleanOpt match { + getSingleAttribute(node, "isVisible").toBooleanOpt match { case Some(isVisible) => Some(isVisible) case None => color.map(c => !c.a.isNearZero) } private def parseTree(tree: XMLNode, branchPoints: Map[Int, List[BranchPoint]], comments: Map[Int, List[Comment]])( implicit m: MessagesProvider): Box[Tree] = { + val treeIdText = getSingleAttribute(tree, "id") for { - id <- getSingleAttribute(tree, "id").toIntOpt ?~ Messages("nml.tree.id.invalid", (tree \ "@id").text) + id <- treeIdText.toIntOpt ?~ Messages("nml.tree.id.invalid", treeIdText) color = parseColor(tree) name = parseName(tree) groupId = parseGroupId(tree) isVisible = parseVisibility(tree, color) - nodes <- (tree \ "nodes" \ "node").map(parseNode).toList.toSingleBox(Messages("nml.tree.elements.invalid", "nodes", id)) - edges <- (tree \ "edges" \ "edge").map(parseEdge).toList.toSingleBox(Messages("nml.tree.elements.invalid", "edges", id)) + nodes <- (tree \ "nodes" \ "node") + .map(parseNode) + .toList + .toSingleBox(Messages("nml.tree.elements.invalid", "nodes", id)) + edges <- (tree \ "edges" \ "edge") + .map(parseEdge) + .toList + .toSingleBox(Messages("nml.tree.elements.invalid", "edges", id)) nodeIds = nodes.map(_.id) treeBranchPoints = nodeIds.flatMap(nodeId => branchPoints.getOrElse(nodeId, List())) treeComments = nodeIds.flatMap(nodeId => comments.getOrElse(nodeId, List())) - createdTimestamp = if (nodes.isEmpty) System.currentTimeMillis() else nodes.minBy(_.createdTimestamp).createdTimestamp + createdTimestamp = if (nodes.isEmpty) System.currentTimeMillis() + else nodes.minBy(_.createdTimestamp).createdTimestamp } yield Tree(id, nodes, edges, color, treeBranchPoints, treeComments, name, createdTimestamp, groupId, isVisible) } @@ -316,7 +312,8 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener commentNode <- comments \ "comment" } yield { for { - nodeId <- getSingleAttribute(commentNode, "node").toIntOpt ?~ Messages("nml.comment.node.invalid", getSingleAttribute(commentNode, "node")) + nodeId <- getSingleAttribute(commentNode, "node").toIntOpt ?~ Messages("nml.comment.node.invalid", + getSingleAttribute(commentNode, "node")) } yield { val content = getSingleAttribute(commentNode, "content") Comment(nodeId, content) @@ -347,9 +344,11 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener branchPointMap.toMap } - private def getSingleAttribute(xmlNode: XMLNode, attribute: String): String = { - xmlNode.attribute(attribute).flatMap(_.headOption.map(_.text)).getOrElse("") - } + private def getSingleAttribute(xmlNode: XMLNode, attribute: String): String = + getSingleAttributeOpt(xmlNode, attribute).getOrElse("") + + private def getSingleAttributeOpt(xmlNode: XMLNode, attribute: String): Option[String] = + xmlNode.attribute(attribute).flatMap(_.headOption.map(_.text)) private def parseEdge(edge: XMLNode)(implicit m: MessagesProvider): Box[Edge] = { val sourceStr = getSingleAttribute(edge, "source") @@ -377,9 +376,10 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener private def parseTimestamp(node: XMLNode) = getSingleAttribute(node, "time").toLongOpt.getOrElse(DEFAULT_TIMESTAMP) - private def parseNode(node: XMLNode)(implicit m: MessagesProvider): Box[Node] = + private def parseNode(node: XMLNode)(implicit m: MessagesProvider): Box[Node] = { + val nodeIdText = getSingleAttribute(node, "id") for { - id <- getSingleAttribute(node, "id").toIntOpt ?~ Messages("nml.node.id.invalid", "", (node \ "@id").text) + id <- nodeIdText.toIntOpt ?~ Messages("nml.node.id.invalid", "", nodeIdText) radius <- getSingleAttribute(node, "radius").toFloatOpt ?~ Messages("nml.node.attribute.invalid", "radius", id) position <- parsePoint3D(node) ?~ Messages("nml.node.attribute.invalid", "position", id) } yield { @@ -391,5 +391,6 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener val rotation = parseRotationForNode(node).getOrElse(NodeDefaults.rotation) Node(id, position, rotation, radius, viewport, resolution, bitDepth, interpolation, timestamp) } + } } diff --git a/frontend/javascripts/oxalis/model/reducers/skeletontracing_reducer.js b/frontend/javascripts/oxalis/model/reducers/skeletontracing_reducer.js index 5014ed8d3b..e62d0380a2 100644 --- a/frontend/javascripts/oxalis/model/reducers/skeletontracing_reducer.js +++ b/frontend/javascripts/oxalis/model/reducers/skeletontracing_reducer.js @@ -65,9 +65,7 @@ function SkeletonTracingReducer(state: OxalisState, action: Action): OxalisState // Warn the user, since this shouldn't happen, but clear the activeNodeId // so that wk is usable. Toast.warning( - `This tracing was initialized with an active node ID, which does not - belong to any tracing (nodeId: ${nodeId}). WebKnossos will fall back to - the last tree instead.`, + `Annotation was initialized with active node ID ${nodeId}, which is not present in the trees. Falling back to last tree instead.`, { timeout: 10000 }, ); activeNodeId = null; diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala index 2b7a6a0c51..ae1a56855d 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala @@ -47,9 +47,8 @@ class DataSourceService @Inject()( var inboxCheckVerboseCounter = 0 - def tick: Unit = { + def tick: Unit = () - } private def skipTrash(path: Path) = !path.toString.contains(".trash") From dcfc8423a41dd34188c0efc6bd80c0a2f551521a Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 15 Oct 2020 13:25:08 +0200 Subject: [PATCH 3/3] changelog --- CHANGELOG.unreleased.md | 1 + .../webknossos/datastore/services/DataSourceService.scala | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 3719894e01..062eb47b70 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -14,6 +14,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released - Hybrid tracings can now be imported directly in the tracing view via drag'n'drop. [#4837](https://github.com/scalableminds/webknossos/pull/4837) - The find data function now works for volume tracings, too. [#4847](https://github.com/scalableminds/webknossos/pull/4847) - Added admins and dataset managers to dataset access list, as they can access all datasets of the organization. [#4862](https://github.com/scalableminds/webknossos/pull/4862) +- Sped up the NML parsing via dashboard import. [#4872](https://github.com/scalableminds/webknossos/pull/4872) ### Changed - Brush circles are now connected with rectangles to provide a continuous stroke even if the brush is moved quickly. [#4785](https://github.com/scalableminds/webknossos/pull/4822) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala index ae1a56855d..c03be0ec20 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala @@ -47,8 +47,11 @@ class DataSourceService @Inject()( var inboxCheckVerboseCounter = 0 - def tick: Unit = - () + def tick: Unit = { + checkInbox(verbose = inboxCheckVerboseCounter == 0) + inboxCheckVerboseCounter += 1 + if (inboxCheckVerboseCounter >= 10) inboxCheckVerboseCounter = 0 + } private def skipTrash(path: Path) = !path.toString.contains(".trash")