diff --git a/conf/messages b/conf/messages index dbd0e788026..a8686b0b760 100644 --- a/conf/messages +++ b/conf/messages @@ -96,6 +96,8 @@ dataSet.dataStore.missing=dataStore missing in the supplied json dataSet.dataSet.missing=dataSet missing in the supplied json dataSet.downloadAlreadyRunning=Sample dataset download is already running. dataSet.alreadyPresent=Sample dataset is already present. +dataSet.noResolutions=Data layer does not contain resolutions +dataSet.sampledOnlyBlack=Sampled data positions contained only black data dataSource.notFound=Datasource not found on datastore server diff --git a/project/Dependencies.scala b/project/Dependencies.scala index d408de353b5..b02a9c52df1 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -43,6 +43,7 @@ object Dependencies { val xmlWriter = "org.glassfish.jaxb" % "txw2" % "2.2.11" val woodstoxXml = "org.codehaus.woodstox" % "wstx-asl" % "3.2.3" val redis = "net.debasishg" %% "redisclient" % "3.9" + val spire = "org.typelevel" %% "spire" % "0.14.1" val sql = Seq( "com.typesafe.slick" %% "slick" % "3.2.3", @@ -78,7 +79,8 @@ object Dependencies { playIterateesStreams, filters, ws, - guice + guice, + spire ) val webknossosTracingstoreDependencies = Seq( diff --git a/util/src/main/scala/com/scalableminds/util/tools/Math.scala b/util/src/main/scala/com/scalableminds/util/tools/Math.scala index 700700c45c0..7c62b087240 100644 --- a/util/src/main/scala/com/scalableminds/util/tools/Math.scala +++ b/util/src/main/scala/com/scalableminds/util/tools/Math.scala @@ -1,5 +1,7 @@ package com.scalableminds.util.tools +import Numeric.Implicits._ + object Math { val RotationMatrixSize3D = 16 @@ -34,4 +36,14 @@ object Math { lower.max(x).min(upper) } + def mean[T: Numeric](xs: Iterable[T]): Double = xs.sum.toDouble / xs.size + + def variance[T: Numeric](xs: Iterable[T]): Double = { + val avg = mean(xs) + + xs.map(_.toDouble).map(a => math.pow(a - avg, 2)).sum / xs.size + } + + def stdDev[T: Numeric](xs: Iterable[T]): Double = math.sqrt(variance(xs)) + } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala index 723bbcf9a0e..c78a587d22b 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala @@ -370,6 +370,22 @@ class BinaryDataController @Inject()( private def formatNeighborList(neighbors: List[Int]): String = "[" + neighbors.mkString(", ") + "]" + def colorStatistics(organizationName: String, dataSetName: String, dataLayerName: String) = Action.async { + implicit request => + accessTokenService + .validateAccess(UserAccessRequest.readDataSources(DataSourceId(dataSetName, organizationName))) { + AllowRemoteOrigin { + for { + (dataSource, dataLayer) <- getDataSourceAndDataLayer(organizationName, dataSetName, dataLayerName) + meanAndStdDev <- findDataService.meanAndStdDev(dataSource, dataLayer) + } yield + Ok( + Json.obj("mean" -> meanAndStdDev._1, "stdDev" -> meanAndStdDev._2) + ) + } + } + } + def findData(organizationName: String, dataSetName: String, dataLayerName: String) = Action.async { implicit request => accessTokenService diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/FindDataService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/FindDataService.scala index b55a60e6efd..f30855dcfe6 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/FindDataService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/FindDataService.scala @@ -7,8 +7,9 @@ import com.scalableminds.util.tools.{Fox, FoxImplicits} import com.scalableminds.webknossos.datastore.models.{DataRequest, VoxelPosition} import com.scalableminds.webknossos.datastore.models.datasource.{DataLayer, DataSource, ElementClass} import com.scalableminds.webknossos.datastore.models.requests.DataServiceDataRequest +import com.scalableminds.util.tools.Math import net.liftweb.common.Full -import play.api.i18n.MessagesProvider +import play.api.i18n.{Messages, MessagesProvider} import scala.concurrent.ExecutionContext import scala.reflect.ClassTag @@ -18,40 +19,41 @@ class FindDataService @Inject()(dataServicesHolder: BinaryDataServiceHolder)(imp val binaryDataService: BinaryDataService = dataServicesHolder.binaryDataService var i = 0 - def findPositionWithData(dataSource: DataSource, dataLayer: DataLayer)(implicit m: MessagesProvider) = + def findPositionWithData(dataSource: DataSource, dataLayer: DataLayer)( + implicit m: MessagesProvider): Fox[Option[(Point3D, Point3D)]] = for { positionAndResolutionOpt <- checkAllPositionsForData(dataSource, dataLayer) } yield positionAndResolutionOpt - private def checkAllPositionsForData(dataSource: DataSource, dataLayer: DataLayer) = { + private def convertData(data: Array[Byte], + elementClass: ElementClass.Value): Array[_ >: Byte with Short with Int with Long] = + elementClass match { + case ElementClass.uint8 => + convertDataImpl[Byte, ByteBuffer](data, DataTypeFunctors[Byte, ByteBuffer](identity, _.get(_), _.toByte)) + case ElementClass.uint16 => + convertDataImpl[Short, ShortBuffer](data, + DataTypeFunctors[Short, ShortBuffer](_.asShortBuffer, _.get(_), _.toShort)) + case ElementClass.uint32 => + convertDataImpl[Int, IntBuffer](data, DataTypeFunctors[Int, IntBuffer](_.asIntBuffer, _.get(_), _.toInt)) + case ElementClass.uint64 => + convertDataImpl[Long, LongBuffer](data, DataTypeFunctors[Long, LongBuffer](_.asLongBuffer, _.get(_), identity)) + } - def convertData(data: Array[Byte]) = - dataLayer.elementClass match { - case ElementClass.uint8 => - convertDataImpl[Byte, ByteBuffer](data, DataTypeFunctors[Byte, ByteBuffer](identity, _.get(_), _.toByte)) - case ElementClass.uint16 => - convertDataImpl[Short, ShortBuffer]( - data, - DataTypeFunctors[Short, ShortBuffer](_.asShortBuffer, _.get(_), _.toShort)) - case ElementClass.uint32 => - convertDataImpl[Int, IntBuffer](data, DataTypeFunctors[Int, IntBuffer](_.asIntBuffer, _.get(_), _.toInt)) - case ElementClass.uint64 => - convertDataImpl[Long, LongBuffer](data, - DataTypeFunctors[Long, LongBuffer](_.asLongBuffer, _.get(_), identity)) - } + private def convertDataImpl[T: ClassTag, B <: Buffer](data: Array[Byte], + dataTypeFunctor: DataTypeFunctors[T, B]): Array[T] = { + val srcBuffer = dataTypeFunctor.getTypedBufferFn(ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN)) + srcBuffer.rewind() + val dstArray = Array.ofDim[T](srcBuffer.remaining()) + dataTypeFunctor.copyDataFn(srcBuffer, dstArray) + dstArray + } - def convertDataImpl[T: ClassTag, B <: Buffer](data: Array[Byte], - dataTypeFunctor: DataTypeFunctors[T, B]): Array[T] = { - val srcBuffer = dataTypeFunctor.getTypedBufferFn(ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN)) - srcBuffer.rewind() - val dstArray = Array.ofDim[T](srcBuffer.remaining()) - dataTypeFunctor.copyDataFn(srcBuffer, dstArray) - dstArray - } + private def checkAllPositionsForData(dataSource: DataSource, + dataLayer: DataLayer): Fox[Option[(Point3D, Point3D)]] = { def getExactDataOffset(data: Array[Byte]): Point3D = { val cubeLength = DataLayer.bucketLength / dataLayer.bytesPerElement - val convertedData = convertData(data) + val convertedData = convertData(data, dataLayer.elementClass) for { z <- 0 until cubeLength y <- 0 until cubeLength @@ -103,7 +105,7 @@ class FindDataService @Inject()(dataServicesHolder: BinaryDataServiceHolder)(imp resolutionIter(createPositions(dataLayer).distinct, dataLayer.resolutions.sortBy(_.maxDim)) } - private def createPositions(dataLayer: DataLayer) = { + private def createPositions(dataLayer: DataLayer, iterationCount: Int = 4) = { def positionCreationIter(remainingRuns: List[Int], currentPositions: List[Point3D]): List[Point3D] = { @@ -140,6 +142,66 @@ class FindDataService @Inject()(dataServicesHolder: BinaryDataServiceHolder)(imp } } - positionCreationIter((1 to 4).toList, List[Point3D]()) + positionCreationIter((1 to iterationCount).toList, List[Point3D]()) + } + + def meanAndStdDev(dataSource: DataSource, dataLayer: DataLayer)( + implicit m: MessagesProvider): Fox[(Double, Double)] = { + + def getDataFor(position: Point3D, resolution: Point3D): Fox[Array[Byte]] = { + val request = DataRequest( + new VoxelPosition(position.x, position.y, position.z, resolution), + DataLayer.bucketLength, + DataLayer.bucketLength, + DataLayer.bucketLength + ) + binaryDataService.handleDataRequest( + DataServiceDataRequest(dataSource, dataLayer, None, request.cuboid(dataLayer), request.settings)) + } + + def concatenateBuckets(buckets: Seq[Array[Byte]]): Array[Byte] = + buckets.foldLeft(Array[Byte]()) { (acc, i) => + { + acc ++ i + } + } + + def convertNonZeroDataToDouble(data: Array[Byte], elementClass: ElementClass.Value): Array[Double] = + elementClass match { + case ElementClass.uint8 => + convertDataImpl[Byte, ByteBuffer](data, DataTypeFunctors[Byte, ByteBuffer](identity, _.get(_), _.toByte)) + .filter(_ != 0) + .map(spire.math.UByte(_).toDouble) + case ElementClass.uint16 => + convertDataImpl[Short, ShortBuffer](data, + DataTypeFunctors[Short, ShortBuffer]( + _.asShortBuffer, + _.get(_), + _.toShort)).filter(_ != 0).map(spire.math.UShort(_).toDouble) + case ElementClass.uint32 => + convertDataImpl[Int, IntBuffer](data, DataTypeFunctors[Int, IntBuffer](_.asIntBuffer, _.get(_), _.toInt)) + .filter(_ != 0) + .map(spire.math.UInt(_).toDouble) + case ElementClass.uint64 => + convertDataImpl[Long, LongBuffer](data, + DataTypeFunctors[Long, LongBuffer](_.asLongBuffer, _.get(_), identity)) + .filter(_ != 0) + .map(spire.math.ULong(_).toDouble) + } + + def meanAndStdDevForPositions(positions: List[Point3D], resolution: Point3D)( + implicit m: MessagesProvider): Fox[(Double, Double)] = + for { + dataBucketWise: Seq[Array[Byte]] <- Fox.serialCombined(positions)(pos => getDataFor(pos, resolution)) + dataConcatenated = concatenateBuckets(dataBucketWise) + dataAsDoubles = convertNonZeroDataToDouble(dataConcatenated, dataLayer.elementClass) + _ <- Fox.bool2Fox(dataAsDoubles.nonEmpty) ?~> "dataSet.sampledOnlyBlack" + } yield (Math.mean(dataAsDoubles), Math.stdDev(dataAsDoubles)) + + for { + _ <- bool2Fox(dataLayer.resolutions.nonEmpty) ?~> "dataSet.noResolutions" + meanAndStdDev <- meanAndStdDevForPositions(createPositions(dataLayer, 2).distinct, + dataLayer.resolutions.minBy(_.maxDim)) + } yield meanAndStdDev } } diff --git a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes index abf1883ca26..655bda4794b 100644 --- a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes +++ b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes @@ -14,6 +14,7 @@ GET /datasets/:organizationName/:dataSetName/layers/:dataLayerName/image GET /datasets/:organizationName/:dataSetName/layers/:dataLayerName/thumbnail.json @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestImageThumbnailJson(organizationName: String, dataSetName: String, dataLayerName: String, width: Int, height: Int, centerX: Option[Int], centerY: Option[Int], centerZ: Option[Int], zoom: Option[Double]) GET /datasets/:organizationName/:dataSetName/layers/:dataLayerName/thumbnail.jpg @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestImageThumbnailJpeg(organizationName: String, dataSetName: String, dataLayerName: String, width: Int, height: Int, centerX: Option[Int], centerY: Option[Int], centerZ: Option[Int], zoom: Option[Double]) GET /datasets/:organizationName/:dataSetName/layers/:dataLayerName/findData @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.findData(organizationName: String, dataSetName: String, dataLayerName: String) +GET /datasets/:organizationName/:dataSetName/layers/:dataLayerName/colorStatistics @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.colorStatistics(organizationName: String, dataSetName: String, dataLayerName: String) # Knossos compatibale routes GET /datasets/:organizationName/:dataSetName/layers/:dataLayerName/mag:resolution/x:x/y:y/z:z/bucket.raw @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestViaKnossos(organizationName: String, dataSetName: String, dataLayerName: String, resolution: Int, x: Int, y: Int, z: Int, cubeSize: Int) diff --git a/yarn.lock b/yarn.lock index 56105a653ee..cac1d868c05 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1363,7 +1363,7 @@ acorn-walk@^6.0.1: resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-6.1.1.tgz#d363b66f5fac5f018ff9c3a1e7b6f8e310cc3913" integrity sha512-OtUw6JUTgxA2QoqqmrmQ7F2NYqiBPi/L2jqHyFtllhOUvXYQXf0Z1CYUinIfyT4bTCGmrA7gX9FvHA81uzCoVw== -acorn@^5.2.1, acorn@^5.5.3, acorn@^5.6.2: +acorn@^5.0.0, acorn@^5.2.1, acorn@^5.5.3, acorn@^5.6.2: version "5.7.3" resolved "https://registry.yarnpkg.com/acorn/-/acorn-5.7.3.tgz#67aa231bf8812974b85235a96771eb6bd07ea279" integrity sha512-T/zvzYRfbVojPWahDsE5evJdHb3oJoQfFbsrKM7w5Zcs++Tr257tia3BmMP8XYVjp1S9RZXQMh7gao96BlqZOw== @@ -10558,7 +10558,7 @@ scheduler@^0.13.3: loose-envify "^1.1.0" object-assign "^4.1.1" -schema-utils@^0.4.0, schema-utils@^0.4.5: +schema-utils@^0.4.0, schema-utils@^0.4.4, schema-utils@^0.4.5: version "0.4.7" resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-0.4.7.tgz#ba74f597d2be2ea880131746ee17d0a093c68187" integrity sha512-v/iwU6wvwGK8HbU9yi3/nhGzP0yGSuhQMzL6ySiec1FSrZZDkhm4noOSWzrNFo/jEc+SJY6jRTwuwbSXJPDUnQ==