Skip to content

Commit

Permalink
Tensorflow segmentation statistics (#3946)
Browse files Browse the repository at this point in the history
* [WIP] provide sampled mean and stdev for color layer data

* convert to unsigned, clean up

* best resolution is actually min, not max

* pretty-backend
  • Loading branch information
fm3 authored Mar 26, 2019
1 parent 608d84c commit 96eb19a
Showing 6 changed files with 124 additions and 29 deletions.
2 changes: 2 additions & 0 deletions conf/messages
Original file line number Diff line number Diff line change
@@ -96,6 +96,8 @@ dataSet.dataStore.missing=dataStore missing in the supplied json
dataSet.dataSet.missing=dataSet missing in the supplied json
dataSet.downloadAlreadyRunning=Sample dataset download is already running.
dataSet.alreadyPresent=Sample dataset is already present.
dataSet.noResolutions=Data layer does not contain resolutions
dataSet.sampledOnlyBlack=Sampled data positions contained only black data

dataSource.notFound=Datasource not found on datastore server

4 changes: 3 additions & 1 deletion project/Dependencies.scala
Original file line number Diff line number Diff line change
@@ -43,6 +43,7 @@ object Dependencies {
val xmlWriter = "org.glassfish.jaxb" % "txw2" % "2.2.11"
val woodstoxXml = "org.codehaus.woodstox" % "wstx-asl" % "3.2.3"
val redis = "net.debasishg" %% "redisclient" % "3.9"
val spire = "org.typelevel" %% "spire" % "0.14.1"

val sql = Seq(
"com.typesafe.slick" %% "slick" % "3.2.3",
@@ -78,7 +79,8 @@ object Dependencies {
playIterateesStreams,
filters,
ws,
guice
guice,
spire
)

val webknossosTracingstoreDependencies = Seq(
12 changes: 12 additions & 0 deletions util/src/main/scala/com/scalableminds/util/tools/Math.scala
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package com.scalableminds.util.tools

import Numeric.Implicits._

object Math {
val RotationMatrixSize3D = 16

@@ -34,4 +36,14 @@ object Math {
lower.max(x).min(upper)
}

def mean[T: Numeric](xs: Iterable[T]): Double = xs.sum.toDouble / xs.size

def variance[T: Numeric](xs: Iterable[T]): Double = {
val avg = mean(xs)

xs.map(_.toDouble).map(a => math.pow(a - avg, 2)).sum / xs.size
}

def stdDev[T: Numeric](xs: Iterable[T]): Double = math.sqrt(variance(xs))

}
Original file line number Diff line number Diff line change
@@ -370,6 +370,22 @@ class BinaryDataController @Inject()(
private def formatNeighborList(neighbors: List[Int]): String =
"[" + neighbors.mkString(", ") + "]"

def colorStatistics(organizationName: String, dataSetName: String, dataLayerName: String) = Action.async {
implicit request =>
accessTokenService
.validateAccess(UserAccessRequest.readDataSources(DataSourceId(dataSetName, organizationName))) {
AllowRemoteOrigin {
for {
(dataSource, dataLayer) <- getDataSourceAndDataLayer(organizationName, dataSetName, dataLayerName)
meanAndStdDev <- findDataService.meanAndStdDev(dataSource, dataLayer)
} yield
Ok(
Json.obj("mean" -> meanAndStdDev._1, "stdDev" -> meanAndStdDev._2)
)
}
}
}

def findData(organizationName: String, dataSetName: String, dataLayerName: String) = Action.async {
implicit request =>
accessTokenService
Original file line number Diff line number Diff line change
@@ -7,8 +7,9 @@ import com.scalableminds.util.tools.{Fox, FoxImplicits}
import com.scalableminds.webknossos.datastore.models.{DataRequest, VoxelPosition}
import com.scalableminds.webknossos.datastore.models.datasource.{DataLayer, DataSource, ElementClass}
import com.scalableminds.webknossos.datastore.models.requests.DataServiceDataRequest
import com.scalableminds.util.tools.Math
import net.liftweb.common.Full
import play.api.i18n.MessagesProvider
import play.api.i18n.{Messages, MessagesProvider}

import scala.concurrent.ExecutionContext
import scala.reflect.ClassTag
@@ -18,40 +19,41 @@ class FindDataService @Inject()(dataServicesHolder: BinaryDataServiceHolder)(imp
val binaryDataService: BinaryDataService = dataServicesHolder.binaryDataService
var i = 0

def findPositionWithData(dataSource: DataSource, dataLayer: DataLayer)(implicit m: MessagesProvider) =
def findPositionWithData(dataSource: DataSource, dataLayer: DataLayer)(
implicit m: MessagesProvider): Fox[Option[(Point3D, Point3D)]] =
for {
positionAndResolutionOpt <- checkAllPositionsForData(dataSource, dataLayer)
} yield positionAndResolutionOpt

private def checkAllPositionsForData(dataSource: DataSource, dataLayer: DataLayer) = {
private def convertData(data: Array[Byte],
elementClass: ElementClass.Value): Array[_ >: Byte with Short with Int with Long] =
elementClass match {
case ElementClass.uint8 =>
convertDataImpl[Byte, ByteBuffer](data, DataTypeFunctors[Byte, ByteBuffer](identity, _.get(_), _.toByte))
case ElementClass.uint16 =>
convertDataImpl[Short, ShortBuffer](data,
DataTypeFunctors[Short, ShortBuffer](_.asShortBuffer, _.get(_), _.toShort))
case ElementClass.uint32 =>
convertDataImpl[Int, IntBuffer](data, DataTypeFunctors[Int, IntBuffer](_.asIntBuffer, _.get(_), _.toInt))
case ElementClass.uint64 =>
convertDataImpl[Long, LongBuffer](data, DataTypeFunctors[Long, LongBuffer](_.asLongBuffer, _.get(_), identity))
}

def convertData(data: Array[Byte]) =
dataLayer.elementClass match {
case ElementClass.uint8 =>
convertDataImpl[Byte, ByteBuffer](data, DataTypeFunctors[Byte, ByteBuffer](identity, _.get(_), _.toByte))
case ElementClass.uint16 =>
convertDataImpl[Short, ShortBuffer](
data,
DataTypeFunctors[Short, ShortBuffer](_.asShortBuffer, _.get(_), _.toShort))
case ElementClass.uint32 =>
convertDataImpl[Int, IntBuffer](data, DataTypeFunctors[Int, IntBuffer](_.asIntBuffer, _.get(_), _.toInt))
case ElementClass.uint64 =>
convertDataImpl[Long, LongBuffer](data,
DataTypeFunctors[Long, LongBuffer](_.asLongBuffer, _.get(_), identity))
}
private def convertDataImpl[T: ClassTag, B <: Buffer](data: Array[Byte],
dataTypeFunctor: DataTypeFunctors[T, B]): Array[T] = {
val srcBuffer = dataTypeFunctor.getTypedBufferFn(ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN))
srcBuffer.rewind()
val dstArray = Array.ofDim[T](srcBuffer.remaining())
dataTypeFunctor.copyDataFn(srcBuffer, dstArray)
dstArray
}

def convertDataImpl[T: ClassTag, B <: Buffer](data: Array[Byte],
dataTypeFunctor: DataTypeFunctors[T, B]): Array[T] = {
val srcBuffer = dataTypeFunctor.getTypedBufferFn(ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN))
srcBuffer.rewind()
val dstArray = Array.ofDim[T](srcBuffer.remaining())
dataTypeFunctor.copyDataFn(srcBuffer, dstArray)
dstArray
}
private def checkAllPositionsForData(dataSource: DataSource,
dataLayer: DataLayer): Fox[Option[(Point3D, Point3D)]] = {

def getExactDataOffset(data: Array[Byte]): Point3D = {
val cubeLength = DataLayer.bucketLength / dataLayer.bytesPerElement
val convertedData = convertData(data)
val convertedData = convertData(data, dataLayer.elementClass)
for {
z <- 0 until cubeLength
y <- 0 until cubeLength
@@ -103,7 +105,7 @@ class FindDataService @Inject()(dataServicesHolder: BinaryDataServiceHolder)(imp
resolutionIter(createPositions(dataLayer).distinct, dataLayer.resolutions.sortBy(_.maxDim))
}

private def createPositions(dataLayer: DataLayer) = {
private def createPositions(dataLayer: DataLayer, iterationCount: Int = 4) = {

def positionCreationIter(remainingRuns: List[Int], currentPositions: List[Point3D]): List[Point3D] = {

@@ -140,6 +142,66 @@ class FindDataService @Inject()(dataServicesHolder: BinaryDataServiceHolder)(imp
}
}

positionCreationIter((1 to 4).toList, List[Point3D]())
positionCreationIter((1 to iterationCount).toList, List[Point3D]())
}

def meanAndStdDev(dataSource: DataSource, dataLayer: DataLayer)(
implicit m: MessagesProvider): Fox[(Double, Double)] = {

def getDataFor(position: Point3D, resolution: Point3D): Fox[Array[Byte]] = {
val request = DataRequest(
new VoxelPosition(position.x, position.y, position.z, resolution),
DataLayer.bucketLength,
DataLayer.bucketLength,
DataLayer.bucketLength
)
binaryDataService.handleDataRequest(
DataServiceDataRequest(dataSource, dataLayer, None, request.cuboid(dataLayer), request.settings))
}

def concatenateBuckets(buckets: Seq[Array[Byte]]): Array[Byte] =
buckets.foldLeft(Array[Byte]()) { (acc, i) =>
{
acc ++ i
}
}

def convertNonZeroDataToDouble(data: Array[Byte], elementClass: ElementClass.Value): Array[Double] =
elementClass match {
case ElementClass.uint8 =>
convertDataImpl[Byte, ByteBuffer](data, DataTypeFunctors[Byte, ByteBuffer](identity, _.get(_), _.toByte))
.filter(_ != 0)
.map(spire.math.UByte(_).toDouble)
case ElementClass.uint16 =>
convertDataImpl[Short, ShortBuffer](data,
DataTypeFunctors[Short, ShortBuffer](
_.asShortBuffer,
_.get(_),
_.toShort)).filter(_ != 0).map(spire.math.UShort(_).toDouble)
case ElementClass.uint32 =>
convertDataImpl[Int, IntBuffer](data, DataTypeFunctors[Int, IntBuffer](_.asIntBuffer, _.get(_), _.toInt))
.filter(_ != 0)
.map(spire.math.UInt(_).toDouble)
case ElementClass.uint64 =>
convertDataImpl[Long, LongBuffer](data,
DataTypeFunctors[Long, LongBuffer](_.asLongBuffer, _.get(_), identity))
.filter(_ != 0)
.map(spire.math.ULong(_).toDouble)
}

def meanAndStdDevForPositions(positions: List[Point3D], resolution: Point3D)(
implicit m: MessagesProvider): Fox[(Double, Double)] =
for {
dataBucketWise: Seq[Array[Byte]] <- Fox.serialCombined(positions)(pos => getDataFor(pos, resolution))
dataConcatenated = concatenateBuckets(dataBucketWise)
dataAsDoubles = convertNonZeroDataToDouble(dataConcatenated, dataLayer.elementClass)
_ <- Fox.bool2Fox(dataAsDoubles.nonEmpty) ?~> "dataSet.sampledOnlyBlack"
} yield (Math.mean(dataAsDoubles), Math.stdDev(dataAsDoubles))

for {
_ <- bool2Fox(dataLayer.resolutions.nonEmpty) ?~> "dataSet.noResolutions"
meanAndStdDev <- meanAndStdDevForPositions(createPositions(dataLayer, 2).distinct,
dataLayer.resolutions.minBy(_.maxDim))
} yield meanAndStdDev
}
}
Original file line number Diff line number Diff line change
@@ -14,6 +14,7 @@ GET /datasets/:organizationName/:dataSetName/layers/:dataLayerName/image
GET /datasets/:organizationName/:dataSetName/layers/:dataLayerName/thumbnail.json @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestImageThumbnailJson(organizationName: String, dataSetName: String, dataLayerName: String, width: Int, height: Int, centerX: Option[Int], centerY: Option[Int], centerZ: Option[Int], zoom: Option[Double])
GET /datasets/:organizationName/:dataSetName/layers/:dataLayerName/thumbnail.jpg @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestImageThumbnailJpeg(organizationName: String, dataSetName: String, dataLayerName: String, width: Int, height: Int, centerX: Option[Int], centerY: Option[Int], centerZ: Option[Int], zoom: Option[Double])
GET /datasets/:organizationName/:dataSetName/layers/:dataLayerName/findData @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.findData(organizationName: String, dataSetName: String, dataLayerName: String)
GET /datasets/:organizationName/:dataSetName/layers/:dataLayerName/colorStatistics @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.colorStatistics(organizationName: String, dataSetName: String, dataLayerName: String)

# Knossos compatibale routes
GET /datasets/:organizationName/:dataSetName/layers/:dataLayerName/mag:resolution/x:x/y:y/z:z/bucket.raw @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestViaKnossos(organizationName: String, dataSetName: String, dataLayerName: String, resolution: Int, x: Int, y: Int, z: Int, cubeSize: Int)

0 comments on commit 96eb19a

Please sign in to comment.