cheap uint64 support for segmentation layer

app/models/binary/DataSetService.scala

@@ -3,9 +3,13 @@ package models.binary
 import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext}
 import com.scalableminds.webknossos.datastore.rpc.RPC
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
+import com.scalableminds.webknossos.datastore.dataformats.wkw.WKWSegmentationLayer
 import com.scalableminds.webknossos.datastore.models.datasource.{
+  AbstractSegmentationLayer,
   DataSourceId,
+  ElementClass,
   GenericDataSource,
+  inbox,
   DataLayerLike => DataLayer
 }
 import com.scalableminds.webknossos.datastore.models.datasource.inbox.{
@@ -254,7 +258,7 @@ class DataSetService @Inject()(organizationDAO: OrganizationDAO,
       dataSourceId = DataSourceId(dataSet.name, organization.name)
     } yield {
       dataLayersBox match {
-        case Full(dataLayers) if (dataLayers.length > 0) =>
+        case Full(dataLayers) if (dataLayers.nonEmpty) =>
           for {
             scale <- dataSet.scale
           } yield GenericDataSource[DataLayer](dataSourceId, dataLayers, scale)
@@ -327,12 +331,13 @@ class DataSetService @Inject()(organizationDAO: OrganizationDAO,
       lastUsedByUser <- lastUsedTimeFor(dataSet._id, requestingUserOpt)
       dataStoreJs <- dataStoreService.publicWrites(dataStore)
       dataSource <- dataSourceFor(dataSet, Some(organization), skipResolutions)
+      dataSourceWith64BitSupport = dataSource.toUsable.map(replaceUint64Layers).getOrElse(dataSource)
       publicationOpt <- Fox.runOptional(dataSet._publication)(publicationDAO.findOne(_))
       publicationJson <- Fox.runOptional(publicationOpt)(publicationService.publicWrites)
     } yield {
       Json.obj(
         "name" -> dataSet.name,
-        "dataSource" -> dataSource,
+        "dataSource" -> dataSourceWith64BitSupport,
         "dataStore" -> dataStoreJs,
         "owningOrganization" -> organization.name,
         "allowedTeams" -> teamsJs,
@@ -352,4 +357,15 @@ class DataSetService @Inject()(organizationDAO: OrganizationDAO,
       )
     }
 
+  private def replaceUint64Layers(dataSource: GenericDataSource[DataLayer]) = {
+    val newLayers = dataSource.dataLayers.map {
+      case l: WKWSegmentationLayer if l.elementClass == ElementClass.uint64 =>
+        l.copy(elementClass = ElementClass.uint32)
+      case l: AbstractSegmentationLayer if l.elementClass == ElementClass.uint64 =>
+        l.copy(elementClass = ElementClass.uint32)
+      case l => l
+    }
+
+    dataSource.copy(dataLayers = newLayers)
+  }
 }

webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayer.scala

@@ -66,7 +66,7 @@ object ElementClass extends Enumeration {
     case ElementClass.uint8  => 1L << 8L
     case ElementClass.uint16 => 1L << 16L
     case ElementClass.uint32 => 1L << 32L
-    case ElementClass.uint64 => 1L << 64L
+    case ElementClass.uint64 => (1L << 63L) - 1
   }
 
   def fromString(s: String): Option[Value] = values.find(_.toString == s)
@@ -211,13 +211,15 @@ case class AbstractSegmentationLayer(
 object AbstractSegmentationLayer {
 
   def from(layer: SegmentationLayerLike): AbstractSegmentationLayer =
-    AbstractSegmentationLayer(layer.name,
-                              layer.category,
-                              layer.boundingBox,
-                              layer.resolutions,
-                              layer.elementClass,
-                              layer.largestSegmentId,
-                              layer.mappings)
+    AbstractSegmentationLayer(
+      layer.name,
+      layer.category,
+      layer.boundingBox,
+      layer.resolutions,
+      layer.elementClass,
+      layer.largestSegmentId,
+      layer.mappings
+    )
 
   implicit val abstractSegmentationLayerFormat = Json.format[AbstractSegmentationLayer]
 }

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala

@@ -4,7 +4,7 @@ import java.nio.file.{Path, Paths}
 
 import com.scalableminds.util.geometry.{Point3D, Vector3I}
 import com.scalableminds.webknossos.datastore.models.BucketPosition
-import com.scalableminds.webknossos.datastore.models.datasource.DataLayer
+import com.scalableminds.webknossos.datastore.models.datasource.{Category, DataLayer, ElementClass}
 import com.scalableminds.webknossos.datastore.models.requests.{
   DataReadInstruction,
   DataServiceDataRequest,
@@ -48,10 +48,14 @@ class BinaryDataService(dataBaseDir: Path, loadTimeout: FiniteDuration, maxCache
     val requestData = requests.zipWithIndex.map {
       case (request, index) =>
         handleDataRequest(request).map { data =>
+          val convertedData =
+            if (request.dataLayer.elementClass == ElementClass.uint64 && request.dataLayer.category == Category.segmentation)
+              convertToUInt32(data)
+            else data
           if (request.settings.halfByte) {
-            (convertToHalfByte(data), index)
+            (convertToHalfByte(convertedData), index)
           } else {
-            (data, index)
+            (convertedData, index)
           }
         }
     }
@@ -153,6 +157,17 @@ class BinaryDataService(dataBaseDir: Path, loadTimeout: FiniteDuration, maxCache
     compressed
   }
 
+  private def convertToUInt32(a: Array[Byte]) = {
+    val result = new Array[Byte](a.length / 2)
+
+    for (i <- a.indices by 8) {
+      for (j <- 0 until 4) {
+        result(i / 2 + j) = a(i + j)
+      }
+    }
+    result
+  }
+
   def clearCache(organizationName: String, dataSetName: String) = {
     def matchingPredicate(cubeKey: CachedCube) =
       cubeKey.dataSourceName == dataSetName