diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index d57d75a003f..4659b1fa0b5 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -14,6 +14,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released - Added support for datasets where layers are transformed individually (with an affine matrix). Transformations can be specified via datasource-properties.json or via JS API (will be ephemeral, then). [#6748](https://github.com/scalableminds/webknossos/pull/6748) - Added list of all respective team members to the administration page for teams. [#6915](https://github.com/scalableminds/webknossos/pull/6915) - Added email notifications for WK worker jobs. [#6918](https://github.com/scalableminds/webknossos/pull/6918) +- Added support for viewing sharded neuroglancer precomputed datasets. [#6920](https://github.com/scalableminds/webknossos/pull/6920) ### Changed - Interpolation during rendering is now more performance intensive, since the rendering approach was changed. Therefore, interpolation is disabled by default. On the flip side, the rendered quality is often higher than it used to be. [#6748](https://github.com/scalableminds/webknossos/pull/6748) diff --git a/MIGRATIONS.released.md b/MIGRATIONS.released.md index b6668ec7efe..767a4221566 100644 --- a/MIGRATIONS.released.md +++ b/MIGRATIONS.released.md @@ -1,5 +1,6 @@ # Migration Guide (Released) -All migrations of WEBKNOSOSS are documented in this file. + +All migrations of WEBKNOSSOS are documented in this file. See `MIGRATIONS.unreleased.md` for the changes which are not yet part of an official release. This project adheres to [Calendar Versioning](http://calver.org/) `0Y.0M.MICRO`. diff --git a/app/models/binary/explore/PrecomputedExplorer.scala b/app/models/binary/explore/PrecomputedExplorer.scala index ca52456fdb7..1de8e1397c0 100644 --- a/app/models/binary/explore/PrecomputedExplorer.scala +++ b/app/models/binary/explore/PrecomputedExplorer.scala @@ -30,7 +30,6 @@ class PrecomputedExplorer extends RemoteLayerExplorer { for { name <- guessNameFromPath(remotePath) firstScale <- precomputedHeader.scales.headOption.toFox - _ <- bool2Fox(firstScale.sharding.isEmpty) ?~> "Failed to read dataset: sharding not supported" boundingBox <- BoundingBox.fromSizeArray(firstScale.size).toFox elementClass: ElementClass.Value <- elementClassFromPrecomputedDataType(precomputedHeader.data_type) ?~> "Unknown data type" smallestResolution = firstScale.resolution diff --git a/test/backend/CompressedMortonCodeTestSuite.scala b/test/backend/CompressedMortonCodeTestSuite.scala new file mode 100644 index 00000000000..14109307909 --- /dev/null +++ b/test/backend/CompressedMortonCodeTestSuite.scala @@ -0,0 +1,37 @@ +package backend + +import com.scalableminds.webknossos.datastore.datareaders.precomputed.CompressedMortonCode +import org.scalatestplus.play.PlaySpec + +class CompressedMortonCodeTestSuite extends PlaySpec { + + "Compressed Morton Code" when { + "Grid size = 10,10,10" should { + val grid_size = Array(10, 10, 10) + "encode 0,0,0" in { + assert(CompressedMortonCode.encode(Array(0, 0, 0), grid_size) == 0) + } + "encode 1,2,3" in { + assert(CompressedMortonCode.encode(Array(1, 2, 3), grid_size) == 53) + } + "encode 9,9,9" in { + assert(CompressedMortonCode.encode(Array(9, 9, 9), grid_size) == 3591) + } + "encode 10,10,10" in { + assert(CompressedMortonCode.encode(Array(10, 10, 10), grid_size) == 3640) + } + } + "Grid size = 2048,2048,1024" should { + val grid_size = Array(2048, 2048, 1024) + "encode 0,0,0" in { + assert(CompressedMortonCode.encode(Array(0, 0, 0), grid_size) == 0) + } + "encode 1,2,3" in { + assert(CompressedMortonCode.encode(Array(1, 2, 3), grid_size) == 53) + } + "encode 1024, 512, 684" in { + assert(CompressedMortonCode.encode(Array(1024, 512, 684), grid_size) == 1887570176) + } + } + } +} diff --git a/util/src/main/scala/com/scalableminds/util/geometry/Vec3Int.scala b/util/src/main/scala/com/scalableminds/util/geometry/Vec3Int.scala index be79e2fb6bc..27ae5a5b8e5 100644 --- a/util/src/main/scala/com/scalableminds/util/geometry/Vec3Int.scala +++ b/util/src/main/scala/com/scalableminds/util/geometry/Vec3Int.scala @@ -63,6 +63,9 @@ case class Vec3Int(x: Int, y: Int, z: Int) { } yield Vec3Int(x, y, z) def product: Int = x * y * z + + def alignWithGridFloor(gridCellSize: Vec3Int): Vec3Int = + this / gridCellSize * gridCellSize } object Vec3Int { diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkReader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkReader.scala index 16f6fff434a..fb9d2346696 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkReader.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkReader.scala @@ -6,6 +6,7 @@ import ucar.ma2.{Array => MultiArray, DataType => MADataType} import java.io.{ByteArrayInputStream, ByteArrayOutputStream, IOException} import javax.imageio.stream.MemoryCacheImageInputStream +import scala.collection.immutable.NumericRange import scala.concurrent.Future import scala.util.Using @@ -28,16 +29,17 @@ class ChunkReader(val header: DatasetHeader, val vaultPath: VaultPath, val chunk lazy val chunkSize: Int = header.chunkSize.toList.product @throws[IOException] - def read(path: String, chunkShape: Array[Int]): Future[MultiArray] = { - val chunkBytesAndShape = readChunkBytesAndShape(path) + def read(path: String, chunkShape: Array[Int], range: Option[NumericRange[Long]]): Future[MultiArray] = { + val chunkBytesAndShape = readChunkBytesAndShape(path, range) chunkTyper.wrapAndType(chunkBytesAndShape.map(_._1), chunkBytesAndShape.flatMap(_._2).getOrElse(chunkShape)) } // Returns bytes (optional, None may later be replaced with fill value) // and chunk shape (optional, only for data formats where each chunk reports its own shape, e.g. N5) - protected def readChunkBytesAndShape(path: String): Option[(Array[Byte], Option[Array[Int]])] = + protected def readChunkBytesAndShape(path: String, + range: Option[NumericRange[Long]]): Option[(Array[Byte], Option[Array[Int]])] = Using.Manager { use => - (vaultPath / path).readBytes().map { bytes => + (vaultPath / path).readBytes(range).map { bytes => val is = use(new ByteArrayInputStream(bytes)) val os = use(new ByteArrayOutputStream()) header.compressorImpl.uncompress(is, os) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkUtils.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkUtils.scala index f6d10acb2c7..630656fbbac 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkUtils.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkUtils.scala @@ -23,7 +23,7 @@ object ChunkUtils extends LazyLogging { } if (numChunks < 0) { logger.warn( - s"Failed to compute zarr chunk indices. array shape ${arrayShape.toList}, chunkShape: ${arrayChunkSize.toList}, requested ${selectedShape.toList} at ${selectedOffset.toList}") + s"Failed to compute chunk indices. array shape ${arrayShape.toList}, chunkShape: ${arrayChunkSize.toList}, requested ${selectedShape.toList} at ${selectedOffset.toList}") } val chunkIndices = new Array[Array[Int]](numChunks) val currentIdx = start.clone diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala index df06c55537a..f6f39a80e97 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala @@ -13,6 +13,7 @@ import ucar.ma2.{InvalidRangeException, Array => MultiArray} import java.io.IOException import java.nio.ByteOrder import java.util +import scala.collection.immutable.NumericRange import scala.concurrent.{ExecutionContext, Future} class DatasetArray(relativePath: DatasetPath, @@ -73,7 +74,7 @@ class DatasetArray(relativePath: DatasetPath, val targetBuffer = MultiArrayUtils.createDataBuffer(header.resolvedDataType, shape) val targetInCOrder: MultiArray = MultiArrayUtils.orderFlippedView(MultiArrayUtils.createArrayWithGivenStorage(targetBuffer, shape.reverse)) - val wasCopiedFox = Fox.serialCombined(chunkIndices) { chunkIndex: Array[Int] => + val copiedFuture = Future.sequence(chunkIndices.map { chunkIndex: Array[Int] => for { sourceChunk: MultiArray <- getSourceChunkDataWithCache(axisOrder.permuteIndices(chunkIndex)) offsetInChunk = computeOffsetInChunk(chunkIndex, offset) @@ -82,21 +83,33 @@ class DatasetArray(relativePath: DatasetPath, flip = header.order != ArrayOrder.C) _ = MultiArrayUtils.copyRange(offsetInChunk, sourceChunkInCOrder, targetInCOrder) } yield () - } + }) for { - _ <- wasCopiedFox + _ <- copiedFuture } yield targetBuffer } } - private def getSourceChunkDataWithCache(chunkIndex: Array[Int]): Future[MultiArray] = { - val chunkFilename = getChunkFilename(chunkIndex) - val chunkFilePath = relativePath.resolve(chunkFilename) - val storeKey = chunkFilePath.storeKey - val chunkShape = header.chunkSizeAtIndex(chunkIndex) + protected def getShardedChunkPathAndRange(chunkIndex: Array[Int])( + implicit ec: ExecutionContext): Future[(VaultPath, NumericRange[Long])] = ??? - chunkContentsCache.getOrLoad(storeKey, key => chunkReader.read(key, chunkShape)) - } + private def getSourceChunkDataWithCache(chunkIndex: Array[Int])(implicit ec: ExecutionContext): Future[MultiArray] = + chunkContentsCache.getOrLoad(chunkIndex.mkString(","), _ => readSourceChunkData(chunkIndex)) + + private def readSourceChunkData(chunkIndex: Array[Int])(implicit ec: ExecutionContext): Future[MultiArray] = + if (header.isSharded) { + for { + (shardPath, chunkRange) <- getShardedChunkPathAndRange(chunkIndex) + chunkShape = header.chunkSizeAtIndex(chunkIndex) + multiArray <- chunkReader.read(shardPath.toString, chunkShape, Some(chunkRange)) + } yield multiArray + } else { + val chunkFilename = getChunkFilename(chunkIndex) + val chunkFilePath = relativePath.resolve(chunkFilename) + val storeKey = chunkFilePath.storeKey + val chunkShape = header.chunkSizeAtIndex(chunkIndex) + chunkReader.read(storeKey, chunkShape, None) + } protected def getChunkFilename(chunkIndex: Array[Int]): String = chunkIndex.mkString(header.dimension_separator.toString) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetHeader.scala index 35741c015a7..caa218365d0 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetHeader.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetHeader.scala @@ -45,4 +45,6 @@ trait DatasetHeader { lazy val rank: Int = datasetShape.length def chunkSizeAtIndex(chunkIndex: Array[Int]): Array[Int] = chunkSize + + def isSharded = false } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/n5/N5ChunkReader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/n5/N5ChunkReader.scala index 3db9782aff8..1e76d3756e0 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/n5/N5ChunkReader.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/n5/N5ChunkReader.scala @@ -5,6 +5,7 @@ import com.scalableminds.webknossos.datastore.datavault.VaultPath import com.typesafe.scalalogging.LazyLogging import java.io.{ByteArrayInputStream, ByteArrayOutputStream} +import scala.collection.immutable.NumericRange import scala.util.Using object N5ChunkReader { @@ -23,7 +24,9 @@ class N5ChunkReader(header: DatasetHeader, vaultPath: VaultPath, typedChunkReade val dataExtractor: N5DataExtractor = new N5DataExtractor - override protected def readChunkBytesAndShape(path: String): Option[(Array[Byte], Option[Array[Int]])] = + override protected def readChunkBytesAndShape( + path: String, + range: Option[NumericRange[Long]]): Option[(Array[Byte], Option[Array[Int]])] = Using.Manager { use => def processBytes(bytes: Array[Byte], expectedElementCount: Int): Array[Byte] = { val is = use(new ByteArrayInputStream(bytes)) @@ -37,7 +40,7 @@ class N5ChunkReader(header: DatasetHeader, vaultPath: VaultPath, typedChunkReade } for { - bytes <- (vaultPath / path).readBytes() + bytes <- (vaultPath / path).readBytes(range) (blockHeader, data) = dataExtractor.readBytesAndHeader(bytes) paddedChunkBytes = processBytes(data, blockHeader.blockSize.product) } yield (paddedChunkBytes, Some(blockHeader.blockSize)) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/precomputed/CompressedMortonCode.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/precomputed/CompressedMortonCode.scala new file mode 100644 index 00000000000..43a6fcb3308 --- /dev/null +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/precomputed/CompressedMortonCode.scala @@ -0,0 +1,39 @@ +package com.scalableminds.webknossos.datastore.datareaders.precomputed + +import scala.math.log10 + +object CompressedMortonCode { + + def log2(x: Double): Double = log10(x) / log10(2.0) + + def encode(position: Array[Int], gridSize: Array[Int]): Long = { + /* + Computes the compressed morton code as per + https://github.com/google/neuroglancer/blob/master/src/neuroglancer/datasource/precomputed/volume.md#compressed-morton-code + https://github.com/google/neuroglancer/blob/162b698f703c86e0b3e92b8d8e0cacb0d3b098df/src/neuroglancer/util/zorder.ts#L72 + */ + val bits = gridSize.map(log2(_).ceil.toInt) + val maxBits = bits.max + var outputBit = 0L + val one = 1L + + var output = 0L + for (bit <- 0 to maxBits) { + if (bit < bits(0)) { + output |= (((position(0) >> bit) & one) << outputBit) + outputBit += 1 + } + if (bit < bits(1)) { + output |= (((position(1) >> bit) & one) << outputBit) + outputBit += 1 + } + if (bit < bits(2)) { + output |= (((position(2) >> bit) & one) << outputBit) + outputBit += 1 + } + } + + output + } + +} diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/precomputed/PrecomputedArray.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/precomputed/PrecomputedArray.scala index 58dbaf27d55..440f1bd1270 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/precomputed/PrecomputedArray.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/precomputed/PrecomputedArray.scala @@ -1,12 +1,19 @@ package com.scalableminds.webknossos.datastore.datareaders.precomputed +import com.scalableminds.util.cache.AlfuFoxCache +import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.datareaders.{AxisOrder, ChunkReader, DatasetArray, DatasetPath} import com.scalableminds.webknossos.datastore.datavault.VaultPath import com.typesafe.scalalogging.LazyLogging import play.api.libs.json.{JsError, JsSuccess, Json} import java.io.IOException +import java.nio.ByteOrder + +import java.nio.ByteBuffer import java.nio.charset.StandardCharsets +import scala.collection.immutable.NumericRange +import scala.concurrent.{ExecutionContext, Future} object PrecomputedArray extends LazyLogging { @throws[IOException] @@ -69,4 +76,178 @@ class PrecomputedArray(relativePath: DatasetPath, .mkString(header.dimension_separator.toString) } + // SHARDING + // Implemented according to https://github.com/google/neuroglancer/blob/master/src/neuroglancer/datasource/precomputed/sharded.md, + // directly adapted from https://github.com/scalableminds/webknossos-connect/blob/master/wkconnect/backends/neuroglancer/sharding.py. + + private val shardIndexCache: AlfuFoxCache[VaultPath, Array[Byte]] = + AlfuFoxCache() + + private val minishardIndexCache: AlfuFoxCache[(VaultPath, Int), Seq[(Long, Long, Long)]] = + AlfuFoxCache() + + private def getHashForChunk(chunkIndex: Array[Int]): Long = + CompressedMortonCode.encode(chunkIndex, header.gridSize) + + private lazy val minishardMask = { + header.precomputedScale.sharding match { + case Some(shardingSpec: ShardingSpecification) => + if (shardingSpec.minishard_bits == 0) { + 0 + } else { + var minishardMask = 1L + for (_ <- 0 until shardingSpec.minishard_bits - 1) { + minishardMask <<= 1 + minishardMask |= 1 + } + minishardMask + } + case None => 0 + } + } + + private lazy val shardMask = { + header.precomputedScale.sharding match { + case Some(shardingSpec: ShardingSpecification) => + val oneMask = Long.MinValue // 0xFFFFFFFFFFFFFFFF + val cursor = shardingSpec.minishard_bits + shardingSpec.shard_bits + val shardMask = ~((oneMask >> cursor) << cursor) + shardMask & (~minishardMask) + case None => 0 + } + } + + private lazy val minishardCount = 1 << header.precomputedScale.sharding.map(_.minishard_bits).getOrElse(0) + + private lazy val shardIndexRange: NumericRange.Exclusive[Long] = { + val end = minishardCount * 16 + Range.Long(0, end, 1) + } + + private def getShardIndex(shardPath: VaultPath)(implicit ec: ExecutionContext): Fox[Array[Byte]] = + shardIndexCache.getOrLoad(shardPath, readShardIndex) + + private def readShardIndex(shardPath: VaultPath)(implicit ec: ExecutionContext): Fox[Array[Byte]] = + Fox.option2Fox(shardPath.readBytes(Some(shardIndexRange))) + + private def parseShardIndex(index: Array[Byte]): Seq[(Long, Long)] = + // See https://github.com/google/neuroglancer/blob/master/src/neuroglancer/datasource/precomputed/sharded.md#shard-index-format + index + .grouped(16) // 16 Bytes: 2 uint64 numbers: start_offset, end_offset + .map((bytes: Array[Byte]) => { + (BigInt(bytes.take(8).reverse).toLong, BigInt(bytes.slice(8, 16).reverse).toLong) // bytes reversed because they are stored little endian + }) + .toSeq + + private def getMinishardInfo(chunkHash: Long): (Long, Long) = + header.precomputedScale.sharding match { + case Some(shardingSpec: ShardingSpecification) => + val rawChunkIdentifier = chunkHash >> shardingSpec.preshift_bits + val chunkIdentifier = shardingSpec.hashFunction(rawChunkIdentifier) + val minishardNumber = chunkIdentifier & minishardMask + val shardNumber = (chunkIdentifier & shardMask) >> shardingSpec.minishard_bits + (shardNumber, minishardNumber) + case None => (0, 0) + } + + private def getPathForShard(shardNumber: Long): VaultPath = { + val shardBits = header.precomputedScale.sharding.map(_.shard_bits.toFloat).getOrElse(0f) + if (shardBits == 0) { + vaultPath / relativePath.storeKey / "0.shard" + } else { + val shardString = String.format(s"%1$$${(shardBits / 4).ceil.toInt}s", shardNumber.toHexString).replace(' ', '0') + vaultPath / relativePath.storeKey / s"$shardString.shard" + } + + } + + private def getMinishardIndexRange(minishardNumber: Int, + parsedShardIndex: Seq[(Long, Long)]): NumericRange.Exclusive[Long] = { + val miniShardIndexStart: Long = (shardIndexRange.end) + parsedShardIndex(minishardNumber)._1 + val miniShardIndexEnd: Long = (shardIndexRange.end) + parsedShardIndex(minishardNumber)._2 + Range.Long(miniShardIndexStart, miniShardIndexEnd, 1) + } + + private def parseMinishardIndex(bytes: Array[Byte]): Seq[(Long, Long, Long)] = { + // Because readBytes already decodes gzip, we don't need to decompress here + /* + From: https://github.com/google/neuroglancer/blob/master/src/neuroglancer/datasource/precomputed/sharded.md#minishard-index-format + The decoded "minishard index" is a binary string of 24*n bytes, specifying a contiguous C-order array of [3, n] + uint64le values. + */ + val n = bytes.length / 24 + val buf = ByteBuffer.allocate(bytes.length) + buf.put(bytes) + + val longArray = new Array[Long](n * 3) + buf.position(0) + buf.order(ByteOrder.LITTLE_ENDIAN) + buf.asLongBuffer().get(longArray) + // longArray is row major / C-order + /* + From: https://github.com/google/neuroglancer/blob/master/src/neuroglancer/datasource/precomputed/sharded.md#minishard-index-format + Values array[0, 0], ..., array[0, n-1] specify the chunk IDs in the minishard, and are delta encoded, such that + array[0, 0] is equal to the ID of the first chunk, and the ID of chunk i is equal to the sum + of array[0, 0], ..., array[0, i]. + */ + val chunkIds = new Array[Long](n) + chunkIds(0) = longArray(0) + for (i <- 1 until n) { + chunkIds(i) = longArray(i) + chunkIds(i - 1) + } + /* + From: https://github.com/google/neuroglancer/blob/master/src/neuroglancer/datasource/precomputed/sharded.md#minishard-index-format + The size of the data for chunk i is stored as array[2, i]. + Values array[1, 0], ..., array[1, n-1] specify the starting offsets in the shard file of the data corresponding to + each chunk, and are also delta encoded relative to the end of the prior chunk, such that the starting offset of the + first chunk is equal to shard_index_end + array[1, 0], and the starting offset of chunk i is the sum of + shard_index_end + array[1, 0], ..., array[1, i] and array[2, 0], ..., array[2, i-1]. + */ + val chunkSizes = longArray.slice(2 * n, 3 * n) + val chunkStartOffsets = new Array[Long](n) + chunkStartOffsets(0) = longArray(n) + for (i <- 1 until n) { + val startOffsetIndex = i + n + chunkStartOffsets(i) = chunkStartOffsets(i - 1) + longArray(startOffsetIndex) + chunkSizes(i - 1) + } + (chunkIds, chunkStartOffsets, chunkSizes).zipped.map((a, b, c) => (a, b, c)) + } + + private def getMinishardIndex(shardPath: VaultPath, minishardNumber: Int)( + implicit ec: ExecutionContext): Fox[Seq[(Long, Long, Long)]] = + minishardIndexCache.getOrLoad((shardPath, minishardNumber), readMinishardIndex) + + private def readMinishardIndex(vaultPathAndMinishardNumber: (VaultPath, Int))( + implicit ec: ExecutionContext): Fox[Seq[(Long, Long, Long)]] = { + val (vaultPath, minishardNumber) = vaultPathAndMinishardNumber + for { + index <- getShardIndex(vaultPath) + parsedIndex = parseShardIndex(index) + minishardIndexRange = getMinishardIndexRange(minishardNumber, parsedIndex) + indexRaw <- vaultPath.readBytes(Some(minishardIndexRange)) + } yield parseMinishardIndex(indexRaw) + } + + private def getChunkRange(chunkId: Long, + minishardIndex: Seq[(Long, Long, Long)]): Option[NumericRange.Exclusive[Long]] = + for { + chunkSpecification <- minishardIndex.find(_._1 == chunkId) + chunkStart = (shardIndexRange.end) + chunkSpecification._2 + chunkEnd = (shardIndexRange.end) + chunkSpecification._2 + chunkSpecification._3 + } yield Range.Long(chunkStart, chunkEnd, 1) + + override def getShardedChunkPathAndRange(chunkIndex: Array[Int])( + implicit ec: ExecutionContext): Future[(VaultPath, NumericRange[Long])] = { + val chunkIdentifier = getHashForChunk(chunkIndex) + val minishardInfo = getMinishardInfo(chunkIdentifier) + val shardPath = getPathForShard(minishardInfo._1) + for { + minishardIndex <- getMinishardIndex(shardPath, minishardInfo._2.toInt) + .toFutureOrThrowException("Could not get minishard index") + chunkRange: NumericRange.Exclusive[Long] <- Fox + .option2Fox(getChunkRange(chunkIdentifier, minishardIndex)) + .toFutureOrThrowException("Chunk range not found in minishard index") + } yield (shardPath, chunkRange) + } + } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/precomputed/PrecomputedHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/precomputed/PrecomputedHeader.scala index c70a366e067..12b9c45c422 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/precomputed/PrecomputedHeader.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/precomputed/PrecomputedHeader.scala @@ -69,9 +69,24 @@ case class PrecomputedScaleHeader(precomputedScale: PrecomputedScale, precompute .min(precomputedScale.size(dim)) (beginOffset, endOffset) }) + + def gridSize: Array[Int] = (chunkSize, precomputedScale.size).zipped.map((c, s) => (s.toDouble / c).ceil.toInt) + + override def isSharded: Boolean = precomputedScale.sharding.isDefined } -case class ShardingSpecification(`@type`: String) +case class ShardingSpecification(`@type`: String, + preshift_bits: Long, + hash: String, + minishard_bits: Int, + shard_bits: Long, + minishard_index_encoding: String = "raw", + data_encoding: String = "raw") { + + def hashFunction(input: Long): Long = + if (hash == "identity") input + else ??? // not implemented: murmurhash3_x86_128 +} object ShardingSpecification extends JsonImplicits { implicit object ShardingSpecificationFormat extends Format[ShardingSpecification] { diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/FindDataService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/FindDataService.scala index c6af6aa836f..eb4ac4cfee3 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/FindDataService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/FindDataService.scala @@ -94,7 +94,8 @@ class FindDataService @Inject()(dataServicesHolder: BinaryDataServiceHolder)(imp } } - positionCreationIter((1 to iterationCount).toList, List[Vec3Int]()) :+ dataLayer.boundingBox.topLeft + val positions = positionCreationIter((1 to iterationCount).toList, List[Vec3Int]()) :+ dataLayer.boundingBox.topLeft + positions.map(_.alignWithGridFloor(Vec3Int.full(DataLayer.bucketLength))).distinct } private def checkAllPositionsForData(dataSource: DataSource,