From a0313db53ec6cd15eb8256d713e72cc4e527231b Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 22 Jan 2024 14:39:43 +0100 Subject: [PATCH 1/4] Include all-zero buckets in volume annotation downloads --- .../tracings/volume/VolumeDataZipHelper.scala | 8 +++-- .../volume/VolumeTracingBucketHelper.scala | 4 +-- .../volume/VolumeTracingService.scala | 14 +++++---- .../tracings/volume/WKWBucketStreamSink.scala | 29 ++++++++++++++----- .../volume/Zarr3BucketStreamSink.scala | 24 ++++++++++----- 5 files changed, 53 insertions(+), 26 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeDataZipHelper.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeDataZipHelper.scala index ab155c3162..30d9f7809f 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeDataZipHelper.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeDataZipHelper.scala @@ -23,7 +23,11 @@ import org.apache.commons.io.IOUtils import java.util.zip.{ZipEntry, ZipFile} import scala.collection.mutable -trait VolumeDataZipHelper extends WKWDataFormatHelper with ByteUtils with BoxImplicits with LazyLogging { +trait VolumeDataZipHelper + extends WKWDataFormatHelper + with VolumeBucketReversionHelper + with BoxImplicits + with LazyLogging { protected def withBucketsFromZip(zipFile: File)(block: (BucketPosition, Array[Byte]) => Unit): Box[Unit] = for { @@ -51,7 +55,7 @@ trait VolumeDataZipHelper extends WKWDataFormatHelper with ByteUtils with BoxImp parseWKWFilePath(fileName.toString).map { bucketPosition: BucketPosition => if (buckets.hasNext) { val data = buckets.next() - if (!isAllZero(data)) { + if (!isRevertedBucket(data)) { block(bucketPosition, data) } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala index c3903cebab..81910e4587 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala @@ -17,9 +17,9 @@ import scala.concurrent.ExecutionContext import scala.concurrent.duration._ trait VolumeBucketReversionHelper { - private def isRevertedBucket(data: Array[Byte]): Boolean = data sameElements Array[Byte](0) + protected def isRevertedBucket(data: Array[Byte]): Boolean = data sameElements Array[Byte](0) - def isRevertedBucket(bucket: VersionedKeyValuePair[Array[Byte]]): Boolean = isRevertedBucket(bucket.value) + protected def isRevertedBucket(bucket: VersionedKeyValuePair[Array[Byte]]): Boolean = isRevertedBucket(bucket.value) } trait VolumeBucketCompression extends LazyLogging { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 22b2e3dc75..840be9dd7b 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -371,13 +371,15 @@ class VolumeTracingService @Inject()( val dataLayer = volumeTracingLayer(tracingId, tracing) val buckets: Iterator[NamedStream] = volumeDataZipFormmat match { case VolumeDataZipFormat.wkw => - new WKWBucketStreamSink(dataLayer)(dataLayer.bucketProvider.bucketStream(Some(tracing.version)), - tracing.resolutions.map(mag => vec3IntFromProto(mag))) + new WKWBucketStreamSink(dataLayer, tracing.fallbackLayer.nonEmpty)( + dataLayer.bucketProvider.bucketStream(Some(tracing.version)), + tracing.resolutions.map(mag => vec3IntFromProto(mag))) case VolumeDataZipFormat.zarr3 => - new Zarr3BucketStreamSink(dataLayer)(dataLayer.bucketProvider.bucketStream(Some(tracing.version)), - tracing.resolutions.map(mag => vec3IntFromProto(mag)), - tracing.additionalAxes, - voxelSize) + new Zarr3BucketStreamSink(dataLayer, tracing.fallbackLayer.nonEmpty)( + dataLayer.bucketProvider.bucketStream(Some(tracing.version)), + tracing.resolutions.map(mag => vec3IntFromProto(mag)), + tracing.additionalAxes, + voxelSize) } val before = Instant.now diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/WKWBucketStreamSink.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/WKWBucketStreamSink.scala index d8ed2b3469..1d70c78c61 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/WKWBucketStreamSink.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/WKWBucketStreamSink.scala @@ -7,28 +7,41 @@ import com.scalableminds.webknossos.datastore.models.BucketPosition import com.scalableminds.webknossos.datastore.models.datasource.DataLayer import com.scalableminds.util.tools.ByteUtils import com.scalableminds.webknossos.wrap.{BlockType, WKWFile, WKWHeader} +import com.typesafe.scalalogging.LazyLogging import java.io.DataOutputStream import scala.concurrent.{ExecutionContext, Future} -class WKWBucketStreamSink(val layer: DataLayer) extends WKWDataFormatHelper with ByteUtils { +class WKWBucketStreamSink(val layer: DataLayer, tracingHasFallbackLayer: Boolean) + extends WKWDataFormatHelper + with VolumeBucketReversionHelper + with LazyLogging + with ByteUtils { def apply(bucketStream: Iterator[(BucketPosition, Array[Byte])], mags: Seq[Vec3Int])( implicit ec: ExecutionContext): Iterator[NamedStream] = { val (voxelType, numChannels) = WKWDataFormat.elementClassToVoxelType(layer.elementClass) val header = WKWHeader(1, DataLayer.bucketLength, BlockType.LZ4, voxelType, numChannels) bucketStream.flatMap { - case (bucket, data) if !isAllZero(data) => - val filePath = wkwFilePath(bucket.toCube(bucket.bucketLength)).toString - Some( - NamedFunctionStream( - filePath, - os => Future.successful(WKWFile.write(os, header, Array(data).iterator)) - )) + case (bucket, data) => + logger.info(f"isRevertedBucket(data): ${isRevertedBucket(data)}, isAllZero(data): ${isAllZero(data)}") + val skipBucket = if (tracingHasFallbackLayer) isRevertedBucket(data) else isAllZero(data) + if (skipBucket) { + // If the tracing has no fallback segmentation, all-zero buckets can be omitted entirely + None + } else { + val filePath = wkwFilePath(bucket.toCube(bucket.bucketLength)).toString + Some( + NamedFunctionStream( + filePath, + os => Future.successful(WKWFile.write(os, header, Array(data).iterator)) + )) + } case _ => None } ++ mags.map { mag => NamedFunctionStream(wkwHeaderFilePath(mag).toString, os => Future.successful(header.writeTo(new DataOutputStream(os), isHeaderFile = true))) } } + } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala index 6195ce6473..2e4d12aa71 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala @@ -22,9 +22,10 @@ import play.api.libs.json.Json import scala.concurrent.{ExecutionContext, Future} // Creates data zip from volume tracings -class Zarr3BucketStreamSink(val layer: VolumeTracingLayer) +class Zarr3BucketStreamSink(val layer: VolumeTracingLayer, tracingHasFallbackLayer: Boolean) extends LazyLogging with ProtoGeometryImplicits + with VolumeBucketReversionHelper with ByteUtils { private lazy val defaultLayerName = "volumeAnnotationData" @@ -69,13 +70,20 @@ class Zarr3BucketStreamSink(val layer: VolumeTracingLayer) dimension_names = Some(Array("c") ++ additionalAxes.map(_.name).toArray ++ Seq("x", "y", "z")) ) bucketStream.flatMap { - case (bucket, data) if !isAllZero(data) => - val filePath = zarrChunkFilePath(defaultLayerName, bucket) - Some( - NamedFunctionStream( - filePath, - os => Future.successful(os.write(compressor.compress(data))) - )) + case (bucket, data) => + val skipBucket = if (tracingHasFallbackLayer) isAllZero(data) else isRevertedBucket(data) + if (skipBucket) { + // If the tracing has no fallback segmentation, all-zero buckets can be omitted entirely + None + } else { + val filePath = zarrChunkFilePath(defaultLayerName, bucket) + Some( + NamedFunctionStream( + filePath, + os => Future.successful(os.write(compressor.compress(data))) + ) + ) + } case _ => None } ++ mags.map { mag => NamedFunctionStream.fromString(zarrHeaderFilePath(defaultLayerName, mag), Json.prettyPrint(Json.toJson(header))) From 3dd812639152bb8164c99d3d75691c12b2ae662d Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 22 Jan 2024 14:41:55 +0100 Subject: [PATCH 2/4] changelog, remove logging --- CHANGELOG.unreleased.md | 1 + .../tracingstore/tracings/volume/WKWBucketStreamSink.scala | 3 --- .../tracingstore/tracings/volume/Zarr3BucketStreamSink.scala | 4 +--- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index d279a260b6..adbc4a1504 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -55,6 +55,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released - Fixed the initialization of the mapping list for agglomerate views if json mappings are present. [#7537](https://github.com/scalableminds/webknossos/pull/7537) - Fixed a bug where uploading ND volume annotations would lead to errors due to parsing of the chunk paths. [#7547](https://github.com/scalableminds/webknossos/pull/7547) - Fixed a bug where listing the annotations of other users would result in empty lists even if there are annotations and you should be allowed to see them. [#7563](https://github.com/scalableminds/webknossos/pull/7563) +- Fixed a bug where all-zero chunks/buckets were omitted when downloading volume annotation even in case of a fallback segmentation layer, where ther zeroed-bucket information is actually needed. [#7576](https://github.com/scalableminds/webknossos/pull/7576) ### Removed - Removed several unused frontend libraries. [#7521](https://github.com/scalableminds/webknossos/pull/7521) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/WKWBucketStreamSink.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/WKWBucketStreamSink.scala index 1d70c78c61..408f2152f9 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/WKWBucketStreamSink.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/WKWBucketStreamSink.scala @@ -7,7 +7,6 @@ import com.scalableminds.webknossos.datastore.models.BucketPosition import com.scalableminds.webknossos.datastore.models.datasource.DataLayer import com.scalableminds.util.tools.ByteUtils import com.scalableminds.webknossos.wrap.{BlockType, WKWFile, WKWHeader} -import com.typesafe.scalalogging.LazyLogging import java.io.DataOutputStream import scala.concurrent.{ExecutionContext, Future} @@ -15,7 +14,6 @@ import scala.concurrent.{ExecutionContext, Future} class WKWBucketStreamSink(val layer: DataLayer, tracingHasFallbackLayer: Boolean) extends WKWDataFormatHelper with VolumeBucketReversionHelper - with LazyLogging with ByteUtils { def apply(bucketStream: Iterator[(BucketPosition, Array[Byte])], mags: Seq[Vec3Int])( @@ -24,7 +22,6 @@ class WKWBucketStreamSink(val layer: DataLayer, tracingHasFallbackLayer: Boolean val header = WKWHeader(1, DataLayer.bucketLength, BlockType.LZ4, voxelType, numChannels) bucketStream.flatMap { case (bucket, data) => - logger.info(f"isRevertedBucket(data): ${isRevertedBucket(data)}, isAllZero(data): ${isAllZero(data)}") val skipBucket = if (tracingHasFallbackLayer) isRevertedBucket(data) else isAllZero(data) if (skipBucket) { // If the tracing has no fallback segmentation, all-zero buckets can be omitted entirely diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala index 2e4d12aa71..7c5cfa4987 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala @@ -16,15 +16,13 @@ import com.scalableminds.webknossos.datastore.geometry.AdditionalAxisProto import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits import com.scalableminds.webknossos.datastore.models.datasource.{DataLayer, DataSourceId, GenericDataSource} import com.scalableminds.webknossos.datastore.models.{AdditionalCoordinate, BucketPosition} -import com.typesafe.scalalogging.LazyLogging import play.api.libs.json.Json import scala.concurrent.{ExecutionContext, Future} // Creates data zip from volume tracings class Zarr3BucketStreamSink(val layer: VolumeTracingLayer, tracingHasFallbackLayer: Boolean) - extends LazyLogging - with ProtoGeometryImplicits + extends ProtoGeometryImplicits with VolumeBucketReversionHelper with ByteUtils { From c6c5233017bfb024302ed7a35d93ec291d2736ad Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 22 Jan 2024 14:48:45 +0100 Subject: [PATCH 3/4] unused import --- .../tracingstore/tracings/volume/VolumeDataZipHelper.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeDataZipHelper.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeDataZipHelper.scala index 30d9f7809f..011aec37e2 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeDataZipHelper.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeDataZipHelper.scala @@ -3,7 +3,7 @@ package com.scalableminds.webknossos.tracingstore.tracings.volume import java.io.{File, FileOutputStream, InputStream} import com.scalableminds.util.geometry.Vec3Int import com.scalableminds.util.io.ZipIO -import com.scalableminds.util.tools.{BoxImplicits, ByteUtils, JsonHelper} +import com.scalableminds.util.tools.{BoxImplicits, JsonHelper} import com.scalableminds.webknossos.datastore.dataformats.wkw.WKWDataFormatHelper import com.scalableminds.webknossos.datastore.datareaders.{ BloscCompressor, From c57c6a0810eb49b62c92a55cbaaa308121fb248e Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 24 Jan 2024 10:23:12 +0100 Subject: [PATCH 4/4] Update CHANGELOG.unreleased.md Co-authored-by: frcroth --- CHANGELOG.unreleased.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index adbc4a1504..ae08527147 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -55,7 +55,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released - Fixed the initialization of the mapping list for agglomerate views if json mappings are present. [#7537](https://github.com/scalableminds/webknossos/pull/7537) - Fixed a bug where uploading ND volume annotations would lead to errors due to parsing of the chunk paths. [#7547](https://github.com/scalableminds/webknossos/pull/7547) - Fixed a bug where listing the annotations of other users would result in empty lists even if there are annotations and you should be allowed to see them. [#7563](https://github.com/scalableminds/webknossos/pull/7563) -- Fixed a bug where all-zero chunks/buckets were omitted when downloading volume annotation even in case of a fallback segmentation layer, where ther zeroed-bucket information is actually needed. [#7576](https://github.com/scalableminds/webknossos/pull/7576) +- Fixed a bug where all-zero chunks/buckets were omitted when downloading volume annotation even in case of a fallback segmentation layer, where their zeroed-bucket information is actually needed. [#7576](https://github.com/scalableminds/webknossos/pull/7576) ### Removed - Removed several unused frontend libraries. [#7521](https://github.com/scalableminds/webknossos/pull/7521)