From 6563e94d81afb6683c5bd24d689d84238dd3eb8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Mon, 7 Oct 2024 19:47:27 +0200 Subject: [PATCH 1/6] fix shape in zarr.json in zarr3 streaming --- .../controllers/ZarrStreamingController.scala | 2 +- .../datareaders/zarr3/Zarr3ArrayHeader.scala | 12 ++++++++++-- .../tracings/volume/Zarr3BucketStreamSink.scala | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala index 2c94c3386e6..dc5ccd18987 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala @@ -342,7 +342,7 @@ class ZarrStreamingController @Inject()( "dataSource.notFound") ~> NOT_FOUND magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> NOT_FOUND - zarrHeader = Zarr3ArrayHeader.fromDataLayer(dataLayer) + zarrHeader = Zarr3ArrayHeader.fromDataLayer(dataLayer, magParsed) } yield Ok(Json.toJson(zarrHeader)) def zArrayPrivateLink(token: Option[String], diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala index 783ed80022f..cfd038f4e46 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala @@ -1,5 +1,6 @@ package com.scalableminds.webknossos.datastore.datareaders.zarr3 +import com.scalableminds.util.geometry.Vec3Int import com.scalableminds.util.tools.{BoxImplicits, JsonHelper} import com.scalableminds.webknossos.datastore.datareaders.ArrayDataType.ArrayDataType import com.scalableminds.webknossos.datastore.datareaders.ArrayOrder.ArrayOrder @@ -251,13 +252,20 @@ object Zarr3ArrayHeader extends JsonImplicits { ) } - def fromDataLayer(dataLayer: DataLayer): Zarr3ArrayHeader = { + def fromDataLayer(dataLayer: DataLayer, mag: Vec3Int): Zarr3ArrayHeader = { val additionalAxes = reorderAdditionalAxes(dataLayer.additionalAxes.getOrElse(Seq.empty)) + // TODOM: Make code together with ZarrHeader.scala DRYer + val xyzBBounds = Array( + // Zarr can't handle data sets that don't start at 0, so we extend the shape to include "true" coords + (dataLayer.boundingBox.width + dataLayer.boundingBox.topLeft.x) / mag.x, + (dataLayer.boundingBox.height + dataLayer.boundingBox.topLeft.y) / mag.y, + (dataLayer.boundingBox.depth + dataLayer.boundingBox.topLeft.z) / mag.z + ) Zarr3ArrayHeader( zarr_format = 3, node_type = "array", // channel, additional axes, XYZ - shape = Array(1) ++ additionalAxes.map(_.highestValue).toArray ++ dataLayer.boundingBox.bottomRight.toArray, + shape = Array(1) ++ additionalAxes.map(_.highestValue).toArray ++ xyzBBounds, data_type = Left(dataLayer.elementClass.toString), chunk_grid = Left( ChunkGridSpecification( diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala index f2baa1f9c80..5b39ca4348e 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala @@ -41,7 +41,7 @@ class Zarr3BucketStreamSink(val layer: VolumeTracingLayer, tracingHasFallbackLay def apply(bucketStream: Iterator[(BucketPosition, Array[Byte])], mags: Seq[Vec3Int], voxelSize: Option[VoxelSize])( implicit ec: ExecutionContext): Iterator[NamedStream] = { - val header = Zarr3ArrayHeader.fromDataLayer(layer) + val header = Zarr3ArrayHeader.fromDataLayer(layer, mags.head) bucketStream.flatMap { case (bucket, data) => val skipBucket = if (tracingHasFallbackLayer) isAllZero(data) else isRevertedBucket(data) From 592ea8ef2797760724681afa9e920c549b5079b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Tue, 8 Oct 2024 13:33:58 +0200 Subject: [PATCH 2/6] remove comment to make code DRYer --- .../datastore/datareaders/zarr3/Zarr3ArrayHeader.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala index cfd038f4e46..377916e9f4b 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala @@ -254,7 +254,6 @@ object Zarr3ArrayHeader extends JsonImplicits { } def fromDataLayer(dataLayer: DataLayer, mag: Vec3Int): Zarr3ArrayHeader = { val additionalAxes = reorderAdditionalAxes(dataLayer.additionalAxes.getOrElse(Seq.empty)) - // TODOM: Make code together with ZarrHeader.scala DRYer val xyzBBounds = Array( // Zarr can't handle data sets that don't start at 0, so we extend the shape to include "true" coords (dataLayer.boundingBox.width + dataLayer.boundingBox.topLeft.x) / mag.x, From ebdfbe510df6053578d143bc71be975302ca73ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Tue, 8 Oct 2024 17:07:08 +0200 Subject: [PATCH 3/6] use default mag when creating zarr3 header file in bucket sink --- .../tracingstore/tracings/volume/Zarr3BucketStreamSink.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala index 5b39ca4348e..1b23b4fffa9 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala @@ -41,7 +41,7 @@ class Zarr3BucketStreamSink(val layer: VolumeTracingLayer, tracingHasFallbackLay def apply(bucketStream: Iterator[(BucketPosition, Array[Byte])], mags: Seq[Vec3Int], voxelSize: Option[VoxelSize])( implicit ec: ExecutionContext): Iterator[NamedStream] = { - val header = Zarr3ArrayHeader.fromDataLayer(layer, mags.head) + val header = Zarr3ArrayHeader.fromDataLayer(layer, mags.headOption.getOrElse(Vec3Int.ones)) bucketStream.flatMap { case (bucket, data) => val skipBucket = if (tracingHasFallbackLayer) isAllZero(data) else isRevertedBucket(data) From dd43f3d9a58dc8d75a1cdd2e9c81c4f8815eb9ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Tue, 8 Oct 2024 18:34:59 +0200 Subject: [PATCH 4/6] add changelog entry --- CHANGELOG.unreleased.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index f722f4c8cbc..0dff4a9965d 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -22,6 +22,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released ### Fixed - Fixed a bug during dataset upload in case the configured `datastore.baseFolder` is an absolute path. [#8098](https://github.com/scalableminds/webknossos/pull/8098) [#8103](https://github.com/scalableminds/webknossos/pull/8103) +- Fixed a bug where the materialize volume annotation job would not start due to a missing parameter. For now the name of the output segmentation layer of the job is autogenerated. [#8116](https://github.com/scalableminds/webknossos/pull/8116) - Fixed a bug where you could not create annotations for public datasets of other organizations. [#8107](https://github.com/scalableminds/webknossos/pull/8107) ### Removed From cd32695aafc773dd71cf9ec78f9697482c2abfc5 Mon Sep 17 00:00:00 2001 From: MichaelBuessemeyer <39529669+MichaelBuessemeyer@users.noreply.github.com> Date: Tue, 8 Oct 2024 18:53:12 +0200 Subject: [PATCH 5/6] Update CHANGELOG.unreleased.md --- CHANGELOG.unreleased.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 0dff4a9965d..460b129a2e4 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -22,7 +22,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released ### Fixed - Fixed a bug during dataset upload in case the configured `datastore.baseFolder` is an absolute path. [#8098](https://github.com/scalableminds/webknossos/pull/8098) [#8103](https://github.com/scalableminds/webknossos/pull/8103) -- Fixed a bug where the materialize volume annotation job would not start due to a missing parameter. For now the name of the output segmentation layer of the job is autogenerated. [#8116](https://github.com/scalableminds/webknossos/pull/8116) +- Fixed a bug where the materialize volume annotation job would not start due to a missing parameter. For now the name of the output segmentation layer of the job is autogenerated. [#8118](https://github.com/scalableminds/webknossos/pull/8118) - Fixed a bug where you could not create annotations for public datasets of other organizations. [#8107](https://github.com/scalableminds/webknossos/pull/8107) ### Removed From 4c16d65a62ede911425b481859702701aa1344f7 Mon Sep 17 00:00:00 2001 From: MichaelBuessemeyer <39529669+MichaelBuessemeyer@users.noreply.github.com> Date: Tue, 8 Oct 2024 18:57:00 +0200 Subject: [PATCH 6/6] Update CHANGELOG.unreleased.md --- CHANGELOG.unreleased.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 460b129a2e4..5ce59f3f007 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -22,7 +22,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released ### Fixed - Fixed a bug during dataset upload in case the configured `datastore.baseFolder` is an absolute path. [#8098](https://github.com/scalableminds/webknossos/pull/8098) [#8103](https://github.com/scalableminds/webknossos/pull/8103) -- Fixed a bug where the materialize volume annotation job would not start due to a missing parameter. For now the name of the output segmentation layer of the job is autogenerated. [#8118](https://github.com/scalableminds/webknossos/pull/8118) +- Fixed a bug that zarr streaming version 3 returned the shape of mag (1, 1, 1) / the finest mag for all mags. [#8116](https://github.com/scalableminds/webknossos/pull/8116) - Fixed a bug where you could not create annotations for public datasets of other organizations. [#8107](https://github.com/scalableminds/webknossos/pull/8107) ### Removed