From 7698a02ae7b6fbbbe0f4bc2b1747b6d676f4f180 Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 29 Sep 2022 21:05:52 +0200 Subject: [PATCH] Always return 404 for Failures in Zarr Streaming (#6515) * Always return 404 for Failures in Zarr Streaming * changelog * use NOT_FUOND constant --- CHANGELOG.unreleased.md | 1 + .../util/mvc/ExtendedController.scala | 6 +- .../controllers/ZarrStreamingController.scala | 2 + .../controllers/VolumeTracingController.scala | 229 +--------------- ...VolumeTracingZarrStreamingController.scala | 256 ++++++++++++++++++ ...alableminds.webknossos.tracingstore.routes | 22 +- 6 files changed, 278 insertions(+), 238 deletions(-) create mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 55dd5449149..0f6d478c76f 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -16,6 +16,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released - Creating tasks in bulk now also supports referencing task types by their summary instead of id. [#6486](https://github.com/scalableminds/webknossos/pull/6486) ### Fixed +- Fixed a bug where some file requests replied with error 400 instead of 404, confusing some zarr clients. [#6515](https://github.com/scalableminds/webknossos/pull/6515) ### Removed diff --git a/util/src/main/scala/com/scalableminds/util/mvc/ExtendedController.scala b/util/src/main/scala/com/scalableminds/util/mvc/ExtendedController.scala index cf50ec2ec9d..c247ad7ca90 100644 --- a/util/src/main/scala/com/scalableminds/util/mvc/ExtendedController.scala +++ b/util/src/main/scala/com/scalableminds/util/mvc/ExtendedController.scala @@ -19,6 +19,8 @@ import scala.concurrent.{ExecutionContext, Future} trait BoxToResultHelpers extends I18nSupport with Formatter with RemoteOriginHelpers { + protected def defaultErrorCode: Int = BAD_REQUEST + def asResult[T <: Result](b: Box[T])(implicit messages: MessagesProvider): Result = { val result = b match { case Full(result) => @@ -26,9 +28,9 @@ trait BoxToResultHelpers extends I18nSupport with Formatter with RemoteOriginHel case ParamFailure(msg, _, chain, statusCode: Int) => new JsonResult(statusCode)(Messages(msg), formatChainOpt(chain)) case ParamFailure(_, _, _, msgs: JsArray) => - new JsonResult(BAD_REQUEST)(jsonMessages(msgs)) + new JsonResult(defaultErrorCode)(jsonMessages(msgs)) case Failure(msg, _, chain) => - new JsonResult(BAD_REQUEST)(Messages(msg), formatChainOpt(chain)) + new JsonResult(defaultErrorCode)(Messages(msg), formatChainOpt(chain)) case Empty => new JsonResult(NOT_FOUND)("Couldn't find the requested resource.") } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala index f994ca7563d..5f6486b20d1 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala @@ -35,6 +35,8 @@ class ZarrStreamingController @Inject()( )(implicit ec: ExecutionContext) extends Controller { + override def defaultErrorCode: Int = NOT_FOUND + val binaryDataService: BinaryDataService = binaryDataServiceHolder.binaryDataService override def allowRemoteOrigin: Boolean = true diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index 8c04cadd9f7..f01a79cb217 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -1,19 +1,12 @@ package com.scalableminds.webknossos.tracingstore.controllers -import java.io.File -import java.nio.{ByteBuffer, ByteOrder} import akka.stream.scaladsl.Source import com.google.inject.Inject import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int} import com.scalableminds.util.tools.ExtendedTypes.ExtendedString import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings} -import com.scalableminds.webknossos.datastore.dataformats.MagLocator -import com.scalableminds.webknossos.datastore.dataformats.zarr.{ZarrCoordinatesParser, ZarrSegmentationLayer} -import com.scalableminds.webknossos.datastore.datareaders.jzarr.{OmeNgffGroupHeader, OmeNgffHeader, ZarrHeader} -import com.scalableminds.webknossos.datastore.datareaders.{ArrayOrder, AxisOrder} import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits -import com.scalableminds.webknossos.datastore.models.datasource.{DataLayer, ElementClass} import com.scalableminds.webknossos.datastore.models.{WebKnossosDataRequest, WebKnossosIsosurfaceRequest} import com.scalableminds.webknossos.datastore.rpc.RPC import com.scalableminds.webknossos.datastore.services.UserAccessRequest @@ -43,16 +36,18 @@ import play.api.libs.iteratee.streams.IterateeStreams import play.api.libs.json.Json import play.api.mvc.{Action, AnyContent, MultipartFormData, PlayBodyParsers} -import scala.concurrent.{ExecutionContext, Future} +import java.io.File +import java.nio.{ByteBuffer, ByteOrder} +import scala.concurrent.ExecutionContext class VolumeTracingController @Inject()( val tracingService: VolumeTracingService, val config: TracingStoreConfig, - val remoteWebKnossosClient: TSRemoteWebKnossosClient, val remoteDataStoreClient: TSRemoteDatastoreClient, val accessTokenService: TracingStoreAccessTokenService, editableMappingService: EditableMappingService, val slackNotificationService: TSSlackNotificationService, + val remoteWebKnossosClient: TSRemoteWebKnossosClient, val rpc: RPC)(implicit val ec: ExecutionContext, val bodyParsers: PlayBodyParsers) extends TracingController[VolumeTracing, VolumeTracings] with ProtoGeometryImplicits @@ -253,222 +248,6 @@ class VolumeTracingController @Inject()( } } - def volumeTracingFolderContent(token: Option[String], tracingId: String): Action[AnyContent] = - Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { - for { - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND - existingMags = tracing.resolutions.map(vec3IntFromProto) - } yield - Ok( - views.html.datastoreZarrDatasourceDir( - "Tracingstore", - "%s".format(tracingId), - List(".zattrs", ".zgroup") ++ existingMags.map(_.toMagLiteral(allowScalar = true)) - )).withHeaders() - } - } - - def volumeTracingFolderContentJson(token: Option[String], tracingId: String): Action[AnyContent] = - Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { - for { - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> 404 - existingMags = tracing.resolutions.map(vec3IntFromProto(_).toMagLiteral(allowScalar = true)) - } yield Ok(Json.toJson(List(".zattrs", ".zgroup") ++ existingMags)) - } - } - - def volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String): Action[AnyContent] = - Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { - for { - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND - - existingMags = tracing.resolutions.map(vec3IntFromProto) - magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND - _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND - } yield - Ok( - views.html.datastoreZarrDatasourceDir( - "Tracingstore", - "%s".format(tracingId), - List(".zarray") - )).withHeaders() - } - } - - def volumeTracingMagFolderContentJson(token: Option[String], tracingId: String, mag: String): Action[AnyContent] = - Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { - for { - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> 404 - - existingMags = tracing.resolutions.map(vec3IntFromProto) - magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> 404 - _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> 404 - } yield Ok(Json.toJson(List(".zarray"))) - } - } - - def zArray(token: Option[String], tracingId: String, mag: String): Action[AnyContent] = Action.async { - implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { - for { - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND - - existingMags = tracing.resolutions.map(vec3IntFromProto) - magParsed <- Vec3Int - .fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND - _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND - - cubeLength = DataLayer.bucketLength - (channels, dtype) = ElementClass.toChannelAndZarrString(tracing.elementClass) - // data request method always decompresses before sending - compressor = None - - shape = Array( - channels, - // Zarr can't handle data sets that don't start at 0, so we extend shape to include "true" coords - (tracing.boundingBox.width + tracing.boundingBox.topLeft.x) / magParsed.x, - (tracing.boundingBox.height + tracing.boundingBox.topLeft.y) / magParsed.y, - (tracing.boundingBox.depth + tracing.boundingBox.topLeft.z) / magParsed.z - ) - - chunks = Array(channels, cubeLength, cubeLength, cubeLength) - - zarrHeader = ZarrHeader(zarr_format = 2, - shape = shape, - chunks = chunks, - compressor = compressor, - dtype = dtype, - order = ArrayOrder.F) - } yield Ok(Json.toJson(zarrHeader)) - } - } - - def zGroup(token: Option[String], tracingId: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { - Future(Ok(Json.toJson(OmeNgffGroupHeader(zarr_format = 2)))) - } - } - - /** - * Handles a request for .zattrs file for a Volume Tracing via a HTTP GET. - * Uses the OME-NGFF standard (see https://ngff.openmicroscopy.org/latest/) - * Used by zarr-streaming. - */ - def zAttrs( - token: Option[String], - tracingId: String, - ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { - for { - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND - - existingMags = tracing.resolutions.map(vec3IntFromProto) - dataSource <- remoteWebKnossosClient.getDataSource(tracing.organizationName, tracing.dataSetName) ~> NOT_FOUND - - omeNgffHeader = OmeNgffHeader.fromNameScaleAndMags(tracingId, - dataSourceScale = dataSource.scale, - mags = existingMags.toList) - } yield Ok(Json.toJson(omeNgffHeader)) - } - } - - def zarrSource(token: Option[String], tracingId: String, tracingName: Option[String]): Action[AnyContent] = - Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { - for { - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> 404 - - zarrLayer = ZarrSegmentationLayer( - name = tracingName.getOrElse(tracingId), - largestSegmentId = tracing.largestSegmentId, - boundingBox = tracing.boundingBox, - elementClass = tracing.elementClass, - mags = tracing.resolutions.toList.map(x => MagLocator(x, None, None, Some(AxisOrder.cxyz))), - mappings = None, - numChannels = Some(if (tracing.elementClass.isuint24) 3 else 1) - ) - } yield Ok(Json.toJson(zarrLayer)) - } - } - - def rawZarrCube(token: Option[String], tracingId: String, mag: String, cxyz: String): Action[AnyContent] = - Action.async { implicit request => - { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { - for { - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND - - existingMags = tracing.resolutions.map(vec3IntFromProto) - magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND - _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND - - (c, x, y, z) <- ZarrCoordinatesParser.parseDotCoordinates(cxyz) ?~> Messages("zarr.invalidChunkCoordinates") ~> NOT_FOUND - _ <- bool2Fox(c == 0) ~> Messages("zarr.invalidFirstChunkCoord") ~> NOT_FOUND - cubeSize = DataLayer.bucketLength - wkRequest = WebKnossosDataRequest( - position = Vec3Int(x, y, z) * cubeSize * magParsed, - mag = magParsed, - cubeSize = cubeSize, - fourBit = Some(false), - applyAgglomerate = None, - version = None - ) - (data, missingBucketIndices) <- if (tracing.getMappingIsEditable) - editableMappingService.volumeData(tracing, List(wkRequest), urlOrHeaderToken(token, request)) - else tracingService.data(tracingId, tracing, List(wkRequest)) - dataWithFallback <- getFallbackLayerDataIfEmpty(tracing, - data, - missingBucketIndices, - magParsed, - Vec3Int(x, y, z), - cubeSize, - urlOrHeaderToken(token, request)) ~> NOT_FOUND - } yield Ok(dataWithFallback) - } - } - } - - private def getFallbackLayerDataIfEmpty(tracing: VolumeTracing, - data: Array[Byte], - missingBucketIndices: List[Int], - mag: Vec3Int, - position: Vec3Int, - cubeSize: Int, - urlToken: Option[String]): Fox[Array[Byte]] = { - def fallbackLayerData(fallbackLayerName: String): Fox[Array[Byte]] = { - val request = WebKnossosDataRequest( - position = position * mag * cubeSize, - mag = mag, - cubeSize = cubeSize, - fourBit = Some(false), - applyAgglomerate = tracing.mappingName, - version = None - ) - for { - organizationName <- tracing.organizationName ?~> "Zarr streaming not supported for legacy volume annotations (organizationName is not set)" - remoteFallbackLayer = RemoteFallbackLayer(organizationName, - tracing.dataSetName, - fallbackLayerName, - tracing.elementClass) - (fallbackData, fallbackMissingBucketIndices) <- remoteDataStoreClient.getData(remoteFallbackLayer, - List(request), - urlToken) - _ <- bool2Fox(fallbackMissingBucketIndices.isEmpty) ?~> "No data at coordinations in fallback layer" - } yield fallbackData - } - - if (missingBucketIndices.nonEmpty) { - for { - fallbackLayer <- tracing.fallbackLayer.toFox ?~> "No data at coordinates, no fallback layer defined" - data <- fallbackLayerData(fallbackLayer) ?~> "No data at coordinates, no fallback layer data at coordinates." - } yield data - } else Fox.successful(data) - } - private def getNeighborIndices(neighbors: List[Int]) = List("NEIGHBORS" -> formatNeighborList(neighbors), "Access-Control-Expose-Headers" -> "NEIGHBORS") diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala new file mode 100644 index 00000000000..6b1f17395f0 --- /dev/null +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala @@ -0,0 +1,256 @@ +package com.scalableminds.webknossos.tracingstore.controllers + +import com.google.inject.Inject +import com.scalableminds.util.geometry.Vec3Int +import com.scalableminds.util.mvc.ExtendedController +import com.scalableminds.util.tools.{Fox, FoxImplicits} +import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing +import com.scalableminds.webknossos.datastore.dataformats.MagLocator +import com.scalableminds.webknossos.datastore.dataformats.zarr.{ZarrCoordinatesParser, ZarrSegmentationLayer} +import com.scalableminds.webknossos.datastore.datareaders.jzarr.{OmeNgffGroupHeader, OmeNgffHeader, ZarrHeader} +import com.scalableminds.webknossos.datastore.datareaders.{ArrayOrder, AxisOrder} +import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits +import com.scalableminds.webknossos.datastore.models.WebKnossosDataRequest +import com.scalableminds.webknossos.datastore.models.datasource.{DataLayer, ElementClass} +import com.scalableminds.webknossos.datastore.services.UserAccessRequest +import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{EditableMappingService, RemoteFallbackLayer} +import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeTracingService +import com.scalableminds.webknossos.tracingstore.{ + TSRemoteDatastoreClient, + TSRemoteWebKnossosClient, + TracingStoreAccessTokenService +} +import play.api.i18n.Messages +import play.api.libs.json.Json +import play.api.mvc.{Action, AnyContent} + +import scala.concurrent.{ExecutionContext, Future} + +class VolumeTracingZarrStreamingController @Inject()( + tracingService: VolumeTracingService, + accessTokenService: TracingStoreAccessTokenService, + editableMappingService: EditableMappingService, + remoteDataStoreClient: TSRemoteDatastoreClient, + remoteWebKnossosClient: TSRemoteWebKnossosClient)(implicit ec: ExecutionContext) + extends ExtendedController + with ProtoGeometryImplicits + with FoxImplicits { + + override def defaultErrorCode: Int = NOT_FOUND + + def volumeTracingFolderContent(token: Option[String], tracingId: String): Action[AnyContent] = + Action.async { implicit request => + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + for { + tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + existingMags = tracing.resolutions.map(vec3IntFromProto) + } yield + Ok( + views.html.datastoreZarrDatasourceDir( + "Tracingstore", + "%s".format(tracingId), + List(".zattrs", ".zgroup") ++ existingMags.map(_.toMagLiteral(allowScalar = true)) + )).withHeaders() + } + } + + def volumeTracingFolderContentJson(token: Option[String], tracingId: String): Action[AnyContent] = + Action.async { implicit request => + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + for { + tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + existingMags = tracing.resolutions.map(vec3IntFromProto(_).toMagLiteral(allowScalar = true)) + } yield Ok(Json.toJson(List(".zattrs", ".zgroup") ++ existingMags)) + } + } + + def volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String): Action[AnyContent] = + Action.async { implicit request => + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + for { + tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + + existingMags = tracing.resolutions.map(vec3IntFromProto) + magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND + _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND + } yield + Ok( + views.html.datastoreZarrDatasourceDir( + "Tracingstore", + "%s".format(tracingId), + List(".zarray") + )).withHeaders() + } + } + + def volumeTracingMagFolderContentJson(token: Option[String], tracingId: String, mag: String): Action[AnyContent] = + Action.async { implicit request => + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + for { + tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + + existingMags = tracing.resolutions.map(vec3IntFromProto) + magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND + _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND + } yield Ok(Json.toJson(List(".zarray"))) + } + } + + def zArray(token: Option[String], tracingId: String, mag: String): Action[AnyContent] = Action.async { + implicit request => + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + for { + tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + + existingMags = tracing.resolutions.map(vec3IntFromProto) + magParsed <- Vec3Int + .fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND + _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND + + cubeLength = DataLayer.bucketLength + (channels, dtype) = ElementClass.toChannelAndZarrString(tracing.elementClass) + // data request method always decompresses before sending + compressor = None + + shape = Array( + channels, + // Zarr can't handle data sets that don't start at 0, so we extend shape to include "true" coords + (tracing.boundingBox.width + tracing.boundingBox.topLeft.x) / magParsed.x, + (tracing.boundingBox.height + tracing.boundingBox.topLeft.y) / magParsed.y, + (tracing.boundingBox.depth + tracing.boundingBox.topLeft.z) / magParsed.z + ) + + chunks = Array(channels, cubeLength, cubeLength, cubeLength) + + zarrHeader = ZarrHeader(zarr_format = 2, + shape = shape, + chunks = chunks, + compressor = compressor, + dtype = dtype, + order = ArrayOrder.F) + } yield Ok(Json.toJson(zarrHeader)) + } + } + + def zGroup(token: Option[String], tracingId: String): Action[AnyContent] = Action.async { implicit request => + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + Future(Ok(Json.toJson(OmeNgffGroupHeader(zarr_format = 2)))) + } + } + + /** + * Handles a request for .zattrs file for a Volume Tracing via a HTTP GET. + * Uses the OME-NGFF standard (see https://ngff.openmicroscopy.org/latest/) + * Used by zarr-streaming. + */ + def zAttrs( + token: Option[String], + tracingId: String, + ): Action[AnyContent] = Action.async { implicit request => + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + for { + tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + + existingMags = tracing.resolutions.map(vec3IntFromProto) + dataSource <- remoteWebKnossosClient.getDataSource(tracing.organizationName, tracing.dataSetName) ~> NOT_FOUND + + omeNgffHeader = OmeNgffHeader.fromNameScaleAndMags(tracingId, + dataSourceScale = dataSource.scale, + mags = existingMags.toList) + } yield Ok(Json.toJson(omeNgffHeader)) + } + } + + def zarrSource(token: Option[String], tracingId: String, tracingName: Option[String]): Action[AnyContent] = + Action.async { implicit request => + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + for { + tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + + zarrLayer = ZarrSegmentationLayer( + name = tracingName.getOrElse(tracingId), + largestSegmentId = tracing.largestSegmentId, + boundingBox = tracing.boundingBox, + elementClass = tracing.elementClass, + mags = tracing.resolutions.toList.map(x => MagLocator(x, None, None, Some(AxisOrder.cxyz))), + mappings = None, + numChannels = Some(if (tracing.elementClass.isuint24) 3 else 1) + ) + } yield Ok(Json.toJson(zarrLayer)) + } + } + + def rawZarrCube(token: Option[String], tracingId: String, mag: String, cxyz: String): Action[AnyContent] = + Action.async { implicit request => + { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + for { + tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + + existingMags = tracing.resolutions.map(vec3IntFromProto) + magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND + _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND + + (c, x, y, z) <- ZarrCoordinatesParser.parseDotCoordinates(cxyz) ?~> Messages("zarr.invalidChunkCoordinates") ~> NOT_FOUND + _ <- bool2Fox(c == 0) ~> Messages("zarr.invalidFirstChunkCoord") ~> NOT_FOUND + cubeSize = DataLayer.bucketLength + wkRequest = WebKnossosDataRequest( + position = Vec3Int(x, y, z) * cubeSize * magParsed, + mag = magParsed, + cubeSize = cubeSize, + fourBit = Some(false), + applyAgglomerate = None, + version = None + ) + (data, missingBucketIndices) <- if (tracing.getMappingIsEditable) + editableMappingService.volumeData(tracing, List(wkRequest), urlOrHeaderToken(token, request)) + else tracingService.data(tracingId, tracing, List(wkRequest)) + dataWithFallback <- getFallbackLayerDataIfEmpty(tracing, + data, + missingBucketIndices, + magParsed, + Vec3Int(x, y, z), + cubeSize, + urlOrHeaderToken(token, request)) ~> NOT_FOUND + } yield Ok(dataWithFallback) + } + } + } + + private def getFallbackLayerDataIfEmpty(tracing: VolumeTracing, + data: Array[Byte], + missingBucketIndices: List[Int], + mag: Vec3Int, + position: Vec3Int, + cubeSize: Int, + urlToken: Option[String]): Fox[Array[Byte]] = { + def fallbackLayerData(fallbackLayerName: String): Fox[Array[Byte]] = { + val request = WebKnossosDataRequest( + position = position * mag * cubeSize, + mag = mag, + cubeSize = cubeSize, + fourBit = Some(false), + applyAgglomerate = tracing.mappingName, + version = None + ) + for { + organizationName <- tracing.organizationName ?~> "Zarr streaming not supported for legacy volume annotations (organizationName is not set)" + remoteFallbackLayer = RemoteFallbackLayer(organizationName, + tracing.dataSetName, + fallbackLayerName, + tracing.elementClass) + (fallbackData, fallbackMissingBucketIndices) <- remoteDataStoreClient.getData(remoteFallbackLayer, + List(request), + urlToken) + _ <- bool2Fox(fallbackMissingBucketIndices.isEmpty) ?~> "No data at coordinations in fallback layer" + } yield fallbackData + } + + if (missingBucketIndices.nonEmpty) { + for { + fallbackLayer <- tracing.fallbackLayer.toFox ?~> "No data at coordinates, no fallback layer defined" + data <- fallbackLayerData(fallbackLayer) ?~> "No data at coordinates, no fallback layer data at coordinates." + } yield data + } else Fox.successful(data) + } +} diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index 45afbe3469f..c30da05c517 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -34,17 +34,17 @@ GET /mapping/:tracingId/updateActionLog @com.scalablemin GET /mapping/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingInfo(token: Option[String], tracingId: String) # Zarr endpoints for volume annotations -GET /volume/zarr/json/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.volumeTracingFolderContentJson(token: Option[String], tracingId: String) -GET /volume/zarr/json/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.volumeTracingMagFolderContentJson(token: Option[String], tracingId: String, mag: String) -GET /volume/zarr/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.volumeTracingFolderContent(token: Option[String], tracingId: String) -GET /volume/zarr/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.volumeTracingFolderContent(token: Option[String], tracingId: String) -GET /volume/zarr/:tracingId/.zgroup @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.zGroup(token: Option[String], tracingId: String) -GET /volume/zarr/:tracingId/.zattrs @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.zAttrs(token: Option[String], tracingId: String) -GET /volume/zarr/:tracingId/zarrSource @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.zarrSource(token: Option[String], tracingId: String, tracingName: Option[String]) -GET /volume/zarr/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String) -GET /volume/zarr/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String) -GET /volume/zarr/:tracingId/:mag/.zarray @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.zArray(token: Option[String], tracingId: String, mag: String) -GET /volume/zarr/:tracingId/:mag/:cxyz @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.rawZarrCube(token: Option[String], tracingId: String, mag: String, cxyz: String) +GET /volume/zarr/json/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContentJson(token: Option[String], tracingId: String) +GET /volume/zarr/json/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContentJson(token: Option[String], tracingId: String, mag: String) +GET /volume/zarr/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], tracingId: String) +GET /volume/zarr/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], tracingId: String) +GET /volume/zarr/:tracingId/.zgroup @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zGroup(token: Option[String], tracingId: String) +GET /volume/zarr/:tracingId/.zattrs @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zAttrs(token: Option[String], tracingId: String) +GET /volume/zarr/:tracingId/zarrSource @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrSource(token: Option[String], tracingId: String, tracingName: Option[String]) +GET /volume/zarr/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String) +GET /volume/zarr/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String) +GET /volume/zarr/:tracingId/:mag/.zarray @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zArray(token: Option[String], tracingId: String, mag: String) +GET /volume/zarr/:tracingId/:mag/:cxyz @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(token: Option[String], tracingId: String, mag: String, cxyz: String) # Skeleton tracings POST /skeleton/save @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.save(token: Option[String])