Return 404 for not-found zarr chunks (#6381)

* fixes #6179 * add NOT_FOUND constants * Merge branch 'master' into zarr-chunk-oob * changelog
scalableminds · Aug 8, 2022 · 5e6097b · 5e6097b
1 parent 509570d
commit 5e6097b
Show file tree

Hide file tree

Showing 6 changed files with 48 additions and 42 deletions.
diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md
@@ -26,6 +26,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - The Zarr directory listings no longer include the current directory “.”. [6359](https://github.com/scalableminds/webknossos/pull/6359)
 - The default name for volume annotation layers with fallback segmentations is now set to the name of their fallback segmentation layer, no longer “Volume”. [#6373](https://github.com/scalableminds/webknossos/pull/6373)
 - The “reload data” button for dataset and annotation layers is now only shown to users who have administrative permissions for the dataset (because the button clears server caches and file handles). [#6380](https://github.com/scalableminds/webknossos/pull/6380)
+- Requests for missing chunks in zarr datasets now return status code 404 instead of 200 with an empty response. [#6381](https://github.com/scalableminds/webknossos/pull/6381)
 
 ### Fixed
 - Fixed a regression where the mapping activation confirmation dialog was never shown. [#6346](https://github.com/scalableminds/webknossos/pull/6346)

diff --git a/conf/messages b/conf/messages
@@ -161,6 +161,7 @@ dataLayer.invalidMag=Supplied “{0}” is not a valid mag format. Please use
 
 zarr.invalidChunkCoordinates=The requested chunk coordinates are in an invalid format. Expected c.x.y.z
 zarr.invalidFirstChunkCoord="First Channel must be 0"
+zarr.chunkNotFound=Could not find the requested chunk
 
 scale.scale=Scale
 scale.invalid=Specify dataset scale like "XX, YY, ZZ"

diff --git a/...tastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala b/...tastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala
@@ -69,7 +69,7 @@ class BinaryDataController @Inject()(
         for {
           (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName,
                                                                                     dataSetName,
-                                                                                    dataLayerName) ~> 404
+                                                                                    dataLayerName) ~> NOT_FOUND
           (data, indices) <- requestData(dataSource, dataLayer, request.body)
           duration = System.currentTimeMillis() - t
           _ = if (duration > 10000)
@@ -114,7 +114,7 @@ class BinaryDataController @Inject()(
       for {
         (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName,
                                                                                   dataSetName,
-                                                                                  dataLayerName) ~> 404
+                                                                                  dataLayerName) ~> NOT_FOUND
         _ <- bool2Fox(!(resolution.isDefined && mag.isDefined)) ?~> "Can only interpret mag or zoomStep. Use only mag instead."
         magFromZoomStep = resolution.map(dataLayer.magFromExponent(_, snapToClosest = true))
         magParsedOpt <- Fox.runOptional(mag)(Vec3Int.fromMagLiteral(_).toFox)
@@ -149,7 +149,7 @@ class BinaryDataController @Inject()(
       for {
         (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName,
                                                                                   dataSetName,
-                                                                                  dataLayerName) ~> 404
+                                                                                  dataLayerName) ~> NOT_FOUND
         request = DataRequest(
           VoxelPosition(x * cubeSize * resolution,
                         y * cubeSize * resolution,
@@ -181,7 +181,7 @@ class BinaryDataController @Inject()(
         (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName,
                                                                                   dataSetName,
                                                                                   dataLayerName) ?~> Messages(
-          "dataSource.notFound") ~> 404
+          "dataSource.notFound") ~> NOT_FOUND
         position = ImageThumbnail.goodThumbnailParameters(dataLayer, width, height, centerX, centerY, centerZ, zoom)
         request = DataRequest(position, width, height, 1)
         (data, _) <- requestData(dataSource, dataLayer, request)
@@ -217,7 +217,7 @@ class BinaryDataController @Inject()(
       for {
         (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName,
                                                                                   dataSetName,
-                                                                                  dataLayerName) ~> 404
+                                                                                  dataLayerName) ~> NOT_FOUND
         segmentationLayer <- tryo(dataLayer.asInstanceOf[SegmentationLayer]).toFox ?~> Messages("dataLayer.notFound")
         mappingRequest = DataServiceMappingRequest(dataSource, segmentationLayer, mappingName)
         result <- mappingService.handleMappingRequest(mappingRequest)
@@ -239,7 +239,7 @@ class BinaryDataController @Inject()(
         for {
           (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName,
                                                                                     dataSetName,
-                                                                                    dataLayerName) ~> 404
+                                                                                    dataLayerName) ~> NOT_FOUND
           segmentationLayer <- tryo(dataLayer.asInstanceOf[SegmentationLayer]).toFox ?~> "dataLayer.mustBeSegmentation"
           isosurfaceRequest = IsosurfaceRequest(
             Some(dataSource),
@@ -281,7 +281,7 @@ class BinaryDataController @Inject()(
         for {
           (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName,
                                                                                     dataSetName,
-                                                                                    dataLayerName) ~> 404
+                                                                                    dataLayerName) ~> NOT_FOUND
           meanAndStdDev <- findDataService.meanAndStdDev(dataSource, dataLayer)
         } yield
           Ok(
@@ -301,7 +301,7 @@ class BinaryDataController @Inject()(
         for {
           (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName,
                                                                                     dataSetName,
-                                                                                    dataLayerName) ~> 404
+                                                                                    dataLayerName) ~> NOT_FOUND
           positionAndResolutionOpt <- findDataService.findPositionWithData(dataSource, dataLayer)
         } yield
           Ok(
@@ -322,7 +322,7 @@ class BinaryDataController @Inject()(
           (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName,
                                                                                     dataSetName,
                                                                                     dataLayerName) ?~> Messages(
-            "dataSource.notFound") ~> 404 ?~> Messages("histogram.layerMissing", dataLayerName)
+            "dataSource.notFound") ~> NOT_FOUND ?~> Messages("histogram.layerMissing", dataLayerName)
           listOfHistograms <- findDataService.createHistogram(dataSource, dataLayer) ?~> Messages("histogram.failed",
                                                                                                   dataLayerName)
         } yield Ok(Json.toJson(listOfHistograms))

diff --git a/...tastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala b/...tastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
@@ -254,7 +254,7 @@ Expects:
         urlOrHeaderToken(token, request)) {
         for {
           previousDataSource <- dataSourceRepository.find(DataSourceId(dataSetName, organizationName)) ?~ Messages(
-            "dataSource.notFound") ~> 404
+            "dataSource.notFound") ~> NOT_FOUND
           (dataSource, messages) <- dataSourceService.exploreDataSource(previousDataSource.id,
                                                                         previousDataSource.toUsable)
           previousDataSourceJson = previousDataSource match {
@@ -458,7 +458,7 @@ Expects:
         for {
           _ <- Fox.successful(())
           dataSource <- dataSourceRepository.find(DataSourceId(dataSetName, organizationName)).toFox ?~> Messages(
-            "dataSource.notFound") ~> 404
+            "dataSource.notFound") ~> NOT_FOUND
           _ <- dataSourceService.updateDataSource(request.body.copy(id = dataSource.id))
         } yield {
           Ok

diff --git a/...tore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala b/...tore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala
@@ -43,7 +43,7 @@ class ZarrStreamingController @Inject()(
                                         urlOrHeaderToken(token, request)) {
         for {
           dataSource <- dataSourceRepository.findUsable(DataSourceId(dataSetName, organizationName)).toFox ?~> Messages(
-            "dataSource.notFound") ~> 404
+            "dataSource.notFound") ~> NOT_FOUND
           layerNames = dataSource.dataLayers.map((dataLayer: DataLayer) => dataLayer.name)
         } yield
           Ok(
@@ -64,7 +64,7 @@ class ZarrStreamingController @Inject()(
                                         urlOrHeaderToken(token, request)) {
         for {
           (_, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, dataSetName, dataLayerName) ?~> Messages(
-            "dataSource.notFound") ~> 404
+            "dataSource.notFound") ~> NOT_FOUND
           mags = dataLayer.resolutions
         } yield
           Ok(
@@ -85,9 +85,9 @@ class ZarrStreamingController @Inject()(
       accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(dataSetName, organizationName)),
                                         urlOrHeaderToken(token, request)) {
         for {
-          (_, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, dataSetName, dataLayerName) ~> 404
-          magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> 404
-          _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> 404
+          (_, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, dataSetName, dataLayerName) ~> NOT_FOUND
+          magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND
+          _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> NOT_FOUND
         } yield
           Ok(
             views.html.datastoreZarrDatasourceDir(
@@ -107,9 +107,10 @@ class ZarrStreamingController @Inject()(
                                       urlOrHeaderToken(token, request)) {
       for {
         (_, dataLayer) <- dataSourceRepository
-          .getDataSourceAndDataLayer(organizationName, dataSetName, dataLayerName) ?~> Messages("dataSource.notFound") ~> 404
-        magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> 404
-        _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> 404
+          .getDataSourceAndDataLayer(organizationName, dataSetName, dataLayerName) ?~> Messages("dataSource.notFound") ~> NOT_FOUND
+        magParsed <- Vec3Int
+          .fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND
+        _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> NOT_FOUND
         cubeLength = DataLayer.bucketLength
         (channels, dtype) = ElementClass.toChannelAndZarrString(dataLayer.elementClass)
         // data request method always decompresses before sending
@@ -165,7 +166,7 @@ class ZarrStreamingController @Inject()(
         (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName,
                                                                                   dataSetName,
                                                                                   dataLayerName) ?~> Messages(
-          "dataSource.notFound") ~> 404
+          "dataSource.notFound") ~> NOT_FOUND
         existingMags = dataLayer.resolutions
 
         omeNgffHeader = OmeNgffHeader.fromDataLayerName(dataLayerName,
@@ -191,11 +192,12 @@ class ZarrStreamingController @Inject()(
       for {
         (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName,
                                                                                   dataSetName,
-                                                                                  dataLayerName) ~> 404
-        (c, x, y, z) <- parseDotCoordinates(cxyz) ?~> "zarr.invalidChunkCoordinates" ~> 404
-        magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> 404
-        _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> 404
-        _ <- bool2Fox(c == 0) ~> "zarr.invalidFirstChunkCoord" ~> 404
+                                                                                  dataLayerName) ~> NOT_FOUND
+        (c, x, y, z) <- parseDotCoordinates(cxyz) ?~> "zarr.invalidChunkCoordinates" ~> NOT_FOUND
+        magParsed <- Vec3Int
+          .fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND
+        _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> NOT_FOUND
+        _ <- bool2Fox(c == 0) ~> "zarr.invalidFirstChunkCoord" ~> NOT_FOUND
         cubeSize = DataLayer.bucketLength
         request = DataServiceDataRequest(
           dataSource,
@@ -212,7 +214,8 @@ class ZarrStreamingController @Inject()(
           ),
           DataServiceRequestSettings(halfByte = false)
         )
-        (data, _) <- binaryDataService.handleDataRequests(List(request))
+        (data, notFoundIndices) <- binaryDataService.handleDataRequests(List(request))
+        _ <- bool2Fox(notFoundIndices.isEmpty) ~> "zarr.chunkNotFound" ~> NOT_FOUND
       } yield Ok(data)
     }
   }
@@ -228,7 +231,7 @@ class ZarrStreamingController @Inject()(
     accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(dataSetName, organizationName)),
                                       urlOrHeaderToken(token, request)) {
       for {
-        dataSource <- dataSourceRepository.findUsable(DataSourceId(dataSetName, organizationName)).toFox ~> 404
+        dataSource <- dataSourceRepository.findUsable(DataSourceId(dataSetName, organizationName)).toFox ~> NOT_FOUND
         dataLayers = dataSource.dataLayers
         zarrLayers = dataLayers.collect({
           case d: WKWDataLayer =>