Read Segment Index Files, Segment Stats for Annot. w/ FallbackL.

CHANGELOG.unreleased.md

@@ -18,6 +18,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Added a new "Split from all neighboring segments" feature for the proofreading mode. [#7611](https://github.com/scalableminds/webknossos/pull/7611)
 - If storage scan is enabled, the measured used storage is now displayed in the dashboard’s dataset detail view. [#7677](https://github.com/scalableminds/webknossos/pull/7677)
 - Prepared support to download full stl meshes via the HTTP api. [#7587](https://github.com/scalableminds/webknossos/pull/7587)
+- You can now place segment index files with your on-disk segmentation layers, which makes segment stats available when viewing these segmentations, and also when working on volume annotations based on these segmentation layers. [#7437](https://github.com/scalableminds/webknossos/pull/7437)
 
 ### Changed
 - Datasets stored in WKW format are no longer loaded with memory mapping, reducing memory demands. [#7528](https://github.com/scalableminds/webknossos/pull/7528)

frontend/javascripts/admin/admin_rest_api.ts

@@ -973,11 +973,9 @@ export function getSegmentVolumes(
   additionalCoordinates: AdditionalCoordinate[] | undefined | null,
   mappingName: string | null | undefined,
 ): Promise<number[]> {
-  let mappingNameNoEmptyString = mappingName;
-  if (mappingName != null && mappingName.length < 1) mappingNameNoEmptyString = null;
   return doWithToken((token) =>
     Request.sendJSONReceiveJSON(`${requestUrl}/segmentStatistics/volume?token=${token}`, {
-      data: { additionalCoordinates, mag, segmentIds, mappingName: mappingNameNoEmptyString },
+      data: { additionalCoordinates, mag, segmentIds, mappingName },
       method: "POST",
     }),
   );
@@ -990,11 +988,9 @@ export function getSegmentBoundingBoxes(
   additionalCoordinates: AdditionalCoordinate[] | undefined | null,
   mappingName: string | null | undefined,
 ): Promise<Array<{ topLeft: Vector3; width: number; height: number; depth: number }>> {
-  let mappingNameNoEmptyString = mappingName;
-  if (mappingName != null && mappingName.length < 1) mappingNameNoEmptyString = null;
   return doWithToken((token) =>
     Request.sendJSONReceiveJSON(`${requestUrl}/segmentStatistics/boundingBox?token=${token}`, {
-      data: { additionalCoordinates, mag, segmentIds, mappingName: mappingNameNoEmptyString },
+      data: { additionalCoordinates, mag, segmentIds, mappingName },
       method: "POST",
     }),
   );

frontend/javascripts/oxalis/model/accessors/dataset_accessor.ts

@@ -833,7 +833,7 @@ export function getEffectiveIntensityRange(
 // sure this info is fetched.
 export function getMaybeSegmentIndexAvailability(
   dataset: APIDataset,
-  layerName: string | undefined,
+  layerName: string | null | undefined,
 ) {
   if (layerName == null) {
     return false;

frontend/javascripts/oxalis/model/actions/dataset_actions.ts

@@ -6,14 +6,15 @@ export type EnsureLayerMappingsAreLoadedAction = ReturnType<
   typeof ensureLayerMappingsAreLoadedAction
 >;
 type SetLayerHasSegmentIndexAction = ReturnType<typeof setLayerHasSegmentIndexAction>;
+export type EnsureSegmentIndexIsLoadedAction = ReturnType<typeof ensureSegmentIndexIsLoadedAction>;
 
 export type DatasetAction =
   | SetDatasetAction
   | SetLayerMappingsAction
   | SetLayerTransformsAction
   | EnsureLayerMappingsAreLoadedAction
   | SetLayerHasSegmentIndexAction
-  | EnsureLayerMappingsAreLoadedAction;
+  | EnsureSegmentIndexIsLoadedAction;
 
 export const setDatasetAction = (dataset: APIDataset) =>
   ({
@@ -47,16 +48,17 @@ export const ensureLayerMappingsAreLoadedAction = (layerName?: string) =>
   ({
     type: "ENSURE_LAYER_MAPPINGS_ARE_LOADED",
     layerName,
-  } as const);
+  }) as const;
 
 export const setLayerHasSegmentIndexAction = (layerName: string, hasSegmentIndex: boolean) =>
   ({
     type: "SET_LAYER_HAS_SEGMENT_INDEX",
     layerName,
     hasSegmentIndex,
-  } as const);
+  }) as const;
 
-export const ensureSegmentIndexIsLoadedAction = () =>
+export const ensureSegmentIndexIsLoadedAction = (layerName: string | null | undefined) =>
   ({
     type: "ENSURE_SEGMENT_INDEX_IS_LOADED",
-  } as const);
+    layerName,
+  }) as const;

frontend/javascripts/oxalis/model/actions/save_actions.ts

@@ -74,7 +74,7 @@ export const setSaveBusyAction = (
     isBusy,
     saveQueueType,
     tracingId,
-  } as const);
+  }) as const;
 
 export const setLastSaveTimestampAction = (saveQueueType: SaveQueueType, tracingId: string) =>
   ({

frontend/javascripts/oxalis/model/sagas/dataset_saga.ts

@@ -7,10 +7,10 @@ import Toast from "libs/toast";
 import messages from "messages";
 import {
   getEnabledLayers,
+  getLayerByName,
   getMaybeSegmentIndexAvailability,
   getResolutionInfo,
   getTransformsForLayer,
-  getVisibleSegmentationLayer,
   invertAndTranspose,
   isLayerVisible,
 } from "../accessors/dataset_accessor";
@@ -19,7 +19,10 @@ import { getViewportExtents } from "../accessors/view_mode_accessor";
 import { V3 } from "libs/mjs";
 import { Identity4x4 } from "oxalis/constants";
 import { hasSegmentIndex } from "oxalis/view/right-border-tabs/segments_tab/segments_view_helper";
-import { setLayerHasSegmentIndexAction } from "../actions/dataset_actions";
+import {
+  EnsureSegmentIndexIsLoadedAction,
+  setLayerHasSegmentIndexAction,
+} from "../actions/dataset_actions";
 
 export function* watchMaximumRenderableLayers(): Saga<void> {
   function* warnMaybe(): Saga<void> {
@@ -154,22 +157,27 @@ export function* watchZ1Downsampling(): Saga<void> {
 }
 
 export function* ensureSegmentIndexIsLoaded(): Saga<void> {
-  function* maybeFetchHasSegmentIndex(): Saga<void> {
-    const visibleSegmentationLayer = yield* select((state) => getVisibleSegmentationLayer(state));
-    const maybeIsSegmentIndexAvailable = yield* select((state) =>
-      getMaybeSegmentIndexAvailability(state.dataset, visibleSegmentationLayer?.name),
+  function* maybeFetchHasSegmentIndex(action: EnsureSegmentIndexIsLoadedAction): Saga<void> {
+    const { layerName } = action;
+    const dataset = yield* select((state) => state.dataset);
+    if (layerName == null) return;
+    const concernedSegmentationLayer = yield* call(getLayerByName, dataset, layerName);
+    const maybeIsSegmentIndexAvailable = yield* call(
+      getMaybeSegmentIndexAvailability,
+      dataset,
+      layerName,
     );
-    if (maybeIsSegmentIndexAvailable == null && visibleSegmentationLayer != null) {
-      const { dataset, tracing } = yield* select((state) => state);
+    if (maybeIsSegmentIndexAvailable == null && concernedSegmentationLayer != null) {
+      const tracing = yield* select((state) => state.tracing);
       const updatedIsSegmentIndexAvailable = yield* call(
         hasSegmentIndex,
-        visibleSegmentationLayer,
+        concernedSegmentationLayer,
         dataset,
         tracing,
       );
       yield* put(
         setLayerHasSegmentIndexAction(
-          visibleSegmentationLayer.name,
+          concernedSegmentationLayer.name,
           updatedIsSegmentIndexAvailable,
         ),
       );

frontend/javascripts/oxalis/view/context_menu.tsx

@@ -1218,7 +1218,7 @@ function ContextMenuInner(propsWithInputRef: Props) {
 
   const { dataset, tracing, flycam } = useSelector((state: OxalisState) => state);
   useEffect(() => {
-    Store.dispatch(ensureSegmentIndexIsLoadedAction());
+    Store.dispatch(ensureSegmentIndexIsLoadedAction(visibleSegmentationLayer?.name));
   }, [visibleSegmentationLayer]);
   const isSegmentIndexAvailable = useSelector((state: OxalisState) =>
     getMaybeSegmentIndexAvailability(state.dataset, visibleSegmentationLayer?.name),

frontend/javascripts/oxalis/view/right-border-tabs/segments_tab/segment_statistics_modal.tsx

@@ -134,7 +134,7 @@ export function SegmentStatisticsModal({
       const maybeVolumeTracing =
         tracingId != null ? getVolumeTracingById(tracing, tracingId) : null;
       const maybeGetMappingName = () => {
-        if (maybeVolumeTracing?.mappingName != null) return maybeVolumeTracing?.mappingName;
+        if (maybeVolumeTracing?.mappingName != null) return maybeVolumeTracing.mappingName;
         const mappingInfo = getMappingInfo(
           temporaryConfiguration.activeMappingByLayer,
           visibleSegmentationLayer?.name,

frontend/javascripts/oxalis/view/right-border-tabs/segments_tab/segments_view.tsx

@@ -421,7 +421,7 @@ class SegmentsView extends React.Component<Props, State> {
       this.pollJobData();
     }
 
-    Store.dispatch(ensureSegmentIndexIsLoadedAction());
+    Store.dispatch(ensureSegmentIndexIsLoadedAction(this.props.visibleSegmentationLayer?.name));
   }
 
   componentDidUpdate(prevProps: Props) {

util/src/main/scala/com/scalableminds/util/io/ZipIO.scala

@@ -1,5 +1,7 @@
 package com.scalableminds.util.io
 
+import com.scalableminds.util.tools.Fox.{box2Fox, future2Fox}
+
 import java.io._
 import java.nio.file.{Files, Path, Paths}
 import java.util.zip.{GZIPOutputStream => DefaultGZIPOutputStream, _}
@@ -164,6 +166,63 @@ object ZipIO extends LazyLogging {
   def withUnziped[A](file: File)(f: (Path, InputStream) => A): Box[List[A]] =
     tryo(new java.util.zip.ZipFile(file)).flatMap(withUnziped(_)((name, is) => Full(f(name, is))))
 
+  def withUnzipedAsync[A](file: File)(f: (Path, InputStream) => Fox[A])(implicit ec: ExecutionContext): Fox[List[A]] =
+    for {
+      zip <- tryo(new java.util.zip.ZipFile(file)).toFox
+      resultList <- withUnzipedAsync(zip)((name, is) => f(name, is))
+    } yield resultList
+
+  def withUnzipedAsync[A](zip: ZipFile,
+                          includeHiddenFiles: Boolean = false,
+                          hiddenFilesWhitelist: List[String] = List(),
+                          truncateCommonPrefix: Boolean = false,
+                          excludeFromPrefix: Option[List[String]] = None)(f: (Path, InputStream) => Fox[A])(
+      implicit ec: ExecutionContext): Fox[List[A]] = {
+
+    val zipEntries = zip.entries.asScala.filter { e: ZipEntry =>
+      !e.isDirectory && (includeHiddenFiles || !isFileHidden(e) || hiddenFilesWhitelist.contains(
+        Paths.get(e.getName).getFileName.toString))
+    }.toList
+
+    val commonPrefix = if (truncateCommonPrefix) {
+      val commonPrefixNotFixed = PathUtils.commonPrefix(zipEntries.map(e => Paths.get(e.getName)))
+      val strippedPrefix =
+        PathUtils.cutOffPathAtLastOccurrenceOf(commonPrefixNotFixed, excludeFromPrefix.getOrElse(List.empty))
+      PathUtils.removeSingleFileNameFromPrefix(strippedPrefix, zipEntries.map(_.getName))
+    } else {
+      Paths.get("")
+    }
+
+    val resultFox = zipEntries.foldLeft[Fox[List[A]]](Fox.successful(List.empty)) { (results, entry) =>
+      results.futureBox.map {
+        case Full(rs) =>
+          val input: InputStream = zip.getInputStream(entry)
+          val path = commonPrefix.relativize(Paths.get(entry.getName))
+          val innerResultFox: Fox[List[A]] = f(path, input).futureBox.map {
+            case Full(result) =>
+              input.close()
+              Full(rs :+ result)
+            case Empty =>
+              input.close()
+              Empty
+            case failure: Failure =>
+              input.close()
+              failure
+          }
+          innerResultFox
+        case e =>
+          e.toFox
+      }.toFox.flatten
+    }
+
+    for {
+      result <- resultFox.futureBox.map { resultBox =>
+        zip.close() // close even if result is not success
+        resultBox
+      }
+    } yield result
+  }
+
   def withUnziped[A](zip: ZipFile,
                      includeHiddenFiles: Boolean = false,
                      hiddenFilesWhitelist: List[String] = List(),

webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala

@@ -1,6 +1,7 @@
 package com.scalableminds.webknossos.datastore.controllers
 
 import com.google.inject.Inject
+import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.scalableminds.webknossos.datastore.ListOfLong.ListOfLong
 import com.scalableminds.webknossos.datastore.helpers.{
@@ -42,15 +43,18 @@ class DataSourceController @Inject()(
     dataSourceService: DataSourceService,
     remoteWebknossosClient: DSRemoteWebknossosClient,
     accessTokenService: DataStoreAccessTokenService,
-    binaryDataServiceHolder: BinaryDataServiceHolder,
+    val binaryDataServiceHolder: BinaryDataServiceHolder,
     connectomeFileService: ConnectomeFileService,
     segmentIndexFileService: SegmentIndexFileService,
     storageUsageService: DSUsedStorageService,
     datasetErrorLoggingService: DatasetErrorLoggingService,
     uploadService: UploadService,
-    composeService: ComposeService
+    composeService: ComposeService,
+    val dsRemoteWebknossosClient: DSRemoteWebknossosClient,
+    val dsRemoteTracingstoreClient: DSRemoteTracingstoreClient,
 )(implicit bodyParsers: PlayBodyParsers, ec: ExecutionContext)
     extends Controller
+    with MeshMappingHelper
     with FoxImplicits {
 
   override def allowRemoteOrigin: Boolean = true
@@ -576,11 +580,6 @@ class DataSourceController @Inject()(
 
   /**
     * Query the segment index file for a single segment
-    * @param token
-    * @param organizationName
-    * @param datasetName
-    * @param dataLayerName
-    * @param segmentId
     * @return List of bucketPositions as positions (not indices) of 32³ buckets in mag
     */
   def getSegmentIndex(token: Option[String],
@@ -592,11 +591,23 @@ class DataSourceController @Inject()(
       accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)),
                                         urlOrHeaderToken(token, request)) {
         for {
-          (topLefts, fileMag) <- segmentIndexFileService.readSegmentIndex(organizationName,
-                                                                          datasetName,
-                                                                          dataLayerName,
-                                                                          segmentId.toLong)
-          bucketPositions = segmentIndexFileService.topLeftsToBucketPositions(topLefts, request.body.mag, fileMag)
+          segmentIds <- segmentIdsForAgglomerateIdIfNeeded(
+            organizationName,
+            datasetName,
+            dataLayerName,
+            request.body.mappingName,
+            request.body.editableMappingTracingId,
+            segmentId.toLong,
+            mappingNameForMeshFile = None,
+            urlOrHeaderToken(token, request)
+          )
+          fileMag <- segmentIndexFileService.readFileMag(organizationName, datasetName, dataLayerName)
+          topLeftsNested: Seq[Array[Vec3Int]] <- Fox.serialCombined(segmentIds)(sId =>
+            segmentIndexFileService.readSegmentIndex(organizationName, datasetName, dataLayerName, sId))
+          topLefts: Array[Vec3Int] = topLeftsNested.toArray.flatten
+          bucketPositions = segmentIndexFileService.topLeftsToDistinctBucketPositions(topLefts,
+                                                                                      request.body.mag,
+                                                                                      fileMag)
           bucketPositionsForCubeSize = bucketPositions
             .map(_.scale(DataLayer.bucketLength)) // bucket positions raw are indices of 32³ buckets
             .map(_ / request.body.cubeSize)
@@ -608,10 +619,6 @@ class DataSourceController @Inject()(
 
   /**
     * Query the segment index file for multiple segments
-    * @param token
-    * @param organizationName
-    * @param datasetName
-    * @param dataLayerName
     * @return List of bucketPositions as indices of 32³ buckets
     */
   def querySegmentIndex(token: Option[String],
@@ -622,14 +629,26 @@ class DataSourceController @Inject()(
       accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)),
                                         urlOrHeaderToken(token, request)) {
         for {
-          segmentIdsAndBucketPositions <- Fox.serialCombined(request.body.segmentIds) { segmentId =>
+          segmentIdsAndBucketPositions <- Fox.serialCombined(request.body.segmentIds) { segmentOrAgglomerateId =>
             for {
-              (topLefts, fileMag) <- segmentIndexFileService.readSegmentIndex(organizationName,
-                                                                              datasetName,
-                                                                              dataLayerName,
-                                                                              segmentId)
-              bucketPositions = segmentIndexFileService.topLeftsToBucketPositions(topLefts, request.body.mag, fileMag)
-            } yield SegmentIndexData(segmentId, bucketPositions.toSeq)
+              segmentIds <- segmentIdsForAgglomerateIdIfNeeded(
+                organizationName,
+                datasetName,
+                dataLayerName,
+                request.body.mappingName,
+                request.body.editableMappingTracingId,
+                segmentOrAgglomerateId,
+                mappingNameForMeshFile = None,
+                urlOrHeaderToken(token, request)
+              )
+              fileMag <- segmentIndexFileService.readFileMag(organizationName, datasetName, dataLayerName)
+              topLeftsNested: Seq[Array[Vec3Int]] <- Fox.serialCombined(segmentIds)(sId =>
+                segmentIndexFileService.readSegmentIndex(organizationName, datasetName, dataLayerName, sId))
+              topLefts: Array[Vec3Int] = topLeftsNested.toArray.flatten
+              bucketPositions = segmentIndexFileService.topLeftsToDistinctBucketPositions(topLefts,
+                                                                                          request.body.mag,
+                                                                                          fileMag)
+            } yield SegmentIndexData(segmentOrAgglomerateId, bucketPositions.toSeq)
           }
         } yield Ok(Json.toJson(segmentIdsAndBucketPositions))
       }