Rescale layers for different voxel sizes on remote import

CHANGELOG.unreleased.md

@@ -14,6 +14,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Added configuration to require users' emails to be verified, added flow to verify emails via link. [#7161](https://github.com/scalableminds/webknossos/pull/7161)
 
 ### Changed
+- When importing a remote dataset and adding another layer with a different voxel size, that layer is now scaled to match the first layer. [#7213](https://github.com/scalableminds/webknossos/pull/7213)
 
 ### Fixed
 

app/models/binary/explore/ExploreRemoteLayerService.scala

@@ -32,7 +32,8 @@ import scala.util.Try
 
 case class ExploreRemoteDatasetParameters(remoteUri: String,
                                           credentialIdentifier: Option[String],
-                                          credentialSecret: Option[String])
+                                          credentialSecret: Option[String],
+                                          preferredVoxelSize: Option[Array[Double]])
 
 object ExploreRemoteDatasetParameters {
   implicit val jsonFormat: OFormat[ExploreRemoteDatasetParameters] = Json.format[ExploreRemoteDatasetParameters]
@@ -43,11 +44,11 @@ class ExploreRemoteLayerService @Inject()(credentialService: CredentialService,
     with LazyLogging {
 
   def exploreRemoteDatasource(
-      urisWithCredentials: List[ExploreRemoteDatasetParameters],
+      parameters: List[ExploreRemoteDatasetParameters],
       requestIdentity: WkEnv#I,
       reportMutable: ListBuffer[String])(implicit ec: ExecutionContext): Fox[GenericDataSource[DataLayer]] =
     for {
-      exploredLayersNested <- Fox.serialCombined(urisWithCredentials)(
+      exploredLayersNested <- Fox.serialCombined(parameters)(
         parameters =>
           exploreRemoteLayersForUri(parameters.remoteUri,
                                     parameters.credentialIdentifier,
@@ -60,9 +61,13 @@ class ExploreRemoteLayerService @Inject()(credentialService: CredentialService,
       rescaledLayers = rescaledLayersAndVoxelSize._1
       voxelSize = rescaledLayersAndVoxelSize._2
       renamedLayers = makeLayerNamesUnique(rescaledLayers)
+      preferredVoxelSize = parameters.flatMap(_.preferredVoxelSize).headOption
+      layersWithCoordinateTransformations = addCoordinateTransformationsToLayers(renamedLayers,
+                                                                                 preferredVoxelSize,
+                                                                                 voxelSize)
       dataSource = GenericDataSource[DataLayer](
         DataSourceId("", ""), // Frontend will prompt user for a good name
-        renamedLayers,
+        layersWithCoordinateTransformations,
         voxelSize
       )
     } yield dataSource
@@ -90,6 +95,24 @@ class ExploreRemoteLayerService @Inject()(credentialService: CredentialService,
     }
   }
 
+  private def addCoordinateTransformationsToLayers(layers: List[DataLayer],
+                                                   preferredVoxelSize: Option[Array[Double]],
+                                                   voxelSize: Vec3Double): List[DataLayer] =
+    layers.map(l => {
+      val coordinateTransformations = coordinateTransformationForVoxelSize(voxelSize, preferredVoxelSize)
+      l match {
+        case l: ZarrDataLayer                => l.copy(coordinateTransformations = coordinateTransformations)
+        case l: ZarrSegmentationLayer        => l.copy(coordinateTransformations = coordinateTransformations)
+        case l: N5DataLayer                  => l.copy(coordinateTransformations = coordinateTransformations)
+        case l: N5SegmentationLayer          => l.copy(coordinateTransformations = coordinateTransformations)
+        case l: PrecomputedDataLayer         => l.copy(coordinateTransformations = coordinateTransformations)
+        case l: PrecomputedSegmentationLayer => l.copy(coordinateTransformations = coordinateTransformations)
+        case l: Zarr3DataLayer               => l.copy(coordinateTransformations = coordinateTransformations)
+        case l: Zarr3SegmentationLayer       => l.copy(coordinateTransformations = coordinateTransformations)
+        case _                               => throw new Exception("Encountered unsupported layer format during explore remote")
+      }
+    })
+
   private def magFromVoxelSize(minVoxelSize: Vec3Double, voxelSize: Vec3Double)(
       implicit ec: ExecutionContext): Fox[Vec3Int] = {
     def isPowerOfTwo(x: Int): Boolean =
@@ -106,6 +129,32 @@ class ExploreRemoteLayerService @Inject()(credentialService: CredentialService,
       _ <- bool2Fox(magGroup.length == 1) ?~> s"detected mags are not unique, found $magGroup"
     } yield ()
 
+  private def coordinateTransformationForVoxelSize(
+      foundVoxelSize: Vec3Double,
+      preferredVoxelSize: Option[Array[Double]]): Option[List[CoordinateTransformation]] =
+    preferredVoxelSize match {
+      case None => None
+      case Some(asArray) =>
+        Vec3Double.fromArray(asArray) match {
+          case None => None
+          case Some(voxelSize) =>
+            val scaleX = foundVoxelSize.x / voxelSize.x
+            val scaleY = foundVoxelSize.y / voxelSize.y
+            val scaleZ = foundVoxelSize.z / voxelSize.z
+            Some(
+              List(
+                CoordinateTransformation(CoordinateTransformationType.affine,
+                                         matrix = Some(
+                                           List(
+                                             List(scaleX, 0, 0, 0),
+                                             List(0, scaleY, 0, 0),
+                                             List(0, 0, scaleZ, 0),
+                                             List(0, 0, 0, 1)
+                                           )))))
+        }
+
+    }
+
   private def rescaleLayersByCommonVoxelSize(layersWithVoxelSizes: List[(DataLayer, Vec3Double)])(
       implicit ec: ExecutionContext): Fox[(List[DataLayer], Vec3Double)] = {
     val allVoxelSizes = layersWithVoxelSizes

frontend/javascripts/admin/admin_rest_api.ts

@@ -1522,16 +1522,26 @@ type ExplorationResult = {
 
 export async function exploreRemoteDataset(
   remoteUris: string[],
-  credentials?: { username: string; pass: string },
+  credentials?: { username: string; pass: string } | null,
+  preferredVoxelSize?: Vector3,
 ): Promise<ExplorationResult> {
   const { dataSource, report } = await Request.sendJSONReceiveJSON("/api/datasets/exploreRemote", {
-    data: credentials
-      ? remoteUris.map((uri) => ({
-          remoteUri: uri.trim(),
+    data: remoteUris.map((uri) => {
+      const extendedUri = {
+        remoteUri: uri.trim(),
+        preferredVoxelSize,
+      };
+
+      if (credentials) {
+        return {
+          ...extendedUri,
           credentialIdentifier: credentials.username,
           credentialSecret: credentials.pass,
-        }))
-      : remoteUris.map((uri) => ({ remoteUri: uri.trim() })),
+        };
+      }
+
+      return extendedUri;
+    }),
   });
   if (report.indexOf("403 Forbidden") !== -1 || report.indexOf("401 Unauthorized") !== -1) {
     Toast.error("The data could not be accessed. Please verify the credentials!");
@@ -2130,7 +2140,7 @@ export function computeIsosurface(
         },
       },
     );
-    const neighbors = Utils.parseAsMaybe(headers.neighbors).getOrElse([]);
+    const neighbors = Utils.parseMaybe(headers.neighbors) || [];
     return {
       buffer,
       neighbors,

frontend/javascripts/admin/dataset/dataset_add_remote_view.tsx

@@ -36,6 +36,7 @@ import Upload, { RcFile, UploadChangeParam, UploadFile } from "antd/lib/upload";
 import { UnlockOutlined } from "@ant-design/icons";
 import { Unicode } from "oxalis/constants";
 import { readFileAsText } from "libs/read_file";
+import * as Utils from "libs/utils";
 
 const { Panel } = Collapse;
 const FormItem = Form.Item;
@@ -403,26 +404,37 @@ function AddZarrLayer({
     const datasourceConfigStr = form.getFieldValue("dataSourceJson");
 
     const { dataSource: newDataSource, report } = await (async () => {
+      // @ts-ignore
+      const preferredVoxelSize = Utils.parseMaybe(datasourceConfigStr)?.scale;
+
       if (showCredentialsFields) {
         if (selectedProtocol === "gs") {
           const credentials =
             fileList.length > 0 ? await parseCredentials(fileList[0]?.originFileObj) : null;
           if (credentials) {
-            return exploreRemoteDataset([datasourceUrl], {
-              username: "",
-              pass: JSON.stringify(credentials),
-            });
+            return exploreRemoteDataset(
+              [datasourceUrl],
+              {
+                username: "",
+                pass: JSON.stringify(credentials),
+              },
+              preferredVoxelSize,
+            );
           } else {
             // Fall through to exploreRemoteDataset without parameters
           }
         } else if (usernameOrAccessKey && passwordOrSecretKey) {
-          return exploreRemoteDataset([datasourceUrl], {
-            username: usernameOrAccessKey,
-            pass: passwordOrSecretKey,
-          });
+          return exploreRemoteDataset(
+            [datasourceUrl],
+            {
+              username: usernameOrAccessKey,
+              pass: passwordOrSecretKey,
+            },
+            preferredVoxelSize,
+          );
         }
       }
-      return exploreRemoteDataset([datasourceUrl]);
+      return exploreRemoteDataset([datasourceUrl], null, preferredVoxelSize);
     })();
     setExploreLog(report);
     if (!newDataSource) {

frontend/javascripts/dashboard/dataset/dataset_settings_data_tab.tsx

@@ -25,7 +25,7 @@ import {
   jsonEditStyle,
 } from "dashboard/dataset/helper_components";
 import { startFindLargestSegmentIdJob } from "admin/admin_rest_api";
-import { jsonStringify, parseAsMaybe } from "libs/utils";
+import { jsonStringify, parseMaybe } from "libs/utils";
 import { DataLayer } from "types/schemas/datasource.types";
 import { getDatasetNameRules, layerNameRules } from "admin/dataset/dataset_components";
 import { useSelector } from "react-redux";
@@ -52,9 +52,7 @@ export const syncDataSourceFields = (
       dataSourceJson: jsonStringify(dataSourceFromSimpleTab),
     });
   } else {
-    const dataSourceFromAdvancedTab = parseAsMaybe(form.getFieldValue("dataSourceJson")).getOrElse(
-      null,
-    );
+    const dataSourceFromAdvancedTab = parseMaybe(form.getFieldValue("dataSourceJson"));
     // Copy from advanced to simple: update form values
     form.setFieldsValue({
       dataSource: dataSourceFromAdvancedTab,

frontend/javascripts/libs/utils.ts

@@ -132,17 +132,17 @@ export function maybe<A, B>(fn: (arg0: A) => B): (arg0: A | null | undefined) =>
   return (nullableA: A | null | undefined) => Maybe.fromNullable(nullableA).map(fn);
 }
 
-export function parseAsMaybe(str: string | null | undefined): Maybe<any> {
+export function parseMaybe(str: string | null | undefined): unknown | null {
   try {
     const parsedJSON = JSON.parse(str || "");
 
     if (parsedJSON != null) {
-      return Maybe.Just(parsedJSON);
+      return parsedJSON;
     } else {
-      return Maybe.Nothing();
+      return null;
     }
   } catch (_exception) {
-    return Maybe.Nothing();
+    return null;
   }
 }
 

frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.ts

@@ -9,7 +9,7 @@ import {
   getResolutionInfo,
 } from "oxalis/model/accessors/dataset_accessor";
 import { getVolumeTracingById } from "oxalis/model/accessors/volumetracing_accessor";
-import { parseAsMaybe } from "libs/utils";
+import { parseMaybe } from "libs/utils";
 import { pushSaveQueueTransaction } from "oxalis/model/actions/save_actions";
 import type { UpdateAction } from "oxalis/model/sagas/update_actions";
 import { updateBucket } from "oxalis/model/sagas/update_actions";
@@ -187,7 +187,7 @@ export async function requestFromStore(
           showErrorToast: false,
         });
       const endTime = window.performance.now();
-      const missingBuckets = parseAsMaybe(headers["missing-buckets"]).getOrElse([]);
+      const missingBuckets = (parseMaybe(headers["missing-buckets"]) || []) as number[];
       const receivedBucketsCount = batch.length - missingBuckets.length;
       const BUCKET_BYTE_LENGTH = constants.BUCKET_SIZE * getByteCountFromLayer(layerInfo);
       getGlobalDataConnectionInfo().log(

webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/CoordinateTransformation.scala

@@ -15,7 +15,7 @@ object ThinPlateSplineCorrespondences {
 
 case class CoordinateTransformation(`type`: CoordinateTransformationType,
                                     matrix: Option[List[List[Double]]],
-                                    correspondences: Option[ThinPlateSplineCorrespondences])
+                                    correspondences: Option[ThinPlateSplineCorrespondences] = None)
 
 object CoordinateTransformation {
   implicit val jsonFormat: OFormat[CoordinateTransformation] = Json.format[CoordinateTransformation]