diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 1f0225d766c..a4be17dee47 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -11,6 +11,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released [Commits](https://github.com/scalableminds/webknossos/compare/24.12.0...HEAD) ### Added +- Added the possibility to configure a rotation for a dataset which can be toggled off and on when viewing and annotating data. [#8159](https://github.com/scalableminds/webknossos/pull/8159) - Added the total volume of a dataset to a tooltip in the dataset info tab. [#8229](https://github.com/scalableminds/webknossos/pull/8229) - Optimized performance of data loading with “fill value“ chunks. [#8271](https://github.com/scalableminds/webknossos/pull/8271) diff --git a/frontend/javascripts/admin/dataset/composition_wizard/04_configure_new_dataset.tsx b/frontend/javascripts/admin/dataset/composition_wizard/04_configure_new_dataset.tsx index 0263630a89d..4cbf5c7cc80 100644 --- a/frontend/javascripts/admin/dataset/composition_wizard/04_configure_new_dataset.tsx +++ b/frontend/javascripts/admin/dataset/composition_wizard/04_configure_new_dataset.tsx @@ -24,7 +24,8 @@ import Toast, { guardedWithErrorToast } from "libs/toast"; import * as Utils from "libs/utils"; import _ from "lodash"; import messages from "messages"; -import { flatToNestedMatrix, getReadableURLPart } from "oxalis/model/accessors/dataset_accessor"; +import { getReadableURLPart } from "oxalis/model/accessors/dataset_accessor"; +import { flatToNestedMatrix } from "oxalis/model/accessors/dataset_layer_transformation_accessor"; import type { OxalisState } from "oxalis/store"; import React, { useState } from "react"; import { useSelector } from "react-redux"; diff --git a/frontend/javascripts/dashboard/dataset/dataset_rotation_form_item.tsx b/frontend/javascripts/dashboard/dataset/dataset_rotation_form_item.tsx new file mode 100644 index 00000000000..3383b20e78c --- /dev/null +++ b/frontend/javascripts/dashboard/dataset/dataset_rotation_form_item.tsx @@ -0,0 +1,189 @@ +import { InfoCircleOutlined } from "@ant-design/icons"; +import { type FormInstance, Row, Col, Slider, InputNumber, Tooltip, Typography, Form } from "antd"; +import FormItem from "antd/es/form/FormItem"; +import { useCallback, useEffect, useMemo } from "react"; +import type { APIDataLayer } from "types/api_flow_types"; +import { FormItemWithInfo } from "./helper_components"; +import { + getRotationMatrixAroundAxis, + fromCenterToOrigin, + IDENTITY_TRANSFORM, + fromOriginToCenter, + AXIS_TO_TRANSFORM_INDEX, + doAllLayersHaveTheSameRotation, +} from "oxalis/model/accessors/dataset_layer_transformation_accessor"; +import BoundingBox from "oxalis/model/bucket_data_handling/bounding_box"; + +const { Text } = Typography; + +type AxisRotationFormItemProps = { + form: FormInstance | undefined; + axis: "x" | "y" | "z"; +}; + +function getDatasetBoundingBoxFromLayers(layers: APIDataLayer[]): BoundingBox | undefined { + if (!layers || layers.length === 0) { + return undefined; + } + let datasetBoundingBox = BoundingBox.fromBoundBoxObject(layers[0].boundingBox); + for (let i = 1; i < layers.length; i++) { + datasetBoundingBox = datasetBoundingBox.extend( + BoundingBox.fromBoundBoxObject(layers[i].boundingBox), + ); + } + return datasetBoundingBox; +} + +export const AxisRotationFormItem: React.FC = ({ + form, + axis, +}: AxisRotationFormItemProps) => { + const dataLayers: APIDataLayer[] = Form.useWatch(["dataSource", "dataLayers"], form); + const datasetBoundingBox = useMemo( + () => getDatasetBoundingBoxFromLayers(dataLayers), + [dataLayers], + ); + // Update the transformations in case the user changes the dataset bounding box. + useEffect(() => { + if ( + datasetBoundingBox == null || + dataLayers[0].coordinateTransformations?.length !== 5 || + !form + ) { + return; + } + const rotationValues = form.getFieldValue(["datasetRotation"]); + const transformations = [ + fromCenterToOrigin(datasetBoundingBox), + getRotationMatrixAroundAxis("x", rotationValues["x"]), + getRotationMatrixAroundAxis("y", rotationValues["y"]), + getRotationMatrixAroundAxis("z", rotationValues["z"]), + fromOriginToCenter(datasetBoundingBox), + ]; + const dataLayersWithUpdatedTransforms = dataLayers.map((layer) => { + return { + ...layer, + coordinateTransformations: transformations, + }; + }); + form.setFieldValue(["dataSource", "dataLayers"], dataLayersWithUpdatedTransforms); + }, [datasetBoundingBox, dataLayers, form]); + + const setMatrixRotationsForAllLayer = useCallback( + (rotationInDegrees: number): void => { + if (!form) { + return; + } + const dataLayers: APIDataLayer[] = form.getFieldValue(["dataSource", "dataLayers"]); + const datasetBoundingBox = getDatasetBoundingBoxFromLayers(dataLayers); + if (datasetBoundingBox == null) { + return; + } + + const rotationInRadians = rotationInDegrees * (Math.PI / 180); + const rotationMatrix = getRotationMatrixAroundAxis(axis, rotationInRadians); + const dataLayersWithUpdatedTransforms: APIDataLayer[] = dataLayers.map((layer) => { + let transformations = layer.coordinateTransformations; + if (transformations == null || transformations.length !== 5) { + transformations = [ + fromCenterToOrigin(datasetBoundingBox), + IDENTITY_TRANSFORM, + IDENTITY_TRANSFORM, + IDENTITY_TRANSFORM, + fromOriginToCenter(datasetBoundingBox), + ]; + } + transformations[AXIS_TO_TRANSFORM_INDEX[axis]] = rotationMatrix; + return { + ...layer, + coordinateTransformations: transformations, + }; + }); + form.setFieldValue(["dataSource", "dataLayers"], dataLayersWithUpdatedTransforms); + }, + [axis, form], + ); + return ( + + + + + + + + + + value != null && setMatrixRotationsForAllLayer(value) + } + /> + + + + ); +}; + +type AxisRotationSettingForDatasetProps = { + form: FormInstance | undefined; +}; + +export type DatasetRotation = { + x: number; + y: number; + z: number; +}; + +export const AxisRotationSettingForDataset: React.FC = ({ + form, +}: AxisRotationSettingForDatasetProps) => { + const dataLayers: APIDataLayer[] = form?.getFieldValue(["dataSource", "dataLayers"]); + const isRotationOnly = useMemo(() => doAllLayersHaveTheSameRotation(dataLayers), [dataLayers]); + + if (!isRotationOnly) { + return ( + + Each layers transformations must be equal and each layer needs exactly 5 affine + transformation with the following schema: +
    +
  • Translation to the origin
    • +
  • Rotation around the x-axis
    • +
  • Rotation around the y-axis
    • +
  • Rotation around the z-axis
    • +
  • Translation back to the original position
    • +
+ To easily enable this setting, delete all coordinateTransformations of all layers in the + advanced tab, save and reload the dataset settings. + + } + > + + Setting a dataset's rotation is only supported when all layers have the same rotation + transformation. + + + ); + } + + return ( +
+ + + +
+ ); +}; diff --git a/frontend/javascripts/dashboard/dataset/dataset_settings_data_tab.tsx b/frontend/javascripts/dashboard/dataset/dataset_settings_data_tab.tsx index e8bbe2aee2a..0f911522978 100644 --- a/frontend/javascripts/dashboard/dataset/dataset_settings_data_tab.tsx +++ b/frontend/javascripts/dashboard/dataset/dataset_settings_data_tab.tsx @@ -33,6 +33,7 @@ import { type APIDataLayer, type APIDataset, APIJobType } from "types/api_flow_t import { useStartAndPollJob } from "admin/job/job_hooks"; import { AllUnits, LongUnitToShortUnitMap, type Vector3 } from "oxalis/constants"; import Toast from "libs/toast"; +import { AxisRotationSettingForDataset } from "./dataset_rotation_form_item"; import type { ArbitraryObject } from "types/globals"; const FormItem = Form.Item; @@ -267,6 +268,12 @@ function SimpleDatasetForm({ + + + + + + diff --git a/frontend/javascripts/dashboard/dataset/dataset_settings_view.tsx b/frontend/javascripts/dashboard/dataset/dataset_settings_view.tsx index da35dcef146..e9adc33d8b7 100644 --- a/frontend/javascripts/dashboard/dataset/dataset_settings_view.tsx +++ b/frontend/javascripts/dashboard/dataset/dataset_settings_view.tsx @@ -43,6 +43,11 @@ import DatasetSettingsDeleteTab from "./dataset_settings_delete_tab"; import DatasetSettingsDataTab, { syncDataSourceFields } from "./dataset_settings_data_tab"; import { defaultContext } from "@tanstack/react-query"; import { getReadableURLPart } from "oxalis/model/accessors/dataset_accessor"; +import type { DatasetRotation } from "./dataset_rotation_form_item"; +import { + doAllLayersHaveTheSameRotation, + getRotationFromTransformationIn90DegreeSteps, +} from "oxalis/model/accessors/dataset_layer_transformation_accessor"; const FormItem = Form.Item; const notImportedYetStatus = "Not imported yet."; @@ -76,6 +81,7 @@ export type FormData = { dataset: APIDataset; defaultConfiguration: DatasetConfiguration; defaultConfigurationLayersJson: string; + datasetRotation?: DatasetRotation; }; class DatasetSettingsView extends React.PureComponent { @@ -194,6 +200,29 @@ class DatasetSettingsView extends React.PureComponent diff --git a/frontend/javascripts/libs/mjs.ts b/frontend/javascripts/libs/mjs.ts index 4c5b6db8a43..043ea92a3ce 100644 --- a/frontend/javascripts/libs/mjs.ts +++ b/frontend/javascripts/libs/mjs.ts @@ -220,6 +220,10 @@ const M4x4 = { r[2] = m[14]; return r; }, + + identity(): Matrix4x4 { + return BareM4x4.identity; + }, }; const V2 = { diff --git a/frontend/javascripts/oxalis/api/api_latest.ts b/frontend/javascripts/oxalis/api/api_latest.ts index 924460157f8..5f1e3cd6bc5 100644 --- a/frontend/javascripts/oxalis/api/api_latest.ts +++ b/frontend/javascripts/oxalis/api/api_latest.ts @@ -84,8 +84,8 @@ import { getMagInfo, getVisibleSegmentationLayer, getMappingInfo, - flatToNestedMatrix, } from "oxalis/model/accessors/dataset_accessor"; +import { flatToNestedMatrix } from "oxalis/model/accessors/dataset_layer_transformation_accessor"; import { getPosition, getActiveMagIndexForLayer, diff --git a/frontend/javascripts/oxalis/constants.ts b/frontend/javascripts/oxalis/constants.ts index 3276804c564..e06fd0ac6f0 100644 --- a/frontend/javascripts/oxalis/constants.ts +++ b/frontend/javascripts/oxalis/constants.ts @@ -15,6 +15,8 @@ export type Vector4 = [number, number, number, number]; export type Vector5 = [number, number, number, number, number]; export type Vector6 = [number, number, number, number, number, number]; +export type NestedMatrix4 = [Vector4, Vector4, Vector4, Vector4]; // Represents a row major matrix. + // For 3D data BucketAddress = x, y, z, mag // For higher dimensional data, BucketAddress = x, y, z, mag, [{name: "t", value: t}, ...] export type BucketAddress = diff --git a/frontend/javascripts/oxalis/controller/scene_controller.ts b/frontend/javascripts/oxalis/controller/scene_controller.ts index 382c6a5a7b3..c6ac4390f17 100644 --- a/frontend/javascripts/oxalis/controller/scene_controller.ts +++ b/frontend/javascripts/oxalis/controller/scene_controller.ts @@ -33,7 +33,6 @@ import { getDatasetBoundingBox, getLayerBoundingBox, getLayerNameToIsDisabled, - getTransformsForLayerOrNull, } from "oxalis/model/accessors/dataset_accessor"; import { getActiveMagIndicesForLayers, getPosition } from "oxalis/model/accessors/flycam_accessor"; import { getSkeletonTracing } from "oxalis/model/accessors/skeletontracing_accessor"; @@ -47,6 +46,7 @@ import { Model } from "oxalis/singletons"; import type { OxalisState, SkeletonTracing, UserBoundingBox } from "oxalis/store"; import Store from "oxalis/store"; import SegmentMeshController from "./segment_mesh_controller"; +import { getTransformsForLayerOrNull } from "oxalis/model/accessors/dataset_layer_transformation_accessor"; const CUBE_COLOR = 0x999999; const LAYER_CUBE_COLOR = 0xffff99; diff --git a/frontend/javascripts/oxalis/geometries/materials/edge_shader.ts b/frontend/javascripts/oxalis/geometries/materials/edge_shader.ts index 5983bdc256a..afe98152132 100644 --- a/frontend/javascripts/oxalis/geometries/materials/edge_shader.ts +++ b/frontend/javascripts/oxalis/geometries/materials/edge_shader.ts @@ -5,13 +5,13 @@ import { listenToStoreProperty } from "oxalis/model/helpers/listener_helpers"; import shaderEditor from "oxalis/model/helpers/shader_editor"; import { Store } from "oxalis/singletons"; import _ from "lodash"; -import { getTransformsForSkeletonLayer } from "oxalis/model/accessors/dataset_accessor"; import { M4x4 } from "libs/mjs"; import { generateCalculateTpsOffsetFunction, generateTpsInitialization, } from "oxalis/shaders/thin_plate_spline.glsl"; import type TPS3D from "libs/thin_plate_spline"; +import { getTransformsForSkeletonLayer } from "oxalis/model/accessors/dataset_layer_transformation_accessor"; class EdgeShader { material: THREE.RawShaderMaterial; diff --git a/frontend/javascripts/oxalis/geometries/materials/node_shader.ts b/frontend/javascripts/oxalis/geometries/materials/node_shader.ts index 84cec0fe626..fce4ac0a3f6 100644 --- a/frontend/javascripts/oxalis/geometries/materials/node_shader.ts +++ b/frontend/javascripts/oxalis/geometries/materials/node_shader.ts @@ -8,13 +8,13 @@ import { Store } from "oxalis/singletons"; import shaderEditor from "oxalis/model/helpers/shader_editor"; import _ from "lodash"; import { formatNumberAsGLSLFloat } from "oxalis/shaders/utils.glsl"; -import { getTransformsForSkeletonLayer } from "oxalis/model/accessors/dataset_accessor"; import { M4x4 } from "libs/mjs"; import { generateCalculateTpsOffsetFunction, generateTpsInitialization, } from "oxalis/shaders/thin_plate_spline.glsl"; import type TPS3D from "libs/thin_plate_spline"; +import { getTransformsForSkeletonLayer } from "oxalis/model/accessors/dataset_layer_transformation_accessor"; export const NodeTypes = { INVALID: 0.0, diff --git a/frontend/javascripts/oxalis/geometries/materials/plane_material_factory.ts b/frontend/javascripts/oxalis/geometries/materials/plane_material_factory.ts index 87c5a6220c7..ad1771f2c07 100644 --- a/frontend/javascripts/oxalis/geometries/materials/plane_material_factory.ts +++ b/frontend/javascripts/oxalis/geometries/materials/plane_material_factory.ts @@ -29,11 +29,8 @@ import { getMappingInfoForSupportedLayer, getVisibleSegmentationLayer, getLayerByName, - invertAndTranspose, - getTransformsForLayer, getMagInfoByLayer, getMagInfo, - getTransformsPerLayer, } from "oxalis/model/accessors/dataset_accessor"; import { getActiveMagIndicesForLayers, @@ -53,6 +50,11 @@ import { CuckooTableVec3 } from "libs/cuckoo/cuckoo_table_vec3"; import { getGlobalLayerIndexForLayerName } from "oxalis/model/bucket_data_handling/layer_rendering_manager"; import { V3 } from "libs/mjs"; import type TPS3D from "libs/thin_plate_spline"; +import { + invertAndTranspose, + getTransformsForLayer, + getTransformsPerLayer, +} from "oxalis/model/accessors/dataset_layer_transformation_accessor"; type ShaderMaterialOptions = { polygonOffset?: boolean; @@ -242,8 +244,7 @@ class PlaneMaterialFactory { this.uniforms.activeMagIndices = { value: Object.values(activeMagIndices), }; - const nativelyRenderedLayerName = - Store.getState().datasetConfiguration.nativelyRenderedLayerName; + const { nativelyRenderedLayerName } = Store.getState().datasetConfiguration; const dataset = Store.getState().dataset; for (const dataLayer of Model.getAllLayers()) { const layerName = sanitizeName(dataLayer.name); diff --git a/frontend/javascripts/oxalis/merger_mode.ts b/frontend/javascripts/oxalis/merger_mode.ts index d934f23596d..e02dd91125e 100644 --- a/frontend/javascripts/oxalis/merger_mode.ts +++ b/frontend/javascripts/oxalis/merger_mode.ts @@ -7,10 +7,8 @@ import type { import type { TreeMap, SkeletonTracing, OxalisState, StoreType } from "oxalis/store"; import type { Vector3 } from "oxalis/constants"; import { cachedDiffTrees } from "oxalis/model/sagas/skeletontracing_saga"; -import { - getInverseSegmentationTransformer, - getVisibleSegmentationLayer, -} from "oxalis/model/accessors/dataset_accessor"; +import { getVisibleSegmentationLayer } from "oxalis/model/accessors/dataset_accessor"; +import { getInverseSegmentationTransformer } from "oxalis/model/accessors/dataset_layer_transformation_accessor"; import { getNodePosition, getSkeletonTracing, diff --git a/frontend/javascripts/oxalis/model/accessors/dataset_accessor.ts b/frontend/javascripts/oxalis/model/accessors/dataset_accessor.ts index 90fac26ea8f..9b742ac1f52 100644 --- a/frontend/javascripts/oxalis/model/accessors/dataset_accessor.ts +++ b/frontend/javascripts/oxalis/model/accessors/dataset_accessor.ts @@ -8,7 +8,6 @@ import type { APIDatasetCompact, APIMaybeUnimportedDataset, APISegmentationLayer, - APISkeletonLayer, ElementClass, } from "types/api_flow_types"; import type { @@ -20,30 +19,15 @@ import type { ActiveMappingInfo, } from "oxalis/store"; import ErrorHandling from "libs/error_handling"; -import { - IdentityTransform, - LongUnitToShortUnitMap, - type Vector3, - type Vector4, - type ViewMode, -} from "oxalis/constants"; +import { LongUnitToShortUnitMap, type Vector3, type ViewMode } from "oxalis/constants"; import constants, { ViewModeValues, Vector3Indicies, MappingStatusEnum } from "oxalis/constants"; import { aggregateBoundingBox, maxValue } from "libs/utils"; import { formatExtentInUnitWithLength, formatNumberToLength } from "libs/format_utils"; import messages from "messages"; import type { DataLayer } from "types/schemas/datasource.types"; import BoundingBox from "../bucket_data_handling/bounding_box"; -import { M4x4, type Matrix4x4, V3 } from "libs/mjs"; +import { V3 } from "libs/mjs"; import { convertToDenseMag, MagInfo } from "../helpers/mag_info"; -import MultiKeyMap from "libs/multi_key_map"; -import { - chainTransforms, - createAffineTransformFromMatrix, - createThinPlateSplineTransform, - invertTransform, - type Transform, - transformPointUnscaled, -} from "../helpers/transformation_helpers"; function _getMagInfo(magnifications: Array): MagInfo { return new MagInfo(magnifications); @@ -715,188 +699,6 @@ export function getMappingInfoForSupportedLayer(state: OxalisState): ActiveMappi ); } -// Returns the transforms (if they exist) for a layer as -// they are defined in the dataset properties. -function _getOriginalTransformsForLayerOrNull( - dataset: APIDataset, - layer: APIDataLayer, -): Transform | null { - const coordinateTransformations = layer.coordinateTransformations; - if (!coordinateTransformations || coordinateTransformations.length === 0) { - return null; - } - if (coordinateTransformations.length > 1) { - console.error( - "Data layer has defined multiple coordinate transforms. This is currently not supported and ignored", - ); - return null; - } - const transformation = coordinateTransformations[0]; - const { type } = transformation; - - if (type === "affine") { - const nestedMatrix = transformation.matrix; - return createAffineTransformFromMatrix(nestedMatrix); - } else if (type === "thin_plate_spline") { - const { source, target } = transformation.correspondences; - - return createThinPlateSplineTransform(source, target, dataset.dataSource.scale.factor); - } - - console.error( - "Data layer has defined a coordinate transform that is not affine or thin_plate_spline. This is currently not supported and ignored", - ); - return null; -} - -function _getTransformsForLayerOrNull( - dataset: APIDataset, - layer: APIDataLayer | APISkeletonLayer, - nativelyRenderedLayerName: string | null, -): Transform | null { - if (layer.category === "skeleton") { - return getTransformsForSkeletonLayerOrNull(dataset, nativelyRenderedLayerName); - } - const layerTransforms = _getOriginalTransformsForLayerOrNull(dataset, layer); - - if (nativelyRenderedLayerName == null) { - // No layer is requested to be rendered natively. Just use the transforms - // as they are in the dataset. - return layerTransforms; - } - - if (nativelyRenderedLayerName === layer.name) { - // This layer should be rendered without any transforms. - return null; - } - - // Apply the inverse of the layer that should be rendered natively - // to the current layers transforms - const nativeLayer = getLayerByName(dataset, nativelyRenderedLayerName, true); - - const transformsOfNativeLayer = _getOriginalTransformsForLayerOrNull(dataset, nativeLayer); - - if (transformsOfNativeLayer == null) { - // The inverse of no transforms, are no transforms. Leave the layer - // transforms untouched. - return layerTransforms; - } - - const inverseNativeTransforms = invertTransform(transformsOfNativeLayer); - return chainTransforms(layerTransforms, inverseNativeTransforms); -} - -function memoizeWithThreeKeys(fn: (a: A, b: B, c: C) => T) { - const map = new MultiKeyMap(); - return (a: A, b: B, c: C): T => { - let res = map.get([a, b, c]); - if (res === undefined) { - res = fn(a, b, c); - map.set([a, b, c], res); - } - return res; - }; -} - -export const getTransformsForLayerOrNull = memoizeWithThreeKeys(_getTransformsForLayerOrNull); -export function getTransformsForLayer( - dataset: APIDataset, - layer: APIDataLayer | APISkeletonLayer, - nativelyRenderedLayerName: string | null, -): Transform { - return ( - getTransformsForLayerOrNull(dataset, layer, nativelyRenderedLayerName || null) || - IdentityTransform - ); -} - -function _getTransformsForSkeletonLayerOrNull( - dataset: APIDataset, - nativelyRenderedLayerName: string | null, -): Transform | null { - if (nativelyRenderedLayerName == null) { - // No layer is requested to be rendered natively. We can use - // each layer's transforms as is. The skeleton layer doesn't have - // a transforms property currently, which is why we return null. - return null; - } - - // Compute the inverse of the layer that should be rendered natively - const nativeLayer = getLayerByName(dataset, nativelyRenderedLayerName, true); - const transformsOfNativeLayer = _getOriginalTransformsForLayerOrNull(dataset, nativeLayer); - - if (transformsOfNativeLayer == null) { - // The inverse of no transforms, are no transforms - return null; - } - - return invertTransform(transformsOfNativeLayer); -} - -export const getTransformsForSkeletonLayerOrNull = memoizeOne(_getTransformsForSkeletonLayerOrNull); - -export function getTransformsForSkeletonLayer( - dataset: APIDataset, - nativelyRenderedLayerName: string | null, -): Transform { - return ( - getTransformsForSkeletonLayerOrNull(dataset, nativelyRenderedLayerName || null) || - IdentityTransform - ); -} - -function _getTransformsPerLayer( - dataset: APIDataset, - nativelyRenderedLayerName: string | null, -): Record { - const transformsPerLayer: Record = {}; - const layers = dataset.dataSource.dataLayers; - for (const layer of layers) { - const transforms = getTransformsForLayer(dataset, layer, nativelyRenderedLayerName); - transformsPerLayer[layer.name] = transforms; - } - - return transformsPerLayer; -} - -export const getTransformsPerLayer = memoizeOne(_getTransformsPerLayer); - -export function getInverseSegmentationTransformer( - state: OxalisState, - segmentationLayerName: string, -) { - const { dataset } = state; - const { nativelyRenderedLayerName } = state.datasetConfiguration; - const layer = getLayerByName(dataset, segmentationLayerName); - const segmentationTransforms = getTransformsForLayer(dataset, layer, nativelyRenderedLayerName); - return transformPointUnscaled(invertTransform(segmentationTransforms)); -} - -export const hasDatasetTransforms = memoizeOne((dataset: APIDataset) => { - const layers = dataset.dataSource.dataLayers; - return layers.some((layer) => _getOriginalTransformsForLayerOrNull(dataset, layer) != null); -}); - -export function flatToNestedMatrix(matrix: Matrix4x4): [Vector4, Vector4, Vector4, Vector4] { - return [ - matrix.slice(0, 4) as Vector4, - matrix.slice(4, 8) as Vector4, - matrix.slice(8, 12) as Vector4, - matrix.slice(12, 16) as Vector4, - ]; -} - -// Transposition is often needed so that the matrix has the right format -// for matrix operations (e.g., on the GPU; but not for ThreeJS). -// Inversion is needed when the position of an "output voxel" (e.g., during -// rendering in the fragment shader) needs to be mapped to its original -// data position (i.e., how it's stored without the transformation). -// Without the inversion, the matrix maps from stored position to the position -// where it should be rendered. -export const invertAndTranspose = _.memoize((mat: Matrix4x4) => { - return M4x4.transpose(M4x4.inverse(mat)); -}); - export function getEffectiveIntensityRange( dataset: APIDataset, layerName: string, diff --git a/frontend/javascripts/oxalis/model/accessors/dataset_layer_transformation_accessor.ts b/frontend/javascripts/oxalis/model/accessors/dataset_layer_transformation_accessor.ts new file mode 100644 index 00000000000..76edc128124 --- /dev/null +++ b/frontend/javascripts/oxalis/model/accessors/dataset_layer_transformation_accessor.ts @@ -0,0 +1,456 @@ +import { M4x4, V3, type Matrix4x4 } from "libs/mjs"; +import { + Identity4x4, + IdentityTransform, + type Vector3, + type NestedMatrix4, + type Vector4, +} from "oxalis/constants"; +import type { OxalisState } from "oxalis/store"; +import * as THREE from "three"; +import type { + AffineTransformation, + APIDataLayer, + APIDataset, + APISkeletonLayer, + CoordinateTransformation, +} from "types/api_flow_types"; +import { mod } from "libs/utils"; +import MultiKeyMap from "libs/multi_key_map"; +import _ from "lodash"; +import memoizeOne from "memoize-one"; +import { + createAffineTransformFromMatrix, + createThinPlateSplineTransform, + chainTransforms, + invertTransform, + transformPointUnscaled, + nestedToFlatMatrix, + type Transform, +} from "../helpers/transformation_helpers"; +import { getLayerByName } from "./dataset_accessor"; +import type BoundingBox from "../bucket_data_handling/bounding_box"; +import { getPosition } from "./flycam_accessor"; + +const IDENTITY_MATRIX = [ + [1, 0, 0, 0], + [0, 1, 0, 0], + [0, 0, 1, 0], + [0, 0, 0, 1], +] as NestedMatrix4; + +export const IDENTITY_TRANSFORM: CoordinateTransformation = { + type: "affine", + matrix: IDENTITY_MATRIX, +}; + +// cf. https://en.wikipedia.org/wiki/Rotation_matrix#In_three_dimensions +const sinusLocationOfRotationInMatrix = { + x: [2, 1], + y: [0, 2], + z: [1, 0], +}; + +const cosineLocationOfRotationInMatrix = { + x: [1, 1], + y: [0, 0], + z: [0, 0], +}; + +export const AXIS_TO_TRANSFORM_INDEX = { + x: 1, + y: 2, + z: 3, +}; + +export function flatToNestedMatrix(matrix: Matrix4x4): NestedMatrix4 { + return [ + matrix.slice(0, 4) as Vector4, + matrix.slice(4, 8) as Vector4, + matrix.slice(8, 12) as Vector4, + matrix.slice(12, 16) as Vector4, + ]; +} + +// This function extracts the rotation in 90 degree steps a the transformation matrix. +// The transformation matrix must only include a rotation around one of the main axis. +export function getRotationFromTransformationIn90DegreeSteps( + transformation: CoordinateTransformation | undefined, + axis: "x" | "y" | "z", +) { + if (transformation && transformation.type !== "affine") { + return 0; + } + const matrix = transformation ? transformation.matrix : IDENTITY_MATRIX; + const cosineLocation = cosineLocationOfRotationInMatrix[axis]; + const sinusLocation = sinusLocationOfRotationInMatrix[axis]; + const sinOfAngle = matrix[sinusLocation[0]][sinusLocation[1]]; + const cosOfAngle = matrix[cosineLocation[0]][cosineLocation[1]]; + const rotation = + Math.abs(cosOfAngle) > 1e-6 // Avoid division by zero + ? Math.atan2(sinOfAngle, cosOfAngle) + : sinOfAngle > 0 + ? Math.PI / 2 + : -Math.PI / 2; + const rotationInDegrees = rotation * (180 / Math.PI); + // Round to multiple of 90 degrees and keep the result positive. + const roundedRotation = mod(Math.round((rotationInDegrees + 360) / 90) * 90, 360); + return roundedRotation; +} + +export function fromCenterToOrigin(bbox: BoundingBox): AffineTransformation { + const center = bbox.getCenter(); + const translationMatrix = new THREE.Matrix4() + .makeTranslation(-center[0], -center[1], -center[2]) + .transpose(); // Column-major to row-major + return { type: "affine", matrix: flatToNestedMatrix(translationMatrix.toArray()) }; +} + +export function fromOriginToCenter(bbox: BoundingBox): AffineTransformation { + const center = bbox.getCenter(); + const translationMatrix = new THREE.Matrix4() + .makeTranslation(center[0], center[1], center[2]) + .transpose(); // Column-major to row-major + return { type: "affine", matrix: flatToNestedMatrix(translationMatrix.toArray()) }; +} +export function getRotationMatrixAroundAxis( + axis: "x" | "y" | "z", + angleInRadians: number, +): AffineTransformation { + const euler = new THREE.Euler(); + euler[axis] = angleInRadians; + const rotationMatrix = new THREE.Matrix4().makeRotationFromEuler(euler).transpose(); // Column-major to row-major + const matrixWithoutNearlyZeroValues = rotationMatrix + .toArray() + // Avoid nearly zero values due to floating point arithmetic inaccuracies. + .map((value) => (Math.abs(value) < Number.EPSILON ? 0 : value)) as Matrix4x4; + return { type: "affine", matrix: flatToNestedMatrix(matrixWithoutNearlyZeroValues) }; +} + +function memoizeWithThreeKeys(fn: (a: A, b: B, c: C) => T) { + const map = new MultiKeyMap(); + return (a: A, b: B, c: C): T => { + let res = map.get([a, b, c]); + if (res === undefined) { + res = fn(a, b, c); + map.set([a, b, c], res); + } + return res; + }; +} + +function memoizeWithTwoKeys(fn: (a: A, b: B) => T) { + const map = new MultiKeyMap(); + return (a: A, b: B): T => { + let res = map.get([a, b]); + if (res === undefined) { + res = fn(a, b); + map.set([a, b], res); + } + return res; + }; +} + +// Returns the transforms (if they exist) for a layer as +// they are defined in the dataset properties. +function _getOriginalTransformsForLayerOrNull( + dataset: APIDataset, + layer: APIDataLayer, +): Transform | null { + const coordinateTransformations = layer.coordinateTransformations; + if (!coordinateTransformations || coordinateTransformations.length === 0) { + return null; + } + + const transforms = coordinateTransformations.map((coordTransformation) => { + const { type } = coordTransformation; + if (type === "affine") { + const nestedMatrix = coordTransformation.matrix; + return createAffineTransformFromMatrix(nestedMatrix); + } else if (type === "thin_plate_spline") { + const { source, target } = coordTransformation.correspondences; + + return createThinPlateSplineTransform(source, target, dataset.dataSource.scale.factor); + } + + console.error( + "Data layer has defined a coordinate transform that is not affine or thin_plate_spline. This is currently not supported and ignored", + ); + return IdentityTransform; + }); + return transforms.reduce(chainTransforms, IdentityTransform); +} + +export const getOriginalTransformsForLayerOrNull = memoizeWithTwoKeys( + _getOriginalTransformsForLayerOrNull, +); + +export function isLayerWithoutTransformationConfigSupport(layer: APIDataLayer | APISkeletonLayer) { + return ( + layer.category === "skeleton" || + (layer.category === "segmentation" && "tracingId" in layer && !layer.fallbackLayer) + ); +} + +function _getTransformsForLayerOrNull( + dataset: APIDataset, + layer: APIDataLayer | APISkeletonLayer, + nativelyRenderedLayerName: string | null, +): Transform | null { + if (isLayerWithoutTransformationConfigSupport(layer)) { + return getTransformsForLayerThatDoesNotSupportTransformationConfigOrNull( + dataset, + nativelyRenderedLayerName, + ); + } + + if (layer.name === nativelyRenderedLayerName) { + // This layer should be rendered without any transforms. + return null; + } + const layerTransforms = getOriginalTransformsForLayerOrNull(dataset, layer as APIDataLayer); + if (nativelyRenderedLayerName == null) { + // No layer is requested to be rendered natively. -> We can use the layer's transforms as is. + return layerTransforms; + } + + // Apply the inverse of the layer that should be rendered natively + // to the current layer's transforms. + const nativeLayer = getLayerByName(dataset, nativelyRenderedLayerName, true); + const transformsOfNativeLayer = getOriginalTransformsForLayerOrNull(dataset, nativeLayer); + + if (transformsOfNativeLayer == null) { + // The inverse of no transforms, are no transforms. Leave the layer + // transforms untouched. + return layerTransforms; + } + + const inverseNativeTransforms = invertTransform(transformsOfNativeLayer); + return chainTransforms(layerTransforms, inverseNativeTransforms); +} + +export const getTransformsForLayerOrNull = memoizeWithThreeKeys(_getTransformsForLayerOrNull); +export function getTransformsForLayer( + dataset: APIDataset, + layer: APIDataLayer | APISkeletonLayer, + nativelyRenderedLayerName: string | null, +): Transform { + return ( + getTransformsForLayerOrNull(dataset, layer, nativelyRenderedLayerName) || IdentityTransform + ); +} + +export function isIdentityTransform(transform: Transform) { + return transform.type === "affine" && _.isEqual(transform.affineMatrix, Identity4x4); +} + +function _getTransformsForLayerThatDoesNotSupportTransformationConfigOrNull( + dataset: APIDataset, + nativelyRenderedLayerName: string | null, +): Transform | null { + const layers = dataset.dataSource.dataLayers; + const allLayersSameRotation = doAllLayersHaveTheSameRotation(layers); + if (nativelyRenderedLayerName == null) { + // No layer is requested to be rendered natively. -> We can use each layer's transforms as is. + if (!allLayersSameRotation) { + // If the dataset's layers do not have a consistent transformation (which only rotates the dataset), + // we cannot guess what transformation should be applied to the layer. + // As skeleton layer and volume layer without fallback don't have a transforms property currently. + return null; + } + + // The skeleton layer / volume layer without fallback needs transformed just like the other layers. + // Thus, we simply use the first usable layer which supports transforms. + const usableReferenceLayer = layers.find( + (layer) => !isLayerWithoutTransformationConfigSupport(layer), + ); + const someLayersTransformsMaybe = usableReferenceLayer + ? getTransformsForLayerOrNull(dataset, usableReferenceLayer, nativelyRenderedLayerName) + : null; + return someLayersTransformsMaybe; + } else if (nativelyRenderedLayerName != null && allLayersSameRotation) { + // If all layers have the same transformations and at least one is rendered natively, this means that all layer should be rendered natively. + return null; + } + + // Compute the inverse of the layer that should be rendered natively. + const nativeLayer = getLayerByName(dataset, nativelyRenderedLayerName, true); + const transformsOfNativeLayer = getOriginalTransformsForLayerOrNull(dataset, nativeLayer); + + if (transformsOfNativeLayer == null) { + // The inverse of no transforms, are no transforms. + return null; + } + + return invertTransform(transformsOfNativeLayer); +} + +export const getTransformsForLayerThatDoesNotSupportTransformationConfigOrNull = memoizeOne( + _getTransformsForLayerThatDoesNotSupportTransformationConfigOrNull, +); + +export function getTransformsForSkeletonLayer( + dataset: APIDataset, + nativelyRenderedLayerName: string | null, +): Transform { + return ( + getTransformsForLayerThatDoesNotSupportTransformationConfigOrNull( + dataset, + nativelyRenderedLayerName, + ) || IdentityTransform + ); +} + +function _getTransformsPerLayer( + dataset: APIDataset, + nativelyRenderedLayerName: string | null, +): Record { + const transformsPerLayer: Record = {}; + const layers = dataset.dataSource.dataLayers; + for (const layer of layers) { + const transforms = getTransformsForLayer(dataset, layer, nativelyRenderedLayerName); + transformsPerLayer[layer.name] = transforms; + } + + return transformsPerLayer; +} + +export const getTransformsPerLayer = memoizeWithTwoKeys(_getTransformsPerLayer); + +export function getInverseSegmentationTransformer( + state: OxalisState, + segmentationLayerName: string, +) { + const { dataset } = state; + const { nativelyRenderedLayerName } = state.datasetConfiguration; + const layer = getLayerByName(dataset, segmentationLayerName); + const segmentationTransforms = getTransformsForLayer(dataset, layer, nativelyRenderedLayerName); + return transformPointUnscaled(invertTransform(segmentationTransforms)); +} + +export const hasDatasetTransforms = memoizeOne((dataset: APIDataset) => { + const layers = dataset.dataSource.dataLayers; + return layers.some((layer) => getOriginalTransformsForLayerOrNull(dataset, layer) != null); +}); + +// Transposition is often needed so that the matrix has the right format +// for matrix operations (e.g., on the GPU; but not for ThreeJS). +// Inversion is needed when the position of an "output voxel" (e.g., during +// rendering in the fragment shader) needs to be mapped to its original +// data position (i.e., how it's stored without the transformation). +// Without the inversion, the matrix maps from stored position to the position +// where it should be rendered. +export const invertAndTranspose = _.memoize((mat: Matrix4x4) => { + return M4x4.transpose(M4x4.inverse(mat)); +}); + +const translation = new THREE.Vector3(); +const scale = new THREE.Vector3(); +const quaternion = new THREE.Quaternion(); +const IDENTITY_QUATERNION = new THREE.Quaternion(); + +const NON_SCALED_VECTOR = new THREE.Vector3(1, 1, 1); + +function isTranslationOnly(transformation?: AffineTransformation) { + if (!transformation) { + return false; + } + const threeMatrix = new THREE.Matrix4() + .fromArray(nestedToFlatMatrix(transformation.matrix)) + .transpose(); + threeMatrix.decompose(translation, quaternion, scale); + return scale.equals(NON_SCALED_VECTOR) && quaternion.equals(IDENTITY_QUATERNION); +} + +function isRotationOnly(transformation?: AffineTransformation) { + if (!transformation) { + return false; + } + const threeMatrix = new THREE.Matrix4() + .fromArray(nestedToFlatMatrix(transformation.matrix)) + .transpose(); + threeMatrix.decompose(translation, quaternion, scale); + return translation.length() === 0 && scale.equals(NON_SCALED_VECTOR); +} + +function hasValidTransformationCount(dataLayers: Array): boolean { + return dataLayers.every((layer) => layer.coordinateTransformations?.length === 5); +} + +function hasOnlyAffineTransformations(dataLayers: Array): boolean { + return dataLayers.every((layer) => + layer.coordinateTransformations?.every((transformation) => transformation.type === "affine"), + ); +} + +function hasValidTransformationPattern(transformations: CoordinateTransformation[]): boolean { + return ( + isTranslationOnly(transformations[0] as AffineTransformation) && + isRotationOnly(transformations[1] as AffineTransformation) && + isRotationOnly(transformations[2] as AffineTransformation) && + isRotationOnly(transformations[3] as AffineTransformation) && + isTranslationOnly(transformations[4] as AffineTransformation) + ); +} + +function _doAllLayersHaveTheSameRotation(dataLayers: Array): boolean { + const firstDataLayerTransformations = dataLayers[0]?.coordinateTransformations; + if (firstDataLayerTransformations == null || firstDataLayerTransformations.length === 0) { + // No transformations in all layers compatible with setting a rotation for the whole dataset. + return dataLayers.every( + (layer) => + layer.coordinateTransformations == null || layer.coordinateTransformations.length === 0, + ); + } + // There should be a translation to the origin, one transformation for each axis and one translation back. => A total of 5 affine transformations. + if (!hasValidTransformationCount(dataLayers) || !hasOnlyAffineTransformations(dataLayers)) { + return false; + } + + if (!hasValidTransformationPattern(firstDataLayerTransformations)) { + return false; + } + for (let i = 1; i < dataLayers.length; i++) { + const transformations = dataLayers[i].coordinateTransformations; + if ( + transformations == null || + !_.isEqual(transformations[0], firstDataLayerTransformations[0]) || + !_.isEqual(transformations[1], firstDataLayerTransformations[1]) || + !_.isEqual(transformations[2], firstDataLayerTransformations[2]) || + !_.isEqual(transformations[3], firstDataLayerTransformations[3]) || + !_.isEqual(transformations[4], firstDataLayerTransformations[4]) + ) { + return false; + } + } + return true; +} + +export const doAllLayersHaveTheSameRotation = _.memoize(_doAllLayersHaveTheSameRotation); + +export function getNewPositionAndZoomChangeFromTransformationChange( + activeTransformation: Transform, + nextTransform: Transform, + state: OxalisState, +) { + // Calculate the difference between the current and the next transformation. + const currentTransformInverted = invertTransform(activeTransformation); + const changeInAppliedTransformation = chainTransforms(currentTransformInverted, nextTransform); + + const currentPosition = getPosition(state.flycam); + const newPosition = transformPointUnscaled(changeInAppliedTransformation)(currentPosition); + + // Also transform a reference coordinate to determine how the scaling + // changed. Then, adapt the zoom accordingly. + + const referenceOffset: Vector3 = [10, 10, 10]; + const secondPosition = V3.add(currentPosition, referenceOffset, [0, 0, 0]); + const newSecondPosition = transformPointUnscaled(changeInAppliedTransformation)(secondPosition); + + const scaleChange = _.mean( + // Only consider XY for now to determine the zoom change (by slicing from 0 to 2) + V3.abs(V3.divide3(V3.sub(newPosition, newSecondPosition), referenceOffset)).slice(0, 2), + ); + + return { newPosition, scaleChange }; +} diff --git a/frontend/javascripts/oxalis/model/accessors/flycam_accessor.ts b/frontend/javascripts/oxalis/model/accessors/flycam_accessor.ts index c3040f57e8f..e2de57c6816 100644 --- a/frontend/javascripts/oxalis/model/accessors/flycam_accessor.ts +++ b/frontend/javascripts/oxalis/model/accessors/flycam_accessor.ts @@ -12,8 +12,6 @@ import { getLayerByName, getMaxZoomStep, getMagInfo, - getTransformsForLayer, - invertAndTranspose, } from "oxalis/model/accessors/dataset_accessor"; import { map3, mod } from "libs/utils"; import Dimensions from "oxalis/model/dimensions"; @@ -37,6 +35,7 @@ import { getMatrixScale, rotateOnAxis } from "../reducers/flycam_reducer"; import type { SmallerOrHigherInfo } from "../helpers/mag_info"; import { getBaseVoxelInUnit } from "oxalis/model/scaleinfo"; import type { AdditionalCoordinate, VoxelSize } from "types/api_flow_types"; +import { invertAndTranspose, getTransformsForLayer } from "./dataset_layer_transformation_accessor"; export const ZOOM_STEP_INTERVAL = 1.1; @@ -194,7 +193,7 @@ const perLayerFnCache: Map = new Map() // Only exported for testing. export const _getDummyFlycamMatrix = memoizeOne((scale: Vector3) => { const scaleMatrix = getMatrixScale(scale); - return rotateOnAxis(M4x4.scale(scaleMatrix, M4x4.identity, []), Math.PI, [0, 0, 1]); + return rotateOnAxis(M4x4.scale(scaleMatrix, M4x4.identity(), []), Math.PI, [0, 0, 1]); }); export function getMoveOffset(state: OxalisState, timeFactor: number) { diff --git a/frontend/javascripts/oxalis/model/accessors/skeletontracing_accessor.ts b/frontend/javascripts/oxalis/model/accessors/skeletontracing_accessor.ts index 657a18ca2e0..24bb09f615b 100644 --- a/frontend/javascripts/oxalis/model/accessors/skeletontracing_accessor.ts +++ b/frontend/javascripts/oxalis/model/accessors/skeletontracing_accessor.ts @@ -23,11 +23,11 @@ import { MISSING_GROUP_ID, } from "oxalis/view/right-border-tabs/tree_hierarchy_view_helpers"; import type { TreeType, Vector3 } from "oxalis/constants"; +import { invertTransform, transformPointUnscaled } from "../helpers/transformation_helpers"; import { + getTransformsForLayerThatDoesNotSupportTransformationConfigOrNull, getTransformsForSkeletonLayer, - getTransformsForSkeletonLayerOrNull, -} from "./dataset_accessor"; -import { invertTransform, transformPointUnscaled } from "../helpers/transformation_helpers"; +} from "./dataset_layer_transformation_accessor"; export function getSkeletonTracing(tracing: Tracing): Maybe { if (tracing.skeleton != null) { @@ -218,7 +218,7 @@ export function getNodeAndTreeOrNull( export function isSkeletonLayerTransformed(state: OxalisState) { return ( - getTransformsForSkeletonLayerOrNull( + getTransformsForLayerThatDoesNotSupportTransformationConfigOrNull( state.dataset, state.datasetConfiguration.nativelyRenderedLayerName, ) != null @@ -231,16 +231,14 @@ export function getNodePosition(node: Node, state: OxalisState): Vector3 { export function transformNodePosition(position: Vector3, state: OxalisState): Vector3 { const dataset = state.dataset; - const nativelyRenderedLayerName = state.datasetConfiguration.nativelyRenderedLayerName; - + const { nativelyRenderedLayerName } = state.datasetConfiguration; const currentTransforms = getTransformsForSkeletonLayer(dataset, nativelyRenderedLayerName); return transformPointUnscaled(currentTransforms)(position); } export function untransformNodePosition(position: Vector3, state: OxalisState): Vector3 { const dataset = state.dataset; - const nativelyRenderedLayerName = state.datasetConfiguration.nativelyRenderedLayerName; - + const { nativelyRenderedLayerName } = state.datasetConfiguration; const currentTransforms = getTransformsForSkeletonLayer(dataset, nativelyRenderedLayerName); return transformPointUnscaled(invertTransform(currentTransforms))(position); } diff --git a/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts b/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts index 35dfe4c2b6b..e6e2ba4882b 100644 --- a/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts +++ b/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts @@ -9,10 +9,7 @@ import { hasAgglomerateMapping, isVolumeAnnotationDisallowedForZoom, } from "oxalis/model/accessors/volumetracing_accessor"; -import { - getTransformsPerLayer, - getVisibleSegmentationLayer, -} from "oxalis/model/accessors/dataset_accessor"; +import { getVisibleSegmentationLayer } from "oxalis/model/accessors/dataset_accessor"; import { isMagRestrictionViolated } from "oxalis/model/accessors/flycam_accessor"; import type { APIOrganization, APIUser } from "types/api_flow_types"; import { @@ -22,6 +19,7 @@ import { } from "admin/organization/pricing_plan_utils"; import { isSkeletonLayerTransformed } from "./skeletontracing_accessor"; import { reuseInstanceOnEquality } from "./accessor_helpers"; +import { getTransformsPerLayer } from "./dataset_layer_transformation_accessor"; const zoomInToUseToolMessage = "Please zoom in further to use this tool. If you want to edit volume data on this zoom level, create an annotation with restricted magnifications from the extended annotation menu in the dashboard."; diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/bounding_box.ts b/frontend/javascripts/oxalis/model/bucket_data_handling/bounding_box.ts index 9769bc95984..2786d7c3684 100644 --- a/frontend/javascripts/oxalis/model/bucket_data_handling/bounding_box.ts +++ b/frontend/javascripts/oxalis/model/bucket_data_handling/bounding_box.ts @@ -5,6 +5,7 @@ import type { BoundingBoxType, OrthoView, Vector2, Vector3, Vector4 } from "oxal import constants, { Vector3Indicies } from "oxalis/constants"; import type { MagInfo } from "../helpers/mag_info"; import Dimensions from "../dimensions"; +import type { BoundingBoxObject } from "oxalis/store"; class BoundingBox { min: Vector3; @@ -24,6 +25,13 @@ class BoundingBox { } } + static fromBoundBoxObject(boundingBox: BoundingBoxObject): BoundingBox { + return new BoundingBox({ + min: boundingBox.topLeft, + max: V3.add(boundingBox.topLeft, [boundingBox.width, boundingBox.height, boundingBox.depth]), + }); + } + getMinUV(activeViewport: OrthoView): Vector2 { const [u, v, _w] = Dimensions.transDim(this.min, activeViewport); return [u, v]; diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/layer_rendering_manager.ts b/frontend/javascripts/oxalis/model/bucket_data_handling/layer_rendering_manager.ts index 8d39d019265..b690a7edf97 100644 --- a/frontend/javascripts/oxalis/model/bucket_data_handling/layer_rendering_manager.ts +++ b/frontend/javascripts/oxalis/model/bucket_data_handling/layer_rendering_manager.ts @@ -11,8 +11,6 @@ import { isLayerVisible, getLayerByName, getMagInfo, - invertAndTranspose, - getTransformsForLayer, } from "oxalis/model/accessors/dataset_accessor"; import AsyncBucketPickerWorker from "oxalis/workers/async_bucket_picker.worker"; import type DataCube from "oxalis/model/bucket_data_handling/data_cube"; @@ -32,6 +30,10 @@ import { getSegmentsForLayer } from "../accessors/volumetracing_accessor"; import { getViewportRects } from "../accessors/view_mode_accessor"; import type { AdditionalCoordinate } from "types/api_flow_types"; import app from "app"; +import { + invertAndTranspose, + getTransformsForLayer, +} from "../accessors/dataset_layer_transformation_accessor"; const CUSTOM_COLORS_TEXTURE_WIDTH = 512; // 256**2 (entries) * 0.25 (load capacity) / 8 (layers) == 2048 buckets/layer diff --git a/frontend/javascripts/oxalis/model/helpers/nml_helpers.ts b/frontend/javascripts/oxalis/model/helpers/nml_helpers.ts index 4312fb8c941..d1cc9ac9593 100644 --- a/frontend/javascripts/oxalis/model/helpers/nml_helpers.ts +++ b/frontend/javascripts/oxalis/model/helpers/nml_helpers.ts @@ -38,7 +38,7 @@ import { location } from "libs/window"; import { coalesce } from "libs/utils"; import type { AdditionalCoordinate } from "types/api_flow_types"; import { getNodePosition } from "../accessors/skeletontracing_accessor"; -import { getTransformsForSkeletonLayer } from "../accessors/dataset_accessor"; +import { getTransformsForSkeletonLayer } from "../accessors/dataset_layer_transformation_accessor"; // NML Defaults const DEFAULT_COLOR: Vector3 = [1, 0, 0]; diff --git a/frontend/javascripts/oxalis/model/helpers/transformation_helpers.ts b/frontend/javascripts/oxalis/model/helpers/transformation_helpers.ts index 01e910b9cfe..1badf5b682b 100644 --- a/frontend/javascripts/oxalis/model/helpers/transformation_helpers.ts +++ b/frontend/javascripts/oxalis/model/helpers/transformation_helpers.ts @@ -2,9 +2,9 @@ import { estimateAffineMatrix4x4 } from "libs/estimate_affine"; import { M4x4 } from "libs/mjs"; import TPS3D from "libs/thin_plate_spline"; import type { Matrix4x4 } from "mjs"; -import type { Vector3, Vector4 } from "oxalis/constants"; +import type { Vector3, NestedMatrix4 } from "oxalis/constants"; -export function nestedToFlatMatrix(matrix: [Vector4, Vector4, Vector4, Vector4]): Matrix4x4 { +export function nestedToFlatMatrix(matrix: NestedMatrix4): Matrix4x4 { return [...matrix[0], ...matrix[1], ...matrix[2], ...matrix[3]]; } @@ -26,9 +26,7 @@ export type Transform = scaledTps: TPS3D; }; -export function createAffineTransformFromMatrix( - nestedMatrix: [Vector4, Vector4, Vector4, Vector4], -): Transform { +export function createAffineTransformFromMatrix(nestedMatrix: NestedMatrix4): Transform { const affineMatrix = nestedToFlatMatrix(nestedMatrix); return { type: "affine", affineMatrix, affineMatrixInv: M4x4.inverse(affineMatrix) }; } diff --git a/frontend/javascripts/oxalis/model/reducers/flycam_reducer.ts b/frontend/javascripts/oxalis/model/reducers/flycam_reducer.ts index 3569f26a462..6d24c78187c 100644 --- a/frontend/javascripts/oxalis/model/reducers/flycam_reducer.ts +++ b/frontend/javascripts/oxalis/model/reducers/flycam_reducer.ts @@ -102,7 +102,7 @@ function resetMatrix(matrix: Matrix4x4, voxelSize: Vector3) { // Save position const position = [matrix[12], matrix[13], matrix[14]]; // Reset rotation - const newMatrix = rotateOnAxis(M4x4.scale(scale, M4x4.identity, []), Math.PI, [0, 0, 1]); + const newMatrix = rotateOnAxis(M4x4.scale(scale, M4x4.identity(), []), Math.PI, [0, 0, 1]); // Restore position newMatrix[12] = position[0]; newMatrix[13] = position[1]; diff --git a/frontend/javascripts/oxalis/model/sagas/dataset_saga.ts b/frontend/javascripts/oxalis/model/sagas/dataset_saga.ts index b10c681ba5c..3e088ba7092 100644 --- a/frontend/javascripts/oxalis/model/sagas/dataset_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/dataset_saga.ts @@ -10,10 +10,12 @@ import { getLayerByName, getMaybeSegmentIndexAvailability, getMagInfo, - getTransformsForLayer, - invertAndTranspose, isLayerVisible, } from "../accessors/dataset_accessor"; +import { + getTransformsForLayer, + invertAndTranspose, +} from "../accessors/dataset_layer_transformation_accessor"; import { getCurrentMag } from "../accessors/flycam_accessor"; import { getViewportExtents } from "../accessors/view_mode_accessor"; import { V3 } from "libs/mjs"; diff --git a/frontend/javascripts/oxalis/model/sagas/quick_select_heuristic_saga.ts b/frontend/javascripts/oxalis/model/sagas/quick_select_heuristic_saga.ts index 8a6956f8cdb..8696fc8473a 100644 --- a/frontend/javascripts/oxalis/model/sagas/quick_select_heuristic_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/quick_select_heuristic_saga.ts @@ -57,8 +57,8 @@ import { getEnabledColorLayers, getLayerBoundingBox, getMagInfo, - getTransformsForLayer, } from "../accessors/dataset_accessor"; +import { getTransformsForLayer } from "../accessors/dataset_layer_transformation_accessor"; import Dimensions, { type DimensionIndices } from "../dimensions"; import { getActiveMagIndexForLayer } from "../accessors/flycam_accessor"; import { updateUserSettingAction } from "../actions/settings_actions"; diff --git a/frontend/javascripts/oxalis/model_initialization.ts b/frontend/javascripts/oxalis/model_initialization.ts index 3be1248d892..4e66ebbab7b 100644 --- a/frontend/javascripts/oxalis/model_initialization.ts +++ b/frontend/javascripts/oxalis/model_initialization.ts @@ -105,6 +105,8 @@ import { isFeatureAllowedByPricingPlan, } from "admin/organization/pricing_plan_utils"; import { convertServerAdditionalAxesToFrontEnd } from "./model/reducers/reducer_helpers"; +import { doAllLayersHaveTheSameRotation } from "./model/accessors/dataset_layer_transformation_accessor"; +import type { Mutable } from "types/globals"; export const HANDLED_ERROR = "error_was_handled"; type DataLayerCollection = Record; @@ -476,6 +478,7 @@ function getMergedDataLayersFromDatasetAndVolumeTracings( const originalLayers = dataset.dataSource.dataLayers; const newLayers = originalLayers.slice(); + const allLayersSameRotation = doAllLayersHaveTheSameRotation(originalLayers); for (const tracing of tracings) { // The tracing always contains the layer information for the user segmentation. @@ -493,6 +496,16 @@ function getMergedDataLayersFromDatasetAndVolumeTracings( const boundingBox = getDatasetBoundingBox(dataset).asServerBoundingBox(); const mags = tracing.mags || []; const tracingHasMagList = mags.length > 0; + let coordinateTransformsMaybe = {}; + if (allLayersSameRotation) { + coordinateTransformsMaybe = { + coordinateTransformations: originalLayers?.[0].coordinateTransformations, + }; + } else if (fallbackLayer?.coordinateTransformations) { + coordinateTransformsMaybe = { + coordinateTransformations: fallbackLayer.coordinateTransformations, + }; + } // Legacy tracings don't have the `tracing.mags` property // since they were created before WK started to maintain multiple magnifications // in volume annotations. Therefore, this code falls back to mag (1, 1, 1) for @@ -514,6 +527,7 @@ function getMergedDataLayersFromDatasetAndVolumeTracings( fallbackLayer: tracing.fallbackLayer, fallbackLayerInfo: fallbackLayer, additionalAxes: convertServerAdditionalAxesToFrontEnd(tracing.additionalAxes), + ...coordinateTransformsMaybe, }; if (fallbackLayerIndex > -1) { newLayers[fallbackLayerIndex] = tracingLayer; @@ -839,13 +853,32 @@ function applyAnnotationSpecificViewConfiguration( * Apply annotation-specific view configurations to the dataset settings which are persisted * per user per dataset. The AnnotationViewConfiguration currently only holds the "isDisabled" * information per layer which should override the isDisabled information in DatasetConfiguration. + * Moreover, due to another annotation nativelyRenderedLayerName might be set to a layer which does + * not exist in this view / annotation. In this case, the nativelyRenderedLayerName should be set to null. */ - if (!annotation) { - return originalDatasetSettings; + const initialDatasetSettings: Mutable = + _.cloneDeep(originalDatasetSettings); + + if (originalDatasetSettings.nativelyRenderedLayerName) { + const isNativelyRenderedNamePresent = + dataset.dataSource.dataLayers.some( + (layer) => + layer.name === originalDatasetSettings.nativelyRenderedLayerName || + (layer.category === "segmentation" && + layer.fallbackLayer === originalDatasetSettings.nativelyRenderedLayerName), + ) || + annotation?.annotationLayers.some( + (layer) => layer.name === originalDatasetSettings.nativelyRenderedLayerName, + ); + if (!isNativelyRenderedNamePresent) { + initialDatasetSettings.nativelyRenderedLayerName = null; + } } - const initialDatasetSettings: DatasetConfiguration = _.cloneDeep(originalDatasetSettings); + if (!annotation) { + return initialDatasetSettings; + } if (annotation.viewConfiguration) { // The annotation already contains a viewConfiguration. Merge that into the diff --git a/frontend/javascripts/oxalis/store.ts b/frontend/javascripts/oxalis/store.ts index 61762134fd8..b3f4dc36f90 100644 --- a/frontend/javascripts/oxalis/store.ts +++ b/frontend/javascripts/oxalis/store.ts @@ -335,11 +335,14 @@ export type DatasetConfiguration = { // that name (or id) should be rendered without any transforms. // This means, that all other layers should be transformed so that // they still correlated with each other. + // If other layers have the same transformation they will also be rendered + // natively as their transform and the inverse transform of the nativelyRenderedLayer + // layer cancel each other out. // If nativelyRenderedLayerName is null, all layers are rendered // as their transforms property signal it. - // Currently, the skeleton layer does not have transforms as a stored - // property. So, to render the skeleton layer natively, nativelyRenderedLayerName - // can be set to null. + // Currently, skeleton layers and volume layers without fallback do not have transforms + // as a stored property. So, to render the skeleton layer natively, + // nativelyRenderedLayerName can be set to null. readonly nativelyRenderedLayerName: string | null; }; diff --git a/frontend/javascripts/oxalis/view/left-border-tabs/layer_settings_tab.tsx b/frontend/javascripts/oxalis/view/left-border-tabs/layer_settings_tab.tsx index 29fb787852b..af67d9d15ee 100644 --- a/frontend/javascripts/oxalis/view/left-border-tabs/layer_settings_tab.tsx +++ b/frontend/javascripts/oxalis/view/left-border-tabs/layer_settings_tab.tsx @@ -60,11 +60,8 @@ import { isColorLayer as getIsColorLayer, getLayerByName, getMagInfo, - getTransformsForLayerOrNull, getWidestMags, getLayerBoundingBox, - getTransformsForLayer, - hasDatasetTransforms, } from "oxalis/model/accessors/dataset_accessor"; import { getMaxZoomValueForMag, getPosition } from "oxalis/model/accessors/flycam_accessor"; import { @@ -118,10 +115,6 @@ import DownsampleVolumeModal from "./modals/downsample_volume_modal"; import Histogram, { isHistogramSupported } from "./histogram_view"; import MappingSettingsView from "./mapping_settings_view"; import { confirmAsync } from "../../../dashboard/dataset/helper_components"; -import { - invertTransform, - transformPointUnscaled, -} from "oxalis/model/helpers/transformation_helpers"; import FastTooltip from "components/fast_tooltip"; import { SortableContext, useSortable, verticalListSortingStrategy } from "@dnd-kit/sortable"; import { DndContext, type DragEndEvent } from "@dnd-kit/core"; @@ -131,6 +124,13 @@ import { getDefaultLayerViewConfiguration, } from "types/schemas/dataset_view_configuration.schema"; import defaultState from "oxalis/default_state"; +import { + getTransformsForLayerOrNull, + hasDatasetTransforms, + isIdentityTransform, + getTransformsForLayer, + getNewPositionAndZoomChangeFromTransformationChange, +} from "oxalis/model/accessors/dataset_layer_transformation_accessor"; type DatasetSettingsProps = { userConfiguration: UserConfiguration; @@ -219,10 +219,14 @@ function TransformationIcon({ layer }: { layer: APIDataLayer | APISkeletonLayer state.datasetConfiguration.nativelyRenderedLayerName, ), ); + const hasLayerTransformsConfigured = useSelector( + (state: OxalisState) => getTransformsForLayerOrNull(state.dataset, layer, null) != null, + ); const showIcon = useSelector((state: OxalisState) => hasDatasetTransforms(state.dataset)); if (!showIcon) { return null; } + const isRenderedNatively = transform == null || isIdentityTransform(transform); const typeToLabel = { affine: "an affine", @@ -235,69 +239,63 @@ function TransformationIcon({ layer }: { layer: APIDataLayer | APISkeletonLayer affine: "icon-affine-transformation.svg", }; + // Cannot toggle transforms on for a layer that has no transforms. + // Layers that cannot have transformations like skeleton layer and volume tracing layers without fallback + // automatically copy to the dataset transformation if all other layers have the same transformation. + const isDisabled = isRenderedNatively && !hasLayerTransformsConfigured; + const toggleLayerTransforms = () => { const state = Store.getState(); - const { nativelyRenderedLayerName } = state.datasetConfiguration; - if ( - layer.category === "skeleton" - ? nativelyRenderedLayerName == null - : nativelyRenderedLayerName === layer.name - ) { - return; - } - // Transform current position using the inverse transform - // so that the user will still look at the same data location. - const currentPosition = getPosition(state.flycam); - const currentTransforms = getTransformsForLayer( + // Get getOriginalTransformsForLayerOrNull is not used to handle layers that do not support configuring a transformation. + const nextNativelyRenderedLayerName = isRenderedNatively ? null : layer.name; + const activeTransformation = getTransformsForLayer( state.dataset, layer, state.datasetConfiguration.nativelyRenderedLayerName, ); - const invertedTransform = invertTransform(currentTransforms); - const newPosition = transformPointUnscaled(invertedTransform)(currentPosition); - - // Also transform a reference coordinate to determine how the scaling - // changed. Then, adapt the zoom accordingly. - const referenceOffset: Vector3 = [10, 10, 10]; - const secondPosition = V3.add(currentPosition, referenceOffset, [0, 0, 0]); - const newSecondPosition = transformPointUnscaled(invertedTransform)(secondPosition); - - const scaleChange = _.mean( - // Only consider XY for now to determine the zoom change (by slicing from 0 to 2) - V3.abs(V3.divide3(V3.sub(newPosition, newSecondPosition), referenceOffset)).slice(0, 2), + const nextTransform = getTransformsForLayer( + state.dataset, + layer, + nextNativelyRenderedLayerName, + ); + const { scaleChange, newPosition } = getNewPositionAndZoomChangeFromTransformationChange( + activeTransformation, + nextTransform, + state, ); dispatch( - updateDatasetSettingAction( - "nativelyRenderedLayerName", - layer.category === "skeleton" ? null : layer.name, - ), + updateDatasetSettingAction("nativelyRenderedLayerName", nextNativelyRenderedLayerName), ); dispatch(setPositionAction(newPosition)); dispatch(setZoomStepAction(state.flycam.zoomStep * scaleChange)); }; + const style = { + width: 14, + height: 14, + marginBottom: 4, + marginRight: 5, + ...(isDisabled + ? { cursor: "not-allowed", opacity: "0.5" } + : { cursor: "pointer", opacity: "1.0" }), + }; + return (
Transformed Layer Icon {} : toggleLayerTransforms} />
@@ -334,8 +332,8 @@ function LayerInfoIconWithTooltip({ Min - {layer.boundingBox.topLeft[0]} - {layer.boundingBox.topLeft[1]} + {layer.boundingBox.topLeft[0]} + {layer.boundingBox.topLeft[1]} {layer.boundingBox.topLeft[2]} @@ -779,8 +777,8 @@ class DatasetSettings extends React.PureComponent { placement="left" > } - hoveredIcon={} + icon={} + hoveredIcon={} onClick={() => { this.setState({ isAddVolumeLayerModalVisible: true, @@ -1167,7 +1165,7 @@ class DatasetSettings extends React.PureComponent { const readableName = "Skeleton"; const skeletonTracing = enforceSkeletonTracing(tracing); const isOnlyAnnotationLayer = tracing.annotationLayers.length === 1; - const { showSkeletons } = skeletonTracing; + const { showSkeletons, tracingId } = skeletonTracing; const activeNodeRadius = getActiveNode(skeletonTracing)?.radius ?? 0; return ( @@ -1217,7 +1215,7 @@ class DatasetSettings extends React.PureComponent { paddingRight: 1, }} > - + {!isOnlyAnnotationLayer ? this.getDeleteAnnotationLayerButton(readableName) : null} diff --git a/frontend/javascripts/test/reducers/flycam_reducer.spec.ts b/frontend/javascripts/test/reducers/flycam_reducer.spec.ts index 546537b97b0..6b71ed3f84d 100644 --- a/frontend/javascripts/test/reducers/flycam_reducer.spec.ts +++ b/frontend/javascripts/test/reducers/flycam_reducer.spec.ts @@ -36,7 +36,7 @@ const initialState = { flycam: { zoomStep: 2, additionalCoordinates: [], - currentMatrix: M4x4.identity, + currentMatrix: M4x4.identity(), spaceDirectionOrtho: [1, 1, 1], }, temporaryConfiguration: { diff --git a/frontend/javascripts/types/api_flow_types.ts b/frontend/javascripts/types/api_flow_types.ts index 958651b32ec..078971f4936 100644 --- a/frontend/javascripts/types/api_flow_types.ts +++ b/frontend/javascripts/types/api_flow_types.ts @@ -19,9 +19,9 @@ import type { Point3, ColorObject, LOG_LEVELS, - Vector4, TreeType, UnitLong, + NestedMatrix4, } from "oxalis/constants"; import type { PricingPlanEnum } from "admin/organization/pricing_plan_utils"; import type { EmptyObject } from "./globals"; @@ -60,15 +60,17 @@ export type ServerAdditionalAxis = { name: string; }; -export type CoordinateTransformation = - | { - type: "affine"; - matrix: [Vector4, Vector4, Vector4, Vector4]; - } - | { - type: "thin_plate_spline"; - correspondences: { source: Vector3[]; target: Vector3[] }; - }; +export type AffineTransformation = { + type: "affine"; + matrix: NestedMatrix4; // Stored in row major order. +}; + +export type ThinPlateSplineTransformation = { + type: "thin_plate_spline"; + correspondences: { source: Vector3[]; target: Vector3[] }; +}; + +export type CoordinateTransformation = AffineTransformation | ThinPlateSplineTransformation; type APIDataLayerBase = { readonly name: string; readonly boundingBox: BoundingBoxObject; @@ -94,8 +96,8 @@ export type APISegmentationLayer = APIDataLayerBase & { export type APIDataLayer = APIColorLayer | APISegmentationLayer; // Only used in rare cases to generalize over actual data layers and -// a skeleton layer. -export type APISkeletonLayer = { category: "skeleton" }; +// a skeleton layer. The name should be the skeleton tracing id to very likely ensure it is unique. +export type APISkeletonLayer = { category: "skeleton"; name: string }; export type LayerLink = { datasetId: string; diff --git a/frontend/javascripts/types/globals.d.ts b/frontend/javascripts/types/globals.d.ts index c19f5fcd353..0cec5eb30a5 100644 --- a/frontend/javascripts/types/globals.d.ts +++ b/frontend/javascripts/types/globals.d.ts @@ -16,3 +16,6 @@ export type ArbitraryObject = Record; export type ArbitraryFunction = (...args: Array) => any; export type Comparator = (arg0: T, arg1: T) => -1 | 0 | 1; export type ArrayElement = A extends readonly (infer T)[] ? T : never; +export type Mutable = { + -readonly [K in keyof T]: T[K]; +}; diff --git a/frontend/javascripts/types/schemas/dataset_view_configuration.schema.ts b/frontend/javascripts/types/schemas/dataset_view_configuration.schema.ts index 46d09bc45a7..02404c8b656 100644 --- a/frontend/javascripts/types/schemas/dataset_view_configuration.schema.ts +++ b/frontend/javascripts/types/schemas/dataset_view_configuration.schema.ts @@ -149,6 +149,9 @@ export const datasetViewConfiguration = { type: "string", }, }, + nativelyRenderedLayerName: { + type: "string", + }, }; export default { $schema: "http://json-schema.org/draft-06/schema#",