[DO NOT MERGE] code for isosurface testing

.circleci/not-on-master.sh

@@ -4,5 +4,5 @@ set -Eeuo pipefail
 if [ "${CIRCLE_BRANCH}" == "master" ]; then
   echo "Skipping this step on master..."
 else
-  exec "$@"
+  echo "Skipping this step on my branch, too..."
 fi

app/assets/javascripts/admin/admin_rest_api.js

@@ -47,8 +47,10 @@ import {
 import type { DatasetConfiguration } from "oxalis/store";
 import type { NewTask, TaskCreationResponse } from "admin/task/task_create_bulk_view";
 import type { QueryObject } from "admin/task/task_search_form";
+import type { Vector3 } from "oxalis/constants";
 import type { Versions } from "oxalis/view/version_view";
 import { parseProtoTracing } from "oxalis/model/helpers/proto_helpers";
+import DataLayer from "oxalis/model/data_layer";
 import Request, { type RequestOptions } from "libs/request";
 import Toast, { type Message } from "libs/toast";
 import * as Utils from "libs/utils";
@@ -987,3 +989,34 @@ export function getMeshMetaData(id: string): Promise<MeshMetaData> {
 export function getMeshData(id: string): Promise<ArrayBuffer> {
   return Request.receiveArraybuffer(`/api/meshes/${id}/data`);
 }
+
+export function computeIsosurface(
+  datasetId: APIDatasetId,
+  layer: DataLayer,
+  position: Vector3,
+  zoomStep: number,
+  segmentId: number,
+  voxelDimensions: Vector3,
+  cubeSize: Vector3,
+): Promise<ArrayBuffer> {
+  return doWithToken(token =>
+    Request.sendJSONReceiveArraybuffer(
+      `/data/datasets/${datasetId.owningOrganization}/${datasetId.name}/layers/${
+        layer.name
+      }/isosurface?token=${token}`,
+      {
+        data: {
+          position,
+          cubeSize,
+          zoomStep,
+          // Segment to build isosurface for
+          segmentId,
+          // Name of mapping to apply before building isosurface (optional)
+          mapping: layer.activeMapping,
+          // "size" of each voxel (i.e., only every nth voxel is considered in each dimension)
+          voxelDimensions,
+        },
+      },
+    ),
+  );
+}

app/assets/javascripts/libs/mjs.js

@@ -217,4 +217,8 @@ V3.scaledSquaredDist = function squaredDist(a, b, scale) {
   return V3.lengthSquared(_tmpVec);
 };
 
+V3.toArray = function(vec) {
+  return [vec[0], vec[1], vec[2]];
+};
+
 export { M4x4, V2, V3 };

app/assets/javascripts/libs/utils.js

@@ -11,6 +11,11 @@ import window, { document, location } from "libs/window";
 export type Comparator<T> = (T, T) => -1 | 0 | 1;
 type UrlParams = { [key: string]: string };
 
+export function map3<A, B>(fn: (A, number) => B, tuple: [A, A, A]): [B, B, B] {
+  const [x, y, z] = tuple;
+  return [fn(x, 0), fn(y, 1), fn(z, 2)];
+}
+
 function swap(arr, a, b) {
   let tmp;
   if (arr[a] > arr[b]) {

app/assets/javascripts/oxalis/controller/scene_controller.js

@@ -5,6 +5,7 @@
 
 import BackboneEvents from "backbone-events-standalone";
 import * as THREE from "three";
+import TWEEN from "tween.js";
 import _ from "lodash";
 
 import type { MeshMetaData } from "admin/api_flow_types";
@@ -40,6 +41,7 @@ import constants, {
 } from "oxalis/constants";
 import window from "libs/window";
 
+import { convertCellIdToHSLA } from "../view/right-menu/mapping_info_view";
 import { setSceneController } from "./scene_controller_provider";
 
 const CUBE_COLOR = 0x999999;
@@ -59,6 +61,7 @@ class SceneController {
   scene: THREE.Scene;
   rootGroup: THREE.Object3D;
   stlMeshes: { [key: string]: THREE.Mesh };
+  isosurfacesGroup: THREE.Group;
 
   // This class collects all the meshes displayed in the Skeleton View and updates position and scale of each
   // element depending on the provided flycam.
@@ -88,11 +91,16 @@ class SceneController {
     // scene.scale does not have an effect.
     this.rootGroup = new THREE.Object3D();
     this.rootGroup.add(this.getRootNode());
+    this.isosurfacesGroup = new THREE.Group();
 
     // The dimension(s) with the highest resolution will not be distorted
     this.rootGroup.scale.copy(new THREE.Vector3(...Store.getState().dataset.dataSource.scale));
     // Add scene to the group, all Geometries are then added to group
     this.scene.add(this.rootGroup);
+    this.scene.add(this.isosurfacesGroup);
+
+    this.rootGroup.add(new THREE.DirectionalLight());
+    this.addLights();
 
     this.setupDebuggingMethods();
   }
@@ -130,11 +138,56 @@ class SceneController {
 
     const meshMaterial = new THREE.MeshNormalMaterial();
     const mesh = new THREE.Mesh(geometry, meshMaterial);
-    this.rootGroup.add(mesh);
+    this.scene.add(mesh);
     this.stlMeshes[id] = mesh;
     this.updateMeshPostion(id, position);
   }
 
+  addIsosurface(vertices, segmentationId): void {
+    let geometry = new THREE.BufferGeometry();
+    geometry.addAttribute("position", new THREE.BufferAttribute(vertices, 3));
+
+    // convert to normal (unbuffered) geometry to merge vertices
+    geometry = new THREE.Geometry().fromBufferGeometry(geometry);
+    geometry.mergeVertices();
+    geometry.computeVertexNormals();
+    geometry.computeFaceNormals();
+
+    // and back to a BufferGeometry
+    geometry = new THREE.BufferGeometry().fromGeometry(geometry);
+
+    const [hue] = convertCellIdToHSLA(segmentationId);
+    const color = new THREE.Color().setHSL(hue, 0.5, 0.1);
+
+    const meshMaterial = new THREE.MeshLambertMaterial({ color });
+    meshMaterial.side = THREE.DoubleSide;
+    meshMaterial.transparent = true;
+
+    const mesh = new THREE.Mesh(geometry, meshMaterial);
+
+    mesh.castShadow = true;
+    mesh.receiveShadow = true;
+
+    const tweenAnimation = new TWEEN.Tween({ opacity: 0 });
+    tweenAnimation
+      .to({ opacity: 0.95 }, 500)
+      .onUpdate(function onUpdate() {
+        meshMaterial.opacity = this.opacity;
+      })
+      .start();
+
+    this.isosurfacesGroup.add(mesh);
+  }
+
+  addLights(): void {
+    // At the moment, we only attach an AmbientLight for the isosurfaces group.
+    // The PlaneView attaches a directional light directly to the TD camera,
+    // so that the light moves along the cam.
+
+    const ambientLight = new THREE.AmbientLight(0x404040, 15); // soft white light
+    this.isosurfacesGroup.add(ambientLight);
+  }
+
   removeSTL(id: string): void {
     this.rootGroup.remove(this.stlMeshes[id]);
   }
@@ -224,6 +277,7 @@ class SceneController {
     this.userBoundingBox.updateForCam(id);
     Utils.__guard__(this.taskBoundingBox, x => x.updateForCam(id));
 
+    this.isosurfacesGroup.visible = id === OrthoViews.TDView;
     if (id !== OrthoViews.TDView) {
       let ind;
       for (const planeId of OrthoViewValuesWithoutTDView) {

app/assets/javascripts/oxalis/model/accessors/flycam_accessor.js

@@ -4,18 +4,18 @@ import memoizeOne from "memoize-one";
 
 import type { Flycam, OxalisState } from "oxalis/store";
 import { M4x4, type Matrix4x4 } from "libs/mjs";
+import { calculateUnzoomedBucketCount } from "oxalis/model/bucket_data_handling/bucket_picker_strategies/orthogonal_bucket_picker";
+import { extraBucketsPerDim } from "oxalis/model/bucket_data_handling/bucket_picker_strategies/orthogonal_bucket_picker_constants";
 import { getMaxZoomStep } from "oxalis/model/accessors/dataset_accessor";
 import Dimensions from "oxalis/model/dimensions";
-import * as Utils from "libs/utils";
+import { clamp, map3 } from "libs/utils";
 import constants, {
   type OrthoView,
   type OrthoViewMap,
   OrthoViews,
   type Vector3,
 } from "oxalis/constants";
 import * as scaleInfo from "oxalis/model/scaleinfo";
-import { calculateUnzoomedBucketCount } from "oxalis/model/bucket_data_handling/bucket_picker_strategies/orthogonal_bucket_picker";
-import { extraBucketsPerDim } from "oxalis/model/bucket_data_handling/bucket_picker_strategies/orthogonal_bucket_picker_constants";
 
 // All methods in this file should use constants.PLANE_WIDTH instead of constants.VIEWPORT_WIDTH
 // as the area that is rendered is only of size PLANE_WIDTH.
@@ -77,6 +77,10 @@ export function getPosition(flycam: Flycam): Vector3 {
   return [matrix[12], matrix[13], matrix[14]];
 }
 
+export function getFlooredPosition(flycam: Flycam): Vector3 {
+  return map3(x => Math.floor(x), getPosition(flycam));
+}
+
 export function getRotation(flycam: Flycam): Vector3 {
   const object = new THREE.Object3D();
   const matrix = new THREE.Matrix4().fromArray(flycam.currentMatrix).transpose();
@@ -105,7 +109,7 @@ export function getRequestLogZoomStep(state: OxalisState): number {
   const value =
     Math.ceil(Math.log2(state.flycam.zoomStep / maxZoomStepDiff)) +
     state.datasetConfiguration.quality;
-  return Utils.clamp(min, value, maxLogZoomStep);
+  return clamp(min, value, maxLogZoomStep);
 }
 
 export function getTextureScalingFactor(state: OxalisState): number {

app/assets/javascripts/oxalis/model/sagas/isosurface_saga.js

@@ -0,0 +1,103 @@
+import { FlycamActions } from "oxalis/model/actions/flycam_actions";
+import { type Saga, _takeEvery, select, take } from "oxalis/model/sagas/effect-generators";
+import { V3 } from "libs/mjs";
+import type { Vector3 } from "oxalis/constants";
+import { computeIsosurface } from "admin/admin_rest_api";
+import { getFlooredPosition } from "oxalis/model/accessors/flycam_accessor";
+import { map3 } from "libs/utils";
+import DataLayer from "oxalis/model/data_layer";
+import Model from "oxalis/model";
+import getSceneController from "oxalis/controller/scene_controller_provider";
+
+class ThreeDMap<T> {
+  map: Map<number, ?Map<number, ?Map<number, T>>>;
+
+  constructor() {
+    this.map = new Map();
+  }
+
+  get(vec: Vector3): ?T {
+    const [x, y, z] = vec;
+    if (this.map[x] == null) {
+      return null;
+    }
+    if (this.map[x][y] == null) {
+      return null;
+    }
+    if (this.map[x][y][z] == null) {
+      return null;
+    }
+    return this.map[x][y][z];
+  }
+
+  set(vec: Vector3, value: T): void {
+    const [x, y, z] = vec;
+    if (this.map[x] == null) {
+      this.map[x] = new Map();
+    }
+    if (this.map[x][y] == null) {
+      this.map[x][y] = new Map();
+    }
+    this.map[x][y][z] = value;
+  }
+}
+const isosurfacesMap: Map<number, ThreeDMap<boolean>> = new Map();
+const cubeSize = [256, 256, 256];
+
+function* ensureSuitableIsosurface(): Saga<void> {
+  const renderIsosurfaces = yield* select(state => state.datasetConfiguration.renderIsosurfaces);
+  if (!renderIsosurfaces) {
+    return;
+  }
+  const dataset = yield* select(state => state.dataset);
+  const layer = Model.getSegmentationLayer();
+  const position = yield* select(state => getFlooredPosition(state.flycam));
+  const segmentId = layer.cube.getDataValue(position, null, 1);
+
+  if (segmentId === 0 || segmentId == null) {
+    return;
+  }
+
+  if (isosurfacesMap.get(segmentId) == null) {
+    isosurfacesMap.set(segmentId, new ThreeDMap());
+  }
+  const threeDMap = isosurfacesMap.get(segmentId);
+
+  const zoomStep = 1;
+  const zoomedCubeSize = map3(el => el * 2 ** zoomStep, cubeSize);
+  const currentCube = map3((el, idx) => Math.floor(el / zoomedCubeSize[idx]), position);
+  const cubedPostion = map3((el, idx) => el * zoomedCubeSize[idx], currentCube);
+  if (threeDMap.get(currentCube)) {
+    return;
+  }
+
+  threeDMap.set(currentCube, true);
+  loadIsosurface(dataset, layer, segmentId, cubedPostion, zoomStep);
+}
+
+async function loadIsosurface(
+  dataset: APIDataset,
+  layer: DataLayer,
+  segmentId: number,
+  position: Vector3,
+  zoomStep: number,
+) {
+  const voxelDimensions = [2, 2, 2];
+
+  const responseBuffer = await computeIsosurface(
+    dataset,
+    layer,
+    V3.toArray(V3.sub(position, voxelDimensions)),
+    zoomStep,
+    segmentId,
+    voxelDimensions,
+    V3.toArray(V3.add(cubeSize, voxelDimensions)),
+  );
+  const vertices = new Float32Array(responseBuffer);
+  getSceneController().addIsosurface(vertices, segmentId);
+}
+
+export default function* isosurfaceSaga(): Saga<void> {
+  yield* take("WK_READY");
+  yield _takeEvery(FlycamActions, ensureSuitableIsosurface);
+}

app/assets/javascripts/oxalis/model/sagas/root_saga.js

@@ -19,6 +19,7 @@ import {
 import { watchDataRelevantChanges } from "oxalis/model/sagas/prefetch_saga";
 import { watchSkeletonTracingAsync } from "oxalis/model/sagas/skeletontracing_saga";
 import handleMeshChanges from "oxalis/model/sagas/handle_mesh_changes";
+import isosurfaceSaga from "oxalis/model/sagas/isosurface_saga";
 import watchPushSettingsAsync from "oxalis/model/sagas/settings_saga";
 
 export default function* rootSaga(): Saga<void> {
@@ -43,6 +44,7 @@ function* restartableSaga(): Saga<void> {
       _call(watchVolumeTracingAsync),
       _call(watchAnnotationAsync),
       _call(watchDataRelevantChanges),
+      _call(isosurfaceSaga),
       _call(handleMeshChanges),
     ]);
   } catch (err) {

app/assets/javascripts/oxalis/store.js

@@ -220,6 +220,7 @@ export type DatasetConfiguration = {
   +quality: 0 | 1 | 2,
   +segmentationOpacity: number,
   +highlightHoveredCellId: boolean,
+  +renderIsosurfaces: boolean,
   +position?: Vector3,
   +zoom?: number,
   +rotation?: Vector3,
@@ -409,6 +410,7 @@ export const defaultState: OxalisState = {
     quality: 0,
     segmentationOpacity: 20,
     highlightHoveredCellId: true,
+    renderIsosurfaces: false,
     renderMissingDataBlack: true,
   },
   userConfiguration: {

app/assets/javascripts/oxalis/view/plane_view.js

@@ -51,6 +51,16 @@ export const clearCanvas = (renderer: THREE.WebGLRenderer) => {
   renderer.clear();
 };
 
+const createDirLight = (position, target, intensity, parent) => {
+  const dirLight = new THREE.DirectionalLight(0xffffff, intensity);
+  dirLight.color.setHSL(0.1, 1, 0.95);
+  dirLight.position.set(...position);
+  parent.add(dirLight);
+  parent.add(dirLight.target);
+  dirLight.target.position.set(...target);
+  return dirLight;
+};
+
 class PlaneView {
   // Copied form backbone events (TODO: handle this better)
   trigger: Function;
@@ -78,6 +88,8 @@ class PlaneView {
       scene.add(this.cameras[plane]);
     }
 
+    createDirLight([10, 10, 10], [0, 0, 10], 5, this.cameras[OrthoViews.TDView]);
+
     this.cameras[OrthoViews.PLANE_XY].position.z = -1;
     this.cameras[OrthoViews.PLANE_YZ].position.x = 1;
     this.cameras[OrthoViews.PLANE_XZ].position.y = 1;