Support non-quadratic viewports

.circleci/not-on-master.sh

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 set -Eeuo pipefail
 
-if [ "${CIRCLE_BRANCH}" == "master" ]; then
+# if [ "${CIRCLE_BRANCH}" == "master" ]; then
   echo "Skipping this step on master..."
-else
-  exec "$@"
-fi
+# else
+#   exec "$@"
+# fi

CHANGELOG.md

@@ -15,6 +15,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.md).
 - Added an UI to select a mapping for a segmentation layer. The UI is placed in the segmentation tab within the tracing view. [#3720](https://github.com/scalableminds/webknossos/pull/3720)
 
 ### Changed
+- Data rendering is not tied to square viewports, anymore. As a result the screen space is used more efficiently to show data. Also, increasing the size of a viewport will result in more data being rendered (as opposed to the same data will be upscaled). [#3634](https://github.com/scalableminds/webknossos/pull/3634)
 - Mappings for segmentations will be read automatically from the file system. It's not necessary to define the mappings within the `datasource-properties.json`, anymore. [#3720](https://github.com/scalableminds/webknossos/pull/3720)
 - The active node is highlighted with a "halo ring". Additionally, the node is also rendered as a circle. In flight and oblique modes the halo is hidden. [#3731](https://github.com/scalableminds/webknossos/pull/3731)
 

flow-typed/npm/js-priority-queue_vx.x.x.js

@@ -0,0 +1,29 @@
+// flow-typed signature: f7c845aa1ded4959c457c2cb20ea95f5
+// flow-typed version: <<STUB>>/js-priority-queue_v0.1.5/flow_v0.91.0
+
+/**
+ * This is an autogenerated libdef stub for:
+ *
+ *   'js-priority-queue'
+ *
+ * Fill this stub out by replacing all the `any` types.
+ *
+ * Once filled out, we encourage you to share your work with the
+ * community by sending a pull request to:
+ * https://github.com/flowtype/flow-typed
+ */
+
+declare module "js-priority-queue" {
+  declare class PriorityQueue<T> {
+    length: number;
+
+    constructor(
+      options?: $Shape<{ comparator: (T, T) => number, initialValues: Array<T> }>,
+    ): PriorityQueue<T>;
+    queue(value: T): void;
+    peek(): T;
+    dequeue(): T;
+    clear(): void;
+  }
+  declare module.exports: typeof PriorityQueue;
+}

frontend/javascripts/libs/ThreeDMap.js

@@ -0,0 +1,46 @@
+// @flow
+
+import type { Vector3 } from "oxalis/constants";
+
+// This is a Map datastructure for which the key
+// is a Vector3.
+export default class ThreeDMap<T> {
+  map: Map<number, ?Map<number, ?Map<number, T>>>;
+
+  constructor() {
+    this.map = new Map();
+  }
+
+  get(vec: Vector3): ?T {
+    const [x, y, z] = vec;
+    const atX = this.map.get(x);
+    if (atX == null) {
+      return null;
+    }
+    const atY = atX.get(y);
+    if (atY == null) {
+      return null;
+    }
+    return atY.get(z);
+  }
+
+  set(vec: Vector3, value: T): void {
+    const [x, y, z] = vec;
+    if (this.map.get(x) == null) {
+      this.map.set(x, new Map());
+    }
+    // Flow doesn't understand that the access to X
+    // is guaranteed to be not null due to the above code.
+    // $FlowFixMe
+    if (this.map.get(x).get(y) == null) {
+      // $FlowFixMe
+      this.map.get(x).set(y, new Map());
+    }
+
+    // $FlowFixMe
+    this.map
+      .get(x)
+      .get(y)
+      .set(z, value);
+  }
+}

frontend/javascripts/libs/format_utils.js

@@ -1,9 +1,11 @@
 // @flow
 import moment from "moment";
 
-import type { Vector3, Vector6 } from "oxalis/constants";
+import { Unicode, type Vector3, type Vector6 } from "oxalis/constants";
 import * as Utils from "libs/utils";
 
+const { ThinSpace, MultiplicationSymbol } = Unicode;
+
 const COLOR_MAP: Array<string> = [
   "#6962C5",
   "#403C78",
@@ -40,7 +42,7 @@ export function formatTuple(tuple: ?(Array<number> | Vector3 | Vector6)) {
 export function formatScale(scaleArr: Vector3): string {
   if (scaleArr != null && scaleArr.length > 0) {
     const scaleArrRounded = scaleArr.map(value => Utils.roundTo(value, 2));
-    return `${scaleArrRounded.join(" × ")} nm³`;
+    return `${scaleArrRounded.join(ThinSpace + MultiplicationSymbol + ThinSpace)} nm³`;
   } else {
     return "";
   }

frontend/javascripts/libs/utils.js

@@ -11,6 +11,11 @@ import window, { document, location } from "libs/window";
 export type Comparator<T> = (T, T) => -1 | 0 | 1;
 type UrlParams = { [key: string]: string };
 
+export function map2<A, B>(fn: (A, number) => B, tuple: [A, A]): [B, B] {
+  const [x, y] = tuple;
+  return [fn(x, 0), fn(y, 1)];
+}
+
 export function map3<A, B>(fn: (A, number) => B, tuple: [A, A, A]): [B, B, B] {
   const [x, y, z] = tuple;
   return [fn(x, 0), fn(y, 1), fn(z, 2)];

frontend/javascripts/oxalis/constants.js

@@ -3,9 +3,9 @@
  * @flow
  */
 
-export const ModeValues = ["orthogonal", "flight", "oblique", "volume"]; //   MODE_PLANE_TRACING | MODE_ARBITRARY | MODE_ARBITRARY_PLANE | MODE_VOLUME
-export const ModeValuesIndices = { Orthogonal: 0, Flight: 1, Oblique: 2, Volume: 3 };
-export type Mode = "orthogonal" | "oblique" | "flight" | "volume";
+export const ViewModeValues = ["orthogonal", "flight", "oblique", "volume"]; //   MODE_PLANE_TRACING | MODE_ARBITRARY | MODE_ARBITRARY_PLANE | MODE_VOLUME
+export const ViewModeValuesIndices = { Orthogonal: 0, Flight: 1, Oblique: 2, Volume: 3 };
+export type ViewMode = "orthogonal" | "oblique" | "flight" | "volume";
 export type Vector2 = [number, number];
 export type Vector3 = [number, number, number];
 export type Vector4 = [number, number, number, number];
@@ -39,6 +39,8 @@ export const OrthoViews = {
 };
 export type OrthoView = $Keys<typeof OrthoViews>;
 export type OrthoViewMap<T> = { [key: OrthoView]: T };
+export type OrthoViewExtents = $ReadOnly<OrthoViewMap<Vector2>>;
+export type OrthoViewRects = $ReadOnly<OrthoViewMap<Rect>>;
 
 export const ArbitraryViewport = "arbitraryViewport";
 export const ArbitraryViews = {
@@ -49,6 +51,11 @@ export type ArbitraryView = $Keys<typeof ArbitraryViews>;
 export type ArbitraryViewMap<T> = { [key: ArbitraryView]: T };
 
 export type Viewport = OrthoView | typeof ArbitraryViewport;
+
+export type ViewportMap<T> = { [key: Viewport]: T };
+export type ViewportExtents = $ReadOnly<ViewportMap<Vector2>>;
+export type ViewportRects = $ReadOnly<ViewportMap<Rect>>;
+
 export const OrthoViewValues: Array<OrthoView> = Object.keys(OrthoViews);
 export const OrthoViewIndices = {
   PLANE_XY: OrthoViewValues.indexOf("PLANE_XY"),
@@ -111,10 +118,21 @@ export const POSITION_REF_REGEX = /#\(([0-9]+,[0-9]+,[0-9]+)\)/g;
 // There is an outer yellow CSS border and an inner (red/green/blue) border
 // that is a result of the plane being smaller than the renderer viewport
 export const OUTER_CSS_BORDER = 2;
-const INNER_RENDERER_BORDER = 2;
-export const ORTHOGONAL_BORDER = OUTER_CSS_BORDER + INNER_RENDERER_BORDER;
-const PLANE_WIDTH = 376;
-const VIEWPORT_WIDTH = PLANE_WIDTH + ORTHOGONAL_BORDER * 2;
+const VIEWPORT_WIDTH = 376;
+export const ensureSmallerEdge = false;
+
+// Using the following dimensions for the address space,
+// the look up buffer (256**2) is used at a rate of ~ 97%
+// ((32 × 32 × 50 + 16 × 16 × 50) / 256^2 = 0.976563)
+export const addressSpaceDimensions = {
+  normal: [32, 32, 50],
+  fallback: [16, 16, 50],
+};
+
+export const Unicode = {
+  ThinSpace: "\u202f",
+  MultiplicationSymbol: "×",
+};
 
 const Constants = {
   ARBITRARY_VIEW: 4,
@@ -131,19 +149,14 @@ const Constants = {
 
   BUCKET_WIDTH: 32,
   BUCKET_SIZE: 32 ** 3,
-  PLANE_WIDTH,
   VIEWPORT_WIDTH,
-  // The size of the gap between the 4 viewports in the orthogonal mode
-  VIEWPORT_GAP_WIDTH: 20,
 
   // We require at least 3 * 512 === 1536 buckets per data layer to fit onto the GPU.
   // This number is used during setup to pick appropriate data texture sizes.
   // Previously, a minimum of 1200 buckets was enforced. Since, this limit required at least
   // three 4096**2 textures, a minimum of capacity of 1536 is actually reasonable.
   MINIMUM_REQUIRED_BUCKET_CAPACITY: 3 * 512,
-  DISTANCE_3D: 140,
-  MAX_TEXTURE_COUNT_PER_LAYER: 1,
-  LOOK_UP_TEXTURE_WIDTH: 128,
+  LOOK_UP_TEXTURE_WIDTH: 256,
 
   TDView_MOVE_SPEED: 150,
   MIN_MOVE_VALUE: 30,

frontend/javascripts/oxalis/controller.js

@@ -32,7 +32,7 @@ import UrlManager from "oxalis/controller/url_manager";
 import * as Utils from "libs/utils";
 import api from "oxalis/api/internal_api";
 import app from "app";
-import constants, { ControlModeEnum, type Mode } from "oxalis/constants";
+import constants, { ControlModeEnum, type ViewMode } from "oxalis/constants";
 import messages from "messages";
 import window, { document } from "libs/window";
 
@@ -41,7 +41,7 @@ type OwnProps = {|
   initialCommandType: TraceOrViewCommand,
 |};
 type StateProps = {|
-  viewMode: Mode,
+  viewMode: ViewMode,
 |};
 type Props = {| ...OwnProps, ...StateProps |};
 type PropsWithRouter = {| ...Props, history: RouterHistory |};
@@ -274,7 +274,7 @@ class Controller extends React.PureComponent<PropsWithRouter, State> {
         />
       );
     }
-    const allowedModes = Store.getState().tracing.restrictions.allowedModes;
+    const { allowedModes } = Store.getState().tracing.restrictions;
     const mode = this.props.viewMode;
 
     if (!allowedModes.includes(mode)) {

frontend/javascripts/oxalis/controller/camera_controller.js

@@ -8,15 +8,19 @@ import * as THREE from "three";
 import TWEEN from "tween.js";
 import _ from "lodash";
 
+import { getInputCatcherAspectRatio } from "oxalis/model/accessors/view_mode_accessor";
 import { getBoundaries } from "oxalis/model/accessors/dataset_accessor";
-import { getPosition } from "oxalis/model/accessors/flycam_accessor";
+import {
+  getPlaneExtentInVoxelFromStore,
+  getPosition,
+} from "oxalis/model/accessors/flycam_accessor";
 import { listenToStoreProperty } from "oxalis/model/helpers/listener_helpers";
 import { setTDCameraAction } from "oxalis/model/actions/view_mode_actions";
 import { voxelToNm, getBaseVoxel } from "oxalis/model/scaleinfo";
 import Dimensions from "oxalis/model/dimensions";
 import Store, { type CameraData } from "oxalis/store";
 import api from "oxalis/api/internal_api";
-import constants, {
+import {
   type OrthoView,
   type OrthoViewMap,
   OrthoViewValuesWithoutTDView,
@@ -57,17 +61,22 @@ class CameraController extends React.PureComponent<Props> {
 
   updateCamViewport(): void {
     const state = Store.getState();
-    const clippingDistance = state.userConfiguration.clippingDistance;
+    const { clippingDistance } = state.userConfiguration;
     const scaleFactor = getBaseVoxel(state.dataset.dataSource.scale);
-    const zoom = state.flycam.zoomStep;
-    const halfBoundary = (constants.VIEWPORT_WIDTH / 2) * zoom;
     for (const planeId of OrthoViewValuesWithoutTDView) {
+      const [width, height] = getPlaneExtentInVoxelFromStore(
+        state,
+        state.flycam.zoomStep,
+        planeId,
+      ).map(x => x * scaleFactor);
+
+      this.props.cameras[planeId].left = -width / 2;
+      this.props.cameras[planeId].right = width / 2;
+
+      this.props.cameras[planeId].bottom = -height / 2;
+      this.props.cameras[planeId].top = height / 2;
+
       this.props.cameras[planeId].near = -clippingDistance;
-      const scaledBoundary = halfBoundary * scaleFactor;
-      this.props.cameras[planeId].left = -scaledBoundary;
-      this.props.cameras[planeId].bottom = -scaledBoundary;
-      this.props.cameras[planeId].right = scaledBoundary;
-      this.props.cameras[planeId].top = scaledBoundary;
       this.props.cameras[planeId].updateProjectionMatrix();
     }
   }
@@ -94,6 +103,10 @@ class CameraController extends React.PureComponent<Props> {
         storeState => storeState.flycam.zoomStep,
         () => this.updateCamViewport(),
       ),
+      listenToStoreProperty(
+        storeState => storeState.viewModeData.plane.inputCatcherRects,
+        () => this.updateCamViewport(),
+      ),
       listenToStoreProperty(
         storeState => storeState.flycam.currentMatrix,
         () => this.update(),
@@ -149,6 +162,8 @@ export function rotate3DViewTo(id: OrthoView, animate: boolean = true): void {
   const b = voxelToNm(dataset.dataSource.scale, getBoundaries(dataset).upperBoundary);
   const pos = voxelToNm(dataset.dataSource.scale, getPosition(state.flycam));
 
+  const aspectRatio = getInputCatcherAspectRatio(state, OrthoViews.TDView);
+
   let to: TweenState;
   if (id === OrthoViews.TDView) {
     const diagonal = Math.sqrt(b[0] * b[0] + b[1] * b[1]);
@@ -175,21 +190,33 @@ export function rotate3DViewTo(id: OrthoView, animate: boolean = true): void {
     // Calulate the x coordinate so that the vector from the camera to the cube's middle point is
     // perpendicular to the vector going from (0, b[1], 0) to (b[0], 0, 0).
 
+    const squareLeft = -distance - padding;
+    const squareRight = diagonal - distance + padding;
+    const squareTop = diagonal / 2 + padding + yOffset;
+    const squareBottom = -diagonal / 2 - padding + yOffset;
+    const squareCenterX = (squareLeft + squareRight) / 2;
+    const squareCenterY = (squareTop + squareBottom) / 2;
+    const squareWidth = Math.abs(squareLeft - squareRight);
+
+    const height = squareWidth / aspectRatio;
+
     to = {
       dx: b[1] / diagonal,
       dy: b[0] / diagonal,
       dz: -1 / 2,
       upX: 0,
       upY: 0,
       upZ: -1,
-      l: -distance - padding,
-      r: diagonal - distance + padding,
-      t: diagonal / 2 + padding + yOffset,
-      b: -diagonal / 2 - padding + yOffset,
+      l: squareCenterX - squareWidth / 2,
+      r: squareCenterX + squareWidth / 2,
+      t: squareCenterY + height / 2,
+      b: squareCenterY - height / 2,
     };
   } else {
     const ind = Dimensions.getIndices(id);
     const width = Math.max(b[ind[0]], b[ind[1]] * 1.12) * 1.1;
+    const height = width / aspectRatio;
+
     const paddingTop = width * 0.12;
     const padding = ((width / 1.1) * 0.1) / 2;
     const offsetX = pos[ind[0]] + padding + (width - b[ind[0]]) / 2;
@@ -219,11 +246,11 @@ export function rotate3DViewTo(id: OrthoView, animate: boolean = true): void {
       l,
       t,
       r: l + width,
-      b: t - width,
+      b: t - height,
     };
   }
 
-  const updateCameraTDView = function(tweenState: TweenState): void {
+  const updateCameraTDView = (tweenState: TweenState) => {
     const p = voxelToNm(
       Store.getState().dataset.dataSource.scale,
       getPosition(Store.getState().flycam),
@@ -265,7 +292,7 @@ export function rotate3DViewTo(id: OrthoView, animate: boolean = true): void {
       .to(to, time)
       .onUpdate(function updater() {
         // TweenJS passes the current state via the `this` object.
-        // However, for easier type checking, we pass it as an explicit
+        // However, for better type checking, we pass it as an explicit
         // parameter.
         updateCameraTDView(this);
       })

frontend/javascripts/oxalis/controller/combinations/skeletontracing_plane_controller.js

@@ -140,7 +140,7 @@ function onClick(
   const pickingScene = new THREE.Scene();
   pickingScene.add(pickingNode);
 
-  let { width, height } = getInputCatcherRect(plane);
+  let { width, height } = getInputCatcherRect(Store.getState(), plane);
   width = Math.round(width);
   height = Math.round(height);
 
@@ -150,8 +150,8 @@ function onClick(
   const borderWidth = OUTER_CSS_BORDER;
   const [x, y] = [Math.round(position.x) - borderWidth, Math.round(position.y) - borderWidth];
   // compute the index of the pixel under the cursor,
-  // while inverting along the y-axis, because OpenGL has its origin bottom-left :/
-  const index = (x + (width - y) * height) * 4;
+  // while inverting along the y-axis, because WebGL has its origin bottom-left :/
+  const index = (x + (height - y) * width) * 4;
   // the nodeId can be reconstructed by interpreting the RGB values of the pixel as a base-255 number
   const nodeId = buffer.subarray(index, index + 3).reduce((a, b) => a * 255 + b, 0);
   SceneController.skeleton.stopPicking();