Render patterns in volume segments to increase distinguishability (#4730

)

* implement anti-aliased patterns for segmentations
* implement patterns for segments to improve distinguishability
* clean up and adapt JS-side color calculation
* fix linting
* fix tests
* tmp: try out other color scheme
* ensure that patterns are rendered equally in all viewports; add grid pattern; add variable for density
* fune-tune complimentary colors in pattern rendering
* Merge branch 'segmentation-patterns' into segmentation-patterns-ng
* integrate patterns with jet color scheme
* Update frontend/javascripts/oxalis/shaders/segmentation.glsl.js
* use primitive roots for generation of pseudo-random segmentation patterns; adapt zoom-dependent scaling
* tune pattern frequency again
* Merge branch 'segmentation-patterns' of github.com:scalableminds/webknossos into segmentation-patterns
* clean up
* Merge branch 'master' of github.com:scalableminds/webknossos into segmentation-patterns
* integrate PR feedback
* implement new color assignment in JS and use 2 as a primitive root go be more robust against numerical instability
* update changelog
* add more comments for stripe and grid pattern generation
* extract util functions into proper glsl utils modules
* Merge branch 'master' of github.com:scalableminds/webknossos into segmentation-patterns
* format
* move glsl-js-mirrors to next to the glsl implementations
* update screenshots for nightly test
* update screenshots

CHANGELOG.unreleased.md

@@ -14,6 +14,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Added the possibility to move nodes in skeleton tracings. This can be done either by pressing CTRL + arrow key or by dragging while holding CTRL. [#4743](https://github.com/scalableminds/webknossos/pull/4743)
 - Added the possibility to delete datasets on disk from webKnossos. Use with care. [#4696](https://github.com/scalableminds/webknossos/pull/4696)
 - Added error toasts for failing bucket requests. [#4740](https://github.com/scalableminds/webknossos/pull/4740)
+- Improved the distinguishability of segments by improving the color generation and also by rendering patterns within the segments. The pattern opacity can be adapted in the layer settings (next to the opacity of the segmentation layer). [#4730](https://github.com/scalableminds/webknossos/pull/4730)
 - Added a list of all projects containing tasks of a specific task type. It's accessible from the task types list view. [#4420](https://github.com/scalableminds/webknossos/pull/4745)
 
 ### Changed

frontend/javascripts/admin/tasktype/recommended_configuration_view.js

@@ -33,6 +33,7 @@ const recommendedConfigByCategory = {
     interpolation: true,
     segmentationOpacity: 0,
     highlightHoveredCellId: false,
+    segmentationPatternOpacity: 40,
     zoom: 0.8,
     renderMissingDataBlack: false,
     loadingStrategy: "BEST_QUALITY_FIRST",

frontend/javascripts/libs/user_settings.schema.js

@@ -33,6 +33,7 @@ export const userSettings = {
   zoom: { type: "number", minimum: 0.005 },
   renderMissingDataBlack: { type: "boolean" },
   brushSize: { type: "number", minimum: 1, maximum: 5000 },
+  segmentationPatternOpacity: { type: "number", minimum: 0, maximum: 100 },
   layoutScaleValue: { type: "number", minimum: 1, maximum: 5 },
   autoSaveLayouts: { type: "boolean" },
   gpuMemoryFactor: { type: "number" },

frontend/javascripts/messages.js

@@ -29,6 +29,7 @@ export const settings = {
   sphericalCapRadius: "Sphere Radius",
   crosshairSize: "Crosshair Size",
   brushSize: "Brush Size",
+  segmentationPatternOpacity: "Pattern Opacity",
   userBoundingBoxes: "Bounding Boxes",
   loadingStrategy: "Loading Strategy",
   loadingStrategyDescription: `You can choose between loading the best quality first

frontend/javascripts/oxalis/api/api_latest.js

@@ -1062,6 +1062,7 @@ class DataApi {
      - layers
      - quality
      - highlightHoveredCellId
+     - segmentationPatternOpacity
      - renderMissingDataBlack
    *
    * @example

frontend/javascripts/oxalis/api/api_v2.js

@@ -672,6 +672,7 @@ class DataApi {
      - layers
      - quality
      - highlightHoveredCellId
+     - segmentationPatternOpacity
      - renderMissingDataBlack
    *
    * @example

frontend/javascripts/oxalis/controller/scene_controller.js

@@ -40,7 +40,7 @@ import constants, {
 } from "oxalis/constants";
 import window from "libs/window";
 
-import { convertCellIdToHSLA } from "../view/right-menu/mapping_info_view";
+import { jsConvertCellIdToHSLA } from "oxalis/shaders/segmentation.glsl.js";
 import { setSceneController } from "./scene_controller_provider";
 
 const CUBE_COLOR = 0x999999;
@@ -169,7 +169,7 @@ class SceneController {
   }
 
   addIsosurfaceFromGeometry(geometry: THREE.Geometry, segmentationId: number): void {
-    const [hue] = convertCellIdToHSLA(segmentationId);
+    const [hue] = jsConvertCellIdToHSLA(segmentationId);
     const color = new THREE.Color().setHSL(hue, 0.5, 0.1);
 
     const meshMaterial = new THREE.MeshLambertMaterial({ color });

frontend/javascripts/oxalis/default_state.js

@@ -43,6 +43,7 @@ const defaultState: OxalisState = {
     quality: 0,
     loadingStrategy: "PROGRESSIVE_QUALITY",
     highlightHoveredCellId: true,
+    segmentationPatternOpacity: 40,
     renderIsosurfaces: false,
     renderMissingDataBlack: true,
   },

frontend/javascripts/oxalis/geometries/materials/plane_material_factory.js

@@ -159,6 +159,10 @@ class PlaneMaterialFactory {
         type: "f",
         value: 0,
       },
+      segmentationPatternOpacity: {
+        type: "f",
+        value: 40,
+      },
       activeCellId: {
         type: "v4",
         value: new THREE.Vector4(0, 0, 0, 0),
@@ -298,6 +302,10 @@ class PlaneMaterialFactory {
         fragmentShader: this.getFragmentShader(),
       }),
     );
+    this.material.extensions = {
+      // Necessary for anti-aliasing via fwidth in shader
+      derivatives: true,
+    };
 
     shaderEditor.addMaterial(this.shaderId, this.material);
 
@@ -477,6 +485,16 @@ class PlaneMaterialFactory {
         ),
       );
 
+      this.storePropertyUnsubscribers.push(
+        listenToStoreProperty(
+          storeState => storeState.datasetConfiguration.segmentationPatternOpacity,
+          segmentationPatternOpacity => {
+            this.uniforms.segmentationPatternOpacity.value = segmentationPatternOpacity;
+          },
+          true,
+        ),
+      );
+
       this.storePropertyUnsubscribers.push(
         listenToStoreProperty(
           storeState => storeState.temporaryConfiguration.hoveredIsosurfaceId,

frontend/javascripts/oxalis/model/helpers/shader_editor.js

@@ -37,7 +37,9 @@ window._setupShaderEditor = (identifier, _shaderType) => {
         top: 0;
         right: 0px;
         z-index: 10000;
-        background: white;`,
+        background: white;
+        font-family: monospace;
+        `,
   );
   input.addEventListener("keydown", evt => {
     if ((evt.keyCode === 10 || evt.keyCode === 13) && evt.ctrlKey) {

frontend/javascripts/oxalis/shaders/main_data_fragment.glsl.js

@@ -70,6 +70,7 @@ const int dataTextureCountPerLayer = <%= dataTextureCountPerLayer %>;
   uniform vec4 activeCellId;
   uniform bool isMouseInActiveViewport;
   uniform float activeVolumeToolIndex;
+  uniform float segmentationPatternOpacity;
 
   <% if (isMappingSupported) { %>
     uniform bool isMappingEnabled;

frontend/javascripts/oxalis/shaders/segmentation.glsl.js

@@ -2,39 +2,153 @@
 import type { ShaderModule } from "./shader_module_system";
 import { binarySearchIndex } from "./mappings.glsl";
 import { getRgbaAtIndex } from "./texture_access.glsl";
-import { hsvToRgb } from "./utils.glsl";
+import {
+  hsvToRgb,
+  jsRgb2hsv,
+  getElementOfPermutation,
+  jsGetElementOfPermutation,
+  aaStep,
+  colormapJet,
+  jsColormapJet,
+} from "./utils.glsl";
 
 export const convertCellIdToRGB: ShaderModule = {
-  requirements: [hsvToRgb, getRgbaAtIndex],
+  requirements: [hsvToRgb, getRgbaAtIndex, getElementOfPermutation, aaStep, colormapJet],
   code: `
     vec3 convertCellIdToRGB(vec4 id) {
-      float golden_ratio = 0.618033988749895;
+      /*
+      This function maps from a segment id to a color with a pattern.
+      For the color, the jet color map is used. For the patterns, we employ the following
+      features:
+      - different shapes (stripes and grid)
+      - different angles
+      - different densities (see frequencyModulator)
+
+      The features are pseudo-randomly combined using getElementOfPermutation.
+      This approach gives us 19 colors  * 2 shapes * 17 angles * 3 densities and therefore
+      1938 different segment styles.
+
+      If custom colors were provided via mappings, the color values are used from there.
+      The patterns are still painted on top of these, though.
+      */
+
       float lastEightBits = id.r;
-      float value = mod( lastEightBits * golden_ratio, 1.0);
+      float significantSegmentIndex = 256.0 * id.g + id.r;
+
+      float colorCount = 19.;
+      float colorIndex = getElementOfPermutation(significantSegmentIndex, colorCount, 2.);
+      float colorValueDecimal = 1.0 / colorCount * colorIndex;
+      float colorValue = rgb2hsv(colormapJet(colorValueDecimal)).x;
+      // For historical reference: the old color generation was: colorValue = mod(lastEightBits * (golden_ratio - 1.0), 1.0);
 
       <% if (isMappingSupported) { %>
         // If the first element of the mapping colors texture is still the initialized
-        // value of -1, no mapping colors have been specified
+        // colorValue of -1, no mapping colors have been specified
         bool hasCustomMappingColors = getRgbaAtIndex(
           <%= segmentationName %>_mapping_color_texture,
           <%= mappingColorTextureWidth %>,
           0.0
         ).r != -1.0;
         if (isMappingEnabled && hasCustomMappingColors) {
-          value = getRgbaAtIndex(
+          colorValue = getRgbaAtIndex(
             <%= segmentationName %>_mapping_color_texture,
             <%= mappingColorTextureWidth %>,
             lastEightBits
           ).r;
         }
       <% } %>
 
-      vec4 HSV = vec4( value, 1.0, 1.0, 1.0 );
+      // The following code scales the world coordinates so that the coordinate frequency is in a "pleasant" range.
+      // Also, when zooming out, coordinates change faster which make the pattern more turbulent. Dividing by the
+      // zoomValue compensates this. Note that the zoom *step* should not be used here, because that value relates to the
+      // three-dimensional dataset. Since the patterns are only 2D, the zoomValue is a better proxy.
+      //
+      // By default, scale everything with fineTunedScale as this seemed a good value during testing.
+      float fineTunedScale = 0.15;
+      // Additionally, apply another scale factor (between 0.5 and 1.5) depending on the segment id.
+      float frequencySequenceLength = 3.;
+      float frequencyModulator = mix(0.5, 1.5, getElementOfPermutation(significantSegmentIndex, frequencySequenceLength, 2.) / frequencySequenceLength);
+      float coordScaling = fineTunedScale * frequencyModulator;
+      // Round the zoomValue so that the pattern frequency only changes at distinct steps. Otherwise, zooming out
+      // wouldn't change the pattern at all, which would feel weird.
+      float zoomAdaption = ceil(zoomValue);
+      vec3 worldCoordUVW = coordScaling * getWorldCoordUVW()  / zoomAdaption;
+
+      float baseVoxelSize = min(min(datasetScale.x, datasetScale.y), datasetScale.z);
+      vec3 datasetScaleUVW = transDim(datasetScale) / baseVoxelSize;
+      worldCoordUVW.x = worldCoordUVW.x * datasetScaleUVW.x;
+      worldCoordUVW.y = worldCoordUVW.y * datasetScaleUVW.y;
+
+      float angleCount = 17.;
+      float angle = 1.0 / angleCount * getElementOfPermutation(significantSegmentIndex, angleCount, 3.0);
+
+      // To produce a stripe or grid pattern, we use the current fragment coordinates
+      // and an angle.
+      // stripeValueA is a value between 0 and 1 which - when rounded - denotes if the current fragment
+      // is in the "bright" or "dark" stripe class.
+      // Similarly, stripeValueB is constructed, with the difference that the angle is orthogonal to
+      // stripeValueA.
+      // When combining both stripe values, a grid can be produced. When only using stripeValueA, a simple
+      // stripe pattern is rendered.
+      float stripeValueA = mix(
+        worldCoordUVW.x,
+        worldCoordUVW.y,
+        angle
+      );
+      float stripeValueB = mix(
+        worldCoordUVW.x,
+        -worldCoordUVW.y,
+        1.0 - angle
+      );
+
+      // useGrid is binary, but we generate a pseudo-random sequence of 13 elements which we map
+      // to ones and zeros. This has the benefit that the periodicity has a prime length.
+      float useGridSequenceLength = 13.;
+      float useGrid = step(mod(getElementOfPermutation(significantSegmentIndex, useGridSequenceLength, 2.0), 2.0), 0.5);
+      // Cast the continuous stripe values to 0 and 1 + a bit of anti-aliasing.
+      float aaStripeValueA = aaStep(stripeValueA);
+      float aaStripeValueB = aaStep(stripeValueB);
+      // Combine both stripe values when a grid should be rendered. Otherwise, only use aaStripeValueA
+      float aaStripeValue = 1.0 - max(aaStripeValueA, useGrid * aaStripeValueB);
+
+      vec4 HSV = vec4(
+        colorValue,
+        1.0 - 0.5 * ((1. - aaStripeValue) * segmentationPatternOpacity / 100.0),
+        1.0 - 0.5 * (aaStripeValue * segmentationPatternOpacity / 100.0),
+        1.0
+      );
+
       return hsvToRgb(HSV);
     }
   `,
 };
 
+// This function mirrors convertCellIdToRGB in the fragment shader of the rendering plane.
+// Output is in [0,1] for H, S, L and A
+export const jsConvertCellIdToHSLA = (id: number, customColors: ?Array<number>): Array<number> => {
+  if (id === 0) {
+    // Return white
+    return [1, 1, 1, 1];
+  }
+
+  let hue;
+
+  if (customColors != null) {
+    const last8Bits = id % 2 ** 8;
+    hue = customColors[last8Bits] || 0;
+  } else {
+    const significantSegmentIndex = id % 2 ** 16;
+
+    const colorCount = 19;
+    const colorIndex = jsGetElementOfPermutation(significantSegmentIndex, colorCount, 2);
+    const colorValueDecimal = (1.0 / colorCount) * colorIndex;
+
+    hue = (1 / 360) * jsRgb2hsv(jsColormapJet(colorValueDecimal))[0];
+  }
+
+  return [hue, 1, 0.5, 0.15];
+};
+
 export const getBrushOverlay: ShaderModule = {
   code: `
     vec4 getBrushOverlay(vec3 worldCoordUVW) {
@@ -75,17 +189,18 @@ export const getSegmentationId: ShaderModule = {
           vec4(0.0, 0.0, 0.0, 0.0)
         );
 
+      // Depending on the packing degree, the returned volume color contains extra values
+      // which would should be ignored (in the binary search as well as when comparing
+      // a cell id with the hovered cell passed via uniforms, for example).
+
+      <% if (segmentationPackingDegree === "4.0") { %>
+        volume_color = vec4(volume_color.r, 0.0, 0.0, 0.0);
+      <% } else if (segmentationPackingDegree === "2.0") { %>
+        volume_color = vec4(volume_color.r, volume_color.g, 0.0, 0.0);
+      <% } %>
 
       <% if (isMappingSupported) { %>
         if (isMappingEnabled) {
-          // Depending on the packing degree, the returned volume color contains extra values
-          // which would make the binary search fail
-
-          <% if (segmentationPackingDegree === "4.0") { %>
-            volume_color = vec4(volume_color.r, 0.0, 0.0, 0.0);
-          <% } else if (segmentationPackingDegree === "2.0") { %>
-            volume_color = vec4(volume_color.r, volume_color.g, 0.0, 0.0);
-          <% } %>
 
           float index = binarySearchIndex(
             <%= segmentationName %>_mapping_lookup_texture,