Histogram followup (#4195)

* add sampling for float layers and better support for uint24 layers #4095 * swapped statements and added changelog entry * prevent error when no data was found * implement float/uint16 histogram support * draw uint24 rgb histograms as one colored histogram * scalafmt * specify step size for histogram slider * fix bucket size calculation for small floats * only show layer-specific options if layer is enabled
scalableminds · Jul 29, 2019 · f22f6a0 · f22f6a0
1 parent 462f5e9
commit f22f6a0
Show file tree

Hide file tree

Showing 10 changed files with 149 additions and 111 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -34,7 +34,8 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.md).
 ### Added
 - Volume tasks with only one finished instance can now be viewed as CompoundTask. [#4167](https://github.com/scalableminds/webknossos/pull/4167)
 - Added the possibility to remove isosurfaces from the 3D viewport by CTRL+Clicking it. [#4185](https://github.com/scalableminds/webknossos/pull/4185)
-- Added support for int16 and uint16 color layers. [#4152](https://github.com/scalableminds/webknossos/pull/4152)
+- Added support for `int16` and `uint16` color layers. [#4152](https://github.com/scalableminds/webknossos/pull/4152)
+- Added histogram support for `int16` and `uint16` color layers. Additionally refined support for `float` color layers. [#4195](https://github.com/scalableminds/webknossos/pull/4195)
 
 ### Changed
 - Volume project download zips are reorganized to contain a zipfile for each annotation (that in turn contains a data.zip and an nml file). [#4167](https://github.com/scalableminds/webknossos/pull/4167)

diff --git a/frontend/javascripts/admin/admin_rest_api.js b/frontend/javascripts/admin/admin_rest_api.js
@@ -918,14 +918,13 @@ export async function getHistogramForLayer(
   datasetId: APIDatasetId,
   layerName: string,
 ): Promise<APIHistogramData> {
-  const { count, histogram } = await doWithToken(token =>
+  return doWithToken(token =>
     Request.receiveJSON(
       `${datastoreUrl}/data/datasets/${datasetId.owningOrganization}/${
         datasetId.name
       }/layers/${layerName}/histogram?token=${token}`,
     ),
   );
-  return { count, histogram };
 }
 
 export async function getMappingsForDatasetLayer(

diff --git a/frontend/javascripts/admin/api_flow_types.js b/frontend/javascripts/admin/api_flow_types.js
@@ -60,7 +60,12 @@ export type APISegmentationLayer = {|
 
 export type APIDataLayer = APIColorLayer | APISegmentationLayer;
 
-export type APIHistogramData = { count: number, histogram: Array<number> };
+export type APIHistogramData = Array<{
+  numberOfElements: number,
+  elementCounts: Array<number>,
+  min: number,
+  max: number,
+}>;
 
 type APIDataSourceBase = {
   +id: {

diff --git a/frontend/javascripts/oxalis/geometries/materials/plane_material_factory.js b/frontend/javascripts/oxalis/geometries/materials/plane_material_factory.js
@@ -33,6 +33,7 @@ import * as Utils from "libs/utils";
 import app from "app";
 import getMainFragmentShader from "oxalis/shaders/main_data_fragment.glsl";
 import shaderEditor from "oxalis/model/helpers/shader_editor";
+import { type ElementClass } from "admin/api_flow_types";
 
 type ShaderMaterialOptions = {
   polygonOffset?: boolean,
@@ -71,18 +72,6 @@ function getPackingDegreeLookup(): { [string]: number } {
   );
 }
 
-function getFloatLayerLookup(): { [string]: false | { min: number, max: number } } {
-  const { dataset } = Store.getState();
-  const colorLayers = getColorLayers(dataset);
-  // keyBy the sanitized layer name as the lookup will happen in the shader using the sanitized layer name
-  const colorLayersObject = _.keyBy(colorLayers, layer => sanitizeName(layer.name));
-  // TODO: Use the correct float value range (from histogram)
-  return _.mapValues(
-    colorLayersObject,
-    layer => getElementClass(dataset, layer.name) === "float" && { min: 0, max: 255 },
-  );
-}
-
 class PlaneMaterialFactory {
   planeID: OrthoView;
   isOrthogonal: boolean;
@@ -421,7 +410,7 @@ class PlaneMaterialFactory {
             const settings = layerSettings[colorLayer.name];
             if (settings != null) {
               const name = sanitizeName(colorLayer.name);
-              this.updateUniformsForLayer(settings, name);
+              this.updateUniformsForLayer(settings, name, colorLayer.elementClass);
             }
           }
           // TODO: Needed?
@@ -521,10 +510,16 @@ class PlaneMaterialFactory {
     this.uniforms.activeCellId.value.set(r, g, b, a);
   }
 
-  updateUniformsForLayer(settings: DatasetLayerConfiguration, name: string): void {
+  updateUniformsForLayer(
+    settings: DatasetLayerConfiguration,
+    name: string,
+    elementClass: ElementClass,
+  ): void {
     const { alpha, intensityRange, isDisabled } = settings;
-    this.uniforms[`${name}_min`].value = intensityRange[0] / 255;
-    this.uniforms[`${name}_max`].value = intensityRange[1] / 255;
+    // In UnsignedByte textures the byte values are scaled to [0, 1], in Float textures they are not
+    const divisor = elementClass === "float" ? 1 : 255;
+    this.uniforms[`${name}_min`].value = intensityRange[0] / divisor;
+    this.uniforms[`${name}_max`].value = intensityRange[1] / divisor;
     this.uniforms[`${name}_alpha`].value = isDisabled ? 0 : alpha / 100;
 
     if (settings.color != null) {
@@ -540,7 +535,6 @@ class PlaneMaterialFactory {
   getFragmentShader(): string {
     const colorLayerNames = getColorLayerNames();
     const packingDegreeLookup = getPackingDegreeLookup();
-    const floatLayerLookup = getFloatLayerLookup();
     const segmentationLayer = Model.getSegmentationLayer();
     const segmentationName = sanitizeName(segmentationLayer ? segmentationLayer.name : "");
     const { dataset } = Store.getState();
@@ -555,7 +549,6 @@ class PlaneMaterialFactory {
     const code = getMainFragmentShader({
       colorLayerNames,
       packingDegreeLookup,
-      floatLayerLookup,
       hasSegmentation,
       segmentationName,
       isMappingSupported: Model.isMappingSupported,

diff --git a/frontend/javascripts/oxalis/shaders/main_data_fragment.glsl.js b/frontend/javascripts/oxalis/shaders/main_data_fragment.glsl.js
@@ -23,7 +23,6 @@ import compileShader from "./shader_module_system";
 type Params = {|
   colorLayerNames: string[],
   packingDegreeLookup: { [string]: number },
-  floatLayerLookup: { [string]: false | { min: number, max: number } },
   hasSegmentation: boolean,
   segmentationName: string,
   isMappingSupported: boolean,
@@ -175,22 +174,14 @@ void main() {
         fallbackGray
       ).xyz;
 
-    <% if (floatLayerLookup[name]) { %>
-      // Adjust the value range of the float values
-      vec3 range = vec3(<%= formatNumberAsGLSLFloat(floatLayerLookup[name].max - floatLayerLookup[name].min) %>);
-      vec3 rangeMin = vec3(<%= formatNumberAsGLSLFloat(floatLayerLookup[name].min) %>);
-      color_value = (color_value + rangeMin) / range;
-    <% } else { %>
-
-      <% if (packingDegreeLookup[name] === 2.0) { %>
-        // Workaround for 16-bit color layers
-        color_value = vec3(color_value.g * 255.0 + color_value.r);
-      <% } %>
-      // Keep the color in bounds of min and max
-      color_value = clamp(color_value, <%= name %>_min, <%= name %>_max);
-      // Scale interval between min and max up to interval from 0 to 255
-      color_value = (color_value - <%= name %>_min) / (<%= name %>_max - <%= name %>_min);
+    <% if (packingDegreeLookup[name] === 2.0) { %>
+      // Workaround for 16-bit color layers
+      color_value = vec3(color_value.g * 256.0 + color_value.r);
     <% } %>
+    // Keep the color in bounds of min and max
+    color_value = clamp(color_value, <%= name %>_min, <%= name %>_max);
+    // Scale the color value according to the histogram settings
+    color_value = (color_value - <%= name %>_min) / (<%= name %>_max - <%= name %>_min);
 
     // Multiply with color and alpha for <%= name %>
     data_color += color_value * <%= name %>_alpha * <%= name %>_color;

diff --git a/frontend/javascripts/oxalis/view/settings/dataset_settings_view.js b/frontend/javascripts/oxalis/view/settings/dataset_settings_view.js
@@ -133,9 +133,11 @@ class DatasetSettings extends React.PureComponent<DatasetSettingsProps, State> {
   ) => {
     const elementClass = getElementClass(this.props.dataset, layerName);
     const { alpha, color, intensityRange, isDisabled } = layer;
-    let histogram = new Array(256).fill(0);
+    let histograms = [
+      { numberOfElements: 256, elementCounts: new Array(256).fill(0), min: 0, max: 255 },
+    ];
     if (this.state.histograms && this.state.histograms[layerName]) {
-      ({ histogram } = this.state.histograms[layerName]);
+      histograms = this.state.histograms[layerName];
     }
 
     return (
@@ -171,29 +173,31 @@ class DatasetSettings extends React.PureComponent<DatasetSettingsProps, State> {
             {this.getFindDataButton(layerName, isDisabled)}
           </Col>
         </Row>
-        {!isDisabled && isHistogramSupported(elementClass) ? (
-          <Histogram
-            data={histogram}
-            min={intensityRange[0]}
-            max={intensityRange[1]}
-            layerName={layerName}
-          />
-        ) : null}
-        <NumberSliderSetting
-          label="Opacity"
-          min={0}
-          max={100}
-          value={alpha}
-          onChange={_.partial(this.props.onChangeLayer, layerName, "alpha")}
-          disabled={isDisabled}
-        />
-        <ColorSetting
-          label="Color"
-          value={Utils.rgbToHex(color)}
-          onChange={_.partial(this.props.onChangeLayer, layerName, "color")}
-          className="ant-btn"
-          disabled={isDisabled}
-        />
+        {isDisabled ? null : (
+          <React.Fragment>
+            {isHistogramSupported(elementClass) ? (
+              <Histogram
+                data={histograms}
+                min={intensityRange[0]}
+                max={intensityRange[1]}
+                layerName={layerName}
+              />
+            ) : null}
+            <NumberSliderSetting
+              label="Opacity"
+              min={0}
+              max={100}
+              value={alpha}
+              onChange={_.partial(this.props.onChangeLayer, layerName, "alpha")}
+            />
+            <ColorSetting
+              label="Color"
+              value={Utils.rgbToHex(color)}
+              onChange={_.partial(this.props.onChangeLayer, layerName, "color")}
+              className="ant-btn"
+            />
+          </React.Fragment>
+        )}
         {!isLastLayer && <Divider />}
       </div>
     );

diff --git a/frontend/javascripts/oxalis/view/settings/histogram_view.js b/frontend/javascripts/oxalis/view/settings/histogram_view.js
@@ -8,9 +8,12 @@ import { connect } from "react-redux";
 import { type DatasetLayerConfiguration } from "oxalis/store";
 import { updateLayerSettingAction } from "oxalis/model/actions/settings_actions";
 import { type ElementClass } from "admin/api_flow_types";
+import type { APIHistogramData } from "admin/api_flow_types";
+import { type Vector3 } from "oxalis/constants";
+import { roundTo } from "libs/utils";
 
 type OwnProps = {|
-  data: Array<number>,
+  data: APIHistogramData,
   layerName: string,
   min: number,
   max: number,
@@ -25,11 +28,12 @@ type HistogramProps = {
   ) => void,
 };
 
+const uint24Colors = [[255, 65, 54], [46, 204, 64], [24, 144, 255]];
 const canvasHeight = 100;
 const canvasWidth = 300;
 
 export function isHistogramSupported(elementClass: ElementClass): boolean {
-  return ["int8", "uint8"].includes(elementClass);
+  return ["int8", "uint8", "int16", "uint16", "float", "uint24"].includes(elementClass);
 }
 
 class Histogram extends React.PureComponent<HistogramProps> {
@@ -44,41 +48,58 @@ class Histogram extends React.PureComponent<HistogramProps> {
     ctx.scale(1, -1);
     ctx.lineWidth = 1;
     ctx.lineJoin = "round";
-    ctx.fillStyle = "rgba(24, 144, 255, 0.1)";
-    ctx.strokeStyle = "#1890ff";
     this.updateCanvas();
   }
 
   componentDidUpdate() {
     this.updateCanvas();
   }
 
-  updateCanvas = () => {
+  updateCanvas() {
     if (this.canvasRef == null) {
       return;
     }
-    const { min, max, data } = this.props;
     const ctx = this.canvasRef.getContext("2d");
     ctx.clearRect(0, 0, canvasWidth, canvasHeight);
-    const maxValue = Math.max(...data);
-    const downscaledData = this.props.data.map(value =>
+    const { data } = this.props;
+    // Compute the overall maximum count, so the RGB curves are scaled correctly relative to each other
+    const maxValue = Math.max(...data.map(({ elementCounts }) => Math.max(...elementCounts)));
+    for (const [i, histogram] of data.entries()) {
+      const color = this.props.data.length > 1 ? uint24Colors[i] : uint24Colors[2];
+      this.drawHistogram(ctx, histogram, maxValue, color);
+    }
+  }
+
+  drawHistogram = (
+    ctx: CanvasRenderingContext2D,
+    histogram: $ElementType<APIHistogramData, number>,
+    maxValue: number,
+    color: Vector3,
+  ) => {
+    const { min, max } = this.props;
+    const { min: minRange, max: maxRange, elementCounts } = histogram;
+    const rangeLength = maxRange - minRange;
+    ctx.fillStyle = `rgba(${color.join(",")}, 0.1)`;
+    ctx.strokeStyle = `rgba(${color.join(",")})`;
+    const downscaledData = elementCounts.map(value =>
       value > 0 ? (Math.log(value) / Math.log(maxValue)) * canvasHeight : 0,
     );
     const activeRegion = new Path2D();
     ctx.beginPath();
     ctx.moveTo(0, downscaledData[0]);
-    activeRegion.moveTo((min / downscaledData.length) * canvasWidth, 0);
+    activeRegion.moveTo(((min - minRange) / rangeLength) * canvasWidth, 0);
     for (let i = 0; i < downscaledData.length; i++) {
       const x = (i / downscaledData.length) * canvasWidth;
-      if (i >= min && i <= max) {
+      const xValue = minRange + i * (rangeLength / downscaledData.length);
+      if (xValue >= min && xValue <= max) {
         activeRegion.lineTo(x, downscaledData[i]);
       }
       ctx.lineTo(x, downscaledData[i]);
     }
     ctx.stroke();
     ctx.closePath();
-    activeRegion.lineTo((max / downscaledData.length) * canvasWidth, 0);
-    activeRegion.lineTo((min / downscaledData.length) * canvasWidth, 0);
+    activeRegion.lineTo(((max - minRange) / rangeLength) * canvasWidth, 0);
+    activeRegion.lineTo(((min - minRange) / rangeLength) * canvasWidth, 0);
     activeRegion.closePath();
     ctx.fill(activeRegion);
   };
@@ -93,7 +114,8 @@ class Histogram extends React.PureComponent<HistogramProps> {
   };
 
   render() {
-    const { min, max } = this.props;
+    const { min, max, data } = this.props;
+    const { min: minRange, max: maxRange } = data[0];
     return (
       <React.Fragment>
         <canvas
@@ -105,13 +127,15 @@ class Histogram extends React.PureComponent<HistogramProps> {
         />
         <Slider
           value={[min, max]}
-          min={0}
-          max={255}
+          min={minRange}
+          max={maxRange}
           range
-          defaultValue={[0, this.props.data.length - 1]}
+          defaultValue={[minRange, maxRange]}
           onChange={this.onThresholdChange}
           onAfterChange={this.onThresholdChange}
           style={{ width: 300, margin: 0, marginBottom: 18 }}
+          step={(maxRange - minRange) / 255}
+          tipFormatter={val => roundTo(val, 2).toString()}
         />
       </React.Fragment>
     );

diff --git a/frontend/javascripts/test/shaders/shader_syntax.spec.js b/frontend/javascripts/test/shaders/shader_syntax.spec.js
@@ -13,7 +13,6 @@ test("Shader syntax: Ortho Mode", t => {
   const code = getMainFragmentShader({
     colorLayerNames: ["color_layer_1", "color_layer_2"],
     packingDegreeLookup: { color_layer_1: 4.0, color_layer_2: 4.0 },
-    floatLayerLookup: { color_layer_1: false, color_layer_2: false },
     hasSegmentation: false,
     segmentationName: "",
     isMappingSupported: true,
@@ -34,7 +33,6 @@ test("Shader syntax: Ortho Mode + Segmentation - Mapping", t => {
   const code = getMainFragmentShader({
     colorLayerNames: ["color_layer_1", "color_layer_2"],
     packingDegreeLookup: { color_layer_1: 4.0, color_layer_2: 4.0, segmentationLayer: 1.0 },
-    floatLayerLookup: { color_layer_1: false, color_layer_2: false },
     hasSegmentation: true,
     segmentationName: "segmentationLayer",
     isMappingSupported: false,
@@ -55,7 +53,6 @@ test("Shader syntax: Ortho Mode + Segmentation + Mapping", t => {
   const code = getMainFragmentShader({
     colorLayerNames: ["color_layer_1", "color_layer_2"],
     packingDegreeLookup: { color_layer_1: 4.0, color_layer_2: 4.0, segmentationLayer: 1.0 },
-    floatLayerLookup: { color_layer_1: false, color_layer_2: false },
     hasSegmentation: true,
     segmentationName: "segmentationLayer",
     isMappingSupported: true,
@@ -76,7 +73,6 @@ test("Shader syntax: Arbitrary Mode (no segmentation available)", t => {
   const code = getMainFragmentShader({
     colorLayerNames: ["color_layer_1", "color_layer_2"],
     packingDegreeLookup: { color_layer_1: 4.0, color_layer_2: 4.0 },
-    floatLayerLookup: { color_layer_1: false, color_layer_2: false },
     hasSegmentation: false,
     segmentationName: "",
     isMappingSupported: true,
@@ -97,7 +93,6 @@ test("Shader syntax: Arbitrary Mode (segmentation available)", t => {
   const code = getMainFragmentShader({
     colorLayerNames: ["color_layer_1", "color_layer_2"],
     packingDegreeLookup: { color_layer_1: 4.0, color_layer_2: 4.0, segmentationLayer: 1.0 },
-    floatLayerLookup: { color_layer_1: false, color_layer_2: false },
     hasSegmentation: true,
     segmentationName: "segmentationLayer",
     isMappingSupported: true,
@@ -118,7 +113,6 @@ test("Shader syntax: Ortho Mode (rgb and float layer)", t => {
   const code = getMainFragmentShader({
     colorLayerNames: ["color_layer_1", "color_layer_2"],
     packingDegreeLookup: { color_layer_1: 1.0, color_layer_2: 4.0 },
-    floatLayerLookup: { color_layer_1: false, color_layer_2: { min: 0, max: 255 } },
     hasSegmentation: false,
     segmentationName: "",
     isMappingSupported: true,