Feature - adaptive downsampling of volume rendering based on framerate (

#591)

python/neuroglancer/viewer_state.py

@@ -1668,6 +1668,10 @@ class ViewerState(JsonObjectWrapper):
     show_axis_lines = showAxisLines = wrapped_property(
         "showAxisLines", optional(bool, True)
     )
+    wire_frame = wireFrame = wrapped_property("wireFrame", optional(bool, False))
+    enable_adaptive_downsampling = enableAdaptiveDownsampling = wrapped_property(
+        "enableAdaptiveDownsampling", optional(bool, True)
+    )
     show_scale_bar = showScaleBar = wrapped_property(
         "showScaleBar", optional(bool, True)
     )

src/data_panel_layout.ts

@@ -92,6 +92,7 @@ export interface ViewerUIState
   showPerspectiveSliceViews: TrackableBoolean;
   showAxisLines: TrackableBoolean;
   wireFrame: TrackableBoolean;
+  enableAdaptiveDownsampling: TrackableBoolean;
   showScaleBar: TrackableBoolean;
   scaleBarOptions: TrackableValue<ScaleBarOptions>;
   visibleLayerRoles: WatchableSet<RenderLayerRole>;
@@ -174,6 +175,7 @@ export function getCommonViewerState(viewer: ViewerUIState) {
     layerManager: viewer.layerManager,
     showAxisLines: viewer.showAxisLines,
     wireFrame: viewer.wireFrame,
+    enableAdaptiveDownsampling: viewer.enableAdaptiveDownsampling,
     visibleLayerRoles: viewer.visibleLayerRoles,
     selectedLayer: viewer.selectedLayer,
     visibility: viewer.visibility,

src/display_context.ts

@@ -14,18 +14,23 @@
  * limitations under the License.
  */
 
+import { debounce } from "lodash-es";
+
 import type { FrameNumberCounter } from "#src/chunk_manager/frontend.js";
 import { TrackableValue } from "#src/trackable_value.js";
 import { animationFrameDebounce } from "#src/util/animation_frame_debounce.js";
 import type { Borrowed } from "#src/util/disposable.js";
 import { RefCounted } from "#src/util/disposable.js";
+import { FramerateMonitor } from "#src/util/framerate.js";
 import type { mat4 } from "#src/util/geom.js";
 import { parseFixedLengthArray, verifyFloat01 } from "#src/util/json.js";
 import { NullarySignal } from "#src/util/signal.js";
 import type { WatchableVisibilityPriority } from "#src/visibility_priority/frontend.js";
 import type { GL } from "#src/webgl/context.js";
 import { initializeWebGL } from "#src/webgl/context.js";
 
+const DELAY_AFTER_CONTINUOUS_CAMERA_MOTION_MS = 300;
+
 export class RenderViewport {
   // Width of visible portion of panel in canvas pixels.
   width = 0;
@@ -390,12 +395,17 @@ export class DisplayContext extends RefCounted implements FrameNumberCounter {
   gl: GL;
   updateStarted = new NullarySignal();
   updateFinished = new NullarySignal();
+  continuousCameraMotionStarted = new NullarySignal();
+  continuousCameraMotionFinished = new NullarySignal();
   changed = this.updateFinished;
   panels = new Set<RenderedPanel>();
   canvasRect: DOMRect | undefined;
   rootRect: DOMRect | undefined;
   resizeGeneration = 0;
   boundsGeneration = -1;
+  private framerateMonitor = new FramerateMonitor();
+
+  private continuousCameraMotionInProgress = false;
 
   // Panels ordered by `drawOrder`.  If length is 0, needs to be recomputed.
   private orderedPanels: RenderedPanel[] = [];
@@ -447,6 +457,25 @@ export class DisplayContext extends RefCounted implements FrameNumberCounter {
 
   private resizeObserver = new ResizeObserver(this.resizeCallback);
 
+  private debouncedEndContinuousCameraMotion = this.registerCancellable(
+    debounce(() => {
+      this.continuousCameraMotionInProgress = false;
+      this.continuousCameraMotionFinished.dispatch();
+    }, DELAY_AFTER_CONTINUOUS_CAMERA_MOTION_MS),
+  );
+
+  flagContinuousCameraMotion() {
+    if (!this.continuousCameraMotionInProgress) {
+      this.continuousCameraMotionStarted.dispatch();
+    }
+    this.continuousCameraMotionInProgress = true;
+    this.debouncedEndContinuousCameraMotion();
+  }
+
+  get isContinuousCameraMotionInProgress() {
+    return this.continuousCameraMotionInProgress;
+  }
+
   constructor(public container: HTMLElement) {
     super();
     const { canvas, resizeObserver } = this;
@@ -557,6 +586,8 @@ export class DisplayContext extends RefCounted implements FrameNumberCounter {
     ++this.frameNumber;
     this.updateStarted.dispatch();
     const gl = this.gl;
+    const ext = this.framerateMonitor.getTimingExtension(gl);
+    const query = this.framerateMonitor.startFrameTimeQuery(gl, ext);
     this.ensureBoundsUpdated();
     this.gl.clearColor(0.0, 0.0, 0.0, 0.0);
     gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
@@ -580,6 +611,8 @@ export class DisplayContext extends RefCounted implements FrameNumberCounter {
     gl.clear(gl.COLOR_BUFFER_BIT);
     this.gl.colorMask(true, true, true, true);
     this.updateFinished.dispatch();
+    this.framerateMonitor.endFrameTimeQuery(gl, ext, query);
+    this.framerateMonitor.grabAnyFinishedQueryResults(gl);
   }
 
   getDepthArray(): Float32Array {
@@ -607,4 +640,8 @@ export class DisplayContext extends RefCounted implements FrameNumberCounter {
     }
     return depthArray;
   }
+
+  getLastFrameTimesInMs(numberOfFrames: number = 10) {
+    return this.framerateMonitor.getLastFrameTimesInMs(numberOfFrames);
+  }
 }

src/layer_group_viewer.ts

@@ -94,6 +94,7 @@ export interface LayerGroupViewerState {
   mouseState: MouseSelectionState;
   showAxisLines: TrackableBoolean;
   wireFrame: TrackableBoolean;
+  enableAdaptiveDownsampling: TrackableBoolean;
   showScaleBar: TrackableBoolean;
   scaleBarOptions: TrackableScaleBarOptions;
   showPerspectiveSliceViews: TrackableBoolean;
@@ -356,6 +357,9 @@ export class LayerGroupViewer extends RefCounted {
   get wireFrame() {
     return this.viewerState.wireFrame;
   }
+  get enableAdaptiveDownsampling() {
+    return this.viewerState.enableAdaptiveDownsampling;
+  }
   get showScaleBar() {
     return this.viewerState.showScaleBar;
   }

src/layer_groups_layout.ts

@@ -407,6 +407,7 @@ function getCommonViewerState(viewer: Viewer) {
     mouseState: viewer.mouseState,
     showAxisLines: viewer.showAxisLines,
     wireFrame: viewer.wireFrame,
+    enableAdaptiveDownsampling: viewer.enableAdaptiveDownsampling,
     showScaleBar: viewer.showScaleBar,
     scaleBarOptions: viewer.scaleBarOptions,
     showPerspectiveSliceViews: viewer.showPerspectiveSliceViews,