DZI viewer component (#29)

Co-authored-by: Lane Sawyer <[email protected]>
AllenInstitute · Oct 2, 2024 · 8cb3cf6 · 8cb3cf6
1 parent 824f293
commit 8cb3cf6
Show file tree

Hide file tree

Showing 29 changed files with 6,386 additions and 4,142 deletions.
diff --git a/docs/examples.md b/docs/examples.md
@@ -1,7 +1,7 @@
 # Examples
 
-Examples are located in the `apps` directory.
+Examples are located in the `examples` directory.
 
 ## Adding New Examples
 
-To add a new example, create a new directory in the `apps` directory, copy the `package.json` from another example, and then start building!
+To add a new example, create a new directory in the `examples` directory, copy the `package.json` from another example, and then start building!
diff --git a/examples/dzi.html b/examples/dzi.html
@@ -0,0 +1,17 @@
+<!doctype html>
+<html>
+    <body>
+        <div
+            id="sidebar"
+            style="top: 0; left: 0; width: 15%; height: 100%; position: absolute"
+        ></div>
+        <div
+            id="main"
+            style="top: 0; left: 15%; width: 85%; height: 100%; position: absolute"
+        />
+    </body>
+</html>
+<script
+    type="module"
+    src="./src/dzi/dzi.ts"
+></script>
diff --git a/examples/index.html b/examples/index.html
@@ -1,17 +1,11 @@
 <!doctype html>
 <html>
     <body>
-        <div
-            id="sidebar"
-            style="top: 0; left: 0; width: 15%; height: 100%; position: absolute"
-        ></div>
-        <canvas
-            id="glCanvas"
-            style="top: 0; left: 15%; width: 85%; height: 100%; position: absolute"
-        />
+        EXAMPLES
+        <br />
+        <ul>
+            <li><a href="/dzi">Deep Zoom Image</a><br /></li>
+            <li><a href="/layers">Layers</a><br /></li>
+        </ul>
     </body>
 </html>
-<script
-    type="module"
-    src="./src/layers.ts"
-></script>
diff --git a/examples/layers.html b/examples/layers.html
@@ -0,0 +1,17 @@
+<!doctype html>
+<html>
+    <body>
+        <div
+            id="sidebar"
+            style="top: 0; left: 0; width: 15%; height: 100%; position: absolute"
+        ></div>
+        <canvas
+            id="glCanvas"
+            style="top: 0; left: 15%; width: 85%; height: 100%; position: absolute"
+        />
+    </body>
+</html>
+<script
+    type="module"
+    src="./src/layers.ts"
+></script>
diff --git a/examples/package.json b/examples/package.json
@@ -23,7 +23,7 @@
             "email": "[email protected]"
         }
     ],
-    "license": "TBD",
+    "license": "BSD-3-Clause",
     "type": "module",
     "main": "lib/index.js",
     "types": "lib/index.d.ts",
@@ -45,6 +45,7 @@
     "dependencies": {
         "@alleninstitute/vis-geometry": "workspace:*",
         "@alleninstitute/vis-scatterbrain": "workspace:*",
+        "@alleninstitute/vis-dzi": "workspace:*",
         "@czi-sds/components": "^20.0.1",
         "@emotion/css": "^11.11.2",
         "@emotion/react": "^11.11.4",

diff --git a/examples/readme.md b/examples/readme.md
@@ -7,15 +7,23 @@
 3. run `pnpm build`
 4. `cd examples/`
 5. `pnpm run dev`
-6. navigate to the running app (default `localhost://5173`)
+6. navigate to the running app (default `localhost://5173`). you can click a link to a specific example, or just use the address bar (`localhost://5173/{path_to_desired_example}`)
 
-## Why?
+## DZI Example
+
+### Why?
+
+A simple proof of concept for displaying DZI (Deep zoom images) using our utilities, as well as demonstrating how to share an Offscreen canvas to render WebGL to multiple client canvases.
+
+## Layers Example
+
+### Why?
 
 the goal of this (rather complicated) example app is not to show off a cool app - rather its goal is to show that we can build complexity by composing simple, focused modules. As we (the AIBS Apps team) have developed ABC-Atlas, we've tried to make sure our visualization code stays general, and that each part does as little as possible. The result is that it was fairly easy to combine those components into this new app, which mixes a (terrible) UI, scatter-plot rendering, polygon-mesh rendering (for annotations) and multi-channel volumetric rendering into independent layers. Although each of these data types appear different, our caching, fetching, visibility determination, and render-scheduling code is the same regardless of the datatype to be rendered. All that is required is to fill in a simple interface, and provide a low-level "renderer" plugin in order to add a new "layer" type.
 
-## Demo Script
+### Demo Script
 
-### Programmatic Configuration
+#### Programmatic Configuration
 
 After starting the app in a browser, you'll be greeted by a blank screen. We're going to demonstrate programmatic access to the features of this demo. The goal here is not to make users invoke command-line arguments, but rather just an easy way for interested parties to "peak under the hood". All the visualizations are configured here via simple json objects - it would not be a stretch to read these configuration options at initialization-time via URL parameters for example.
 

diff --git a/examples/src/common/loaders/ome-zarr/zarr-data.ts b/examples/src/common/loaders/ome-zarr/zarr-data.ts
@@ -120,7 +120,7 @@ export function pickBestScale(
     const choice = datasets.reduce(
         (bestSoFar, cur) =>
             dstToDesired(vxlPitch(planeSizeInVoxels(plane, axes, bestSoFar)!), pxPitch) >
-                dstToDesired(vxlPitch(planeSizeInVoxels(plane, axes, cur)!), pxPitch)
+            dstToDesired(vxlPitch(planeSizeInVoxels(plane, axes, cur)!), pxPitch)
                 ? cur
                 : bestSoFar,
         datasets[0]
@@ -135,9 +135,9 @@ export function sizeInUnits(
     plane:
         | AxisAlignedPlane
         | {
-            u: OmeDimension;
-            v: OmeDimension;
-        },
+              u: OmeDimension;
+              v: OmeDimension;
+          },
     axes: readonly AxisDesc[],
     dataset: DatasetWithShape
 ): vec2 | undefined {

diff --git a/examples/src/dzi/app.tsx b/examples/src/dzi/app.tsx
@@ -0,0 +1,6 @@
+import React from 'react';
+import { TwoClientsPOC } from './double';
+
+export function AppUi() {
+    return <TwoClientsPOC />;
+}
diff --git a/examples/src/dzi/double.tsx b/examples/src/dzi/double.tsx
@@ -0,0 +1,70 @@
+import { useContext, useEffect, useMemo, useRef, useState } from 'react';
+import { RenderServerProvider } from './render-server-provider';
+import React from 'react';
+import { DziView } from './dziView';
+import type { DziImage, DziRenderSettings } from '@alleninstitute/vis-dzi';
+import { Box2D, Vec2, type box2D } from '@alleninstitute/vis-geometry';
+
+const example: DziImage = {
+    format: 'jpeg',
+    imagesUrl:
+        'https://idk-etl-prod-download-bucket.s3.amazonaws.com/idf-23-10-pathology-images/pat_images_HPW332DMO29NC92JPWA/H20.33.029-A12-I6-primary/H20.33.029-A12-I6-primary_files/',
+    overlap: 1,
+    size: {
+        width: 13446,
+        height: 11596,
+    },
+    tileSize: 512,
+};
+const exampleDzi: DziImage = {
+    imagesUrl: 'https://openseadragon.github.io/example-images/highsmith/highsmith_files/',
+    format: 'jpg',
+    overlap: 2,
+    size: {
+        width: 7026,
+        height: 9221,
+    },
+    tileSize: 256,
+};
+const exampleSettings: DziRenderSettings = {
+    camera: {
+        screenSize: [500, 500],
+        view: Box2D.create([0, 0], [1, 1]),
+    },
+};
+
+export function TwoClientsPOC() {
+    const [view, setView] = useState<box2D>(Box2D.create([0, 0], [1, 1]));
+    const zoom = (e: React.WheelEvent<HTMLCanvasElement>) => {
+        const scale = e.deltaY > 0 ? 1.1 : 0.9;
+        const m = Box2D.midpoint(view);
+        const v = Box2D.translate(Box2D.scale(Box2D.translate(view, Vec2.scale(m, -1)), [scale, scale]), m);
+        setView(v);
+    };
+    const overlay = useRef<HTMLImageElement>(new Image());
+    useEffect(() => {
+        overlay.current.onload = () => {
+            console.log('loaded svg!');
+        };
+        overlay.current.src =
+            'https://idk-etl-prod-download-bucket.s3.amazonaws.com/idf-22-07-pathology-image-move/pat_images_JGCXWER774NLNWX2NNR/7179-A6-I6-MTG-classified/annotation.svg';
+    }, []);
+    return (
+        <RenderServerProvider>
+            <DziView
+                id="left"
+                svgOverlay={overlay.current}
+                dzi={example}
+                camera={{ ...exampleSettings.camera, view }}
+                wheel={zoom}
+            />
+            <DziView
+                id="right"
+                dzi={exampleDzi}
+                svgOverlay={overlay.current}
+                camera={{ ...exampleSettings.camera, view }}
+                wheel={zoom}
+            />
+        </RenderServerProvider>
+    );
+}
diff --git a/examples/src/dzi/dzi.ts b/examples/src/dzi/dzi.ts
@@ -0,0 +1,5 @@
+import { createRoot } from 'react-dom/client';
+import { AppUi } from './app';
+
+const uiroot = createRoot(document.getElementById('main')!);
+uiroot.render(AppUi());
diff --git a/examples/src/dzi/dziView.tsx b/examples/src/dzi/dziView.tsx
@@ -0,0 +1,110 @@
+import { useContext, useEffect, useRef, useState } from 'react';
+import {
+    buildDziRenderer,
+    type DziImage,
+    type DziRenderSettings,
+    type DziTile,
+    type GpuProps as CachedPixels,
+    buildAsyncDziRenderer,
+} from '@alleninstitute/vis-dzi';
+import React from 'react';
+import { buildAsyncRenderer, type RenderFrameFn } from '@alleninstitute/vis-scatterbrain';
+import { isEqual } from 'lodash';
+import { renderServerContext } from './render-server-provider';
+import { Vec2, type vec2 } from '@alleninstitute/vis-geometry';
+
+type Props = {
+    id: string;
+    dzi: DziImage;
+    svgOverlay: HTMLImageElement;
+    wheel: (e: React.WheelEvent<HTMLCanvasElement>) => void;
+} & DziRenderSettings;
+
+function buildCompositor(svg: HTMLImageElement, settings: DziRenderSettings) {
+    return (ctx: CanvasRenderingContext2D, image: ImageData) => {
+        const { width, height } = svg;
+        const { camera } = settings;
+        const svgSize: vec2 = [width, height];
+        const start = Vec2.mul(camera.view.minCorner, svgSize);
+        const wh = Vec2.sub(Vec2.mul(camera.view.maxCorner, svgSize), start);
+        const [sx, sy] = start;
+        const [sw, sh] = wh;
+        // first, draw the results from webGL
+        ctx.putImageData(image, 0, 0);
+        // then add our svg overlay
+        ctx.drawImage(svg, sx, sy, sw, sh, 0, 0, ctx.canvas.width, ctx.canvas.height);
+    };
+}
+
+export function DziView(props: Props) {
+    const { svgOverlay, camera, dzi, wheel, id } = props;
+    const server = useContext(renderServerContext);
+    const cnvs = useRef<HTMLCanvasElement>(null);
+
+    // this is a demo, so rather than work hard to have a referentially stable camera,
+    // we just memoize it like so to prevent over-rendering
+    const [cam, setCam] = useState(camera);
+    useEffect(() => {
+        if (!isEqual(cam, camera)) {
+            setCam(camera);
+        }
+    }, [camera]);
+
+    // the renderer needs WebGL for us to create it, and WebGL needs a canvas to exist, and that canvas needs to be the same canvas forever
+    // hence the awkwardness of refs + an effect to initialize the whole hting
+    const renderer =
+        useRef<
+            ReturnType<typeof buildAsyncRenderer<DziImage, DziTile, DziRenderSettings, string, string, CachedPixels>>
+        >();
+
+    useEffect(() => {
+        if (server && server.regl) {
+            renderer.current = buildAsyncDziRenderer(server.regl);
+        }
+        return () => {
+            if (cnvs.current) {
+                server?.destroyClient(cnvs.current);
+            }
+        };
+    }, [server]);
+
+    useEffect(() => {
+        if (server && renderer.current && cnvs.current) {
+            const renderMyData: RenderFrameFn<DziImage, DziTile> = (target, cache, callback) => {
+                if (renderer.current) {
+                    // erase the frame before we start drawing on it
+                    return renderer.current(dzi, { camera: cam }, callback, target, cache);
+                }
+                return null;
+            };
+            const compose = buildCompositor(svgOverlay, { camera: cam });
+            server.beginRendering(
+                renderMyData,
+                (e) => {
+                    switch (e.status) {
+                        case 'begin':
+                            server.regl?.clear({ framebuffer: e.target, color: [0, 0, 0, 0], depth: 1 });
+                            break;
+                        case 'progress':
+                            // wanna see the tiles as they arrive?
+                            e.server.copyToClient(compose);
+                            break;
+                        case 'finished': {
+                            e.server.copyToClient(compose);
+                        }
+                    }
+                },
+                cnvs.current
+            );
+        }
+    }, [server, renderer.current, cnvs.current, cam]);
+    return (
+        <canvas
+            id={id}
+            ref={cnvs}
+            onWheel={wheel}
+            width={camera.screenSize[0]}
+            height={camera.screenSize[1]}
+        ></canvas>
+    );
+}
diff --git a/examples/src/dzi/render-server-provider.tsx b/examples/src/dzi/render-server-provider.tsx
@@ -0,0 +1,13 @@
+import { RenderServer } from '@alleninstitute/vis-scatterbrain';
+import React, { createContext, useEffect, useRef, type PropsWithChildren } from 'react';
+
+export const renderServerContext = createContext<RenderServer | null>(null);
+
+export function RenderServerProvider(props: PropsWithChildren<{}>) {
+    const server = useRef<RenderServer>();
+    const { children } = props;
+    useEffect(() => {
+        server.current = new RenderServer([2048, 2048], []);
+    }, []);
+    return <renderServerContext.Provider value={server.current ?? null}>{children}</renderServerContext.Provider>;
+}
diff --git a/examples/vite.config.ts b/examples/vite.config.ts
@@ -3,6 +3,14 @@ import react from '@vitejs/plugin-react-swc';
 import path from 'path';
 // https://vitejs.dev/config/
 export default defineConfig({
+    build: {
+        rollupOptions: {
+            input: {
+                layers: path.resolve(__dirname, './layers.html'),
+                dzi: path.resolve(__dirname, './dzi.html'),
+            },
+        },
+    },
     plugins: [react()],
     resolve: {
         alias: {