Merge pull request #5896 from mozilla/feature/bit-pdf

Add pdf support

src/bit-components.js

@@ -137,6 +137,14 @@ export const MediaImage = defineComponent({
 });
 MediaImage.cacheKey[$isStringType] = true;
 
+export const NetworkedPDF = defineComponent({
+  pageNumber: Types.ui8
+});
+export const MediaPDF = defineComponent({
+  pageNumber: Types.ui8
+});
+MediaPDF.map = new Map();
+
 export const MediaVideo = defineComponent({
   autoPlay: Types.ui8
 });
@@ -178,6 +186,14 @@ export const ObjectMenu = defineComponent({
   scaleButtonRef: Types.eid,
   targetRef: Types.eid
 });
+// TODO: Store this data elsewhere, since only one or two will ever exist.
+export const PDFMenu = defineComponent({
+  prevButtonRef: Types.eid,
+  nextButtonRef: Types.eid,
+  pageLabelRef: Types.eid,
+  targetRef: Types.eid,
+  clearTargetTimer: Types.f64
+});
 export const ObjectMenuTarget = defineComponent();
 export const NetworkDebug = defineComponent();
 export const NetworkDebugRef = defineComponent({

src/bit-systems/media-loading.ts

@@ -6,15 +6,23 @@ import { ErrorObject } from "../prefabs/error-object";
 import { LoadingObject } from "../prefabs/loading-object";
 import { animate } from "../utils/animate";
 import { setNetworkedDataWithoutRoot } from "../utils/assign-network-ids";
-import { cancelable, coroutine, crClearTimeout, crTimeout, makeCancelable } from "../utils/coroutine";
+import { crClearTimeout, crNextFrame, crTimeout } from "../utils/coroutine";
+import { ClearFunction, JobRunner, withRollback } from "../utils/coroutine-utils";
 import { easeOutQuadratic } from "../utils/easing";
 import { renderAsEntity } from "../utils/jsx-entity";
 import { loadImage } from "../utils/load-image";
 import { loadModel } from "../utils/load-model";
+import { loadPDF } from "../utils/load-pdf";
 import { loadVideo } from "../utils/load-video";
 import { MediaType, mediaTypeName, resolveMediaInfo } from "../utils/media-utils";
 import { EntityID } from "../utils/networking-types";
 
+export function* waitForMediaLoaded(world: HubsWorld, eid: EntityID) {
+  while (hasComponent(world, MediaLoader, eid)) {
+    yield crNextFrame();
+  }
+}
+
 const loaderForMediaType = {
   [MediaType.IMAGE]: (
     world: HubsWorld,
@@ -25,7 +33,8 @@ const loaderForMediaType = {
   [MediaType.MODEL]: (
     world: HubsWorld,
     { accessibleUrl, contentType }: { accessibleUrl: string; contentType: string }
-  ) => loadModel(world, accessibleUrl, contentType, true)
+  ) => loadModel(world, accessibleUrl, contentType, true),
+  [MediaType.PDF]: (world: HubsWorld, { accessibleUrl }: { accessibleUrl: string }) => loadPDF(world, accessibleUrl)
 };
 
 export const MEDIA_LOADER_FLAGS = {
@@ -44,46 +53,45 @@ function resizeAndRecenter(world: HubsWorld, media: EntityID, eid: EntityID) {
   const box = new THREE.Box3();
   box.setFromObject(mediaObj);
 
-  let scale = 1;
+  let scalar = 1;
   if (resize) {
     const size = new THREE.Vector3();
     box.getSize(size);
-    const multiplier = hasComponent(world, GLTFModel, media) ? 0.5 : 1.0;
-    scale = multiplier / Math.max(size.x, size.y, size.z);
-    mediaObj.scale.setScalar(scale);
+    scalar = 1 / Math.max(size.x, size.y, size.z);
+    if (hasComponent(world, GLTFModel, media)) scalar = scalar * 0.5;
+    mediaObj.scale.multiplyScalar(scalar);
     mediaObj.matrixNeedsUpdate = true;
   }
 
   if (recenter) {
     const center = new THREE.Vector3();
     box.getCenter(center);
-    mediaObj.position.copy(center).multiplyScalar(-1 * scale);
+    mediaObj.position.copy(center).multiplyScalar(-1 * scalar);
     mediaObj.matrixNeedsUpdate = true;
   }
 }
 
 export function* animateScale(world: HubsWorld, media: EntityID) {
   const mediaObj = world.eid2obj.get(media)!;
-
   const onAnimate = ([position, scale]: [Vector3, Vector3]) => {
     mediaObj.position.copy(position);
     mediaObj.scale.copy(scale);
     mediaObj.matrixNeedsUpdate = true;
   };
-
-  const startScale = new Vector3().setScalar(0.0001);
-  const endScale = new Vector3().setScalar(mediaObj.scale.x);
-
-  const startPosition = new Vector3().copy(mediaObj.position).multiplyScalar(startScale.x);
+  const scalar = 0.001;
+  const startScale = new Vector3().copy(mediaObj.scale).multiplyScalar(scalar);
+  const endScale = new Vector3().copy(mediaObj.scale);
+  // The animation should affect the mediaObj as if its parent were being scaled:
+  // If mediaObj is offset from its parent (e.g. because it was recentered),
+  // then its position relative to its parent also needs to be scaled.
+  const startPosition = new Vector3().copy(mediaObj.position).multiplyScalar(scalar);
   const endPosition = new Vector3().copy(mediaObj.position);
-
-  // Set the initial position and yield one frame
-  // because the first frame that we render a new object is slow
+  // Animate once to set the initial state, then yield one frame
+  // because the first render of the new object may be slow
   // TODO: We could move uploading textures to the GPU to the loader,
   //       so that we don't hitch here
   onAnimate([startPosition, startScale]);
-  yield Promise.resolve();
-
+  yield crNextFrame();
   yield* animate({
     properties: [
       [startPosition, endPosition],
@@ -127,9 +135,9 @@ function* loadMedia(world: HubsWorld, eid: EntityID) {
     loadingObjEid = renderAsEntity(world, LoadingObject());
     add(world, loadingObjEid, eid);
   }, 400);
-  yield makeCancelable(() => loadingObjEid && removeEntity(world, loadingObjEid));
+  yield withRollback(Promise.resolve(), () => loadingObjEid && removeEntity(world, loadingObjEid));
   const src = APP.getString(MediaLoader.src[eid]);
-  let media;
+  let media: EntityID;
   try {
     const urlData = (yield resolveMediaInfo(src)) as MediaInfo;
     const loader = urlData.mediaType && loaderForMediaType[urlData.mediaType];
@@ -146,43 +154,34 @@ function* loadMedia(world: HubsWorld, eid: EntityID) {
   return media;
 }
 
-function* loadAndAnimateMedia(world: HubsWorld, eid: EntityID, signal: AbortSignal) {
-  const { value: media, canceled } = yield* cancelable(loadMedia(world, eid), signal);
-  if (!canceled) {
-    resizeAndRecenter(world, media, eid);
-    if (MediaLoader.flags[eid] & MEDIA_LOADER_FLAGS.IS_OBJECT_MENU_TARGET) {
-      addComponent(world, ObjectMenuTarget, eid);
-    }
-    add(world, media, eid);
-    setNetworkedDataWithoutRoot(world, APP.getString(Networked.id[eid])!, media);
-    if (MediaLoader.flags[eid] & MEDIA_LOADER_FLAGS.ANIMATE_LOAD) {
-      yield* animateScale(world, media);
-    }
-    removeComponent(world, MediaLoader, eid);
+function* loadAndAnimateMedia(world: HubsWorld, eid: EntityID, clearRollbacks: ClearFunction) {
+  if (MediaLoader.flags[eid] & MEDIA_LOADER_FLAGS.IS_OBJECT_MENU_TARGET) {
+    addComponent(world, ObjectMenuTarget, eid);
   }
+  const media = yield* loadMedia(world, eid);
+  clearRollbacks(); // After this point, normal entity cleanup will takes care of things
+
+  resizeAndRecenter(world, media, eid);
+  add(world, media, eid);
+  setNetworkedDataWithoutRoot(world, APP.getString(Networked.id[eid])!, media);
+  if (MediaLoader.flags[eid] & MEDIA_LOADER_FLAGS.ANIMATE_LOAD) {
+    yield* animateScale(world, media);
+  }
+  removeComponent(world, MediaLoader, eid);
 }
 
-const jobs = new Set();
-const abortControllers = new Map();
+const jobs = new JobRunner();
 const mediaLoaderQuery = defineQuery([MediaLoader]);
 const mediaLoaderEnterQuery = enterQuery(mediaLoaderQuery);
 const mediaLoaderExitQuery = exitQuery(mediaLoaderQuery);
 export function mediaLoadingSystem(world: HubsWorld) {
   mediaLoaderEnterQuery(world).forEach(function (eid) {
-    const ac = new AbortController();
-    abortControllers.set(eid, ac);
-    jobs.add(coroutine(loadAndAnimateMedia(world, eid, ac.signal)));
+    jobs.add(eid, clearRollbacks => loadAndAnimateMedia(world, eid, clearRollbacks));
   });
 
   mediaLoaderExitQuery(world).forEach(function (eid) {
-    const ac = abortControllers.get(eid);
-    ac.abort();
-    abortControllers.delete(eid);
+    jobs.stop(eid);
   });
 
-  jobs.forEach((c: any) => {
-    if (c().done) {
-      jobs.delete(c);
-    }
-  });
+  jobs.tick();
 }

src/bit-systems/object-spawner.ts

@@ -1,25 +1,19 @@
-import { addComponent, defineQuery, enterQuery, exitQuery, hasComponent } from "bitecs";
+import { addComponent, defineQuery, enterQuery, exitQuery } from "bitecs";
 import { HubsWorld } from "../app";
-import { FloatyObject, Held, HeldRemoteRight, Interacted, MediaLoader, ObjectSpawner } from "../bit-components";
+import { FloatyObject, Held, HeldRemoteRight, Interacted, ObjectSpawner } from "../bit-components";
 import { FLOATY_OBJECT_FLAGS } from "../systems/floaty-object-system";
 import { sleep } from "../utils/async-utils";
-import { coroutine, crNextFrame } from "../utils/coroutine";
+import { coroutine } from "../utils/coroutine";
 import { createNetworkedEntity } from "../utils/create-networked-entity";
 import { EntityID } from "../utils/networking-types";
 import { setMatrixWorld } from "../utils/three-utils";
-import { animateScale } from "./media-loading";
+import { animateScale, waitForMediaLoaded } from "./media-loading";
 
 export enum OBJECT_SPAWNER_FLAGS {
   /** Apply gravity to spawned objects */
   APPLY_GRAVITY = 1 << 0
 }
 
-function* waitForMediaLoaded(world: HubsWorld, eid: EntityID) {
-  while (hasComponent(world, MediaLoader, eid)) {
-    yield crNextFrame();
-  }
-}
-
 function* spawnObjectJob(world: HubsWorld, spawner: EntityID) {
   const spawned = createNetworkedEntity(world, "media", {
     src: APP.getString(ObjectSpawner.src[spawner]),

src/bit-systems/pdf-menu-system.ts

@@ -0,0 +1,102 @@
+import { defineQuery, entityExists, hasComponent } from "bitecs";
+import { Text } from "troika-three-text";
+import type { HubsWorld } from "../app";
+import { HoveredRemoteRight, Interacted, MediaPDF, NetworkedPDF, PDFMenu } from "../bit-components";
+import { anyEntityWith, findAncestorWithComponent } from "../utils/bit-utils";
+import type { EntityID } from "../utils/networking-types";
+import { takeOwnership } from "../utils/take-ownership";
+import { setMatrixWorld } from "../utils/three-utils";
+import { PDFResourcesMap } from "./pdf-system";
+
+function clicked(world: HubsWorld, eid: EntityID) {
+  return hasComponent(world, Interacted, eid);
+}
+
+function findPDFMenuTarget(world: HubsWorld, menu: EntityID, sceneIsFrozen: boolean) {
+  if (PDFMenu.targetRef[menu] && !entityExists(world, PDFMenu.targetRef[menu])) {
+    // Clear the invalid entity reference. (The pdf entity was removed).
+    PDFMenu.targetRef[menu] = 0;
+  }
+
+  if (sceneIsFrozen) {
+    PDFMenu.targetRef[menu] = 0;
+    return;
+  }
+
+  const hovered = hoveredQuery(world);
+  const target = hovered.map(eid => findAncestorWithComponent(world, MediaPDF, eid))[0] || 0;
+  if (target) {
+    PDFMenu.targetRef[menu] = target;
+    PDFMenu.clearTargetTimer[menu] = world.time.elapsed + 1000;
+    return;
+  }
+
+  if (hovered.some(eid => findAncestorWithComponent(world, PDFMenu, eid))) {
+    PDFMenu.clearTargetTimer[menu] = world.time.elapsed + 1000;
+    return;
+  }
+
+  if (world.time.elapsed > PDFMenu.clearTargetTimer[menu]) {
+    PDFMenu.targetRef[menu] = 0;
+    return;
+  }
+}
+
+function moveToTarget(world: HubsWorld, menu: EntityID) {
+  const targetObj = world.eid2obj.get(PDFMenu.targetRef[menu])!;
+  targetObj.updateMatrices();
+  const menuObj = world.eid2obj.get(menu)!;
+  setMatrixWorld(menuObj, targetObj.matrixWorld);
+}
+
+function wrapAround(n: number, min: number, max: number) {
+  // Wrap around [min, max] inclusively
+  // Assumes that n is only 1 more than max or 1 less than min
+  return n < min ? max : n > max ? min : n;
+}
+
+function setPage(world: HubsWorld, eid: EntityID, pageNumber: number) {
+  takeOwnership(world, eid);
+  NetworkedPDF.pageNumber[eid] = wrapAround(pageNumber, 1, PDFResourcesMap.get(eid)!.pdf.numPages);
+}
+
+function handleClicks(world: HubsWorld, menu: EntityID) {
+  if (clicked(world, PDFMenu.nextButtonRef[menu])) {
+    const pdf = PDFMenu.targetRef[menu];
+    setPage(world, pdf, NetworkedPDF.pageNumber[pdf] + 1);
+  } else if (clicked(world, PDFMenu.prevButtonRef[menu])) {
+    const pdf = PDFMenu.targetRef[menu];
+    setPage(world, pdf, NetworkedPDF.pageNumber[pdf] - 1);
+  }
+}
+
+function flushToObject3Ds(world: HubsWorld, menu: EntityID, frozen: boolean) {
+  const target = PDFMenu.targetRef[menu];
+  const visible = !!(target && !frozen);
+
+  const obj = world.eid2obj.get(menu)!;
+  obj.visible = visible;
+
+  [PDFMenu.prevButtonRef[menu], PDFMenu.nextButtonRef[menu]].forEach(buttonRef => {
+    const buttonObj = world.eid2obj.get(buttonRef)!;
+    // Parent visibility doesn't block raycasting, so we must set each button to be invisible
+    // TODO: Ensure that children of invisible entities aren't raycastable
+    buttonObj.visible = visible;
+  });
+
+  if (target) {
+    const numPages = PDFResourcesMap.get(target)!.pdf.numPages;
+    (world.eid2obj.get(PDFMenu.pageLabelRef[menu]) as Text).text = `${NetworkedPDF.pageNumber[target]} / ${numPages}`;
+  }
+}
+
+const hoveredQuery = defineQuery([HoveredRemoteRight]);
+export function pdfMenuSystem(world: HubsWorld, sceneIsFrozen: boolean) {
+  const menu = anyEntityWith(world, PDFMenu)!;
+  findPDFMenuTarget(world, menu, sceneIsFrozen);
+  if (PDFMenu.targetRef[menu]) {
+    moveToTarget(world, menu);
+    handleClicks(world, menu);
+  }
+  flushToObject3Ds(world, menu, sceneIsFrozen);
+}