From 0a70f7a58dbb22d683773f8b9034806ac99f2701 Mon Sep 17 00:00:00 2001 From: netpro2k Date: Wed, 25 Jan 2023 13:18:31 -0800 Subject: [PATCH 01/11] Add Material components and fix resolving noderefs --- src/app.ts | 3 + src/bit-components.js | 1 + src/components/gltf-model-plus.js | 24 ++++---- src/inflators/model.tsx | 82 ++++++++++++++++++++++----- src/systems/remove-object3D-system.js | 19 +++++-- src/utils/jsx-entity.ts | 12 +++- src/utils/three-utils.js | 5 ++ types/three.d.ts | 3 + 8 files changed, 114 insertions(+), 35 deletions(-) diff --git a/src/app.ts b/src/app.ts index f141956660..4095a477ac 100644 --- a/src/app.ts +++ b/src/app.ts @@ -13,6 +13,7 @@ import { EffectComposer, EffectPass } from "postprocessing"; import { Audio, AudioListener, + Material, Object3D, PerspectiveCamera, PositionalAudio, @@ -48,6 +49,7 @@ export interface HubsWorld extends IWorld { deletedNids: Set; nid2eid: Map; eid2obj: Map; + eid2mat: Map; time: { delta: number; elapsed: number; tick: number }; } @@ -113,6 +115,7 @@ export class App { this.store = store; // TODO: Create accessor / update methods for these maps / set this.world.eid2obj = new Map(); + this.world.eid2mat = new Map(); this.world.nid2eid = new Map(); this.world.deletedNids = new Set(); diff --git a/src/bit-components.js b/src/bit-components.js index 267c070b6d..c2fc8de60d 100644 --- a/src/bit-components.js +++ b/src/bit-components.js @@ -197,3 +197,4 @@ export const ObjectSpawner = defineComponent({ export const Billboard = defineComponent({ onlyY: Types.ui8 }); +export const MaterialTag = defineComponent(); diff --git a/src/components/gltf-model-plus.js b/src/components/gltf-model-plus.js index c87d8dce63..c823976384 100644 --- a/src/components/gltf-model-plus.js +++ b/src/components/gltf-model-plus.js @@ -12,7 +12,7 @@ import { getCustomGLTFParserURLResolver } from "../utils/media-url-utils"; import nextTick from "../utils/next-tick"; import { promisifyWorker } from "../utils/promisify-worker.js"; import qsTruthy from "../utils/qs_truthy"; -import { cloneObject3D } from "../utils/three-utils"; +import { cloneObject3D, disposeMaterial } from "../utils/three-utils"; import SketchfabZipWorker from "../workers/sketchfab-zip.worker.js"; THREE.Mesh.prototype.raycast = acceleratedRaycast; @@ -757,18 +757,9 @@ export async function loadGLTF(src, contentType, onProgress, jsonPreprocessor) { disposables.add(obj.geometry); } - if (obj.material) { - if (obj.material.map) disposables.add(obj.material.map); - if (obj.material.lightMap) disposables.add(obj.material.lightMap); - if (obj.material.bumpMap) disposables.add(obj.material.bumpMap); - if (obj.material.normalMap) disposables.add(obj.material.normalMap); - if (obj.material.specularMap) disposables.add(obj.material.specularMap); - if (obj.material.envMap) disposables.add(obj.material.envMap); - if (obj.material.aoMap) disposables.add(obj.material.aoMap); - if (obj.material.metalnessMap) disposables.add(obj.material.metalnessMap); - if (obj.material.roughnessMap) disposables.add(obj.material.roughnessMap); - if (obj.material.emissiveMap) disposables.add(obj.material.emissiveMap); - } + mapMaterials(obj, function (m) { + disposables.add(m); + }); const mozHubsComponents = obj.userData.gltfExtensions?.MOZ_hubs_components; if (mozHubsComponents) { @@ -784,9 +775,14 @@ export async function loadGLTF(src, contentType, onProgress, jsonPreprocessor) { } }); + scene.associations = gltf.parser.associations; scene.dispose = function dispose() { disposables.forEach(disposable => { - disposable.dispose(); + if (disposable.isMaterial) { + disposeMaterial(disposable); + } else { + disposable.dispose(); + } }); }; }); diff --git a/src/inflators/model.tsx b/src/inflators/model.tsx index e9798e87b0..e5608d96c8 100644 --- a/src/inflators/model.tsx +++ b/src/inflators/model.tsx @@ -1,8 +1,10 @@ -import { addComponent, addEntity } from "bitecs"; -import { Object3D } from "three"; +import { addComponent, addEntity, hasComponent } from "bitecs"; +import { Material, Object3D } from "three"; import { HubsWorld } from "../app"; -import { GLTFModel } from "../bit-components"; -import { addObject3DComponent, gltfInflatorExists, gltfInflators } from "../utils/jsx-entity"; +import { GLTFModel, MaterialTag } from "../bit-components"; +import { addMaterialComponent, addObject3DComponent, gltfInflatorExists, gltfInflators } from "../utils/jsx-entity"; +import { mapMaterials } from "../utils/material-utils"; +import { EntityID } from "../utils/networking-types"; function camelCase(s: string) { return s.replace(/-(\w)/g, (_, m) => m.toUpperCase()); @@ -10,21 +12,71 @@ function camelCase(s: string) { export type ModelParams = { model: Object3D }; +// These components are all handled in some special way, not through inflators +const ignoredComponents = ["visible", "frustum", "frustrum", "shadow", "networked"]; + +function inflateComponents( + world: HubsWorld, + eid: number, + components: { [componentName: string]: any }, + idx2eid: Map +) { + Object.keys(components).forEach(name => { + const inflatorName = camelCase(name); + if (ignoredComponents.includes(inflatorName)) return; + if (!gltfInflatorExists(inflatorName)) { + console.warn(`Failed to inflate unknown component called ${inflatorName}`, components[name]); + return; + } + + const props = components[name]; + Object.keys(components[name]).forEach(propName => { + const value = props[propName]; + const linkType = value?.__mhc_link_type; + if (linkType) { + if (linkType !== "node") { + throw new Error("Non node link types should be resolved before inflateModel is called"); + } + const existingEid = idx2eid.get(value.index); + if (existingEid) { + props[propName] = existingEid; + } else { + props[propName] = addEntity(world); + idx2eid.set(value.index, props[propName]); + } + } + }); + gltfInflators[inflatorName](world, eid, props); + }); +} + export function inflateModel(world: HubsWorld, rootEid: number, { model }: ModelParams) { const swap: [old: Object3D, replacement: Object3D][] = []; + const idx2eid = new Map(); model.traverse(obj => { - const components = obj.userData.gltfExtensions?.MOZ_hubs_components || {}; - const eid = obj === model ? rootEid : addEntity(world); - Object.keys(components).forEach(name => { - const inflatorName = camelCase(name); - if (inflatorName === "visible" || inflatorName === "frustum" || inflatorName === "shadow") { - // These are handled below - } else if (!gltfInflatorExists(inflatorName)) { - console.warn(`Failed to inflate unknown component called ${inflatorName}`, components[name]); - } else { - gltfInflators[inflatorName](world, eid, components[name]); - } + const gltfIndex: number | undefined = obj.userData.gltfIndex; + + let eid: number; + if (obj === model) { + eid = rootEid; + } else if (gltfIndex !== undefined && idx2eid.has(gltfIndex)) { + eid = idx2eid.get(gltfIndex)!; + } else { + eid = addEntity(world); + } + + if (gltfIndex !== undefined) idx2eid.set(gltfIndex, eid); + + const components = obj.userData.gltfExtensions?.MOZ_hubs_components; + if (components) inflateComponents(world, eid, components, idx2eid); + + mapMaterials(obj, function (mat: Material) { + const eid = mat.eid || addEntity(world); + if (!hasComponent(world, MaterialTag, eid)) addMaterialComponent(world, eid, mat); + const components = mat.userData.gltfExtensions?.MOZ_hubs_components; + if (components) inflateComponents(world, eid, components, idx2eid); }); + const replacement = world.eid2obj.get(eid); if (replacement) { if (obj.type !== "Object3D") { diff --git a/src/systems/remove-object3D-system.js b/src/systems/remove-object3D-system.js index a9c1c2c09d..ff6e635f5b 100644 --- a/src/systems/remove-object3D-system.js +++ b/src/systems/remove-object3D-system.js @@ -10,7 +10,8 @@ import { Slice9, Text, VideoMenu, - Skybox + Skybox, + MaterialTag } from "../bit-components"; import { gltfCache } from "../components/gltf-model-plus"; import { releaseTextureByKey } from "../utils/load-texture"; @@ -78,11 +79,17 @@ const cleanupSkyboxes = cleanupObjOnExit(Skybox, obj => { // When we remove an AFRAME entity, AFRAME will call `removeEntity` for all of its descendants, // which means we will remove each descendent from its parent. const exitedObject3DQuery = exitQuery(defineQuery([Object3DTag])); +const exitedMaterialQuery = exitQuery(defineQuery([MaterialTag])); export function removeObject3DSystem(world) { - function removeFromMap(eid) { + function removeObjFromMap(eid) { const o = world.eid2obj.get(eid); world.eid2obj.delete(eid); - o.eid = null; + o.eid = 0; + } + function removeFromMatMap(eid) { + const m = world.eid2mat.get(eid); + world.eid2mat.delete(eid); + m.eid = 0; } // TODO write removeObject3DEntity to do this work up-front, @@ -105,6 +112,7 @@ export function removeObject3DSystem(world) { }); // cleanup any component specific resources + // NOTE These are done here as opposed to in other systems because the eid2obj and eid2mat maps are cleared of removed entities at the end of this system. cleanupGLTFs(world); cleanupSlice9s(world); cleanupTexts(world); @@ -116,6 +124,7 @@ export function removeObject3DSystem(world) { cleanupSkyboxes(world); // Finally remove all the entities we just removed from the eid2obj map - entities.forEach(removeFromMap); - exitedObject3DQuery(world).forEach(removeFromMap); + entities.forEach(removeObjFromMap); + exitedObject3DQuery(world).forEach(removeObjFromMap); + exitedMaterialQuery(world).forEach(removeFromMatMap); } diff --git a/src/utils/jsx-entity.ts b/src/utils/jsx-entity.ts index 671c463f14..8621a35981 100644 --- a/src/utils/jsx-entity.ts +++ b/src/utils/jsx-entity.ts @@ -57,7 +57,7 @@ import { import { inflateSpawnpoint, inflateWaypoint, WaypointParams } from "../inflators/waypoint"; import { inflateReflectionProbe, ReflectionProbeParams } from "../inflators/reflection-probe"; import { HubsWorld } from "../app"; -import { Group, Object3D, Texture, VideoTexture } from "three"; +import { Group, Material, Object3D, Texture, VideoTexture } from "three"; import { AlphaMode } from "./create-image-mesh"; import { MediaLoaderParams } from "../inflators/media-loader"; import { preload } from "./preload"; @@ -174,6 +174,16 @@ export function swapObject3DComponent(world: HubsWorld, eid: number, obj: Object return eid; } +export function addMaterialComponent(world: HubsWorld, eid: number, mat: Material) { + if (hasComponent(world, MaterialTag, eid)) { + throw new Error("Tried to add an Material tag to an entity that already has one"); + } + addComponent(world, MaterialTag, eid); + world.eid2mat.set(eid, mat); + mat.eid = eid; + return eid; +} + // TODO HACK gettting internal bitecs symbol, should expose an API to check a properties type const $isEidType = Object.getOwnPropertySymbols(CameraTool.screenRef).find(s => s.description === "isEidType"); diff --git a/src/utils/three-utils.js b/src/utils/three-utils.js index b250524395..160b5dabe3 100644 --- a/src/utils/three-utils.js +++ b/src/utils/three-utils.js @@ -1,3 +1,5 @@ +import { removeEntity } from "bitecs"; + const tempVector3 = new THREE.Vector3(); const tempQuaternion = new THREE.Quaternion(); @@ -31,6 +33,9 @@ export function disposeMaterial(mtrl) { if (mtrl.roughnessMap) mtrl.roughnessMap.dispose(); if (mtrl.emissiveMap) mtrl.emissiveMap.dispose(); mtrl.dispose(); + if (mtrl.eid) { + removeEntity(APP.world, mtrl.eid); + } } export function disposeNode(node) { diff --git a/types/three.d.ts b/types/three.d.ts index 9bf129b187..a49f1aac19 100644 --- a/types/three.d.ts +++ b/types/three.d.ts @@ -9,6 +9,9 @@ declare module "three" { el?: AElement; updateMatrices: (forceLocalUpdate?: boolean, forceWorldUpdate?: boolean, skipParents?: boolean) => void; } + interface Material { + eid?: number; + } interface Mesh { reflectionProbeMode: "static" | "dynamic" | false; } From 0824568b554d185040e403ce367ddeadf866f54b Mon Sep 17 00:00:00 2001 From: netpro2k Date: Wed, 25 Jan 2023 13:29:12 -0800 Subject: [PATCH 02/11] video-texture-target & video-texture-source --- src/bit-components.js | 8 ++ src/bit-systems/camera-tool.js | 64 +----------- src/bit-systems/video-texture.ts | 132 +++++++++++++++++++++++++ src/components/video-texture-target.js | 4 +- src/inflators/video-texture-target.ts | 24 +++++ src/systems/hubs-systems.ts | 2 + src/utils/jsx-entity.ts | 11 ++- types/aframe.d.ts | 13 ++- 8 files changed, 194 insertions(+), 64 deletions(-) create mode 100644 src/bit-systems/video-texture.ts create mode 100644 src/inflators/video-texture-target.ts diff --git a/src/bit-components.js b/src/bit-components.js index c2fc8de60d..e807a1933d 100644 --- a/src/bit-components.js +++ b/src/bit-components.js @@ -198,3 +198,11 @@ export const Billboard = defineComponent({ onlyY: Types.ui8 }); export const MaterialTag = defineComponent(); +export const VideoTextureSource = defineComponent({ + fps: Types.ui8, + resolution: [Types.ui16, 2] +}); +export const VideoTextureTarget = defineComponent({ + srcNode: Types.eid, + flags: Types.ui8 +}); diff --git a/src/bit-systems/camera-tool.js b/src/bit-systems/camera-tool.js index 88e387ced2..bad9baa5c4 100644 --- a/src/bit-systems/camera-tool.js +++ b/src/bit-systems/camera-tool.js @@ -19,6 +19,7 @@ import { SOUND_CAMERA_TOOL_COUNTDOWN, SOUND_CAMERA_TOOL_TOOK_SNAPSHOT } from ".. import { paths } from "../systems/userinput/paths"; import { ObjectTypes } from "../object-types"; import { anyEntityWith } from "../utils/bit-utils"; +import { updateRenderTarget } from "./video-texture"; // Prefer h264 if available due to faster decoding speec on most platforms const videoCodec = ["h264", "vp9,opus", "vp8,opus", "vp9", "vp8"].find( @@ -129,64 +130,6 @@ function endRecording(world, camera, cancel) { APP.hubChannel.endRecording(); } -function updateRenderTarget(world, camera) { - const sceneEl = AFRAME.scenes[0]; - const renderer = AFRAME.scenes[0].renderer; - - const tmpVRFlag = renderer.xr.enabled; - renderer.xr.enabled = false; - - // TODO we are doing this because aframe uses this hook for tock. - // Namely to capture what camera was rendering. We don't actually use that in any of our tocks. - // Also tock can likely go away as a concept since we can just direclty order things after render in raf if we want to. - const tmpOnAfterRender = sceneEl.object3D.onAfterRender; - delete sceneEl.object3D.onAfterRender; - - // TODO this assumption is now not true since we are not running after render. We should probably just permentently turn of autoUpdate and run matrix updates at a point we wnat to. - // The entire scene graph matrices should already be updated - // in tick(). They don't need to be recomputed again in tock(). - const tmpAutoUpdate = sceneEl.object3D.autoUpdate; - sceneEl.object3D.autoUpdate = false; - - const bubbleSystem = AFRAME.scenes[0].systems["personal-space-bubble"]; - const boneVisibilitySystem = AFRAME.scenes[0].systems["hubs-systems"].boneVisibilitySystem; - - if (bubbleSystem) { - for (let i = 0, l = bubbleSystem.invaders.length; i < l; i++) { - bubbleSystem.invaders[i].disable(); - } - // HACK, bone visibility typically takes a tick to update, but since we want to be able - // to have enable() and disable() be reflected this frame, we need to do it immediately. - boneVisibilitySystem.tick(); - // scene.autoUpdate will be false so explicitly update the world matrices - boneVisibilitySystem.updateMatrices(); - } - - const renderTarget = renderTargets.get(camera); - renderTarget.needsUpdate = false; - renderTarget.lastUpdated = world.time.elapsed; - - const tmpRenderTarget = renderer.getRenderTarget(); - renderer.setRenderTarget(renderTarget); - renderer.clearDepth(); - renderer.render(sceneEl.object3D, world.eid2obj.get(CameraTool.cameraRef[camera])); - renderer.setRenderTarget(tmpRenderTarget); - - renderer.xr.enabled = tmpVRFlag; - sceneEl.object3D.onAfterRender = tmpOnAfterRender; - sceneEl.object3D.autoUpdate = tmpAutoUpdate; - - if (bubbleSystem) { - for (let i = 0, l = bubbleSystem.invaders.length; i < l; i++) { - bubbleSystem.invaders[i].enable(); - } - // HACK, bone visibility typically takes a tick to update, but since we want to be able - // to have enable() and disable() be reflected this frame, we need to do it immediately. - boneVisibilitySystem.tick(); - boneVisibilitySystem.updateMatrices(); - } -} - function updateUI(world, camera) { const snapMenuObj = world.eid2obj.get(CameraTool.snapMenuRef[camera]); const snapBtnObj = world.eid2obj.get(CameraTool.snapRef[camera]); @@ -404,7 +347,10 @@ export function cameraToolSystem(world) { world.time.tick % allCameras.length === i && elapsed > renderTarget.lastUpdated + VIEWFINDER_UPDATE_RATE) ) { - updateRenderTarget(world, camera); + // TODO camera tool may be able to just direclty use video-texture-target/source + updateRenderTarget(world, renderTarget, CameraTool.cameraRef[camera]); + renderTarget.needsUpdate = false; + renderTarget.lastUpdated = world.time.elapsed; if (CameraTool.state[camera] === CAMERA_STATE.RECORDING_VIDEO) { videoRecorders.get(camera).captureFrame(renderTargets.get(camera)); } diff --git a/src/bit-systems/video-texture.ts b/src/bit-systems/video-texture.ts new file mode 100644 index 0000000000..444a80b049 --- /dev/null +++ b/src/bit-systems/video-texture.ts @@ -0,0 +1,132 @@ +import { defineQuery, enterQuery, exitQuery } from "bitecs"; +import { + Camera, + LinearFilter, + MeshStandardMaterial, + NearestFilter, + PerspectiveCamera, + RGBAFormat, + sRGBEncoding, + WebGLRenderTarget +} from "three"; +import { HubsWorld } from "../app"; +import { MaterialTag, VideoTextureSource, VideoTextureTarget } from "../bit-components"; +import { Layers } from "../camera-layers"; +import { EntityID } from "../utils/networking-types"; + +function noop() {} + +export function updateRenderTarget(world: HubsWorld, renderTarget: WebGLRenderTarget, camera: EntityID) { + const sceneEl = AFRAME.scenes[0]; + const renderer = AFRAME.scenes[0].renderer; + + const tmpVRFlag = renderer.xr.enabled; + renderer.xr.enabled = false; + + // TODO we are doing this because aframe uses this hook for tock. + // Namely to capture what camera was rendering. We don't actually use that in any of our tocks. + // Also tock can likely go away as a concept since we can just direclty order things after render in raf if we want to. + const tmpOnAfterRender = sceneEl.object3D.onAfterRender; + sceneEl.object3D.onAfterRender = noop; + + const bubbleSystem = AFRAME.scenes[0].systems["personal-space-bubble"]; + const boneVisibilitySystem = AFRAME.scenes[0].systems["hubs-systems"].boneVisibilitySystem; + + if (bubbleSystem) { + for (let i = 0, l = bubbleSystem.invaders.length; i < l; i++) { + bubbleSystem.invaders[i].disable(); + } + // HACK, bone visibility typically takes a tick to update, but since we want to be able + // to have enable() and disable() be reflected this frame, we need to do it immediately. + boneVisibilitySystem.tick(); + // scene.autoUpdate will be false so explicitly update the world matrices + boneVisibilitySystem.updateMatrices(); + } + + const tmpRenderTarget = renderer.getRenderTarget(); + renderer.setRenderTarget(renderTarget); + renderer.clearDepth(); + renderer.render(sceneEl.object3D, world.eid2obj.get(camera)! as Camera); + renderer.setRenderTarget(tmpRenderTarget); + + renderer.xr.enabled = tmpVRFlag; + sceneEl.object3D.onAfterRender = tmpOnAfterRender; + + if (bubbleSystem) { + for (let i = 0, l = bubbleSystem.invaders.length; i < l; i++) { + bubbleSystem.invaders[i].enable(); + } + // HACK, bone visibility typically takes a tick to update, but since we want to be able + // to have enable() and disable() be reflected this frame, we need to do it immediately. + boneVisibilitySystem.tick(); + boneVisibilitySystem.updateMatrices(); + } +} + +interface SourceData { + renderTarget: WebGLRenderTarget; + lastUpdated: number; +} + +const sourceData = new Map(); + +const videoTextureSourceQuery = defineQuery([VideoTextureSource]); +const enteredVideoTextureSourcesQuery = enterQuery(videoTextureSourceQuery); +const exitedVideoTextureSourcesQuery = exitQuery(videoTextureSourceQuery); +const enteredVideoTextureTargetsQuery = enterQuery(defineQuery([VideoTextureTarget, MaterialTag])); +export function videoTextureSystem(world: HubsWorld) { + enteredVideoTextureSourcesQuery(world).forEach(function (eid) { + const camera = world.eid2obj.get(eid)! as PerspectiveCamera; + if (!(camera && camera.isCamera)) { + console.error("video-texture-source added to an entity without a camera"); + return; + } + + camera.layers.enable(Layers.CAMERA_LAYER_THIRD_PERSON_ONLY); + + const resolution = VideoTextureSource.resolution[eid]; + camera.aspect = resolution[0] / resolution[1]; + + // TODO currently if a video-texture-source tries to render itself it will fail with a warning. + // If we want to support this we will need 2 render targets to swap back and forth. + const renderTarget = new WebGLRenderTarget(resolution[0], resolution[1], { + format: RGBAFormat, + minFilter: LinearFilter, + magFilter: NearestFilter, + encoding: sRGBEncoding + }); + + // Since we are rendering directly to a texture we need to flip it vertically + // See https://github.com/mozilla/hubs/pull/4126#discussion_r610120237 + renderTarget.texture.matrixAutoUpdate = false; + renderTarget.texture.matrix.scale(1, -1); + renderTarget.texture.matrix.translate(0, 1); + + sourceData.set(eid, { renderTarget, lastUpdated: 0 }); + }); + exitedVideoTextureSourcesQuery(world).forEach(function (eid) { + const data = sourceData.get(eid); + if (data) { + data.renderTarget.dispose(); + sourceData.delete(eid); + } + }); + enteredVideoTextureTargetsQuery(world).forEach(function (eid) { + const data = sourceData.get(VideoTextureTarget.srcNode[eid]); + if (!data) { + console.error("video-texture-target unable to get source"); + return; + } + const mat = world.eid2mat.get(eid)! as MeshStandardMaterial; + mat.color.setHex(0x00ff00); + mat.map = data.renderTarget.texture; + mat.emissiveMap = data.renderTarget.texture; + }); + videoTextureSourceQuery(world).forEach(function (eid) { + const data = sourceData.get(eid); + if (data && world.time.elapsed > data.lastUpdated + 1000 / VideoTextureSource.fps[eid]) { + updateRenderTarget(world, data.renderTarget, eid); + data.lastUpdated = world.time.elapsed; + } + }); +} diff --git a/src/components/video-texture-target.js b/src/components/video-texture-target.js index eabd47e9c9..331c57f012 100644 --- a/src/components/video-texture-target.js +++ b/src/components/video-texture-target.js @@ -20,13 +20,13 @@ AFRAME.registerComponent("video-texture-source", { init() { this.camera = findNode(this.el.object3D, n => n.isCamera); - this.camera.layers.enable(Layers.CAMERA_LAYER_THIRD_PERSON_ONLY); - if (!this.camera) { console.warn("video-texture-source added to an entity without a camera"); return; } + this.camera.layers.enable(Layers.CAMERA_LAYER_THIRD_PERSON_ONLY); + this.camera.aspect = this.data.resolution[0] / this.data.resolution[1]; // TODO currently if a video-texture-source tries to render itself it will fail with a warning. diff --git a/src/inflators/video-texture-target.ts b/src/inflators/video-texture-target.ts new file mode 100644 index 0000000000..0ce5be6e23 --- /dev/null +++ b/src/inflators/video-texture-target.ts @@ -0,0 +1,24 @@ +import { addComponent } from "bitecs"; +import { HubsWorld } from "../app"; +import { VideoTextureTarget } from "../bit-components"; +import { EntityID } from "../utils/networking-types"; + +export const VIDEO_TEXTURE_TARGET_FLAGS = { + TARGET_BASE_MAP: 1 << 0, + TARGET_EMISSIVE_MAP: 1 << 1 +}; + +export interface VideoTextureTargetParams { + srcNode: EntityID; + targetBaseColorMap: boolean; + targetEmissiveMap: boolean; +} + +export function inflateVideoTextureTarget(world: HubsWorld, eid: number, props: VideoTextureTargetParams) { + addComponent(world, VideoTextureTarget, eid); + VideoTextureTarget.srcNode[eid] = props.srcNode; + let flags = 0; + if (props.targetBaseColorMap) flags |= VIDEO_TEXTURE_TARGET_FLAGS.TARGET_BASE_MAP; + if (props.targetEmissiveMap) flags |= VIDEO_TEXTURE_TARGET_FLAGS.TARGET_EMISSIVE_MAP; + VideoTextureTarget.flags[eid] = flags; +} diff --git a/src/systems/hubs-systems.ts b/src/systems/hubs-systems.ts index 0c899769da..64b904b441 100644 --- a/src/systems/hubs-systems.ts +++ b/src/systems/hubs-systems.ts @@ -61,6 +61,7 @@ import qsTruthy from "../utils/qs_truthy"; import { waypointSystem } from "../bit-systems/waypoint"; import { objectSpawnerSystem } from "../bit-systems/object-spawner"; import { billboardSystem } from "../bit-systems/billboard"; +import { videoTextureSystem } from "../bit-systems/video-texture"; declare global { interface Window { @@ -233,6 +234,7 @@ export function mainTick(xrFrame: XRFrame, renderer: WebGLRenderer, scene: Scene hubsSystems.audioZonesSystem.tick(hubsSystems.el); hubsSystems.gainSystem.tick(); hubsSystems.nameTagSystem.tick(); + videoTextureSystem(world); deleteEntitySystem(world, aframeSystems.userinput); destroyAtExtremeDistanceSystem(world); diff --git a/src/utils/jsx-entity.ts b/src/utils/jsx-entity.ts index 8621a35981..aabd45011c 100644 --- a/src/utils/jsx-entity.ts +++ b/src/utils/jsx-entity.ts @@ -34,7 +34,9 @@ import { NetworkDebug, WaypointPreview, NetworkedFloatyObject, - Billboard + Billboard, + MaterialTag, + VideoTextureSource } from "../bit-components"; import { inflateMediaLoader } from "../inflators/media-loader"; import { inflateMediaFrame } from "../inflators/media-frame"; @@ -65,6 +67,7 @@ import { DirectionalLightParams, inflateDirectionalLight } from "../inflators/di import { ProjectionMode } from "./projection-mode"; import { inflateSkybox, SkyboxParams } from "../inflators/skybox"; import { inflateSpawner, SpawnerParams } from "../inflators/spawner"; +import { inflateVideoTextureTarget, VideoTextureTargetParams } from "../inflators/video-texture-target"; preload( new Promise(resolve => { @@ -323,6 +326,8 @@ export interface GLTFComponentData extends ComponentData { navMesh?: true; waypoint?: WaypointParams; spawner: SpawnerParams; + videoTextureTarget: VideoTextureTargetParams; + videoTextureSource: { fps: number; resolution: [x: number, y: number] }; // deprecated spawnPoint?: true; @@ -410,7 +415,9 @@ export const gltfInflators: Required<{ [K in keyof GLTFComponentData]: InflatorF background: inflateBackground, spawnPoint: inflateSpawnpoint, skybox: inflateSkybox, - spawner: inflateSpawner + spawner: inflateSpawner, + videoTextureTarget: inflateVideoTextureTarget, + videoTextureSource: createDefaultInflator(VideoTextureSource), }; function jsxInflatorExists(name: string): name is keyof JSXComponentData { diff --git a/types/aframe.d.ts b/types/aframe.d.ts index 7a041fd8d4..8bfb280087 100644 --- a/types/aframe.d.ts +++ b/types/aframe.d.ts @@ -1,5 +1,5 @@ declare module "aframe" { - import { Scene, Clock, Object3D, Mesh } from "three"; + import { Scene, Clock, Object3D, Mesh, WebGLRenderer } from "three"; interface AElement extends HTMLElement { object3D: Object3D; @@ -84,9 +84,19 @@ declare module "aframe" { mesh?: Mesh; } + interface PersonalSpaceInvader extends AComponent { + enable(): void; + disable(): void; + } + + interface PersonalSpaceBubbleSystem extends ASystem { + invaders: PersonalSpaceInvader[]; + } + interface AScene extends AElement { object3D: Scene; renderStarted: boolean; + renderer: WebGLRenderer; tick(time: number, delta: number): void; isPlaying: boolean; behaviors: { @@ -96,6 +106,7 @@ declare module "aframe" { systemNames: Array; systems: { "hubs-systems": HubsSystems; + "personal-space-bubble": PersonalSpaceBubbleSystem; userinput: UserInputSystem; /** @deprecated see bit-interaction-system */ interaction: InteractionSystem; From 07c1156a21ba18ffaf8c632092514459dbbfedf6 Mon Sep 17 00:00:00 2001 From: netpro2k Date: Wed, 25 Jan 2023 13:30:50 -0800 Subject: [PATCH 03/11] Implement uv-scroll --- src/bit-components.js | 5 ++++ src/bit-systems/uv-scroll.ts | 55 ++++++++++++++++++++++++++++++++++++ src/inflators/uv-scroll.ts | 20 +++++++++++++ src/systems/hubs-systems.ts | 2 ++ src/utils/jsx-entity.ts | 3 ++ 5 files changed, 85 insertions(+) create mode 100644 src/bit-systems/uv-scroll.ts create mode 100644 src/inflators/uv-scroll.ts diff --git a/src/bit-components.js b/src/bit-components.js index e807a1933d..1d7c72a61d 100644 --- a/src/bit-components.js +++ b/src/bit-components.js @@ -198,6 +198,11 @@ export const Billboard = defineComponent({ onlyY: Types.ui8 }); export const MaterialTag = defineComponent(); +export const UVScroll = defineComponent({ + speed: [Types.f32, 2], + increment: [Types.f32, 2], + offset: [Types.f32, 2] +}); export const VideoTextureSource = defineComponent({ fps: Types.ui8, resolution: [Types.ui16, 2] diff --git a/src/bit-systems/uv-scroll.ts b/src/bit-systems/uv-scroll.ts new file mode 100644 index 0000000000..a74b997810 --- /dev/null +++ b/src/bit-systems/uv-scroll.ts @@ -0,0 +1,55 @@ +import { addComponent, defineQuery, hasComponent, removeComponent } from "bitecs"; +import { Material, Mesh, MeshBasicMaterial } from "three"; +import { HubsWorld } from "../app"; +import { MaterialTag, Object3DTag, UVScroll } from "../bit-components"; +import { mapMaterials } from "../utils/material-utils"; + +// We wanted uv-scroll to be a component on materials not objects. The original uv-scroll component predated +// the concept of "material components". Also in AFRAME, material components ended up being on objects anyway +// so it didn't make a practical difference. This corrects the behaviour to act how we want. +const uvScrollObjectsQuery = defineQuery([UVScroll, Object3DTag]); +function migrateLegacyComponents(world: HubsWorld) { + uvScrollObjectsQuery(world).forEach(function (eid) { + const obj = world.eid2obj.get(eid)!; + const mat = (obj as Mesh).material; + // TODO We will warn once we modify the Blender addon to put these components on materials instead + // console.warn( + // "The uv-scroll component should be added directly to materials not objects, transferring to object's material" + // ); + if (!mat) { + console.error("uv-scroll component added to an object without a Material"); + } else { + mapMaterials(obj, function (mat: Material) { + if (hasComponent(world, UVScroll, mat.eid!)) { + console.warn( + "Multiple uv-scroll instances added to objects sharing a material, only the speed/increment from the first one will have any effect" + ); + } else { + addComponent(world, UVScroll, mat.eid!); + UVScroll.speed[mat.eid!].set(UVScroll.speed[eid]); + UVScroll.increment[mat.eid!].set(UVScroll.increment[eid]); + } + }); + } + removeComponent(world, UVScroll, eid); + }); +} + +const uvScrollQuery = defineQuery([UVScroll, MaterialTag]); +export function uvScrollSystem(world: HubsWorld) { + migrateLegacyComponents(world); + uvScrollQuery(world).forEach(function (eid) { + const map = (world.eid2mat.get(eid)! as MeshBasicMaterial).map; + if (!map) return; // This would not exactly be expected to happen but is not a "bug" either. There is just no work to do in this case. + + const offset = UVScroll.offset[eid]; + const speed = UVScroll.speed[eid]; + const scale = world.time.delta / 1000; + offset[0] = (offset[0] + speed[0] * scale) % 1.0; + offset[1] = (offset[1] + speed[1] * scale) % 1.0; + + const increment = UVScroll.increment[eid]; + map.offset.x = increment[0] ? offset[0] - (offset[0] % increment[0]) : offset[0]; + map.offset.y = increment[1] ? offset[1] - (offset[1] % increment[1]) : offset[1]; + }); +} diff --git a/src/inflators/uv-scroll.ts b/src/inflators/uv-scroll.ts new file mode 100644 index 0000000000..2b2c6c523c --- /dev/null +++ b/src/inflators/uv-scroll.ts @@ -0,0 +1,20 @@ +import { addComponent } from "bitecs"; +import { HubsWorld } from "../app"; +import { UVScroll } from "../bit-components"; + +export interface UVScrollParams { + speed: { + x: number; + y: number; + }; + increment?: { + x: number; + y: number; + }; +} + +export function inflateUVScroll(world: HubsWorld, eid: number, props: UVScrollParams) { + addComponent(world, UVScroll, eid); + UVScroll.speed[eid].set([props.speed.x, props.speed.y]); + UVScroll.increment[eid].set([props.increment?.x || 0, props.increment?.y || 0]); +} diff --git a/src/systems/hubs-systems.ts b/src/systems/hubs-systems.ts index 64b904b441..33a06964d1 100644 --- a/src/systems/hubs-systems.ts +++ b/src/systems/hubs-systems.ts @@ -62,6 +62,7 @@ import { waypointSystem } from "../bit-systems/waypoint"; import { objectSpawnerSystem } from "../bit-systems/object-spawner"; import { billboardSystem } from "../bit-systems/billboard"; import { videoTextureSystem } from "../bit-systems/video-texture"; +import { uvScrollSystem } from "../bit-systems/uv-scroll"; declare global { interface Window { @@ -226,6 +227,7 @@ export function mainTick(xrFrame: XRFrame, renderer: WebGLRenderer, scene: Scene hubsSystems.menuAnimationSystem.tick(t); hubsSystems.spriteSystem.tick(t, dt); hubsSystems.uvScrollSystem.tick(dt); + uvScrollSystem(world); hubsSystems.shadowSystem.tick(); objectMenuSystem(world, sceneEl.is("frozen"), APP.hubChannel!); videoMenuSystem(world, aframeSystems.userinput); diff --git a/src/utils/jsx-entity.ts b/src/utils/jsx-entity.ts index aabd45011c..25652e0dfd 100644 --- a/src/utils/jsx-entity.ts +++ b/src/utils/jsx-entity.ts @@ -68,6 +68,7 @@ import { ProjectionMode } from "./projection-mode"; import { inflateSkybox, SkyboxParams } from "../inflators/skybox"; import { inflateSpawner, SpawnerParams } from "../inflators/spawner"; import { inflateVideoTextureTarget, VideoTextureTargetParams } from "../inflators/video-texture-target"; +import { inflateUVScroll, UVScrollParams } from "../inflators/uv-scroll"; preload( new Promise(resolve => { @@ -326,6 +327,7 @@ export interface GLTFComponentData extends ComponentData { navMesh?: true; waypoint?: WaypointParams; spawner: SpawnerParams; + uvScroll: UVScrollParams; videoTextureTarget: VideoTextureTargetParams; videoTextureSource: { fps: number; resolution: [x: number, y: number] }; @@ -418,6 +420,7 @@ export const gltfInflators: Required<{ [K in keyof GLTFComponentData]: InflatorF spawner: inflateSpawner, videoTextureTarget: inflateVideoTextureTarget, videoTextureSource: createDefaultInflator(VideoTextureSource), + uvScroll: inflateUVScroll }; function jsxInflatorExists(name: string): name is keyof JSXComponentData { From d90f9ec12231356c209a384853cebcb2c0ea3f3d Mon Sep 17 00:00:00 2001 From: netpro2k Date: Wed, 25 Jan 2023 13:31:13 -0800 Subject: [PATCH 04/11] Add Material entities to ECS sidebar --- .../debug-panel/ECSSidebar.js | 54 ++++++++++++++----- .../debug-panel/ECSSidebar.scss | 5 ++ 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/src/react-components/debug-panel/ECSSidebar.js b/src/react-components/debug-panel/ECSSidebar.js index f7009e66e4..c9c9b8300e 100644 --- a/src/react-components/debug-panel/ECSSidebar.js +++ b/src/react-components/debug-panel/ECSSidebar.js @@ -51,6 +51,25 @@ function Object3DItem(props) { ); } +function MaterialItem(props) { + const { mat, setSelectedObj } = props; + const displayName = formatObjectName(mat); + return ( +
+
{ + e.preventDefault(); + setSelectedObj(mat); + }} + > + {displayName} + {` [${mat.eid}]`} +
+
+ ); +} + export function formatComponentProps(eid, component) { const formatted = Object.keys(component).reduce((str, k, i, arr) => { const val = component[k][eid]; @@ -123,6 +142,7 @@ function RefreshButton({ onClick }) { } const object3dQuery = defineQuery([bitComponents.Object3DTag]); +const materialQuery = defineQuery([bitComponents.MaterialTag]); function ECSDebugSidebar({ onClose, toggleObjExpand, @@ -135,6 +155,7 @@ function ECSDebugSidebar({ const orphaned = object3dQuery(APP.world) .map(eid => APP.world.eid2obj.get(eid)) .filter(o => !o.parent); + const materials = materialQuery(APP.world).map(eid => APP.world.eid2mat.get(eid)); return (
- - {orphaned.map(o => ( +
- ))} +
+
+ {orphaned.map(o => ( + + ))} +
+
+ {materials.map(m => ( + + ))} +
{selectedObj && }
diff --git a/src/react-components/debug-panel/ECSSidebar.scss b/src/react-components/debug-panel/ECSSidebar.scss index 40e77bd7ee..152655c15d 100644 --- a/src/react-components/debug-panel/ECSSidebar.scss +++ b/src/react-components/debug-panel/ECSSidebar.scss @@ -6,6 +6,11 @@ overflow: hidden; height: 100%; } + + section { + padding: 8px 0; + } + .object-list, .object-properties { height: 50%; overflow: auto; From ebcbae27ea2d530ed9130b82645ddea8973159ab Mon Sep 17 00:00:00 2001 From: netpro2k Date: Wed, 25 Jan 2023 15:01:33 -0800 Subject: [PATCH 05/11] Fix legacy noderefs --- src/components/gltf-model-plus.js | 15 +++++++++++++++ src/gltf-component-mappings.js | 14 ++------------ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/components/gltf-model-plus.js b/src/components/gltf-model-plus.js index c823976384..426b480839 100644 --- a/src/components/gltf-model-plus.js +++ b/src/components/gltf-model-plus.js @@ -505,6 +505,21 @@ class GLTFHubsComponentsExtension { for (const componentName in ext) { const props = ext[componentName]; for (const propName in props) { + // These components had a variant before mhc_link_type existed that just directly pointed at the node index, fix them + if ( + ((componentName === "video-texture-target" && propName === "srcNode") || + (componentName === "audio-target" && propName === "srcNode")) && + typeof props[propName] === "number" + ) { + console.warn( + `Found an outdated ${componentName} 'srcNode' property, fixing. Make sure you are using the latest exporter.` + ); + props[propName] = { + __mhc_link_type: "node", + index: props[propName] + }; + } + const value = props[propName]; const type = value?.__mhc_link_type; if (type && value.index !== undefined) { diff --git a/src/gltf-component-mappings.js b/src/gltf-component-mappings.js index 00d7bf835e..7c742edfaf 100644 --- a/src/gltf-component-mappings.js +++ b/src/gltf-component-mappings.js @@ -467,12 +467,7 @@ AFRAME.GLTFModelPlus.registerComponent( let srcEl; if (srcNode !== undefined) { - // indexToEntityMap should be considered depredcated. These references are now resovled by the GLTFHubsComponentExtension - if (typeof srcNode === "number") { - srcEl = indexToEntityMap[srcNode]; - } else { - srcEl = srcNode?.el; - } + srcEl = srcNode?.el; if (!srcEl) { console.warn( `Error inflating gltf component "video-texture-srcEl": Couldn't find srcEl entity with index ${srcNode}` @@ -501,12 +496,7 @@ AFRAME.GLTFModelPlus.registerComponent( let srcEl; if (srcNode !== undefined) { - // indexToEntityMap should be considered depredcated. These references are now resovled by the GLTFHubsComponentExtension - if (typeof srcNode === "number") { - srcEl = indexToEntityMap[srcNode]; - } else { - srcEl = srcNode?.el; - } + srcEl = srcNode?.el; if (!srcEl) { console.warn( `Error inflating gltf component ${componentName}: Couldn't find srcEl entity with index ${srcNode}` From e062620d55d06490d5e08838b5e85a15084775e3 Mon Sep 17 00:00:00 2001 From: netpro2k Date: Wed, 25 Jan 2023 15:08:30 -0800 Subject: [PATCH 06/11] Fix some issues in videoTextureSystem --- src/bit-systems/video-texture.ts | 77 ++++++++++++++++++++++++-------- 1 file changed, 59 insertions(+), 18 deletions(-) diff --git a/src/bit-systems/video-texture.ts b/src/bit-systems/video-texture.ts index 444a80b049..1f83fc8c2f 100644 --- a/src/bit-systems/video-texture.ts +++ b/src/bit-systems/video-texture.ts @@ -7,12 +7,15 @@ import { PerspectiveCamera, RGBAFormat, sRGBEncoding, + Texture, WebGLRenderTarget } from "three"; import { HubsWorld } from "../app"; import { MaterialTag, VideoTextureSource, VideoTextureTarget } from "../bit-components"; import { Layers } from "../camera-layers"; +import { VIDEO_TEXTURE_TARGET_FLAGS } from "../inflators/video-texture-target"; import { EntityID } from "../utils/networking-types"; +import { findNode } from "../utils/three-utils"; function noop() {} @@ -66,20 +69,36 @@ export function updateRenderTarget(world: HubsWorld, renderTarget: WebGLRenderTa interface SourceData { renderTarget: WebGLRenderTarget; lastUpdated: number; + camera: EntityID; } -const sourceData = new Map(); +interface TargetData { + originalMap: Texture | null; + originalEmissiveMap: Texture | null; +} + +const sourceDataMap = new Map(); +const targetDataMap = new Map(); const videoTextureSourceQuery = defineQuery([VideoTextureSource]); const enteredVideoTextureSourcesQuery = enterQuery(videoTextureSourceQuery); const exitedVideoTextureSourcesQuery = exitQuery(videoTextureSourceQuery); -const enteredVideoTextureTargetsQuery = enterQuery(defineQuery([VideoTextureTarget, MaterialTag])); + +const videoTextureTargetQuery = defineQuery([VideoTextureTarget, MaterialTag]); +const enteredVideoTextureTargetsQuery = enterQuery(videoTextureTargetQuery); +const exitedVideoTextureTargetsQuery = exitQuery(videoTextureTargetQuery); export function videoTextureSystem(world: HubsWorld) { enteredVideoTextureSourcesQuery(world).forEach(function (eid) { - const camera = world.eid2obj.get(eid)! as PerspectiveCamera; + let camera = world.eid2obj.get(eid)! as PerspectiveCamera; if (!(camera && camera.isCamera)) { - console.error("video-texture-source added to an entity without a camera"); - return; + const actualCamera = findNode(camera, (o: any) => o.isCamera); + if (actualCamera) { + console.warn("video-texture-source should be added directly to a camera, not it's ancestor."); + camera = actualCamera as PerspectiveCamera; + } else { + console.error("video-texture-source added to an entity without a camera"); + return; + } } camera.layers.enable(Layers.CAMERA_LAYER_THIRD_PERSON_ONLY); @@ -102,30 +121,52 @@ export function videoTextureSystem(world: HubsWorld) { renderTarget.texture.matrix.scale(1, -1); renderTarget.texture.matrix.translate(0, 1); - sourceData.set(eid, { renderTarget, lastUpdated: 0 }); + sourceDataMap.set(eid, { renderTarget, lastUpdated: 0, camera: camera.eid! }); }); exitedVideoTextureSourcesQuery(world).forEach(function (eid) { - const data = sourceData.get(eid); - if (data) { - data.renderTarget.dispose(); - sourceData.delete(eid); + const srcData = sourceDataMap.get(eid); + if (srcData) { + srcData.renderTarget.dispose(); + sourceDataMap.delete(eid); } }); + enteredVideoTextureTargetsQuery(world).forEach(function (eid) { - const data = sourceData.get(VideoTextureTarget.srcNode[eid]); - if (!data) { - console.error("video-texture-target unable to get source"); + const srcData = sourceDataMap.get(VideoTextureTarget.srcNode[eid]); + const targetData: TargetData = { originalMap: null, originalEmissiveMap: null }; + targetDataMap.set(eid, targetData); + + if (!srcData) { + console.error("video-texture-target unable to find source"); return; } + + const mat = world.eid2mat.get(eid)! as MeshStandardMaterial; + if (VideoTextureTarget.flags[eid] & VIDEO_TEXTURE_TARGET_FLAGS.TARGET_BASE_MAP) { + targetData.originalMap = mat.map; + mat.map = srcData.renderTarget.texture; + } + if (VideoTextureTarget.flags[eid] & VIDEO_TEXTURE_TARGET_FLAGS.TARGET_EMISSIVE_MAP) { + targetData.originalEmissiveMap = mat.emissiveMap; + mat.emissiveMap = srcData.renderTarget.texture; + } + }); + exitedVideoTextureTargetsQuery(world).forEach(function (eid) { + const targetData = targetDataMap.get(VideoTextureTarget.srcNode[eid])!; const mat = world.eid2mat.get(eid)! as MeshStandardMaterial; - mat.color.setHex(0x00ff00); - mat.map = data.renderTarget.texture; - mat.emissiveMap = data.renderTarget.texture; + if (VideoTextureTarget.flags[eid] & VIDEO_TEXTURE_TARGET_FLAGS.TARGET_BASE_MAP) { + mat.map = targetData.originalMap; + } + if (VideoTextureTarget.flags[eid] & VIDEO_TEXTURE_TARGET_FLAGS.TARGET_EMISSIVE_MAP) { + mat.emissiveMap = targetData.originalMap; + } + targetDataMap.delete(eid); }); + videoTextureSourceQuery(world).forEach(function (eid) { - const data = sourceData.get(eid); + const data = sourceDataMap.get(eid); if (data && world.time.elapsed > data.lastUpdated + 1000 / VideoTextureSource.fps[eid]) { - updateRenderTarget(world, data.renderTarget, eid); + updateRenderTarget(world, data.renderTarget, data.camera); data.lastUpdated = world.time.elapsed; } }); From 17683e1824b758b96d25216a0c26337f6377df1c Mon Sep 17 00:00:00 2001 From: netpro2k Date: Thu, 26 Jan 2023 00:13:36 -0800 Subject: [PATCH 07/11] Fix ref resolving in renderAsEntity --- src/inflators/model.tsx | 2 +- src/utils/jsx-entity.ts | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/inflators/model.tsx b/src/inflators/model.tsx index e5608d96c8..98a681e736 100644 --- a/src/inflators/model.tsx +++ b/src/inflators/model.tsx @@ -30,7 +30,7 @@ function inflateComponents( } const props = components[name]; - Object.keys(components[name]).forEach(propName => { + Object.keys(props).forEach(propName => { const value = props[propName]; const linkType = value?.__mhc_link_type; if (linkType) { diff --git a/src/utils/jsx-entity.ts b/src/utils/jsx-entity.ts index 25652e0dfd..6d8d98d751 100644 --- a/src/utils/jsx-entity.ts +++ b/src/utils/jsx-entity.ts @@ -188,9 +188,6 @@ export function addMaterialComponent(world: HubsWorld, eid: number, mat: Materia return eid; } -// TODO HACK gettting internal bitecs symbol, should expose an API to check a properties type -const $isEidType = Object.getOwnPropertySymbols(CameraTool.screenRef).find(s => s.description === "isEidType"); - const createDefaultInflator = (C: Component, defaults = {}): InflatorFn => { return (world, eid, componentProps) => { componentProps = Object.assign({}, defaults, componentProps); @@ -207,8 +204,6 @@ const createDefaultInflator = (C: Component, defaults = {}): InflatorFn => { throw new TypeError(`Expected ${propName} to be a string, got an ${typeof value} (${value})`); } prop[eid] = APP.getSid(value); - } else if (prop[$isEidType!]) { - prop[eid] = resolveRef(world, value); } else { prop[eid] = value; } @@ -437,6 +432,13 @@ export function renderAsEntity(world: HubsWorld, entityDef: EntityDef) { if (!jsxInflatorExists(name)) { throw new Error(`Failed to inflate unknown component called ${name}`); } + const props = entityDef.components[name]; + for (const propName in props) { + const value = props[propName]; + if (value instanceof Ref) { + props[propName] = resolveRef(world, value); + } + } jsxInflators[name](world, eid, entityDef.components[name]); }); From ec78d8bc4f9a999ada62f3e8f1166d56cae153fa Mon Sep 17 00:00:00 2001 From: netpro2k Date: Thu, 26 Jan 2023 00:16:35 -0800 Subject: [PATCH 08/11] srcNode -> source --- src/bit-components.js | 2 +- src/bit-systems/video-texture.ts | 4 ++-- src/inflators/video-texture-target.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/bit-components.js b/src/bit-components.js index 1d7c72a61d..9f6f7100bc 100644 --- a/src/bit-components.js +++ b/src/bit-components.js @@ -208,6 +208,6 @@ export const VideoTextureSource = defineComponent({ resolution: [Types.ui16, 2] }); export const VideoTextureTarget = defineComponent({ - srcNode: Types.eid, + source: Types.eid, flags: Types.ui8 }); diff --git a/src/bit-systems/video-texture.ts b/src/bit-systems/video-texture.ts index 1f83fc8c2f..76d73700c8 100644 --- a/src/bit-systems/video-texture.ts +++ b/src/bit-systems/video-texture.ts @@ -132,7 +132,7 @@ export function videoTextureSystem(world: HubsWorld) { }); enteredVideoTextureTargetsQuery(world).forEach(function (eid) { - const srcData = sourceDataMap.get(VideoTextureTarget.srcNode[eid]); + const srcData = sourceDataMap.get(VideoTextureTarget.source[eid]); const targetData: TargetData = { originalMap: null, originalEmissiveMap: null }; targetDataMap.set(eid, targetData); @@ -152,7 +152,7 @@ export function videoTextureSystem(world: HubsWorld) { } }); exitedVideoTextureTargetsQuery(world).forEach(function (eid) { - const targetData = targetDataMap.get(VideoTextureTarget.srcNode[eid])!; + const targetData = targetDataMap.get(VideoTextureTarget.source[eid])!; const mat = world.eid2mat.get(eid)! as MeshStandardMaterial; if (VideoTextureTarget.flags[eid] & VIDEO_TEXTURE_TARGET_FLAGS.TARGET_BASE_MAP) { mat.map = targetData.originalMap; diff --git a/src/inflators/video-texture-target.ts b/src/inflators/video-texture-target.ts index 0ce5be6e23..ad515ff8da 100644 --- a/src/inflators/video-texture-target.ts +++ b/src/inflators/video-texture-target.ts @@ -16,7 +16,7 @@ export interface VideoTextureTargetParams { export function inflateVideoTextureTarget(world: HubsWorld, eid: number, props: VideoTextureTargetParams) { addComponent(world, VideoTextureTarget, eid); - VideoTextureTarget.srcNode[eid] = props.srcNode; + VideoTextureTarget.source[eid] = props.srcNode; let flags = 0; if (props.targetBaseColorMap) flags |= VIDEO_TEXTURE_TARGET_FLAGS.TARGET_BASE_MAP; if (props.targetEmissiveMap) flags |= VIDEO_TEXTURE_TARGET_FLAGS.TARGET_EMISSIVE_MAP; From 0f464deaa534e116bcfec2c23d4d99b317f7bfc4 Mon Sep 17 00:00:00 2001 From: netpro2k Date: Thu, 26 Jan 2023 14:14:44 -0800 Subject: [PATCH 09/11] Fix changing sources in videoTextureSystem --- src/bit-systems/video-texture.ts | 120 ++++++++++++++++++++----------- types/three.d.ts | 11 ++- 2 files changed, 88 insertions(+), 43 deletions(-) diff --git a/src/bit-systems/video-texture.ts b/src/bit-systems/video-texture.ts index 76d73700c8..d5d93b5d60 100644 --- a/src/bit-systems/video-texture.ts +++ b/src/bit-systems/video-texture.ts @@ -1,7 +1,8 @@ -import { defineQuery, enterQuery, exitQuery } from "bitecs"; +import { defineQuery, enterQuery, entityExists, exitQuery, removeComponent } from "bitecs"; import { Camera, LinearFilter, + Material, MeshStandardMaterial, NearestFilter, PerspectiveCamera, @@ -17,6 +18,20 @@ import { VIDEO_TEXTURE_TARGET_FLAGS } from "../inflators/video-texture-target"; import { EntityID } from "../utils/networking-types"; import { findNode } from "../utils/three-utils"; +interface SourceData { + renderTarget: WebGLRenderTarget; + camera: EntityID; + lastUpdated: number; + needsUpdate: boolean; +} + +interface TargetData { + originalMap: Texture | null; + originalEmissiveMap: Texture | null; + originalBeforeRender: typeof Material.prototype.onBeforeRender; + boundTo: EntityID; +} + function noop() {} export function updateRenderTarget(world: HubsWorld, renderTarget: WebGLRenderTarget, camera: EntityID) { @@ -66,15 +81,46 @@ export function updateRenderTarget(world: HubsWorld, renderTarget: WebGLRenderTa } } -interface SourceData { - renderTarget: WebGLRenderTarget; - lastUpdated: number; - camera: EntityID; +function bindMaterial(world: HubsWorld, eid: EntityID) { + const srcData = sourceDataMap.get(VideoTextureTarget.source[eid]); + if (!srcData) { + console.error("video-texture-target unable to find source"); + VideoTextureTarget.source[eid] = 0; + return; + } + + const mat = world.eid2mat.get(eid)! as MeshStandardMaterial; + const targetData: TargetData = { + originalMap: mat.map, + originalEmissiveMap: mat.emissiveMap, + originalBeforeRender: mat.onBeforeRender, + boundTo: VideoTextureTarget.source[eid] + }; + mat.onBeforeRender = function () { + // Only update when a target were in view last frame + // This is safe because this system always runs before render and invalid sources are unbound + sourceDataMap.get(VideoTextureTarget.source[eid])!.needsUpdate = true; + }; + if (VideoTextureTarget.flags[eid] & VIDEO_TEXTURE_TARGET_FLAGS.TARGET_BASE_MAP) { + mat.map = srcData.renderTarget.texture; + } + if (VideoTextureTarget.flags[eid] & VIDEO_TEXTURE_TARGET_FLAGS.TARGET_EMISSIVE_MAP) { + mat.emissiveMap = srcData.renderTarget.texture; + } + targetDataMap.set(eid, targetData); } -interface TargetData { - originalMap: Texture | null; - originalEmissiveMap: Texture | null; +function unbindMaterial(world: HubsWorld, eid: EntityID) { + const targetData = targetDataMap.get(eid)!; + const mat = world.eid2mat.get(eid)! as MeshStandardMaterial; + if (VideoTextureTarget.flags[eid] & VIDEO_TEXTURE_TARGET_FLAGS.TARGET_BASE_MAP) { + mat.map = targetData.originalMap; + } + if (VideoTextureTarget.flags[eid] & VIDEO_TEXTURE_TARGET_FLAGS.TARGET_EMISSIVE_MAP) { + mat.emissiveMap = targetData.originalMap; + } + mat.onBeforeRender = targetData.originalBeforeRender; + targetDataMap.delete(eid); } const sourceDataMap = new Map(); @@ -85,7 +131,6 @@ const enteredVideoTextureSourcesQuery = enterQuery(videoTextureSourceQuery); const exitedVideoTextureSourcesQuery = exitQuery(videoTextureSourceQuery); const videoTextureTargetQuery = defineQuery([VideoTextureTarget, MaterialTag]); -const enteredVideoTextureTargetsQuery = enterQuery(videoTextureTargetQuery); const exitedVideoTextureTargetsQuery = exitQuery(videoTextureTargetQuery); export function videoTextureSystem(world: HubsWorld) { enteredVideoTextureSourcesQuery(world).forEach(function (eid) { @@ -97,6 +142,7 @@ export function videoTextureSystem(world: HubsWorld) { camera = actualCamera as PerspectiveCamera; } else { console.error("video-texture-source added to an entity without a camera"); + removeComponent(world, VideoTextureSource, eid); return; } } @@ -121,7 +167,7 @@ export function videoTextureSystem(world: HubsWorld) { renderTarget.texture.matrix.scale(1, -1); renderTarget.texture.matrix.translate(0, 1); - sourceDataMap.set(eid, { renderTarget, lastUpdated: 0, camera: camera.eid! }); + sourceDataMap.set(eid, { renderTarget, lastUpdated: 0, camera: camera.eid!, needsUpdate: false }); }); exitedVideoTextureSourcesQuery(world).forEach(function (eid) { const srcData = sourceDataMap.get(eid); @@ -131,43 +177,33 @@ export function videoTextureSystem(world: HubsWorld) { } }); - enteredVideoTextureTargetsQuery(world).forEach(function (eid) { - const srcData = sourceDataMap.get(VideoTextureTarget.source[eid]); - const targetData: TargetData = { originalMap: null, originalEmissiveMap: null }; - targetDataMap.set(eid, targetData); - - if (!srcData) { - console.error("video-texture-target unable to find source"); - return; - } - - const mat = world.eid2mat.get(eid)! as MeshStandardMaterial; - if (VideoTextureTarget.flags[eid] & VIDEO_TEXTURE_TARGET_FLAGS.TARGET_BASE_MAP) { - targetData.originalMap = mat.map; - mat.map = srcData.renderTarget.texture; - } - if (VideoTextureTarget.flags[eid] & VIDEO_TEXTURE_TARGET_FLAGS.TARGET_EMISSIVE_MAP) { - targetData.originalEmissiveMap = mat.emissiveMap; - mat.emissiveMap = srcData.renderTarget.texture; - } - }); exitedVideoTextureTargetsQuery(world).forEach(function (eid) { - const targetData = targetDataMap.get(VideoTextureTarget.source[eid])!; - const mat = world.eid2mat.get(eid)! as MeshStandardMaterial; - if (VideoTextureTarget.flags[eid] & VIDEO_TEXTURE_TARGET_FLAGS.TARGET_BASE_MAP) { - mat.map = targetData.originalMap; - } - if (VideoTextureTarget.flags[eid] & VIDEO_TEXTURE_TARGET_FLAGS.TARGET_EMISSIVE_MAP) { - mat.emissiveMap = targetData.originalMap; - } + const isBound = targetDataMap.has(eid); + if (isBound && entityExists(world, eid)) unbindMaterial(world, eid); targetDataMap.delete(eid); }); + videoTextureTargetQuery(world).forEach(function (eid) { + const source = VideoTextureTarget.source[eid]; + const isBound = targetDataMap.has(eid); + if (isBound) { + if (!source || !entityExists(world, source)) { + unbindMaterial(world, eid); + VideoTextureTarget.source[eid] = 0; + } else if (source !== targetDataMap.get(eid)!.boundTo) { + unbindMaterial(world, eid); + bindMaterial(world, eid); + } + } else if (source && entityExists(world, source)) { + bindMaterial(world, eid); + } + }); videoTextureSourceQuery(world).forEach(function (eid) { - const data = sourceDataMap.get(eid); - if (data && world.time.elapsed > data.lastUpdated + 1000 / VideoTextureSource.fps[eid]) { - updateRenderTarget(world, data.renderTarget, data.camera); - data.lastUpdated = world.time.elapsed; + const sourceData = sourceDataMap.get(eid)!; + if (sourceData.needsUpdate && world.time.elapsed > sourceData.lastUpdated + 1000 / VideoTextureSource.fps[eid]) { + updateRenderTarget(world, sourceData.renderTarget, sourceData.camera); + sourceData.lastUpdated = world.time.elapsed; + sourceData.needsUpdate = false; } }); } diff --git a/types/three.d.ts b/types/three.d.ts index a49f1aac19..39c6fd7838 100644 --- a/types/three.d.ts +++ b/types/three.d.ts @@ -1,5 +1,5 @@ import { AElement } from "aframe"; -import { Object3D, Mesh } from "three"; +import { Object3D, Mesh, WebGLRenderer, Scene, Camera } from "three"; declare module "three" { interface Object3D { @@ -9,8 +9,17 @@ declare module "three" { el?: AElement; updateMatrices: (forceLocalUpdate?: boolean, forceWorldUpdate?: boolean, skipParents?: boolean) => void; } + type GeometryGroup = { start: number; count: number; materialIndex: number }; interface Material { eid?: number; + onBeforeRender: ( + renderer: WebGLRenderer, + scene: Scene, + camera: Camera, + geometry: Geometry, + obj: Object3D, + group: GeometryGroup + ) => void; } interface Mesh { reflectionProbeMode: "static" | "dynamic" | false; From 1d5ebd723919359ac7cf1f8c15ddd7096fd6f150 Mon Sep 17 00:00:00 2001 From: netpro2k Date: Thu, 26 Jan 2023 14:23:21 -0800 Subject: [PATCH 10/11] Delint --- src/gltf-component-mappings.js | 94 ++++++++++++++++------------------ 1 file changed, 44 insertions(+), 50 deletions(-) diff --git a/src/gltf-component-mappings.js b/src/gltf-component-mappings.js index 7c742edfaf..7e307bb2a6 100644 --- a/src/gltf-component-mappings.js +++ b/src/gltf-component-mappings.js @@ -462,7 +462,7 @@ AFRAME.GLTFModelPlus.registerComponent("audio-settings", "audio-settings", (el, AFRAME.GLTFModelPlus.registerComponent( "video-texture-target", "video-texture-target", - (el, componentName, componentData, _components, indexToEntityMap) => { + (el, componentName, componentData) => { const { targetBaseColorMap, targetEmissiveMap, srcNode } = componentData; let srcEl; @@ -488,61 +488,55 @@ AFRAME.GLTFModelPlus.registerComponent("video-texture-source", "video-texture-so AFRAME.GLTFModelPlus.registerComponent("text", "text"); -AFRAME.GLTFModelPlus.registerComponent( - "audio-target", - "audio-target", - (el, componentName, componentData, _components, indexToEntityMap) => { - const { srcNode } = componentData; +AFRAME.GLTFModelPlus.registerComponent("audio-target", "audio-target", (el, componentName, componentData) => { + const { srcNode } = componentData; - let srcEl; - if (srcNode !== undefined) { - srcEl = srcNode?.el; - if (!srcEl) { - console.warn( - `Error inflating gltf component ${componentName}: Couldn't find srcEl entity with index ${srcNode}` - ); - } + let srcEl; + if (srcNode !== undefined) { + srcEl = srcNode?.el; + if (!srcEl) { + console.warn(`Error inflating gltf component ${componentName}: Couldn't find srcEl entity with index ${srcNode}`); } + } - if (componentData.positional !== undefined) { - // This is an old version of the audio-target component, which had built-in audio parameters. - // The way we are handling it is wrong. If a user created a scene in spoke with this old version - // of this component, all of these parameters will be present whether the user explicitly set - // the values for them or not. But really, they should only count as "overrides" if the user - // meant for them to take precendence over the app and scene defaults. - // TODO: Fix this issue. One option is to just ignore this component data, which might break old scenes - // but simplifying the handling. Another option is to compare the component data here with - // the "defaults" and only save the values that are different from the defaults. However, - // this loses information if the user changed the scene settings but wanted this specific - // node to use the "defaults". - // I don't see a perfect solution here and would prefer not to handle the "legacy" components. - APP.audioOverrides.set(el, { - audioType: componentData.positional ? AudioType.PannerNode : AudioType.Stereo, - distanceModel: componentData.distanceModel, - rolloffFactor: componentData.rolloffFactor, - refDistance: componentData.refDistance, - maxDistance: componentData.maxDistance, - coneInnerAngle: componentData.coneInnerAngle, - coneOuterAngle: componentData.coneOuterAngle, - coneOuterGain: componentData.coneOuterGain, - gain: componentData.gain - }); - APP.sourceType.set(el, SourceType.AUDIO_TARGET); + if (componentData.positional !== undefined) { + // This is an old version of the audio-target component, which had built-in audio parameters. + // The way we are handling it is wrong. If a user created a scene in spoke with this old version + // of this component, all of these parameters will be present whether the user explicitly set + // the values for them or not. But really, they should only count as "overrides" if the user + // meant for them to take precendence over the app and scene defaults. + // TODO: Fix this issue. One option is to just ignore this component data, which might break old scenes + // but simplifying the handling. Another option is to compare the component data here with + // the "defaults" and only save the values that are different from the defaults. However, + // this loses information if the user changed the scene settings but wanted this specific + // node to use the "defaults". + // I don't see a perfect solution here and would prefer not to handle the "legacy" components. + APP.audioOverrides.set(el, { + audioType: componentData.positional ? AudioType.PannerNode : AudioType.Stereo, + distanceModel: componentData.distanceModel, + rolloffFactor: componentData.rolloffFactor, + refDistance: componentData.refDistance, + maxDistance: componentData.maxDistance, + coneInnerAngle: componentData.coneInnerAngle, + coneOuterAngle: componentData.coneOuterAngle, + coneOuterGain: componentData.coneOuterGain, + gain: componentData.gain + }); + APP.sourceType.set(el, SourceType.AUDIO_TARGET); - const audio = APP.audios.get(el); - if (audio) { - updateAudioSettings(el, audio); - } + const audio = APP.audios.get(el); + if (audio) { + updateAudioSettings(el, audio); } - - el.setAttribute(componentName, { - minDelay: componentData.minDelay, - maxDelay: componentData.maxDelay, - debug: componentData.debug, - srcEl - }); } -); + + el.setAttribute(componentName, { + minDelay: componentData.minDelay, + maxDelay: componentData.maxDelay, + debug: componentData.debug, + srcEl + }); +}); AFRAME.GLTFModelPlus.registerComponent("zone-audio-source", "zone-audio-source"); AFRAME.GLTFModelPlus.registerComponent("audio-params", "audio-params", (el, componentName, componentData) => { From ffb27af6382b230497ec5f3091594be86d9e0d94 Mon Sep 17 00:00:00 2001 From: netpro2k Date: Mon, 30 Jan 2023 13:00:21 -0800 Subject: [PATCH 11/11] Fix merge --- src/systems/remove-object3D-system.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/systems/remove-object3D-system.js b/src/systems/remove-object3D-system.js index 3a1900bc11..0b9cacb618 100644 --- a/src/systems/remove-object3D-system.js +++ b/src/systems/remove-object3D-system.js @@ -12,7 +12,7 @@ import { Text, VideoMenu, Skybox, - MaterialTag + MaterialTag, SimpleWater } from "../bit-components"; import { gltfCache } from "../components/gltf-model-plus";