From 6dd79bc41e94738334c398f27d76f9eef4f68214 Mon Sep 17 00:00:00 2001 From: xiaoiver Date: Wed, 22 May 2024 15:22:49 +0800 Subject: [PATCH] fix: support cameras array (#1688) * fix: support cameras array * chore: disable enableDirtyRectangleRenderingDebug by default * fix: set viewport * chore: commit changeset --- .changeset/blue-ants-decide.md | 5 + __tests__/demos/3d/force.ts | 135 ++++----- __tests__/demos/3d/webar.ts | 6 +- __tests__/main.ts | 25 -- packages/g-mobile-webgl/package.json | 2 +- packages/g-plugin-3d/package.json | 2 +- .../g-plugin-device-renderer/package.json | 2 +- .../src/RenderGraphPlugin.ts | 257 +++++++++++------- packages/g-webgl/package.json | 2 +- packages/g-webgpu/package.json | 2 +- pnpm-lock.yaml | 36 +-- 11 files changed, 262 insertions(+), 212 deletions(-) create mode 100644 .changeset/blue-ants-decide.md diff --git a/.changeset/blue-ants-decide.md b/.changeset/blue-ants-decide.md new file mode 100644 index 000000000..bdd070569 --- /dev/null +++ b/.changeset/blue-ants-decide.md @@ -0,0 +1,5 @@ +--- +'@antv/g-plugin-device-renderer': patch +--- + +Support multiple viewport. diff --git a/__tests__/demos/3d/force.ts b/__tests__/demos/3d/force.ts index 5a6b80141..48ac1e268 100644 --- a/__tests__/demos/3d/force.ts +++ b/__tests__/demos/3d/force.ts @@ -1642,6 +1642,7 @@ export async function force(context) { const material = new MeshPhongMaterial(device, { shininess: 30, }); + // material.polygonOffset = true; // @see https://antv.vision/en/docs/specification/language/palette#%E5%88%86%E7%B1%BB%E8%89%B2%E6%9D%BF const colorPalette = [ @@ -1683,78 +1684,78 @@ export async function force(context) { sphere.style.fill = fill; }); - const icon = new Image({ - style: { - x: node.x + 310, - y: node.y + 250, - z: node.z - 1, - width: 10, - height: 10, - src: 'https://gw.alipayobjects.com/mdn/rms_6ae20b/afts/img/A*N4ZMS7gHsUIAAAAAAAAAAABkARQnAQ', - isBillboard: true, - }, - }); - canvas.appendChild(icon); + // const icon = new Image({ + // style: { + // x: node.x + 310, + // y: node.y + 250, + // z: node.z - 1, + // width: 10, + // height: 10, + // src: 'https://gw.alipayobjects.com/mdn/rms_6ae20b/afts/img/A*N4ZMS7gHsUIAAAAAAAAAAABkARQnAQ', + // isBillboard: true, + // }, + // }); + // canvas.appendChild(icon); - const circle = new Circle({ - style: { - cx: node.x + 310, - cy: node.y + 250, - cz: node.z - 2, - r: 2.5, - fill: 'red', - isBillboard: true, - }, - }); - canvas.appendChild(circle); + // const circle = new Circle({ + // style: { + // cx: node.x + 310, + // cy: node.y + 250, + // cz: node.z - 2, + // r: 2.5, + // fill: 'red', + // isBillboard: true, + // }, + // }); + // canvas.appendChild(circle); - const label = new Text({ - style: { - x: node.x + 310, - y: node.y + 250, - z: node.z + 1, - fontFamily: 'sans-serif', - text: node.id, - fontSize: 6, - fill: 'black', - isBillboard: true, - }, - }); + // const label = new Text({ + // style: { + // x: node.x + 310, + // y: node.y + 250, + // z: node.z + 1, + // fontFamily: 'sans-serif', + // text: node.id, + // fontSize: 6, + // fill: 'black', + // isBillboard: true, + // }, + // }); - const rect = new Rect({ - style: { - x: node.x + 310, - y: node.y + 250, - z: node.z, - width: label.getBBox().width, - height: 8, - fill: 'grey', - isBillboard: true, - fillOpacity: 0.6, - }, - }); - canvas.appendChild(rect); - canvas.appendChild(label); + // const rect = new Rect({ + // style: { + // x: node.x + 310, + // y: node.y + 250, + // z: node.z, + // width: label.getBBox().width, + // height: 8, + // fill: 'grey', + // isBillboard: true, + // fillOpacity: 0.6, + // }, + // }); + // canvas.appendChild(rect); + // canvas.appendChild(label); }); - dataset.links.forEach((edge) => { - const { source, target } = edge; - const line = new Line({ - style: { - x1: source.x + 300, - y1: source.y + 250, - z1: source.z, - x2: target.x + 300, - y2: target.y + 250, - z2: target.z, - stroke: 'black', - lineWidth: 2, - opacity: 0.5, - isBillboard: true, // 始终面向屏幕 - }, - }); - canvas.appendChild(line); - }); + // dataset.links.forEach((edge) => { + // const { source, target } = edge; + // const line = new Line({ + // style: { + // x1: source.x + 300, + // y1: source.y + 250, + // z1: source.z, + // x2: target.x + 300, + // y2: target.y + 250, + // z2: target.z, + // stroke: 'black', + // lineWidth: 2, + // opacity: 0.5, + // isBillboard: true, // 始终面向屏幕 + // }, + // }); + // canvas.appendChild(line); + // }); // add a directional light into scene const light = new DirectionalLight({ diff --git a/__tests__/demos/3d/webar.ts b/__tests__/demos/3d/webar.ts index 2f8c51c55..c543fe659 100644 --- a/__tests__/demos/3d/webar.ts +++ b/__tests__/demos/3d/webar.ts @@ -49,12 +49,14 @@ export async function ar(context) { }); // cube.setOrigin(300, 250, 200); - cube.setPosition(300, 250, 200); + cube.setPosition(300, 250, -200); canvas.appendChild(cube); + // Called every time a XRSession requests that a new frame be drawn. + // @see https://github.com/immersive-web/webxr-samples/blob/main/immersive-ar-session.html#L173 canvas.addEventListener(CanvasEvent.AFTER_RENDER, () => { - cube.rotate(0, 0, 0); + cube.rotate(0, 0.2, 0); }); canvas.getConfig().disableHitTesting = true; diff --git a/__tests__/main.ts b/__tests__/main.ts index c46aebfd6..41c29fdd5 100644 --- a/__tests__/main.ts +++ b/__tests__/main.ts @@ -244,31 +244,6 @@ function createSpecRender(object) { }); } - if ( - selectRenderer.value === 'canvas' && - renderer.getConfig().enableDirtyRectangleRendering && - renderer.getConfig().enableDirtyRectangleRenderingDebug - ) { - // display dirty rectangle - const $dirtyRectangle = document.createElement('div'); - $dirtyRectangle.style.cssText = ` - position: absolute; - pointer-events: none; - background: rgba(255, 0, 0, 0.5); - `; - $div.appendChild($dirtyRectangle); - canvas.addEventListener(CanvasEvent.DIRTY_RECTANGLE, (e) => { - const { dirtyRect } = e.detail; - const { x, y, width, height } = dirtyRect; - const dpr = window.devicePixelRatio; - // convert from canvas coords to viewport - $dirtyRectangle.style.left = `${x / dpr}px`; - $dirtyRectangle.style.top = `${y / dpr}px`; - $dirtyRectangle.style.width = `${width / dpr}px`; - $dirtyRectangle.style.height = `${height / dpr}px`; - }); - } - container.append($div); }; }; diff --git a/packages/g-mobile-webgl/package.json b/packages/g-mobile-webgl/package.json index dd47a6e62..aa4c030b6 100644 --- a/packages/g-mobile-webgl/package.json +++ b/packages/g-mobile-webgl/package.json @@ -45,7 +45,7 @@ "@antv/g-plugin-html-renderer": "workspace:*", "@antv/g-plugin-image-loader": "workspace:*", "@antv/g-plugin-mobile-interaction": "workspace:*", - "@antv/g-device-api": "^1.3.6", + "@antv/g-device-api": "^1.6.10", "@antv/util": "^3.3.5", "tslib": "^2.5.3" }, diff --git a/packages/g-plugin-3d/package.json b/packages/g-plugin-3d/package.json index 3fe82785f..1eb0e2ff6 100644 --- a/packages/g-plugin-3d/package.json +++ b/packages/g-plugin-3d/package.json @@ -41,7 +41,7 @@ "@antv/g-lite": "workspace:*", "@antv/g-plugin-device-renderer": "workspace:*", "@antv/g-shader-components": "workspace:*", - "@antv/g-device-api": "^1.3.6", + "@antv/g-device-api": "^1.6.10", "gl-matrix": "^3.4.3", "tslib": "^2.5.3" }, diff --git a/packages/g-plugin-device-renderer/package.json b/packages/g-plugin-device-renderer/package.json index da73124b7..75ec27dd1 100644 --- a/packages/g-plugin-device-renderer/package.json +++ b/packages/g-plugin-device-renderer/package.json @@ -43,7 +43,7 @@ "@antv/g-plugin-image-loader": "workspace:*", "@antv/g-shader-components": "workspace:*", "@antv/util": "^3.3.5", - "@antv/g-device-api": "^1.3.6", + "@antv/g-device-api": "^1.6.10", "@webgpu/types": "^0.1.6", "earcut": "^2.2.3", "eventemitter3": "^5.0.1", diff --git a/packages/g-plugin-device-renderer/src/RenderGraphPlugin.ts b/packages/g-plugin-device-renderer/src/RenderGraphPlugin.ts index 4ffc9071e..4b8790ee0 100644 --- a/packages/g-plugin-device-renderer/src/RenderGraphPlugin.ts +++ b/packages/g-plugin-device-renderer/src/RenderGraphPlugin.ts @@ -39,7 +39,7 @@ import { import type { BatchManager } from './renderer'; import type { TexturePool } from './TexturePool'; import { DeviceRendererPluginOptions } from './interfaces'; -import { mat4 } from 'gl-matrix'; +import { mat4, vec3 } from 'gl-matrix'; // scene uniform block index export const SceneUniformBufferIndex = 0; @@ -93,6 +93,18 @@ export class RenderGraphPlugin implements RenderingPlugin { private capturePromise: Promise | undefined; private resolveCapturePromise: (dataURL: string) => void; + /** + * An array of sub cameras used in VR scenes. + * @see https://threejs.org/docs/#api/en/cameras/ArrayCamera + */ + private cameras: { + viewport: XRViewport; + projectionMatrix: mat4; + viewMatrix: mat4; + cameraPosition: vec3; + isOrtho: boolean; + }[] = []; + getDevice(): Device { return this.device; } @@ -258,8 +270,6 @@ export class RenderGraphPlugin implements RenderingPlugin { config.disableRenderHooks = false; }); - // let usedViewport: XRViewport | undefined; - /** * build frame graph at the beginning of each frame */ @@ -269,7 +279,6 @@ export class RenderGraphPlugin implements RenderingPlugin { const session = frame?.session; const { width, height } = this.context.config; if (session) { - const camera = this.context.camera; // Assumed to be a XRWebGLLayer for now. let layer = session.renderState.baseLayer; if (!layer) { @@ -292,29 +301,64 @@ export class RenderGraphPlugin implements RenderingPlugin { // XRFrame.getViewerPose can return null while the session attempts to establish tracking. const pose = frame.getViewerPose(referenceSpace); if (pose) { - // const p = pose.transform.position; // In mobile AR, we only have one view. - const view = pose.views[0]; - // const viewport = session.renderState.baseLayer!.getViewport(view)!; - // usedViewport = viewport; - // @ts-ignore - const cameraMatrix = mat4.fromValues(...view.transform.matrix); - cameraMatrix[12] *= width; - cameraMatrix[13] *= -height; - cameraMatrix[14] *= 500; - - cameraMatrix[12] += width / 2; - cameraMatrix[13] += height / 2; - cameraMatrix[14] += 500 / 2; - - // @ts-ignore - const projectionMatrix = mat4.fromValues(...view.projectionMatrix); - mat4.scale(projectionMatrix, projectionMatrix, [1, -1, 1]); // flipY - - // @ts-ignore - camera.setProjectionMatrix(projectionMatrix); - camera.setMatrix(cameraMatrix); + pose.views.forEach((view, i) => { + const viewport = + session.renderState.baseLayer!.getViewport(view)!; + + // @ts-ignore + const cameraMatrix = mat4.fromValues(...view.transform.matrix); + cameraMatrix[12] *= width; + cameraMatrix[13] *= height; + cameraMatrix[14] *= 500; + + cameraMatrix[12] += width / 2; + cameraMatrix[13] -= height / 2; + cameraMatrix[14] += 500 / 2; + + // Use this matrix without modification or decomposition + // @see https://immersive-web.github.io/webxr/#dom-xrview-projectionmatrix + const projectionMatrix = mat4.fromValues( + // @ts-ignore + ...view.projectionMatrix, + ); + // mat4.scale(projectionMatrix, projectionMatrix, [1, -1, 1]); // flipY + + const viewMatrix = mat4.invert(mat4.create(), cameraMatrix); + mat4.scale(viewMatrix, viewMatrix, vec3.fromValues(1, -1, 1)); + + // @see https://github.com/immersive-web/webxr-samples/blob/main/js/render/core/renderer.js#L651 + const { x, y, z } = view.transform.position; + this.cameras[i] = { + viewport: { + x: viewport.x / layer.framebufferWidth, + y: viewport.y / layer.framebufferHeight, + width: viewport.width / layer.framebufferWidth, + height: viewport.height / layer.framebufferHeight, + }, + projectionMatrix, + viewMatrix, + cameraPosition: [x, y, z], + isOrtho: false, + }; + }); } + } else { + const camera = this.context.camera; + this.cameras = [ + { + viewport: { + x: 0, + y: 0, + width: 1, + height: 1, + }, + projectionMatrix: camera.getPerspective(), + viewMatrix: camera.getViewTransform(), + cameraPosition: camera.getPosition(), + isOrtho: camera.isOrtho(), + }, + ]; } const canvas = this.swapChain.getCanvas() as HTMLCanvasElement; @@ -375,21 +419,38 @@ export class RenderGraphPlugin implements RenderingPlugin { // main render pass this.builder.pushPass((pass) => { pass.setDebugName('Main Render Pass'); - // if (usedViewport) { - // const { x, y, width: vw, height: vh } = usedViewport; - // // pass.setViewport(x, y, vw / width / 2, vh / height / 2); - // console.log(x, y, vw, vh, width, height); - // } pass.attachRenderTargetID(RGAttachmentSlot.Color0, mainColorTargetID); pass.attachRenderTargetID( RGAttachmentSlot.DepthStencil, mainDepthTargetID, ); - pass.exec((passRenderer) => { - this.renderLists.world.drawOnPassRenderer( - renderInstManager.renderCache, - passRenderer, - ); + + pass.exec((passRenderer, scope) => { + this.cameras.forEach(({ viewport }) => { + const { x, y, width, height } = viewport; + + const { viewportW, viewportH } = scope['currentPass']; + + // @see https://github.com/immersive-web/webxr-samples/blob/main/js/render/core/renderer.js#L757 + passRenderer.setViewport( + x * viewportW, + y * viewportH, + width * viewportW, + height * viewportH, + ); + + // console.log( + // x * viewportW, + // y * viewportH, + // width * viewportW, + // height * viewportH, + // ); + + this.renderLists.world.drawOnPassRenderer( + renderInstManager.renderCache, + passRenderer, + ); + }); }); }); @@ -416,68 +477,74 @@ export class RenderGraphPlugin implements RenderingPlugin { RenderGraphPlugin.tag, (frame: XRFrame) => { const renderInstManager = this.renderHelper.renderInstManager; - - // TODO: time for GPU Animation - // const timeInMilliseconds = window.performance.now(); - - // Push our outer template, which contains the dynamic UBO bindings... - const template = this.renderHelper.pushTemplateRenderInst(); - // SceneParams: binding = 0, ObjectParams: binding = 1 - template.setBindingLayout({ numUniformBuffers: 2, numSamplers: 0 }); - template.setMegaStateFlags( - setAttachmentStateSimple( - { - depthWrite: true, - blendConstant: TransparentBlack, - }, - { - rgbBlendMode: BlendMode.ADD, - alphaBlendMode: BlendMode.ADD, - rgbBlendSrcFactor: BlendFactor.SRC_ALPHA, - alphaBlendSrcFactor: BlendFactor.ONE, - rgbBlendDstFactor: BlendFactor.ONE_MINUS_SRC_ALPHA, - alphaBlendDstFactor: BlendFactor.ONE_MINUS_SRC_ALPHA, - }, - ), - ); - - // Update Scene Params const { width, height } = this.context.config; - const camera = this.context.camera; - template.setUniforms(SceneUniformBufferIndex, [ - { - name: SceneUniform.PROJECTION_MATRIX, - value: camera.getPerspective(), - }, - { - name: SceneUniform.VIEW_MATRIX, - value: camera.getViewTransform(), - }, - { - name: SceneUniform.CAMERA_POSITION, - value: camera.getPosition(), - }, - { - name: SceneUniform.DEVICE_PIXEL_RATIO, - value: this.context.contextService.getDPR(), - }, - { - name: SceneUniform.VIEWPORT, - value: [width, height], - }, - { - name: SceneUniform.IS_ORTHO, - value: camera.isOrtho() ? 1 : 0, - }, - { - name: SceneUniform.IS_PICKING, - value: 0, - }, - ]); - - this.batchManager.render(this.renderLists.world); + this.cameras.forEach( + ({ + viewport, + cameraPosition, + viewMatrix, + projectionMatrix, + isOrtho, + }) => { + const { width: normalizedW, height: normalizedH } = viewport; + + // Push our outer template, which contains the dynamic UBO bindings... + const template = this.renderHelper.pushTemplateRenderInst(); + // SceneParams: binding = 0, ObjectParams: binding = 1 + template.setBindingLayout({ numUniformBuffers: 2, numSamplers: 0 }); + template.setMegaStateFlags( + setAttachmentStateSimple( + { + depthWrite: true, + blendConstant: TransparentBlack, + }, + { + rgbBlendMode: BlendMode.ADD, + alphaBlendMode: BlendMode.ADD, + rgbBlendSrcFactor: BlendFactor.SRC_ALPHA, + alphaBlendSrcFactor: BlendFactor.ONE, + rgbBlendDstFactor: BlendFactor.ONE_MINUS_SRC_ALPHA, + alphaBlendDstFactor: BlendFactor.ONE_MINUS_SRC_ALPHA, + }, + ), + ); - renderInstManager.popTemplateRenderInst(); + template.setUniforms(SceneUniformBufferIndex, [ + { + name: SceneUniform.PROJECTION_MATRIX, + value: projectionMatrix, + }, + { + name: SceneUniform.VIEW_MATRIX, + value: viewMatrix, + }, + { + name: SceneUniform.CAMERA_POSITION, + value: cameraPosition, + }, + { + name: SceneUniform.DEVICE_PIXEL_RATIO, + value: this.context.contextService.getDPR(), + }, + { + name: SceneUniform.VIEWPORT, + value: [width * normalizedW, height * normalizedH], + }, + { + name: SceneUniform.IS_ORTHO, + value: isOrtho ? 1 : 0, + }, + { + name: SceneUniform.IS_PICKING, + value: 0, + }, + ]); + + this.batchManager.render(this.renderLists.world); + + renderInstManager.popTemplateRenderInst(); + }, + ); this.renderHelper.prepareToRender(); this.renderHelper.renderGraph.execute(); diff --git a/packages/g-webgl/package.json b/packages/g-webgl/package.json index 4758e0b1b..53abfc6c5 100644 --- a/packages/g-webgl/package.json +++ b/packages/g-webgl/package.json @@ -44,7 +44,7 @@ "@antv/g-plugin-html-renderer": "workspace:*", "@antv/g-plugin-image-loader": "workspace:*", "@antv/util": "^3.3.5", - "@antv/g-device-api": "^1.3.6", + "@antv/g-device-api": "^1.6.10", "tslib": "^2.5.3" }, "devDependencies": { diff --git a/packages/g-webgpu/package.json b/packages/g-webgpu/package.json index 928d1cea6..33f255282 100644 --- a/packages/g-webgpu/package.json +++ b/packages/g-webgpu/package.json @@ -43,7 +43,7 @@ "@antv/g-plugin-dom-interaction": "workspace:*", "@antv/g-plugin-html-renderer": "workspace:*", "@antv/g-plugin-image-loader": "workspace:*", - "@antv/g-device-api": "^1.3.6", + "@antv/g-device-api": "^1.6.10", "@antv/util": "^3.3.5", "@webgpu/types": "^0.1.6", "tslib": "^2.5.3" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f3be6a19e..93c1180f9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -117,7 +117,7 @@ importers: version: 5.3.0 playwright: specifier: latest - version: 1.40.1 + version: 1.44.0 pngjs: specifier: ^6.0.0 version: 6.0.0 @@ -465,8 +465,8 @@ importers: packages/g-mobile-webgl: dependencies: '@antv/g-device-api': - specifier: ^1.3.6 - version: 1.3.9 + specifier: ^1.6.10 + version: 1.6.10 '@antv/g-lite': specifier: workspace:* version: link:../g-lite @@ -517,8 +517,8 @@ importers: packages/g-plugin-3d: dependencies: '@antv/g-device-api': - specifier: ^1.3.6 - version: 1.3.9 + specifier: ^1.6.10 + version: 1.6.10 '@antv/g-lite': specifier: workspace:* version: link:../g-lite @@ -703,8 +703,8 @@ importers: packages/g-plugin-device-renderer: dependencies: '@antv/g-device-api': - specifier: ^1.3.6 - version: 1.3.9 + specifier: ^1.6.10 + version: 1.6.10 '@antv/g-lite': specifier: workspace:* version: link:../g-lite @@ -1045,8 +1045,8 @@ importers: packages/g-webgl: dependencies: '@antv/g-device-api': - specifier: ^1.3.6 - version: 1.3.9 + specifier: ^1.6.10 + version: 1.6.10 '@antv/g-lite': specifier: workspace:* version: link:../g-lite @@ -1082,8 +1082,8 @@ importers: packages/g-webgpu: dependencies: '@antv/g-device-api': - specifier: ^1.3.6 - version: 1.3.9 + specifier: ^1.6.10 + version: 1.6.10 '@antv/g-lite': specifier: workspace:* version: link:../g-lite @@ -1168,8 +1168,8 @@ packages: '@jridgewell/trace-mapping': 0.3.20 dev: true - /@antv/g-device-api@1.3.9: - resolution: {integrity: sha512-73ilvcF6ToHzNl8w0dA/nE7s5e174/NzP93stfaXsHUDsZ0UItFWOH1TXHhuyrmGmhIAhKVF4sFXtk82MjIO6Q==} + /@antv/g-device-api@1.6.10: + resolution: {integrity: sha512-cuIEwIKvq8QdtZ+Ix/Mv9K8lJEAB2nrvPJPhH/Pj2FQeLRQiWGwHyyzsdgz4+uqAvEwtd2EX7lJ0A2Dwzl3NXA==} dependencies: '@antv/util': 3.3.5 '@webgpu/types': 0.1.34 @@ -7589,18 +7589,18 @@ packages: find-up: 4.1.0 dev: true - /playwright-core@1.40.1: - resolution: {integrity: sha512-+hkOycxPiV534c4HhpfX6yrlawqVUzITRKwHAmYfmsVreltEl6fAZJ3DPfLMOODw0H3s1Itd6MDCWmP1fl/QvQ==} + /playwright-core@1.44.0: + resolution: {integrity: sha512-ZTbkNpFfYcGWohvTTl+xewITm7EOuqIqex0c7dNZ+aXsbrLj0qI8XlGKfPpipjm0Wny/4Lt4CJsWJk1stVS5qQ==} engines: {node: '>=16'} hasBin: true dev: true - /playwright@1.40.1: - resolution: {integrity: sha512-2eHI7IioIpQ0bS1Ovg/HszsN/XKNwEG1kbzSDDmADpclKc7CyqkHw7Mg2JCz/bbCxg25QUPcjksoMW7JcIFQmw==} + /playwright@1.44.0: + resolution: {integrity: sha512-F9b3GUCLQ3Nffrfb6dunPOkE5Mh68tR7zN32L4jCk4FjQamgesGay7/dAAe1WaMEGV04DkdJfcJzjoCKygUaRQ==} engines: {node: '>=16'} hasBin: true dependencies: - playwright-core: 1.40.1 + playwright-core: 1.44.0 optionalDependencies: fsevents: 2.3.2 dev: true