fix: sync proj & view matrix from webxr session

.changeset/stale-eyes-provide.md

@@ -0,0 +1,5 @@
+---
+'@antv/g-plugin-device-renderer': patch
+---
+
+Sync projection & view matrix from webxr session.

__tests__/demos/3d/force.ts

@@ -4,7 +4,14 @@ import {
   forceManyBody,
   forceCenter,
 } from 'd3-force-3d';
-import { Line, Text, Rect, Image, Circle } from '../../../packages/g';
+import {
+  Line,
+  Text,
+  Rect,
+  Image,
+  Circle,
+  CanvasEvent,
+} from '../../../packages/g';
 import {
   SphereGeometry,
   MeshPhongMaterial,
@@ -13,6 +20,7 @@ import {
   Plugin as Plugin3D,
 } from '../../../packages/g-plugin-3d';
 import { Plugin as PluginControl } from '../../../packages/g-plugin-control';
+import { ARButton, DeviceRenderer } from '../../../packages/g-webgl';
 
 // https://bl.ocks.org/vasturiano/f59675656258d3f490e9faa40828c0e7
 const dataset = {
@@ -1601,7 +1609,7 @@ const dataset = {
 };
 
 export async function force(context) {
-  const { canvas, renderer } = context;
+  const { canvas, renderer, container } = context;
 
   // wait for canvas' initialization complete
   await canvas.ready;
@@ -1655,16 +1663,17 @@ export async function force(context) {
     // create a mesh
     const sphere = new Mesh({
       style: {
-        x: node.x + 300,
-        y: node.y + 250,
-        z: node.z,
+        // x: node.x + 300,
+        // y: node.y + 250,
+        // z: node.z,
         fill,
         opacity: 1,
         geometry: sphereGeometry,
         material,
         cursor: 'pointer',
       },
     });
+    sphere.setPosition(node.x + 300, node.y + 250, node.z);
     canvas.appendChild(sphere);
 
     sphere.addEventListener('mouseenter', () => {
@@ -1758,7 +1767,17 @@ export async function force(context) {
 
   // adjust camera's position
   const camera = canvas.getCamera();
-  camera.setPerspective(0.1, 5000, 45, 600 / 500);
+  camera.setPerspective(0.1, 5000, 45, 640 / 640);
+
+  canvas.addEventListener(CanvasEvent.AFTER_RENDER, () => {
+    canvas.document.documentElement.setOrigin(320, 320, 0);
+    canvas.document.documentElement.rotate(0, 0.1, 0);
+  });
+
+  canvas.getConfig().disableHitTesting = true;
+
+  const $button = ARButton.createButton(canvas, renderer, {});
+  container.appendChild($button);
 }
 
 force.initRenderer = (renderer) => {

__tests__/demos/3d/webar.ts

@@ -27,6 +27,10 @@ export async function ar(context) {
     width: 200,
     height: 200,
     depth: 200,
+
+    // width: 1,
+    // height: 1,
+    // depth: 1,
   });
   const basicMaterial = new MeshBasicMaterial(device, {
     // wireframe: true,
@@ -42,6 +46,7 @@ export async function ar(context) {
     },
   });
 
+  // cube.setOrigin(300, 250, 200);
   cube.setPosition(300, 250, 200);
 
   canvas.appendChild(cube);

packages/g-plugin-device-renderer/src/RenderGraphPlugin.ts

@@ -39,6 +39,7 @@ import {
 import type { BatchManager } from './renderer';
 import type { TexturePool } from './TexturePool';
 import { DeviceRendererPluginOptions } from './interfaces';
+import { mat4 } from 'gl-matrix';
 
 // scene uniform block index
 export const SceneUniformBufferIndex = 0;
@@ -257,16 +258,18 @@ export class RenderGraphPlugin implements RenderingPlugin {
       config.disableRenderHooks = false;
     });
 
+    // let usedViewport: XRViewport | undefined;
+
     /**
      * build frame graph at the beginning of each frame
      */
     renderingService.hooks.beginFrame.tap(
       RenderGraphPlugin.tag,
       (frame: XRFrame) => {
         const session = frame?.session;
-        // const { width, height } = this.context.config;
+        const { width, height } = this.context.config;
         if (session) {
-          // const camera = this.context.camera;
+          const camera = this.context.camera;
           // Assumed to be a XRWebGLLayer for now.
           let layer = session.renderState.baseLayer;
           if (!layer) {
@@ -291,24 +294,26 @@ export class RenderGraphPlugin implements RenderingPlugin {
           if (pose) {
             // const p = pose.transform.position;
             // In mobile AR, we only have one view.
-            // const view = pose.views[0];
+            const view = pose.views[0];
             // const viewport = session.renderState.baseLayer!.getViewport(view)!;
-            // Use the view's transform matrix and projection matrix
-            // const viewMatrix = mat4.invert(mat4.create(), view.transform.matrix);
-            // const cameraMatrix = view.transform.matrix;
-            // const projectionMatrix = view.projectionMatrix;
+            // usedViewport = viewport;
+            // @ts-ignore
+            const cameraMatrix = mat4.fromValues(...view.transform.matrix);
+            cameraMatrix[12] *= width;
+            cameraMatrix[13] *= height;
+            cameraMatrix[14] *= 500;
+
+            cameraMatrix[12] += width / 2;
+            cameraMatrix[13] += height / 2;
+            cameraMatrix[14] += 500 / 2;
+
             // @ts-ignore
-            // camera.setProjectionMatrix(projectionMatrix);
-            // camera.setViewOffset(
-            //   camera.getView().fullWidth,
-            //   camera.getView().fullHeight,
-            //   0,
-            //   0,
-            //   viewport.width,
-            //   viewport.height,
-            // );
-            // camera.setMatrix(cameraMatrix);
-            // console.log(viewport, camera.getView());
+            const projectionMatrix = mat4.fromValues(...view.projectionMatrix);
+            mat4.scale(projectionMatrix, projectionMatrix, [1, -1, 1]); // flipY
+
+            // @ts-ignore
+            camera.setProjectionMatrix(projectionMatrix);
+            camera.setMatrix(cameraMatrix);
           }
         }
 
@@ -370,6 +375,11 @@ export class RenderGraphPlugin implements RenderingPlugin {
         // main render pass
         this.builder.pushPass((pass) => {
           pass.setDebugName('Main Render Pass');
+          // if (usedViewport) {
+          //   const { x, y, width: vw, height: vh } = usedViewport;
+          //   // pass.setViewport(x, y, vw / width / 2, vh / height / 2);
+          //   console.log(x, y, vw, vh, width, height);
+          // }
           pass.attachRenderTargetID(RGAttachmentSlot.Color0, mainColorTargetID);
           pass.attachRenderTargetID(
             RGAttachmentSlot.DepthStencil,
@@ -434,13 +444,6 @@ export class RenderGraphPlugin implements RenderingPlugin {
         // Update Scene Params
         const { width, height } = this.context.config;
         const camera = this.context.camera;
-
-        // console.log(
-        //   camera.getPerspective(),
-        //   camera.getViewTransform(),
-        //   camera.getPosition(),
-        // );
-
         template.setUniforms(SceneUniformBufferIndex, [
           {
             name: SceneUniform.PROJECTION_MATRIX,