Add WebXR capturing functionality to Spector.js, and WebXR sample

package.json

@@ -34,16 +34,17 @@
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "devDependencies": {
-    "ts-loader": "^9.3.1",
+    "@types/webxr": "^0.5.1",
     "concat-cli": "^4.0.0",
     "css-loader": "^6.7.1",
     "exports-loader": "^4.0.0",
     "http-server": "^14.1.1",
     "livereload": "^0.9.3",
-    "sass": "^1.53.0",
     "npm-run-all": "^4.1.5",
+    "sass": "^1.53.0",
     "sass-loader": "^13.0.2",
     "style-loader": "^3.3.1",
+    "ts-loader": "^9.3.1",
     "tslint": "^6.1.3",
     "typescript": "^4.7.4",
     "webpack": "^5.73.0",

sample/assets/js/injectSpector.js

@@ -1,4 +1,9 @@
-var spector = new SPECTOR.Spector();
+var spectorOptions = {};
+if ((window.location + "").indexOf("webxr") !== -1) {
+	spectorOptions.enableXRCapture = true;
+}
+
+var spector = new SPECTOR.Spector(spectorOptions);
 window.spector = spector;
 spector.displayUI();
 

sample/js/webxr.js

@@ -0,0 +1,43 @@
+let renderCanvas = document.getElementById('renderCanvas');
+
+let createScene = async function () {
+    let scene = new BABYLON.Scene(engine);
+    let camera = new BABYLON.FreeCamera("camera1", new BABYLON.Vector3(0, 5, -10), scene);
+    camera.setTarget(BABYLON.Vector3.Zero());
+    camera.attachControl(renderCanvas, true);
+    let light = new BABYLON.HemisphericLight("light1", new BABYLON.Vector3(0, 1, 0), scene);
+    light.intensity = 0.7;
+    let sphere = BABYLON.MeshBuilder.CreateSphere("sphere1", { segments: 16, diameter: 2 }, scene);
+    sphere.position.y = 1;
+
+    const env = scene.createDefaultEnvironment();
+
+    // here we add XR support
+    const xr = await scene.createDefaultXRExperienceAsync({
+        floorMeshes: [env.ground],
+    });
+
+    // we can't access the Spector UI while in WebXR, so we take a capture whenever the trigger
+    // is pressed instead.
+    let previousTriggerPressed = false;
+    xr.input.onControllerAddedObservable.add((controller) => {
+        let triggerComponent = controller.gamepadController.components["xr-standard-trigger"];
+        triggerComponent.onButtonStateChanged.add((stateObject) => {
+            if (stateObject.pressed && !previousTriggerPressed) {
+                spector.captureXRContext();
+            }
+            previousTriggerPressed = stateObject.pressed;
+        });
+    });
+
+    return scene;
+};
+
+let engine = new BABYLON.Engine(renderCanvas);
+createScene(engine, renderCanvas).then((scene) => {
+
+    engine.runRenderLoop(function () {
+        scene.render();
+    });
+
+});

src/backend/spies/timeSpy.ts

@@ -25,7 +25,7 @@ export class TimeSpy {
     public readonly onFrameEnd: Observable<TimeSpy>;
     public readonly onError: Observable<string>;
 
-    private readonly spiedScope: { [name: string]: any };
+    private spiedScope: { [name: string]: any };
     private readonly lastSixtyFramesDuration: number[];
 
     private lastSixtyFramesCurrentIndex: number;
@@ -62,10 +62,18 @@ export class TimeSpy {
         this.speedRatio = ratio;
     }
 
+    public static getRequestAnimationFrameFunctionNames(): string[] {
+        return [...TimeSpy.requestAnimationFrameFunctions];
+    }
+
     public addRequestAnimationFrameFunctionName(functionName: string): void {
         TimeSpy.requestAnimationFrameFunctions.push(functionName);
     }
 
+    public getSpiedScope() {
+        return this.spiedScope;
+    }
+
     public setSpiedScope(spiedScope: { [name: string]: any }): void {
         this.spiedScope = spiedScope;
     }
@@ -97,8 +105,7 @@ export class TimeSpy {
             });
         }
     }
-
-    private spyRequestAnimationFrame(functionName: string, owner: any): void {
+    public spyRequestAnimationFrame(functionName: string, owner: any): void {
         // Needs both this.
         // tslint:disable-next-line
         const self = this;

src/backend/spies/xrSpy.ts

@@ -0,0 +1,99 @@
+import { XRSessionSpector } from "../../polyfill/XRSessionSpector";
+import { XRWebGLBindingSpector } from "../../polyfill/XRWebGLBindingSpector";
+import { XRWebGLLayerSpector } from "../../polyfill/XRWebGLLayerSpector";
+import { OriginFunctionHelper } from "../utils/originFunctionHelper";
+import { TimeSpy } from "./timeSpy";
+
+export class XRSpy {
+    public currentXRSession: XRSessionSpector | undefined;
+    private timeSpy: TimeSpy;
+    constructor(timeSpy: TimeSpy) {
+        this.timeSpy = timeSpy;
+        this.init();
+    }
+
+    public spyXRSession(session: XRSessionSpector) {
+        if (this.currentXRSession) {
+            this.unspyXRSession();
+        }
+        for (const Spy of TimeSpy.getRequestAnimationFrameFunctionNames()) {
+            OriginFunctionHelper.resetOriginFunction(this.timeSpy.getSpiedScope(), Spy);
+        }
+        this.timeSpy.spyRequestAnimationFrame("requestAnimationFrame", session);
+        this.currentXRSession = session;
+    }
+
+    public unspyXRSession() {
+        if (!this.currentXRSession) {
+            return;
+        }
+
+        OriginFunctionHelper.resetOriginFunction(this.currentXRSession, "requestAnimationFrame");
+        this.currentXRSession = undefined;
+        // listen to the regular frames again.
+        for (const Spy of TimeSpy.getRequestAnimationFrameFunctionNames()) {
+            this.timeSpy.spyRequestAnimationFrame(Spy, this.timeSpy.getSpiedScope());
+        }
+    }
+
+    private init(): void {
+        if (!navigator.xr) {
+            return;
+        }
+
+        (window as any).XRWebGLLayer = XRWebGLLayerSpector;
+        (window as any).XRWebGLBinding = XRWebGLBindingSpector;
+
+        // polyfill request session so Spector gets access to the session object.
+        const existingRequestSession = navigator.xr.requestSession;
+        Object.defineProperty(navigator.xr, "requestSessionInternal", { writable: true });
+        (navigator.xr as any).requestSessionInternal = existingRequestSession;
+
+        const newRequestSession = (
+            sessionMode: XRSessionMode,
+            sessionInit?: any
+        ): Promise<XRSession> => {
+            const modifiedSessionPromise = (mode: XRSessionMode, init?: any): Promise<XRSession> => {
+                return (navigator.xr as any).requestSessionInternal(mode, init).then((session: XRSession) => {
+                    // listen to the XR Session here! When we do that, we'll stop listening to window.requestAnimationFrame
+                    // and start listening to session.requestAnimationFrame
+
+                    // Feed the gl context through the session
+                    const spectorSession = session as XRSessionSpector;
+                    spectorSession._updateRenderState = session.updateRenderState;
+                    spectorSession.updateRenderState = async (
+                        renderStateInit?: XRRenderStateInit
+                    ): Promise<void> => {
+                        if (renderStateInit.baseLayer) {
+                            const polyfilledBaseLayer =
+                                renderStateInit.baseLayer as XRWebGLLayerSpector;
+                            spectorSession.glContext = polyfilledBaseLayer.getContext();
+                        }
+
+                        if (renderStateInit.layers) {
+                            for (const layer of renderStateInit.layers) {
+                                const layerAny: any = layer;
+                                if (layerAny.glContext) {
+                                    spectorSession.glContext = layerAny.glContext;
+                                }
+                            }
+                        }
+                        return spectorSession._updateRenderState(renderStateInit);
+                    };
+
+                    this.spyXRSession(spectorSession);
+                    session.addEventListener("end", () => {
+                        this.unspyXRSession();
+                    });
+                    return Promise.resolve(session);
+                });
+            };
+            return modifiedSessionPromise(sessionMode, sessionInit);
+        };
+
+
+        Object.defineProperty(navigator.xr, "requestSession", { writable: true });
+        (navigator.xr as any).requestSession = newRequestSession;
+    }
+
+}

src/backend/states/parameterState.ts

@@ -65,6 +65,11 @@ export abstract class ParameterState extends BaseState {
 
             for (const parameter of this.parameters[version - 1]) {
                 const value = this.readParameterFromContext(parameter);
+                if (value === null || value === undefined) {
+                    const stringValue = this.stringifyParameterValue(value, parameter);
+                    this.currentState[parameter.constant.name] = stringValue;
+                    continue;
+                }
                 const tag = WebGlObjects.getWebGlObjectTag(value);
                 if (tag) {
                     this.currentState[parameter.constant.name] = tag;

src/backend/utils/originFunctionHelper.ts

@@ -16,6 +16,23 @@ export class OriginFunctionHelper {
         object[originFunctionName] = object[functionName];
     }
 
+    public static resetOriginFunction(object: any, functionName: string): any {
+        if (!object) {
+            return;
+        }
+
+        if (!object[functionName]) {
+            return;
+        }
+        const originFunctionName = this.getOriginFunctionName(functionName);
+        if (!object[originFunctionName]) {
+            return;
+        }
+
+        object[functionName] = object[originFunctionName];
+        delete object[originFunctionName];
+    }
+
     public static storePrototypeOriginFunction(object: any, functionName: string): void {
         if (!object) {
             return;

src/backend/utils/readPixelsHelper.ts

@@ -33,6 +33,12 @@ export class ReadPixelsHelper {
         // Empty error list.
         gl.getError();
 
+        // If type is UNSIGNED_NORMALIZED, we passed in a component type that isn't a pixel format type.
+        // So we have to convert it to a valid pixel format type.
+        if (type === WebGlConstants.UNSIGNED_NORMALIZED.value) {
+            type = WebGlConstants.UNSIGNED_BYTE.value;
+        }
+
         // prepare destination storage.
         const size = width * height * 4;
         let pixels: ArrayBufferView;

src/polyfill/XRSessionSpector.ts

@@ -0,0 +1,5 @@
+export interface XRSessionSpector extends XRSession {
+    glContext: WebGLRenderingContext | WebGL2RenderingContext;
+
+    _updateRenderState: typeof XRSession.prototype.updateRenderState;
+}

src/polyfill/XRWebGLBindingSpector.ts

@@ -0,0 +1,13 @@
+export class XRWebGLBindingSpector extends XRWebGLBinding {
+    private glContext: WebGLRenderingContext | WebGL2RenderingContext;
+    constructor(session: XRSession, context: WebGLRenderingContext) {
+        super(session, context);
+        this.glContext = context;
+    }
+
+    public createProjectionLayer(init?: XRProjectionLayerInit): XRProjectionLayer {
+        const layer = super.createProjectionLayer(init);
+        (layer as any).glContext = this.glContext;
+        return layer;
+    }
+}

src/polyfill/XRWebGLLayerSpector.ts

@@ -0,0 +1,15 @@
+export class XRWebGLLayerSpector extends XRWebGLLayer {
+    private glContext: WebGLRenderingContext | WebGL2RenderingContext;
+    constructor(
+        session: XRSession,
+        context: WebGLRenderingContext | WebGL2RenderingContext,
+        layerInit?: XRWebGLLayerInit
+    ) {
+        super(session, context, layerInit);
+        this.glContext = context;
+    }
+
+    public getContext() {
+        return this.glContext;
+    }
+}