feat(WebXR): Add support for head-mounted augmented reality displays

Documentation/content/docs/develop_webxr.md

@@ -40,6 +40,8 @@ vtk.js supports virtual and augmented reality rendering via the [WebXR device AP
 
 [![VR Cone Example][VrCone]](../examples/VR.html)
 [![SkyboxViewer Example][SkyboxViewerVR]](../examples/SkyboxViewer.html?fileURL=https://data.kitware.com/api/v1/file/5ae8a89c8d777f0685796bae/download)
+[![GeometryViewer Brain Blood Vessels][GeometryViewerBrainBloodVessels]](../examples/GeometryViewer/GeometryViewer.html?fileURL=[https://data.kitware.com/api/v1/file/61f041f14acac99f42c2ff9a/download,https://data.kitware.com/api/v1/file/61f042024acac99f42c2ffa6/download,https://data.kitware.com/api/v1/file/61f042b74acac99f42c30079/download])
+[![GeometryViewer Chest CT][GeometryViewerchestCT]](../examples/GeometryViewer/GeometryViewer.html?fileURL=[https://data.kitware.com/api/v1/file/61f044354acac99f42c30276/download,https://data.kitware.com/api/v1/file/61f0440f4acac99f42c30191/download,https://data.kitware.com/api/v1/file/61f044204acac99f42c30267/download])
 [![XR Volume Example][WebXRVolume]](../examples/WebXRVolume.html)
 [![XR Volumetric Example][HeadFullVolume]](../examples/WebXRHeadFullVolumeCVR.html)
 [![XR Hybrid Example][ChestCTHybrid]](../examples/WebXRChestCTBlendedCVR.html)
@@ -74,6 +76,24 @@ vtk.js supports virtual and augmented reality rendering via the [WebXR device AP
 
 ### For Developers
 
+#### Building and Running Examples
+
+Webpack can be used to run examples on local XR hardware. The following command will
+serve the "AR" example over a self-signed HTTPS connection:
+
+```
+path/to/vtk-js> npm run example:https -- AR
+...
+<i> [webpack-dev-server] Project is running at:
+<i> [webpack-dev-server] Loopback: https://localhost:9999/
+<i> [webpack-dev-server] On Your Network (IPv4): https://xxx.xxx.xxx.xxx:9999/
+```
+
+The example can then be launched on the WebXR device by navigating in a compatible browser to the
+address given at `https://xxx.xxx.xxx.xxx:9999` in the output.
+
+#### Emulating XR Hardware
+
 Developers without access to XR hardware may find it convenient to install and use the [Mozilla WebXR emulator](https://github.com/MozillaReality/WebXR-emulator-extension) in their browser.
 
 - Install the WebXR extension on either Chrome or Firefox.
@@ -83,6 +103,29 @@ Developers without access to XR hardware may find it convenient to install and u
 
 While WebXR has broad industry support, it is not yet implemented in all browsers. Developers may include the [WebXR polyfill](https://github.com/immersive-web/webxr-polyfill) in their projects for backwards compatibility with the deprecated WebVR API.
 
+#### Selecting the XR Session Type
+
+A WebXR device may be compatible with more than one type of session. For instance, the Meta Quest 2 headset supports fully immersive virtual reality sessions as well as augmented reality sessions with passthrough.
+
+Several vtk.js XR-capable examples support more than one type of XR session. For these examples, the desired session type may be requested by adding the parameter `&xrSessionType=<value>` to the end of the example URL. Session type values are defined in the [OpenGL RenderWindow](https://github.com/Kitware/vtk-js/blob/master/Sources/Rendering/OpenGL/RenderWindow/Constants.js).
+
+### Frequently Asked Questions
+
+#### What mixed reality devices are supported by VTK.js?
+
+VTK.js supports rendering to most devices that support the WebXR API. Rendering with VTK.js has been tested on the following devices:
+
+- Meta Quest 2 (VR, AR)
+- HP Reverb G2 (VR)
+- Samsung Galaxy mobile device (AR)
+- Apple iPhone devices, with Mozille Reality browser app (AR)
+- Microsoft HoloLens 2 (AR, but considered as a VR device)
+- Looking Glass Factory displays (holographic)
+
+#### Where can I find more information about planned VTK.js WebXR development?
+
+The roadmap for VTK.js WebXR feature development and support is maintained in the [VTK.js GitHub issue tracker](https://github.com/Kitware/vtk-js/issues/2571).
+
 
 [ArCone]: ../docs/gallery/ArCone.jpg
 [GeometryViewer]: ../docs/gallery/GeometryViewer.jpg

Examples/Applications/GeometryViewer/index.js

@@ -430,25 +430,24 @@ function createPipeline(fileName, fileContents) {
   // --------------------------------------------------------------------
 
   function toggleXR() {
-    if (requestedXrSessionType === XrSessionTypes.MobileAR) {
-      fullScreenRenderWindow.setBackground([...background, 0]);
-    }
-
     if (immersionSelector.textContent.startsWith('Start')) {
       fullScreenRenderWindow
         .getApiSpecificRenderWindow()
         .startXR(requestedXrSessionType);
-      immersionSelector.textContent =
-        requestedXrSessionType === XrSessionTypes.MobileAR
-          ? 'Exit AR'
-          : 'Exit VR';
+      immersionSelector.textContent = [
+        XrSessionTypes.HmdAR,
+        XrSessionTypes.MobileAR,
+      ].includes(requestedXrSessionType)
+        ? 'Exit AR'
+        : 'Exit VR';
     } else {
-      fullScreenRenderWindow.setBackground([...background, 255]);
       fullScreenRenderWindow.getApiSpecificRenderWindow().stopXR();
-      immersionSelector.textContent =
-        requestedXrSessionType === XrSessionTypes.MobileAR
-          ? 'Start AR'
-          : 'Start VR';
+      immersionSelector.textContent = [
+        XrSessionTypes.HmdAR,
+        XrSessionTypes.MobileAR,
+      ].includes(requestedXrSessionType)
+        ? 'Start AR'
+        : 'Start VR';
     }
   }
   immersionSelector.addEventListener('click', toggleXR);

Examples/Applications/GeometryViewer/index.md

@@ -19,4 +19,9 @@ Virtual reality, augmented reality, and holographic viewing is supported for Web
 - [Brain Blood Vessels (holographic)](https://kitware.github.io/vtk-js/examples/GeometryViewer/index.html?xrSessionType=2&fileURL=[https://data.kitware.com/api/v1/file/61f041f14acac99f42c2ff9a/download,https://data.kitware.com/api/v1/file/61f042024acac99f42c2ffa6/download,https://data.kitware.com/api/v1/file/61f042b74acac99f42c30079/download]) 57.71 MB
 - [Chest CT (holographic)](https://kitware.github.io/vtk-js/examples/GeometryViewer/index.html?xrSessionType=2&fileURL=[https://data.kitware.com/api/v1/file/61f044354acac99f42c30276/download,https://data.kitware.com/api/v1/file/61f0440f4acac99f42c30191/download,https://data.kitware.com/api/v1/file/61f044204acac99f42c30267/download]) 63.75 MB
 
+The following links provide augmented reality examples for head-mounted displays:
+- [Earth.vtp (AR headset)](https://kitware.github.io/vtk-js/examples/GeometryViewer/GeometryViewer.html?xrSessionType=3fileURL=https://data.kitware.com/api/v1/item/59ee68d98d777f31ac64784b/download) 1.17 MB
+- [Brain Blood Vessels (AR headset)](https://kitware.github.io/vtk-js/examples/GeometryViewer/index.html?xrSessionType=3&fileURL=[https://data.kitware.com/api/v1/file/61f041f14acac99f42c2ff9a/download,https://data.kitware.com/api/v1/file/61f042024acac99f42c2ffa6/download,https://data.kitware.com/api/v1/file/61f042b74acac99f42c30079/download]) 57.71 MB
+- [Chest CT (AR headset)](https://kitware.github.io/vtk-js/examples/GeometryViewer/index.html?xrSessionType=3&fileURL=[https://data.kitware.com/api/v1/file/61f044354acac99f42c30276/download,https://data.kitware.com/api/v1/file/61f0440f4acac99f42c30191/download,https://data.kitware.com/api/v1/file/61f044204acac99f42c30267/download]) 63.75 MB
+
 [HTML]: https://kitware.github.io/vtk-js/examples/GeometryViewer/GeometryViewer.html

Examples/Geometry/AR/index.js

@@ -8,6 +8,7 @@ import vtkCalculator from '@kitware/vtk.js/Filters/General/Calculator';
 import vtkConeSource from '@kitware/vtk.js/Filters/Sources/ConeSource';
 import vtkFullScreenRenderWindow from '@kitware/vtk.js/Rendering/Misc/FullScreenRenderWindow';
 import vtkMapper from '@kitware/vtk.js/Rendering/Core/Mapper';
+import vtkURLExtract from '@kitware/vtk.js/Common/Core/URLExtract';
 import { AttributeTypes } from '@kitware/vtk.js/Common/DataModel/DataSetAttributes/Constants';
 import { FieldDataTypes } from '@kitware/vtk.js/Common/DataModel/DataSet/Constants';
 import { XrSessionTypes } from '@kitware/vtk.js/Rendering/OpenGL/RenderWindow/Constants';
@@ -19,6 +20,13 @@ import '@kitware/vtk.js/IO/Core/DataAccessHelper/JSZipDataAccessHelper';
 
 import controlPanel from './controller.html';
 
+// ----------------------------------------------------------------------------
+// Parse URL parameters
+// ----------------------------------------------------------------------------
+const userParams = vtkURLExtract.extractURLParameters();
+const requestedXrSessionType =
+  userParams.xrSessionType ?? XrSessionTypes.MobileAR;
+
 // ----------------------------------------------------------------------------
 // Standard rendering code setup
 // ----------------------------------------------------------------------------
@@ -85,13 +93,11 @@ arbutton.disabled = !fullScreenRenderer
 
 arbutton.addEventListener('click', (e) => {
   if (arbutton.textContent === 'Start AR') {
-    fullScreenRenderer.setBackground([0, 0, 0, 0]);
     fullScreenRenderer
       .getApiSpecificRenderWindow()
-      .startXR(XrSessionTypes.MobileAR);
+      .startXR(requestedXrSessionType);
     arbutton.textContent = 'Exit AR';
   } else {
-    fullScreenRenderer.setBackground([0, 0, 0, 255]);
     fullScreenRenderer.getApiSpecificRenderWindow().stopXR();
     arbutton.textContent = 'Start AR';
   }

Examples/Geometry/AR/index.md

@@ -0,0 +1,14 @@
+
+## Viewing Scenes with WebXR
+
+vtk.js supports augmented reality rendering via the [WebXR device API](https://www.w3.org/TR/webxr/) for dedicated AR headsets and most modern mobile devices.
+
+See the list of supported devices on the [VTK.js WebXR Examples](../docs/develop_webxr.html) page.
+
+## Augmented Reality Head-Mounted Display
+
+The default viewer above assumes that a mobile device is used for augmented reality rendering.
+
+[Click here](AR/index.html?xrSessionType=3) to run the example on augmented reality head-mounted displays such as a Meta Quest 2 headset.
+
+The Microsoft HoloLens 2 headset is treated as a virtual reality device. [Click here](VR/index.html) to run the example on a Microsoft HoloLens 2 headset.

Examples/Volume/WebXRChestCTBlendedCVR/index.js

@@ -140,13 +140,9 @@ HttpDataAccessHelper.fetchBinary(fileURL).then((fileContents) => {
   }
   xrButton.addEventListener('click', () => {
     if (xrButton.textContent === enterText) {
-      if (xrSessionType === XrSessionTypes.MobileAR) {
-        fullScreenRenderer.setBackground([0, 0, 0, 0]);
-      }
       fullScreenRenderer.getApiSpecificRenderWindow().startXR(xrSessionType);
       xrButton.textContent = exitText;
     } else {
-      fullScreenRenderer.setBackground([...background, 255]);
       fullScreenRenderer.getApiSpecificRenderWindow().stopXR();
       xrButton.textContent = enterText;
     }

Examples/Volume/WebXRHeadFullVolumeCVR/index.js

@@ -161,15 +161,11 @@ HttpDataAccessHelper.fetchBinary(fileURL).then((fileContents) => {
   }
   xrButton.addEventListener('click', () => {
     if (xrButton.textContent === enterText) {
-      if (requestedXrSessionType === XrSessionTypes.MobileAR) {
-        fullScreenRenderer.setBackground([0, 0, 0, 0]);
-      }
       fullScreenRenderer
         .getApiSpecificRenderWindow()
         .startXR(requestedXrSessionType);
       xrButton.textContent = exitText;
     } else {
-      fullScreenRenderer.setBackground([...background, 255]);
       fullScreenRenderer.getApiSpecificRenderWindow().stopXR();
       xrButton.textContent = enterText;
     }

Examples/Volume/WebXRHeadGradientCVR/index.js

@@ -158,15 +158,11 @@ HttpDataAccessHelper.fetchBinary(fileURL).then((fileContents) => {
   }
   xrButton.addEventListener('click', () => {
     if (xrButton.textContent === enterText) {
-      if (requestedXrSessionType === XrSessionTypes.MobileAR) {
-        fullScreenRenderer.setBackground([0, 0, 0, 0]);
-      }
       fullScreenRenderer
         .getApiSpecificRenderWindow()
         .startXR(requestedXrSessionType);
       xrButton.textContent = exitText;
     } else {
-      fullScreenRenderer.setBackground([...background, 255]);
       fullScreenRenderer.getApiSpecificRenderWindow().stopXR();
       xrButton.textContent = enterText;
     }

Examples/Volume/WebXRVolume/index.js

@@ -186,15 +186,11 @@ HttpDataAccessHelper.fetchBinary(fileURL).then((fileContents) => {
   }
   xrButton.addEventListener('click', () => {
     if (xrButton.textContent === enterText) {
-      if (requestedXrSessionType === XrSessionTypes.MobileAR) {
-        fullScreenRenderer.setBackground([0, 0, 0, 0]);
-      }
       fullScreenRenderer
         .getApiSpecificRenderWindow()
         .startXR(requestedXrSessionType);
       xrButton.textContent = exitText;
     } else {
-      fullScreenRenderer.setBackground([...background, 255]);
       fullScreenRenderer.getApiSpecificRenderWindow().stopXR();
       xrButton.textContent = enterText;
     }

Examples/Volume/WebXRVolume/index.md

@@ -14,12 +14,19 @@ Volume rendering can be processor-intensive. Smaller volumes may give better per
 
 ### Looking Glass Holographic Support
 
-Holographic scenes can be rendered to a Looking Glass display by specifying the `xrSessionType` in the example URL.
+Holographic scenes can be rendered to a Looking Glass display by specifying `xrSessionType=2` in the example URL.
 
 - [binary-head.vti (holographic)](https://kitware.github.io/vtk-js/examples/WebXRVolume/WebXRVolume.html?xrSessionType=2&fileURL=https://data.kitware.com/api/v1/file/59de9dca8d777f31ac641dc2/download) 15.6 MB
 - [3DUS-fetus.vti (holographic)](https://kitware.github.io/vtk-js/examples/WebXRVolume/WebXRVolume.html?xrSessionType=2&fileURL=https://data.kitware.com/api/v1/file/63fe43217b0dfcc98f66a85a/download) 30.22 MB
 - [Kitware_CTA_Head_and_Neck.vti (holographic)](https://kitware.github.io/vtk-js/examples/WebXRVolume/WebXRVolume.html?xrSessionType=2&fileURL=https://data.kitware.com/api/v1/file/63fe3f237b0dfcc98f66a857/download&colorPreset=CT-Cardiac2&resliceVolume=false) 5.025 MB
 
+### Augmented Reality Head-Mounted Displays
+
+Augmented reality scenes can be rendered to a dedicated head-mounted display by specifying `xrSessionType=3` in the example URL.
+
+- [binary-head.vti](https://kitware.github.io/vtk-js/examples/WebXRVolume/WebXRVolume.html?xrSessionType=3&fileURL=https://data.kitware.com/api/v1/file/59de9dca8d777f31ac641dc2/download) 15.6 MB
+- [Kitware_CTA_Head_and_Neck.vti](https://kitware.github.io/vtk-js/examples/WebXRVolume/WebXRVolume.html?xrSessionType=3&fileURL=https://data.kitware.com/api/v1/file/63fe3f237b0dfcc98f66a857/download&colorPreset=CT-Cardiac2&resliceVolume=true) 5.025 MB
+- [3DUS-fetus.vti (holographic)](https://kitware.github.io/vtk-js/examples/WebXRVolume/WebXRVolume.html?xrSessionType=3&fileURL=https://data.kitware.com/api/v1/file/63fe43217b0dfcc98f66a85a/download) 30.22 MB
 
 ### See Also
 

Sources/Rendering/OpenGL/RenderWindow/Constants.js

@@ -2,6 +2,7 @@ export const XrSessionTypes = {
   HmdVR: 0, // Head-mounted display (HMD), two-camera virtual reality session
   MobileAR: 1, // Mobile device, single-camera augmented reality session
   LookingGlassVR: 2, // Looking Glass hologram display, N-camera virtual reality session
+  HmdAR: 3,
 };
 
 export default {

Sources/Rendering/OpenGL/RenderWindow/index.js

@@ -26,14 +26,8 @@ const SCREENSHOT_PLACEHOLDER = {
 };
 
 const DEFAULT_RESET_FACTORS = {
-  vr: {
-    rescaleFactor: 1.0,
-    translateZ: -0.7, // 0.7 m forward from the camera
-  },
-  ar: {
-    rescaleFactor: 0.25, // scale down AR for viewing comfort by default
-    translateZ: -0.5, // 0.5 m forward from the camera
-  },
+  rescaleFactor: 0.25, // isotropic scale factor reduces apparent size of objects
+  translateZ: -1.5, // default translation initializes object in front of camera
 };
 
 function checkRenderTargetSupport(gl, format, type) {
@@ -302,10 +296,13 @@ function vtkOpenGLRenderWindow(publicAPI, model) {
 
     model.xrSessionType =
       xrSessionType !== undefined ? xrSessionType : XrSessionTypes.HmdVR;
-    const isAR = xrSessionType === XrSessionTypes.MobileAR;
-    const sessionType = isAR ? 'immersive-ar' : 'immersive-vr';
+    const isXrSessionAR = [
+      XrSessionTypes.HmdAR,
+      XrSessionTypes.MobileAR,
+    ].includes(model.xrSessionType);
+    const sessionType = isXrSessionAR ? 'immersive-ar' : 'immersive-vr';
     if (!navigator.xr.isSessionSupported(sessionType)) {
-      if (isAR) {
+      if (isXrSessionAR) {
         throw new Error('Device does not support AR session');
       } else {
         throw new Error('VR display is not available');
@@ -345,39 +342,33 @@ function vtkOpenGLRenderWindow(publicAPI, model) {
         model.xrReferenceSpace = refSpace;
       });
 
+      // Initialize transparent background for augmented reality session
+      const isXrSessionAR = [
+        XrSessionTypes.HmdAR,
+        XrSessionTypes.MobileAR,
+      ].includes(model.xrSessionType);
+      if (isXrSessionAR) {
+        const ren = model.renderable.getRenderers()[0];
+        model.preXrSessionBackground = ren.getBackground();
+        ren.setBackground([0, 0, 0, 0]);
+      }
+
       publicAPI.resetXRScene();
 
       model.renderable.getInteractor().switchToXRAnimation();
       model.xrSceneFrame = model.xrSession.requestAnimationFrame(
         publicAPI.xrRender
       );
     } else {
-      throw new Error('Failed to enter VR with a null xrSession.');
+      throw new Error('Failed to enter XR with a null xrSession.');
     }
   };
 
   publicAPI.resetXRScene = (
-    inputRescaleFactor = DEFAULT_RESET_FACTORS.vr.rescaleFactor,
-    inputTranslateZ = DEFAULT_RESET_FACTORS.vr.translateZ
+    rescaleFactor = DEFAULT_RESET_FACTORS.rescaleFactor,
+    translateZ = DEFAULT_RESET_FACTORS.translateZ
   ) => {
     // Adjust world-to-physical parameters for different modalities
-    // Default parameter values are for HMD VR
-    let rescaleFactor = inputRescaleFactor;
-    let translateZ = inputTranslateZ;
-
-    const isXrSessionAR = model.xrSessionType === XrSessionTypes.MobileAR;
-    if (
-      isXrSessionAR &&
-      rescaleFactor === DEFAULT_RESET_FACTORS.vr.rescaleFactor
-    ) {
-      // Scale down by default in AR
-      rescaleFactor = DEFAULT_RESET_FACTORS.ar.rescaleFactor;
-    }
-
-    if (isXrSessionAR && translateZ === DEFAULT_RESET_FACTORS.vr.translateZ) {
-      // Default closer to the camera in AR
-      translateZ = DEFAULT_RESET_FACTORS.ar.translateZ;
-    }
 
     const ren = model.renderable.getRenderers()[0];
     ren.resetCamera();
@@ -386,9 +377,9 @@ function vtkOpenGLRenderWindow(publicAPI, model) {
     let physicalScale = camera.getPhysicalScale();
     const physicalTranslation = camera.getPhysicalTranslation();
 
+    const rescaledTranslateZ = translateZ * physicalScale;
     physicalScale /= rescaleFactor;
-    translateZ *= physicalScale;
-    physicalTranslation[2] += translateZ;
+    physicalTranslation[2] += rescaledTranslateZ;
 
     camera.setPhysicalScale(physicalScale);
     camera.setPhysicalTranslation(physicalTranslation);
@@ -422,6 +413,12 @@ function vtkOpenGLRenderWindow(publicAPI, model) {
 
     // Reset to default canvas
     const ren = model.renderable.getRenderers()[0];
+
+    if (model.preXrSessionBackground != null) {
+      ren.setBackground(model.preXrSessionBackground);
+      model.preXrSessionBackground = null;
+    }
+
     ren.getActiveCamera().setProjectionMatrix(null);
     ren.resetCamera();
 
@@ -431,6 +428,10 @@ function vtkOpenGLRenderWindow(publicAPI, model) {
 
   publicAPI.xrRender = async (t, frame) => {
     const xrSession = frame.session;
+    const isXrSessionHMD = [
+      XrSessionTypes.HmdVR,
+      XrSessionTypes.HmdAR,
+    ].includes(model.xrSessionType);
 
     model.renderable
       .getInteractor()
@@ -466,9 +467,7 @@ function vtkOpenGLRenderWindow(publicAPI, model) {
       xrPose.views.forEach((view, index) => {
         const viewport = glLayer.getViewport(view);
 
-        // TODO: Appropriate handling for AR passthrough on HMDs
-        // with two eyes will require further investigation.
-        if (model.xrSessionType === XrSessionTypes.HmdVR) {
+        if (isXrSessionHMD) {
           if (view.eye === 'left') {
             ren.setViewport(0, 0, 0.5, 1.0);
           } else if (view.eye === 'right') {