Auto merge of #22699 - Manishearth:render_state, r=jdm

Update XR code to support null layers and new spec changes

Last time we landed an incomplete PR (https://github.com/servo/servo/pull/22649) that corrected the promise model but left XR sessions in panicky states.

This updates the code to not panic all the time, and also includes changes from https://github.com/immersive-web/webxr/pull/458

r? @jdm

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/22699)
<!-- Reviewable:end -->

components/script/dom/mod.rs

@@ -538,6 +538,7 @@ pub mod xr;
 pub mod xrframe;
 pub mod xrlayer;
 pub mod xrreferencespace;
+pub mod xrrenderstate;
 pub mod xrrigidtransform;
 pub mod xrsession;
 pub mod xrspace;

components/script/dom/vrdisplay.rs

@@ -13,7 +13,9 @@ use crate::dom::bindings::codegen::Bindings::VRLayerBinding::VRLayer;
 use crate::dom::bindings::codegen::Bindings::WebGLRenderingContextBinding::WebGLRenderingContextMethods;
 use crate::dom::bindings::codegen::Bindings::WindowBinding::FrameRequestCallback;
 use crate::dom::bindings::codegen::Bindings::WindowBinding::WindowMethods;
+use crate::dom::bindings::codegen::Bindings::XRRenderStateBinding::XRRenderStateInit;
 use crate::dom::bindings::codegen::Bindings::XRSessionBinding::XRFrameRequestCallback;
+use crate::dom::bindings::codegen::Bindings::XRWebGLLayerBinding::XRWebGLLayerMethods;
 use crate::dom::bindings::error::Error;
 use crate::dom::bindings::inheritance::Castable;
 use crate::dom::bindings::num::Finite;
@@ -34,10 +36,11 @@ use crate::dom::vrstageparameters::VRStageParameters;
 use crate::dom::webglrenderingcontext::WebGLRenderingContext;
 use crate::dom::xrframe::XRFrame;
 use crate::dom::xrsession::XRSession;
+use crate::dom::xrwebgllayer::XRWebGLLayer;
 use crate::script_runtime::CommonScriptMsg;
 use crate::script_runtime::ScriptThreadEventCategory::WebVREvent;
 use crate::task_source::{TaskSource, TaskSourceName};
-use canvas_traits::webgl::{webgl_channel, WebGLReceiver, WebVRCommand};
+use canvas_traits::webgl::{webgl_channel, WebGLMsgSender, WebGLReceiver, WebVRCommand};
 use crossbeam_channel::{unbounded, Sender};
 use dom_struct::dom_struct;
 use ipc_channel::ipc::IpcSender;
@@ -58,6 +61,7 @@ pub struct VRDisplay {
     depth_near: Cell<f64>,
     depth_far: Cell<f64>,
     presenting: Cell<bool>,
+    has_raf_thread: Cell<bool>,
     left_eye_params: MutDom<VREyeParameters>,
     right_eye_params: MutDom<VREyeParameters>,
     capabilities: MutDom<VRDisplayCapabilities>,
@@ -74,6 +78,10 @@ pub struct VRDisplay {
     raf_callback_list: DomRefCell<Vec<(u32, Option<Rc<FrameRequestCallback>>)>>,
     #[ignore_malloc_size_of = "closures are hard"]
     xr_raf_callback_list: DomRefCell<Vec<(u32, Option<Rc<XRFrameRequestCallback>>)>>,
+    /// When there isn't any layer_ctx the RAF thread needs to be "woken up"
+    raf_wakeup_sender: DomRefCell<Option<Sender<()>>>,
+    #[ignore_malloc_size_of = "Rc is hard"]
+    pending_renderstate_updates: DomRefCell<Vec<(XRRenderStateInit, Rc<Promise>)>>,
     // Compositor VRFrameData synchonization
     frame_data_status: Cell<VRFrameDataStatus>,
     #[ignore_malloc_size_of = "closures are hard"]
@@ -88,6 +96,7 @@ pub struct VRDisplay {
 unsafe_no_jsmanaged_fields!(WebVRDisplayData);
 unsafe_no_jsmanaged_fields!(WebVRFrameData);
 unsafe_no_jsmanaged_fields!(WebVRLayer);
+unsafe_no_jsmanaged_fields!(VRFrameDataStatus);
 
 #[derive(Clone, Copy, Eq, MallocSizeOf, PartialEq)]
 enum VRFrameDataStatus {
@@ -96,7 +105,18 @@ enum VRFrameDataStatus {
     Exit,
 }
 
-unsafe_no_jsmanaged_fields!(VRFrameDataStatus);
+#[derive(Clone, MallocSizeOf)]
+struct VRRAFUpdate {
+    depth_near: f64,
+    depth_far: f64,
+    /// WebGL API sender
+    api_sender: Option<WebGLMsgSender>,
+    /// Number uniquely identifying the WebGL context
+    /// so that we may setup/tear down VR compositors as things change
+    context_id: usize,
+}
+
+type VRRAFUpdateSender = Sender<Result<VRRAFUpdate, ()>>;
 
 impl VRDisplay {
     fn new_inherited(global: &GlobalScope, display: WebVRDisplayData) -> VRDisplay {
@@ -111,6 +131,7 @@ impl VRDisplay {
             depth_near: Cell::new(0.01),
             depth_far: Cell::new(10000.0),
             presenting: Cell::new(false),
+            has_raf_thread: Cell::new(false),
             left_eye_params: MutDom::new(&*VREyeParameters::new(
                 display.left_eye_parameters.clone(),
                 &global,
@@ -130,6 +151,8 @@ impl VRDisplay {
             next_raf_id: Cell::new(1),
             raf_callback_list: DomRefCell::new(vec![]),
             xr_raf_callback_list: DomRefCell::new(vec![]),
+            raf_wakeup_sender: DomRefCell::new(None),
+            pending_renderstate_updates: DomRefCell::new(vec![]),
             frame_data_status: Cell::new(VRFrameDataStatus::Waiting),
             frame_data_receiver: DomRefCell::new(None),
             running_display_raf: Cell::new(false),
@@ -416,7 +439,10 @@ impl VRDisplayMethods for VRDisplay {
         let display_id = self.display.borrow().display_id;
         let layer = self.layer.borrow();
         let msg = WebVRCommand::SubmitFrame(display_id, layer.left_bounds, layer.right_bounds);
-        self.layer_ctx.get().unwrap().send_vr_command(msg);
+        self.layer_ctx
+            .get()
+            .expect("SubmitFrame can only be called when there is a webgl layer")
+            .send_vr_command(msg);
     }
 
     // https://w3c.github.io/webvr/spec/1.1/#dom-vrdisplay-getlayers
@@ -567,21 +593,87 @@ impl VRDisplay {
             .fire(self.global().upcast::<EventTarget>());
     }
 
+    fn api_sender(&self) -> Option<WebGLMsgSender> {
+        self.layer_ctx.get().map(|c| c.webgl_sender())
+    }
+
+    fn context_id(&self) -> usize {
+        self.layer_ctx
+            .get()
+            .map(|c| &*c as *const WebGLRenderingContext as usize)
+            .unwrap_or(0)
+    }
+
+    fn vr_raf_update(&self) -> VRRAFUpdate {
+        VRRAFUpdate {
+            depth_near: self.depth_near.get(),
+            depth_far: self.depth_far.get(),
+            api_sender: self.api_sender(),
+            context_id: self.context_id(),
+        }
+    }
+
+    pub fn queue_renderstate(&self, state: &XRRenderStateInit, promise: Rc<Promise>) {
+        // can't clone dictionaries
+        let new_state = XRRenderStateInit {
+            depthNear: state.depthNear,
+            depthFar: state.depthFar,
+            baseLayer: state.baseLayer.clone(),
+        };
+        self.pending_renderstate_updates
+            .borrow_mut()
+            .push((new_state, promise));
+
+        if let Some(ref wakeup) = *self.raf_wakeup_sender.borrow() {
+            let _ = wakeup.send(());
+        }
+    }
+
+    fn process_renderstate_queue(&self) {
+        let mut updates = self.pending_renderstate_updates.borrow_mut();
+
+        debug_assert!(updates.is_empty() || self.xr_session.get().is_some());
+        for update in updates.drain(..) {
+            if let Some(near) = update.0.depthNear {
+                self.depth_near.set(*near);
+            }
+            if let Some(far) = update.0.depthFar {
+                self.depth_far.set(*far);
+            }
+            if let Some(ref layer) = update.0.baseLayer {
+                self.xr_session.get().unwrap().set_layer(&layer);
+                let layer = layer.downcast::<XRWebGLLayer>().unwrap();
+                self.layer_ctx.set(Some(&layer.Context()));
+            }
+            update.1.resolve_native(&());
+        }
+    }
+
     fn init_present(&self) {
         self.presenting.set(true);
         let xr = self.global().as_window().Navigator().Xr();
         xr.set_active_immersive_session(&self);
+        self.process_renderstate_queue();
+        if self.has_raf_thread.get() {
+            return;
+        }
+        self.has_raf_thread.set(true);
         let (sync_sender, sync_receiver) = webgl_channel().unwrap();
         *self.frame_data_receiver.borrow_mut() = Some(sync_receiver);
 
         let display_id = self.display.borrow().display_id;
-        let api_sender = self.layer_ctx.get().unwrap().webgl_sender();
+        let mut api_sender = self.api_sender();
+        let mut context_id = self.context_id();
         let js_sender = self.global().script_chan();
         let address = Trusted::new(&*self);
-        let near_init = self.depth_near.get();
-        let far_init = self.depth_far.get();
+        let mut near = self.depth_near.get();
+        let mut far = self.depth_far.get();
         let pipeline_id = self.global().pipeline_id();
 
+        let (raf_sender, raf_receiver) = unbounded();
+        let (wakeup_sender, wakeup_receiver) = unbounded();
+        *self.raf_wakeup_sender.borrow_mut() = Some(wakeup_sender);
+
         // The render loop at native headset frame rate is implemented using a dedicated thread.
         // Every loop iteration syncs pose data with the HMD, submits the pixels to the display and waits for Vsync.
         // Both the requestAnimationFrame call of a VRDisplay in the JavaScript thread and the VRSyncPoses call
@@ -591,40 +683,74 @@ impl VRDisplay {
         thread::Builder::new()
             .name("WebVR_RAF".into())
             .spawn(move || {
-                let (raf_sender, raf_receiver) = unbounded();
-                let mut near = near_init;
-                let mut far = far_init;
-
                 // Initialize compositor
-                api_sender
-                    .send_vr(WebVRCommand::Create(display_id))
-                    .unwrap();
-                loop {
-                    // Run RAF callbacks on JavaScript thread
-                    let this = address.clone();
-                    let sender = raf_sender.clone();
-                    let task = Box::new(task!(handle_vrdisplay_raf: move || {
-                        this.root().handle_raf(&sender);
-                    }));
-                    // NOTE: WebVR spec doesn't specify what task source we should use. Is
-                    // dom-manipulation a good choice long term?
-                    js_sender
-                        .send(CommonScriptMsg::Task(
-                            WebVREvent,
-                            task,
-                            Some(pipeline_id),
-                            TaskSourceName::DOMManipulation,
-                        ))
+                if let Some(ref api_sender) = api_sender {
+                    api_sender
+                        .send_vr(WebVRCommand::Create(display_id))
                         .unwrap();
+                }
+                loop {
+                    if let Some(ref api_sender) = api_sender {
+                        // Run RAF callbacks on JavaScript thread
+                        let this = address.clone();
+                        let sender = raf_sender.clone();
+                        let task = Box::new(task!(handle_vrdisplay_raf: move || {
+                            this.root().handle_raf(&sender);
+                        }));
+                        // NOTE: WebVR spec doesn't specify what task source we should use. Is
+                        // dom-manipulation a good choice long term?
+                        js_sender
+                            .send(CommonScriptMsg::Task(
+                                WebVREvent,
+                                task,
+                                Some(pipeline_id),
+                                TaskSourceName::DOMManipulation,
+                            ))
+                            .unwrap();
 
-                    // Run Sync Poses in parallell on Render thread
-                    let msg = WebVRCommand::SyncPoses(display_id, near, far, sync_sender.clone());
-                    api_sender.send_vr(msg).unwrap();
+                        // Run Sync Poses in parallell on Render thread
+                        let msg =
+                            WebVRCommand::SyncPoses(display_id, near, far, sync_sender.clone());
+                        api_sender.send_vr(msg).unwrap();
+                    } else {
+                        let _ = wakeup_receiver.recv();
+                        let sender = raf_sender.clone();
+                        let this = address.clone();
+                        let task = Box::new(task!(flush_renderstate_queue: move || {
+                            let this = this.root();
+                            this.process_renderstate_queue();
+                            sender.send(Ok(this.vr_raf_update())).unwrap();
+                        }));
+                        js_sender
+                            .send(CommonScriptMsg::Task(
+                                WebVREvent,
+                                task,
+                                Some(pipeline_id),
+                                TaskSourceName::DOMManipulation,
+                            ))
+                            .unwrap();
+                    }
 
                     // Wait until both SyncPoses & RAF ends
-                    if let Ok(depth) = raf_receiver.recv().unwrap() {
-                        near = depth.0;
-                        far = depth.1;
+                    if let Ok(update) = raf_receiver.recv().unwrap() {
+                        near = update.depth_near;
+                        far = update.depth_far;
+                        if update.context_id != context_id {
+                            if let Some(ref api_sender) = update.api_sender {
+                                api_sender
+                                    .send_vr(WebVRCommand::Create(display_id))
+                                    .unwrap();
+                            }
+                            if let Some(ref api_sender) = api_sender {
+                                // shut down old vr compositor
+                                api_sender
+                                    .send_vr(WebVRCommand::Release(display_id))
+                                    .unwrap();
+                            }
+                            context_id = update.context_id;
+                        }
+
+                        api_sender = update.api_sender;
                     } else {
                         // Stop thread
                         // ExitPresent called or some error happened
@@ -640,12 +766,13 @@ impl VRDisplay {
         let xr = self.global().as_window().Navigator().Xr();
         xr.deactivate_session();
         *self.frame_data_receiver.borrow_mut() = None;
-
-        let api_sender = self.layer_ctx.get().unwrap().webgl_sender();
-        let display_id = self.display.borrow().display_id;
-        api_sender
-            .send_vr(WebVRCommand::Release(display_id))
-            .unwrap();
+        self.has_raf_thread.set(false);
+        if let Some(api_sender) = self.api_sender() {
+            let display_id = self.display.borrow().display_id;
+            api_sender
+                .send_vr(WebVRCommand::Release(display_id))
+                .unwrap();
+        }
     }
 
     // Only called when the JSContext is destroyed while presenting.
@@ -677,7 +804,7 @@ impl VRDisplay {
         self.frame_data_status.set(status);
     }
 
-    fn handle_raf(&self, end_sender: &Sender<Result<(f64, f64), ()>>) {
+    fn handle_raf(&self, end_sender: &VRRAFUpdateSender) {
         self.frame_data_status.set(VRFrameDataStatus::Waiting);
 
         let now = self.global().as_window().Performance().Now();
@@ -717,12 +844,11 @@ impl VRDisplay {
             }
         }
 
+        self.process_renderstate_queue();
         match self.frame_data_status.get() {
             VRFrameDataStatus::Synced => {
                 // Sync succeeded. Notify RAF thread.
-                end_sender
-                    .send(Ok((self.depth_near.get(), self.depth_far.get())))
-                    .unwrap();
+                end_sender.send(Ok(self.vr_raf_update())).unwrap();
             },
             VRFrameDataStatus::Exit | VRFrameDataStatus::Waiting => {
                 // ExitPresent called or some error ocurred.

components/script/dom/webidls/XRRenderState.webidl

@@ -0,0 +1,17 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+// https://immersive-web.github.io/webxr/#xrrenderstate-interface
+
+dictionary XRRenderStateInit {
+  double depthNear;
+  double depthFar;
+  XRLayer baseLayer;
+};
+
+[SecureContext, Exposed=Window, Pref="dom.webxr.enabled"] interface XRRenderState {
+  readonly attribute double depthNear;
+  readonly attribute double depthFar;
+  readonly attribute XRLayer? baseLayer;
+};

components/script/dom/webidls/XRSession.webidl

@@ -19,15 +19,14 @@ interface XRSession : EventTarget {
   // readonly attribute XRPresentationContext outputContext;
   readonly attribute XREnvironmentBlendMode environmentBlendMode;
 
-  attribute double depthNear;
-  attribute double depthFar;
-  attribute XRLayer? baseLayer;
+  readonly attribute XRRenderState renderState;
 
   // // Methods
   Promise<XRReferenceSpace> requestReferenceSpace(XRReferenceSpaceOptions options);
 
   // FrozenArray<XRInputSource> getInputSources();
 
+  Promise<void> updateRenderState(optional XRRenderStateInit state);
   long requestAnimationFrame(XRFrameRequestCallback callback);
   void cancelAnimationFrame(long handle);
 

components/script/dom/xrrenderstate.rs

@@ -0,0 +1,67 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+use crate::dom::bindings::codegen::Bindings::XRRenderStateBinding::{self, XRRenderStateMethods};
+use crate::dom::bindings::num::Finite;
+use crate::dom::bindings::reflector::reflect_dom_object;
+use crate::dom::bindings::reflector::Reflector;
+use crate::dom::bindings::root::{DomRoot, MutNullableDom};
+use crate::dom::globalscope::GlobalScope;
+use crate::dom::xrlayer::XRLayer;
+
+use dom_struct::dom_struct;
+use std::cell::Cell;
+
+#[dom_struct]
+pub struct XRRenderState {
+    reflector_: Reflector,
+    depth_near: Cell<f64>,
+    depth_far: Cell<f64>,
+    layer: MutNullableDom<XRLayer>,
+}
+
+impl XRRenderState {
+    pub fn new_inherited(
+        depth_near: f64,
+        depth_far: f64,
+        layer: Option<&XRLayer>,
+    ) -> XRRenderState {
+        XRRenderState {
+            reflector_: Reflector::new(),
+            depth_near: Cell::new(depth_near),
+            depth_far: Cell::new(depth_far),
+            layer: MutNullableDom::new(layer),
+        }
+    }
+
+    pub fn new(
+        global: &GlobalScope,
+        depth_near: f64,
+        depth_far: f64,
+        layer: Option<&XRLayer>,
+    ) -> DomRoot<XRRenderState> {
+        reflect_dom_object(
+            Box::new(XRRenderState::new_inherited(depth_near, depth_far, layer)),
+            global,
+            XRRenderStateBinding::Wrap,
+        )
+    }
+}
+
+impl XRRenderStateMethods for XRRenderState {
+    /// https://immersive-web.github.io/webxr/#dom-xrrenderstate-depthnear
+    fn DepthNear(&self) -> Finite<f64> {
+        Finite::wrap(self.depth_near.get())
+    }
+
+    /// https://immersive-web.github.io/webxr/#dom-xrrenderstate-depthfar
+    fn DepthFar(&self) -> Finite<f64> {
+        Finite::wrap(self.depth_far.get())
+    }
+
+    /// https://immersive-web.github.io/webxr/#dom-xrrenderstate-baselayer
+    fn GetBaseLayer(&self) -> Option<DomRoot<XRLayer>> {
+        self.layer.get()
+    }
+}

components/script/dom/xrsession.rs

@@ -4,18 +4,15 @@
 
 use crate::dom::bindings::codegen::Bindings::VRDisplayBinding::VRDisplayMethods;
 use crate::dom::bindings::codegen::Bindings::XRBinding::XRSessionMode;
+use crate::dom::bindings::codegen::Bindings::XRRenderStateBinding::XRRenderStateInit;
 use crate::dom::bindings::codegen::Bindings::XRSessionBinding;
 use crate::dom::bindings::codegen::Bindings::XRSessionBinding::XREnvironmentBlendMode;
 use crate::dom::bindings::codegen::Bindings::XRSessionBinding::XRFrameRequestCallback;
 use crate::dom::bindings::codegen::Bindings::XRSessionBinding::XRReferenceSpaceOptions;
 use crate::dom::bindings::codegen::Bindings::XRSessionBinding::XRReferenceSpaceType;
 use crate::dom::bindings::codegen::Bindings::XRSessionBinding::XRSessionMethods;
-use crate::dom::bindings::codegen::Bindings::XRWebGLLayerBinding::XRWebGLLayerMethods;
 use crate::dom::bindings::error::Error;
-use crate::dom::bindings::inheritance::Castable;
-use crate::dom::bindings::num::Finite;
-use crate::dom::bindings::reflector::reflect_dom_object;
-use crate::dom::bindings::reflector::DomObject;
+use crate::dom::bindings::reflector::{reflect_dom_object, DomObject};
 use crate::dom::bindings::root::{Dom, DomRoot, MutNullableDom};
 use crate::dom::eventtarget::EventTarget;
 use crate::dom::globalscope::GlobalScope;
@@ -23,8 +20,8 @@ use crate::dom::promise::Promise;
 use crate::dom::vrdisplay::VRDisplay;
 use crate::dom::xrlayer::XRLayer;
 use crate::dom::xrreferencespace::XRReferenceSpace;
+use crate::dom::xrrenderstate::XRRenderState;
 use crate::dom::xrstationaryreferencespace::XRStationaryReferenceSpace;
-use crate::dom::xrwebgllayer::XRWebGLLayer;
 use dom_struct::dom_struct;
 use std::rc::Rc;
 
@@ -58,49 +55,34 @@ impl XRSession {
     pub fn xr_present(&self, p: Rc<Promise>) {
         self.display.xr_present(self, None, Some(p));
     }
+
+    pub fn set_layer(&self, layer: &XRLayer) {
+        self.base_layer.set(Some(layer))
+    }
 }
 
 impl XRSessionMethods for XRSession {
-    /// https://immersive-web.github.io/webxr/#dom-xrsession-depthnear
-    fn DepthNear(&self) -> Finite<f64> {
-        self.display.DepthNear()
-    }
-
-    /// https://immersive-web.github.io/webxr/#dom-xrsession-depthfar
-    fn DepthFar(&self) -> Finite<f64> {
-        self.display.DepthFar()
-    }
-
-    /// https://immersive-web.github.io/webxr/#dom-xrsession-depthnear
-    fn SetDepthNear(&self, d: Finite<f64>) {
-        self.display.SetDepthNear(d)
-    }
-
-    /// https://immersive-web.github.io/webxr/#dom-xrsession-depthfar
-    fn SetDepthFar(&self, d: Finite<f64>) {
-        self.display.SetDepthFar(d)
-    }
-
     /// https://immersive-web.github.io/webxr/#dom-xrsession-mode
     fn Mode(&self) -> XRSessionMode {
         XRSessionMode::Immersive_vr
     }
 
-    /// https://immersive-web.github.io/webxr/#dom-xrsession-baselayer
-    fn SetBaseLayer(&self, layer: Option<&XRLayer>) {
-        self.base_layer.set(layer);
-        if let Some(layer) = layer {
-            let layer = layer.downcast::<XRWebGLLayer>().unwrap();
-            self.display.xr_present(&self, Some(&layer.Context()), None);
-        } else {
-            // steps unknown
-            // https://github.com/immersive-web/webxr/issues/453
-        }
+    // https://immersive-web.github.io/webxr/#dom-xrsession-renderstate
+    fn RenderState(&self) -> DomRoot<XRRenderState> {
+        // XXXManishearth maybe cache this
+        XRRenderState::new(
+            &self.global(),
+            *self.display.DepthNear(),
+            *self.display.DepthFar(),
+            self.base_layer.get().as_ref().map(|l| &**l),
+        )
     }
 
-    /// https://immersive-web.github.io/webxr/#dom-xrsession-baselayer
-    fn GetBaseLayer(&self) -> Option<DomRoot<XRLayer>> {
-        self.base_layer.get()
+    /// https://immersive-web.github.io/webxr/#dom-xrsession-requestanimationframe
+    fn UpdateRenderState(&self, init: &XRRenderStateInit) -> Rc<Promise> {
+        let p = Promise::new(&self.global());
+        self.display.queue_renderstate(init, p.clone());
+        p
     }
 
     /// https://immersive-web.github.io/webxr/#dom-xrsession-requestanimationframe