Auto merge of #23292 - Manishearth:input, r=asajeffrey

Add support for XRInputSource and target ray spaces

Untested, but compiles.

r? @jdm or @asajeffrey

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/23292)
<!-- Reviewable:end -->

components/script/dom/bindings/trace.rs

@@ -138,7 +138,7 @@ use tendril::{StrTendril, TendrilSink};
 use time::{Duration, Timespec};
 use uuid::Uuid;
 use webrender_api::{DocumentId, ImageKey, RenderApiSender};
-use webvr_traits::WebVRGamepadHand;
+use webvr_traits::{WebVRGamepadData, WebVRGamepadHand, WebVRGamepadState};
 
 /// A trait to allow tracing (only) DOM objects.
 pub unsafe trait JSTraceable {
@@ -478,7 +478,7 @@ unsafe_no_jsmanaged_fields!(WebGLVertexArrayId);
 unsafe_no_jsmanaged_fields!(WebGLVersion);
 unsafe_no_jsmanaged_fields!(WebGLSLVersion);
 unsafe_no_jsmanaged_fields!(MediaList);
-unsafe_no_jsmanaged_fields!(WebVRGamepadHand);
+unsafe_no_jsmanaged_fields!(WebVRGamepadData, WebVRGamepadState, WebVRGamepadHand);
 unsafe_no_jsmanaged_fields!(ScriptToConstellationChan);
 unsafe_no_jsmanaged_fields!(InteractiveMetrics);
 unsafe_no_jsmanaged_fields!(InteractiveWindow);

components/script/dom/mod.rs

@@ -540,6 +540,7 @@ pub mod xmlhttprequestupload;
 pub mod xmlserializer;
 pub mod xr;
 pub mod xrframe;
+pub mod xrinputsource;
 pub mod xrlayer;
 pub mod xrpose;
 pub mod xrreferencespace;

components/script/dom/vrdisplay.rs

@@ -22,7 +22,7 @@ use crate::dom::bindings::inheritance::Castable;
 use crate::dom::bindings::num::Finite;
 use crate::dom::bindings::refcounted::{Trusted, TrustedPromise};
 use crate::dom::bindings::reflector::{reflect_dom_object, DomObject};
-use crate::dom::bindings::root::{DomRoot, MutDom, MutNullableDom};
+use crate::dom::bindings::root::{Dom, DomRoot, MutDom, MutNullableDom};
 use crate::dom::bindings::str::DOMString;
 use crate::dom::event::Event;
 use crate::dom::eventtarget::EventTarget;
@@ -36,6 +36,7 @@ use crate::dom::vrpose::VRPose;
 use crate::dom::vrstageparameters::VRStageParameters;
 use crate::dom::webglrenderingcontext::WebGLRenderingContext;
 use crate::dom::xrframe::XRFrame;
+use crate::dom::xrinputsource::XRInputSource;
 use crate::dom::xrsession::XRSession;
 use crate::dom::xrwebgllayer::XRWebGLLayer;
 use crate::script_runtime::CommonScriptMsg;
@@ -47,11 +48,12 @@ use dom_struct::dom_struct;
 use ipc_channel::ipc::IpcSender;
 use profile_traits::ipc;
 use std::cell::Cell;
+use std::collections::HashMap;
 use std::mem;
 use std::ops::Deref;
 use std::rc::Rc;
 use std::thread;
-use webvr_traits::{WebVRDisplayData, WebVRDisplayEvent, WebVRFrameData, WebVRFutureFrameData};
+use webvr_traits::{WebVRDisplayData, WebVRDisplayEvent, WebVRFrameData, WebVRPoseInformation};
 use webvr_traits::{WebVRLayer, WebVRMsg};
 
 #[dom_struct]
@@ -86,12 +88,16 @@ pub struct VRDisplay {
     // Compositor VRFrameData synchonization
     frame_data_status: Cell<VRFrameDataStatus>,
     #[ignore_malloc_size_of = "closures are hard"]
-    frame_data_receiver: DomRefCell<Option<WebGLReceiver<Result<WebVRFutureFrameData, ()>>>>,
+    frame_data_receiver: DomRefCell<Option<WebGLReceiver<Result<WebVRPoseInformation, ()>>>>,
     running_display_raf: Cell<bool>,
     paused: Cell<bool>,
     stopped_on_pause: Cell<bool>,
     /// Whether or not this is XR mode, and the session
     xr_session: MutNullableDom<XRSession>,
+    /// Have inputs been initialized? (i.e, has getInputSources() been called?)
+    /// XR only
+    initialized_inputs: Cell<bool>,
+    input_sources: DomRefCell<HashMap<u32, Dom<XRInputSource>>>,
 }
 
 unsafe_no_jsmanaged_fields!(WebVRDisplayData);
@@ -115,6 +121,8 @@ struct VRRAFUpdate {
     /// Number uniquely identifying the WebGL context
     /// so that we may setup/tear down VR compositors as things change
     context_id: usize,
+    /// Do we need input data?
+    needs_inputs: bool,
 }
 
 type VRRAFUpdateSender = Sender<Result<VRRAFUpdate, ()>>;
@@ -164,6 +172,8 @@ impl VRDisplay {
             // When the VR Resume event is received and the flag is set, VR presentation automatically restarts.
             stopped_on_pause: Cell::new(false),
             xr_session: MutNullableDom::default(),
+            initialized_inputs: Cell::new(false),
+            input_sources: DomRefCell::new(HashMap::new()),
         }
     }
 
@@ -627,6 +637,7 @@ impl VRDisplay {
             depth_far: self.depth_far.get(),
             api_sender: self.api_sender(),
             context_id: self.context_id(),
+            needs_inputs: self.initialized_inputs.get(),
         }
     }
 
@@ -690,6 +701,7 @@ impl VRDisplay {
         let (raf_sender, raf_receiver) = unbounded();
         let (wakeup_sender, wakeup_receiver) = unbounded();
         *self.raf_wakeup_sender.borrow_mut() = Some(wakeup_sender);
+        let mut needs_inputs = false;
 
         // The render loop at native headset frame rate is implemented using a dedicated thread.
         // Every loop iteration syncs pose data with the HMD, submits the pixels to the display and waits for Vsync.
@@ -726,8 +738,13 @@ impl VRDisplay {
                             .unwrap();
 
                         // Run Sync Poses in parallell on Render thread
-                        let msg =
-                            WebVRCommand::SyncPoses(display_id, near, far, sync_sender.clone());
+                        let msg = WebVRCommand::SyncPoses(
+                            display_id,
+                            near,
+                            far,
+                            needs_inputs,
+                            sync_sender.clone(),
+                        );
                         api_sender.send_vr(msg).unwrap();
                     } else {
                         let _ = wakeup_receiver.recv();
@@ -752,6 +769,7 @@ impl VRDisplay {
                     if let Ok(update) = raf_receiver.recv().unwrap() {
                         near = update.depth_near;
                         far = update.depth_far;
+                        needs_inputs = update.needs_inputs;
                         if update.context_id != context_id {
                             if let Some(ref api_sender) = update.api_sender {
                                 api_sender
@@ -808,8 +826,16 @@ impl VRDisplay {
     fn sync_frame_data(&self) {
         let status = if let Some(receiver) = self.frame_data_receiver.borrow().as_ref() {
             match receiver.recv().unwrap() {
-                Ok(future_data) => {
-                    *self.frame_data.borrow_mut() = future_data.block();
+                Ok(pose) => {
+                    *self.frame_data.borrow_mut() = pose.frame.block();
+                    if self.initialized_inputs.get() {
+                        let inputs = self.input_sources.borrow();
+                        for (id, state) in pose.gamepads {
+                            if let Some(input) = inputs.get(&id) {
+                                input.update_state(state);
+                            }
+                        }
+                    }
                     VRFrameDataStatus::Synced
                 },
                 Err(()) => VRFrameDataStatus::Exit,
@@ -909,6 +935,53 @@ impl VRDisplay {
             pair.1 = None;
         }
     }
+
+    /// Initialize XRInputSources
+    fn initialize_inputs(&self) {
+        if self.initialized_inputs.get() {
+            return;
+        }
+        self.initialized_inputs.set(true);
+
+        let (sender, receiver) = ipc::channel(self.global().time_profiler_chan().clone()).unwrap();
+        let display = self.display.borrow().display_id;
+        self.webvr_thread()
+            .send(WebVRMsg::GetGamepadsForDisplay(display, sender))
+            .unwrap();
+        match receiver.recv().unwrap() {
+            Ok(gamepads) => {
+                let global = self.global();
+                let session = self
+                    .xr_session
+                    .get()
+                    .expect("initialize_inputs called on a VR session");
+                let roots: Vec<_> = gamepads
+                    .into_iter()
+                    .map(|g| {
+                        (
+                            g.1.gamepad_id,
+                            XRInputSource::new(&global, &session, g.0, g.1),
+                        )
+                    })
+                    .collect();
+
+                let mut inputs = self.input_sources.borrow_mut();
+                for (id, root) in &roots {
+                    inputs.insert(*id, Dom::from_ref(&root));
+                }
+            },
+            Err(_) => {},
+        }
+    }
+
+    pub fn get_input_sources(&self) -> Vec<DomRoot<XRInputSource>> {
+        self.initialize_inputs();
+        self.input_sources
+            .borrow()
+            .iter()
+            .map(|(_, x)| DomRoot::from_ref(&**x))
+            .collect()
+    }
 }
 
 // WebVR Spec: If the number of values in the leftBounds/rightBounds arrays

components/script/dom/webidls/XRInputSource.webidl

@@ -0,0 +1,26 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+// https://immersive-web.github.io/webxr/#xrinputsource-interface
+
+enum XRHandedness {
+  "none",
+  "left",
+  "right"
+};
+
+enum XRTargetRayMode {
+  "gaze",
+  "tracked-pointer",
+  "screen"
+};
+
+[SecureContext, Exposed=Window, Pref="dom.webxr.enabled"]
+interface XRInputSource {
+  readonly attribute XRHandedness handedness;
+  // [SameObject] readonly attribute XRTargetRayMode targetRayMode;
+  [SameObject] readonly attribute XRSpace targetRaySpace;
+  // [SameObject] readonly attribute XRSpace? gripSpace;
+  // [SameObject] readonly attribute Gamepad? gamepad;
+};

components/script/dom/webidls/XRSession.webidl

@@ -20,12 +20,15 @@ interface XRSession : EventTarget {
   readonly attribute XREnvironmentBlendMode environmentBlendMode;
 
   readonly attribute XRRenderState renderState;
-  readonly attribute XRSpace viewerSpace;
+  [SameObject] readonly attribute XRSpace viewerSpace;
 
   // // Methods
   Promise<XRReferenceSpace> requestReferenceSpace(XRReferenceSpaceOptions options);
 
+  // workaround until we have FrozenArray
+  // see https://github.com/servo/servo/issues/10427#issuecomment-449593626
   // FrozenArray<XRInputSource> getInputSources();
+  sequence<XRInputSource> getInputSources();
 
   Promise<void> updateRenderState(optional XRRenderStateInit state);
   long requestAnimationFrame(XRFrameRequestCallback callback);

components/script/dom/xrinputsource.rs

@@ -0,0 +1,83 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+use crate::dom::bindings::cell::DomRefCell;
+use crate::dom::bindings::codegen::Bindings::XRInputSourceBinding;
+use crate::dom::bindings::codegen::Bindings::XRInputSourceBinding::{
+    XRHandedness, XRInputSourceMethods,
+};
+use crate::dom::bindings::reflector::{reflect_dom_object, DomObject, Reflector};
+use crate::dom::bindings::root::{Dom, DomRoot, MutNullableDom};
+use crate::dom::globalscope::GlobalScope;
+use crate::dom::xrsession::XRSession;
+use crate::dom::xrspace::XRSpace;
+use dom_struct::dom_struct;
+use webvr_traits::{WebVRGamepadData, WebVRGamepadHand, WebVRGamepadState, WebVRPose};
+
+#[dom_struct]
+pub struct XRInputSource {
+    reflector: Reflector,
+    session: Dom<XRSession>,
+    #[ignore_malloc_size_of = "Defined in rust-webvr"]
+    data: WebVRGamepadData,
+    #[ignore_malloc_size_of = "Defined in rust-webvr"]
+    state: DomRefCell<WebVRGamepadState>,
+    target_ray_space: MutNullableDom<XRSpace>,
+}
+
+impl XRInputSource {
+    pub fn new_inherited(
+        session: &XRSession,
+        data: WebVRGamepadData,
+        state: WebVRGamepadState,
+    ) -> XRInputSource {
+        XRInputSource {
+            reflector: Reflector::new(),
+            session: Dom::from_ref(session),
+            data,
+            state: DomRefCell::new(state),
+            target_ray_space: Default::default(),
+        }
+    }
+
+    pub fn new(
+        global: &GlobalScope,
+        session: &XRSession,
+        data: WebVRGamepadData,
+        state: WebVRGamepadState,
+    ) -> DomRoot<XRInputSource> {
+        reflect_dom_object(
+            Box::new(XRInputSource::new_inherited(session, data, state)),
+            global,
+            XRInputSourceBinding::Wrap,
+        )
+    }
+
+    pub fn update_state(&self, state: WebVRGamepadState) {
+        *self.state.borrow_mut() = state;
+    }
+
+    pub fn pose(&self) -> WebVRPose {
+        self.state.borrow().pose
+    }
+}
+
+impl XRInputSourceMethods for XRInputSource {
+    /// https://immersive-web.github.io/webxr/#dom-xrinputsource-handedness
+    fn Handedness(&self) -> XRHandedness {
+        match self.data.hand {
+            WebVRGamepadHand::Unknown => XRHandedness::None,
+            WebVRGamepadHand::Left => XRHandedness::Left,
+            WebVRGamepadHand::Right => XRHandedness::Right,
+        }
+    }
+
+    /// https://immersive-web.github.io/webxr/#dom-xrinputsource-targetrayspace
+    fn TargetRaySpace(&self) -> DomRoot<XRSpace> {
+        self.target_ray_space.or_init(|| {
+            let global = self.global();
+            XRSpace::new_inputspace(&global, &self.session, &self)
+        })
+    }
+}

components/script/dom/xrreferencespace.rs

@@ -110,7 +110,7 @@ impl XRReferenceSpace {
             // non-subclassed XRReferenceSpaces exist, obtained via the "identity"
             // type. These are equivalent to the viewer pose and follow the headset
             // around
-            XRSpace::viewer_pose_from_frame_data(base_pose)
+            XRSpace::pose_to_transform(&base_pose.pose)
         }
     }
 }

components/script/dom/xrrigidtransform.rs

@@ -94,7 +94,7 @@ impl XRRigidTransformMethods for XRRigidTransform {
     }
     // https://immersive-web.github.io/webxr/#dom-xrrigidtransform-orientation
     fn Orientation(&self) -> DomRoot<DOMPointReadOnly> {
-        self.position.or_init(|| {
+        self.orientation.or_init(|| {
             let r = &self.transform.rotation;
             DOMPointReadOnly::new(&self.global(), r.i, r.j, r.k, r.r)
         })

components/script/dom/xrsession.rs

@@ -19,6 +19,7 @@ use crate::dom::eventtarget::EventTarget;
 use crate::dom::globalscope::GlobalScope;
 use crate::dom::promise::Promise;
 use crate::dom::vrdisplay::VRDisplay;
+use crate::dom::xrinputsource::XRInputSource;
 use crate::dom::xrlayer::XRLayer;
 use crate::dom::xrreferencespace::XRReferenceSpace;
 use crate::dom::xrrenderstate::XRRenderState;
@@ -33,6 +34,7 @@ pub struct XRSession {
     display: Dom<VRDisplay>,
     base_layer: MutNullableDom<XRLayer>,
     blend_mode: XREnvironmentBlendMode,
+    viewer_space: MutNullableDom<XRSpace>,
 }
 
 impl XRSession {
@@ -43,6 +45,7 @@ impl XRSession {
             base_layer: Default::default(),
             // we don't yet support any AR devices
             blend_mode: XREnvironmentBlendMode::Opaque,
+            viewer_space: Default::default(),
         }
     }
 
@@ -86,7 +89,8 @@ impl XRSessionMethods for XRSession {
 
     // https://immersive-web.github.io/webxr/#dom-xrsession-viewerspace
     fn ViewerSpace(&self) -> DomRoot<XRSpace> {
-        XRSpace::new_viewerspace(&self.global(), &self)
+        self.viewer_space
+            .or_init(|| XRSpace::new_viewerspace(&self.global(), &self))
     }
 
     /// https://immersive-web.github.io/webxr/#dom-xrsession-requestanimationframe
@@ -153,4 +157,9 @@ impl XRSessionMethods for XRSession {
 
         p
     }
+
+    /// https://immersive-web.github.io/webxr/#dom-xrsession-getinputsources
+    fn GetInputSources(&self) -> Vec<DomRoot<XRInputSource>> {
+        self.display.get_input_sources()
+    }
 }

components/script/dom/xrspace.rs

@@ -5,20 +5,22 @@
 use crate::dom::bindings::codegen::Bindings::XRSpaceBinding;
 use crate::dom::bindings::inheritance::Castable;
 use crate::dom::bindings::reflector::reflect_dom_object;
-use crate::dom::bindings::root::{Dom, DomRoot};
+use crate::dom::bindings::root::{Dom, DomRoot, MutNullableDom};
 use crate::dom::eventtarget::EventTarget;
 use crate::dom::globalscope::GlobalScope;
+use crate::dom::xrinputsource::XRInputSource;
 use crate::dom::xrreferencespace::XRReferenceSpace;
 use crate::dom::xrsession::XRSession;
 use dom_struct::dom_struct;
 use euclid::{RigidTransform3D, Rotation3D, Vector3D};
-use webvr_traits::WebVRFrameData;
+use webvr_traits::{WebVRFrameData, WebVRPose};
 
 #[dom_struct]
 pub struct XRSpace {
     eventtarget: EventTarget,
     session: Dom<XRSession>,
     is_viewerspace: bool,
+    input_source: MutNullableDom<XRInputSource>,
 }
 
 impl XRSpace {
@@ -27,6 +29,7 @@ impl XRSpace {
             eventtarget: EventTarget::new_inherited(),
             session: Dom::from_ref(session),
             is_viewerspace: false,
+            input_source: Default::default(),
         }
     }
 
@@ -35,6 +38,7 @@ impl XRSpace {
             eventtarget: EventTarget::new_inherited(),
             session: Dom::from_ref(session),
             is_viewerspace: true,
+            input_source: Default::default(),
         }
     }
 
@@ -45,6 +49,27 @@ impl XRSpace {
             XRSpaceBinding::Wrap,
         )
     }
+
+    fn new_inputspace_inner(session: &XRSession, input: &XRInputSource) -> XRSpace {
+        XRSpace {
+            eventtarget: EventTarget::new_inherited(),
+            session: Dom::from_ref(session),
+            is_viewerspace: false,
+            input_source: MutNullableDom::new(Some(input)),
+        }
+    }
+
+    pub fn new_inputspace(
+        global: &GlobalScope,
+        session: &XRSession,
+        input: &XRInputSource,
+    ) -> DomRoot<XRSpace> {
+        reflect_dom_object(
+            Box::new(XRSpace::new_inputspace_inner(session, input)),
+            global,
+            XRSpaceBinding::Wrap,
+        )
+    }
 }
 
 impl XRSpace {
@@ -57,16 +82,18 @@ impl XRSpace {
         if let Some(reference) = self.downcast::<XRReferenceSpace>() {
             reference.get_pose(base_pose)
         } else if self.is_viewerspace {
-            XRSpace::viewer_pose_from_frame_data(base_pose)
+            XRSpace::pose_to_transform(&base_pose.pose)
+        } else if let Some(source) = self.input_source.get() {
+            XRSpace::pose_to_transform(&source.pose())
         } else {
             unreachable!()
         }
     }
 
-    pub fn viewer_pose_from_frame_data(data: &WebVRFrameData) -> RigidTransform3D<f64> {
-        let pos = data.pose.position.unwrap_or([0., 0., 0.]);
+    pub fn pose_to_transform(pose: &WebVRPose) -> RigidTransform3D<f64> {
+        let pos = pose.position.unwrap_or([0., 0., 0.]);
         let translation = Vector3D::new(pos[0] as f64, pos[1] as f64, pos[2] as f64);
-        let orient = data.pose.orientation.unwrap_or([0., 0., 0., 0.]);
+        let orient = pose.orientation.unwrap_or([0., 0., 0., 0.]);
         let rotation = Rotation3D::quaternion(
             orient[0] as f64,
             orient[1] as f64,

components/script/dom/xrstationaryreferencespace.rs

@@ -55,7 +55,7 @@ impl XRStationaryReferenceSpace {
     ///
     /// Does not apply originOffset, use get_viewer_pose on XRReferenceSpace instead
     pub fn get_unoffset_viewer_pose(&self, viewer_pose: &WebVRFrameData) -> RigidTransform3D<f64> {
-        let viewer_pose = XRSpace::viewer_pose_from_frame_data(viewer_pose);
+        let viewer_pose = XRSpace::pose_to_transform(&viewer_pose.pose);
         // all math is in column-vector notation
         // we use the following equation to verify correctness here:
         // get_viewer_pose(space) = get_pose(space).inverse() * get_pose(viewer_space)
@@ -113,7 +113,7 @@ impl XRStationaryReferenceSpace {
             },
             XRStationaryReferenceSpaceSubtype::Position_disabled => {
                 // This space follows the user around, but does not mirror the user's orientation
-                let viewer_pose = XRSpace::viewer_pose_from_frame_data(viewer_pose);
+                let viewer_pose = XRSpace::pose_to_transform(&viewer_pose.pose);
                 viewer_pose.translation.into()
             },
         }