Auto merge of #26316 - Manishearth:hands, r=asajeffrey

Add experimental hand tracking support

Depends on https://github.com/servo/webxr/pull/162

Adds support for [the experimental hand tracking API](https://github.com/immersive-web/webxr-hands-input/blob/master/explainer.md) (with some tweaks made that I intend to upstream).

This needs https://github.com/servo/webxr/pull/163 to actually run on any backend, however that depends on some openxrs changes.

If folks want to try this out, patch in
https://github.com/servo/webxr/pull/163 and run https://manishearth.net/sand/three.js/examples/webxr_vr_paint.html . You also need to toggle the `dom.webxr.hand` pref.

<!-- Please describe your changes on the following line: -->

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `___` with appropriate data: -->
- [x] `./mach build -d` does not report any errors
- [x] `./mach test-tidy` does not report any errors
- [ ] These changes fix #___ (GitHub issue number if applicable)

<!-- Either: -->
- [ ] There are tests for these changes OR
- [x] These changes do not require tests because this is an experimental API

<!-- Also, please make sure that "Allow edits from maintainers" checkbox is checked, so that we can help you if you get stuck somewhere along the way.-->

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->
This commit is contained in:
bors-servo 2020-04-28 08:33:19 -04:00 committed by GitHub
commit 1f34c55c23
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 383 additions and 5 deletions

4
Cargo.lock generated
View file

@ -6427,7 +6427,7 @@ dependencies = [
[[package]]
name = "webxr"
version = "0.0.1"
source = "git+https://github.com/servo/webxr#6ead41b15b0c72ef8bd98af0c09f4fefec888aac"
source = "git+https://github.com/servo/webxr#eae68436697131122504b035746daa3a157b36b4"
dependencies = [
"android_injected_glue",
"bindgen",
@ -6450,7 +6450,7 @@ dependencies = [
[[package]]
name = "webxr-api"
version = "0.0.1"
source = "git+https://github.com/servo/webxr#6ead41b15b0c72ef8bd98af0c09f4fefec888aac"
source = "git+https://github.com/servo/webxr#eae68436697131122504b035746daa3a157b36b4"
dependencies = [
"euclid",
"ipc-channel",

View file

@ -299,6 +299,10 @@ mod gen {
test: bool,
#[serde(default)]
glwindow: bool,
hands: {
#[serde(default)]
enabled: bool,
},
layers: {
enabled: bool,
}

View file

@ -157,8 +157,8 @@ use webgpu::{
WebGPUPipelineLayout, WebGPUQueue, WebGPUShaderModule,
};
use webrender_api::{DocumentId, ImageKey};
use webxr_api::Ray;
use webxr_api::SwapChainId as WebXRSwapChainId;
use webxr_api::{Finger, Hand, Ray};
unsafe_no_jsmanaged_fields!(Tm);
@ -554,6 +554,7 @@ unsafe_no_jsmanaged_fields!(
webxr_api::Frame,
webxr_api::InputSource,
webxr_api::InputId,
webxr_api::Joint,
webxr_api::HitTestId,
webxr_api::HitTestResult
);
@ -881,6 +882,58 @@ where
}
}
unsafe impl<J> JSTraceable for Hand<J>
where
J: JSTraceable,
{
#[inline]
unsafe fn trace(&self, trc: *mut JSTracer) {
// exhaustive match so we don't miss new fields
let Hand {
ref wrist,
ref thumb_metacarpal,
ref thumb_phalanx_proximal,
ref thumb_phalanx_distal,
ref thumb_phalanx_tip,
ref index,
ref middle,
ref ring,
ref little,
} = *self;
wrist.trace(trc);
thumb_metacarpal.trace(trc);
thumb_phalanx_proximal.trace(trc);
thumb_phalanx_distal.trace(trc);
thumb_phalanx_tip.trace(trc);
index.trace(trc);
middle.trace(trc);
ring.trace(trc);
little.trace(trc);
}
}
unsafe impl<J> JSTraceable for Finger<J>
where
J: JSTraceable,
{
#[inline]
unsafe fn trace(&self, trc: *mut JSTracer) {
// exhaustive match so we don't miss new fields
let Finger {
ref metacarpal,
ref phalanx_proximal,
ref phalanx_intermediate,
ref phalanx_distal,
ref phalanx_tip,
} = *self;
metacarpal.trace(trc);
phalanx_proximal.trace(trc);
phalanx_intermediate.trace(trc);
phalanx_distal.trace(trc);
phalanx_tip.trace(trc);
}
}
/// Holds a set of JSTraceables that need to be rooted
struct RootedTraceableSet {
set: Vec<*const dyn JSTraceable>,

View file

@ -269,6 +269,7 @@ impl FakeXRDeviceMethods for FakeXRDevice {
id,
supports_grip: true,
profiles,
hand_support: None,
};
let init = MockInputInit {

View file

@ -572,12 +572,15 @@ pub mod xmlhttprequesteventtarget;
pub mod xmlhttprequestupload;
pub mod xmlserializer;
pub mod xrframe;
pub mod xrhand;
pub mod xrhittestresult;
pub mod xrhittestsource;
pub mod xrinputsource;
pub mod xrinputsourcearray;
pub mod xrinputsourceevent;
pub mod xrinputsourceschangeevent;
pub mod xrjointpose;
pub mod xrjointspace;
pub mod xrlayer;
pub mod xrmediabinding;
pub mod xrpose;

View file

@ -10,5 +10,6 @@ interface XRFrame {
[Throws] XRViewerPose? getViewerPose(XRReferenceSpace referenceSpace);
[Throws] XRPose? getPose(XRSpace space, XRSpace relativeTo);
[Pref="dom.webxr.hands.enabled", Throws] XRJointPose? getJointPose(XRJointSpace space, XRSpace relativeTo);
sequence<XRHitTestResult> getHitTestResults(XRHitTestSource hitTestSource);
};

View file

@ -0,0 +1,41 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
// https://github.com/immersive-web/webxr-hands-input/blob/master/explainer.md
[SecureContext, Exposed=Window, Pref="dom.webxr.hands.enabled"]
interface XRHand {
readonly attribute long length;
getter XRJointSpace(unsigned long index);
const unsigned long WRIST = 0;
const unsigned long THUMB_METACARPAL = 1;
const unsigned long THUMB_PHALANX_PROXIMAL = 2;
const unsigned long THUMB_PHALANX_DISTAL = 3;
const unsigned long THUMB_PHALANX_TIP = 4;
const unsigned long INDEX_METACARPAL = 5;
const unsigned long INDEX_PHALANX_PROXIMAL = 6;
const unsigned long INDEX_PHALANX_INTERMEDIATE = 7;
const unsigned long INDEX_PHALANX_DISTAL = 8;
const unsigned long INDEX_PHALANX_TIP = 9;
const unsigned long MIDDLE_METACARPAL = 10;
const unsigned long MIDDLE_PHALANX_PROXIMAL = 11;
const unsigned long MIDDLE_PHALANX_INTERMEDIATE = 12;
const unsigned long MIDDLE_PHALANX_DISTAL = 13;
const unsigned long MIDDLE_PHALANX_TIP = 14;
const unsigned long RING_METACARPAL = 15;
const unsigned long RING_PHALANX_PROXIMAL = 16;
const unsigned long RING_PHALANX_INTERMEDIATE = 17;
const unsigned long RING_PHALANX_DISTAL = 18;
const unsigned long RING_PHALANX_TIP = 19;
const unsigned long LITTLE_METACARPAL = 20;
const unsigned long LITTLE_PHALANX_PROXIMAL = 21;
const unsigned long LITTLE_PHALANX_INTERMEDIATE = 22;
const unsigned long LITTLE_PHALANX_DISTAL = 23;
const unsigned long LITTLE_PHALANX_TIP = 24;
};

View file

@ -24,4 +24,7 @@ interface XRInputSource {
[SameObject] readonly attribute XRSpace? gripSpace;
// [SameObject] readonly attribute Gamepad? gamepad;
/* [SameObject] */ readonly attribute /* FrozenArray<DOMString> */ any profiles;
[Pref="dom.webxr.hands.enabled"]
readonly attribute XRHand? hand;
};

View file

@ -0,0 +1,10 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
// https://github.com/immersive-web/webxr-hands-input/blob/master/explainer.md
[SecureContext, Exposed=Window, Pref="dom.webxr.hands.enabled"]
interface XRJointPose: XRPose {
readonly attribute float? radius;
};

View file

@ -0,0 +1,8 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
// https://github.com/immersive-web/webxr-hands-input/blob/master/explainer.md
[SecureContext, Exposed=Window, Pref="dom.webxr.hands.enabled"]
interface XRJointSpace: XRSpace {};

View file

@ -10,6 +10,8 @@ use crate::dom::bindings::root::{Dom, DomRoot};
use crate::dom::globalscope::GlobalScope;
use crate::dom::xrhittestresult::XRHitTestResult;
use crate::dom::xrhittestsource::XRHitTestSource;
use crate::dom::xrjointpose::XRJointPose;
use crate::dom::xrjointspace::XRJointSpace;
use crate::dom::xrpose::XRPose;
use crate::dom::xrreferencespace::XRReferenceSpace;
use crate::dom::xrsession::{ApiPose, XRSession};
@ -112,6 +114,38 @@ impl XRFrameMethods for XRFrame {
Ok(Some(XRPose::new(&self.global(), pose)))
}
/// https://immersive-web.github.io/webxr/#dom-xrframe-getpose
fn GetJointPose(
&self,
space: &XRJointSpace,
relative_to: &XRSpace,
) -> Result<Option<DomRoot<XRJointPose>>, Error> {
if self.session != space.upcast::<XRSpace>().session() ||
self.session != relative_to.session()
{
return Err(Error::InvalidState);
}
if !self.active.get() {
return Err(Error::InvalidState);
}
let joint_frame = if let Some(frame) = space.frame(&self.data) {
frame
} else {
return Ok(None);
};
let relative_to = if let Some(r) = self.get_pose(relative_to) {
r
} else {
return Ok(None);
};
let pose = relative_to.inverse().pre_transform(&joint_frame.pose);
Ok(Some(XRJointPose::new(
&self.global(),
pose.cast_unit(),
Some(joint_frame.radius),
)))
}
/// https://immersive-web.github.io/hit-test/#dom-xrframe-gethittestresults
fn GetHitTestResults(&self, source: &XRHitTestSource) -> Vec<DomRoot<XRHitTestResult>> {
self.data

View file

@ -0,0 +1,88 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
use crate::dom::bindings::codegen::Bindings::XRHandBinding::{XRHandConstants, XRHandMethods};
use crate::dom::bindings::reflector::{reflect_dom_object, Reflector};
use crate::dom::bindings::root::{Dom, DomRoot};
use crate::dom::globalscope::GlobalScope;
use crate::dom::xrinputsource::XRInputSource;
use crate::dom::xrjointspace::XRJointSpace;
use dom_struct::dom_struct;
use webxr_api::{FingerJoint, Hand, Joint};
#[dom_struct]
pub struct XRHand {
reflector_: Reflector,
#[ignore_malloc_size_of = "defined in webxr"]
source: Dom<XRInputSource>,
#[ignore_malloc_size_of = "partially defind in webxr"]
spaces: Hand<Dom<XRJointSpace>>,
}
impl XRHand {
fn new_inherited(source: &XRInputSource, spaces: &Hand<DomRoot<XRJointSpace>>) -> XRHand {
XRHand {
reflector_: Reflector::new(),
source: Dom::from_ref(source),
spaces: spaces.map(|j, _| j.as_ref().map(|j| Dom::from_ref(&**j))),
}
}
pub fn new(global: &GlobalScope, source: &XRInputSource, support: Hand<()>) -> DomRoot<XRHand> {
let id = source.id();
let session = source.session();
let spaces = support
.map(|field, joint| field.map(|_| XRJointSpace::new(global, session, id, joint)));
reflect_dom_object(Box::new(XRHand::new_inherited(source, &spaces)), global)
}
}
impl XRHandMethods for XRHand {
/// https://github.com/immersive-web/webxr-hands-input/blob/master/explainer.md
fn Length(&self) -> i32 {
XRHandConstants::LITTLE_PHALANX_TIP as i32 + 1
}
/// https://github.com/immersive-web/webxr-hands-input/blob/master/explainer.md
fn IndexedGetter(&self, joint_index: u32) -> Option<DomRoot<XRJointSpace>> {
let joint = match joint_index {
XRHandConstants::WRIST => Joint::Wrist,
XRHandConstants::THUMB_METACARPAL => Joint::ThumbMetacarpal,
XRHandConstants::THUMB_PHALANX_PROXIMAL => Joint::ThumbPhalanxProximal,
XRHandConstants::THUMB_PHALANX_DISTAL => Joint::ThumbPhalanxDistal,
XRHandConstants::THUMB_PHALANX_TIP => Joint::ThumbPhalanxTip,
XRHandConstants::INDEX_METACARPAL => Joint::Index(FingerJoint::Metacarpal),
XRHandConstants::INDEX_PHALANX_PROXIMAL => Joint::Index(FingerJoint::PhalanxProximal),
XRHandConstants::INDEX_PHALANX_INTERMEDIATE => {
Joint::Index(FingerJoint::PhalanxIntermediate)
},
XRHandConstants::INDEX_PHALANX_DISTAL => Joint::Index(FingerJoint::PhalanxDistal),
XRHandConstants::INDEX_PHALANX_TIP => Joint::Index(FingerJoint::PhalanxTip),
XRHandConstants::MIDDLE_METACARPAL => Joint::Middle(FingerJoint::Metacarpal),
XRHandConstants::MIDDLE_PHALANX_PROXIMAL => Joint::Middle(FingerJoint::PhalanxProximal),
XRHandConstants::MIDDLE_PHALANX_INTERMEDIATE => {
Joint::Middle(FingerJoint::PhalanxIntermediate)
},
XRHandConstants::MIDDLE_PHALANX_DISTAL => Joint::Middle(FingerJoint::PhalanxDistal),
XRHandConstants::MIDDLE_PHALANX_TIP => Joint::Middle(FingerJoint::PhalanxTip),
XRHandConstants::RING_METACARPAL => Joint::Ring(FingerJoint::Metacarpal),
XRHandConstants::RING_PHALANX_PROXIMAL => Joint::Ring(FingerJoint::PhalanxProximal),
XRHandConstants::RING_PHALANX_INTERMEDIATE => {
Joint::Ring(FingerJoint::PhalanxIntermediate)
},
XRHandConstants::RING_PHALANX_DISTAL => Joint::Ring(FingerJoint::PhalanxDistal),
XRHandConstants::RING_PHALANX_TIP => Joint::Ring(FingerJoint::PhalanxTip),
XRHandConstants::LITTLE_METACARPAL => Joint::Little(FingerJoint::Metacarpal),
XRHandConstants::LITTLE_PHALANX_PROXIMAL => Joint::Little(FingerJoint::PhalanxProximal),
XRHandConstants::LITTLE_PHALANX_INTERMEDIATE => {
Joint::Little(FingerJoint::PhalanxIntermediate)
},
XRHandConstants::LITTLE_PHALANX_DISTAL => Joint::Little(FingerJoint::PhalanxDistal),
XRHandConstants::LITTLE_PHALANX_TIP => Joint::Little(FingerJoint::PhalanxTip),
// XXXManishearth should this be a TypeError?
_ => return None,
};
self.spaces.get(joint).map(|j| DomRoot::from_ref(&**j))
}
}

View file

@ -8,6 +8,7 @@ use crate::dom::bindings::codegen::Bindings::XRInputSourceBinding::{
use crate::dom::bindings::reflector::{reflect_dom_object, DomObject, Reflector};
use crate::dom::bindings::root::{Dom, DomRoot, MutNullableDom};
use crate::dom::globalscope::GlobalScope;
use crate::dom::xrhand::XRHand;
use crate::dom::xrsession::XRSession;
use crate::dom::xrspace::XRSpace;
use crate::realms::enter_realm;
@ -24,10 +25,9 @@ pub struct XRInputSource {
session: Dom<XRSession>,
#[ignore_malloc_size_of = "Defined in rust-webxr"]
info: InputSource,
#[ignore_malloc_size_of = "Defined in rust-webxr"]
target_ray_space: MutNullableDom<XRSpace>,
#[ignore_malloc_size_of = "Defined in rust-webxr"]
grip_space: MutNullableDom<XRSpace>,
hand: MutNullableDom<XRHand>,
#[ignore_malloc_size_of = "mozjs"]
profiles: Heap<JSVal>,
}
@ -40,6 +40,7 @@ impl XRInputSource {
info,
target_ray_space: Default::default(),
grip_space: Default::default(),
hand: Default::default(),
profiles: Heap::default(),
}
}
@ -68,6 +69,10 @@ impl XRInputSource {
pub fn id(&self) -> InputId {
self.info.id
}
pub fn session(&self) -> &XRSession {
&self.session
}
}
impl XRInputSourceMethods for XRInputSource {
@ -112,4 +117,16 @@ impl XRInputSourceMethods for XRInputSource {
fn Profiles(&self, _cx: JSContext) -> JSVal {
self.profiles.get()
}
// https://github.com/immersive-web/webxr-hands-input/blob/master/explainer.md
fn GetHand(&self) -> Option<DomRoot<XRHand>> {
if let Some(ref hand) = self.info.hand_support {
Some(
self.hand
.or_init(|| XRHand::new(&self.global(), &self, hand.clone())),
)
} else {
None
}
}
}

View file

@ -0,0 +1,48 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
use crate::dom::bindings::codegen::Bindings::XRJointPoseBinding::XRJointPoseMethods;
use crate::dom::bindings::num::Finite;
use crate::dom::bindings::reflector::reflect_dom_object;
use crate::dom::bindings::root::DomRoot;
use crate::dom::globalscope::GlobalScope;
use crate::dom::xrpose::XRPose;
use crate::dom::xrrigidtransform::XRRigidTransform;
use crate::dom::xrsession::ApiRigidTransform;
use dom_struct::dom_struct;
#[dom_struct]
pub struct XRJointPose {
pose: XRPose,
radius: Option<f32>,
}
impl XRJointPose {
fn new_inherited(transform: &XRRigidTransform, radius: Option<f32>) -> XRJointPose {
XRJointPose {
pose: XRPose::new_inherited(transform),
radius,
}
}
#[allow(unsafe_code)]
pub fn new(
global: &GlobalScope,
pose: ApiRigidTransform,
radius: Option<f32>,
) -> DomRoot<XRJointPose> {
let transform = XRRigidTransform::new(global, pose);
reflect_dom_object(
Box::new(XRJointPose::new_inherited(&transform, radius)),
global,
)
}
}
impl XRJointPoseMethods for XRJointPose {
/// https://immersive-web.github.io/webxr/#dom-XRJointPose-views
fn GetRadius(&self) -> Option<Finite<f32>> {
self.radius.map(Finite::wrap)
}
}

View file

@ -0,0 +1,60 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
use crate::dom::bindings::reflector::reflect_dom_object;
use crate::dom::bindings::root::DomRoot;
use crate::dom::globalscope::GlobalScope;
use crate::dom::xrsession::{ApiPose, XRSession};
use crate::dom::xrspace::XRSpace;
use dom_struct::dom_struct;
use euclid::RigidTransform3D;
use webxr_api::{BaseSpace, Frame, InputId, Joint, JointFrame, Space};
#[dom_struct]
pub struct XRJointSpace {
xrspace: XRSpace,
#[ignore_malloc_size_of = "defined in rust-webxr"]
input: InputId,
#[ignore_malloc_size_of = "defined in rust-webxr"]
joint: Joint,
}
impl XRJointSpace {
pub fn new_inherited(session: &XRSession, input: InputId, joint: Joint) -> XRJointSpace {
XRJointSpace {
xrspace: XRSpace::new_inherited(session),
input,
joint,
}
}
#[allow(unused)]
pub fn new(
global: &GlobalScope,
session: &XRSession,
input: InputId,
joint: Joint,
) -> DomRoot<XRJointSpace> {
reflect_dom_object(Box::new(Self::new_inherited(session, input, joint)), global)
}
pub fn space(&self) -> Space {
let base = BaseSpace::Joint(self.input, self.joint);
let offset = RigidTransform3D::identity();
Space { base, offset }
}
pub fn frame<'a>(&self, frame: &'a Frame) -> Option<&'a JointFrame> {
frame
.inputs
.iter()
.find(|i| i.id == self.input)
.and_then(|i| i.hand.as_ref())
.and_then(|h| h.get(self.joint))
}
pub fn get_pose(&self, frame: &Frame) -> Option<ApiPose> {
self.frame(frame).map(|f| f.pose).map(|t| t.cast_unit())
}
}

View file

@ -8,6 +8,7 @@ use crate::dom::bindings::root::{Dom, DomRoot, MutNullableDom};
use crate::dom::eventtarget::EventTarget;
use crate::dom::globalscope::GlobalScope;
use crate::dom::xrinputsource::XRInputSource;
use crate::dom::xrjointspace::XRJointSpace;
use crate::dom::xrreferencespace::XRReferenceSpace;
use crate::dom::xrsession::{cast_transform, ApiPose, XRSession};
use dom_struct::dom_struct;
@ -61,6 +62,8 @@ impl XRSpace {
pub fn space(&self) -> Space {
if let Some(rs) = self.downcast::<XRReferenceSpace>() {
rs.space()
} else if let Some(j) = self.downcast::<XRJointSpace>() {
j.space()
} else if let Some(source) = self.input_source.get() {
let base = if self.is_grip_space {
BaseSpace::Grip(source.id())
@ -86,6 +89,8 @@ impl XRSpace {
pub fn get_pose(&self, base_pose: &Frame) -> Option<ApiPose> {
if let Some(reference) = self.downcast::<XRReferenceSpace>() {
reference.get_pose(base_pose)
} else if let Some(joint) = self.downcast::<XRJointSpace>() {
joint.get_pose(base_pose)
} else if let Some(source) = self.input_source.get() {
// XXXManishearth we should be able to request frame information
// for inputs when necessary instead of always loading it

View file

@ -102,6 +102,7 @@ WEBIDL_STANDARDS = [
b"//webaudio.github.io",
b"//immersive-web.github.io/",
b"//github.com/immersive-web/webxr-test-api/",
b"//github.com/immersive-web/webxr-hands-input/",
b"//gpuweb.github.io",
# Not a URL
b"// This interface is entirely internal to Servo, and should not be" +

View file

@ -34,6 +34,7 @@
"dom.webvtt.enabled": false,
"dom.webxr.enabled": true,
"dom.webxr.glwindow": true,
"dom.webxr.hands.enabled": false,
"dom.webxr.layers.enabled": false,
"dom.webxr.test": false,
"dom.worklet.timeout_ms": 10,