Use a laser pointer rather than the trackpad to control web content.

ports/libmlservo/src/lib.rs

@@ -190,7 +190,7 @@ pub unsafe extern "C" fn heartbeat_servo(servo: *mut ServoInstance) {
 
 // How far does the cursor have to move for it to count as a drag rather than a click?
 // (In device pixels squared, to avoid taking a sqrt when calculating move distance.)
-const DRAG_CUTOFF_SQUARED: f32 = 100.0;
+const DRAG_CUTOFF_SQUARED: f32 = 900.0;
 
 // How much should we scale scrolling by?
 const SCROLL_SCALE: f32 = 3.0;

support/magicleap/Servo2D/Servo2D.mlproject

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="ASCII"?>
-<mlproject:mlproject xmlns:mlproject="http://www.magicleap.com/uidesigner/mlproject" generated="false">
+<mlproject:mlproject xmlns:mlproject="http://www.magicleap.com/uidesigner/mlproject" generated="true">
   <designFile path="scenes/Servo2D.design"/>
   <pipelineDirectory path="pipeline"/>
 </mlproject:mlproject>

support/magicleap/Servo2D/code/inc.gen/scenesGen.h

@@ -22,10 +22,12 @@
 #include <SceneDescriptor.h>
 
 namespace Servo2D_exportedNodes {
+  extern const std::string contentPanel;
   extern const std::string content;
   extern const std::string backButton;
   extern const std::string fwdButton;
   extern const std::string urlBar;
+  extern const std::string laser;
 }
 
 namespace scenes {

support/magicleap/Servo2D/code/inc/Servo2D.h

@@ -2,16 +2,20 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+#include <GLES/gl.h>
 #include <lumin/LandscapeApp.h>
 #include <lumin/Prism.h>
 #include <lumin/event/ServerEvent.h>
 #include <lumin/event/GestureInputEventData.h>
 #include <lumin/event/KeyInputEventData.h>
 #include <lumin/event/ControlTouchPadInputEventData.h>
+#include <lumin/event/ControlPose6DofInputEventData.h>
+#include <lumin/node/LineNode.h>
 #include <lumin/node/QuadNode.h>
 #include <lumin/resource/PlanarResource.h>
 #include <lumin/ui/KeyboardDefines.h>
 #include <lumin/ui/node/UiButton.h>
+#include <lumin/ui/node/UiPanel.h>
 #include <lumin/ui/node/UiTextEdit.h>
 #include <SceneDescriptor.h>
 
@@ -100,22 +104,32 @@ protected:
    * Handle events from the server
    */
   virtual bool eventListener(lumin::ServerEvent* event) override;
-  bool touchpadEventListener(lumin::ControlTouchPadInputEventData* event);
+  bool pose6DofEventListener(lumin::ControlPose6DofInputEventData* event);
   void urlBarEventListener();
   bool gestureEventListener(lumin::GestureInputEventData* event);
 
   /**
-   * Get the current cursor position, with respect to the viewport.
+   * Convert a point in prism coordinates to viewport coordinates
+   * (ignoring the z value).
    */
-  glm::vec2 viewportCursorPosition();
+  glm::vec2 viewportPosition(glm::vec3 prism_pos);
   bool pointInsideViewport(glm::vec2 pt);
 
+  /**
+   * Redraw the laser. Returns the laser endpoint, in viewport coordinates.
+   */
+  glm::vec2 redrawLaser();
+
 private:
   lumin::Prism* prism_ = nullptr;  // represents the bounded space where the App renders.
   lumin::PlanarResource* plane_ = nullptr; // the plane we're rendering into
   lumin::QuadNode* content_node_ = nullptr; // the node containing the plane
+  lumin::ui::UiPanel* content_panel_ = nullptr; // the panel containing the node
   lumin::ui::UiButton* back_button_ = nullptr; // the back button
   lumin::ui::UiButton* fwd_button_ = nullptr; // the forward button
   lumin::ui::UiTextEdit* url_bar_ = nullptr; // the URL bar
+  lumin::LineNode* laser_ = nullptr; // The laser pointer
+  glm::vec3 controller_position_; // The last recorded position of the controller (in world coords)
+  glm::quat controller_orientation_; // The last recorded orientation of the controller (in world coords)
   ServoInstance* servo_ = nullptr; // the servo instance we're embedding
 };

support/magicleap/Servo2D/code/src.gen/scenesGen.cpp

@@ -20,19 +20,23 @@
 #include <scenesGen.h>
 
 namespace Servo2D_exportedNodes {
+  const std::string contentPanel = "contentPanel";
   const std::string content = "content";
   const std::string backButton = "backButton";
   const std::string fwdButton = "fwdButton";
   const std::string urlBar = "urlBar";
+  const std::string laser = "laser";
 }
 
 namespace scenes {
 
   const SceneDescriptor::ExportedNodeReferences Servo2D_exportedNodesMap = {
+    {"contentPanel", Servo2D_exportedNodes::contentPanel},
     {"content", Servo2D_exportedNodes::content},
     {"backButton", Servo2D_exportedNodes::backButton},
     {"fwdButton", Servo2D_exportedNodes::fwdButton},
-    {"urlBar", Servo2D_exportedNodes::urlBar}
+    {"urlBar", Servo2D_exportedNodes::urlBar},
+    {"laser", Servo2D_exportedNodes::laser}
   };
 
   const SceneDescriptor Servo2D(

support/magicleap/Servo2D/code/src/Servo2D.cpp

@@ -22,9 +22,12 @@ const int VIEWPORT_H = 500;
 const float HIDPI = 1.0;
 
 // The prism dimensions (in m).
-const float PRISM_W = 0.5;
+const float PRISM_W = 2.0;
-const float PRISM_H = 0.5;
+const float PRISM_H = 2.0;
-const float PRISM_D = 0.5;
+const float PRISM_D = 2.0;
+
+// The length of the laser pointer (in m).
+const float LASER_LENGTH = 10.0;
 
 // A function which calls the ML logger, suitable for passing into Servo
 typedef void (*MLLogger)(MLLogLevel lvl, char* msg);
@@ -103,6 +106,14 @@ int Servo2D::init() {
   }
   content_node_->setTriggerable(true);
 
+  content_panel_ = lumin::ui::UiPanel::CastFrom(prism_->findNode(Servo2D_exportedNodes::contentPanel, root_node));
+  if (!content_panel_) {
+    ML_LOG(Error, "Servo2D Failed to get content panel");
+    abort();
+    return 1;
+  }
+  lumin::ui::UiPanel::RequestFocus(content_panel_);
+
   lumin::ResourceIDType plane_id = prism_->createPlanarEGLResourceId();
   if (!plane_id) {
     ML_LOG(Error, "Servo2D Failed to create EGL resource");
@@ -161,6 +172,15 @@ int Servo2D::init() {
     return 1;
   }
   url_bar_->onFocusLostSub(std::bind(&Servo2D::urlBarEventListener, this));
+
+  // Add the laser pointer
+  laser_ = lumin::LineNode::CastFrom(prism_->findNode(Servo2D_exportedNodes::laser, root_node));
+  if (!laser_) {
+    ML_LOG(Error, "Servo2D Failed to get laser");
+    abort();
+    return 1;
+  }
+
   return 0;
 }
 
@@ -209,28 +229,28 @@ void Servo2D::instanceInitialScenes() {
 }
 
 bool Servo2D::updateLoop(float fDelta) {
-  // Hook into servo
+  glm::vec2 pos = redrawLaser();
+  move_servo(servo_, pos.x, pos.y);
   heartbeat_servo(servo_);
-
-  // Return true for your app to continue running, false to terminate the app.
   return true;
 }
 
 bool Servo2D::eventListener(lumin::ServerEvent* event) {
   // Dispatch based on event type
-  switch (event->getServerEventType()) {
+  lumin::ServerEventType typ = event->getServerEventType();
-    case lumin::ServerEventType::kControlTouchPadInputEvent:
+  switch (typ) {
-      return touchpadEventListener(static_cast<lumin::ControlTouchPadInputEventData*>(event));
     case lumin::ServerEventType::kGestureInputEvent:
       return gestureEventListener(static_cast<lumin::GestureInputEventData*>(event));
+    case lumin::ServerEventType::kControlPose6DofInputEvent:
+      return pose6DofEventListener(static_cast<lumin::ControlPose6DofInputEventData*>(event));
     default:
       return false;
   }
 }
 
-glm::vec2 Servo2D::viewportCursorPosition() {
+glm::vec2 Servo2D::viewportPosition(glm::vec3 prism_pos) {
   // Get the cursor position relative to the origin of the content node (in m)
-  glm::vec3 pos = lumin::ui::Cursor::GetPosition(prism_) - content_node_->getPrismPosition();
+  glm::vec3 pos = prism_pos - content_node_->getPrismPosition();
 
   // Get the size of the content node (in m)
   glm::vec2 sz = content_node_->getSize();
@@ -246,21 +266,56 @@ bool Servo2D::pointInsideViewport(glm::vec2 pt) {
    return (0 <= pt.x && 0 <= pt.y && pt.x <= VIEWPORT_W && pt.y <= VIEWPORT_H);
 }
 
-bool Servo2D::touchpadEventListener(lumin::ControlTouchPadInputEventData* event) {
+bool Servo2D::pose6DofEventListener(lumin::ControlPose6DofInputEventData* event) {
-  // Only respond when the cursor is enabled
+  // Get the controller position in world coordinates
-  if (!lumin::ui::Cursor::IsEnabled(prism_)) {
+  event->get6DofPosition(controller_position_.x, controller_position_.y, controller_position_.z);
+  // Get the controller orientation
+  event->getQuaternion(controller_orientation_.w, controller_orientation_.x,
+                       controller_orientation_.y, controller_orientation_.z);
+  // Bubble up to any other 6DOF handlers
   return false;
 }
 
-  // Only respond when the cursor is inside the viewport
+glm::vec2 Servo2D::redrawLaser() {
-  glm::vec2 pos = viewportCursorPosition();
+  // Return (-1, -1) if the laser doesn't intersect z=0
-  if (!pointInsideViewport(pos)) {
+  glm::vec2 result = glm::vec2(-1.0, -1.0);
-    return false;
+
+  // Convert to prism coordinates
+  glm::vec3 position = glm::inverse(prism_->getTransform()) * glm::vec4(controller_position_, 1.0f);
+  glm::quat orientation = glm::inverse(prism_->getRotation()) * controller_orientation_;
+
+  // 1m in the direction of the controller
+  glm::vec3 direction = orientation * glm::vec3(0.0f, 0.0f, -1.0f);
+  // The endpoint of the laser, in prism coordinates
+  glm::vec3 endpoint = position + direction * LASER_LENGTH;
+
+  // The laser color
+  glm::vec4 color = glm::vec4(0.0, 0.0, 0.0, 0.0);
+
+  // Check to see if the cursor is over the content
+  glm::vec2 cursor = viewportPosition(lumin::ui::Cursor::GetPosition(prism_));
+
+  // Is the laser active and does the laser intersect z=0?
+  if (pointInsideViewport(cursor) && ((position.z < 0) ^ (endpoint.z < 0))) {
+    // How far along the laser did it intersect?
+    float ratio = 1.0 / (1.0 - (endpoint.z / position.z));
+    // The intersection point
+    glm::vec3 intersection = ((1 - ratio) * position) + (ratio * endpoint);
+    // Is the intersection inside the viewport?
+    result = viewportPosition(intersection);
+    if (pointInsideViewport(result)) {
+      color = glm::vec4(0.0, 1.0, 0.0, 1.0);
+      endpoint = intersection;
+    } else {
+      color = glm::vec4(1.0, 0.0, 0.0, 1.0);
+    }
   }
 
-  // Inform Servo of the move
+  laser_->clearPoints();
-  move_servo(servo_, pos.x, pos.y);
+  laser_->addPoints(position);
-  return true;
+  laser_->addPoints(endpoint);
+  laser_->setColor(color);
+  return result;
 }
 
 bool Servo2D::gestureEventListener(lumin::GestureInputEventData* event) {
@@ -276,12 +331,13 @@ bool Servo2D::gestureEventListener(lumin::GestureInputEventData* event) {
   }
 
   // Only respond when the cursor is inside the viewport
-  glm::vec2 pos = viewportCursorPosition();
+  glm::vec2 cursor = viewportPosition(lumin::ui::Cursor::GetPosition(prism_));
-  if (!pointInsideViewport(pos)) {
+  if (!pointInsideViewport(cursor)) {
     return false;
   }
 
   // Inform Servo of the trigger
+  glm::vec2 pos = redrawLaser();
   trigger_servo(servo_, pos.x, pos.y, typ == lumin::input::GestureType::TriggerDown);
   return true;
 }

support/magicleap/Servo2D/manifest.xml

@@ -16,5 +16,6 @@
         ml:portal_folder="Icon/Portal/" />
     </component>
     <uses-privilege ml:name="Internet"/>
+    <uses-privilege ml:name="ControllerPose"/>
   </application>
 </manifest>

support/magicleap/Servo2D/scenes/Servo2D.design

@@ -1,23 +1,39 @@
 <?xml version="1.0" encoding="ASCII"?>
-<design:rootNode xmlns:design="http://www.magicleap.com/uidesigner/rcp/document/design" name="root" nodeTypeId="lumin.root" modelId="lumin" version="1.7.2">
+<design:rootNode xmlns:design="http://www.magicleap.com/uidesigner/rcp/document/design" name="root" nodeTypeId="lumin.root" modelId="lumin" version="1.8.0">
   <property id="name" value="root"/>
-  <node name="content" nodeTypeId="lumin.quad">
+  <node name="contentPanel" nodeTypeId="lumin.ui.panel">
-    <property id="color">
+    <property id="cursorInitialPosition"/>
-      <property id="rgb" value="1 1 1"/>
+    <property id="edgeConstraint"/>
-      <property id="alpha" value="1.0"/>
+    <property id="externalName" value="contentPanel"/>
+    <property id="gravityWellProperties">
+      <property id="boundaryShape">
+        <property id="size"/>
+        <property id="offset"/>
       </property>
+    </property>
+    <property id="name" value="contentPanel"/>
+    <property id="panelShape">
+      <property id="size">
+        <property id="x" value="0.5"/>
+        <property id="y" value="0.44"/>
+      </property>
+      <property id="offset"/>
+    </property>
+    <property id="position">
+      <property id="y" value="0.06"/>
+    </property>
+    <property id="rotation"/>
+    <property id="scale"/>
+    <node name="content" nodeTypeId="lumin.quad">
       <property id="externalName" value="content"/>
       <property id="name" value="content"/>
       <property id="position">
         <property id="x" value="-0.25"/>
-      <property id="y" value="-0.19"/>
+        <property id="y" value="-0.22"/>
+        <property id="z" value="-0.0"/>
       </property>
       <property id="rotation"/>
-    <property id="scale">
+      <property id="scale"/>
-      <property id="x" value="1.0"/>
-      <property id="y" value="1.0"/>
-      <property id="z" value="1.0"/>
-    </property>
       <property id="size">
         <property id="x" value="0.5"/>
         <property id="y" value="0.44"/>
@@ -36,6 +52,7 @@
         <property id="w"/>
       </property>
     </node>
+  </node>
   <node name="uiLinearLayout1" nodeTypeId="lumin.ui.linearLayout">
     <property id="alignment">
       <property id="horizontalAlignment" value="Center"/>
@@ -61,11 +78,7 @@
       <property id="y" value="-0.2"/>
     </property>
     <property id="rotation"/>
-    <property id="scale">
+    <property id="scale"/>
-      <property id="x" value="1.0"/>
-      <property id="y" value="1.0"/>
-      <property id="z" value="1.0"/>
-    </property>
     <property id="size">
       <property id="x" value="0.5"/>
       <property id="y" value="0.05"/>
@@ -79,26 +92,14 @@
         </property>
       </property>
       <property id="height" value="0.1"/>
-      <property id="iconColor">
-        <property id="rgb" value="1 1 1"/>
-        <property id="alpha" value="1.0"/>
-      </property>
       <property id="iconSize"/>
       <property id="name" value="backButton"/>
       <property id="position">
         <property id="y" value="-0.6"/>
       </property>
       <property id="rotation"/>
-      <property id="scale">
+      <property id="scale"/>
-        <property id="x" value="1.0"/>
-        <property id="y" value="1.0"/>
-        <property id="z" value="1.0"/>
-      </property>
       <property id="text" value="Back"/>
-      <property id="textColor">
-        <property id="rgb" value="1 1 1"/>
-        <property id="alpha" value="1.0"/>
-      </property>
       <property id="textSize" value="0.05"/>
       <property id="textSizeChanged" value="true"/>
       <property id="width" value="0.1"/>
@@ -112,24 +113,12 @@
         </property>
       </property>
       <property id="height" value="0.1"/>
-      <property id="iconColor">
-        <property id="rgb" value="1 1 1"/>
-        <property id="alpha" value="1.0"/>
-      </property>
       <property id="iconSize"/>
       <property id="name" value="fwdButton"/>
       <property id="position"/>
       <property id="rotation"/>
-      <property id="scale">
+      <property id="scale"/>
-        <property id="x" value="1.0"/>
-        <property id="y" value="1.0"/>
-        <property id="z" value="1.0"/>
-      </property>
       <property id="text" value="Fwd"/>
-      <property id="textColor">
-        <property id="rgb" value="1 1 1"/>
-        <property id="alpha" value="1.0"/>
-      </property>
       <property id="textSize" value="0.05"/>
       <property id="textSizeChanged" value="true"/>
       <property id="width" value="0.1"/>
@@ -146,24 +135,12 @@
         </property>
       </property>
       <property id="height" value="0.05"/>
-      <property id="hintTextColor">
-        <property id="rgb" value="1 1 1"/>
-        <property id="alpha" value="1.0"/>
-      </property>
       <property id="name" value="urlBar"/>
       <property id="position"/>
       <property id="rotation"/>
-      <property id="scale">
+      <property id="scale"/>
-        <property id="x" value="1.0"/>
-        <property id="y" value="1.0"/>
-        <property id="z" value="1.0"/>
-      </property>
       <property id="scrollBarVisibilityMode" value="Off"/>
       <property id="text" value="https://servo.org/"/>
-      <property id="textColor">
-        <property id="rgb" value="1 1 1"/>
-        <property id="alpha" value="1.0"/>
-      </property>
       <property id="textPadding">
         <property id="top" value="0.003"/>
         <property id="right" value="0.003"/>
@@ -174,4 +151,21 @@
       <property id="width" value="0.6"/>
     </node>
   </node>
+  <node name="laser" nodeTypeId="lumin.line">
+    <property id="color" value="0 0 0 0"/>
+    <property id="externalName" value="laser"/>
+    <property id="name" value="laser"/>
+    <property id="opaque" value="false"/>
+    <property id="points">
+      <property id="0"/>
+      <property id="1">
+        <property id="x" value="1.0"/>
+        <property id="y" value="1.0"/>
+        <property id="z" value="1.0"/>
+      </property>
+    </property>
+    <property id="position"/>
+    <property id="rotation"/>
+    <property id="scale"/>
+  </node>
 </design:rootNode>

support/magicleap/Servo2D/scenes/Servo2D.scene.xml

@@ -1,6 +1,5 @@
 <ObjectModel name="Servo2D" version="1">
     <TransformNode/>
-    <QuadNode castShadow="false" name="content" pos="-0.250000, -0.190000, 0.000000" receiveShadow="false" shader="UnlitColorTex2d" size="0.500000, 0.440000"/>
     <UiLinearLayout alignment="Top, Center" gravityWellEnabled="false" gravityWellRoundness="0.000000" gravityWellSize="0.000000, 0.000000" gravityWellSnap="ClosestEdge" itemAlignment="Center, Left" itemPadding="0.000000, 0.010000, 0.000000, 0.010000" name="uiLinearLayout1" orientation="Horizontal" pos="0.000000, -0.200000, 0.000000" size="0.500000, 0.050000">
         <Content>
             <UiButton gravityWellEnabled="false" gravityWellRoundness="0.000000" gravityWellSize="0.000000, 0.000000" gravityWellSnap="ClosestEdge" name="backButton" pos="0.000000, -0.600000, 0.000000" size="0.100000, 0.100000" text="Back" textSize="0.050000"/>
@@ -9,7 +8,11 @@
             <UiButton gravityWellEnabled="false" gravityWellRoundness="0.000000" gravityWellSize="0.000000, 0.000000" gravityWellSnap="ClosestEdge" name="fwdButton" size="0.100000, 0.100000" text="Fwd" textSize="0.050000"/>
         </Content>
         <Content>
-            <UiTextEdit alignment="Center, Left" font="" gravityWellEnabled="false" gravityWellRoundness="0.000000" gravityWellSize="0.000000, 0.000000" gravityWellSnap="ClosestEdge" name="urlBar" scrollSpeed="0.500000" size="0.600000, 0.050000" text="https://servo.org/" textSize="0.050000"/>
+            <UiTextEdit alignment="Center, Left" gravityWellEnabled="false" gravityWellRoundness="0.000000" gravityWellSize="0.000000, 0.000000" gravityWellSnap="ClosestEdge" name="urlBar" scrollSpeed="0.500000" size="0.600000, 0.050000" text="https://servo.org/" textSize="0.050000"/>
         </Content>
     </UiLinearLayout>
+    <LineNode castShadow="false" color="0.000000, 0.000000, 0.000000, 0.000000" name="laser" opaque="false" points="0.000000, 0.000000, 0.000000, 1.000000, 1.000000, 1.000000" receiveShadow="false" shader="Line"/>
+    <UiPanel gravityWellEnabled="false" gravityWellRoundness="0.000000" gravityWellSize="0.000000, 0.000000" gravityWellSnap="ClosestEdge" name="contentPanel" pos="0.000000, 0.060000, 0.000000" shape="[size:[0.5,0.44], roundness: 0, offset[0,0,0]]">
+        <QuadNode castShadow="false" name="content" pos="-0.250000, -0.220000, -0.000000" receiveShadow="false" shader="UnlitColorTex2d" size="0.500000, 0.440000"/>
+    </UiPanel>
 </ObjectModel>