Auto merge of #8975 - mbrubeck:touch-refactor, r=glennw

Add pinch zoom gesture for touch-screen devices

r? @glennw

Notes:
* Our layer system doesn't yet support zooming `<iframe mozbrowser>` separately, so this can only zoom the root window.
* There are some issues that cause some tiles to disappear while zooming. I hope this will be fixed by WebRender.
* Sorry this is mostly in one giant commit. I did the refactoring and adding new functionality in tandem, and didn't really end up with logical or compilable intermediate commits. Hopefully the new code is easier to read than the old stuff.

<!-- Reviewable:start -->
[<img src="https://reviewable.io/review_button.png" height=40 alt="Review on Reviewable"/>](https://reviewable.io/reviews/servo/servo/8975)
<!-- Reviewable:end -->

components/compositing/compositor.rs

@@ -27,7 +27,7 @@ use layers::rendergl;
 use layers::rendergl::RenderContext;
 use layers::scene::Scene;
 use layout_traits::LayoutControlChan;
-use msg::compositor_msg::{Epoch, EventResult, FrameTreeId, LayerId, LayerKind};
+use msg::compositor_msg::{Epoch, FrameTreeId, LayerId, LayerKind};
 use msg::compositor_msg::{LayerProperties, ScrollPolicy};
 use msg::constellation_msg::{AnimationState, Image, PixelFormat};
 use msg::constellation_msg::{Key, KeyModifiers, KeyState, LoadData, MouseButton};
@@ -48,6 +48,7 @@ use std::sync::mpsc::Sender;
 use style_traits::viewport::ViewportConstraints;
 use surface_map::SurfaceMap;
 use time::{precise_time_ns, precise_time_s};
+use touch::{TouchHandler, TouchAction};
 use url::Url;
 use util::geometry::{PagePx, ScreenPx, ViewportPx};
 use util::opts;
@@ -78,9 +79,6 @@ const BUFFER_MAP_SIZE: usize = 10000000;
 const MAX_ZOOM: f32 = 8.0;
 const MIN_ZOOM: f32 = 0.1;
 
-/// Minimum number of ScreenPx to begin touch scrolling.
-const TOUCH_PAN_MIN_SCREEN_PX: f32 = 20.0;
-
 /// Holds the state when running reftests that determines when it is
 /// safe to save the output image.
 #[derive(Copy, Clone, PartialEq)]
@@ -179,10 +177,10 @@ pub struct IOCompositor<Window: WindowMethods> {
     fragment_point: Option<Point2D<f32>>,
 
     /// Touch input state machine
-    touch_gesture_state: TouchState,
+    touch_handler: TouchHandler,
 
-    /// Pending scroll events.
+    /// Pending scroll/zoom events.
-    pending_scroll_events: Vec<ScrollEvent>,
+    pending_scroll_zoom_events: Vec<ScrollZoomEvent>,
 
     /// Used by the logic that determines when it is safe to output an
     /// image for the reftest framework.
@@ -199,30 +197,12 @@ pub struct IOCompositor<Window: WindowMethods> {
     last_mouse_move_recipient: Option<PipelineId>,
 }
 
-/// The states of the touch input state machine.
+pub struct ScrollZoomEvent {
-///
+    /// Change the pinch zoom level by this factor
-/// TODO: Currently Add support for "flinging" (scrolling inertia), pinch zooming.
+    magnification: f32,
-enum TouchState {
+    /// Scroll by this offset
-    /// Not tracking any touch point
-    Nothing,
-    /// A touchstart event was dispatched to the page, but the response wasn't received yet.
-    /// Contains the number of active touch points and the initial touch location.
-    WaitingForScript(u32, TypedPoint2D<DevicePixel, f32>),
-    /// Script is consuming the current touch sequence; don't perform default actions.
-    /// Contains the number of active touch points.
-    DefaultPrevented(u32),
-    /// A single touch point is active and may perform click or pan default actions.
-    /// Contains the initial touch location.
-    Touching(TypedPoint2D<DevicePixel, f32>),
-    /// A single touch point is active and has started panning. Contains the latest touch location.
-    Panning(TypedPoint2D<DevicePixel, f32>),
-    /// A multi-touch gesture is in progress. Contains the number of active touch points.
-    MultiTouch(u32),
-}
-
-
-pub struct ScrollEvent {
     delta: TypedPoint2D<DevicePixel, f32>,
+    /// Apply changes to the frame at this location
     cursor: TypedPoint2D<DevicePixel, i32>,
 }
 
@@ -347,8 +327,8 @@ impl<Window: WindowMethods> IOCompositor<Window> {
             channel_to_self: state.sender.clone_compositor_proxy(),
             scrolling_timer: ScrollingTimerProxy::new(state.sender),
             composition_request: CompositionRequest::NoCompositingNecessary,
-            touch_gesture_state: TouchState::Nothing,
+            touch_handler: TouchHandler::new(),
-            pending_scroll_events: Vec::new(),
+            pending_scroll_zoom_events: Vec::new(),
             composite_target: composite_target,
             shutdown_state: ShutdownState::NotShuttingDown,
             page_zoom: ScaleFactor::new(1.0),
@@ -556,19 +536,7 @@ impl<Window: WindowMethods> IOCompositor<Window> {
             }
 
             (Msg::TouchEventProcessed(result), ShutdownState::NotShuttingDown) => {
-                match self.touch_gesture_state {
+                self.touch_handler.on_event_processed(result);
-                    TouchState::WaitingForScript(n, p) => {
-                        self.touch_gesture_state = match result {
-                            EventResult::DefaultPrevented => TouchState::DefaultPrevented(n),
-                            EventResult::DefaultAllowed => if n > 1 {
-                                TouchState::MultiTouch(n)
-                            } else {
-                                TouchState::Touching(p)
-                            }
-                        };
-                    }
-                    _ => {}
-                }
             }
 
             (Msg::SetCursor(cursor), ShutdownState::NotShuttingDown) => {
@@ -1220,25 +1188,7 @@ impl<Window: WindowMethods> IOCompositor<Window> {
     }
 
     fn on_touch_down(&mut self, identifier: TouchId, point: TypedPoint2D<DevicePixel, f32>) {
-        self.touch_gesture_state = match self.touch_gesture_state {
+        self.touch_handler.on_touch_down(identifier, point);
-            TouchState::Nothing => {
-                TouchState::WaitingForScript(1, point)
-            }
-            TouchState::Touching(_) |
-            TouchState::Panning(_) => {
-                // Was a single-touch sequence; now a multi-touch sequence:
-                TouchState::MultiTouch(2)
-            }
-            TouchState::WaitingForScript(n, p) => {
-                TouchState::WaitingForScript(n + 1, p)
-            }
-            TouchState::DefaultPrevented(n) => {
-                TouchState::DefaultPrevented(n + 1)
-            }
-            TouchState::MultiTouch(n) => {
-                TouchState::MultiTouch(n + 1)
-            }
-        };
         if let Some(result) = self.find_topmost_layer_at_point(point / self.scene.scale) {
             result.layer.send_event(self, TouchEvent(TouchEventType::Down, identifier,
                                                      result.point.to_untyped()));
@@ -1246,89 +1196,49 @@ impl<Window: WindowMethods> IOCompositor<Window> {
     }
 
     fn on_touch_move(&mut self, identifier: TouchId, point: TypedPoint2D<DevicePixel, f32>) {
-        match self.touch_gesture_state {
+        match self.touch_handler.on_touch_move(identifier, point) {
-            TouchState::Nothing => warn!("Got unexpected touch move event"),
+            TouchAction::Scroll(delta) => {
-
-            TouchState::WaitingForScript(..) => {
-                // TODO: Queue events while waiting for script?
-            }
-            TouchState::Touching(p0) => {
-                let delta = point - p0;
-                let px: TypedPoint2D<ScreenPx, _> = delta / self.device_pixels_per_screen_px();
-                let px = px.to_untyped();
-
-                if px.x.abs() > TOUCH_PAN_MIN_SCREEN_PX ||
-                   px.y.abs() > TOUCH_PAN_MIN_SCREEN_PX
-                {
-                    self.touch_gesture_state = TouchState::Panning(point);
-                    self.on_scroll_window_event(delta, point.cast().unwrap());
-                }
-            }
-            TouchState::Panning(p0) => {
-                let delta = point - p0;
                 self.on_scroll_window_event(delta, point.cast().unwrap());
-                self.touch_gesture_state = TouchState::Panning(point);
             }
-            TouchState::DefaultPrevented(_) => {
+            TouchAction::Zoom(magnification, scroll_delta) => {
-                // Send the event to script.
+                let cursor = Point2D::typed(-1, -1);  // Make sure this hits the base layer.
+                self.pending_scroll_zoom_events.push(ScrollZoomEvent {
+                    magnification: magnification,
+                    delta: scroll_delta,
+                    cursor: cursor,
+                });
+                self.composite_if_necessary(CompositingReason::Zoom);
+            }
+            TouchAction::DispatchEvent => {
                 if let Some(result) = self.find_topmost_layer_at_point(point / self.scene.scale) {
                     result.layer.send_event(self, TouchEvent(TouchEventType::Move, identifier,
                                                              result.point.to_untyped()));
                 }
             }
-            TouchState::MultiTouch(_) => {
+            _ => {}
-                // TODO: Pinch zooming.
-            }
         }
     }
 
     fn on_touch_up(&mut self, identifier: TouchId, point: TypedPoint2D<DevicePixel, f32>) {
-        // Send the event to script.
         if let Some(result) = self.find_topmost_layer_at_point(point / self.scene.scale) {
             result.layer.send_event(self, TouchEvent(TouchEventType::Up, identifier,
                                                      result.point.to_untyped()));
         }
-
+        match self.touch_handler.on_touch_up(identifier, point) {
-        self.touch_gesture_state = match self.touch_gesture_state {
+            TouchAction::Click => {
-            TouchState::Touching(_) => {
-                // TODO: If the duration exceeds some threshold, send a contextmenu event instead.
-                // TODO: Don't send a click if preventDefault is called on the touchend event.
                 self.simulate_mouse_click(point);
-                TouchState::Nothing
             }
-            TouchState::Nothing |
+            _ => {}
-            TouchState::Panning(_) |
+        }
-            TouchState::WaitingForScript(1, _) |
-            TouchState::DefaultPrevented(1) |
-            TouchState::MultiTouch(1) => {
-                TouchState::Nothing
-            }
-            TouchState::WaitingForScript(n, p) => TouchState::WaitingForScript(n - 1, p),
-            TouchState::DefaultPrevented(n) => TouchState::DefaultPrevented(n - 1),
-            TouchState::MultiTouch(n) => TouchState::MultiTouch(n - 1),
-        };
     }
 
     fn on_touch_cancel(&mut self, identifier: TouchId, point: TypedPoint2D<DevicePixel, f32>) {
         // Send the event to script.
+        self.touch_handler.on_touch_cancel(identifier, point);
         if let Some(result) = self.find_topmost_layer_at_point(point / self.scene.scale) {
             result.layer.send_event(self, TouchEvent(TouchEventType::Cancel, identifier,
                                                      result.point.to_untyped()));
         }
-
-        self.touch_gesture_state = match self.touch_gesture_state {
-            TouchState::Nothing |
-            TouchState::Touching(_) |
-            TouchState::Panning(_) |
-            TouchState::WaitingForScript(1, _) |
-            TouchState::DefaultPrevented(1) |
-            TouchState::MultiTouch(1) => {
-                TouchState::Nothing
-            }
-            TouchState::WaitingForScript(n, p) => TouchState::WaitingForScript(n - 1, p),
-            TouchState::DefaultPrevented(n) => TouchState::DefaultPrevented(n - 1),
-            TouchState::MultiTouch(n) => TouchState::MultiTouch(n - 1),
-        };
     }
 
     /// http://w3c.github.io/touch-events/#mouse-events
@@ -1348,7 +1258,8 @@ impl<Window: WindowMethods> IOCompositor<Window> {
     fn on_scroll_window_event(&mut self,
                               delta: TypedPoint2D<DevicePixel, f32>,
                               cursor: TypedPoint2D<DevicePixel, i32>) {
-        self.pending_scroll_events.push(ScrollEvent {
+        self.pending_scroll_zoom_events.push(ScrollZoomEvent {
+            magnification: 1.0,
             delta: delta,
             cursor: cursor,
         });
@@ -1357,20 +1268,30 @@ impl<Window: WindowMethods> IOCompositor<Window> {
     }
 
     fn process_pending_scroll_events(&mut self) {
-        let had_scroll_events = self.pending_scroll_events.len() > 0;
+        let had_events = self.pending_scroll_zoom_events.len() > 0;
-        for scroll_event in std_mem::replace(&mut self.pending_scroll_events,
+        for event in std_mem::replace(&mut self.pending_scroll_zoom_events,
                                              Vec::new()) {
-            let delta = scroll_event.delta / self.scene.scale;
+            let delta = event.delta / self.scene.scale;
-            let cursor = scroll_event.cursor.as_f32() / self.scene.scale;
+            let cursor = event.cursor.as_f32() / self.scene.scale;
 
             if let Some(ref mut layer) = self.scene.root {
                 layer.handle_scroll_event(delta, cursor);
             }
 
+            if event.magnification != 1.0 {
+                self.zoom_action = true;
+                self.zoom_time = precise_time_s();
+                self.viewport_zoom = ScaleFactor::new(
+                    (self.viewport_zoom.get() * event.magnification)
+                    .min(self.max_viewport_zoom.as_ref().map_or(MAX_ZOOM, ScaleFactor::get))
+                    .max(self.min_viewport_zoom.as_ref().map_or(MIN_ZOOM, ScaleFactor::get)));
+                self.update_zoom_transform();
+            }
+
             self.perform_updates_after_scroll();
         }
 
-        if had_scroll_events {
+        if had_events {
             self.send_viewport_rects_for_all_layers();
         }
     }
@@ -1498,36 +1419,13 @@ impl<Window: WindowMethods> IOCompositor<Window> {
         self.send_window_size();
     }
 
-    // TODO(pcwalton): I think this should go through the same queuing as scroll events do.
+    /// Simulate a pinch zoom
     fn on_pinch_zoom_window_event(&mut self, magnification: f32) {
-        use num::Float;
+        self.pending_scroll_zoom_events.push(ScrollZoomEvent {
-
+            magnification: magnification,
-        self.zoom_action = true;
+            delta: Point2D::typed(0.0, 0.0), // TODO: Scroll to keep the center in view?
-        self.zoom_time = precise_time_s();
+            cursor:  Point2D::typed(-1, -1), // Make sure this hits the base layer.
-        let old_viewport_zoom = self.viewport_zoom;
+        });
-
-        let viewport_zoom = ScaleFactor::new((self.viewport_zoom.get() * magnification)
-            .min(self.max_viewport_zoom.as_ref().map_or(MAX_ZOOM, ScaleFactor::get))
-            .max(self.min_viewport_zoom.as_ref().map_or(MIN_ZOOM, ScaleFactor::get)));
-        self.viewport_zoom = viewport_zoom;
-
-        self.update_zoom_transform();
-
-        // Scroll as needed
-        let window_size = self.window_size.as_f32();
-        let page_delta: TypedPoint2D<LayerPixel, f32> = Point2D::typed(
-            window_size.width.get() * (viewport_zoom.inv() - old_viewport_zoom.inv()).get() * 0.5,
-            window_size.height.get() * (viewport_zoom.inv() - old_viewport_zoom.inv()).get() * 0.5);
-
-        let cursor = Point2D::typed(-1f32, -1f32);  // Make sure this hits the base layer.
-        match self.scene.root {
-            Some(ref mut layer) => {
-                layer.handle_scroll_event(page_delta, cursor);
-            }
-            None => { }
-        }
-
-        self.send_viewport_rects_for_all_layers();
         self.composite_if_necessary(CompositingReason::Zoom);
     }
 

components/compositing/lib.rs

@@ -71,6 +71,7 @@ pub mod sandboxing;
 mod scrolling;
 mod surface_map;
 mod timer_scheduler;
+mod touch;
 pub mod windowing;
 
 /// Messages from the compositor to the constellation.

components/compositing/touch.rs

@@ -0,0 +1,237 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use euclid::point::TypedPoint2D;
+use euclid::scale_factor::ScaleFactor;
+use layers::geometry::DevicePixel;
+use msg::compositor_msg::EventResult;
+use script_traits::TouchId;
+use self::TouchState::*;
+
+/// Minimum number of ScreenPx to begin touch scrolling.
+const TOUCH_PAN_MIN_SCREEN_PX: f32 = 20.0;
+
+pub struct TouchHandler {
+    pub state: TouchState,
+    pub active_touch_points: Vec<TouchPoint>,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct TouchPoint {
+    pub id: TouchId,
+    pub point: TypedPoint2D<DevicePixel, f32>
+}
+
+impl TouchPoint {
+    pub fn new(id: TouchId, point: TypedPoint2D<DevicePixel, f32>) -> Self {
+        TouchPoint { id: id, point: point }
+    }
+}
+
+/// The states of the touch input state machine.
+///
+/// TODO: Add support for "flinging" (scrolling inertia)
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum TouchState {
+    /// Not tracking any touch point
+    Nothing,
+    /// A touchstart event was dispatched to the page, but the response wasn't received yet.
+    /// Contains the initial touch point.
+    WaitingForScript,
+    /// Script is consuming the current touch sequence; don't perform default actions.
+    DefaultPrevented,
+    /// A single touch point is active and may perform click or pan default actions.
+    /// Contains the initial touch location.
+    Touching,
+    /// A single touch point is active and has started panning.
+    Panning,
+    /// A two-finger pinch zoom gesture is active.
+    Pinching,
+    /// A multi-touch gesture is in progress. Contains the number of active touch points.
+    MultiTouch,
+}
+
+/// The action to take in response to a touch event
+#[derive(Clone, Copy, Debug)]
+pub enum TouchAction {
+    /// Simulate a mouse click.
+    Click,
+    /// Scroll by the provided offset.
+    Scroll(TypedPoint2D<DevicePixel, f32>),
+    /// Zoom by a magnification factor and scroll by the provided offset.
+    Zoom(f32, TypedPoint2D<DevicePixel, f32>),
+    /// Send a JavaScript event to content.
+    DispatchEvent,
+    /// Don't do anything.
+    NoAction,
+}
+
+impl TouchHandler {
+    pub fn new() -> Self {
+        TouchHandler {
+            state: Nothing,
+            active_touch_points: Vec::new(),
+        }
+    }
+
+    pub fn on_touch_down(&mut self, id: TouchId, point: TypedPoint2D<DevicePixel, f32>) {
+        let point = TouchPoint::new(id, point);
+        self.active_touch_points.push(point);
+
+        self.state = match self.state {
+            Nothing               => WaitingForScript,
+            Touching | Panning    => Pinching,
+            WaitingForScript      => WaitingForScript,
+            DefaultPrevented      => DefaultPrevented,
+            Pinching | MultiTouch => MultiTouch,
+        };
+    }
+
+    pub fn on_touch_move(&mut self, id: TouchId, point: TypedPoint2D<DevicePixel, f32>)
+                         -> TouchAction {
+        let idx = match self.active_touch_points.iter_mut().position(|t| t.id == id) {
+            Some(i) => i,
+            None => {
+                warn!("Got a touchmove event for a non-active touch point");
+                return TouchAction::NoAction;
+            }
+        };
+        let old_point = self.active_touch_points[idx].point;
+
+        let action = match self.state {
+            Touching => {
+                let delta = point - old_point;
+                // TODO let delta: TypedPoint2D<ScreenPx, _> = delta / self.device_pixels_per_screen_px();
+
+                if delta.x.get().abs() > TOUCH_PAN_MIN_SCREEN_PX ||
+                   delta.y.get().abs() > TOUCH_PAN_MIN_SCREEN_PX
+                {
+                    self.state = Panning;
+                    TouchAction::Scroll(delta)
+                } else {
+                    TouchAction::NoAction
+                }
+            }
+            Panning => {
+                let delta = point - old_point;
+                TouchAction::Scroll(delta)
+            }
+            DefaultPrevented => {
+                TouchAction::DispatchEvent
+            }
+            Pinching => {
+                let (d0, c0) = self.pinch_distance_and_center();
+                self.active_touch_points[idx].point = point;
+                let (d1, c1) = self.pinch_distance_and_center();
+
+                let magnification = d1 / d0;
+                let scroll_delta = c1 - c0 * ScaleFactor::new(magnification);
+
+                TouchAction::Zoom(magnification, scroll_delta)
+            }
+            WaitingForScript => TouchAction::NoAction,
+            MultiTouch => TouchAction::NoAction,
+            Nothing => unreachable!(),
+        };
+
+        // If we're still waiting to see whether this is a click or pan, remember the original
+        // location.  Otherwise, update the touch point with the latest location.
+        if self.state != Touching && self.state != WaitingForScript {
+            self.active_touch_points[idx].point = point;
+        }
+        action
+    }
+
+    pub fn on_touch_up(&mut self, id: TouchId, _point: TypedPoint2D<DevicePixel, f32>)
+                       -> TouchAction {
+        match self.active_touch_points.iter().position(|t| t.id == id) {
+            Some(i) => {
+                self.active_touch_points.swap_remove(i);
+            }
+            None => {
+                warn!("Got a touch up event for a non-active touch point");
+            }
+        }
+        match self.state {
+            Touching => {
+                // FIXME: If the duration exceeds some threshold, send a contextmenu event instead.
+                // FIXME: Don't send a click if preventDefault is called on the touchend event.
+                self.state = Nothing;
+                TouchAction::Click
+            }
+            Nothing | Panning => {
+                self.state = Nothing;
+                TouchAction::NoAction
+            }
+            Pinching => {
+                self.state = Panning;
+                TouchAction::NoAction
+            }
+            WaitingForScript | DefaultPrevented | MultiTouch => {
+                if self.active_touch_points.is_empty() {
+                    self.state = Nothing;
+                }
+                TouchAction::NoAction
+            }
+        }
+    }
+
+    pub fn on_touch_cancel(&mut self, id: TouchId, _point: TypedPoint2D<DevicePixel, f32>) {
+        match self.active_touch_points.iter().position(|t| t.id == id) {
+            Some(i) => {
+                self.active_touch_points.swap_remove(i);
+            }
+            None => {
+                warn!("Got a touchcancel event for a non-active touch point");
+                return;
+            }
+        }
+        match self.state {
+            Nothing => {}
+            Touching | Panning => {
+                self.state = Nothing;
+            }
+            Pinching => {
+                self.state = Panning;
+            }
+            WaitingForScript | DefaultPrevented | MultiTouch => {
+                if self.active_touch_points.is_empty() {
+                    self.state = Nothing;
+                }
+            }
+        }
+    }
+
+    pub fn on_event_processed(&mut self, result: EventResult) {
+        match self.state {
+            WaitingForScript => self.state = match result {
+                EventResult::DefaultPrevented => DefaultPrevented,
+                EventResult::DefaultAllowed => match self.touch_count() {
+                    1 => Touching,
+                    2 => Pinching,
+                    _ => MultiTouch,
+                }
+            },
+            _ => {}
+        }
+    }
+
+    fn touch_count(&self) -> usize {
+        self.active_touch_points.len()
+    }
+
+    fn pinch_distance_and_center(&self) -> (f32, TypedPoint2D<DevicePixel, f32>) {
+        debug_assert!(self.touch_count() == 2);
+        let p0 = self.active_touch_points[0].point;
+        let p1 = self.active_touch_points[1].point;
+        let center = (p0 + p1) / ScaleFactor::new(2.0);
+
+        let d = p0 - p1;
+        let dx = d.x.get();
+        let dy = d.y.get();
+        let distance = f32::sqrt(dx * dx + dy * dy);
+
+        (distance, center)
+    }
+}