Auto merge of #7179 - connorimes:add-heartbeats, r=larsbergstrom

Integrate with simple Heartbeats

This PR adds Heartbeats capability to servo.  Heartbeats are used for detailed performance and power/energy profiling.  We will add the power/energy readings in the future.

New dependencies are introduced which need in-depth reviews.  I'm the only one who has had eyes on any of this, and I have limited resources for testing cross-platform compatibility.
* https://github.com/libheartbeats/heartbeats-simple - provides native C libraries from a shared code base:
 * hbs[-static] - performance monitoring
 * hbs-acc[-static] - performance with accuracy monitoring
 * hbs-pow[-static] - performance with power/energy monitoring (the one we're using)
 * hbs-acc-pow[-static] - performance with accuracy and power/energy monitoring 
* https://github.com/connorimes/heartbeats-simple-sys provides rust wrappers for the native C libraries above - one crate for each + a common crate.  These link with the *-static versions of the heartbeats libraries.
* https://github.com/connorimes/heartbeats-simple-rust provides rust abstractions over the -sys crates above - one crate for each.

The new `heartbeats` module in the `profile` crate looks for environment variables telling it to use heartbeats for each ProfilerCategory and where to put log files.  (Of course, if somebody knows how to iterate over the enum instead of hardcoding each one, that would be fantastic.)  If the environment variables aren't set for particular categories, heartbeats aren't created or used.

An interface change is made in the `profile_traits` crate to pass both the start and end time in a `ProfilerMsg` instead of just the elapsed time.  Later we will add energy readings as well.

<!-- Reviewable:start -->
[<img src="https://reviewable.io/review_button.png" height=40 alt="Review on Reviewable"/>](https://reviewable.io/reviews/servo/servo/7179)
<!-- Reviewable:end -->

components/profile/Cargo.toml

@@ -13,6 +13,9 @@ path = "../profile_traits"
 [dependencies.util]
 path = "../util"
 
+[dependencies.hbs-pow]
+git = "https://github.com/libheartbeats/heartbeats-simple-rust.git"
+
 [dependencies.ipc-channel]
 git = "https://github.com/pcwalton/ipc-channel"
 

components/profile/heartbeats.rs

@@ -0,0 +1,120 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+
+use hbs_pow::HeartbeatPow as Heartbeat;
+use hbs_pow::HeartbeatPowContext as HeartbeatContext;
+use profile_traits::time::ProfilerCategory;
+use std::collections::HashMap;
+use std::env::var_os;
+use std::error::Error;
+use std::fs::File;
+use std::mem;
+
+
+static mut HBS: Option<*mut HashMap<ProfilerCategory, Heartbeat>> = None;
+
+/// Initialize heartbeats
+pub fn init() {
+    let mut hbs: HashMap<ProfilerCategory, Heartbeat> = HashMap::new();
+    maybe_create_heartbeat(&mut hbs, ProfilerCategory::Compositing);
+    maybe_create_heartbeat(&mut hbs, ProfilerCategory::LayoutPerform);
+    maybe_create_heartbeat(&mut hbs, ProfilerCategory::LayoutStyleRecalc);
+    maybe_create_heartbeat(&mut hbs, ProfilerCategory::LayoutRestyleDamagePropagation);
+    maybe_create_heartbeat(&mut hbs, ProfilerCategory::LayoutNonIncrementalReset);
+    maybe_create_heartbeat(&mut hbs, ProfilerCategory::LayoutSelectorMatch);
+    maybe_create_heartbeat(&mut hbs, ProfilerCategory::LayoutTreeBuilder);
+    maybe_create_heartbeat(&mut hbs, ProfilerCategory::LayoutDamagePropagate);
+    maybe_create_heartbeat(&mut hbs, ProfilerCategory::LayoutGeneratedContent);
+    maybe_create_heartbeat(&mut hbs, ProfilerCategory::LayoutMain);
+    maybe_create_heartbeat(&mut hbs, ProfilerCategory::LayoutParallelWarmup);
+    maybe_create_heartbeat(&mut hbs, ProfilerCategory::LayoutShaping);
+    maybe_create_heartbeat(&mut hbs, ProfilerCategory::LayoutDispListBuild);
+    maybe_create_heartbeat(&mut hbs, ProfilerCategory::PaintingPerTile);
+    maybe_create_heartbeat(&mut hbs, ProfilerCategory::PaintingPrepBuff);
+    maybe_create_heartbeat(&mut hbs, ProfilerCategory::Painting);
+    maybe_create_heartbeat(&mut hbs, ProfilerCategory::ImageDecoding);
+    unsafe {
+        HBS = Some(mem::transmute(Box::new(hbs)));
+    }
+}
+
+/// Log regmaining buffer data and cleanup heartbeats
+pub fn cleanup() {
+    unsafe {
+        if let Some(hbs) = HBS {
+            let mut h: Box<HashMap<ProfilerCategory, Heartbeat>> = mem::transmute(hbs);
+            for (_, mut v) in h.iter_mut() {
+                // log any remaining heartbeat records before dropping
+                log_heartbeat_records(v);
+            }
+            h.clear();
+        }
+        HBS = None;
+    }
+}
+
+/// Issue a heartbeat (if one exists) for the given category
+pub fn maybe_heartbeat(category: &ProfilerCategory,
+                       start_time: u64,
+                       end_time: u64,
+                       start_energy: u64,
+                       end_energy: u64) {
+    unsafe {
+        if let Some(map) = HBS {
+            if let Some(mut h) = (*map).get_mut(category) {
+                (*h).heartbeat(0, 1, start_time, end_time, start_energy, end_energy);
+            }
+        }
+    }
+}
+
+/// Create a heartbeat if the correct environment variable is set
+fn maybe_create_heartbeat(hbs: &mut HashMap<ProfilerCategory, Heartbeat>,
+                          category: ProfilerCategory) {
+    static WINDOW_SIZE_DEFAULT: usize = 20;
+    if let Some(_) = var_os(format!("SERVO_HEARTBEAT_ENABLE_{:?}", category)) {
+        // get optional log file
+        let logfile: Option<File> = var_os(format!("SERVO_HEARTBEAT_LOG_{:?}", category))
+                                    .and_then(|name| File::create(name).ok());
+        // get window size
+        let window_size: usize = match var_os(format!("SERVO_HEARTBEAT_WINDOW_{:?}", category)) {
+            Some(w) => match w.into_string() {
+                Ok(s) => s.parse::<usize>().unwrap_or(WINDOW_SIZE_DEFAULT),
+                _ => WINDOW_SIZE_DEFAULT,
+            },
+            None => WINDOW_SIZE_DEFAULT,
+        };
+        // create the heartbeat
+        match Heartbeat::new(window_size, Some(heartbeat_window_callback), logfile) {
+            Ok(hb) => {
+                debug!("Created heartbeat for {:?}", category);
+                hbs.insert(category, hb);
+            },
+            Err(e) => warn!("Failed to create heartbeat for {:?}: {}", category, e),
+        }
+    };
+}
+
+/// Log heartbeat records up to the buffer index
+fn log_heartbeat_records(hb: &mut Heartbeat) {
+    match hb.log_to_buffer_index() {
+        Ok(_) => (),
+        Err(e) => warn!("Failed to write heartbeat log: {}", Error::description(&e)),
+    }
+}
+
+/// Callback function used to log the window buffer.
+/// When this is called from native C, the heartbeat is safely locked
+extern fn heartbeat_window_callback(hb: *const HeartbeatContext) {
+    unsafe {
+        if let Some(map) = HBS {
+            for (_, v) in (*map).iter_mut() {
+                if &v.hb as *const HeartbeatContext == hb {
+                    log_heartbeat_records(v);
+                }
+            }
+        }
+    }
+}

components/profile/lib.rs

@@ -8,6 +8,7 @@
 
 #[macro_use] extern crate log;
 
+extern crate hbs_pow;
 extern crate ipc_channel;
 extern crate libc;
 #[macro_use]
@@ -21,3 +22,5 @@ extern crate util;
 
 pub mod mem;
 pub mod time;
+
+mod heartbeats;

components/profile/time.rs

@@ -4,6 +4,7 @@
 
 //! Timing functions.
 
+use heartbeats;
 use ipc_channel::ipc::{self, IpcReceiver};
 use profile_traits::time::{ProfilerCategory, ProfilerChan, ProfilerMsg, TimerMetadata};
 use std::borrow::ToOwned;
@@ -124,6 +125,7 @@ impl Profiler {
                 });
             }
         }
+        heartbeats::init();
 
         ProfilerChan(chan)
     }
@@ -161,13 +163,20 @@ impl Profiler {
 
     fn handle_msg(&mut self, msg: ProfilerMsg) -> bool {
         match msg.clone() {
-            ProfilerMsg::Time(k, t) => self.find_or_insert(k, t),
+            ProfilerMsg::Time(k, t) => {
+                heartbeats::maybe_heartbeat(&k.0, t.0, t.1, 0, 0);
+                let ms = (t.1 - t.0) as f64 / 1000000f64;
+                self.find_or_insert(k, ms);
+            },
             ProfilerMsg::Print => match self.last_msg {
                 // only print if more data has arrived since the last printout
                 Some(ProfilerMsg::Time(..)) => self.print_buckets(),
                 _ => ()
             },
-            ProfilerMsg::Exit => return false,
+            ProfilerMsg::Exit => {
+                heartbeats::cleanup();
+                return false;
+            },
         };
         self.last_msg = Some(msg);
         true

components/profile_traits/time.rs

@@ -29,7 +29,7 @@ impl ProfilerChan {
 #[derive(Clone, Deserialize, Serialize)]
 pub enum ProfilerMsg {
     /// Normal message used for reporting time
-    Time((ProfilerCategory, Option<TimerMetadata>), f64),
+    Time((ProfilerCategory, Option<TimerMetadata>), (u64, u64)),
     /// Message used to force print the profiling metrics
     Print,
     /// Tells the profiler to shut down.
@@ -37,7 +37,7 @@ pub enum ProfilerMsg {
 }
 
 #[repr(u32)]
-#[derive(PartialEq, Clone, PartialOrd, Eq, Ord, Deserialize, Serialize)]
+#[derive(PartialEq, Clone, PartialOrd, Eq, Ord, Deserialize, Serialize, Debug, Hash)]
 pub enum ProfilerCategory {
     Compositing,
     LayoutPerform,
@@ -83,13 +83,12 @@ pub fn profile<T, F>(category: ProfilerCategory,
     let start_time = precise_time_ns();
     let val = callback();
     let end_time = precise_time_ns();
-    let ms = (end_time - start_time) as f64 / 1000000f64;
     let meta = meta.map(|(url, iframe, reflow_type)|
         TimerMetadata {
             url: url.serialize(),
             iframe: iframe == TimerMetadataFrameType::IFrame,
             incremental: reflow_type == TimerMetadataReflowType::Incremental,
         });
-    profiler_chan.send(ProfilerMsg::Time((category, meta), ms));
+    profiler_chan.send(ProfilerMsg::Time((category, meta), (start_time, end_time)));
     return val;
 }

components/servo/Cargo.lock

@@ -678,6 +678,33 @@ dependencies = [
  "libc 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "hbs-common-sys"
+version = "0.1.0"
+source = "git+https://github.com/libheartbeats/heartbeats-simple-sys.git#4476aa6ad4e2dfded87251d6069d9e747a54d9d8"
+dependencies = [
+ "libc 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "hbs-pow"
+version = "0.1.0"
+source = "git+https://github.com/libheartbeats/heartbeats-simple-rust.git#70ad49c810da3842e12eef2075d58552f1f6c707"
+dependencies = [
+ "hbs-pow-sys 0.1.0 (git+https://github.com/libheartbeats/heartbeats-simple-sys.git)",
+ "libc 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "hbs-pow-sys"
+version = "0.1.0"
+source = "git+https://github.com/libheartbeats/heartbeats-simple-sys.git#4476aa6ad4e2dfded87251d6069d9e747a54d9d8"
+dependencies = [
+ "hbs-common-sys 0.1.0 (git+https://github.com/libheartbeats/heartbeats-simple-sys.git)",
+ "libc 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "pkg-config 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "heapsize"
 version = "0.1.2"
@@ -1193,6 +1220,7 @@ source = "git+https://github.com/servo/rust-png#a3569ca11ea54e5d6152ee80d7d39b27
 name = "profile"
 version = "0.0.1"
 dependencies = [
+ "hbs-pow 0.1.0 (git+https://github.com/libheartbeats/heartbeats-simple-rust.git)",
  "ipc-channel 0.1.0 (git+https://github.com/pcwalton/ipc-channel)",
  "libc 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",

ports/cef/Cargo.lock

@@ -670,6 +670,33 @@ dependencies = [
  "libc 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "hbs-common-sys"
+version = "0.1.0"
+source = "git+https://github.com/libheartbeats/heartbeats-simple-sys.git#4476aa6ad4e2dfded87251d6069d9e747a54d9d8"
+dependencies = [
+ "libc 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "hbs-pow"
+version = "0.1.0"
+source = "git+https://github.com/libheartbeats/heartbeats-simple-rust.git#70ad49c810da3842e12eef2075d58552f1f6c707"
+dependencies = [
+ "hbs-pow-sys 0.1.0 (git+https://github.com/libheartbeats/heartbeats-simple-sys.git)",
+ "libc 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "hbs-pow-sys"
+version = "0.1.0"
+source = "git+https://github.com/libheartbeats/heartbeats-simple-sys.git#4476aa6ad4e2dfded87251d6069d9e747a54d9d8"
+dependencies = [
+ "hbs-common-sys 0.1.0 (git+https://github.com/libheartbeats/heartbeats-simple-sys.git)",
+ "libc 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "pkg-config 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "heapsize"
 version = "0.1.2"
@@ -1171,6 +1198,7 @@ source = "git+https://github.com/servo/rust-png#a3569ca11ea54e5d6152ee80d7d39b27
 name = "profile"
 version = "0.0.1"
 dependencies = [
+ "hbs-pow 0.1.0 (git+https://github.com/libheartbeats/heartbeats-simple-rust.git)",
  "ipc-channel 0.1.0 (git+https://github.com/pcwalton/ipc-channel)",
  "libc 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",

ports/gonk/Cargo.lock

@@ -554,6 +554,33 @@ dependencies = [
  "libc 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "hbs-common-sys"
+version = "0.1.0"
+source = "git+https://github.com/libheartbeats/heartbeats-simple-sys.git#4476aa6ad4e2dfded87251d6069d9e747a54d9d8"
+dependencies = [
+ "libc 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "hbs-pow"
+version = "0.1.0"
+source = "git+https://github.com/libheartbeats/heartbeats-simple-rust.git#70ad49c810da3842e12eef2075d58552f1f6c707"
+dependencies = [
+ "hbs-pow-sys 0.1.0 (git+https://github.com/libheartbeats/heartbeats-simple-sys.git)",
+ "libc 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "hbs-pow-sys"
+version = "0.1.0"
+source = "git+https://github.com/libheartbeats/heartbeats-simple-sys.git#4476aa6ad4e2dfded87251d6069d9e747a54d9d8"
+dependencies = [
+ "hbs-common-sys 0.1.0 (git+https://github.com/libheartbeats/heartbeats-simple-sys.git)",
+ "libc 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "pkg-config 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "heapsize"
 version = "0.1.2"
@@ -1020,6 +1047,7 @@ source = "git+https://github.com/servo/rust-png#a3569ca11ea54e5d6152ee80d7d39b27
 name = "profile"
 version = "0.0.1"
 dependencies = [
+ "hbs-pow 0.1.0 (git+https://github.com/libheartbeats/heartbeats-simple-rust.git)",
  "ipc-channel 0.1.0 (git+https://github.com/pcwalton/ipc-channel)",
  "libc 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",