From aef8537d7537cd886ce0a03e2c845b1c8dd54b6d Mon Sep 17 00:00:00 2001 From: webbeef Date: Fri, 4 Apr 2025 22:42:12 -0700 Subject: [PATCH] Make the memory reporting multi-process aware (#35863) So far the memory reporter aggregates reports from all processes, and runs the system reporter only in the main process. Instead it is desirable to have per-process reports. We do so by: - creating a ProcessReports struct that holds includes the pid in addition to the reports themselves. - running the system memory reporter also in content processes. - updating the about:memory page to create one report per process, and add useful information like the pid and the urls loaded in a given process. --- - [X] `./mach build -d` does not report any errors - [X] `./mach test-tidy` does not report any errors ![image](https://github.com/user-attachments/assets/0bafe140-539d-4d6a-8316-639309a22d4a) Signed-off-by: webbeef --- Cargo.lock | 2 + components/constellation/Cargo.toml | 1 + components/constellation/constellation.rs | 4 +- components/constellation/pipeline.rs | 22 ++ components/constellation/process_manager.rs | 15 +- components/net/resource_thread.rs | 6 +- components/profile/Cargo.toml | 1 + components/profile/lib.rs | 3 +- components/profile/mem.rs | 376 ++------------------ components/profile/system_reporter.rs | 332 +++++++++++++++++ components/script/dom/workerglobalscope.rs | 3 +- components/script/script_thread.rs | 5 +- components/servo/lib.rs | 2 + components/shared/profile/mem.rs | 34 +- resources/about-memory.html | 169 +++++---- 15 files changed, 551 insertions(+), 424 deletions(-) create mode 100644 components/profile/system_reporter.rs diff --git a/Cargo.lock b/Cargo.lock index 4606333681b..a47e5445490 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1162,6 +1162,7 @@ dependencies = [ "net", "net_traits", "parking_lot", + "profile", "profile_traits", "script_layout_interface", "script_traits", @@ -5828,6 +5829,7 @@ dependencies = [ "base", "ipc-channel", "libc", + "log", "parking_lot", "profile_traits", "regex", diff --git a/components/constellation/Cargo.toml b/components/constellation/Cargo.toml index 1b3402eaf56..9bc1576fa62 100644 --- a/components/constellation/Cargo.toml +++ b/components/constellation/Cargo.toml @@ -39,6 +39,7 @@ media = { path = "../media" } net = { path = "../net" } net_traits = { workspace = true } parking_lot = { workspace = true } +profile = { path = "../profile" } profile_traits = { workspace = true } script_layout_interface = { workspace = true } script_traits = { workspace = true } diff --git a/components/constellation/constellation.rs b/components/constellation/constellation.rs index 69547e3f6a0..f176f3b60d0 100644 --- a/components/constellation/constellation.rs +++ b/components/constellation/constellation.rs @@ -712,7 +712,7 @@ where // namespace 0 for the embedder, and 0 for the constellation next_pipeline_namespace_id: PipelineNamespaceId(2), time_profiler_chan: state.time_profiler_chan, - mem_profiler_chan: state.mem_profiler_chan, + mem_profiler_chan: state.mem_profiler_chan.clone(), phantom: PhantomData, webdriver: WebDriverData::new(), document_states: HashMap::new(), @@ -739,7 +739,7 @@ where active_media_session: None, rippy_data, user_content_manager: state.user_content_manager, - process_manager: ProcessManager::new(), + process_manager: ProcessManager::new(state.mem_profiler_chan), }; constellation.run(); diff --git a/components/constellation/pipeline.rs b/components/constellation/pipeline.rs index 66cf3b097e0..fc988f6e408 100644 --- a/components/constellation/pipeline.rs +++ b/components/constellation/pipeline.rs @@ -32,6 +32,8 @@ use media::WindowGLContext; use net::image_cache::ImageCacheImpl; use net_traits::ResourceThreads; use net_traits::image_cache::ImageCache; +use profile::system_reporter; +use profile_traits::mem::{ProfilerMsg, Reporter}; use profile_traits::{mem as profile_mem, time}; use script_layout_interface::{LayoutFactory, ScriptThreadFactory}; use script_traits::{ @@ -591,4 +593,24 @@ impl UnprivilegedPipelineContent { pub fn prefs(&self) -> &Preferences { &self.prefs } + + pub fn register_system_memory_reporter(&self) { + // Register the system memory reporter, which will run on its own thread. It never needs to + // be unregistered, because as long as the memory profiler is running the system memory + // reporter can make measurements. + let (system_reporter_sender, system_reporter_receiver) = + ipc::channel().expect("failed to create ipc channel"); + ROUTER.add_typed_route( + system_reporter_receiver, + Box::new(|message| { + if let Ok(request) = message { + system_reporter::collect_reports(request); + } + }), + ); + self.mem_profiler_chan.send(ProfilerMsg::RegisterReporter( + format!("system-content-{}", std::process::id()), + Reporter(system_reporter_sender), + )); + } } diff --git a/components/constellation/process_manager.rs b/components/constellation/process_manager.rs index 5e747a7764b..7da2e9c63df 100644 --- a/components/constellation/process_manager.rs +++ b/components/constellation/process_manager.rs @@ -6,6 +6,7 @@ use std::process::Child; use crossbeam_channel::{Receiver, Select}; use log::{debug, warn}; +use profile_traits::mem::{ProfilerChan, ProfilerMsg}; pub enum Process { Unsandboxed(Child), @@ -37,11 +38,15 @@ type ProcessReceiver = Receiver>; pub(crate) struct ProcessManager { processes: Vec<(Process, ProcessReceiver)>, + mem_profiler_chan: ProfilerChan, } impl ProcessManager { - pub fn new() -> Self { - Self { processes: vec![] } + pub fn new(mem_profiler_chan: ProfilerChan) -> Self { + Self { + processes: vec![], + mem_profiler_chan, + } } pub fn add(&mut self, receiver: ProcessReceiver, process: Process) { @@ -63,6 +68,12 @@ impl ProcessManager { pub fn remove(&mut self, index: usize) { let (mut process, _) = self.processes.swap_remove(index); debug!("Removing process pid={}", process.pid()); + // Unregister this process system memory profiler + self.mem_profiler_chan + .send(ProfilerMsg::UnregisterReporter(format!( + "system-content-{}", + process.pid() + ))); process.wait(); } } diff --git a/components/net/resource_thread.rs b/components/net/resource_thread.rs index effcf7b8e26..28934c94091 100644 --- a/components/net/resource_thread.rs +++ b/components/net/resource_thread.rs @@ -32,7 +32,9 @@ use net_traits::{ FetchChannels, FetchTaskTarget, ResourceFetchTiming, ResourceThreads, ResourceTimingType, WebSocketDomAction, WebSocketNetworkEvent, }; -use profile_traits::mem::{ProfilerChan as MemProfilerChan, Report, ReportKind, ReportsChan}; +use profile_traits::mem::{ + ProcessReports, ProfilerChan as MemProfilerChan, Report, ReportKind, ReportsChan, +}; use profile_traits::path; use profile_traits::time::ProfilerChan; use rustls::RootCertStore; @@ -297,7 +299,7 @@ impl ResourceChannelManager { size: private_cache.size_of(&mut ops), }; - msg.send(vec![public_report, private_report]); + msg.send(ProcessReports::new(vec![public_report, private_report])); } fn cancellation_listener(&self, request_id: RequestId) -> Option> { diff --git a/components/profile/Cargo.toml b/components/profile/Cargo.toml index 977b6aee34f..8a8d0775711 100644 --- a/components/profile/Cargo.toml +++ b/components/profile/Cargo.toml @@ -14,6 +14,7 @@ path = "lib.rs" [dependencies] base = { workspace = true } ipc-channel = { workspace = true } +log = { workspace = true } parking_lot = { workspace = true } profile_traits = { workspace = true } serde = { workspace = true } diff --git a/components/profile/lib.rs b/components/profile/lib.rs index 6321decd9c0..733bcce1278 100644 --- a/components/profile/lib.rs +++ b/components/profile/lib.rs @@ -4,7 +4,8 @@ #![deny(unsafe_code)] -#[allow(unsafe_code)] pub mod mem; +#[allow(unsafe_code)] +pub mod system_reporter; pub mod time; pub mod trace_dump; diff --git a/components/profile/mem.rs b/components/profile/mem.rs index 34930eb17ca..258099b9685 100644 --- a/components/profile/mem.rs +++ b/components/profile/mem.rs @@ -10,9 +10,13 @@ use std::thread; use ipc_channel::ipc::{self, IpcReceiver}; use ipc_channel::router::ROUTER; +use log::debug; use profile_traits::mem::{ MemoryReportResult, ProfilerChan, ProfilerMsg, Report, Reporter, ReporterRequest, ReportsChan, }; +use serde::Serialize; + +use crate::system_reporter; pub struct Profiler { /// The port through which messages are received. @@ -22,9 +26,6 @@ pub struct Profiler { reporters: HashMap, } -const JEMALLOC_HEAP_ALLOCATED_STR: &str = "jemalloc-heap-allocated"; -const SYSTEM_HEAP_ALLOCATED_STR: &str = "system-heap-allocated"; - impl Profiler { pub fn create() -> ProfilerChan { let (chan, port) = ipc::channel().unwrap(); @@ -53,7 +54,7 @@ impl Profiler { }), ); mem_profiler_chan.send(ProfilerMsg::RegisterReporter( - "system".to_owned(), + "system-main".to_owned(), Reporter(system_reporter_sender), )); @@ -78,6 +79,7 @@ impl Profiler { fn handle_msg(&mut self, msg: ProfilerMsg) -> bool { match msg { ProfilerMsg::RegisterReporter(name, reporter) => { + debug!("Registering memory reporter: {}", name); // Panic if it has already been registered. let name_clone = name.clone(); match self.reporters.insert(name, reporter) { @@ -87,6 +89,7 @@ impl Profiler { }, ProfilerMsg::UnregisterReporter(name) => { + debug!("Unregistering memory reporter: {}", name); // Panic if it hasn't previously been registered. match self.reporters.remove(&name) { Some(_) => true, @@ -95,8 +98,28 @@ impl Profiler { }, ProfilerMsg::Report(sender) => { + let main_pid = std::process::id(); + + #[derive(Serialize)] + struct JsonReport { + pid: u32, + #[serde(rename = "isMainProcess")] + is_main_process: bool, + reports: Vec, + } + let reports = self.collect_reports(); - let content = serde_json::to_string(&reports) + // Turn the pid -> reports map into a vector and add the + // hint to find the main process. + let json_reports: Vec = reports + .into_iter() + .map(|(pid, reports)| JsonReport { + pid, + reports, + is_main_process: pid == main_pid, + }) + .collect(); + let content = serde_json::to_string(&json_reports) .unwrap_or_else(|_| "{ error: \"failed to create memory report\"}".to_owned()); let _ = sender.send(MemoryReportResult { content }); true @@ -106,347 +129,20 @@ impl Profiler { } } - fn collect_reports(&self) -> Vec { - let mut result = vec![]; + /// Returns a map of pid -> reports + fn collect_reports(&self) -> HashMap> { + let mut result = HashMap::new(); + for reporter in self.reporters.values() { let (chan, port) = ipc::channel().unwrap(); reporter.collect_reports(ReportsChan(chan)); if let Ok(mut reports) = port.recv() { - result.append(&mut reports); + result + .entry(reports.pid) + .or_insert(vec![]) + .append(&mut reports.reports); } } result } } - -//--------------------------------------------------------------------------- - -mod system_reporter { - #[cfg(not(any(target_os = "windows", target_env = "ohos")))] - use std::ffi::CString; - #[cfg(not(any(target_os = "windows", target_env = "ohos")))] - use std::mem::size_of; - #[cfg(not(any(target_os = "windows", target_env = "ohos")))] - use std::ptr::null_mut; - - #[cfg(all(target_os = "linux", target_env = "gnu"))] - use libc::c_int; - #[cfg(not(any(target_os = "windows", target_env = "ohos")))] - use libc::{c_void, size_t}; - use profile_traits::mem::{Report, ReportKind, ReporterRequest}; - use profile_traits::path; - #[cfg(target_os = "macos")] - use task_info::task_basic_info::{resident_size, virtual_size}; - - use super::{JEMALLOC_HEAP_ALLOCATED_STR, SYSTEM_HEAP_ALLOCATED_STR}; - - /// Collects global measurements from the OS and heap allocators. - pub fn collect_reports(request: ReporterRequest) { - let mut reports = vec![]; - { - let mut report = |path, size| { - if let Some(size) = size { - reports.push(Report { - path, - kind: ReportKind::NonExplicitSize, - size, - }); - } - }; - - // Virtual and physical memory usage, as reported by the OS. - report(path!["vsize"], vsize()); - report(path!["resident"], resident()); - - // Memory segments, as reported by the OS. - for seg in resident_segments() { - report(path!["resident-according-to-smaps", seg.0], Some(seg.1)); - } - - // Total number of bytes allocated by the application on the system - // heap. - report(path![SYSTEM_HEAP_ALLOCATED_STR], system_heap_allocated()); - - // The descriptions of the following jemalloc measurements are taken - // directly from the jemalloc documentation. - - // "Total number of bytes allocated by the application." - report( - path![JEMALLOC_HEAP_ALLOCATED_STR], - jemalloc_stat("stats.allocated"), - ); - - // "Total number of bytes in active pages allocated by the application. - // This is a multiple of the page size, and greater than or equal to - // |stats.allocated|." - report(path!["jemalloc-heap-active"], jemalloc_stat("stats.active")); - - // "Total number of bytes in chunks mapped on behalf of the application. - // This is a multiple of the chunk size, and is at least as large as - // |stats.active|. This does not include inactive chunks." - report(path!["jemalloc-heap-mapped"], jemalloc_stat("stats.mapped")); - } - - request.reports_channel.send(reports); - } - - #[cfg(all(target_os = "linux", target_env = "gnu"))] - unsafe extern "C" { - fn mallinfo() -> struct_mallinfo; - } - - #[cfg(all(target_os = "linux", target_env = "gnu"))] - #[repr(C)] - pub struct struct_mallinfo { - arena: c_int, - ordblks: c_int, - smblks: c_int, - hblks: c_int, - hblkhd: c_int, - usmblks: c_int, - fsmblks: c_int, - uordblks: c_int, - fordblks: c_int, - keepcost: c_int, - } - - #[cfg(all(target_os = "linux", target_env = "gnu"))] - fn system_heap_allocated() -> Option { - let info: struct_mallinfo = unsafe { mallinfo() }; - - // The documentation in the glibc man page makes it sound like |uordblks| would suffice, - // but that only gets the small allocations that are put in the brk heap. We need |hblkhd| - // as well to get the larger allocations that are mmapped. - // - // These fields are unfortunately |int| and so can overflow (becoming negative) if memory - // usage gets high enough. So don't report anything in that case. In the non-overflow case - // we cast the two values to usize before adding them to make sure the sum also doesn't - // overflow. - if info.hblkhd < 0 || info.uordblks < 0 { - None - } else { - Some(info.hblkhd as usize + info.uordblks as usize) - } - } - - #[cfg(not(all(target_os = "linux", target_env = "gnu")))] - fn system_heap_allocated() -> Option { - None - } - - #[cfg(not(any(target_os = "windows", target_env = "ohos")))] - use tikv_jemalloc_sys::mallctl; - - #[cfg(not(any(target_os = "windows", target_env = "ohos")))] - fn jemalloc_stat(value_name: &str) -> Option { - // Before we request the measurement of interest, we first send an "epoch" - // request. Without that jemalloc gives cached statistics(!) which can be - // highly inaccurate. - let epoch_name = "epoch"; - let epoch_c_name = CString::new(epoch_name).unwrap(); - let mut epoch: u64 = 0; - let epoch_ptr = &mut epoch as *mut _ as *mut c_void; - let mut epoch_len = size_of::() as size_t; - - let value_c_name = CString::new(value_name).unwrap(); - let mut value: size_t = 0; - let value_ptr = &mut value as *mut _ as *mut c_void; - let mut value_len = size_of::() as size_t; - - // Using the same values for the `old` and `new` parameters is enough - // to get the statistics updated. - let rv = unsafe { - mallctl( - epoch_c_name.as_ptr(), - epoch_ptr, - &mut epoch_len, - epoch_ptr, - epoch_len, - ) - }; - if rv != 0 { - return None; - } - - let rv = unsafe { - mallctl( - value_c_name.as_ptr(), - value_ptr, - &mut value_len, - null_mut(), - 0, - ) - }; - if rv != 0 { - return None; - } - - Some(value as usize) - } - - #[cfg(any(target_os = "windows", target_env = "ohos"))] - fn jemalloc_stat(_value_name: &str) -> Option { - None - } - - #[cfg(target_os = "linux")] - fn page_size() -> usize { - unsafe { ::libc::sysconf(::libc::_SC_PAGESIZE) as usize } - } - - #[cfg(target_os = "linux")] - fn proc_self_statm_field(field: usize) -> Option { - use std::fs::File; - use std::io::Read; - - let mut f = File::open("/proc/self/statm").ok()?; - let mut contents = String::new(); - f.read_to_string(&mut contents).ok()?; - let s = contents.split_whitespace().nth(field)?; - let npages = s.parse::().ok()?; - Some(npages * page_size()) - } - - #[cfg(target_os = "linux")] - fn vsize() -> Option { - proc_self_statm_field(0) - } - - #[cfg(target_os = "linux")] - fn resident() -> Option { - proc_self_statm_field(1) - } - - #[cfg(target_os = "macos")] - fn vsize() -> Option { - virtual_size() - } - - #[cfg(target_os = "macos")] - fn resident() -> Option { - resident_size() - } - - #[cfg(not(any(target_os = "linux", target_os = "macos")))] - fn vsize() -> Option { - None - } - - #[cfg(not(any(target_os = "linux", target_os = "macos")))] - fn resident() -> Option { - None - } - - #[cfg(target_os = "linux")] - fn resident_segments() -> Vec<(String, usize)> { - use std::collections::HashMap; - use std::collections::hash_map::Entry; - use std::fs::File; - use std::io::{BufRead, BufReader}; - - use regex::Regex; - - // The first line of an entry in /proc//smaps looks just like an entry - // in /proc//maps: - // - // address perms offset dev inode pathname - // 02366000-025d8000 rw-p 00000000 00:00 0 [heap] - // - // Each of the following lines contains a key and a value, separated - // by ": ", where the key does not contain either of those characters. - // For example: - // - // Rss: 132 kB - - let f = match File::open("/proc/self/smaps") { - Ok(f) => BufReader::new(f), - Err(_) => return vec![], - }; - - let seg_re = Regex::new( - r"^[:xdigit:]+-[:xdigit:]+ (....) [:xdigit:]+ [:xdigit:]+:[:xdigit:]+ \d+ +(.*)", - ) - .unwrap(); - let rss_re = Regex::new(r"^Rss: +(\d+) kB").unwrap(); - - // We record each segment's resident size. - let mut seg_map: HashMap = HashMap::new(); - - #[derive(PartialEq)] - enum LookingFor { - Segment, - Rss, - } - let mut looking_for = LookingFor::Segment; - - let mut curr_seg_name = String::new(); - - // Parse the file. - for line in f.lines() { - let line = match line { - Ok(line) => line, - Err(_) => continue, - }; - if looking_for == LookingFor::Segment { - // Look for a segment info line. - let cap = match seg_re.captures(&line) { - Some(cap) => cap, - None => continue, - }; - let perms = cap.get(1).unwrap().as_str(); - let pathname = cap.get(2).unwrap().as_str(); - - // Construct the segment name from its pathname and permissions. - curr_seg_name.clear(); - if pathname.is_empty() || pathname.starts_with("[stack:") { - // Anonymous memory. Entries marked with "[stack:nnn]" - // look like thread stacks but they may include other - // anonymous mappings, so we can't trust them and just - // treat them as entirely anonymous. - curr_seg_name.push_str("anonymous"); - } else { - curr_seg_name.push_str(pathname); - } - curr_seg_name.push_str(" ("); - curr_seg_name.push_str(perms); - curr_seg_name.push(')'); - - looking_for = LookingFor::Rss; - } else { - // Look for an "Rss:" line. - let cap = match rss_re.captures(&line) { - Some(cap) => cap, - None => continue, - }; - let rss = cap.get(1).unwrap().as_str().parse::().unwrap() * 1024; - - if rss > 0 { - // Aggregate small segments into "other". - let seg_name = if rss < 512 * 1024 { - "other".to_owned() - } else { - curr_seg_name.clone() - }; - match seg_map.entry(seg_name) { - Entry::Vacant(entry) => { - entry.insert(rss); - }, - Entry::Occupied(mut entry) => *entry.get_mut() += rss, - } - } - - looking_for = LookingFor::Segment; - } - } - - // Note that the sum of all these segments' RSS values differs from the "resident" - // measurement obtained via /proc//statm in resident(). It's unclear why this - // difference occurs; for some processes the measurements match, but for Servo they do not. - seg_map.into_iter().collect() - } - - #[cfg(not(target_os = "linux"))] - fn resident_segments() -> Vec<(String, usize)> { - vec![] - } -} diff --git a/components/profile/system_reporter.rs b/components/profile/system_reporter.rs new file mode 100644 index 00000000000..f41b837666f --- /dev/null +++ b/components/profile/system_reporter.rs @@ -0,0 +1,332 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#[cfg(not(any(target_os = "windows", target_env = "ohos")))] +use std::ffi::CString; +#[cfg(not(any(target_os = "windows", target_env = "ohos")))] +use std::mem::size_of; +#[cfg(not(any(target_os = "windows", target_env = "ohos")))] +use std::ptr::null_mut; + +#[cfg(all(target_os = "linux", target_env = "gnu"))] +use libc::c_int; +#[cfg(not(any(target_os = "windows", target_env = "ohos")))] +use libc::{c_void, size_t}; +use profile_traits::mem::{ProcessReports, Report, ReportKind, ReporterRequest}; +use profile_traits::path; +#[cfg(target_os = "macos")] +use task_info::task_basic_info::{resident_size, virtual_size}; + +const JEMALLOC_HEAP_ALLOCATED_STR: &str = "jemalloc-heap-allocated"; +const SYSTEM_HEAP_ALLOCATED_STR: &str = "system-heap-allocated"; + +/// Collects global measurements from the OS and heap allocators. +pub fn collect_reports(request: ReporterRequest) { + let mut reports = vec![]; + { + let mut report = |path, size| { + if let Some(size) = size { + reports.push(Report { + path, + kind: ReportKind::NonExplicitSize, + size, + }); + } + }; + + // Virtual and physical memory usage, as reported by the OS. + report(path!["vsize"], vsize()); + report(path!["resident"], resident()); + + // Memory segments, as reported by the OS. + for seg in resident_segments() { + report(path!["resident-according-to-smaps", seg.0], Some(seg.1)); + } + + // Total number of bytes allocated by the application on the system + // heap. + report(path![SYSTEM_HEAP_ALLOCATED_STR], system_heap_allocated()); + + // The descriptions of the following jemalloc measurements are taken + // directly from the jemalloc documentation. + + // "Total number of bytes allocated by the application." + report( + path![JEMALLOC_HEAP_ALLOCATED_STR], + jemalloc_stat("stats.allocated"), + ); + + // "Total number of bytes in active pages allocated by the application. + // This is a multiple of the page size, and greater than or equal to + // |stats.allocated|." + report(path!["jemalloc-heap-active"], jemalloc_stat("stats.active")); + + // "Total number of bytes in chunks mapped on behalf of the application. + // This is a multiple of the chunk size, and is at least as large as + // |stats.active|. This does not include inactive chunks." + report(path!["jemalloc-heap-mapped"], jemalloc_stat("stats.mapped")); + } + + request.reports_channel.send(ProcessReports::new(reports)); +} + +#[cfg(all(target_os = "linux", target_env = "gnu"))] +unsafe extern "C" { + fn mallinfo() -> struct_mallinfo; +} + +#[cfg(all(target_os = "linux", target_env = "gnu"))] +#[repr(C)] +pub struct struct_mallinfo { + arena: c_int, + ordblks: c_int, + smblks: c_int, + hblks: c_int, + hblkhd: c_int, + usmblks: c_int, + fsmblks: c_int, + uordblks: c_int, + fordblks: c_int, + keepcost: c_int, +} + +#[cfg(all(target_os = "linux", target_env = "gnu"))] +fn system_heap_allocated() -> Option { + let info: struct_mallinfo = unsafe { mallinfo() }; + + // The documentation in the glibc man page makes it sound like |uordblks| would suffice, + // but that only gets the small allocations that are put in the brk heap. We need |hblkhd| + // as well to get the larger allocations that are mmapped. + // + // These fields are unfortunately |int| and so can overflow (becoming negative) if memory + // usage gets high enough. So don't report anything in that case. In the non-overflow case + // we cast the two values to usize before adding them to make sure the sum also doesn't + // overflow. + if info.hblkhd < 0 || info.uordblks < 0 { + None + } else { + Some(info.hblkhd as usize + info.uordblks as usize) + } +} + +#[cfg(not(all(target_os = "linux", target_env = "gnu")))] +fn system_heap_allocated() -> Option { + None +} + +#[cfg(not(any(target_os = "windows", target_env = "ohos")))] +use tikv_jemalloc_sys::mallctl; + +#[cfg(not(any(target_os = "windows", target_env = "ohos")))] +fn jemalloc_stat(value_name: &str) -> Option { + // Before we request the measurement of interest, we first send an "epoch" + // request. Without that jemalloc gives cached statistics(!) which can be + // highly inaccurate. + let epoch_name = "epoch"; + let epoch_c_name = CString::new(epoch_name).unwrap(); + let mut epoch: u64 = 0; + let epoch_ptr = &mut epoch as *mut _ as *mut c_void; + let mut epoch_len = size_of::() as size_t; + + let value_c_name = CString::new(value_name).unwrap(); + let mut value: size_t = 0; + let value_ptr = &mut value as *mut _ as *mut c_void; + let mut value_len = size_of::() as size_t; + + // Using the same values for the `old` and `new` parameters is enough + // to get the statistics updated. + let rv = unsafe { + mallctl( + epoch_c_name.as_ptr(), + epoch_ptr, + &mut epoch_len, + epoch_ptr, + epoch_len, + ) + }; + if rv != 0 { + return None; + } + + let rv = unsafe { + mallctl( + value_c_name.as_ptr(), + value_ptr, + &mut value_len, + null_mut(), + 0, + ) + }; + if rv != 0 { + return None; + } + + Some(value as usize) +} + +#[cfg(any(target_os = "windows", target_env = "ohos"))] +fn jemalloc_stat(_value_name: &str) -> Option { + None +} + +#[cfg(target_os = "linux")] +fn page_size() -> usize { + unsafe { ::libc::sysconf(::libc::_SC_PAGESIZE) as usize } +} + +#[cfg(target_os = "linux")] +fn proc_self_statm_field(field: usize) -> Option { + use std::fs::File; + use std::io::Read; + + let mut f = File::open("/proc/self/statm").ok()?; + let mut contents = String::new(); + f.read_to_string(&mut contents).ok()?; + let s = contents.split_whitespace().nth(field)?; + let npages = s.parse::().ok()?; + Some(npages * page_size()) +} + +#[cfg(target_os = "linux")] +fn vsize() -> Option { + proc_self_statm_field(0) +} + +#[cfg(target_os = "linux")] +fn resident() -> Option { + proc_self_statm_field(1) +} + +#[cfg(target_os = "macos")] +fn vsize() -> Option { + virtual_size() +} + +#[cfg(target_os = "macos")] +fn resident() -> Option { + resident_size() +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn vsize() -> Option { + None +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn resident() -> Option { + None +} + +#[cfg(target_os = "linux")] +fn resident_segments() -> Vec<(String, usize)> { + use std::collections::HashMap; + use std::collections::hash_map::Entry; + use std::fs::File; + use std::io::{BufRead, BufReader}; + + use regex::Regex; + + // The first line of an entry in /proc//smaps looks just like an entry + // in /proc//maps: + // + // address perms offset dev inode pathname + // 02366000-025d8000 rw-p 00000000 00:00 0 [heap] + // + // Each of the following lines contains a key and a value, separated + // by ": ", where the key does not contain either of those characters. + // For example: + // + // Rss: 132 kB + + let f = match File::open("/proc/self/smaps") { + Ok(f) => BufReader::new(f), + Err(_) => return vec![], + }; + + let seg_re = Regex::new( + r"^[:xdigit:]+-[:xdigit:]+ (....) [:xdigit:]+ [:xdigit:]+:[:xdigit:]+ \d+ +(.*)", + ) + .unwrap(); + let rss_re = Regex::new(r"^Rss: +(\d+) kB").unwrap(); + + // We record each segment's resident size. + let mut seg_map: HashMap = HashMap::new(); + + #[derive(PartialEq)] + enum LookingFor { + Segment, + Rss, + } + let mut looking_for = LookingFor::Segment; + + let mut curr_seg_name = String::new(); + + // Parse the file. + for line in f.lines() { + let line = match line { + Ok(line) => line, + Err(_) => continue, + }; + if looking_for == LookingFor::Segment { + // Look for a segment info line. + let cap = match seg_re.captures(&line) { + Some(cap) => cap, + None => continue, + }; + let perms = cap.get(1).unwrap().as_str(); + let pathname = cap.get(2).unwrap().as_str(); + + // Construct the segment name from its pathname and permissions. + curr_seg_name.clear(); + if pathname.is_empty() || pathname.starts_with("[stack:") { + // Anonymous memory. Entries marked with "[stack:nnn]" + // look like thread stacks but they may include other + // anonymous mappings, so we can't trust them and just + // treat them as entirely anonymous. + curr_seg_name.push_str("anonymous"); + } else { + curr_seg_name.push_str(pathname); + } + curr_seg_name.push_str(" ("); + curr_seg_name.push_str(perms); + curr_seg_name.push(')'); + + looking_for = LookingFor::Rss; + } else { + // Look for an "Rss:" line. + let cap = match rss_re.captures(&line) { + Some(cap) => cap, + None => continue, + }; + let rss = cap.get(1).unwrap().as_str().parse::().unwrap() * 1024; + + if rss > 0 { + // Aggregate small segments into "other". + let seg_name = if rss < 512 * 1024 { + "other".to_owned() + } else { + curr_seg_name.clone() + }; + match seg_map.entry(seg_name) { + Entry::Vacant(entry) => { + entry.insert(rss); + }, + Entry::Occupied(mut entry) => *entry.get_mut() += rss, + } + } + + looking_for = LookingFor::Segment; + } + } + + // Note that the sum of all these segments' RSS values differs from the "resident" + // measurement obtained via /proc//statm in resident(). It's unclear why this + // difference occurs; for some processes the measurements match, but for Servo they do not. + seg_map.into_iter().collect() +} + +#[cfg(not(target_os = "linux"))] +fn resident_segments() -> Vec<(String, usize)> { + vec![] +} diff --git a/components/script/dom/workerglobalscope.rs b/components/script/dom/workerglobalscope.rs index 6e70ffe34e0..3937f8fbedb 100644 --- a/components/script/dom/workerglobalscope.rs +++ b/components/script/dom/workerglobalscope.rs @@ -24,6 +24,7 @@ use net_traits::request::{ CredentialsMode, Destination, InsecureRequestsPolicy, ParserMetadata, RequestBuilder as NetRequestInit, }; +use profile_traits::mem::ProcessReports; use script_traits::WorkerGlobalScopeInit; use servo_url::{MutableOrigin, ServoUrl}; use timers::TimerScheduler; @@ -535,7 +536,7 @@ impl WorkerGlobalScope { CommonScriptMsg::CollectReports(reports_chan) => { let cx = self.get_cx(); let reports = cx.get_reports(format!("url({})", self.get_url())); - reports_chan.send(reports); + reports_chan.send(ProcessReports::new(reports)); }, } true diff --git a/components/script/script_thread.rs b/components/script/script_thread.rs index 5b5aa4b9254..33f07f98b41 100644 --- a/components/script/script_thread.rs +++ b/components/script/script_thread.rs @@ -71,7 +71,7 @@ use net_traits::{ ResourceFetchTiming, ResourceThreads, ResourceTimingType, }; use percent_encoding::percent_decode; -use profile_traits::mem::ReportsChan; +use profile_traits::mem::{ProcessReports, ReportsChan}; use profile_traits::time::ProfilerCategory; use profile_traits::time_profile; use script_layout_interface::{ @@ -427,6 +427,7 @@ impl ScriptThreadFactory for ScriptThread { memory_profiler_sender.run_with_memory_reporting( || { script_thread.start(CanGc::note()); + let _ = script_thread .senders .content_process_shutdown_sender @@ -2426,7 +2427,7 @@ impl ScriptThread { document.window().layout().collect_reports(&mut reports); } - reports_chan.send(reports); + reports_chan.send(ProcessReports::new(reports)); } /// Updates iframe element after a change in visibility diff --git a/components/servo/lib.rs b/components/servo/lib.rs index ef4d8b446fd..5a87bde3dc0 100644 --- a/components/servo/lib.rs +++ b/components/servo/lib.rs @@ -1182,6 +1182,8 @@ pub fn run_content_process(token: String) { let background_hang_monitor_register = content.register_with_background_hang_monitor(); let layout_factory = Arc::new(layout_thread_2020::LayoutFactoryImpl()); + content.register_system_memory_reporter(); + content.start_all::( true, layout_factory, diff --git a/components/shared/profile/mem.rs b/components/shared/profile/mem.rs index 2b13ccb7715..daeadc242c9 100644 --- a/components/shared/profile/mem.rs +++ b/components/shared/profile/mem.rs @@ -140,16 +140,36 @@ pub struct Report { pub size: usize, } +/// A set of reports belonging to a process. +#[derive(Debug, Deserialize, Serialize)] +pub struct ProcessReports { + /// The set of reports. + pub reports: Vec, + + /// The process id. + pub pid: u32, +} + +impl ProcessReports { + /// Adopt these reports and configure the process pid. + pub fn new(reports: Vec) -> Self { + Self { + reports, + pid: std::process::id(), + } + } +} + /// A channel through which memory reports can be sent. #[derive(Clone, Debug, Deserialize, Serialize)] -pub struct ReportsChan(pub IpcSender>); +pub struct ReportsChan(pub IpcSender); impl ReportsChan { /// Send `report` on this `IpcSender`. /// /// Panics if the send fails. - pub fn send(&self, report: Vec) { - self.0.send(report).unwrap(); + pub fn send(&self, reports: ProcessReports) { + self.0.send(reports).unwrap(); } } @@ -172,12 +192,8 @@ pub struct Reporter(pub IpcSender); impl Reporter { /// Collect one or more memory reports. Returns true on success, and false on failure. - pub fn collect_reports(&self, reports_chan: ReportsChan) { - self.0 - .send(ReporterRequest { - reports_channel: reports_chan, - }) - .unwrap() + pub fn collect_reports(&self, reports_channel: ReportsChan) { + self.0.send(ReporterRequest { reports_channel }).unwrap() } } diff --git a/resources/about-memory.html b/resources/about-memory.html index 380cee3d6ec..7e6e4c188e3 100644 --- a/resources/about-memory.html +++ b/resources/about-memory.html @@ -63,6 +63,96 @@ return result; } + function reportsForProcess(processReport) { + let explicitRoot = {}; + let nonExplicitRoot = {}; + + let jemallocHeapReportedSize = 0; + let systemHeapReportedSize = 0; + + let jemallocHeapAllocatedSize = NaN; + let systemHeapAllocatedSize = NaN; + + // In content processes, get the list of urls. + let urls = new Set(); + + processReport.reports.forEach((report) => { + if (report.path[0].startsWith("url(")) { + // This can be a list of urls. + let path_urls = report.path[0].slice(4, -1).split(", "); + path_urls.forEach((url) => urls.add(url)); + } + + // Add "explicit" to the start of the path, when appropriate. + if (report.kind.startsWith("Explicit")) { + report.path.unshift("explicit"); + } + + // Update the reported fractions of the heaps, when appropriate. + if (report.kind == "ExplicitJemallocHeapSize") { + jemallocHeapReportedSize += report.size; + } else if (report.kind == "ExplicitSystemHeapSize") { + systemHeapReportedSize += report.size; + } + + // Record total size of the heaps, when we see them. + if (report.path.length == 1) { + if (report.path[0] == "jemalloc-heap-allocated") { + jemallocHeapAllocatedSize = report.size; + } else if (report.path[0] == "system-heap-allocated") { + systemHeapAllocatedSize = report.size; + } + } + + // Insert this report at the proper position. + insertNode( + report.kind.startsWith("Explicit") ? explicitRoot : nonExplicitRoot, + report + ); + }); + + // Compute and insert the heap-unclassified values. + if (!isNaN(jemallocHeapAllocatedSize)) { + insertNode(explicitRoot, { + path: ["explicit", "jemalloc-heap-unclassified"], + size: jemallocHeapAllocatedSize - jemallocHeapReportedSize, + }); + } + if (!isNaN(systemHeapAllocatedSize)) { + insertNode(explicitRoot, { + path: ["explicit", "system-heap-unclassified"], + size: systemHeapAllocatedSize - systemHeapReportedSize, + }); + } + + // Create the DOM structure for each process report: + //

...

 ...

+ let container = document.createElement("div"); + container.classList.add("process"); + let reportTitle = document.createElement("h4"); + reportTitle.textContent = `${ + processReport.isMainProcess ? "Main Process" : "Content Process" + } (pid ${processReport.pid}) ${[...urls.values()].join(", ")}`; + + container.append(reportTitle); + let reportNode = document.createElement("pre"); + reportNode.classList.add("report"); + container.append(reportNode); + + reportNode.append(convertNodeToDOM(explicitRoot.explicit, "explicit")); + + for (let prop in nonExplicitRoot) { + reportNode.append(convertNodeToDOM(nonExplicitRoot[prop], prop)); + } + + // Make sure we always put the main process first. + if (processReport.isMainProcess) { + window.reports.prepend(container); + } else { + window.reports.append(container); + } + } + function start() { window.startButton.onclick = async () => { let content = await navigator.servo.reportMemory(); @@ -71,70 +161,15 @@ console.error(reports.error); return; } - window.report.innerHTML = ""; - window.report.classList.remove("hidden"); + window.reports.innerHTML = ""; + window.reports.classList.remove("hidden"); - let explicitRoot = {}; - let nonExplicitRoot = {}; - - let jemallocHeapReportedSize = 0; - let systemHeapReportedSize = 0; - - let jemallocHeapAllocatedSize = NaN; - let systemHeapAllocatedSize = NaN; - - reports.forEach((report) => { - // Add "explicit" to the start of the path, when appropriate. - if (report.kind.startsWith("Explicit")) { - report.path.unshift("explicit"); - } - - // Update the reported fractions of the heaps, when appropriate. - if (report.kind == "ExplicitJemallocHeapSize") { - jemallocHeapReportedSize += report.size; - } else if (report.kind == "ExplicitSystemHeapSize") { - systemHeapReportedSize += report.size; - } - - // Record total size of the heaps, when we see them. - if (report.path.length == 1) { - if (report.path[0] == "jemalloc-heap-allocated") { - jemallocHeapAllocatedSize = report.size; - } else if (report.path[0] == "system-heap-allocated") { - systemHeapAllocatedSize = report.size; - } - } - - // Insert this report at the proper position. - insertNode( - report.kind.startsWith("Explicit") - ? explicitRoot - : nonExplicitRoot, - report - ); - }); - - // Compute and insert the heap-unclassified values. - if (!isNaN(jemallocHeapAllocatedSize)) { - insertNode(explicitRoot, { - path: ["explicit", "jemalloc-heap-unclassified"], - size: jemallocHeapAllocatedSize - jemallocHeapReportedSize, - }); - } - if (!isNaN(systemHeapAllocatedSize)) { - insertNode(explicitRoot, { - path: ["explicit", "system-heap-unclassified"], - size: systemHeapAllocatedSize - systemHeapReportedSize, - }); + if (!Array.isArray(reports)) { + console.error("Unexpected memory report format!"); + return; } - window.report.append( - convertNodeToDOM(explicitRoot.explicit, "explicit") - ); - - for (let prop in nonExplicitRoot) { - window.report.append(convertNodeToDOM(nonExplicitRoot[prop], prop)); - } + reports.forEach(reportsForProcess); }; } @@ -152,15 +187,19 @@ cursor: pointer; } - #report { - line-height: 1.5em; + div.process { + margin: 0.5em; border: 2px solid gray; border-radius: 10px; padding: 5px; background-color: lightgray; } - #report > details { + .report { + line-height: 1.5em; + } + + .report > details { margin-bottom: 1em; } @@ -172,6 +211,6 @@

Memory Reports

- +