Stop relying on linking details of std’s default allocator

We’ve been bitten before by symbol names changing:
https://github.com/servo/heapsize/pull/46
and upstream is planning to stop using jemalloc by default:
https://github.com/rust-lang/rust/issues/33082#issuecomment-309781465

So use the (relatively) new `#[global_allocator]` attribute
to explicitly select the system allocator on Windows
and jemalloc (now in an external crate) on other platforms.
This choice matches current defaults.
This commit is contained in:
Simon Sapin 2017-10-18 13:38:07 +02:00
parent 4c538b642e
commit 959ce482dd
20 changed files with 158 additions and 62 deletions

40
Cargo.lock generated
View file

@ -347,6 +347,11 @@ dependencies = [
"unicode-normalization 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "cc"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "cexpr"
version = "0.2.0"
@ -1110,6 +1115,7 @@ dependencies = [
"range 0.0.1",
"serde 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
"servo-fontconfig 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"servo_allocator 0.0.1",
"servo_arc 0.0.1",
"servo_atoms 0.0.1",
"servo_geometry 0.0.1",
@ -1439,6 +1445,24 @@ name = "itoa"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "jemalloc-sys"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cc 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.23 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "jemallocator"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"jemalloc-sys 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.23 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "jpeg-decoder"
version = "0.1.13"
@ -1571,6 +1595,7 @@ dependencies = [
"script_traits 0.0.1",
"selectors 0.19.0",
"serde_json 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
"servo_allocator 0.0.1",
"servo_arc 0.0.1",
"servo_atoms 0.0.1",
"servo_config 0.0.1",
@ -1733,7 +1758,6 @@ dependencies = [
"euclid 0.15.3 (registry+https://github.com/rust-lang/crates.io-index)",
"hashglobe 0.1.0",
"js 0.1.6 (git+https://github.com/servo/rust-mozjs)",
"kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"servo_arc 0.0.1",
"smallbitvec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
"smallvec 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2363,6 +2387,7 @@ dependencies = [
"heartbeats-simple 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"influent 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"ipc-channel 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"jemalloc-sys 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.23 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
"profile_traits 0.0.1",
@ -2381,6 +2406,7 @@ dependencies = [
"ipc-channel 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"profile 0.0.1",
"profile_traits 0.0.1",
"servo_allocator 0.0.1",
]
[[package]]
@ -2630,6 +2656,7 @@ dependencies = [
"selectors 0.19.0",
"serde 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
"servo_allocator 0.0.1",
"servo_arc 0.0.1",
"servo_atoms 0.0.1",
"servo_config 0.0.1",
@ -2921,6 +2948,14 @@ dependencies = [
"url 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "servo_allocator"
version = "0.0.1"
dependencies = [
"jemallocator 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "servo_arc"
version = "0.0.1"
@ -3782,6 +3817,7 @@ dependencies = [
"checksum byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c40977b0ee6b9885c9013cd41d9feffdd22deb3bb4dc3a71d901cc7a77de18c8"
"checksum bytes 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c129aff112dcc562970abb69e2508b40850dd24c274761bb50fb8a0067ba6c27"
"checksum caseless 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8950b075cff75cdabadee97148a8b5816c7cf62e5948a6005b5255d564b42fe7"
"checksum cc 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2c674f0870e3dbd4105184ea035acb1c32c8ae69939c9e228d2b11bbfe29efad"
"checksum cexpr 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "393a5f0088efbe41f9d1fcd062f24e83c278608420e62109feb2c8abee07de7d"
"checksum cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c819a1287eb618df47cc647173c5c4c66ba19d888a6e50d605672aed3140de"
"checksum cgl 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "86765cb42c2a2c497e142af72517c1b4d7ae5bb2f25dfa77a5c69642f2342d89"
@ -3873,6 +3909,8 @@ dependencies = [
"checksum ipc-channel 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c10ed089b1921b01ef342c736a37ee0788eeb9a5f373bb2df1ba88d01125064f"
"checksum itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4833d6978da405305126af4ac88569b5d71ff758581ce5a987dbfa3755f694fc"
"checksum itoa 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "eb2f404fbc66fd9aac13e998248505e7ecb2ad8e44ab6388684c5fb11c6c251c"
"checksum jemalloc-sys 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94fb624d7e8345e5c42caab8d1db6ec925fdadff3fd0cb7dd781b41be8442828"
"checksum jemallocator 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1850725977c344d63af66e8fd00857646e3ec936c490cd63667860b7b03ab5c1"
"checksum jpeg-decoder 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "2805ccb10ffe4d10e06ef68a158ff94c255211ecbae848fbde2146b098f93ce7"
"checksum js 0.1.6 (git+https://github.com/servo/rust-mozjs)" = "<none>"
"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"

View file

@ -0,0 +1,18 @@
[package]
name = "servo_allocator"
version = "0.0.1"
authors = ["The Servo Project Developers"]
license = "MPL-2.0"
publish = false
[lib]
path = "lib.rs"
[features]
unstable = ["kernel32-sys", "jemallocator"]
[target.'cfg(not(windows))'.dependencies]
jemallocator = { version = "0.1.3", optional = true }
[target.'cfg(windows)'.dependencies]
kernel32-sys = { version = "0.2.1", optional = true }

View file

@ -0,0 +1,62 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
//! Selecting the default global allocator for Servo
#![cfg_attr(all(feature = "unstable", windows), feature(alloc_system, allocator_api))]
#![cfg_attr(feature = "unstable", feature(global_allocator))]
#[cfg(feature = "unstable")]
#[global_allocator]
static ALLOC: platform::Allocator = platform::Allocator;
pub use platform::usable_size;
#[cfg(all(feature = "unstable", not(windows)))]
mod platform {
extern crate jemallocator;
pub use self::jemallocator::Jemalloc as Allocator;
use std::os::raw::c_void;
/// Get the size of a heap block.
pub unsafe extern "C" fn usable_size(ptr: *const c_void) -> usize {
jemallocator::usable_size(ptr)
}
}
#[cfg(all(feature = "unstable", windows))]
mod platform {
extern crate alloc_system;
extern crate kernel32;
pub use self::alloc_system::System as Allocator;
use self::kernel32::{GetProcessHeap, HeapSize, HeapValidate};
use std::os::raw::c_void;
/// Get the size of a heap block.
pub unsafe extern "C" fn usable_size(mut ptr: *const c_void) -> usize {
let heap = GetProcessHeap();
if HeapValidate(heap, 0, ptr) == 0 {
ptr = *(ptr as *const *const c_void).offset(-1);
}
HeapSize(heap, 0, ptr) as usize
}
}
#[cfg(not(feature = "unstable"))]
mod platform {
use std::os::raw::c_void;
/// Without `#[global_allocator]` we cannot be certain of what allocator is used
/// or how it is linked. We therefore disable memory reporting. (Return zero.)
pub unsafe extern "C" fn usable_size(_ptr: *const c_void) -> usize {
0
}
}

View file

@ -53,6 +53,7 @@ core-text = "7.0"
[target.'cfg(any(target_os = "linux", target_os = "android"))'.dependencies]
freetype = "0.3"
servo_allocator = {path = "../allocator"}
[target.'cfg(target_os = "linux")'.dependencies]
servo-fontconfig = "0.2.1"

View file

@ -27,8 +27,8 @@ extern crate fnv;
#[cfg(target_os = "linux")]
extern crate fontconfig;
extern crate fontsan;
#[cfg(any(target_os = "linux", target_os = "android"))]
extern crate freetype;
#[cfg(any(target_os = "linux", target_os = "android"))] extern crate freetype;
#[cfg(any(target_os = "linux", target_os = "android"))] extern crate servo_allocator;
extern crate gfx_traits;
// Eventually we would like the shaper to be pluggable, as many operating systems have their own

View file

@ -8,7 +8,8 @@ use freetype::freetype::FT_Library;
use freetype::freetype::FT_Memory;
use freetype::freetype::FT_MemoryRec_;
use freetype::freetype::FT_New_Library;
use malloc_size_of::{malloc_size_of, MallocSizeOf, MallocSizeOfOps};
use malloc_size_of::{MallocSizeOf, MallocSizeOfOps};
use servo_allocator::usable_size;
use std::mem;
use std::os::raw::{c_long, c_void};
use std::ptr;
@ -31,7 +32,7 @@ extern fn ft_alloc(mem: FT_Memory, req_size: c_long) -> *mut c_void {
mem::forget(vec);
unsafe {
let actual_size = malloc_size_of(ptr as *const _);
let actual_size = usable_size(ptr as *const _);
let user = (*mem).user as *mut User;
(*user).size += actual_size;
}
@ -41,7 +42,7 @@ extern fn ft_alloc(mem: FT_Memory, req_size: c_long) -> *mut c_void {
extern fn ft_free(mem: FT_Memory, ptr: *mut c_void) {
unsafe {
let actual_size = malloc_size_of(ptr as *const _);
let actual_size = usable_size(ptr as *const _);
let user = (*mem).user as *mut User;
(*user).size -= actual_size;
@ -50,13 +51,14 @@ extern fn ft_free(mem: FT_Memory, ptr: *mut c_void) {
}
}
extern fn ft_realloc(mem: FT_Memory, _cur_size: c_long, new_req_size: c_long,
extern fn ft_realloc(mem: FT_Memory, old_size: c_long, new_req_size: c_long,
old_ptr: *mut c_void) -> *mut c_void {
let old_actual_size;
let mut vec;
unsafe {
old_actual_size = malloc_size_of(old_ptr as *const _);
vec = Vec::<u8>::from_raw_parts(old_ptr as *mut u8, old_actual_size, old_actual_size);
old_actual_size = usable_size(old_ptr as *const _);
let old_size = old_size as usize;
vec = Vec::<u8>::from_raw_parts(old_ptr as *mut u8, old_size, old_size);
};
let new_req_size = new_req_size as usize;
@ -71,7 +73,7 @@ extern fn ft_realloc(mem: FT_Memory, _cur_size: c_long, new_req_size: c_long,
mem::forget(vec);
unsafe {
let new_actual_size = malloc_size_of(new_ptr as *const _);
let new_actual_size = usable_size(new_ptr as *const _);
let user = (*mem).user as *mut User;
(*user).size += new_actual_size;
(*user).size -= old_actual_size;

View file

@ -40,6 +40,7 @@ script_layout_interface = {path = "../script_layout_interface"}
script_traits = {path = "../script_traits"}
selectors = { path = "../selectors" }
serde_json = "1.0"
servo_allocator = {path = "../allocator"}
servo_arc = {path = "../servo_arc"}
servo_atoms = {path = "../atoms"}
servo_config = {path = "../config"}

View file

@ -39,6 +39,7 @@ extern crate script_layout_interface;
extern crate script_traits;
extern crate selectors;
extern crate serde_json;
extern crate servo_allocator;
extern crate servo_arc;
extern crate servo_atoms;
extern crate servo_config;
@ -84,7 +85,7 @@ use layout::webrender_helpers::WebRenderDisplayListConverter;
use layout::wrapper::LayoutNodeLayoutData;
use layout_traits::LayoutThreadFactory;
use libc::c_void;
use malloc_size_of::{malloc_size_of, MallocSizeOf, MallocSizeOfOps};
use malloc_size_of::{MallocSizeOf, MallocSizeOfOps};
use metrics::{PaintTimeMetrics, ProfilerMetadataFactory};
use msg::constellation_msg::PipelineId;
use msg::constellation_msg::TopLevelBrowsingContextId;
@ -775,7 +776,7 @@ impl LayoutThread {
let mut reports = vec![];
// Servo uses vanilla jemalloc, which doesn't have a
// malloc_enclosing_size_of function.
let mut ops = MallocSizeOfOps::new(malloc_size_of, None, None);
let mut ops = MallocSizeOfOps::new(::servo_allocator::usable_size, None, None);
// FIXME(njn): Just measuring the display tree for now.
let rw_data = possibly_locked_rw_data.lock();

View file

@ -8,9 +8,6 @@ publish = false
[lib]
path = "lib.rs"
[target.'cfg(windows)'.dependencies]
kernel32-sys = "0.2.1"
[features]
servo = ["js", "string_cache", "url", "webrender_api", "xml5ever"]

View file

@ -49,8 +49,6 @@ extern crate euclid;
extern crate hashglobe;
#[cfg(feature = "servo")]
extern crate js;
#[cfg(target_os = "windows")]
extern crate kernel32;
extern crate servo_arc;
extern crate smallbitvec;
extern crate smallvec;
@ -63,8 +61,6 @@ extern crate webrender_api;
#[cfg(feature = "servo")]
extern crate xml5ever;
#[cfg(target_os = "windows")]
use kernel32::{GetProcessHeap, HeapSize, HeapValidate};
use std::hash::{BuildHasher, Hash};
use std::mem::size_of;
use std::ops::Range;
@ -146,33 +142,6 @@ impl MallocSizeOfOps {
}
}
/// Get the size of a heap block.
#[cfg(not(target_os = "windows"))]
pub unsafe extern "C" fn malloc_size_of(ptr: *const c_void) -> usize {
// The C prototype is `je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr)`. On some
// platforms `JEMALLOC_USABLE_SIZE_CONST` is `const` and on some it is empty. But in practice
// this function doesn't modify the contents of the block that `ptr` points to, so we use
// `*const c_void` here.
extern "C" {
#[cfg_attr(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android"),
link_name = "je_malloc_usable_size")]
fn malloc_usable_size(ptr: *const c_void) -> usize;
}
malloc_usable_size(ptr)
}
/// Get the size of a heap block.
#[cfg(target_os = "windows")]
pub unsafe extern "C" fn malloc_size_of(mut ptr: *const c_void) -> usize {
let heap = GetProcessHeap();
if HeapValidate(heap, 0, ptr) == 0 {
ptr = *(ptr as *const *const c_void).offset(-1);
}
HeapSize(heap, 0, ptr) as usize
}
/// Trait for measuring the "deep" heap usage of a data structure. This is the
/// most commonly-used of the traits.
pub trait MallocSizeOf {

View file

@ -10,7 +10,7 @@ name = "profile"
path = "lib.rs"
[features]
unstable = []
unstable = ["jemalloc-sys"]
[dependencies]
profile_traits = {path = "../profile_traits"}
@ -31,3 +31,4 @@ regex = "0.2"
[target.'cfg(not(target_os = "windows"))'.dependencies]
libc = "0.2"
jemalloc-sys = {version = "0.1.3", optional = true}

View file

@ -2,17 +2,14 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#![cfg_attr(all(feature = "unstable", not(target_os = "windows")), feature(alloc_jemalloc))]
#![deny(unsafe_code)]
#[allow(unused_extern_crates)]
#[cfg(all(feature = "unstable", not(target_os = "windows")))]
extern crate alloc_jemalloc;
extern crate heartbeats_simple;
extern crate influent;
extern crate ipc_channel;
#[allow(unused_extern_crates)]
#[cfg(all(feature = "unstable", not(target_os = "windows")))]
extern crate jemalloc_sys;
#[cfg(not(target_os = "windows"))]
extern crate libc;
#[macro_use]

View file

@ -354,7 +354,7 @@ impl ReportsForest {
mod system_reporter {
#[cfg(all(feature = "unstable", not(target_os = "windows")))]
use libc::{c_char, c_void, size_t};
use libc::{c_void, size_t};
#[cfg(target_os = "linux")]
use libc::c_int;
use profile_traits::mem::{Report, ReportKind, ReporterRequest};
@ -460,11 +460,7 @@ mod system_reporter {
}
#[cfg(all(feature = "unstable", not(target_os = "windows")))]
extern {
#[cfg_attr(any(target_os = "macos", target_os = "android"), link_name = "je_mallctl")]
fn mallctl(name: *const c_char, oldp: *mut c_void, oldlenp: *mut size_t,
newp: *mut c_void, newlen: size_t) -> ::libc::c_int;
}
use jemalloc_sys::mallctl;
#[cfg(all(feature = "unstable", not(target_os = "windows")))]
fn jemalloc_stat(value_name: &str) -> Option<usize> {

View file

@ -13,7 +13,7 @@ path = "lib.rs"
[features]
debugmozjs = ['js/debugmozjs']
unstable = []
unstable = ["servo_allocator/unstable"]
[build-dependencies]
cmake = "0.1"
@ -77,6 +77,7 @@ script_plugins = {path = "../script_plugins"}
script_traits = {path = "../script_traits"}
selectors = { path = "../selectors" }
serde = "1.0"
servo_allocator = {path = "../allocator"}
servo_arc = {path = "../servo_arc"}
servo_atoms = {path = "../atoms"}
servo_config = {path = "../config"}

View file

@ -78,6 +78,7 @@ extern crate script_layout_interface;
extern crate script_traits;
extern crate selectors;
extern crate serde;
extern crate servo_allocator;
extern crate servo_arc;
#[macro_use] extern crate servo_atoms;
extern crate servo_config;

View file

@ -73,7 +73,7 @@ use js::jsapi::{JSAutoCompartment, JSContext, JS_SetWrapObjectCallbacks};
use js::jsapi::{JSTracer, SetWindowProxyClass};
use js::jsval::UndefinedValue;
use js::rust::Runtime;
use malloc_size_of::{malloc_size_of, MallocSizeOfOps};
use malloc_size_of::MallocSizeOfOps;
use mem::malloc_size_of_including_self;
use metrics::PaintTimeMetrics;
use microtask::{MicrotaskQueue, Microtask};
@ -1506,7 +1506,7 @@ impl ScriptThread {
let mut reports = vec![];
// Servo uses vanilla jemalloc, which doesn't have a
// malloc_enclosing_size_of function.
let mut ops = MallocSizeOfOps::new(malloc_size_of, None, None);
let mut ops = MallocSizeOfOps::new(::servo_allocator::usable_size, None, None);
for (_, document) in self.documents.borrow().iter() {
let current_url = document.url();

View file

@ -37,7 +37,7 @@ energy-profiling = ["libservo/energy-profiling"]
debugmozjs = ["libservo/debugmozjs"]
googlevr = ["libservo/googlevr"]
oculusvr = ["libservo/oculusvr"]
unstable = ["libservo/unstable"]
unstable = ["libservo/unstable", "profile_tests/unstable"]
[dependencies]
backtrace = "0.3"

View file

@ -626,6 +626,10 @@ def check_rust(file_name, lines):
+ decl_found.format(crate_name))
prev_crate[indent] = crate_name
if line == "}":
for i in [i for i in prev_crate.keys() if i > indent]:
del prev_crate[i]
# check alphabetical order of feature attributes in lib.rs files
if is_lib_rs_file:
match = re.search(r"#!\[feature\((.*)\)\]", line)

View file

@ -9,7 +9,13 @@ name = "profile_tests"
path = "lib.rs"
doctest = false
[features]
unstable = ["servo_allocator/unstable"]
[dependencies]
ipc-channel = "0.9"
profile = {path = "../../../components/profile"}
profile_traits = {path = "../../../components/profile_traits"}
# Work around https://github.com/alexcrichton/jemallocator/issues/19
servo_allocator = {path = "../../../components/allocator"}

View file

@ -5,6 +5,7 @@
extern crate ipc_channel;
extern crate profile;
extern crate profile_traits;
extern crate servo_allocator;
#[cfg(test)]
mod time;