From dd1b43b8909178044b80ee1aa7e1ad1bce2b13d7 Mon Sep 17 00:00:00 2001 From: Josh Matthews Date: Thu, 8 Mar 2018 16:58:23 -0500 Subject: [PATCH] Add tool to chart memory usage from reports. --- components/profile/mem.rs | 8 +- etc/memory_chart.html | 112 ++++++++++++++++++++++++++ etc/memory_reports_over_time.py | 136 ++++++++++++++++++++++++++++++++ 3 files changed, 255 insertions(+), 1 deletion(-) create mode 100644 etc/memory_chart.html create mode 100644 etc/memory_reports_over_time.py diff --git a/components/profile/mem.rs b/components/profile/mem.rs index 9d09853d551..bb2a3873a91 100644 --- a/components/profile/mem.rs +++ b/components/profile/mem.rs @@ -12,6 +12,7 @@ use std::borrow::ToOwned; use std::cmp::Ordering; use std::collections::HashMap; use std::thread; +use std::time::Instant; use time::duration_from_seconds; pub struct Profiler { @@ -20,6 +21,9 @@ pub struct Profiler { /// Registered memory reporters. reporters: HashMap, + + /// Instant at which this profiler was created. + created: Instant, } const JEMALLOC_HEAP_ALLOCATED_STR: &'static str = "jemalloc-heap-allocated"; @@ -69,6 +73,7 @@ impl Profiler { Profiler { port: port, reporters: HashMap::new(), + created: Instant::now(), } } @@ -111,7 +116,8 @@ impl Profiler { } fn handle_print_msg(&self) { - println!("Begin memory reports"); + let elapsed = self.created.elapsed(); + println!("Begin memory reports {}", elapsed.as_secs()); println!("|"); // Collect reports from memory reporters. diff --git a/etc/memory_chart.html b/etc/memory_chart.html new file mode 100644 index 00000000000..f639a93ca04 --- /dev/null +++ b/etc/memory_chart.html @@ -0,0 +1,112 @@ + + + + diff --git a/etc/memory_reports_over_time.py b/etc/memory_reports_over_time.py new file mode 100644 index 00000000000..44a969d2d57 --- /dev/null +++ b/etc/memory_reports_over_time.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python + +# Copyright 2018 The Servo Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import json +import os +import sys +import tempfile +import webbrowser + + +def extract_memory_reports(lines): + in_report = False + report_lines = [] + times = [] + for line in lines: + if line.startswith('Begin memory reports'): + in_report = True + report_lines += [[]] + times += [line.strip().split()[-1]] + elif line == 'End memory reports\n': + in_report = False + elif in_report: + report_lines[-1].append(line.strip()) + return (report_lines, times) + + +def parse_memory_report(lines): + reports = {} + parents = [] + last_separator_index = None + for line in lines: + assert(line[0] == '|') + line = line[1:] + if not line: + continue + separator_index = line.index('--') + if last_separator_index and separator_index <= last_separator_index: + while parents and parents[-1][1] >= separator_index: + parents.pop() + + amount, unit, _, name = line.split() + + dest_report = reports + for (parent, index) in parents: + dest_report = dest_report[parent]['children'] + dest_report[name] = { + 'amount': amount, + 'unit': unit, + 'children': {} + } + + parents += [(name, separator_index)] + last_separator_index = separator_index + return reports + + +def transform_report_for_test(report): + transformed = {} + remaining = list(report.items()) + while remaining: + (name, value) = remaining.pop() + transformed[name] = '%s %s' % (value['amount'], value['unit']) + remaining += map(lambda (k, v): (name + '/' + k, v), list(value['children'].items())) + return transformed + + +def test(): + input = '''| +| 23.89 MiB -- explicit +| 21.35 MiB -- jemalloc-heap-unclassified +| 2.54 MiB -- url(https://servo.org/) +| 2.16 MiB -- js +| 1.00 MiB -- gc-heap +| 0.77 MiB -- decommitted +| 1.00 MiB -- non-heap +| 0.27 MiB -- layout-thread +| 0.27 MiB -- stylist +| 0.12 MiB -- dom-tree +| +| 25.18 MiB -- jemalloc-heap-active''' + + expected = { + 'explicit': '23.89 MiB', + 'explicit/jemalloc-heap-unclassified': '21.35 MiB', + 'explicit/url(https://servo.org/)': '2.54 MiB', + 'explicit/url(https://servo.org/)/js': '2.16 MiB', + 'explicit/url(https://servo.org/)/js/gc-heap': '1.00 MiB', + 'explicit/url(https://servo.org/)/js/gc-heap/decommitted': '0.77 MiB', + 'explicit/url(https://servo.org/)/js/non-heap': '1.00 MiB', + 'explicit/url(https://servo.org/)/layout-thread': '0.27 MiB', + 'explicit/url(https://servo.org/)/layout-thread/stylist': '0.27 MiB', + 'explicit/url(https://servo.org/)/dom-tree': '0.12 MiB', + 'jemalloc-heap-active': '25.18 MiB', + } + report = parse_memory_report(input.split('\n')) + transformed = transform_report_for_test(report) + assert(sorted(transformed.keys()) == sorted(expected.keys())) + for k, v in transformed.items(): + assert(v == expected[k]) + return 0 + + +def usage(): + print('%s --test - run automated tests' % sys.argv[0]) + print('%s file - extract all memory reports that are present in file' % sys.argv[0]) + return 1 + + +if __name__ == "__main__": + if len(sys.argv) == 1: + sys.exit(usage()) + + if sys.argv[1] == '--test': + sys.exit(test()) + + with open(sys.argv[1]) as f: + lines = f.readlines() + (reports, times) = extract_memory_reports(lines) + json_reports = [] + for (report_lines, seconds) in zip(reports, times): + report = parse_memory_report(report_lines) + json_reports += [{'seconds': seconds, 'report': report}] + with tempfile.NamedTemporaryFile(delete=False) as output: + thisdir = os.path.dirname(os.path.abspath(__file__)) + with open(os.path.join(thisdir, 'memory_chart.html')) as template: + content = template.read() + output.write(content.replace('[/* json data */]', json.dumps(json_reports))) + webbrowser.open_new_tab('file://' + output.name)