From 3dd91a28c769e9fe31fad9e07e7ed87c49ab54a7 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Thu, 12 May 2016 18:03:16 -0700 Subject: [PATCH] Bug 1272541 - Expose more data from SystemResourceMonitor.as_dict; r=ahal We add some system information including processor count and memory sizes. We also add an "overall" section describing total resource usage. This (surprisingly) wasn't defined. This commit is the first in a series to reconcile the differences between the JSON format in mozsystemmonitor and what `mach build` writes so we can write a single tool to visualize the data. MozReview-Commit-ID: 9mdbKxeV9Ta --HG-- extra : rebase_source : 3aadf5e83c91ba9553595f3da77ed7ca0e4d5541 --- .../mozsystemmonitor/resourcemonitor.py | 48 +++++++++++++------ .../test/test_resource_monitor.py | 7 ++- 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/testing/mozbase/mozsystemmonitor/mozsystemmonitor/resourcemonitor.py b/testing/mozbase/mozsystemmonitor/mozsystemmonitor/resourcemonitor.py index e7a7b52fd019..9efe1572d081 100644 --- a/testing/mozbase/mozsystemmonitor/mozsystemmonitor/resourcemonitor.py +++ b/testing/mozbase/mozsystemmonitor/mozsystemmonitor/resourcemonitor.py @@ -38,9 +38,11 @@ class PsutilStub(object): def virtual_memory(self): return self.svmem(0, 0, 0, 0, 0, 0, 0, 0, 0) + # psutil will raise NotImplementedError if the platform is not supported. try: import psutil + have_psutil = True except Exception: try: # The PsutilStub should get us time intervals, at least @@ -48,6 +50,8 @@ except Exception: except Exception: psutil = None + have_psutil = False + from contextlib import contextmanager def get_disk_io_counters(): @@ -535,7 +539,7 @@ class SystemResourceMonitor(object): The returned dict has the following keys: - version - Integer version number being rendered. Currently 1. + version - Integer version number being rendered. Currently 2. cpu_times_fields - A list of the names of the CPU times fields. io_fields - A list of the names of the I/O fields. virt_fields - A list of the names of the virtual memory fields. @@ -547,6 +551,10 @@ class SystemResourceMonitor(object): phases - A list of dicts describing phases. Each phase looks a lot like an entry from samples (see below). Some phases may not have data recorded against them, so some keys may be None. + overall - A dict representing overall resource usage. This resembles + a sample entry. + system - Contains additional information about the system including + number of processors and amount of memory. Each entry in the sample list is a dict with the following keys: @@ -567,13 +575,14 @@ class SystemResourceMonitor(object): """ o = dict( - version=1, + version=2, cpu_times_fields=list(self._cpu_times_type._fields), io_fields=list(self._io_type._fields), virt_fields=list(self._virt_type._fields), swap_fields=list(self._swap_type._fields), samples=[], phases=[], + system={}, ) def populate_derived(e): @@ -590,6 +599,19 @@ class SystemResourceMonitor(object): e['cpu_times_total'] = sum(e['cpu_times_sum']) + def phase_entry(name, start, end): + e = dict( + name=name, + start=start, + end=end, + duration=end - start, + cpu_percent_cores=self.aggregate_cpu_percent(phase=name), + cpu_times=[list(c) for c in + self.aggregate_cpu_times(phase=name)], + io=list(self.aggregate_io(phase=name)), + ) + populate_derived(e) + return e for m in self.measurements: e = dict( @@ -609,26 +631,24 @@ class SystemResourceMonitor(object): o['start'] = o['samples'][0]['start'] o['end'] = o['samples'][-1]['end'] o['duration'] = o['end'] - o['start'] + o['overall'] = phase_entry(None, o['start'], o['end']) else: o['start'] = None o['end'] = None o['duration'] = None + o['overall'] = None o['events'] = [list(ev) for ev in self.events] for phase, v in self.phases.items(): - e = dict( - name=phase, - start=v[0], - end=v[1], - duration=v[1] - v[0], - cpu_percent_cores=self.aggregate_cpu_percent(phase=phase), - cpu_times=[list(c) for c in - self.aggregate_cpu_times(phase=phase)], - io=list(self.aggregate_io(phase=phase)), - ) + o['phases'].append(phase_entry(phase, v[0], v[1])) - populate_derived(e) - o['phases'].append(e) + if have_psutil: + o['system'].update(dict( + cpu_logical_count=psutil.cpu_count(logical=True), + cpu_physical_count=psutil.cpu_count(logical=False), + swap_total=psutil.swap_memory()[0], + vmem_total=psutil.virtual_memory()[0], + )) return o diff --git a/testing/mozbase/mozsystemmonitor/mozsystemmonitor/test/test_resource_monitor.py b/testing/mozbase/mozsystemmonitor/mozsystemmonitor/test/test_resource_monitor.py index ade7d1a8eb01..1f8ccd155200 100644 --- a/testing/mozbase/mozsystemmonitor/mozsystemmonitor/test/test_resource_monitor.py +++ b/testing/mozbase/mozsystemmonitor/mozsystemmonitor/test/test_resource_monitor.py @@ -169,6 +169,11 @@ class TestResourceMonitor(unittest.TestCase): d = monitor.as_dict() - self.assertEqual(d['version'], 1) + self.assertEqual(d['version'], 2) self.assertEqual(len(d['events']), 2) self.assertEqual(len(d['phases']), 2) + self.assertIn('system', d) + self.assertIsInstance(d['system'], dict) + self.assertIsInstance(d['overall'], dict) + self.assertIn('duration', d['overall']) + self.assertIn('cpu_times', d['overall'])