Bug 1272541 - Expose more data from SystemResourceMonitor.as_dict; r=ahal

We add some system information including processor count
and memory sizes. We also add an "overall" section describing
total resource usage. This (surprisingly) wasn't defined.

This commit is the first in a series to reconcile the differences
between the JSON format in mozsystemmonitor and what
`mach build` writes so we can write a single tool to visualize
the data.

MozReview-Commit-ID: 9mdbKxeV9Ta

--HG--
extra : rebase_source : 3aadf5e83c91ba9553595f3da77ed7ca0e4d5541

testing/mozbase/mozsystemmonitor/mozsystemmonitor/resourcemonitor.py

@@ -38,9 +38,11 @@ class PsutilStub(object):
     def virtual_memory(self):
         return self.svmem(0, 0, 0, 0, 0, 0, 0, 0, 0)
 
+
 # psutil will raise NotImplementedError if the platform is not supported.
 try:
     import psutil
+    have_psutil = True
 except Exception:
     try:
         # The PsutilStub should get us time intervals, at least
@@ -48,6 +50,8 @@ except Exception:
     except Exception:
         psutil = None
 
+    have_psutil = False
+
 from contextlib import contextmanager
 
 def get_disk_io_counters():
@@ -535,7 +539,7 @@ class SystemResourceMonitor(object):
 
         The returned dict has the following keys:
 
-          version - Integer version number being rendered. Currently 1.
+          version - Integer version number being rendered. Currently 2.
           cpu_times_fields - A list of the names of the CPU times fields.
           io_fields - A list of the names of the I/O fields.
           virt_fields - A list of the names of the virtual memory fields.
@@ -547,6 +551,10 @@ class SystemResourceMonitor(object):
           phases - A list of dicts describing phases. Each phase looks a lot
             like an entry from samples (see below). Some phases may not have
             data recorded against them, so some keys may be None.
+          overall - A dict representing overall resource usage. This resembles
+            a sample entry.
+          system - Contains additional information about the system including
+            number of processors and amount of memory.
 
         Each entry in the sample list is a dict with the following keys:
 
@@ -567,13 +575,14 @@ class SystemResourceMonitor(object):
         """
 
         o = dict(
-            version=1,
+            version=2,
             cpu_times_fields=list(self._cpu_times_type._fields),
             io_fields=list(self._io_type._fields),
             virt_fields=list(self._virt_type._fields),
             swap_fields=list(self._swap_type._fields),
             samples=[],
             phases=[],
+            system={},
         )
 
         def populate_derived(e):
@@ -590,6 +599,19 @@ class SystemResourceMonitor(object):
 
                 e['cpu_times_total'] = sum(e['cpu_times_sum'])
 
+        def phase_entry(name, start, end):
+            e = dict(
+                name=name,
+                start=start,
+                end=end,
+                duration=end - start,
+                cpu_percent_cores=self.aggregate_cpu_percent(phase=name),
+                cpu_times=[list(c) for c in
+                           self.aggregate_cpu_times(phase=name)],
+                io=list(self.aggregate_io(phase=name)),
+            )
+            populate_derived(e)
+            return e
 
         for m in self.measurements:
             e = dict(
@@ -609,26 +631,24 @@ class SystemResourceMonitor(object):
             o['start'] = o['samples'][0]['start']
             o['end'] = o['samples'][-1]['end']
             o['duration'] = o['end'] - o['start']
+            o['overall'] = phase_entry(None, o['start'], o['end'])
         else:
             o['start'] = None
             o['end'] = None
             o['duration'] = None
+            o['overall'] = None
 
         o['events'] = [list(ev) for ev in self.events]
 
         for phase, v in self.phases.items():
-            e = dict(
-                name=phase,
-                start=v[0],
-                end=v[1],
-                duration=v[1] - v[0],
-                cpu_percent_cores=self.aggregate_cpu_percent(phase=phase),
-                cpu_times=[list(c) for c in
-                    self.aggregate_cpu_times(phase=phase)],
-                io=list(self.aggregate_io(phase=phase)),
-            )
+            o['phases'].append(phase_entry(phase, v[0], v[1]))
 
-            populate_derived(e)
-            o['phases'].append(e)
+        if have_psutil:
+            o['system'].update(dict(
+                cpu_logical_count=psutil.cpu_count(logical=True),
+                cpu_physical_count=psutil.cpu_count(logical=False),
+                swap_total=psutil.swap_memory()[0],
+                vmem_total=psutil.virtual_memory()[0],
+            ))
 
         return o

testing/mozbase/mozsystemmonitor/mozsystemmonitor/test/test_resource_monitor.py

@@ -169,6 +169,11 @@ class TestResourceMonitor(unittest.TestCase):
 
         d = monitor.as_dict()
 
-        self.assertEqual(d['version'], 1)
+        self.assertEqual(d['version'], 2)
         self.assertEqual(len(d['events']), 2)
         self.assertEqual(len(d['phases']), 2)
+        self.assertIn('system', d)
+        self.assertIsInstance(d['system'], dict)
+        self.assertIsInstance(d['overall'], dict)
+        self.assertIn('duration', d['overall'])
+        self.assertIn('cpu_times', d['overall'])