Bug 873720 - Part 2: Move mach's build monitoring logic into mozbuild core; r=ted

python/mozbuild/mozbuild/controller/building.py

@@ -0,0 +1,147 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import getpass
+import os
+import sys
+import time
+
+from collections import namedtuple
+
+try:
+    import psutil
+except ImportError:
+    psutil = None
+
+from ..compilation.warnings import (
+    WarningsCollector,
+    WarningsDatabase,
+)
+
+
+BuildOutputResult = namedtuple('BuildOutputResult',
+    ('warning', 'state_changed', 'for_display'))
+
+
+class BuildMonitor(object):
+    """Monitors the output of the build."""
+
+    def __init__(self, topobjdir, warnings_path):
+        """Create a new monitor.
+
+        warnings_path is a path of a warnings database to use.
+        """
+        self._warnings_path = warnings_path
+
+        self.warnings_database = WarningsDatabase()
+        if os.path.exists(warnings_path):
+            try:
+                self.warnings_database.load_from_file(warnings_path)
+            except ValueError:
+                os.remove(warnings_path)
+
+        self._warnings_collector = WarningsCollector(
+            database=self.warnings_database, objdir=topobjdir)
+
+    def start(self):
+        """Record the start of the build."""
+        self.start_time = time.time()
+        self._finder_start_cpu = self._get_finder_cpu_usage()
+
+    def on_line(self, line):
+        """Consume a line of output from the build system.
+
+        This will parse the line for state and determine whether more action is
+        needed.
+
+        Returns a BuildOutputResult instance.
+
+        In this named tuple, warning will be an object describing a new parsed
+        warning. Otherwise it will be None.
+
+        state_changed indicates whether the build system changed state with
+        this line. If the build system changed state, the caller may want to
+        query this instance for the current state in order to update UI, etc.
+
+        for_display is a boolean indicating whether the line is relevant to the
+        user. This is typically used to filter whether the line should be
+        presented to the user.
+        """
+        if line.startswith('BUILDSTATUS'):
+            return BuildOutputResult(None, True, False)
+
+        try:
+            warning = self._warnings_collector.process_line(line)
+        except:
+            pass
+
+        return BuildOutputResult(warning, False, True)
+
+    def finish(self):
+        """Record the end of the build."""
+        self.end_time = time.time()
+        self._finder_end_cpu = self._get_finder_cpu_usage()
+        self.elapsed = self.end_time - self.start_time
+
+        self.warnings_database.prune()
+        self.warnings_database.save_to_file(self._warnings_path)
+
+    def _get_finder_cpu_usage(self):
+        """Obtain the CPU usage of the Finder app on OS X.
+
+        This is used to detect high CPU usage.
+        """
+        if not sys.platform.startswith('darwin'):
+            return None
+
+        if not psutil:
+            return None
+
+        for proc in psutil.process_iter():
+            if proc.name != 'Finder':
+                continue
+
+            if proc.username != getpass.getuser():
+                continue
+
+            # Try to isolate system finder as opposed to other "Finder"
+            # processes.
+            if not proc.exe.endswith('CoreServices/Finder.app/Contents/MacOS/Finder'):
+                continue
+
+            return proc.get_cpu_times()
+
+        return None
+
+    def have_high_finder_usage(self):
+        """Determine whether there was high Finder CPU usage during the build.
+
+        Returns True if there was high Finder CPU usage, False if there wasn't,
+        or None if there is nothing to report.
+        """
+        if not self._finder_start_cpu:
+            return None, None
+
+        # We only measure if the measured range is sufficiently long.
+        if self.elapsed < 15:
+            return None, None
+
+        if not self._finder_end_cpu:
+            return None, None
+
+        start = self._finder_start_cpu
+        end = self._finder_end_cpu
+
+        start_total = start.user + start.system
+        end_total = end.user + end.system
+
+        cpu_seconds = end_total - start_total
+
+        # If Finder used more than 25% of 1 core during the build, report an
+        # error.
+        finder_percent = cpu_seconds / self.elapsed * 100
+
+        return finder_percent > 25, finder_percent

python/mozbuild/mozbuild/mach_commands.py

@@ -4,12 +4,9 @@
 
 from __future__ import print_function, unicode_literals
 
-import getpass
 import logging
 import operator
 import os
-import sys
-import time
 
 from mach.decorators import (
     CommandArgument,
@@ -57,38 +54,23 @@ class Build(MachCommandBase):
     @CommandArgument('-v', '--verbose', action='store_true',
         help='Verbose output for what commands the build is running.')
     def build(self, what=None, disable_extra_make_dependencies=None, jobs=0, verbose=False):
-        # This code is only meant to be temporary until the more robust tree
+        from mozbuild.controller.building import BuildMonitor
-        # building code in bug 780329 lands.
-        from mozbuild.compilation.warnings import WarningsCollector
-        from mozbuild.compilation.warnings import WarningsDatabase
         from mozbuild.util import resolve_target_to_make
 
         warnings_path = self._get_state_filename('warnings.json')
-        warnings_database = WarningsDatabase()
+        monitor = BuildMonitor(self.topobjdir, warnings_path)
-
-        if os.path.exists(warnings_path):
-            try:
-                warnings_database.load_from_file(warnings_path)
-            except ValueError:
-                os.remove(warnings_path)
-
-        warnings_collector = WarningsCollector(database=warnings_database,
-            objdir=self.topobjdir)
 
         def on_line(line):
-            try:
+            warning, state_changed, relevant = monitor.on_line(line)
-                warning = warnings_collector.process_line(line)
-                if warning:
-                    self.log(logging.INFO, 'compiler_warning', warning,
-                        'Warning: {flag} in {filename}: {message}')
-            except:
-                # This will get logged in the more robust implementation.
-                pass
 
-            self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
+            if warning:
+                self.log(logging.INFO, 'compiler_warning', warning,
+                    'Warning: {flag} in {filename}: {message}')
 
-        finder_start_cpu = self._get_finder_cpu_usage()
+            if relevant:
-        time_start = time.time()
+                self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
+
+        monitor.start()
 
         if what:
             top_make = os.path.join(self.topobjdir, 'Makefile')
@@ -141,17 +123,15 @@ class Build(MachCommandBase):
                 silent=not verbose)
 
             self.log(logging.WARNING, 'warning_summary',
-                {'count': len(warnings_collector.database)},
+                {'count': len(monitor.warnings_database)},
                 '{count} compiler warnings present.')
 
-        warnings_database.prune()
+        monitor.finish()
-        warnings_database.save_to_file(warnings_path)
+        high_finder, finder_percent = monitor.have_high_finder_usage()
+        if high_finder:
+            print(FINDER_SLOW_MESSAGE % finder_percent)
 
-        time_end = time.time()
+        long_build = monitor.elapsed > 600
-        time_elapsed = time_end - time_start
-        self._handle_finder_cpu_usage(time_elapsed, finder_start_cpu)
-
-        long_build = time_elapsed > 600
 
         if status:
             return status
@@ -176,61 +156,6 @@ class Build(MachCommandBase):
 
         return status
 
-    def _get_finder_cpu_usage(self):
-        """Obtain the CPU usage of the Finder app on OS X.
-
-        This is used to detect high CPU usage.
-        """
-        if not sys.platform.startswith('darwin'):
-            return None
-
-        try:
-            import psutil
-        except ImportError:
-            return None
-
-        for proc in psutil.process_iter():
-            if proc.name != 'Finder':
-                continue
-
-            if proc.username != getpass.getuser():
-                continue
-
-            # Try to isolate system finder as opposed to other "Finder"
-            # processes.
-            if not proc.exe.endswith('CoreServices/Finder.app/Contents/MacOS/Finder'):
-                continue
-
-            return proc.get_cpu_times()
-
-        return None
-
-    def _handle_finder_cpu_usage(self, elapsed, start):
-        if not start:
-            return
-
-        # We only measure if the measured range is sufficiently long.
-        if elapsed < 15:
-            return
-
-        end = self._get_finder_cpu_usage()
-        if not end:
-            return
-
-        start_total = start.user + start.system
-        end_total = end.user + end.system
-
-        cpu_seconds = end_total - start_total
-
-        # If Finder used more than 25% of 1 core during the build, report an
-        # error.
-        finder_percent = cpu_seconds / elapsed * 100
-        if finder_percent < 25:
-            return
-
-        print(FINDER_SLOW_MESSAGE % finder_percent)
-
-
     @Command('configure', category='build',
         description='Configure the tree (run configure and config.status).')
     def configure(self):