Bug 1209463 - [test] Use a common logger across suites from |mach test| and other test commands, r=gbrown

In order for |mach test| and |mach mochitest| to log an overall summary,
every test harness invocation they make needs to use the same structured
logger (otherwise the affected suite will be missing from the summary).

MozReview-Commit-ID: 8LJw7r8SItk

--HG--
extra : rebase_source : 1417dce3817bae94ad61a5250065c6cbc35857e4

layout/tools/reftest/runreftest.py

@@ -241,6 +241,10 @@ class RefTest(object):
         if self.log:
             return
 
+        self.log = getattr(options, 'log', None)
+        if self.log:
+            return
+
         mozlog.commandline.log_formatters["tbpl"] = (ReftestFormatter,
                                                      "Reftest specific formatter for the"
                                                      "benefit of legacy log parsers and"

testing/mach_commands.py

@@ -17,6 +17,7 @@ from mach.decorators import (
     CommandArgument,
     CommandProvider,
     Command,
+    SettingsProvider,
 )
 
 from mozbuild.base import MachCommandBase, MachCommandConditions as conditions
@@ -51,6 +52,23 @@ The following test suites and aliases are supported: %s
 TEST_HELP = TEST_HELP.strip()
 
 
+@SettingsProvider
+class TestConfig(object):
+
+    @classmethod
+    def config_settings(cls):
+        from mozlog.commandline import log_formatters
+        from mozlog.structuredlog import log_levels
+        format_desc = "The default format to use when running tests with `mach test`."
+        format_choices = log_formatters.keys()
+        level_desc = "The default log level to use when running tests with `mach test`."
+        level_choices = [l.lower() for l in log_levels]
+        return [
+            ('test.format', 'string', format_desc, 'tbpl', {'choices': format_choices}),
+            ('test.level', 'string', level_desc, 'info', {'choices': level_choices}),
+        ]
+
+
 @CommandProvider
 class Test(MachCommandBase):
     @Command('test', category='testing',
@@ -84,7 +102,11 @@ class Test(MachCommandBase):
         you specify a directory with xpcshell and browser chrome mochitests,
         both harnesses will be invoked.
         """
+        from mozlog.commandline import log_formatters
+        from mozlog.handlers import StreamHandler, LogLevelFilter
+        from mozlog.structuredlog import StructuredLogger
         from moztest.resolve import TestResolver, TEST_FLAVORS, TEST_SUITES
+
         resolver = self._spawn(TestResolver)
         run_suites, run_tests = resolver.resolve_metadata(what)
 
@@ -92,14 +114,23 @@ class Test(MachCommandBase):
             print(UNKNOWN_TEST)
             return 1
 
+        # Create shared logger
+        formatter = log_formatters[self._mach_context.settings['test']['format']][0]()
+
+        level = self._mach_context.settings['test']['level']
+        log = StructuredLogger('mach-test')
+        log.add_handler(StreamHandler(sys.stdout, LogLevelFilter(formatter, level)))
+
         status = None
         for suite_name in run_suites:
             suite = TEST_SUITES[suite_name]
+            kwargs = suite['kwargs']
+            kwargs['log'] = log
 
             if 'mach_command' in suite:
                 res = self._mach_context.commands.dispatch(
                     suite['mach_command'], self._mach_context,
-                    argv=extra_args, **suite['kwargs'])
+                    argv=extra_args, **kwargs)
                 if res:
                     status = res
 
@@ -121,6 +152,7 @@ class Test(MachCommandBase):
                 continue
 
             kwargs = dict(m['kwargs'])
+            kwargs['log'] = log
             kwargs['subsuite'] = subsuite
 
             res = self._mach_context.commands.dispatch(
@@ -129,6 +161,7 @@ class Test(MachCommandBase):
             if res:
                 status = res
 
+        log.shutdown()
         return status
 
 
@@ -142,9 +175,12 @@ class MachCommands(MachCommandBase):
                      'executed.')
     def run_cppunit_test(self, **params):
         from mozlog import commandline
-        log = commandline.setup_logging("cppunittest",
-                                        {},
-                                        {"tbpl": sys.stdout})
+
+        log = params.get('log')
+        if not log:
+            log = commandline.setup_logging("cppunittest",
+                                            {},
+                                            {"tbpl": sys.stdout})
 
         # See if we have crash symbols
         symbols_path = os.path.join(self.distdir, 'crashreporter-symbols')

testing/marionette/mach_commands.py

@@ -57,9 +57,11 @@ def run_marionette(tests, binary=None, topsrcdir=None, **kwargs):
 
     parser.verify_usage(args)
 
-    args.logger = commandline.setup_logging("Marionette Unit Tests",
-                                            args,
-                                            {"mach": sys.stdout})
+    args.logger = kwargs.get('log')
+    if not args.logger:
+        args.logger = commandline.setup_logging("Marionette Unit Tests",
+                                                args,
+                                                {"mach": sys.stdout})
     failed = MarionetteHarness(MarionetteTestRunner, args=vars(args)).run()
     if failed > 0:
         return 1

testing/mochitest/mach_commands.py

@@ -286,6 +286,9 @@ class MachCommands(MachCommandBase):
              parser=setup_argument_parser)
     def run_mochitest_general(self, flavor=None, test_objects=None, resolve_tests=True, **kwargs):
         from mochitest_options import ALL_FLAVORS
+        from mozlog.commandline import log_formatters
+        from mozlog.handlers import StreamHandler, LogLevelFilter
+        from mozlog.structuredlog import StructuredLogger
 
         buildapp = None
         for app in SUPPORTED_APPS:
@@ -304,6 +307,13 @@ class MachCommands(MachCommandBase):
         else:
             flavors = [f for f, v in ALL_FLAVORS.iteritems() if buildapp in v['enabled_apps']]
 
+        if not kwargs.get('log'):
+            # Create shared logger
+            formatter = log_formatters[self._mach_context.settings['test']['format']][0]()
+            level = self._mach_context.settings['test']['level']
+            kwargs['log'] = StructuredLogger('mach-mochitest')
+            kwargs['log'].add_handler(StreamHandler(sys.stdout, LogLevelFilter(formatter, level)))
+
         from mozbuild.controller.building import BuildDriver
         self._ensure_state_subdir_exists('.')
 
@@ -419,7 +429,10 @@ class MachCommands(MachCommandBase):
             if result == -1:
                 break
 
-        # TODO consolidate summaries from all suites
+        # Only shutdown the logger if we created it
+        if kwargs['log'].name == 'mach-mochitest':
+            kwargs['log'].shutdown()
+
         return overall
 
 

testing/mochitest/runrobocop.py

@@ -42,7 +42,7 @@ class RobocopTestRunner(MochitestDesktop):
         """
            Simple one-time initialization.
         """
-        MochitestDesktop.__init__(self, options.flavor, options)
+        MochitestDesktop.__init__(self, options.flavor, vars(options))
 
         self.auto = automation
         self.dm = devmgr

testing/mochitest/runtests.py

@@ -860,10 +860,13 @@ class MochitestDesktop(object):
         self.start_script_kwargs = {}
         self.urlOpts = []
 
-        commandline.log_formatters["tbpl"] = (
-            MochitestFormatter,
-            "Mochitest specific tbpl formatter")
-        self.log = commandline.setup_logging("mochitest", logger_options, {"tbpl": sys.stdout})
+        if logger_options.get('log'):
+            self.log = logger_options['log']
+        else:
+            commandline.log_formatters["tbpl"] = (
+                MochitestFormatter,
+                "Mochitest specific tbpl formatter")
+            self.log = commandline.setup_logging("mochitest", logger_options, {"tbpl": sys.stdout})
 
         self.message_logger = MessageLogger(
                 logger=self.log, buffering=quiet, structured=True)

testing/mochitest/runtestsremote.py

@@ -29,7 +29,7 @@ class MochiRemote(MochitestDesktop):
     logMessages = []
 
     def __init__(self, automation, devmgr, options):
-        MochitestDesktop.__init__(self, options.flavor, options)
+        MochitestDesktop.__init__(self, options.flavor, vars(options))
 
         self._automation = automation
         self._dm = devmgr

testing/web-platform/tests/tools/wptrunner/wptrunner/wptlogging.py

@@ -4,10 +4,16 @@ import threading
 from StringIO import StringIO
 from multiprocessing import Queue
 
-from mozlog import commandline, stdadapter
+from mozlog import commandline, stdadapter, set_default_logger
+from mozlog.structuredlog import StructuredLogger
 
 def setup(args, defaults):
-    logger = commandline.setup_logging("web-platform-tests", args, defaults)
+    logger = args.pop('log', None)
+    if logger:
+        set_default_logger(logger)
+        StructuredLogger._logger_states["web-platform-tests"] = logger._state
+    else:
+        logger = commandline.setup_logging("web-platform-tests", args, defaults)
     setup_stdlib_logger()
 
     for name in args.keys():

testing/xpcshell/mach_commands.py

@@ -248,10 +248,11 @@ class MachCommands(MachCommandBase):
         # case the tree wasn't built with mach).
         self._ensure_state_subdir_exists('.')
 
-        params['log'] = structured.commandline.setup_logging("XPCShellTests",
-                                                             params,
-                                                             {"mach": sys.stdout},
-                                                             {"verbose": True})
+        if not params.get('log'):
+            params['log'] = structured.commandline.setup_logging("XPCShellTests",
+                                                                 params,
+                                                                 {"mach": sys.stdout},
+                                                                 {"verbose": True})
 
         if not params['threadCount']:
             params['threadCount'] = int((cpu_count() * 3) / 2)