Bug 1209463 - [test] Use a common logger across suites from |mach test| and other test commands, r=gbrown

In order for |mach test| and |mach mochitest| to log an overall summary,
every test harness invocation they make needs to use the same structured
logger (otherwise the affected suite will be missing from the summary).

MozReview-Commit-ID: 8LJw7r8SItk

--HG--
extra : rebase_source : 1417dce3817bae94ad61a5250065c6cbc35857e4

layout/tools/reftest/runreftest.py

@@ -241,6 +241,10 @@ class RefTest(object):
         if self.log:
             return
 
+        self.log = getattr(options, 'log', None)
+        if self.log:
+            return
+
         mozlog.commandline.log_formatters["tbpl"] = (ReftestFormatter,
                                                      "Reftest specific formatter for the"
                                                      "benefit of legacy log parsers and"

testing/mach_commands.py

@@ -17,6 +17,7 @@ from mach.decorators import (
     CommandArgument,
     CommandProvider,
     Command,
+    SettingsProvider,
 )
 
 from mozbuild.base import MachCommandBase, MachCommandConditions as conditions
@@ -51,6 +52,23 @@ The following test suites and aliases are supported: %s
 TEST_HELP = TEST_HELP.strip()
 
 
+@SettingsProvider
+class TestConfig(object):
+
+    @classmethod
+    def config_settings(cls):
+        from mozlog.commandline import log_formatters
+        from mozlog.structuredlog import log_levels
+        format_desc = "The default format to use when running tests with `mach test`."
+        format_choices = log_formatters.keys()
+        level_desc = "The default log level to use when running tests with `mach test`."
+        level_choices = [l.lower() for l in log_levels]
+        return [
+            ('test.format', 'string', format_desc, 'tbpl', {'choices': format_choices}),
+            ('test.level', 'string', level_desc, 'info', {'choices': level_choices}),
+        ]
+
+
 @CommandProvider
 class Test(MachCommandBase):
     @Command('test', category='testing',
@@ -84,7 +102,11 @@ class Test(MachCommandBase):
         you specify a directory with xpcshell and browser chrome mochitests,
         both harnesses will be invoked.
         """
+        from mozlog.commandline import log_formatters
+        from mozlog.handlers import StreamHandler, LogLevelFilter
+        from mozlog.structuredlog import StructuredLogger
         from moztest.resolve import TestResolver, TEST_FLAVORS, TEST_SUITES
+
         resolver = self._spawn(TestResolver)
         run_suites, run_tests = resolver.resolve_metadata(what)
 
@@ -92,14 +114,23 @@ class Test(MachCommandBase):
             print(UNKNOWN_TEST)
             return 1
 
+        # Create shared logger
+        formatter = log_formatters[self._mach_context.settings['test']['format']][0]()
+
+        level = self._mach_context.settings['test']['level']
+        log = StructuredLogger('mach-test')
+        log.add_handler(StreamHandler(sys.stdout, LogLevelFilter(formatter, level)))
+
         status = None
         for suite_name in run_suites:
             suite = TEST_SUITES[suite_name]
+            kwargs = suite['kwargs']
+            kwargs['log'] = log
 
             if 'mach_command' in suite:
                 res = self._mach_context.commands.dispatch(
                     suite['mach_command'], self._mach_context,
-                    argv=extra_args, **suite['kwargs'])
+                    argv=extra_args, **kwargs)
                 if res:
                     status = res
 
@@ -121,6 +152,7 @@ class Test(MachCommandBase):
                 continue
 
             kwargs = dict(m['kwargs'])
+            kwargs['log'] = log
             kwargs['subsuite'] = subsuite
 
             res = self._mach_context.commands.dispatch(
@@ -129,6 +161,7 @@ class Test(MachCommandBase):
             if res:
                 status = res
 
+        log.shutdown()
         return status
 
 
@@ -142,6 +175,9 @@ class MachCommands(MachCommandBase):
                      'executed.')
     def run_cppunit_test(self, **params):
         from mozlog import commandline
+
+        log = params.get('log')
+        if not log:
             log = commandline.setup_logging("cppunittest",
                                             {},
                                             {"tbpl": sys.stdout})

testing/marionette/mach_commands.py

@@ -57,6 +57,8 @@ def run_marionette(tests, binary=None, topsrcdir=None, **kwargs):
 
     parser.verify_usage(args)
 
+    args.logger = kwargs.get('log')
+    if not args.logger:
         args.logger = commandline.setup_logging("Marionette Unit Tests",
                                                 args,
                                                 {"mach": sys.stdout})

testing/mochitest/mach_commands.py

@@ -286,6 +286,9 @@ class MachCommands(MachCommandBase):
              parser=setup_argument_parser)
     def run_mochitest_general(self, flavor=None, test_objects=None, resolve_tests=True, **kwargs):
         from mochitest_options import ALL_FLAVORS
+        from mozlog.commandline import log_formatters
+        from mozlog.handlers import StreamHandler, LogLevelFilter
+        from mozlog.structuredlog import StructuredLogger
 
         buildapp = None
         for app in SUPPORTED_APPS:
@@ -304,6 +307,13 @@ class MachCommands(MachCommandBase):
         else:
             flavors = [f for f, v in ALL_FLAVORS.iteritems() if buildapp in v['enabled_apps']]
 
+        if not kwargs.get('log'):
+            # Create shared logger
+            formatter = log_formatters[self._mach_context.settings['test']['format']][0]()
+            level = self._mach_context.settings['test']['level']
+            kwargs['log'] = StructuredLogger('mach-mochitest')
+            kwargs['log'].add_handler(StreamHandler(sys.stdout, LogLevelFilter(formatter, level)))
+
         from mozbuild.controller.building import BuildDriver
         self._ensure_state_subdir_exists('.')
 
@@ -419,7 +429,10 @@ class MachCommands(MachCommandBase):
             if result == -1:
                 break
 
-        # TODO consolidate summaries from all suites
+        # Only shutdown the logger if we created it
+        if kwargs['log'].name == 'mach-mochitest':
+            kwargs['log'].shutdown()
+
         return overall
 
 

testing/mochitest/runrobocop.py

@@ -42,7 +42,7 @@ class RobocopTestRunner(MochitestDesktop):
         """
            Simple one-time initialization.
         """
-        MochitestDesktop.__init__(self, options.flavor, options)
+        MochitestDesktop.__init__(self, options.flavor, vars(options))
 
         self.auto = automation
         self.dm = devmgr

testing/mochitest/runtests.py

@@ -860,6 +860,9 @@ class MochitestDesktop(object):
         self.start_script_kwargs = {}
         self.urlOpts = []
 
+        if logger_options.get('log'):
+            self.log = logger_options['log']
+        else:
             commandline.log_formatters["tbpl"] = (
                 MochitestFormatter,
                 "Mochitest specific tbpl formatter")

testing/mochitest/runtestsremote.py

@@ -29,7 +29,7 @@ class MochiRemote(MochitestDesktop):
     logMessages = []
 
     def __init__(self, automation, devmgr, options):
-        MochitestDesktop.__init__(self, options.flavor, options)
+        MochitestDesktop.__init__(self, options.flavor, vars(options))
 
         self._automation = automation
         self._dm = devmgr

testing/web-platform/tests/tools/wptrunner/wptrunner/wptlogging.py

@@ -4,9 +4,15 @@ import threading
 from StringIO import StringIO
 from multiprocessing import Queue
 
-from mozlog import commandline, stdadapter
+from mozlog import commandline, stdadapter, set_default_logger
+from mozlog.structuredlog import StructuredLogger
 
 def setup(args, defaults):
+    logger = args.pop('log', None)
+    if logger:
+        set_default_logger(logger)
+        StructuredLogger._logger_states["web-platform-tests"] = logger._state
+    else:
         logger = commandline.setup_logging("web-platform-tests", args, defaults)
     setup_stdlib_logger()
 

testing/xpcshell/mach_commands.py

@@ -248,6 +248,7 @@ class MachCommands(MachCommandBase):
         # case the tree wasn't built with mach).
         self._ensure_state_subdir_exists('.')
 
+        if not params.get('log'):
             params['log'] = structured.commandline.setup_logging("XPCShellTests",
                                                                  params,
                                                                  {"mach": sys.stdout},