From 3e4692475991f7af62fcf9422eb4c026eff62827 Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Thu, 23 Nov 2017 13:04:19 -0500 Subject: [PATCH] Bug 1209463 - [test] Use a common logger across suites from |mach test| and other test commands, r=gbrown In order for |mach test| and |mach mochitest| to log an overall summary, every test harness invocation they make needs to use the same structured logger (otherwise the affected suite will be missing from the summary). MozReview-Commit-ID: 8LJw7r8SItk --HG-- extra : rebase_source : 1417dce3817bae94ad61a5250065c6cbc35857e4 --- layout/tools/reftest/runreftest.py | 4 ++ testing/mach_commands.py | 44 +++++++++++++++++-- testing/marionette/mach_commands.py | 8 ++-- testing/mochitest/mach_commands.py | 15 ++++++- testing/mochitest/runrobocop.py | 2 +- testing/mochitest/runtests.py | 11 +++-- testing/mochitest/runtestsremote.py | 2 +- .../tools/wptrunner/wptrunner/wptlogging.py | 10 ++++- testing/xpcshell/mach_commands.py | 9 ++-- 9 files changed, 85 insertions(+), 20 deletions(-) diff --git a/layout/tools/reftest/runreftest.py b/layout/tools/reftest/runreftest.py index ff43445c4a8a..105454605650 100644 --- a/layout/tools/reftest/runreftest.py +++ b/layout/tools/reftest/runreftest.py @@ -241,6 +241,10 @@ class RefTest(object): if self.log: return + self.log = getattr(options, 'log', None) + if self.log: + return + mozlog.commandline.log_formatters["tbpl"] = (ReftestFormatter, "Reftest specific formatter for the" "benefit of legacy log parsers and" diff --git a/testing/mach_commands.py b/testing/mach_commands.py index 2bb0f4385820..7d389a8cbe50 100644 --- a/testing/mach_commands.py +++ b/testing/mach_commands.py @@ -17,6 +17,7 @@ from mach.decorators import ( CommandArgument, CommandProvider, Command, + SettingsProvider, ) from mozbuild.base import MachCommandBase, MachCommandConditions as conditions @@ -51,6 +52,23 @@ The following test suites and aliases are supported: %s TEST_HELP = TEST_HELP.strip() +@SettingsProvider +class TestConfig(object): + + @classmethod + def config_settings(cls): + from mozlog.commandline import log_formatters + from mozlog.structuredlog import log_levels + format_desc = "The default format to use when running tests with `mach test`." + format_choices = log_formatters.keys() + level_desc = "The default log level to use when running tests with `mach test`." + level_choices = [l.lower() for l in log_levels] + return [ + ('test.format', 'string', format_desc, 'tbpl', {'choices': format_choices}), + ('test.level', 'string', level_desc, 'info', {'choices': level_choices}), + ] + + @CommandProvider class Test(MachCommandBase): @Command('test', category='testing', @@ -84,7 +102,11 @@ class Test(MachCommandBase): you specify a directory with xpcshell and browser chrome mochitests, both harnesses will be invoked. """ + from mozlog.commandline import log_formatters + from mozlog.handlers import StreamHandler, LogLevelFilter + from mozlog.structuredlog import StructuredLogger from moztest.resolve import TestResolver, TEST_FLAVORS, TEST_SUITES + resolver = self._spawn(TestResolver) run_suites, run_tests = resolver.resolve_metadata(what) @@ -92,14 +114,23 @@ class Test(MachCommandBase): print(UNKNOWN_TEST) return 1 + # Create shared logger + formatter = log_formatters[self._mach_context.settings['test']['format']][0]() + + level = self._mach_context.settings['test']['level'] + log = StructuredLogger('mach-test') + log.add_handler(StreamHandler(sys.stdout, LogLevelFilter(formatter, level))) + status = None for suite_name in run_suites: suite = TEST_SUITES[suite_name] + kwargs = suite['kwargs'] + kwargs['log'] = log if 'mach_command' in suite: res = self._mach_context.commands.dispatch( suite['mach_command'], self._mach_context, - argv=extra_args, **suite['kwargs']) + argv=extra_args, **kwargs) if res: status = res @@ -121,6 +152,7 @@ class Test(MachCommandBase): continue kwargs = dict(m['kwargs']) + kwargs['log'] = log kwargs['subsuite'] = subsuite res = self._mach_context.commands.dispatch( @@ -129,6 +161,7 @@ class Test(MachCommandBase): if res: status = res + log.shutdown() return status @@ -142,9 +175,12 @@ class MachCommands(MachCommandBase): 'executed.') def run_cppunit_test(self, **params): from mozlog import commandline - log = commandline.setup_logging("cppunittest", - {}, - {"tbpl": sys.stdout}) + + log = params.get('log') + if not log: + log = commandline.setup_logging("cppunittest", + {}, + {"tbpl": sys.stdout}) # See if we have crash symbols symbols_path = os.path.join(self.distdir, 'crashreporter-symbols') diff --git a/testing/marionette/mach_commands.py b/testing/marionette/mach_commands.py index d1fcfaa2f66b..ebef5de00207 100644 --- a/testing/marionette/mach_commands.py +++ b/testing/marionette/mach_commands.py @@ -57,9 +57,11 @@ def run_marionette(tests, binary=None, topsrcdir=None, **kwargs): parser.verify_usage(args) - args.logger = commandline.setup_logging("Marionette Unit Tests", - args, - {"mach": sys.stdout}) + args.logger = kwargs.get('log') + if not args.logger: + args.logger = commandline.setup_logging("Marionette Unit Tests", + args, + {"mach": sys.stdout}) failed = MarionetteHarness(MarionetteTestRunner, args=vars(args)).run() if failed > 0: return 1 diff --git a/testing/mochitest/mach_commands.py b/testing/mochitest/mach_commands.py index b00cd5c5a11b..ee2ba67477f3 100644 --- a/testing/mochitest/mach_commands.py +++ b/testing/mochitest/mach_commands.py @@ -286,6 +286,9 @@ class MachCommands(MachCommandBase): parser=setup_argument_parser) def run_mochitest_general(self, flavor=None, test_objects=None, resolve_tests=True, **kwargs): from mochitest_options import ALL_FLAVORS + from mozlog.commandline import log_formatters + from mozlog.handlers import StreamHandler, LogLevelFilter + from mozlog.structuredlog import StructuredLogger buildapp = None for app in SUPPORTED_APPS: @@ -304,6 +307,13 @@ class MachCommands(MachCommandBase): else: flavors = [f for f, v in ALL_FLAVORS.iteritems() if buildapp in v['enabled_apps']] + if not kwargs.get('log'): + # Create shared logger + formatter = log_formatters[self._mach_context.settings['test']['format']][0]() + level = self._mach_context.settings['test']['level'] + kwargs['log'] = StructuredLogger('mach-mochitest') + kwargs['log'].add_handler(StreamHandler(sys.stdout, LogLevelFilter(formatter, level))) + from mozbuild.controller.building import BuildDriver self._ensure_state_subdir_exists('.') @@ -419,7 +429,10 @@ class MachCommands(MachCommandBase): if result == -1: break - # TODO consolidate summaries from all suites + # Only shutdown the logger if we created it + if kwargs['log'].name == 'mach-mochitest': + kwargs['log'].shutdown() + return overall diff --git a/testing/mochitest/runrobocop.py b/testing/mochitest/runrobocop.py index 8b913247e193..fe0c3ce9176b 100644 --- a/testing/mochitest/runrobocop.py +++ b/testing/mochitest/runrobocop.py @@ -42,7 +42,7 @@ class RobocopTestRunner(MochitestDesktop): """ Simple one-time initialization. """ - MochitestDesktop.__init__(self, options.flavor, options) + MochitestDesktop.__init__(self, options.flavor, vars(options)) self.auto = automation self.dm = devmgr diff --git a/testing/mochitest/runtests.py b/testing/mochitest/runtests.py index d226dfd844b0..00bbab14d25b 100644 --- a/testing/mochitest/runtests.py +++ b/testing/mochitest/runtests.py @@ -860,10 +860,13 @@ class MochitestDesktop(object): self.start_script_kwargs = {} self.urlOpts = [] - commandline.log_formatters["tbpl"] = ( - MochitestFormatter, - "Mochitest specific tbpl formatter") - self.log = commandline.setup_logging("mochitest", logger_options, {"tbpl": sys.stdout}) + if logger_options.get('log'): + self.log = logger_options['log'] + else: + commandline.log_formatters["tbpl"] = ( + MochitestFormatter, + "Mochitest specific tbpl formatter") + self.log = commandline.setup_logging("mochitest", logger_options, {"tbpl": sys.stdout}) self.message_logger = MessageLogger( logger=self.log, buffering=quiet, structured=True) diff --git a/testing/mochitest/runtestsremote.py b/testing/mochitest/runtestsremote.py index 5c121ff0c277..3fa4046f0776 100644 --- a/testing/mochitest/runtestsremote.py +++ b/testing/mochitest/runtestsremote.py @@ -29,7 +29,7 @@ class MochiRemote(MochitestDesktop): logMessages = [] def __init__(self, automation, devmgr, options): - MochitestDesktop.__init__(self, options.flavor, options) + MochitestDesktop.__init__(self, options.flavor, vars(options)) self._automation = automation self._dm = devmgr diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptlogging.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptlogging.py index 4d320617859c..003a6c6113cb 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptlogging.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptlogging.py @@ -4,10 +4,16 @@ import threading from StringIO import StringIO from multiprocessing import Queue -from mozlog import commandline, stdadapter +from mozlog import commandline, stdadapter, set_default_logger +from mozlog.structuredlog import StructuredLogger def setup(args, defaults): - logger = commandline.setup_logging("web-platform-tests", args, defaults) + logger = args.pop('log', None) + if logger: + set_default_logger(logger) + StructuredLogger._logger_states["web-platform-tests"] = logger._state + else: + logger = commandline.setup_logging("web-platform-tests", args, defaults) setup_stdlib_logger() for name in args.keys(): diff --git a/testing/xpcshell/mach_commands.py b/testing/xpcshell/mach_commands.py index e78abec0accb..18f62eedc5ea 100644 --- a/testing/xpcshell/mach_commands.py +++ b/testing/xpcshell/mach_commands.py @@ -248,10 +248,11 @@ class MachCommands(MachCommandBase): # case the tree wasn't built with mach). self._ensure_state_subdir_exists('.') - params['log'] = structured.commandline.setup_logging("XPCShellTests", - params, - {"mach": sys.stdout}, - {"verbose": True}) + if not params.get('log'): + params['log'] = structured.commandline.setup_logging("XPCShellTests", + params, + {"mach": sys.stdout}, + {"verbose": True}) if not params['threadCount']: params['threadCount'] = int((cpu_count() * 3) / 2)