From ac95649c98d4d6b09fbd5e2a7f57f25a45b76d44 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Thu, 19 Apr 2018 19:19:31 +0200 Subject: [PATCH] Bug 1431753 - Add a test-coverage test suite, similar to test-verify, that runs tests in isolation in coverage mode. r=gbrown,jmaher --HG-- extra : rebase_source : 8e151ddc81d528f85e2a4acbd7623935d4246b4f --- python/mozbuild/mozbuild/schedules.py | 2 + taskcluster/ci/test/misc.yml | 39 ++++ taskcluster/ci/test/test-sets.yml | 4 + taskcluster/ci/test/web-platform.yml | 19 ++ taskcluster/taskgraph/transforms/tests.py | 2 +- .../configs/web_platform_tests/prod_config.py | 2 +- .../web_platform_tests/prod_config_windows.py | 2 +- .../prod_config_windows_taskcluster.py | 2 +- .../mozilla/testing/codecoverage.py | 184 ++++++++++++------ .../mozilla/testing/per_test_base.py | 164 ++++++++-------- .../mozilla/testing/verify_tools.py | 67 +++++++ .../scripts/android_emulator_unittest.py | 38 ++-- .../mozharness/scripts/desktop_unittest.py | 61 ++++-- .../mozharness/scripts/web_platform_tests.py | 74 ++++--- 14 files changed, 444 insertions(+), 216 deletions(-) create mode 100644 testing/mozharness/mozharness/mozilla/testing/verify_tools.py diff --git a/python/mozbuild/mozbuild/schedules.py b/python/mozbuild/mozbuild/schedules.py index 35afa289fe6f..e64aa7828006 100644 --- a/python/mozbuild/mozbuild/schedules.py +++ b/python/mozbuild/mozbuild/schedules.py @@ -24,6 +24,8 @@ INCLUSIVE_COMPONENTS = [ 'jittest', 'test-verify', 'test-verify-wpt', + 'test-coverage', + 'test-coverage-wpt', 'jsreftest', ] INCLUSIVE_COMPONENTS = sorted(INCLUSIVE_COMPONENTS) diff --git a/taskcluster/ci/test/misc.yml b/taskcluster/ci/test/misc.yml index 6f46cf1f85f7..52d8951518e4 100644 --- a/taskcluster/ci/test/misc.yml +++ b/taskcluster/ci/test/misc.yml @@ -97,3 +97,42 @@ test-verify: no-read-buildbot-config: true extra-options: - --verify + +test-coverage: + description: "Per-test coverage" + suite: test-coverage + treeherder-symbol: TC + loopback-video: true + instance-size: default + max-run-time: 10800 + allow-software-gl-layers: false + run-on-projects: + by-test-platform: + # do not run on beta or release: usually just confirms earlier results + linux64-ccov/.*: ['trunk', 'try'] + windows10-64-ccov/debug: ['trunk', 'try'] + default: [] + tier: + by-test-platform: + windows10-64-asan.*: 3 + default: 2 + mozharness: + script: + by-test-platform: + android.*: android_emulator_unittest.py + default: desktop_unittest.py + config: + by-test-platform: + android.*: + - android/android_common.py + - android/androidarm_4_3.py + linux.*: + - unittests/linux_unittest.py + - remove_executables.py + macosx.*: + - unittests/mac_unittest.py + windows.*: + - unittests/win_taskcluster_unittest.py + no-read-buildbot-config: true + extra-options: + - --per-test-coverage diff --git a/taskcluster/ci/test/test-sets.yml b/taskcluster/ci/test/test-sets.yml index f960fca28a25..4c0a7754df18 100644 --- a/taskcluster/ci/test/test-sets.yml +++ b/taskcluster/ci/test/test-sets.yml @@ -37,6 +37,8 @@ common-tests: - reftest - reftest-no-accel - telemetry-tests-client + - test-coverage + - test-coverage-wpt - test-verify - test-verify-wpt - xpcshell @@ -185,6 +187,8 @@ windows-tests: - mochitest-webgl - reftest - reftest-no-accel + - test-coverage + - test-coverage-wpt - test-verify - test-verify-wpt - web-platform-tests diff --git a/taskcluster/ci/test/web-platform.yml b/taskcluster/ci/test/web-platform.yml index 4e953fa7365e..f552294753f4 100644 --- a/taskcluster/ci/test/web-platform.yml +++ b/taskcluster/ci/test/web-platform.yml @@ -163,3 +163,22 @@ test-verify-wpt: mozharness: extra-options: - --verify + +test-coverage-wpt: + description: "Per web-platform test coverage" + suite: test-coverage-wpt + treeherder-symbol: TCw + max-run-time: 10800 + run-on-projects: + by-test-platform: + # do not run on beta or release: usually just confirms earlier results + linux64-ccov/.*: ['trunk', 'try'] + windows10-64-ccov/debug: ['trunk', 'try'] + default: [] + tier: + by-test-platform: + windows10-64-asan.*: 3 + default: 2 + mozharness: + extra-options: + - --per-test-coverage diff --git a/taskcluster/taskgraph/transforms/tests.py b/taskcluster/taskgraph/transforms/tests.py index 9a67df843ceb..72169ad88b68 100644 --- a/taskcluster/taskgraph/transforms/tests.py +++ b/taskcluster/taskgraph/transforms/tests.py @@ -673,7 +673,7 @@ def handle_suite_category(config, tests): script = test['mozharness']['script'] category_arg = None - if suite == 'test-verify': + if suite == 'test-verify' or suite == 'test-coverage': pass elif script == 'android_emulator_unittest.py': category_arg = '--test-suite' diff --git a/testing/mozharness/configs/web_platform_tests/prod_config.py b/testing/mozharness/configs/web_platform_tests/prod_config.py index fe1c905d9957..a195d8cf928d 100644 --- a/testing/mozharness/configs/web_platform_tests/prod_config.py +++ b/testing/mozharness/configs/web_platform_tests/prod_config.py @@ -36,6 +36,6 @@ config = { # this would normally be in "exes", but "exes" is clobbered by remove_executables "geckodriver": "%(abs_test_bin_dir)s/geckodriver", - "verify_category": "web-platform", + "per_test_category": "web-platform", } diff --git a/testing/mozharness/configs/web_platform_tests/prod_config_windows.py b/testing/mozharness/configs/web_platform_tests/prod_config_windows.py index 3cb77adc5896..576a74d0dc84 100644 --- a/testing/mozharness/configs/web_platform_tests/prod_config_windows.py +++ b/testing/mozharness/configs/web_platform_tests/prod_config_windows.py @@ -42,5 +42,5 @@ config = { "download_minidump_stackwalk": True, - "verify_category": "web-platform", + "per_test_category": "web-platform", } diff --git a/testing/mozharness/configs/web_platform_tests/prod_config_windows_taskcluster.py b/testing/mozharness/configs/web_platform_tests/prod_config_windows_taskcluster.py index b7d262986caa..ac5c69dca30a 100644 --- a/testing/mozharness/configs/web_platform_tests/prod_config_windows_taskcluster.py +++ b/testing/mozharness/configs/web_platform_tests/prod_config_windows_taskcluster.py @@ -64,5 +64,5 @@ config = { "download_minidump_stackwalk": True, - "verify_category": "web-platform", + "per_test_category": "web-platform", } diff --git a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py index 68089c151bc5..b76984e06b6c 100644 --- a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py +++ b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py @@ -14,6 +14,8 @@ from mozharness.base.script import ( PreScriptAction, PostScriptAction, ) +from mozharness.mozilla.testing.per_test_base import SingleTestMixin + _here = os.path.abspath(os.path.dirname(__file__)) _tooltool_path = os.path.normpath(os.path.join(_here, '..', '..', '..', @@ -27,6 +29,12 @@ code_coverage_config_options = [ "default": False, "help": "Whether gcov c++ code coverage should be run." }], + [["--per-test-coverage"], + {"action": "store_true", + "dest": "per_test_coverage", + "default": False, + "help": "Whether per-test coverage should be collected." + }], [["--disable-ccov-upload"], {"action": "store_true", "dest": "disable_ccov_upload", @@ -42,7 +50,7 @@ code_coverage_config_options = [ ] -class CodeCoverageMixin(object): +class CodeCoverageMixin(SingleTestMixin): """ Mixin for setting GCOV_PREFIX during test execution, packaging up the resulting .gcda files and uploading them to blobber. @@ -51,6 +59,9 @@ class CodeCoverageMixin(object): jsvm_dir = None prefix = None + def __init__(self): + super(CodeCoverageMixin, self).__init__() + @property def code_coverage_enabled(self): try: @@ -62,6 +73,13 @@ class CodeCoverageMixin(object): except (AttributeError, KeyError, TypeError): return False + @property + def per_test_coverage(self): + try: + return bool(self.config.get('per_test_coverage')) + except (AttributeError, KeyError, TypeError): + return False + @property def ccov_upload_disabled(self): try: @@ -82,8 +100,8 @@ class CodeCoverageMixin(object): except (AttributeError, KeyError, TypeError): return False - @PreScriptAction('run-tests') - def _set_gcov_prefix(self, action): + @PostScriptAction('download-and-extract') + def setup_coverage_tools(self, action, success=None): if not self.code_coverage_enabled: return @@ -98,14 +116,6 @@ class CodeCoverageMixin(object): os.environ['GCOV_PREFIX_STRIP'] = str(strip_count) - # Set the GCOV directory. - self.gcov_dir = tempfile.mkdtemp() - os.environ['GCOV_PREFIX'] = self.gcov_dir - - # Set JSVM directory. - self.jsvm_dir = tempfile.mkdtemp() - os.environ['JS_CODE_COVERAGE_OUTPUT_DIR'] = self.jsvm_dir - # Install grcov on the test machine # Get the path to the build machines gcno files. self.url_to_gcno = self.query_build_dir_url('target.code-coverage-gcno.zip') @@ -133,6 +143,97 @@ class CodeCoverageMixin(object): with tarfile.open(os.path.join(self.grcov_dir, tar_file)) as tar: tar.extractall(self.grcov_dir) + # Download the gcno archive from the build machine. + self.download_file(self.url_to_gcno, parent_dir=self.grcov_dir) + + # Download the chrome-map.json file from the build machine. + self.download_file(self.url_to_chrome_map, parent_dir=self.grcov_dir) + + @PostScriptAction('download-and-extract') + def find_tests_for_coverage(self, action, success=None): + """ + For each file modified on this push, determine if the modified file + is a test, by searching test manifests. Populate self.verify_suites + with test files, organized by suite. + + This depends on test manifests, so can only run after test zips have + been downloaded and extracted. + """ + if not self.per_test_coverage: + return + + self.find_modified_tests() + # TODO: Add tests that haven't been run for a while (a week? N pushes?) + + @property + def coverage_args(self): + return [] + + def set_coverage_env(self, env): + # Set the GCOV directory. + gcov_dir = tempfile.mkdtemp() + env['GCOV_PREFIX'] = gcov_dir + + # Set JSVM directory. + jsvm_dir = tempfile.mkdtemp() + env['JS_CODE_COVERAGE_OUTPUT_DIR'] = jsvm_dir + + return (gcov_dir, jsvm_dir) + + @PreScriptAction('run-tests') + def _set_gcov_prefix(self, action): + if not self.code_coverage_enabled: + return + + if self.per_test_coverage: + return + + self.gcov_dir, self.jsvm_dir = self.set_coverage_env(os.environ) + + def parse_coverage_artifacts(self, gcov_dir, jsvm_dir): + jsvm_output_file = 'jsvm_lcov_output.info' + grcov_output_file = 'grcov_lcov_output.info' + + dirs = self.query_abs_dirs() + + # Zip gcda files (will be given in input to grcov). + file_path_gcda = os.path.join(os.getcwd(), 'code-coverage-gcda.zip') + self.run_command(['zip', '-q', '-0', '-r', file_path_gcda, '.'], cwd=gcov_dir) + + sys.path.append(dirs['abs_test_install_dir']) + sys.path.append(os.path.join(dirs['abs_test_install_dir'], 'mozbuild/codecoverage')) + + from lcov_rewriter import LcovFileRewriter + jsvm_files = [os.path.join(jsvm_dir, e) for e in os.listdir(jsvm_dir)] + rewriter = LcovFileRewriter(os.path.join(self.grcov_dir, 'chrome-map.json')) + rewriter.rewrite_files(jsvm_files, jsvm_output_file, '') + + # Run grcov on the zipped .gcno and .gcda files. + grcov_command = [ + os.path.join(self.grcov_dir, 'grcov'), + '-t', 'lcov', + '-p', self.prefix, + '--ignore-dir', 'gcc*', + '--ignore-dir', 'vs2017_*', + os.path.join(self.grcov_dir, 'target.code-coverage-gcno.zip'), file_path_gcda + ] + + if mozinfo.os == 'win': + grcov_command += ['--llvm'] + + # 'grcov_output' will be a tuple, the first variable is the path to the lcov output, + # the other is the path to the standard error output. + tmp_output_file, _ = self.get_output_from_command( + grcov_command, + silent=True, + save_tmpfiles=True, + return_type='files', + throw_exception=True, + ) + shutil.move(tmp_output_file, grcov_output_file) + + return grcov_output_file, jsvm_output_file + @PostScriptAction('run-tests') def _package_coverage_data(self, action, success=None): if self.jsd_code_coverage_enabled: @@ -156,65 +257,26 @@ class CodeCoverageMixin(object): if not self.code_coverage_enabled: return + if self.per_test_coverage: + return + del os.environ['GCOV_PREFIX_STRIP'] del os.environ['GCOV_PREFIX'] del os.environ['JS_CODE_COVERAGE_OUTPUT_DIR'] if not self.ccov_upload_disabled: + grcov_output_file, jsvm_output_file = self.parse_coverage_artifacts(self.gcov_dir, self.jsvm_dir) + dirs = self.query_abs_dirs() - # Zip gcda files (will be given in input to grcov). - file_path_gcda = os.path.join(os.getcwd(), 'code-coverage-gcda.zip') - self.run_command(['zip', '-q', '-0', '-r', file_path_gcda, '.'], cwd=self.gcov_dir) - - sys.path.append(dirs['abs_test_install_dir']) - sys.path.append(os.path.join(dirs['abs_test_install_dir'], 'mozbuild/codecoverage')) - - # Download the chrome-map.json file from the build machine. - self.download_file(self.url_to_chrome_map) - - from lcov_rewriter import LcovFileRewriter - jsvm_files = [os.path.join(self.jsvm_dir, e) for e in os.listdir(self.jsvm_dir)] - rewriter = LcovFileRewriter('chrome-map.json') - rewriter.rewrite_files(jsvm_files, 'jsvm_lcov_output.info', '') - - # Package JSVM coverage data. - file_path_jsvm = os.path.join(dirs['abs_blob_upload_dir'], 'code-coverage-jsvm.zip') - self.run_command(['zip', '-q', file_path_jsvm, 'jsvm_lcov_output.info']) - - # GRCOV post-processing - # Download the gcno from the build machine. - self.download_file(self.url_to_gcno, parent_dir=self.grcov_dir) - - # Run grcov on the zipped .gcno and .gcda files. - grcov_command = [ - os.path.join(self.grcov_dir, 'grcov'), - '-t', 'lcov', - '-p', self.prefix, - '--ignore-dir', 'gcc*', - '--ignore-dir', 'vs2017_*', - os.path.join(self.grcov_dir, 'target.code-coverage-gcno.zip'), file_path_gcda - ] - - if mozinfo.os == 'win': - grcov_command += ['--llvm'] - - # 'grcov_output' will be a tuple, the first variable is the path to the lcov output, - # the other is the path to the standard error output. - grcov_output, _ = self.get_output_from_command( - grcov_command, - silent=True, - save_tmpfiles=True, - return_type='files', - throw_exception=True, - ) - output_file_name = 'grcov_lcov_output.info' - shutil.move(grcov_output, os.path.join(self.grcov_dir, output_file_name)) - # Zip the grcov output and upload it. self.run_command( - ['zip', '-q', os.path.join(dirs['abs_blob_upload_dir'], 'code-coverage-grcov.zip'), output_file_name], - cwd=self.grcov_dir + ['zip', '-q', os.path.join(dirs['abs_blob_upload_dir'], 'code-coverage-grcov.zip'), grcov_output_file] + ) + + # Zip the JSVM coverage data and upload it. + self.run_command( + ['zip', '-q', os.path.join(dirs['abs_blob_upload_dir'], 'code-coverage-jsvm.zip'), jsvm_output_file] ) shutil.rmtree(self.gcov_dir) diff --git a/testing/mozharness/mozharness/mozilla/testing/per_test_base.py b/testing/mozharness/mozharness/mozilla/testing/per_test_base.py index b37efa12bf3c..5e9a2ceae2b4 100644 --- a/testing/mozharness/mozharness/mozilla/testing/per_test_base.py +++ b/testing/mozharness/mozharness/mozilla/testing/per_test_base.py @@ -14,22 +14,13 @@ import mozinfo from manifestparser import TestManifest from mozharness.base.script import PostScriptAction -verify_config_options = [ - [["--verify"], - {"action": "store_true", - "dest": "verify", - "default": "False", - "help": "Run additional verification on modified tests." - }], -] - -class VerifyToolsMixin(object): - """Utility functions for test verification.""" +class SingleTestMixin(object): + """Utility functions for per-test testing like test verification and per-test coverage.""" def __init__(self): - self.verify_suites = {} - self.verify_downloaded = False + self.suites = {} + self.tests_downloaded = False self.reftest_test_dir = None self.jsreftest_test_dir = None @@ -47,7 +38,7 @@ class VerifyToolsMixin(object): man = TestManifest([path], strict=False) active = man.active_tests(exists=False, disabled=True, filters=[], **mozinfo.info) # Remove disabled tests. Also, remove tests with the same path as - # disabled tests, even if they are not disabled, since test-verify + # disabled tests, even if they are not disabled, since per-test mode # specifies tests by path (it cannot distinguish between two or more # tests with the same path specified in multiple manifests). disabled = [t['relpath'] for t in active if 'disabled' in t] @@ -55,7 +46,7 @@ class VerifyToolsMixin(object): for t in active if 'disabled' not in t and \ t['relpath'] not in disabled} tests_by_path.update(new_by_path) - self.info("Verification updated with manifest %s" % path) + self.info("Per-test run updated with manifest %s" % path) ref_manifests = [ (os.path.join(dirs['abs_reftest_dir'], 'tests', 'layout', 'reftests', 'reftest.list'), 'reftest'), @@ -69,7 +60,7 @@ class VerifyToolsMixin(object): man = manifest.ReftestManifest() man.load(path) tests_by_path.update({os.path.relpath(t,self.reftest_test_dir):(suite,None) for t in man.files}) - self.info("Verification updated with manifest %s" % path) + self.info("Per-test run updated with manifest %s" % path) suite = 'jsreftest' self.jsreftest_test_dir = os.path.join(dirs['abs_test_install_dir'], 'jsreftest', 'tests') @@ -89,7 +80,7 @@ class VerifyToolsMixin(object): tests_by_path.update({relpath:(suite,None)}) else: self.warning("unexpected jsreftest test format: %s" % str(t)) - self.info("Verification updated with manifest %s" % path) + self.info("Per-test run updated with manifest %s" % path) # for each changed file, determine if it is a test file, and what suite it is in for file in changed_files: @@ -98,7 +89,7 @@ class VerifyToolsMixin(object): file = file.replace(posixpath.sep, os.sep) entry = tests_by_path.get(file) if entry: - self.info("Verification found test %s" % file) + self.info("Per-test run found test %s" % file) subsuite_mapping = { ('browser-chrome', 'clipboard') : 'browser-chrome-clipboard', ('chrome', 'clipboard') : 'chrome-clipboard', @@ -115,11 +106,11 @@ class VerifyToolsMixin(object): suite = subsuite_mapping[entry] else: suite = entry[0] - suite_files = self.verify_suites.get(suite) + suite_files = self.suites.get(suite) if not suite_files: suite_files = [] suite_files.append(file) - self.verify_suites[suite] = suite_files + self.suites[suite] = suite_files def _find_wpt_tests(self, dirs, changed_files): # Setup sys.path to include all the dependencies required to import @@ -146,31 +137,26 @@ class VerifyToolsMixin(object): repo_path = repo_path.replace(os.sep, posixpath.sep) if repo_path in changed_files: self.info("found web-platform test file '%s', type %s" % (path, type)) - suite_files = self.verify_suites.get(type) + suite_files = self.suites.get(type) if not suite_files: suite_files = [] path = os.path.join(tests_path, path) suite_files.append(path) - self.verify_suites[type] = suite_files + self.suites[type] = suite_files - @PostScriptAction('download-and-extract') - def find_tests_for_verification(self, action, success=None): + def find_modified_tests(self): """ For each file modified on this push, determine if the modified file - is a test, by searching test manifests. Populate self.verify_suites + is a test, by searching test manifests. Populate self.suites with test files, organized by suite. This depends on test manifests, so can only run after test zips have been downloaded and extracted. """ - - if self.config.get('verify') != True: - return - repository = os.environ.get("GECKO_HEAD_REPOSITORY") revision = os.environ.get("GECKO_HEAD_REV") if not repository or not revision: - self.warning("unable to verify tests: no repo or revision!") + self.warning("unable to run tests in per-test mode: no repo or revision!") return [] def get_automationrelevance(): @@ -186,7 +172,7 @@ class VerifyToolsMixin(object): # FIXME(emilio): Need to update test expectations. mozinfo.update({'stylo': True}) mozinfo.update({'verify': True}) - self.info("Verification using mozinfo: %s" % str(mozinfo.info)) + self.info("Per-test run using mozinfo: %s" % str(mozinfo.info)) # determine which files were changed on this push url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'), revision) @@ -198,91 +184,93 @@ class VerifyToolsMixin(object): desc=c['desc'].splitlines()[0].encode('ascii', 'ignore'))) changed_files |= set(c['files']) - if self.config.get('verify_category') == "web-platform": + if self.config.get('per_test_category') == "web-platform": self._find_wpt_tests(dirs, changed_files) else: self._find_misc_tests(dirs, changed_files) - self.verify_downloaded = True + self.tests_downloaded = True - def query_verify_args(self, suite): + def query_args(self, suite): """ For the specified suite, return an array of command line arguments to - be passed to test harnesses when running in verify mode. + be passed to test harnesses when running in per-test mode. Each array element is an array of command line arguments for a modified test in the suite. """ + # not in verify or per-test coverage mode: run once, with no additional args + if not self.per_test_coverage and not self.verify_enabled: + return [[]] - # Limit each test harness run to 15 minutes, to avoid task timeouts - # when verifying long-running tests. - MAX_TIME_PER_TEST = 900 + references = re.compile(r"(-ref|-notref|-noref|-noref.)\.") + files = [] + jsreftest_extra_dir = os.path.join('js', 'src', 'tests') + # For some suites, the test path needs to be updated before passing to + # the test harness. + for file in self.suites.get(suite): + if (self.config.get('per_test_category') != "web-platform" and + suite in ['reftest', 'crashtest']): + file = os.path.join(self.reftest_test_dir, file) + if suite == 'reftest': + # Special handling for modified reftest reference files: + # - if both test and reference modified, run the test file + # - if only reference modified, run the test file + nonref = references.sub('.', file) + if nonref != file: + file = None + if nonref not in files and os.path.exists(nonref): + file = nonref + elif (self.config.get('per_test_category') != "web-platform" and + suite == 'jsreftest'): + file = os.path.relpath(file, jsreftest_extra_dir) + file = os.path.join(self.jsreftest_test_dir, file) + + if file is None: + continue + + file = file.replace(os.sep, posixpath.sep) + files.append(file) + + self.info("Per-test file(s) for '%s': %s" % (suite, files)) + + args = [] + for file in files: + cur = [] + + cur.extend(self.coverage_args) + cur.extend(self.verify_args) + + cur.append(file) + args.append(cur) - if self.config.get('verify') != True: - # not in verify mode: run once, with no additional args - args = [[]] - else: - # in verify mode, run nothing by default (unsupported suite or no files modified) - args = [] - # otherwise, run once for each file in requested suite - references = re.compile(r"(-ref|-notref|-noref|-noref.)\.") - files = [] - jsreftest_extra_dir = os.path.join('js', 'src', 'tests') - # For some suites, the test path needs to be updated before passing to - # the test harness. - for file in self.verify_suites.get(suite): - if (self.config.get('verify_category') != "web-platform" and - suite in ['reftest', 'crashtest']): - file = os.path.join(self.reftest_test_dir, file) - elif (self.config.get('verify_category') != "web-platform" and - suite == 'jsreftest'): - file = os.path.relpath(file, jsreftest_extra_dir) - file = os.path.join(self.jsreftest_test_dir, file) - file = file.replace(os.sep, posixpath.sep) - files.append(file) - for file in files: - if self.config.get('verify_category') == "web-platform": - args.append(['--verify-log-full', '--verify', file]) - else: - if suite == 'reftest': - # Special handling for modified reftest reference files: - # - if both test and reference modified, verify the test file - # - if only reference modified, verify the test file - nonref = references.sub('.', file) - if nonref != file: - file = None - if nonref not in files and os.path.exists(nonref): - file = nonref - if file: - args.append(['--verify-max-time=%d' % MAX_TIME_PER_TEST, '--verify', file]) - self.info("Verification file(s) for '%s': %s" % (suite, files)) return args - def query_verify_category_suites(self, category, all_suites): + def query_per_test_category_suites(self, category, all_suites): """ - In verify mode, determine which suites are active, for the given + In per-test mode, determine which suites are active, for the given suite category. """ suites = None - if self.config.get('verify') == True: - if self.config.get('verify_category') == "web-platform": - suites = self.verify_suites.keys() - elif all_suites and self.verify_downloaded: + if self.verify_enabled or self.per_test_coverage: + if self.config.get('per_test_category') == "web-platform": + suites = self.suites.keys() + elif all_suites and self.tests_downloaded: suites = dict((key, all_suites.get(key)) for key in - self.verify_suites if key in all_suites.keys()) + self.suites if key in all_suites.keys()) else: # Until test zips are downloaded, manifests are not available, # so it is not possible to determine which suites are active/ - # required for verification; assume all suites from supported + # required for per-test mode; assume all suites from supported # suite categories are required. if category in ['mochitest', 'xpcshell', 'reftest']: suites = all_suites return suites - def log_verify_status(self, test_name, tbpl_status, log_level): + def log_per_test_status(self, test_name, tbpl_status, log_level): """ - Log verification status of a single test. This will display in the - Job Details pane in treeherder - a convenient summary of verification. + Log status of a single test. This will display in the + Job Details pane in treeherder - a convenient summary of per-test mode. Special test name formatting is needed because treeherder truncates lines that are too long, and may remove duplicates after truncation. """ @@ -298,6 +286,6 @@ class VerifyToolsMixin(object): new = os.path.join(tail, new) test_name = os.path.join('...', previous or new) test_name = test_name.rstrip(os.path.sep) - self.log("TinderboxPrint: Verification of %s
: %s" % + self.log("TinderboxPrint: Per-test run of %s
: %s" % (test_name, tbpl_status), level=log_level) diff --git a/testing/mozharness/mozharness/mozilla/testing/verify_tools.py b/testing/mozharness/mozharness/mozilla/testing/verify_tools.py new file mode 100644 index 000000000000..db60e14cc074 --- /dev/null +++ b/testing/mozharness/mozharness/mozilla/testing/verify_tools.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** + +from mozharness.base.script import PostScriptAction +from mozharness.mozilla.testing.per_test_base import SingleTestMixin + + +verify_config_options = [ + [["--verify"], + {"action": "store_true", + "dest": "verify", + "default": False, + "help": "Run additional verification on modified tests." + }], +] + + +class VerifyToolsMixin(SingleTestMixin): + """Utility functions for test verification.""" + + def __init__(self): + super(VerifyToolsMixin, self).__init__() + + @property + def verify_enabled(self): + try: + return bool(self.config.get('verify')) + except (AttributeError, KeyError, TypeError): + return False + + @PostScriptAction('download-and-extract') + def find_tests_for_verification(self, action, success=None): + """ + For each file modified on this push, determine if the modified file + is a test, by searching test manifests. Populate self.verify_suites + with test files, organized by suite. + + This depends on test manifests, so can only run after test zips have + been downloaded and extracted. + """ + + if not self.verify_enabled: + return + + self.find_modified_tests() + + @property + def verify_args(self): + if not self.verify_enabled: + return [] + + # Limit each test harness run to 15 minutes, to avoid task timeouts + # when executing long-running tests. + MAX_TIME_PER_TEST = 900 + + if self.config.get('per_test_category') == "web-platform": + args = ['--verify-log-full'] + else: + args = ['--verify-max-time=%d' % MAX_TIME_PER_TEST] + + args.append('--verify') + + return args diff --git a/testing/mozharness/scripts/android_emulator_unittest.py b/testing/mozharness/scripts/android_emulator_unittest.py index 02a64d5d6600..174cf90bddba 100644 --- a/testing/mozharness/scripts/android_emulator_unittest.py +++ b/testing/mozharness/scripts/android_emulator_unittest.py @@ -471,7 +471,7 @@ class AndroidEmulatorTest(TestingMixin, EmulatorMixin, BaseScript, MozbaseMixin) try_options, try_tests = self.try_args(self.test_suite) cmd.extend(try_options) - if self.config.get('verify') is not True: + if self.verify_enabled or self.per_test_coverage: cmd.extend(self.query_tests_args( self.config["suite_definitions"][self.test_suite].get("tests"), None, @@ -740,7 +740,7 @@ class AndroidEmulatorTest(TestingMixin, EmulatorMixin, BaseScript, MozbaseMixin) def _query_suites(self): if self.test_suite: return [(self.test_suite, self.test_suite)] - # test-verification: determine test suites to be verified + # per-test mode: determine test suites to run all = [('mochitest', {'plain': 'mochitest', 'chrome': 'mochitest-chrome', 'plain-clipboard': 'mochitest-plain-clipboard', @@ -749,7 +749,7 @@ class AndroidEmulatorTest(TestingMixin, EmulatorMixin, BaseScript, MozbaseMixin) ('xpcshell', {'xpcshell': 'xpcshell'})] suites = [] for (category, all_suites) in all: - cat_suites = self.query_verify_category_suites(category, all_suites) + cat_suites = self.query_per_test_category_suites(category, all_suites) for k in cat_suites.keys(): suites.append((k, cat_suites[k])) return suites @@ -758,7 +758,7 @@ class AndroidEmulatorTest(TestingMixin, EmulatorMixin, BaseScript, MozbaseMixin) if self.test_suite: categories = [self.test_suite] else: - # test-verification + # per-test mode categories = ['mochitest', 'reftest', 'xpcshell'] return categories @@ -767,12 +767,12 @@ class AndroidEmulatorTest(TestingMixin, EmulatorMixin, BaseScript, MozbaseMixin) Run the tests """ self.start_time = datetime.datetime.now() - max_verify_time = datetime.timedelta(minutes=60) + max_per_test_time = datetime.timedelta(minutes=60) - verify_args = [] + per_test_args = [] suites = self._query_suites() minidump = self.query_minidump_stackwalk() - for (verify_suite, suite) in suites: + for (per_test_suite, suite) in suites: self.test_suite = suite cmd = self._build_command() @@ -788,24 +788,24 @@ class AndroidEmulatorTest(TestingMixin, EmulatorMixin, BaseScript, MozbaseMixin) env['MINIDUMP_SAVE_PATH'] = self.query_abs_dirs()['abs_blob_upload_dir'] env['RUST_BACKTRACE'] = 'full' - for verify_args in self.query_verify_args(verify_suite): - if (datetime.datetime.now() - self.start_time) > max_verify_time: - # Verification has run out of time. That is okay! Stop running - # tests so that a task timeout is not triggered, and so that + for per_test_args in self.query_args(per_test_suite): + if (datetime.datetime.now() - self.start_time) > max_per_test_time: + # Running tests has run out of time. That is okay! Stop running + # them so that a task timeout is not triggered, and so that # (partial) results are made available in a timely manner. - self.info("TinderboxPrint: Verification too long: " - "Not all tests were verified.
") - # Signal verify time exceeded, to break out of suites and + self.info("TinderboxPrint: Running tests took too long: " + "Not all tests were executed.
") + # Signal per-test time exceeded, to break out of suites and # suite categories loops also. return False final_cmd = copy.copy(cmd) - if len(verify_args) > 0: - # in verify mode, remove any chunk arguments from command + if len(per_test_args) > 0: + # in per-test mode, remove any chunk arguments from command for arg in final_cmd: if 'total-chunk' in arg or 'this-chunk' in arg: final_cmd.remove(arg) - final_cmd.extend(verify_args) + final_cmd.extend(per_test_args) self.info("Running on %s the command %s" % (self.emulator["name"], subprocess.list2cmdline(final_cmd))) @@ -823,9 +823,9 @@ class AndroidEmulatorTest(TestingMixin, EmulatorMixin, BaseScript, MozbaseMixin) self.info("##### %s log ends" % self.test_suite) - if len(verify_args) > 0: + if len(per_test_args) > 0: self.buildbot_status(tbpl_status, level=log_level) - self.log_verify_status(verify_args[-1], tbpl_status, log_level) + self.log_per_test_status(per_test_args[-1], tbpl_status, log_level) else: self.buildbot_status(tbpl_status, level=log_level) self.log("The %s suite: %s ran with return status: %s" % diff --git a/testing/mozharness/scripts/desktop_unittest.py b/testing/mozharness/scripts/desktop_unittest.py index 3ba4a2ad419d..ae04993ea687 100755 --- a/testing/mozharness/scripts/desktop_unittest.py +++ b/testing/mozharness/scripts/desktop_unittest.py @@ -479,7 +479,7 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix if c.get('run_all_suites'): # needed if you dont specify any suites suites = all_suites else: - suites = self.query_verify_category_suites(category, all_suites) + suites = self.query_per_test_category_suites(category, all_suites) return suites @@ -767,9 +767,9 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix abs_app_dir = self.query_abs_app_dir() abs_res_dir = self.query_abs_res_dir() - max_verify_time = timedelta(minutes=60) - max_verify_tests = 10 - verified_tests = 0 + max_per_test_time = timedelta(minutes=60) + max_per_test_tests = 10 + executed_tests = 0 if suites: self.info('#### Running %s suites' % suite_category) @@ -789,7 +789,7 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix } if isinstance(suites[suite], dict): options_list = suites[suite].get('options', []) - if self.config.get('verify') is True: + if self.verify_enabled or self.per_test_coverage: tests_list = [] else: tests_list = suites[suite].get('tests', []) @@ -848,33 +848,56 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix env = self.query_env(partial_env=env, log_level=INFO) cmd_timeout = self.get_timeout_for_category(suite_category) - for verify_args in self.query_verify_args(suite): - if (datetime.now() - self.start_time) > max_verify_time: - # Verification has run out of time. That is okay! Stop running - # tests so that a task timeout is not triggered, and so that + # Run basic startup/shutdown test to collect baseline coverage. + # This way, after we run a test, we can generate a diff between the + # full coverage of the test and the baseline coverage and only get + # the coverage data specific to the test. + if self.per_test_coverage: + gcov_dir, jsvm_dir = self.set_coverage_env(env) + # TODO: Run basic startup/shutdown test to collect baseline coverage. + # grcov_file, jsvm_file = self.parse_coverage_artifacts(gcov_dir, jsvm_dir) + # shutil.rmtree(gcov_dir) + # shutil.rmtree(jsvm_dir) + # TODO: Parse coverage report + + for per_test_args in self.query_args(suite): + if (datetime.now() - self.start_time) > max_per_test_time: + # Running tests has run out of time. That is okay! Stop running + # them so that a task timeout is not triggered, and so that # (partial) results are made available in a timely manner. - self.info("TinderboxPrint: Verification too long: Not all tests " - "were verified.
") - # Signal verify time exceeded, to break out of suites and + self.info("TinderboxPrint: Running tests took too long: Not all tests " + "were executed.
") + # Signal per-test time exceeded, to break out of suites and # suite categories loops also. return False - if verified_tests >= max_verify_tests: + if executed_tests >= max_per_test_tests: # When changesets are merged between trees or many tests are # otherwise updated at once, there probably is not enough time - # to verify all tests, and attempting to do so may cause other + # to run all tests, and attempting to do so may cause other # problems, such as generating too much log output. self.info("TinderboxPrint: Too many modified tests: Not all tests " - "were verified.
") + "were executed.
") return False - verified_tests = verified_tests + 1 + executed_tests = executed_tests + 1 final_cmd = copy.copy(cmd) - final_cmd.extend(verify_args) + final_cmd.extend(per_test_args) + + if self.per_test_coverage: + gcov_dir, jsvm_dir = self.set_coverage_env(env) + return_code = self.run_command(final_cmd, cwd=dirs['abs_work_dir'], output_timeout=cmd_timeout, output_parser=parser, env=env) + if self.per_test_coverage: + grcov_file, jsvm_file = self.parse_coverage_artifacts(gcov_dir, jsvm_dir) + shutil.rmtree(gcov_dir) + shutil.rmtree(jsvm_dir) + # TODO: Parse coverage report + # TODO: Diff this coverage report with the baseline one + # mochitest, reftest, and xpcshell suites do not return # appropriate return codes. Therefore, we must parse the output # to determine what the tbpl_status and worst_log_level must @@ -895,8 +918,8 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix parser.append_tinderboxprint_line(suite_name) self.buildbot_status(tbpl_status, level=log_level) - if len(verify_args) > 0: - self.log_verify_status(verify_args[-1], tbpl_status, log_level) + if len(per_test_args) > 0: + self.log_per_test_status(per_test_args[-1], tbpl_status, log_level) else: self.log("The %s suite: %s ran with return status: %s" % (suite_category, suite, tbpl_status), level=log_level) diff --git a/testing/mozharness/scripts/web_platform_tests.py b/testing/mozharness/scripts/web_platform_tests.py index 53668cbce3ea..97f998bb5804 100755 --- a/testing/mozharness/scripts/web_platform_tests.py +++ b/testing/mozharness/scripts/web_platform_tests.py @@ -6,6 +6,7 @@ # ***** END LICENSE BLOCK ***** import copy import os +import shutil import sys from datetime import datetime, timedelta @@ -318,44 +319,60 @@ class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin, CodeCovera env = self.query_env(partial_env=env, log_level=INFO) start_time = datetime.now() - max_verify_time = timedelta(minutes=60) - max_verify_tests = 10 - verified_tests = 0 + max_per_test_time = timedelta(minutes=60) + max_per_test_tests = 10 + executed_tests = 0 - if self.config.get("verify") is True: - verify_suites = self.query_verify_category_suites(None, None) - if "wdspec" in verify_suites: + if self.per_test_coverage or self.verify_enabled: + suites = self.query_per_test_category_suites(None, None) + if "wdspec" in suites: # geckodriver is required for wdspec, but not always available geckodriver_path = self._query_geckodriver() if not geckodriver_path or not os.path.isfile(geckodriver_path): - verify_suites.remove("wdspec") - self.info("Test verification skipping 'wdspec' tests - no geckodriver") + suites.remove("wdspec") + self.info("Skipping 'wdspec' tests - no geckodriver") else: test_types = self.config.get("test_type", []) - verify_suites = [None] - for verify_suite in verify_suites: - if verify_suite: - test_types = [verify_suite] - for verify_args in self.query_verify_args(verify_suite): - if (datetime.now() - start_time) > max_verify_time: - # Verification has run out of time. That is okay! Stop running - # tests so that a task timeout is not triggered, and so that + suites = [None] + for suite in suites: + if suite: + test_types = [suite] + + # Run basic startup/shutdown test to collect baseline coverage. + # This way, after we run a test, we can generate a diff between the + # full coverage of the test and the baseline coverage and only get + # the coverage data specific to the test. + if self.per_test_coverage: + gcov_dir, jsvm_dir = self.set_coverage_env(env) + # TODO: Run basic startup/shutdown test to collect baseline coverage. + # grcov_file, jsvm_file = self.parse_coverage_artifacts(gcov_dir, jsvm_dir) + # shutil.rmtree(gcov_dir) + # shutil.rmtree(jsvm_dir) + # TODO: Parse coverage report + + for per_test_args in self.query_args(suite): + if (datetime.now() - start_time) > max_per_test_time: + # Running tests has run out of time. That is okay! Stop running + # them so that a task timeout is not triggered, and so that # (partial) results are made available in a timely manner. - self.info("TinderboxPrint: Verification too long: Not all tests " - "were verified.
") + self.info("TinderboxPrint: Running tests took too long: Not all tests " + "were executed.
") return - if verified_tests >= max_verify_tests: + if executed_tests >= max_per_test_tests: # When changesets are merged between trees or many tests are # otherwise updated at once, there probably is not enough time - # to verify all tests, and attempting to do so may cause other + # to run all tests, and attempting to do so may cause other # problems, such as generating too much log output. self.info("TinderboxPrint: Too many modified tests: Not all tests " - "were verified.
") + "were executed.
") return - verified_tests = verified_tests + 1 + executed_tests = executed_tests + 1 cmd = self._query_cmd(test_types) - cmd.extend(verify_args) + cmd.extend(per_test_args) + + if self.per_test_coverage: + gcov_dir, jsvm_dir = self.set_coverage_env(env) return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'], @@ -363,11 +380,18 @@ class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin, CodeCovera output_parser=parser, env=env) + if self.per_test_coverage: + grcov_file, jsvm_file = self.parse_coverage_artifacts(gcov_dir, jsvm_dir) + shutil.rmtree(gcov_dir) + shutil.rmtree(jsvm_dir) + # TODO: Parse coverage report + # TODO: Diff this coverage report with the baseline one + tbpl_status, log_level = parser.evaluate_parser(return_code) self.buildbot_status(tbpl_status, level=log_level) - if len(verify_args) > 0: - self.log_verify_status(verify_args[-1], tbpl_status, log_level) + if len(per_test_args) > 0: + self.log_per_test_status(per_test_args[-1], tbpl_status, log_level) # main {{{1