зеркало из https://github.com/mozilla/gecko-dev.git
1169 строки
42 KiB
Python
1169 строки
42 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
"""
|
|
Runs the reftest test harness.
|
|
"""
|
|
import json
|
|
import multiprocessing
|
|
import os
|
|
import platform
|
|
import posixpath
|
|
import re
|
|
import shutil
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
from collections import defaultdict
|
|
from datetime import datetime, timedelta
|
|
|
|
SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
|
|
if SCRIPT_DIRECTORY not in sys.path:
|
|
sys.path.insert(0, SCRIPT_DIRECTORY)
|
|
|
|
import mozcrash
|
|
import mozdebug
|
|
import mozfile
|
|
import mozinfo
|
|
import mozleak
|
|
import mozlog
|
|
import mozprocess
|
|
import mozprofile
|
|
import mozrunner
|
|
from manifestparser import TestManifest
|
|
from manifestparser import filters as mpf
|
|
from mozrunner.utils import get_stack_fixer_function, test_environment
|
|
from mozscreenshot import dump_screen, printstatus
|
|
from six import reraise, string_types
|
|
from six.moves import range
|
|
|
|
try:
|
|
from marionette_driver.addons import Addons
|
|
from marionette_driver.marionette import Marionette
|
|
except ImportError as e: # noqa
|
|
# Defer ImportError until attempt to use Marionette.
|
|
# Python 3 deletes the exception once the except block
|
|
# is exited. Save a version to raise later.
|
|
e_save = ImportError(str(e))
|
|
|
|
def reraise_(*args, **kwargs):
|
|
raise (e_save) # noqa
|
|
|
|
Marionette = reraise_
|
|
|
|
import reftestcommandline
|
|
from output import OutputHandler, ReftestFormatter
|
|
|
|
here = os.path.abspath(os.path.dirname(__file__))
|
|
|
|
try:
|
|
from mozbuild.base import MozbuildObject
|
|
|
|
build_obj = MozbuildObject.from_environment(cwd=here)
|
|
except ImportError:
|
|
build_obj = None
|
|
|
|
|
|
def categoriesToRegex(categoryList):
|
|
return "\\(" + ", ".join(["(?P<%s>\\d+) %s" % c for c in categoryList]) + "\\)"
|
|
|
|
|
|
summaryLines = [
|
|
("Successful", [("pass", "pass"), ("loadOnly", "load only")]),
|
|
(
|
|
"Unexpected",
|
|
[
|
|
("fail", "unexpected fail"),
|
|
("pass", "unexpected pass"),
|
|
("asserts", "unexpected asserts"),
|
|
("fixedAsserts", "unexpected fixed asserts"),
|
|
("failedLoad", "failed load"),
|
|
("exception", "exception"),
|
|
],
|
|
),
|
|
(
|
|
"Known problems",
|
|
[
|
|
("knownFail", "known fail"),
|
|
("knownAsserts", "known asserts"),
|
|
("random", "random"),
|
|
("skipped", "skipped"),
|
|
("slow", "slow"),
|
|
],
|
|
),
|
|
]
|
|
|
|
|
|
if sys.version_info[0] == 3:
|
|
|
|
def reraise_(tp_, value_, tb_=None):
|
|
if value_ is None:
|
|
value_ = tp_()
|
|
if value_.__traceback__ is not tb_:
|
|
raise value_.with_traceback(tb_)
|
|
raise value_
|
|
|
|
|
|
else:
|
|
exec("def reraise_(tp_, value_, tb_=None):\n raise tp_, value_, tb_\n")
|
|
|
|
|
|
def update_mozinfo():
|
|
"""walk up directories to find mozinfo.json update the info"""
|
|
# TODO: This should go in a more generic place, e.g. mozinfo
|
|
|
|
path = SCRIPT_DIRECTORY
|
|
dirs = set()
|
|
while path != os.path.expanduser("~"):
|
|
if path in dirs:
|
|
break
|
|
dirs.add(path)
|
|
path = os.path.split(path)[0]
|
|
mozinfo.find_and_update_from_json(*dirs)
|
|
|
|
|
|
# Python's print is not threadsafe.
|
|
printLock = threading.Lock()
|
|
|
|
|
|
class ReftestThread(threading.Thread):
|
|
def __init__(self, cmdargs):
|
|
threading.Thread.__init__(self)
|
|
self.cmdargs = cmdargs
|
|
self.summaryMatches = {}
|
|
self.retcode = -1
|
|
for text, _ in summaryLines:
|
|
self.summaryMatches[text] = None
|
|
|
|
def run(self):
|
|
with printLock:
|
|
print("Starting thread with", self.cmdargs)
|
|
sys.stdout.flush()
|
|
process = subprocess.Popen(self.cmdargs, stdout=subprocess.PIPE)
|
|
for chunk in self.chunkForMergedOutput(process.stdout):
|
|
with printLock:
|
|
print(chunk, end=" ")
|
|
sys.stdout.flush()
|
|
self.retcode = process.wait()
|
|
|
|
def chunkForMergedOutput(self, logsource):
|
|
"""Gather lines together that should be printed as one atomic unit.
|
|
Individual test results--anything between 'REFTEST TEST-START' and
|
|
'REFTEST TEST-END' lines--are an atomic unit. Lines with data from
|
|
summaries are parsed and the data stored for later aggregation.
|
|
Other lines are considered their own atomic units and are permitted
|
|
to intermix freely."""
|
|
testStartRegex = re.compile("^REFTEST TEST-START")
|
|
testEndRegex = re.compile("^REFTEST TEST-END")
|
|
summaryHeadRegex = re.compile("^REFTEST INFO \\| Result summary:")
|
|
summaryRegexFormatString = (
|
|
"^REFTEST INFO \\| (?P<message>{text}): (?P<total>\\d+) {regex}"
|
|
)
|
|
summaryRegexStrings = [
|
|
summaryRegexFormatString.format(
|
|
text=text, regex=categoriesToRegex(categories)
|
|
)
|
|
for (text, categories) in summaryLines
|
|
]
|
|
summaryRegexes = [re.compile(regex) for regex in summaryRegexStrings]
|
|
|
|
for line in logsource:
|
|
if testStartRegex.search(line) is not None:
|
|
chunkedLines = [line]
|
|
for lineToBeChunked in logsource:
|
|
chunkedLines.append(lineToBeChunked)
|
|
if testEndRegex.search(lineToBeChunked) is not None:
|
|
break
|
|
yield "".join(chunkedLines)
|
|
continue
|
|
|
|
haveSuppressedSummaryLine = False
|
|
for regex in summaryRegexes:
|
|
match = regex.search(line)
|
|
if match is not None:
|
|
self.summaryMatches[match.group("message")] = match
|
|
haveSuppressedSummaryLine = True
|
|
break
|
|
if haveSuppressedSummaryLine:
|
|
continue
|
|
|
|
if summaryHeadRegex.search(line) is None:
|
|
yield line
|
|
|
|
|
|
class ReftestResolver(object):
|
|
def defaultManifest(self, suite):
|
|
return {
|
|
"reftest": "reftest.list",
|
|
"crashtest": "crashtests.list",
|
|
"jstestbrowser": "jstests.list",
|
|
}[suite]
|
|
|
|
def directoryManifest(self, suite, path):
|
|
return os.path.join(path, self.defaultManifest(suite))
|
|
|
|
def findManifest(self, suite, test_file, subdirs=True):
|
|
"""Return a tuple of (manifest-path, filter-string) for running test_file.
|
|
|
|
test_file is a path to a test or a manifest file
|
|
"""
|
|
rv = []
|
|
default_manifest = self.defaultManifest(suite)
|
|
relative_path = None
|
|
if not os.path.isabs(test_file):
|
|
relative_path = test_file
|
|
test_file = self.absManifestPath(test_file)
|
|
|
|
if os.path.isdir(test_file):
|
|
for dirpath, dirnames, filenames in os.walk(test_file):
|
|
if default_manifest in filenames:
|
|
rv.append((os.path.join(dirpath, default_manifest), None))
|
|
# We keep recursing into subdirectories which means that in the case
|
|
# of include directives we get the same manifest multiple times.
|
|
# However reftest.js will only read each manifest once
|
|
|
|
if (
|
|
len(rv) == 0
|
|
and relative_path
|
|
and suite == "jstestbrowser"
|
|
and build_obj
|
|
):
|
|
# The relative path can be from staging area.
|
|
staged_js_dir = os.path.join(
|
|
build_obj.topobjdir, "dist", "test-stage", "jsreftest"
|
|
)
|
|
staged_file = os.path.join(staged_js_dir, "tests", relative_path)
|
|
return self.findManifest(suite, staged_file, subdirs)
|
|
elif test_file.endswith(".list"):
|
|
if os.path.exists(test_file):
|
|
rv = [(test_file, None)]
|
|
else:
|
|
dirname, pathname = os.path.split(test_file)
|
|
found = True
|
|
while not os.path.exists(os.path.join(dirname, default_manifest)):
|
|
dirname, suffix = os.path.split(dirname)
|
|
pathname = posixpath.join(suffix, pathname)
|
|
if os.path.dirname(dirname) == dirname:
|
|
found = False
|
|
break
|
|
if found:
|
|
rv = [
|
|
(
|
|
os.path.join(dirname, default_manifest),
|
|
r".*(?:/|\\)%s(?:[#?].*)?$" % pathname.replace("?", "\?"),
|
|
)
|
|
]
|
|
|
|
return rv
|
|
|
|
def absManifestPath(self, path):
|
|
return os.path.normpath(os.path.abspath(path))
|
|
|
|
def manifestURL(self, options, path):
|
|
return "file://%s" % path
|
|
|
|
def resolveManifests(self, options, tests):
|
|
suite = options.suite
|
|
manifests = {}
|
|
for testPath in tests:
|
|
for manifest, filter_str in self.findManifest(suite, testPath):
|
|
if manifest not in manifests:
|
|
manifests[manifest] = set()
|
|
manifests[manifest].add(filter_str)
|
|
manifests_by_url = {}
|
|
for key in manifests.keys():
|
|
id = os.path.relpath(
|
|
os.path.abspath(os.path.dirname(key)), options.topsrcdir
|
|
)
|
|
id = id.replace(os.sep, posixpath.sep)
|
|
if None in manifests[key]:
|
|
manifests[key] = (None, id)
|
|
else:
|
|
manifests[key] = ("|".join(list(manifests[key])), id)
|
|
url = self.manifestURL(options, key)
|
|
manifests_by_url[url] = manifests[key]
|
|
return manifests_by_url
|
|
|
|
|
|
class RefTest(object):
|
|
oldcwd = os.getcwd()
|
|
resolver_cls = ReftestResolver
|
|
use_marionette = True
|
|
|
|
def __init__(self, suite):
|
|
update_mozinfo()
|
|
self.lastTestSeen = None
|
|
self.lastTest = None
|
|
self.haveDumpedScreen = False
|
|
self.resolver = self.resolver_cls()
|
|
self.log = None
|
|
self.outputHandler = None
|
|
self.testDumpFile = os.path.join(tempfile.gettempdir(), "reftests.json")
|
|
|
|
self.run_by_manifest = True
|
|
if suite in ("crashtest", "jstestbrowser"):
|
|
self.run_by_manifest = False
|
|
|
|
def _populate_logger(self, options):
|
|
if self.log:
|
|
return
|
|
|
|
self.log = getattr(options, "log", None)
|
|
if self.log:
|
|
return
|
|
|
|
mozlog.commandline.log_formatters["tbpl"] = (
|
|
ReftestFormatter,
|
|
"Reftest specific formatter for the"
|
|
"benefit of legacy log parsers and"
|
|
"tools such as the reftest analyzer",
|
|
)
|
|
fmt_options = {}
|
|
if not options.log_tbpl_level and os.environ.get("MOZ_REFTEST_VERBOSE"):
|
|
options.log_tbpl_level = fmt_options["level"] = "debug"
|
|
self.log = mozlog.commandline.setup_logging(
|
|
"reftest harness", options, {"tbpl": sys.stdout}, fmt_options
|
|
)
|
|
|
|
def getFullPath(self, path):
|
|
"Get an absolute path relative to self.oldcwd."
|
|
return os.path.normpath(os.path.join(self.oldcwd, os.path.expanduser(path)))
|
|
|
|
def createReftestProfile(
|
|
self,
|
|
options,
|
|
tests=None,
|
|
manifests=None,
|
|
server="localhost",
|
|
port=0,
|
|
profile_to_clone=None,
|
|
prefs=None,
|
|
):
|
|
"""Sets up a profile for reftest.
|
|
|
|
:param options: Object containing command line options
|
|
:param tests: List of test objects to run
|
|
:param manifests: List of manifest files to parse (only takes effect
|
|
if tests were not passed in)
|
|
:param server: Server name to use for http tests
|
|
:param profile_to_clone: Path to a profile to use as the basis for the
|
|
test profile
|
|
:param prefs: Extra preferences to set in the profile
|
|
"""
|
|
locations = mozprofile.permissions.ServerLocations()
|
|
locations.add_host(server, scheme="http", port=port)
|
|
locations.add_host(server, scheme="https", port=port)
|
|
|
|
sandbox_whitelist_paths = options.sandboxReadWhitelist
|
|
if platform.system() == "Linux" or platform.system() in (
|
|
"Windows",
|
|
"Microsoft",
|
|
):
|
|
# Trailing slashes are needed to indicate directories on Linux and Windows
|
|
sandbox_whitelist_paths = map(
|
|
lambda p: os.path.join(p, ""), sandbox_whitelist_paths
|
|
)
|
|
|
|
addons = []
|
|
if not self.use_marionette:
|
|
addons.append(options.reftestExtensionPath)
|
|
|
|
if options.specialPowersExtensionPath is not None:
|
|
if not self.use_marionette:
|
|
addons.append(options.specialPowersExtensionPath)
|
|
|
|
# Install distributed extensions, if application has any.
|
|
distExtDir = os.path.join(
|
|
options.app[: options.app.rfind(os.sep)], "distribution", "extensions"
|
|
)
|
|
if os.path.isdir(distExtDir):
|
|
for f in os.listdir(distExtDir):
|
|
addons.append(os.path.join(distExtDir, f))
|
|
|
|
# Install custom extensions.
|
|
for f in options.extensionsToInstall:
|
|
addons.append(self.getFullPath(f))
|
|
|
|
kwargs = {
|
|
"addons": addons,
|
|
"locations": locations,
|
|
"whitelistpaths": sandbox_whitelist_paths,
|
|
}
|
|
if profile_to_clone:
|
|
profile = mozprofile.Profile.clone(profile_to_clone, **kwargs)
|
|
else:
|
|
profile = mozprofile.Profile(**kwargs)
|
|
|
|
# First set prefs from the base profiles under testing/profiles.
|
|
|
|
# In test packages used in CI, the profile_data directory is installed
|
|
# in the SCRIPT_DIRECTORY.
|
|
profile_data_dir = os.path.join(SCRIPT_DIRECTORY, "profile_data")
|
|
# If possible, read profile data from topsrcdir. This prevents us from
|
|
# requiring a re-build to pick up newly added extensions in the
|
|
# <profile>/extensions directory.
|
|
if build_obj:
|
|
path = os.path.join(build_obj.topsrcdir, "testing", "profiles")
|
|
if os.path.isdir(path):
|
|
profile_data_dir = path
|
|
# Still not found? Look for testing/profiles relative to layout/tools/reftest.
|
|
if not os.path.isdir(profile_data_dir):
|
|
path = os.path.abspath(
|
|
os.path.join(SCRIPT_DIRECTORY, "..", "..", "..", "testing", "profiles")
|
|
)
|
|
if os.path.isdir(path):
|
|
profile_data_dir = path
|
|
|
|
with open(os.path.join(profile_data_dir, "profiles.json"), "r") as fh:
|
|
base_profiles = json.load(fh)["reftest"]
|
|
|
|
for name in base_profiles:
|
|
path = os.path.join(profile_data_dir, name)
|
|
profile.merge(path)
|
|
|
|
# Second set preferences for communication between our command line
|
|
# arguments and the reftest harness. Preferences that are required for
|
|
# reftest to work should instead be set under srcdir/testing/profiles.
|
|
prefs = prefs or {}
|
|
prefs["reftest.timeout"] = options.timeout * 1000
|
|
if options.logFile:
|
|
prefs["reftest.logFile"] = options.logFile
|
|
if options.ignoreWindowSize:
|
|
prefs["reftest.ignoreWindowSize"] = True
|
|
if options.shuffle:
|
|
prefs["reftest.shuffle"] = True
|
|
if options.repeat:
|
|
prefs["reftest.repeat"] = options.repeat
|
|
if options.runUntilFailure:
|
|
prefs["reftest.runUntilFailure"] = True
|
|
if not options.repeat:
|
|
prefs["reftest.repeat"] = 30
|
|
if options.verify:
|
|
prefs["reftest.verify"] = True
|
|
if options.cleanupCrashes:
|
|
prefs["reftest.cleanupPendingCrashes"] = True
|
|
prefs["reftest.focusFilterMode"] = options.focusFilterMode
|
|
prefs["reftest.logLevel"] = options.log_tbpl_level or "info"
|
|
prefs["reftest.suite"] = options.suite
|
|
prefs["gfx.font_rendering.ahem_antialias_none"] = True
|
|
# Run the "deferred" font-loader immediately, because if it finishes
|
|
# mid-test, the extra reflow that is triggered can disrupt the test.
|
|
prefs["gfx.font_loader.delay"] = 0
|
|
# Ensure bundled fonts are activated, even if not enabled by default
|
|
# on the platform, so that tests can rely on them.
|
|
prefs["gfx.bundled-fonts.activate"] = 1
|
|
# Disable dark scrollbars because it's semi-transparent.
|
|
prefs["widget.disable-dark-scrollbar"] = True
|
|
prefs["reftest.isCoverageBuild"] = mozinfo.info.get("ccov", False)
|
|
|
|
# config specific flags
|
|
prefs["sandbox.apple_silicon"] = mozinfo.info.get("apple_silicon", False)
|
|
|
|
# Set tests to run or manifests to parse.
|
|
if tests:
|
|
testlist = os.path.join(profile.profile, "reftests.json")
|
|
with open(testlist, "w") as fh:
|
|
json.dump(tests, fh)
|
|
prefs["reftest.tests"] = testlist
|
|
elif manifests:
|
|
prefs["reftest.manifests"] = json.dumps(manifests)
|
|
|
|
# Unconditionally update the e10s pref, default True
|
|
prefs["browser.tabs.remote.autostart"] = True
|
|
if not options.e10s:
|
|
prefs["browser.tabs.remote.autostart"] = False
|
|
|
|
# default fission to True
|
|
prefs["fission.autostart"] = True
|
|
if options.disableFission:
|
|
prefs["fission.autostart"] = False
|
|
|
|
if not self.run_by_manifest:
|
|
if options.totalChunks:
|
|
prefs["reftest.totalChunks"] = options.totalChunks
|
|
if options.thisChunk:
|
|
prefs["reftest.thisChunk"] = options.thisChunk
|
|
|
|
# Bug 1262954: For winXP + e10s disable acceleration
|
|
if (
|
|
platform.system() in ("Windows", "Microsoft")
|
|
and "5.1" in platform.version()
|
|
and options.e10s
|
|
):
|
|
prefs["layers.acceleration.disabled"] = True
|
|
|
|
# Bug 1300355: Disable canvas cache for win7 as it uses
|
|
# too much memory and causes OOMs.
|
|
if (
|
|
platform.system() in ("Windows", "Microsoft")
|
|
and "6.1" in platform.version()
|
|
):
|
|
prefs["reftest.nocache"] = True
|
|
|
|
if options.marionette:
|
|
# options.marionette can specify host:port
|
|
port = options.marionette.split(":")[1]
|
|
prefs["marionette.port"] = int(port)
|
|
|
|
# Enable tracing output for detailed failures in case of
|
|
# failing connection attempts, and hangs (bug 1397201)
|
|
prefs["remote.log.level"] = "Trace"
|
|
|
|
# Third, set preferences passed in via the command line.
|
|
for v in options.extraPrefs:
|
|
thispref = v.split("=")
|
|
if len(thispref) < 2:
|
|
print("Error: syntax error in --setpref=" + v)
|
|
sys.exit(1)
|
|
prefs[thispref[0]] = thispref[1].strip()
|
|
|
|
for pref in prefs:
|
|
prefs[pref] = mozprofile.Preferences.cast(prefs[pref])
|
|
profile.set_preferences(prefs)
|
|
|
|
if os.path.join(here, "chrome") not in options.extraProfileFiles:
|
|
options.extraProfileFiles.append(os.path.join(here, "chrome"))
|
|
|
|
self.copyExtraFilesToProfile(options, profile)
|
|
|
|
self.log.info(
|
|
"Running with e10s: {}".format(prefs["browser.tabs.remote.autostart"])
|
|
)
|
|
self.log.info("Running with fission: {}".format(prefs["fission.autostart"]))
|
|
|
|
return profile
|
|
|
|
def environment(self, **kwargs):
|
|
kwargs["log"] = self.log
|
|
return test_environment(**kwargs)
|
|
|
|
def buildBrowserEnv(self, options, profileDir):
|
|
browserEnv = self.environment(
|
|
xrePath=options.xrePath, debugger=options.debugger
|
|
)
|
|
browserEnv["XPCOM_DEBUG_BREAK"] = "stack"
|
|
if options.topsrcdir:
|
|
browserEnv["MOZ_DEVELOPER_REPO_DIR"] = options.topsrcdir
|
|
if hasattr(options, "topobjdir"):
|
|
browserEnv["MOZ_DEVELOPER_OBJ_DIR"] = options.topobjdir
|
|
|
|
if mozinfo.info["asan"]:
|
|
# Disable leak checking for reftests for now
|
|
if "ASAN_OPTIONS" in browserEnv:
|
|
browserEnv["ASAN_OPTIONS"] += ":detect_leaks=0"
|
|
else:
|
|
browserEnv["ASAN_OPTIONS"] = "detect_leaks=0"
|
|
|
|
# Set environment defaults for jstestbrowser. Keep in sync with the
|
|
# defaults used in js/src/tests/lib/tests.py.
|
|
if options.suite == "jstestbrowser":
|
|
browserEnv["TZ"] = "PST8PDT"
|
|
browserEnv["LC_ALL"] = "en_US.UTF-8"
|
|
|
|
for v in options.environment:
|
|
ix = v.find("=")
|
|
if ix <= 0:
|
|
print("Error: syntax error in --setenv=" + v)
|
|
return None
|
|
browserEnv[v[:ix]] = v[ix + 1 :]
|
|
|
|
# Enable leaks detection to its own log file.
|
|
self.leakLogFile = os.path.join(profileDir, "runreftest_leaks.log")
|
|
browserEnv["XPCOM_MEM_BLOAT_LOG"] = self.leakLogFile
|
|
|
|
# TODO: this is always defined (as part of --enable-webrender which is default)
|
|
# can we make this default in the browser?
|
|
browserEnv["MOZ_ACCELERATED"] = "1"
|
|
|
|
if options.headless:
|
|
browserEnv["MOZ_HEADLESS"] = "1"
|
|
|
|
return browserEnv
|
|
|
|
def cleanup(self, profileDir):
|
|
if profileDir:
|
|
shutil.rmtree(profileDir, True)
|
|
|
|
def verifyTests(self, tests, options):
|
|
"""
|
|
Support --verify mode: Run test(s) many times in a variety of
|
|
configurations/environments in an effort to find intermittent
|
|
failures.
|
|
"""
|
|
|
|
self._populate_logger(options)
|
|
|
|
# Number of times to repeat test(s) when running with --repeat
|
|
VERIFY_REPEAT = 10
|
|
# Number of times to repeat test(s) when running test in separate browser
|
|
VERIFY_REPEAT_SINGLE_BROWSER = 5
|
|
|
|
def step1():
|
|
options.repeat = VERIFY_REPEAT
|
|
options.runUntilFailure = True
|
|
result = self.runTests(tests, options)
|
|
return result
|
|
|
|
def step2():
|
|
options.repeat = 0
|
|
options.runUntilFailure = False
|
|
for i in range(VERIFY_REPEAT_SINGLE_BROWSER):
|
|
result = self.runTests(tests, options)
|
|
if result != 0:
|
|
break
|
|
return result
|
|
|
|
def step3():
|
|
options.repeat = VERIFY_REPEAT
|
|
options.runUntilFailure = True
|
|
options.environment.append("MOZ_CHAOSMODE=0xfb")
|
|
result = self.runTests(tests, options)
|
|
options.environment.remove("MOZ_CHAOSMODE=0xfb")
|
|
return result
|
|
|
|
def step4():
|
|
options.repeat = 0
|
|
options.runUntilFailure = False
|
|
options.environment.append("MOZ_CHAOSMODE=0xfb")
|
|
for i in range(VERIFY_REPEAT_SINGLE_BROWSER):
|
|
result = self.runTests(tests, options)
|
|
if result != 0:
|
|
break
|
|
options.environment.remove("MOZ_CHAOSMODE=0xfb")
|
|
return result
|
|
|
|
steps = [
|
|
("1. Run each test %d times in one browser." % VERIFY_REPEAT, step1),
|
|
(
|
|
"2. Run each test %d times in a new browser each time."
|
|
% VERIFY_REPEAT_SINGLE_BROWSER,
|
|
step2,
|
|
),
|
|
(
|
|
"3. Run each test %d times in one browser, in chaos mode."
|
|
% VERIFY_REPEAT,
|
|
step3,
|
|
),
|
|
(
|
|
"4. Run each test %d times in a new browser each time, in chaos mode."
|
|
% VERIFY_REPEAT_SINGLE_BROWSER,
|
|
step4,
|
|
),
|
|
]
|
|
|
|
stepResults = {}
|
|
for (descr, step) in steps:
|
|
stepResults[descr] = "not run / incomplete"
|
|
|
|
startTime = datetime.now()
|
|
maxTime = timedelta(seconds=options.verify_max_time)
|
|
finalResult = "PASSED"
|
|
for (descr, step) in steps:
|
|
if (datetime.now() - startTime) > maxTime:
|
|
self.log.info("::: Test verification is taking too long: Giving up!")
|
|
self.log.info(
|
|
"::: So far, all checks passed, but not all checks were run."
|
|
)
|
|
break
|
|
self.log.info(":::")
|
|
self.log.info('::: Running test verification step "%s"...' % descr)
|
|
self.log.info(":::")
|
|
result = step()
|
|
if result != 0:
|
|
stepResults[descr] = "FAIL"
|
|
finalResult = "FAILED!"
|
|
break
|
|
stepResults[descr] = "Pass"
|
|
|
|
self.log.info(":::")
|
|
self.log.info("::: Test verification summary for:")
|
|
self.log.info(":::")
|
|
for test in tests:
|
|
self.log.info("::: " + test)
|
|
self.log.info(":::")
|
|
for descr in sorted(stepResults.keys()):
|
|
self.log.info("::: %s : %s" % (descr, stepResults[descr]))
|
|
self.log.info(":::")
|
|
self.log.info("::: Test verification %s" % finalResult)
|
|
self.log.info(":::")
|
|
|
|
return result
|
|
|
|
def runTests(self, tests, options, cmdargs=None):
|
|
cmdargs = cmdargs or []
|
|
self._populate_logger(options)
|
|
self.outputHandler = OutputHandler(
|
|
self.log, options.utilityPath, options.symbolsPath
|
|
)
|
|
|
|
if options.cleanupCrashes:
|
|
mozcrash.cleanup_pending_crash_reports()
|
|
|
|
manifests = self.resolver.resolveManifests(options, tests)
|
|
if options.filter:
|
|
manifests[""] = (options.filter, None)
|
|
|
|
if not getattr(options, "runTestsInParallel", False):
|
|
return self.runSerialTests(manifests, options, cmdargs)
|
|
|
|
cpuCount = multiprocessing.cpu_count()
|
|
|
|
# We have the directive, technology, and machine to run multiple test instances.
|
|
# Experimentation says that reftests are not overly CPU-intensive, so we can run
|
|
# multiple jobs per CPU core.
|
|
#
|
|
# Our Windows machines in automation seem to get upset when we run a lot of
|
|
# simultaneous tests on them, so tone things down there.
|
|
if sys.platform == "win32":
|
|
jobsWithoutFocus = cpuCount
|
|
else:
|
|
jobsWithoutFocus = 2 * cpuCount
|
|
|
|
totalJobs = jobsWithoutFocus + 1
|
|
perProcessArgs = [sys.argv[:] for i in range(0, totalJobs)]
|
|
|
|
host = "localhost"
|
|
port = 2828
|
|
if options.marionette:
|
|
host, port = options.marionette.split(":")
|
|
|
|
# First job is only needs-focus tests. Remaining jobs are
|
|
# non-needs-focus and chunked.
|
|
perProcessArgs[0].insert(-1, "--focus-filter-mode=needs-focus")
|
|
for (chunkNumber, jobArgs) in enumerate(perProcessArgs[1:], start=1):
|
|
jobArgs[-1:-1] = [
|
|
"--focus-filter-mode=non-needs-focus",
|
|
"--total-chunks=%d" % jobsWithoutFocus,
|
|
"--this-chunk=%d" % chunkNumber,
|
|
"--marionette=%s:%d" % (host, port),
|
|
]
|
|
port += 1
|
|
|
|
for jobArgs in perProcessArgs:
|
|
try:
|
|
jobArgs.remove("--run-tests-in-parallel")
|
|
except Exception:
|
|
pass
|
|
jobArgs[0:0] = [sys.executable, "-u"]
|
|
|
|
threads = [ReftestThread(args) for args in perProcessArgs[1:]]
|
|
for t in threads:
|
|
t.start()
|
|
|
|
while True:
|
|
# The test harness in each individual thread will be doing timeout
|
|
# handling on its own, so we shouldn't need to worry about any of
|
|
# the threads hanging for arbitrarily long.
|
|
for t in threads:
|
|
t.join(10)
|
|
if not any(t.is_alive() for t in threads):
|
|
break
|
|
|
|
# Run the needs-focus tests serially after the other ones, so we don't
|
|
# have to worry about races between the needs-focus tests *actually*
|
|
# needing focus and the dummy windows in the non-needs-focus tests
|
|
# trying to focus themselves.
|
|
focusThread = ReftestThread(perProcessArgs[0])
|
|
focusThread.start()
|
|
focusThread.join()
|
|
|
|
# Output the summaries that the ReftestThread filters suppressed.
|
|
summaryObjects = [defaultdict(int) for s in summaryLines]
|
|
for t in threads:
|
|
for (summaryObj, (text, categories)) in zip(summaryObjects, summaryLines):
|
|
threadMatches = t.summaryMatches[text]
|
|
for (attribute, description) in categories:
|
|
amount = int(threadMatches.group(attribute) if threadMatches else 0)
|
|
summaryObj[attribute] += amount
|
|
amount = int(threadMatches.group("total") if threadMatches else 0)
|
|
summaryObj["total"] += amount
|
|
|
|
print("REFTEST INFO | Result summary:")
|
|
for (summaryObj, (text, categories)) in zip(summaryObjects, summaryLines):
|
|
details = ", ".join(
|
|
[
|
|
"%d %s" % (summaryObj[attribute], description)
|
|
for (attribute, description) in categories
|
|
]
|
|
)
|
|
print(
|
|
"REFTEST INFO | "
|
|
+ text
|
|
+ ": "
|
|
+ str(summaryObj["total"])
|
|
+ " ("
|
|
+ details
|
|
+ ")"
|
|
)
|
|
|
|
return int(any(t.retcode != 0 for t in threads))
|
|
|
|
def handleTimeout(self, timeout, proc, utilityPath, debuggerInfo):
|
|
"""handle process output timeout"""
|
|
# TODO: bug 913975 : _processOutput should call self.processOutputLine
|
|
# one more time one timeout (I think)
|
|
self.log.error(
|
|
"%s | application timed out after %d seconds with no output"
|
|
% (self.lastTestSeen, int(timeout))
|
|
)
|
|
self.log.warning("Force-terminating active process(es).")
|
|
self.killAndGetStack(
|
|
proc, utilityPath, debuggerInfo, dump_screen=not debuggerInfo
|
|
)
|
|
|
|
def dumpScreen(self, utilityPath):
|
|
if self.haveDumpedScreen:
|
|
self.log.info(
|
|
"Not taking screenshot here: see the one that was previously logged"
|
|
)
|
|
return
|
|
self.haveDumpedScreen = True
|
|
dump_screen(utilityPath, self.log)
|
|
|
|
def killAndGetStack(self, process, utilityPath, debuggerInfo, dump_screen=False):
|
|
"""
|
|
Kill the process, preferrably in a way that gets us a stack trace.
|
|
Also attempts to obtain a screenshot before killing the process
|
|
if specified.
|
|
"""
|
|
|
|
if dump_screen:
|
|
self.dumpScreen(utilityPath)
|
|
|
|
if mozinfo.info.get("crashreporter", True) and not debuggerInfo:
|
|
if mozinfo.isWin:
|
|
# We should have a "crashinject" program in our utility path
|
|
crashinject = os.path.normpath(
|
|
os.path.join(utilityPath, "crashinject.exe")
|
|
)
|
|
if os.path.exists(crashinject):
|
|
status = subprocess.Popen([crashinject, str(process.pid)]).wait()
|
|
printstatus("crashinject", status)
|
|
if status == 0:
|
|
return
|
|
else:
|
|
try:
|
|
process.kill(sig=signal.SIGABRT)
|
|
except OSError:
|
|
# https://bugzilla.mozilla.org/show_bug.cgi?id=921509
|
|
self.log.info("Can't trigger Breakpad, process no longer exists")
|
|
return
|
|
self.log.info("Can't trigger Breakpad, just killing process")
|
|
process.kill()
|
|
|
|
def runApp(
|
|
self,
|
|
options,
|
|
cmdargs=None,
|
|
timeout=None,
|
|
debuggerInfo=None,
|
|
symbolsPath=None,
|
|
valgrindPath=None,
|
|
valgrindArgs=None,
|
|
valgrindSuppFiles=None,
|
|
**profileArgs
|
|
):
|
|
|
|
if cmdargs is None:
|
|
cmdargs = []
|
|
cmdargs = cmdargs[:]
|
|
|
|
if self.use_marionette:
|
|
cmdargs.append("-marionette")
|
|
|
|
binary = options.app
|
|
profile = self.createReftestProfile(options, **profileArgs)
|
|
|
|
# browser environment
|
|
env = self.buildBrowserEnv(options, profile.profile)
|
|
|
|
def timeoutHandler():
|
|
self.handleTimeout(timeout, proc, options.utilityPath, debuggerInfo)
|
|
|
|
interactive = False
|
|
debug_args = None
|
|
if debuggerInfo:
|
|
interactive = debuggerInfo.interactive
|
|
debug_args = [debuggerInfo.path] + debuggerInfo.args
|
|
|
|
def record_last_test(message):
|
|
"""Records the last test seen by this harness for the benefit of crash logging."""
|
|
|
|
def testid(test):
|
|
if " " in test:
|
|
return test.split(" ")[0]
|
|
return test
|
|
|
|
if message["action"] == "test_start":
|
|
self.lastTestSeen = testid(message["test"])
|
|
elif message["action"] == "test_end":
|
|
if self.lastTest and message["test"] == self.lastTest:
|
|
self.lastTestSeen = "Last test finished"
|
|
else:
|
|
self.lastTestSeen = "{} (finished)".format(testid(message["test"]))
|
|
|
|
self.log.add_handler(record_last_test)
|
|
|
|
kp_kwargs = {
|
|
"kill_on_timeout": False,
|
|
"cwd": SCRIPT_DIRECTORY,
|
|
"onTimeout": [timeoutHandler],
|
|
"processOutputLine": [self.outputHandler],
|
|
}
|
|
|
|
if mozinfo.isWin or mozinfo.isMac:
|
|
# Prevents log interleaving on Windows at the expense of losing
|
|
# true log order. See bug 798300 and bug 1324961 for more details.
|
|
kp_kwargs["processStderrLine"] = [self.outputHandler]
|
|
|
|
if interactive:
|
|
# If an interactive debugger is attached,
|
|
# don't use timeouts, and don't capture ctrl-c.
|
|
timeout = None
|
|
signal.signal(signal.SIGINT, lambda sigid, frame: None)
|
|
|
|
runner_cls = mozrunner.runners.get(
|
|
mozinfo.info.get("appname", "firefox"), mozrunner.Runner
|
|
)
|
|
runner = runner_cls(
|
|
profile=profile,
|
|
binary=binary,
|
|
process_class=mozprocess.ProcessHandlerMixin,
|
|
cmdargs=cmdargs,
|
|
env=env,
|
|
process_args=kp_kwargs,
|
|
)
|
|
runner.start(
|
|
debug_args=debug_args, interactive=interactive, outputTimeout=timeout
|
|
)
|
|
proc = runner.process_handler
|
|
self.outputHandler.proc_name = "GECKO({})".format(proc.pid)
|
|
|
|
# Used to defer a possible IOError exception from Marionette
|
|
marionette_exception = None
|
|
|
|
if self.use_marionette:
|
|
marionette_args = {
|
|
"socket_timeout": options.marionette_socket_timeout,
|
|
"startup_timeout": options.marionette_startup_timeout,
|
|
"symbols_path": options.symbolsPath,
|
|
}
|
|
if options.marionette:
|
|
host, port = options.marionette.split(":")
|
|
marionette_args["host"] = host
|
|
marionette_args["port"] = int(port)
|
|
|
|
try:
|
|
marionette = Marionette(**marionette_args)
|
|
marionette.start_session()
|
|
|
|
addons = Addons(marionette)
|
|
if options.specialPowersExtensionPath:
|
|
addons.install(options.specialPowersExtensionPath, temp=True)
|
|
|
|
addons.install(options.reftestExtensionPath, temp=True)
|
|
|
|
marionette.delete_session()
|
|
except IOError:
|
|
# Any IOError as thrown by Marionette means that something is
|
|
# wrong with the process, like a crash or the socket is no
|
|
# longer open. We defer raising this specific error so that
|
|
# post-test checks for leaks and crashes are performed and
|
|
# reported first.
|
|
marionette_exception = sys.exc_info()
|
|
|
|
status = runner.wait()
|
|
runner.process_handler = None
|
|
self.outputHandler.proc_name = None
|
|
|
|
if status:
|
|
msg = (
|
|
"TEST-UNEXPECTED-FAIL | %s | application terminated with exit code %s"
|
|
% (self.lastTestSeen, status)
|
|
)
|
|
# use process_output so message is logged verbatim
|
|
self.log.process_output(None, msg)
|
|
|
|
crashed = mozcrash.log_crashes(
|
|
self.log,
|
|
os.path.join(profile.profile, "minidumps"),
|
|
options.symbolsPath,
|
|
test=self.lastTestSeen,
|
|
)
|
|
if not status and crashed:
|
|
status = 1
|
|
|
|
runner.cleanup()
|
|
self.cleanup(profile.profile)
|
|
|
|
if marionette_exception is not None:
|
|
exc, value, tb = marionette_exception
|
|
raise reraise(exc, value, tb)
|
|
|
|
self.log.info("Process mode: {}".format("e10s" if options.e10s else "non-e10s"))
|
|
return status
|
|
|
|
def getActiveTests(self, manifests, options, testDumpFile=None):
|
|
# These prefs will cause reftest.jsm to parse the manifests,
|
|
# dump the resulting tests to a file, and exit.
|
|
prefs = {
|
|
"reftest.manifests": json.dumps(manifests),
|
|
"reftest.manifests.dumpTests": testDumpFile or self.testDumpFile,
|
|
}
|
|
cmdargs = []
|
|
self.runApp(options, cmdargs=cmdargs, prefs=prefs)
|
|
|
|
if not os.path.isfile(self.testDumpFile):
|
|
print("Error: parsing manifests failed!")
|
|
sys.exit(1)
|
|
|
|
with open(self.testDumpFile, "r") as fh:
|
|
tests = json.load(fh)
|
|
|
|
if os.path.isfile(self.testDumpFile):
|
|
mozfile.remove(self.testDumpFile)
|
|
|
|
for test in tests:
|
|
# Name and path are expected by manifestparser, but not used in reftest.
|
|
test["name"] = test["path"] = test["url1"]
|
|
|
|
mp = TestManifest(strict=False)
|
|
mp.tests = tests
|
|
|
|
filters = []
|
|
if options.totalChunks:
|
|
filters.append(
|
|
mpf.chunk_by_manifest(options.thisChunk, options.totalChunks)
|
|
)
|
|
|
|
tests = mp.active_tests(exists=False, filters=filters)
|
|
return tests
|
|
|
|
def runSerialTests(self, manifests, options, cmdargs=None):
|
|
debuggerInfo = None
|
|
if options.debugger:
|
|
debuggerInfo = mozdebug.get_debugger_info(
|
|
options.debugger, options.debuggerArgs, options.debuggerInteractive
|
|
)
|
|
|
|
def run(**kwargs):
|
|
if kwargs.get("tests"):
|
|
self.lastTest = kwargs["tests"][-1]["identifier"]
|
|
if not isinstance(self.lastTest, string_types):
|
|
self.lastTest = " ".join(self.lastTest)
|
|
|
|
status = self.runApp(
|
|
options,
|
|
manifests=manifests,
|
|
cmdargs=cmdargs,
|
|
# We generally want the JS harness or marionette
|
|
# to handle timeouts if they can.
|
|
# The default JS harness timeout is currently
|
|
# 300 seconds (default options.timeout).
|
|
# The default Marionette socket timeout is
|
|
# currently 360 seconds.
|
|
# Give the JS harness extra time to deal with
|
|
# its own timeouts and try to usually exceed
|
|
# the 360 second marionette socket timeout.
|
|
# See bug 479518 and bug 1414063.
|
|
timeout=options.timeout + 70.0,
|
|
debuggerInfo=debuggerInfo,
|
|
symbolsPath=options.symbolsPath,
|
|
**kwargs
|
|
)
|
|
|
|
mozleak.process_leak_log(
|
|
self.leakLogFile,
|
|
leak_thresholds=options.leakThresholds,
|
|
stack_fixer=get_stack_fixer_function(
|
|
options.utilityPath, options.symbolsPath
|
|
),
|
|
)
|
|
return status
|
|
|
|
if not self.run_by_manifest:
|
|
return run()
|
|
|
|
tests = self.getActiveTests(manifests, options)
|
|
tests_by_manifest = defaultdict(list)
|
|
ids_by_manifest = defaultdict(list)
|
|
for t in tests:
|
|
tests_by_manifest[t["manifest"]].append(t)
|
|
test_id = t["identifier"]
|
|
if not isinstance(test_id, string_types):
|
|
test_id = " ".join(test_id)
|
|
ids_by_manifest[t["manifestID"]].append(test_id)
|
|
|
|
self.log.suite_start(ids_by_manifest, name=options.suite)
|
|
|
|
overall = 0
|
|
for manifest, tests in tests_by_manifest.items():
|
|
self.log.info("Running tests in {}".format(manifest))
|
|
status = run(tests=tests)
|
|
overall = overall or status
|
|
|
|
self.log.suite_end(extra={"results": self.outputHandler.results})
|
|
return overall
|
|
|
|
def copyExtraFilesToProfile(self, options, profile):
|
|
"Copy extra files or dirs specified on the command line to the testing profile."
|
|
profileDir = profile.profile
|
|
for f in options.extraProfileFiles:
|
|
abspath = self.getFullPath(f)
|
|
if os.path.isfile(abspath):
|
|
if os.path.basename(abspath) == "user.js":
|
|
extra_prefs = mozprofile.Preferences.read_prefs(abspath)
|
|
profile.set_preferences(extra_prefs)
|
|
elif os.path.basename(abspath).endswith(".dic"):
|
|
hyphDir = os.path.join(profileDir, "hyphenation")
|
|
if not os.path.exists(hyphDir):
|
|
os.makedirs(hyphDir)
|
|
shutil.copy2(abspath, hyphDir)
|
|
else:
|
|
shutil.copy2(abspath, profileDir)
|
|
elif os.path.isdir(abspath):
|
|
dest = os.path.join(profileDir, os.path.basename(abspath))
|
|
shutil.copytree(abspath, dest)
|
|
else:
|
|
self.log.warning(
|
|
"runreftest.py | Failed to copy %s to profile" % abspath
|
|
)
|
|
continue
|
|
|
|
|
|
def run_test_harness(parser, options):
|
|
reftest = RefTest(options.suite)
|
|
parser.validate(options, reftest)
|
|
|
|
# We have to validate options.app here for the case when the mach
|
|
# command is able to find it after argument parsing. This can happen
|
|
# when running from a tests archive.
|
|
if not options.app:
|
|
parser.error("could not find the application path, --appname must be specified")
|
|
|
|
options.app = reftest.getFullPath(options.app)
|
|
if not os.path.exists(options.app):
|
|
parser.error(
|
|
"Error: Path %(app)s doesn't exist. Are you executing "
|
|
"$objdir/_tests/reftest/runreftest.py?" % {"app": options.app}
|
|
)
|
|
|
|
if options.xrePath is None:
|
|
options.xrePath = os.path.dirname(options.app)
|
|
|
|
if options.verify:
|
|
result = reftest.verifyTests(options.tests, options)
|
|
else:
|
|
result = reftest.runTests(options.tests, options)
|
|
|
|
return result
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = reftestcommandline.DesktopArgumentsParser()
|
|
options = parser.parse_args()
|
|
sys.exit(run_test_harness(parser, options))
|