# # ***** BEGIN LICENSE BLOCK ***** # Version: MPL 1.1/GPL 2.0/LGPL 2.1 # # The contents of this file are subject to the Mozilla Public License Version # 1.1 (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # http://www.mozilla.org/MPL/ # # Software distributed under the License is distributed on an "AS IS" basis, # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License # for the specific language governing rights and limitations under the # License. # # The Original Code is mozilla.org code. # # The Initial Developer of the Original Code is # Mozilla Foundation. # Portions created by the Initial Developer are Copyright (C) 1998 # the Initial Developer. All Rights Reserved. # # Contributor(s): # Robert Sayre # Jeff Walden # # Alternatively, the contents of this file may be used under the terms of # either the GNU General Public License Version 2 or later (the "GPL"), or # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), # in which case the provisions of the GPL or the LGPL are applicable instead # of those above. If you wish to allow use of your version of this file only # under the terms of either the GPL or the LGPL, and not to allow others to # use your version of this file under the terms of the MPL, indicate your # decision by deleting the provisions above and replace them with the notice # and other provisions required by the GPL or the LGPL. If you do not delete # the provisions above, a recipient may use your version of this file under # the terms of any one of the MPL, the GPL or the LGPL. # # ***** END LICENSE BLOCK ***** """ Runs the Mochitest test harness. """ from datetime import datetime import logging import optparse import os import os.path import re import sys import time from urllib import quote_plus as encodeURIComponent import urllib2 import commands import automation # Path to the test script on the server TEST_SERVER_HOST = "localhost:8888" TEST_PATH = "/tests/" CHROME_PATH = "/redirect.html"; A11Y_PATH = "/redirect-a11y.html" TESTS_URL = "http://" + TEST_SERVER_HOST + TEST_PATH CHROMETESTS_URL = "http://" + TEST_SERVER_HOST + CHROME_PATH A11YTESTS_URL = "http://" + TEST_SERVER_HOST + A11Y_PATH SERVER_SHUTDOWN_URL = "http://" + TEST_SERVER_HOST + "/server/shutdown" # main browser chrome URL, same as browser.chromeURL pref #ifdef MOZ_SUITE BROWSER_CHROME_URL = "chrome://navigator/content/navigator.xul" #else BROWSER_CHROME_URL = "chrome://browser/content/browser.xul" #endif # Max time in seconds to wait for server startup before tests will fail -- if # this seems big, it's mostly for debug machines where cold startup # (particularly after a build) takes forever. SERVER_STARTUP_TIMEOUT = 45 INFINITY = 1.0e3000 oldcwd = os.getcwd() SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0]))) os.chdir(SCRIPT_DIRECTORY) PROFILE_DIRECTORY = os.path.abspath("./mochitesttestingprofile") LEAK_REPORT_FILE = PROFILE_DIRECTORY + "/" + "leaks-report.log" log = logging.getLogger() ####################### # COMMANDLINE OPTIONS # ####################### class MochitestOptions(optparse.OptionParser): """Parses Mochitest commandline options.""" def __init__(self, **kwargs): optparse.OptionParser.__init__(self, **kwargs) defaults = {} self.add_option("--close-when-done", action = "store_true", dest = "closeWhenDone", help = "close the application when tests are done running") defaults["closeWhenDone"] = False self.add_option("--appname", action = "store", type = "string", dest = "app", help = "absolute path to application, overriding default") defaults["app"] = automation.DEFAULT_APP self.add_option("--log-file", action = "store", type = "string", dest = "logFile", metavar = "FILE", help = "file to which logging occurs") defaults["logFile"] = "" self.add_option("--autorun", action = "store_true", dest = "autorun", help = "start running tests when the application starts") defaults["autorun"] = False LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR", "FATAL") LEVEL_STRING = ", ".join(LOG_LEVELS) self.add_option("--console-level", action = "store", type = "choice", dest = "consoleLevel", choices = LOG_LEVELS, metavar = "LEVEL", help = "one of %s to determine the level of console " "logging" % LEVEL_STRING) defaults["consoleLevel"] = None self.add_option("--file-level", action = "store", type = "choice", dest = "fileLevel", choices = LOG_LEVELS, metavar = "LEVEL", help = "one of %s to determine the level of file " "logging if a file has been specified, defaulting " "to INFO" % LEVEL_STRING) defaults["fileLevel"] = "INFO" self.add_option("--chrome", action = "store_true", dest = "chrome", help = "run chrome Mochitests") defaults["chrome"] = False self.add_option("--test-path", action = "store", type = "string", dest = "testPath", help = "start in the given directory's tests") defaults["testPath"] = "" self.add_option("--browser-chrome", action = "store_true", dest = "browserChrome", help = "run browser chrome Mochitests") defaults["browserChrome"] = False self.add_option("--a11y", action = "store_true", dest = "a11y", help = "run accessibility Mochitests"); self.add_option("--setenv", action = "append", type = "string", dest = "environment", metavar = "NAME=VALUE", help = "sets the given variable in the application's " "environment") defaults["environment"] = [] self.add_option("--browser-arg", action = "append", type = "string", dest = "browserArgs", metavar = "ARG", help = "provides an argument to the test application") defaults["browserArgs"] = [] self.add_option("--leak-threshold", action = "store", type = "int", dest = "leakThreshold", metavar = "THRESHOLD", help = "fail if the number of bytes leaked through " "refcounted objects (or bytes in classes with " "MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) is greater " "than the given number") defaults["leakThreshold"] = INFINITY self.add_option("--fatal-assertions", action = "store_true", dest = "fatalAssertions", help = "abort testing whenever an assertion is hit " "(requires a debug build to be effective)") defaults["fatalAssertions"] = False # -h, --help are automatically handled by OptionParser self.set_defaults(**defaults) usage = """\ Usage instructions for runtests.py. All arguments are optional. If --chrome is specified, chrome tests will be run instead of web content tests. If --browser-chrome is specified, browser-chrome tests will be run instead of web content tests. See for details on the logging levels.""" self.set_usage(usage) ####################### # HTTP SERVER SUPPORT # ####################### class MochitestServer: "Web server used to serve Mochitests, for closer fidelity to the real web." def __init__(self, options): self._closeWhenDone = options.closeWhenDone def start(self): "Run the Mochitest server, returning the process ID of the server." env = dict(os.environ) if automation.UNIXISH: env["LD_LIBRARY_PATH"] = automation.DIST_BIN env["MOZILLA_FIVE_HOME"] = automation.DIST_BIN env["XPCOM_DEBUG_BREAK"] = "warn" args = ["-v", "170", "-f", "./" + "httpd.js", "-f", "./" + "server.js"] xpcshell = automation.DIST_BIN + "/" + "xpcshell"; self._process = automation.Process(xpcshell, args, env = env) pid = self._process.pid if pid < 0: print "Error starting server." sys.exit(2) log.info("Server pid: %d", pid) def ensureReady(self, timeout): assert timeout >= 0 aliveFile = PROFILE_DIRECTORY + "/" + "server_alive.txt" i = 0 while i < timeout: if os.path.exists(aliveFile): break time.sleep(1) i += 1 else: print "Timed out while waiting for server startup." self.stop() sys.exit(1) def stop(self): try: c = urllib2.urlopen(SERVER_SHUTDOWN_URL) c.read() c.close() self._process.wait() except: self._process.kill() ################# # MAIN FUNCTION # ################# def main(): parser = MochitestOptions() options, args = parser.parse_args() # If the leak threshold wasn't explicitly set, we override the default of # infinity when the set of tests we're running are known to leak only a # particular amount. If for some reason you don't want a new leak threshold # enforced, just pass an explicit --leak-threshold=N to prevent the override. maybeForceLeakThreshold(options) if not os.path.exists(options.app): msg = """\ Error: Path %(app)s doesn't exist. Are you executing $objdir/_tests/testing/mochitest/runtests.py?""" print msg % {"app": options.app} sys.exit(1) # browser environment browserEnv = dict(os.environ) # These variables are necessary for correct application startup; change # via the commandline at your own risk. browserEnv["NO_EM_RESTART"] = "1" browserEnv["XPCOM_DEBUG_BREAK"] = "warn" if automation.UNIXISH: browserEnv["LD_LIBRARY_PATH"] = automation.DIST_BIN browserEnv["MOZILLA_FIVE_HOME"] = automation.DIST_BIN browserEnv["GNOME_DISABLE_CRASH_DIALOG"] = "1" for v in options.environment: ix = v.find("=") if ix <= 0: print "Error: syntax error in --setenv=" + v sys.exit(1) browserEnv[v[:ix]] = v[ix + 1:] automation.initializeProfile(PROFILE_DIRECTORY) manifest = addChromeToProfile(options) server = MochitestServer(options) server.start() # If we're lucky, the server has fully started by now, and all paths are # ready, etc. However, xpcshell cold start times suck, at least for debug # builds. We'll try to connect to the server for awhile, and if we fail, # we'll try to kill the server and exit with an error. server.ensureReady(SERVER_STARTUP_TIMEOUT) # URL parameters to test URL: # # autorun -- kick off tests automatically # closeWhenDone -- runs quit.js after tests # logFile -- logs test run to an absolute path # # consoleLevel, fileLevel: set the logging level of the console and # file logs, if activated. # testURL = TESTS_URL + options.testPath urlOpts = [] if options.chrome: testURL = CHROMETESTS_URL if options.testPath: urlOpts.append("testPath=" + encodeURIComponent(options.testPath)) elif options.a11y: testURL = A11YTESTS_URL if options.testPath: urlOpts.append("testPath=" + encodeURIComponent(options.testPath)) elif options.browserChrome: testURL = "about:blank" # allow relative paths for logFile if options.logFile: options.logFile = os.path.normpath(os.path.join(oldcwd, options.logFile)) if options.browserChrome: makeTestConfig(options) else: if options.autorun: urlOpts.append("autorun=1") if options.closeWhenDone: urlOpts.append("closeWhenDone=1") if options.logFile: urlOpts.append("logFile=" + encodeURIComponent(options.logFile)) urlOpts.append("fileLevel=" + encodeURIComponent(options.fileLevel)) if options.consoleLevel: urlOpts.append("consoleLevel=" + encodeURIComponent(options.consoleLevel)) if len(urlOpts) > 0: testURL += "?" + "&".join(urlOpts) browserEnv["XPCOM_MEM_BLOAT_LOG"] = LEAK_REPORT_FILE if options.fatalAssertions: browserEnv["XPCOM_DEBUG_BREAK"] = "stack-and-abort" start = automation.runApp(testURL, browserEnv, options.app, PROFILE_DIRECTORY, options.browserArgs) # Server's no longer needed, and perhaps more importantly, anything it might # spew to console shouldn't disrupt the leak information table we print next. server.stop() if not os.path.exists(LEAK_REPORT_FILE): log.info("WARNING refcount logging is off, so leaks can't be detected!") else: leaks = open(LEAK_REPORT_FILE, "r") for line in leaks: log.info(line.rstrip()) leaks.close() threshold = options.leakThreshold leaks = open(LEAK_REPORT_FILE, "r") # Per-Inst Leaked Total Rem ... # 0 TOTAL 17 192 419115886 2 ... # 833 nsTimerImpl 60 120 24726 2 ... lineRe = re.compile(r"^\s*\d+\s+(?P\S+)\s+" r"(?P-?\d+)\s+(?P-?\d+)\s+" r"\d+\s+(?P-?\d+)") thresholdExceeded = False seenTotal = False prefix = "TEST-PASS" for line in leaks: matches = lineRe.match(line) if not matches: continue name = matches.group("name") size = int(matches.group("size")) bytesLeaked = int(matches.group("bytesLeaked")) numLeaked = int(matches.group("numLeaked")) if size < 0 or bytesLeaked < 0 or numLeaked < 0: log.info("TEST-UNEXPECTED-FAIL | runtests-leaks | negative leaks caught!") if "TOTAL" == name: seenTotal = True if bytesLeaked > threshold: thresholdExceeded = True prefix = "TEST-UNEXPECTED-FAIL" log.info("TEST-UNEXPECTED-FAIL | runtests-leaks | leaked %d bytes during test execution (should " "have leaked no more than %d bytes)", bytesLeaked, threshold) elif bytesLeaked > 0: log.info("TEST-PASS | runtests-leaks | WARNING leaked %d bytes during test execution", bytesLeaked) else: log.info("TEST-PASS | runtests-leaks | no leaks detected!") else: if numLeaked != 0: if numLeaked > 1: instance = "instances" rest = " each (%s bytes total)" % matches.group("bytesLeaked") else: instance = "instance" rest = "" log.info("%(prefix)s | runtests-leaks | leaked %(numLeaked)d %(instance)s of %(name)s " "with size %(size)s bytes%(rest)s" % { "prefix": prefix, "numLeaked": numLeaked, "instance": instance, "name": name, "size": matches.group("size"), "rest": rest }) if not seenTotal: log.info("TEST-UNEXPECTED-FAIL | runtests-leaks | missing output line for total leaks!") leaks.close() # print test run times finish = datetime.now() log.info(" started: %s", str(start)) log.info("finished: %s", str(finish)) # delete the profile and manifest os.remove(manifest) # hanging due to non-halting threads is no fun; assume we hit the errors we # were going to hit already and exit with a success code sys.exit(0) ####################### # CONFIGURATION SETUP # ####################### def maybeForceLeakThreshold(options): """ Modifies an unset leak threshold if it is known that a particular leak threshold can be successfully forced for the particular Mochitest type and platform in use. """ if options.leakThreshold == INFINITY: if options.chrome: # We don't leak running the --chrome tests. options.leakThreshold = 0 elif options.browserChrome: # We still leak a nondeterministic amount running browser-chrome tests. pass else: # We leak nothing on Windows or OS X running normal Mochitests. if automation.IS_MAC or automation.IS_WIN32: options.leakThreshold = 0 def makeTestConfig(options): "Creates a test configuration file for customizing test execution." def boolString(b): if b: return "true" return "false" logFile = options.logFile.replace("\\", "\\\\") testPath = options.testPath.replace("\\", "\\\\") content = """\ ({ autoRun: %(autorun)s, closeWhenDone: %(closeWhenDone)s, logPath: "%(logPath)s", testPath: "%(testPath)s" })""" % {"autorun": boolString(options.autorun), "closeWhenDone": boolString(options.closeWhenDone), "logPath": logFile, "testPath": testPath} config = open(PROFILE_DIRECTORY + "/" + "testConfig.js", "w") config.write(content) config.close() def addChromeToProfile(options): "Adds MochiKit chrome tests to the profile." chromedir = PROFILE_DIRECTORY + "/" + "chrome" os.mkdir(chromedir) chrome = [] part = """ @namespace url("http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"); /* set default namespace to XUL */ toolbar, toolbarpalette { background-color: rgb(235, 235, 235) !important; } toolbar#nav-bar { background-image: none !important; } """ chrome.append(part) # write userChrome.css chromeFile = open(PROFILE_DIRECTORY + "/" + "userChrome.css", "a") chromeFile.write("".join(chrome)) chromeFile.close() # register our chrome dir chrometestDir = os.path.abspath(".") + "/" if automation.IS_WIN32: chrometestDir = "file:///" + chrometestDir.replace("\\", "/") (path, leaf) = os.path.split(options.app) manifest = path + "/" + "chrome/mochikit.manifest" manifestFile = open(manifest, "w") manifestFile.write("content mochikit " + chrometestDir + " contentaccessible=yes\n") if options.browserChrome: overlayLine = "overlay " + BROWSER_CHROME_URL + " " \ "chrome://mochikit/content/browser-test-overlay.xul\n" manifestFile.write(overlayLine) manifestFile.close() return manifest ######### # DO IT # ######### if __name__ == "__main__": main()