gecko-dev/testing/mochitest/runtests.py.in

#
# ***** BEGIN LICENSE BLOCK *****
# Version: MPL 1.1/GPL 2.0/LGPL 2.1
#
# The contents of this file are subject to the Mozilla Public License Version
# 1.1 (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
# http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS" basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.
#
# The Original Code is mozilla.org code.
#
# The Initial Developer of the Original Code is
# Mozilla Foundation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Robert Sayre <sayrer@gmail.com>
#   Jeff Walden <jwalden+bmo@mit.edu>
#   Serge Gautherie <sgautherie.bz@free.fr>
#
# Alternatively, the contents of this file may be used under the terms of
# either the GNU General Public License Version 2 or later (the "GPL"), or
# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
# in which case the provisions of the GPL or the LGPL are applicable instead
# of those above. If you wish to allow use of your version of this file only
# under the terms of either the GPL or the LGPL, and not to allow others to
# use your version of this file under the terms of the MPL, indicate your
# decision by deleting the provisions above and replace them with the notice
# and other provisions required by the GPL or the LGPL. If you do not delete
# the provisions above, a recipient may use your version of this file under
# the terms of any one of the MPL, the GPL or the LGPL.
#
# ***** END LICENSE BLOCK *****

"""
Runs the Mochitest test harness.
"""

from datetime import datetime
import optparse
import os
import os.path
import sys
import time
import shutil
from urllib import quote_plus as encodeURIComponent
import urllib2
import commands
import automation
from automationutils import addCommonOptions, processLeakLog

# Path to the test script on the server
TEST_SERVER_HOST = "localhost:8888"
TEST_PATH = "/tests/"
CHROME_PATH = "/redirect.html";
A11Y_PATH = "/redirect-a11y.html"
TESTS_URL = "http://" + TEST_SERVER_HOST + TEST_PATH
CHROMETESTS_URL = "http://" + TEST_SERVER_HOST + CHROME_PATH
A11YTESTS_URL = "http://" + TEST_SERVER_HOST + A11Y_PATH
SERVER_SHUTDOWN_URL = "http://" + TEST_SERVER_HOST + "/server/shutdown"
# main browser chrome URL, same as browser.chromeURL pref
#ifdef MOZ_SUITE
BROWSER_CHROME_URL = "chrome://navigator/content/navigator.xul"
#else
BROWSER_CHROME_URL = "chrome://browser/content/browser.xul"
#endif

# Max time in seconds to wait for server startup before tests will fail -- if
# this seems big, it's mostly for debug machines where cold startup
# (particularly after a build) takes forever.
if automation.IS_DEBUG_BUILD:
    SERVER_STARTUP_TIMEOUT = 180
else:
    SERVER_STARTUP_TIMEOUT = 45

oldcwd = os.getcwd()
SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
os.chdir(SCRIPT_DIRECTORY)

PROFILE_DIRECTORY = os.path.abspath("./mochitesttestingprofile")

LEAK_REPORT_FILE = os.path.join(PROFILE_DIRECTORY, "runtests_leaks.log")

# Map of debugging programs to information about them, like default arguments
# and whether or not they are interactive.
DEBUGGER_INFO = {
  # gdb requires that you supply the '--args' flag in order to pass arguments
  # after the executable name to the executable.
  "gdb": {
    "interactive": True,
    "args": "-q --args"
  },

  # valgrind doesn't explain much about leaks unless you set the
  # '--leak-check=full' flag.
  "valgrind": {
    "interactive": False,
    "args": "--leak-check=full"
  }
}

#######################
# COMMANDLINE OPTIONS #
#######################

class MochitestOptions(optparse.OptionParser):
  """Parses Mochitest commandline options."""
  def __init__(self, **kwargs):
    optparse.OptionParser.__init__(self, **kwargs)
    defaults = {}

    # we want to pass down everything from automation.__all__
    addCommonOptions(self, defaults=dict(zip(automation.__all__, [getattr(automation, x) for x in automation.__all__])))

    self.add_option("--close-when-done",
                    action = "store_true", dest = "closeWhenDone",
                    help = "close the application when tests are done running")
    defaults["closeWhenDone"] = False

    self.add_option("--appname",
                    action = "store", type = "string", dest = "app",
                    help = "absolute path to application, overriding default")
    defaults["app"] = os.path.join(SCRIPT_DIRECTORY, automation.DEFAULT_APP)

    self.add_option("--utility-path",
                    action = "store", type = "string", dest = "utilityPath",
                    help = "absolute path to directory containing utility programs (xpcshell, ssltunnel, certutil)")
    defaults["utilityPath"] = automation.DIST_BIN

    self.add_option("--certificate-path",
                    action = "store", type = "string", dest = "certPath",
                    help = "absolute path to directory containing certificate store to use testing profile")
    defaults["certPath"] = automation.CERTS_SRC_DIR

    self.add_option("--log-file",
                    action = "store", type = "string",
                    dest = "logFile", metavar = "FILE",
                    help = "file to which logging occurs")
    defaults["logFile"] = ""

    self.add_option("--autorun",
                    action = "store_true", dest = "autorun",
                    help = "start running tests when the application starts")
    defaults["autorun"] = False

    self.add_option("--total-chunks",
                    type = "int", dest = "totalChunks",
                    help = "how many chunks to split the tests up into")
    defaults["totalChunks"] = None

    self.add_option("--this-chunk",
                    type = "int", dest = "thisChunk",
                    help = "which chunk to run")
    defaults["thisChunk"] = None

    self.add_option("--chunk-by-dir",
                    type = "int", dest = "chunkByDir",
                    help = "group tests together in the same chunk that are in the same top chunkByDir directories")
    defaults["chunkByDir"] = 0

    self.add_option("--shuffle",
                    dest = "shuffle",
                    action = "store_true",
                    help = "randomize test order")
    defaults["shuffle"] = False

    LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR", "FATAL")
    LEVEL_STRING = ", ".join(LOG_LEVELS)

    self.add_option("--console-level",
                    action = "store", type = "choice", dest = "consoleLevel",
                    choices = LOG_LEVELS, metavar = "LEVEL",
                    help = "one of %s to determine the level of console "
                           "logging" % LEVEL_STRING)
    defaults["consoleLevel"] = None

    self.add_option("--file-level", 
                    action = "store", type = "choice", dest = "fileLevel",
                    choices = LOG_LEVELS, metavar = "LEVEL",
                    help = "one of %s to determine the level of file "
                           "logging if a file has been specified, defaulting "
                           "to INFO" % LEVEL_STRING)
    defaults["fileLevel"] = "INFO"

    self.add_option("--chrome",
                    action = "store_true", dest = "chrome",
                    help = "run chrome Mochitests")
    defaults["chrome"] = False

    self.add_option("--test-path",
                    action = "store", type = "string", dest = "testPath",
                    help = "start in the given directory's tests")
    defaults["testPath"] = ""

    self.add_option("--browser-chrome",
                    action = "store_true", dest = "browserChrome",
                    help = "run browser chrome Mochitests")
    defaults["browserChrome"] = False

    self.add_option("--a11y",
                    action = "store_true", dest = "a11y",
                    help = "run accessibility Mochitests");

    self.add_option("--setenv",
                    action = "append", type = "string",
                    dest = "environment", metavar = "NAME=VALUE",
                    help = "sets the given variable in the application's "
                           "environment")
    defaults["environment"] = []
    
    self.add_option("--setpref",
                    action = "append", type = "string",
                    dest = "extraPrefs", metavar = "PREF=VALUE",
                    help = "defines an extra user preference")
    defaults["extraPrefs"] = []    
    
    self.add_option("--browser-arg",
                    action = "append", type = "string",
                    dest = "browserArgs", metavar = "ARG",
                    help = "provides an argument to the test application")
    defaults["browserArgs"] = []

    self.add_option("--leak-threshold",
                    action = "store", type = "int",
                    dest = "leakThreshold", metavar = "THRESHOLD",
                    help = "fail if the number of bytes leaked through "
                           "refcounted objects (or bytes in classes with "
                           "MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) is greater "
                           "than the given number")
    defaults["leakThreshold"] = 0

    self.add_option("--fatal-assertions",
                    action = "store_true", dest = "fatalAssertions",
                    help = "abort testing whenever an assertion is hit "
                           "(requires a debug build to be effective)")
    defaults["fatalAssertions"] = False

    self.add_option("--extra-profile-file",
                    action = "append", dest = "extraProfileFiles",
                    help = "copy specified files/dirs to testing profile")
    defaults["extraProfileFiles"] = []

    self.add_option("--debugger",
                    action = "store", dest = "debugger",
                    help = "use the given debugger to launch the application")

    self.add_option("--debugger-args",
                    action = "store", dest = "debuggerArgs",
                    help = "pass the given args to the debugger _before_ "
                           "the application on the command line")

    self.add_option("--debugger-interactive",
                    action = "store_true", dest = "debuggerInteractive",
                    help = "prevents the test harness from redirecting stdout "
                           "and stderr for interactive debuggers")

    # -h, --help are automatically handled by OptionParser

    self.set_defaults(**defaults)

    usage = """\
Usage instructions for runtests.py.
All arguments are optional.
If --chrome is specified, chrome tests will be run instead of web content tests.
If --browser-chrome is specified, browser-chrome tests will be run instead of web content tests.
See <http://mochikit.com/doc/html/MochiKit/Logging.html> for details on the logging levels."""
    self.set_usage(usage)



#######################
# HTTP SERVER SUPPORT #
#######################

class MochitestServer:
  "Web server used to serve Mochitests, for closer fidelity to the real web."

  def __init__(self, options):
    self._closeWhenDone = options.closeWhenDone
    self._utilityPath = options.utilityPath
    self._xrePath = options.xrePath

  def start(self):
    "Run the Mochitest server, returning the process ID of the server."
    
    env = automation.environment(xrePath = self._xrePath)
    env["XPCOM_DEBUG_BREAK"] = "warn"
    if automation.IS_WIN32:
      env["PATH"] = env["PATH"] + ";" + self._xrePath

    args = ["-g", self._xrePath,
            "-v", "170",
            "-f", "./" + "httpd.js",
            "-f", "./" + "server.js"]

    xpcshell = os.path.join(self._utilityPath,
                            "xpcshell" + automation.BIN_SUFFIX)
    self._process = automation.Process([xpcshell] + args, env = env)
    pid = self._process.pid
    if pid < 0:
      print "Error starting server."
      sys.exit(2)
    automation.log.info("INFO | runtests.py | Server pid: %d", pid)
    

  def ensureReady(self, timeout):
    assert timeout >= 0

    aliveFile = os.path.join(PROFILE_DIRECTORY, "server_alive.txt")
    i = 0
    while i < timeout:
      if os.path.exists(aliveFile):
        break
      time.sleep(1)
      i += 1
    else:
      print "Timed out while waiting for server startup."
      self.stop()
      sys.exit(1)

  def stop(self):
    try:
      c = urllib2.urlopen(SERVER_SHUTDOWN_URL)
      c.read()
      c.close()
      self._process.wait()
    except:
      self._process.kill()

def getFullPath(path):
  "Get an absolute path relative to oldcwd."
  return os.path.normpath(os.path.join(oldcwd, os.path.expanduser(path)))

def searchPath(path):
  "Go one step beyond getFullPath and try the various folders in PATH"
  # Try looking in the current working directory first.
  newpath = getFullPath(path)
  if os.path.exists(newpath):
    return newpath

  # At this point we have to fail if a directory was given (to prevent cases
  # like './gdb' from matching '/usr/bin/./gdb').
  if not os.path.dirname(path):
    for dir in os.environ['PATH'].split(os.pathsep):
      newpath = os.path.join(dir, path)
      if os.path.exists(newpath):
        return newpath
  return None

#################
# MAIN FUNCTION #
#################

def main():
  parser = MochitestOptions()
  options, args = parser.parse_args()

  if options.totalChunks is not None and options.thisChunk is None:
    parser.error("thisChunk must be specified when totalChunks is specified")

  if options.totalChunks:
    if not 1 <= options.thisChunk <= options.totalChunks:
       parser.error("thisChunk must be between 1 and totalChunks")

  if options.xrePath is None:
    # default xrePath to the app path if not provided
    # but only if an app path was explicitly provided
    if options.app != parser.defaults['app']:
      options.xrePath = os.path.dirname(options.app)
    else:
      # otherwise default to dist/bin
      options.xrePath = automation.DIST_BIN

  # allow relative paths
  options.xrePath = getFullPath(options.xrePath)

  options.app = getFullPath(options.app)
  if not os.path.exists(options.app):
    msg = """\
Error: Path %(app)s doesn't exist.
Are you executing $objdir/_tests/testing/mochitest/runtests.py?"""
    print msg % {"app": options.app}
    sys.exit(1)

  options.utilityPath = getFullPath(options.utilityPath)
  options.certPath = getFullPath(options.certPath)
  options.symbolsPath = getFullPath(options.symbolsPath)

  debuggerInfo = None

  if options.debugger:
    debuggerPath = searchPath(options.debugger)
    if not debuggerPath:
      print "Error: Path %s doesn't exist." % options.debugger
      sys.exit(1)

    debuggerName = os.path.basename(debuggerPath).lower()

    def getDebuggerInfo(type, default):
      if debuggerName in DEBUGGER_INFO and type in DEBUGGER_INFO[debuggerName]:
        return DEBUGGER_INFO[debuggerName][type]
      return default

    debuggerInfo = {
      "path": debuggerPath,
      "interactive" : getDebuggerInfo("interactive", False),
      "args": getDebuggerInfo("args", "").split()
    }

    if options.debuggerArgs:
      debuggerInfo["args"] = options.debuggerArgs.split()
    if options.debuggerInteractive:
      debuggerInfo["interactive"] = options.debuggerInteractive

  # browser environment
  browserEnv = automation.environment(xrePath = options.xrePath)

  # These variables are necessary for correct application startup; change
  # via the commandline at your own risk.
  browserEnv["XPCOM_DEBUG_BREAK"] = "stack"

  for v in options.environment:
    ix = v.find("=")
    if ix <= 0:
      print "Error: syntax error in --setenv=" + v
      sys.exit(1)
    browserEnv[v[:ix]] = v[ix + 1:]

  automation.initializeProfile(PROFILE_DIRECTORY, options.extraPrefs)
  manifest = addChromeToProfile(options)
  copyExtraFilesToProfile(options)
  server = MochitestServer(options)
  server.start()

  # If we're lucky, the server has fully started by now, and all paths are
  # ready, etc.  However, xpcshell cold start times suck, at least for debug
  # builds.  We'll try to connect to the server for awhile, and if we fail,
  # we'll try to kill the server and exit with an error.
  server.ensureReady(SERVER_STARTUP_TIMEOUT)

  # URL parameters to test URL:
  #
  # autorun -- kick off tests automatically
  # closeWhenDone -- runs quit.js after tests
  # logFile -- logs test run to an absolute path
  # totalChunks -- how many chunks to split tests into
  # thisChunk -- which chunk to run
  #
  
  # consoleLevel, fileLevel: set the logging level of the console and
  # file logs, if activated.
  # <http://mochikit.com/doc/html/MochiKit/Logging.html>

  testURL = TESTS_URL + options.testPath
  urlOpts = []
  if options.chrome:
    testURL = CHROMETESTS_URL
    if options.testPath:
      urlOpts.append("testPath=" + encodeURIComponent(options.testPath))
  elif options.a11y:
    testURL = A11YTESTS_URL
    if options.testPath:
      urlOpts.append("testPath=" + encodeURIComponent(options.testPath))
  elif options.browserChrome:
    testURL = "about:blank"

  # allow relative paths for logFile
  if options.logFile:
    options.logFile = getFullPath(options.logFile)
  if options.browserChrome:
    makeTestConfig(options)
  else:
    if options.autorun:
      urlOpts.append("autorun=1")
    if options.closeWhenDone:
      urlOpts.append("closeWhenDone=1")
    if options.logFile:
      urlOpts.append("logFile=" + encodeURIComponent(options.logFile))
      urlOpts.append("fileLevel=" + encodeURIComponent(options.fileLevel))
    if options.consoleLevel:
      urlOpts.append("consoleLevel=" + encodeURIComponent(options.consoleLevel))
    if options.totalChunks:
        urlOpts.append("totalChunks=%d" % options.totalChunks)
        urlOpts.append("thisChunk=%d" % options.thisChunk)
        if options.chunkByDir:
            urlOpts.append("chunkByDir=%d" % options.chunkByDir)
    if options.shuffle:
        urlOpts.append("shuffle=1")
    if len(urlOpts) > 0:
      testURL += "?" + "&".join(urlOpts)

  browserEnv["XPCOM_MEM_BLOAT_LOG"] = LEAK_REPORT_FILE

  if options.fatalAssertions:
    browserEnv["XPCOM_DEBUG_BREAK"] = "stack-and-abort"

  # run once with -silent to let the extension manager do its thing
  # and then exit the app
  automation.log.info("INFO | runtests.py | Performing extension manager registration: start.\n")
  # Don't care about this |status|: |runApp()| reporting it should be enough.
  status = automation.runApp(None, browserEnv, options.app,
                             PROFILE_DIRECTORY, ["-silent"],
                             utilityPath = options.utilityPath,
                             xrePath = options.xrePath,
                             symbolsPath=options.symbolsPath)
  # We don't care to call |processLeakLog()| for this step.
  automation.log.info("\nINFO | runtests.py | Performing extension manager registration: end.")

  # Remove the leak detection file so it can't "leak" to the tests run.
  # The file is not there if leak logging was not enabled in the application build.
  if os.path.exists(LEAK_REPORT_FILE):
    os.remove(LEAK_REPORT_FILE)

  # then again to actually run reftest
  automation.log.info("INFO | runtests.py | Running tests: start.\n")
  status = automation.runApp(testURL, browserEnv, options.app,
                             PROFILE_DIRECTORY, options.browserArgs,
                             runSSLTunnel = True,
                             utilityPath = options.utilityPath,
                             xrePath = options.xrePath,
                             certPath=options.certPath,
                             debuggerInfo=debuggerInfo,
                             symbolsPath=options.symbolsPath)

  # Server's no longer needed, and perhaps more importantly, anything it might
  # spew to console shouldn't disrupt the leak information table we print next.
  server.stop()

  processLeakLog(LEAK_REPORT_FILE, options.leakThreshold)
  automation.log.info("\nINFO | runtests.py | Running tests: end.")

  # delete the profile and manifest
  os.remove(manifest)

  # hanging due to non-halting threads is no fun; assume we hit the errors we
  # were going to hit already and exit.
  sys.exit(status)



#######################
# CONFIGURATION SETUP #
#######################

def makeTestConfig(options):
  "Creates a test configuration file for customizing test execution."
  def boolString(b):
    if b:
      return "true"
    return "false"

  logFile = options.logFile.replace("\\", "\\\\")
  testPath = options.testPath.replace("\\", "\\\\")
  content = """\
({
  autoRun: %(autorun)s,
  closeWhenDone: %(closeWhenDone)s,
  logPath: "%(logPath)s",
  testPath: "%(testPath)s"
})""" % {"autorun": boolString(options.autorun),
         "closeWhenDone": boolString(options.closeWhenDone),
         "logPath": logFile,
         "testPath": testPath}

  config = open(os.path.join(PROFILE_DIRECTORY, "testConfig.js"), "w")
  config.write(content)
  config.close() 


def addChromeToProfile(options):
  "Adds MochiKit chrome tests to the profile."

  chromedir = os.path.join(PROFILE_DIRECTORY, "chrome")
  os.mkdir(chromedir)

  chrome = []

  part = """
@namespace url("http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"); /* set default namespace to XUL */
toolbar,
toolbarpalette {
  background-color: rgb(235, 235, 235) !important;
}
toolbar#nav-bar {
  background-image: none !important;
}
"""
  chrome.append(part)



  # write userChrome.css
  chromeFile = open(os.path.join(PROFILE_DIRECTORY, "userChrome.css"), "a")
  chromeFile.write("".join(chrome))
  chromeFile.close()


  # register our chrome dir
  chrometestDir = os.path.abspath(".") + "/"
  if automation.IS_WIN32:
    chrometestDir = "file:///" + chrometestDir.replace("\\", "/")


  (path, leaf) = os.path.split(options.app)
  manifest = os.path.join(path, "chrome", "mochikit.manifest")
  manifestFile = open(manifest, "w")
  manifestFile.write("content mochikit " + chrometestDir + " contentaccessible=yes\n")
  if options.browserChrome:
    overlayLine = "overlay " + BROWSER_CHROME_URL + " " \
                  "chrome://mochikit/content/browser-test-overlay.xul\n"
    manifestFile.write(overlayLine)
  manifestFile.close()

  return manifest

def copyExtraFilesToProfile(options):
  "Copy extra files or dirs specified on the command line to the testing profile."
  for f in options.extraProfileFiles:
    abspath = getFullPath(f)
    dest = os.path.join(PROFILE_DIRECTORY, os.path.basename(abspath))
    if os.path.isdir(abspath):
      shutil.copytree(abspath, dest)
    else:
      shutil.copy(abspath, dest)

#########
# DO IT #
#########

if __name__ == "__main__":
  main()