pjs/testing/mochitest/runtests.py.in

#
# ***** BEGIN LICENSE BLOCK *****
# Version: MPL 1.1/GPL 2.0/LGPL 2.1
#
# The contents of this file are subject to the Mozilla Public License Version
# 1.1 (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
# http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS" basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.
#
# The Original Code is mozilla.org code.
#
# The Initial Developer of the Original Code is
# Mozilla Foundation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Robert Sayre <sayrer@gmail.com>
#   Jeff Walden <jwalden+bmo@mit.edu>
#
# Alternatively, the contents of this file may be used under the terms of
# either the GNU General Public License Version 2 or later (the "GPL"), or
# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
# in which case the provisions of the GPL or the LGPL are applicable instead
# of those above. If you wish to allow use of your version of this file only
# under the terms of either the GPL or the LGPL, and not to allow others to
# use your version of this file under the terms of the MPL, indicate your
# decision by deleting the provisions above and replace them with the notice
# and other provisions required by the GPL or the LGPL. If you do not delete
# the provisions above, a recipient may use your version of this file under
# the terms of any one of the MPL, the GPL or the LGPL.
#
# ***** END LICENSE BLOCK *****

"""
Runs the Mochitest test harness.
"""

from datetime import datetime
import logging
import optparse
import os
import os.path
import re
import sys
import time
from urllib import quote_plus as encodeURIComponent
import urllib2
import commands
import automation

# Path to the test script on the server
TEST_SERVER_HOST = "localhost:8888"
TEST_PATH = "/tests/"
CHROME_PATH = "/redirect.html";
A11Y_PATH = "/redirect-a11y.html"
TESTS_URL = "http://" + TEST_SERVER_HOST + TEST_PATH
CHROMETESTS_URL = "http://" + TEST_SERVER_HOST + CHROME_PATH
A11YTESTS_URL = "http://" + TEST_SERVER_HOST + A11Y_PATH
SERVER_SHUTDOWN_URL = "http://" + TEST_SERVER_HOST + "/server/shutdown"
# main browser chrome URL, same as browser.chromeURL pref
#ifdef MOZ_SUITE
BROWSER_CHROME_URL = "chrome://navigator/content/navigator.xul"
#else
BROWSER_CHROME_URL = "chrome://browser/content/browser.xul"
#endif

# Max time in seconds to wait for server startup before tests will fail -- if
# this seems big, it's mostly for debug machines where cold startup
# (particularly after a build) takes forever.
SERVER_STARTUP_TIMEOUT = 45

INFINITY = 1.0e3000

oldcwd = os.getcwd()
SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
os.chdir(SCRIPT_DIRECTORY)

PROFILE_DIRECTORY = os.path.abspath("./mochitesttestingprofile")

LEAK_REPORT_FILE = PROFILE_DIRECTORY + "/" + "leaks-report.log"

log = logging.getLogger()


#######################
# COMMANDLINE OPTIONS #
#######################

class MochitestOptions(optparse.OptionParser):
  """Parses Mochitest commandline options."""
  def __init__(self, **kwargs):
    optparse.OptionParser.__init__(self, **kwargs)
    defaults = {}

    self.add_option("--close-when-done",
                    action = "store_true", dest = "closeWhenDone",
                    help = "close the application when tests are done running")
    defaults["closeWhenDone"] = False

    self.add_option("--appname",
                    action = "store", type = "string", dest = "app",
                    help = "absolute path to application, overriding default")
    defaults["app"] = automation.DEFAULT_APP

    self.add_option("--log-file",
                    action = "store", type = "string",
                    dest = "logFile", metavar = "FILE",
                    help = "file to which logging occurs")
    defaults["logFile"] = ""

    self.add_option("--autorun",
                    action = "store_true", dest = "autorun",
                    help = "start running tests when the application starts")
    defaults["autorun"] = False

    LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR", "FATAL")
    LEVEL_STRING = ", ".join(LOG_LEVELS)

    self.add_option("--console-level",
                    action = "store", type = "choice", dest = "consoleLevel",
                    choices = LOG_LEVELS, metavar = "LEVEL",
                    help = "one of %s to determine the level of console "
                           "logging" % LEVEL_STRING)
    defaults["consoleLevel"] = None

    self.add_option("--file-level", 
                    action = "store", type = "choice", dest = "fileLevel",
                    choices = LOG_LEVELS, metavar = "LEVEL",
                    help = "one of %s to determine the level of file "
                           "logging if a file has been specified, defaulting "
                           "to INFO" % LEVEL_STRING)
    defaults["fileLevel"] = "INFO"

    self.add_option("--chrome",
                    action = "store_true", dest = "chrome",
                    help = "run chrome Mochitests")
    defaults["chrome"] = False

    self.add_option("--test-path",
                    action = "store", type = "string", dest = "testPath",
                    help = "start in the given directory's tests")
    defaults["testPath"] = ""

    self.add_option("--browser-chrome",
                    action = "store_true", dest = "browserChrome",
                    help = "run browser chrome Mochitests")
    defaults["browserChrome"] = False

    self.add_option("--a11y",
                    action = "store_true", dest = "a11y",
                    help = "run accessibility Mochitests");

    self.add_option("--setenv",
                    action = "append", type = "string",
                    dest = "environment", metavar = "NAME=VALUE",
                    help = "sets the given variable in the application's "
                           "environment")
    defaults["environment"] = []
    
    self.add_option("--browser-arg",
                    action = "append", type = "string",
                    dest = "browserArgs", metavar = "ARG",
                    help = "provides an argument to the test application")
    defaults["browserArgs"] = []

    self.add_option("--leak-threshold",
                    action = "store", type = "int",
                    dest = "leakThreshold", metavar = "THRESHOLD",
                    help = "fail if the number of bytes leaked through "
                           "refcounted objects (or bytes in classes with "
                           "MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) is greater "
                           "than the given number")
    defaults["leakThreshold"] = INFINITY

    self.add_option("--fatal-assertions",
                    action = "store_true", dest = "fatalAssertions",
                    help = "abort testing whenever an assertion is hit "
                           "(requires a debug build to be effective)")
    defaults["fatalAssertions"] = False

    # -h, --help are automatically handled by OptionParser

    self.set_defaults(**defaults)

    usage = """\
Usage instructions for runtests.py.
All arguments are optional.
If --chrome is specified, chrome tests will be run instead of web content tests.
If --browser-chrome is specified, browser-chrome tests will be run instead of web content tests.
See <http://mochikit.com/doc/html/MochiKit/Logging.html> for details on the logging levels."""
    self.set_usage(usage)



#######################
# HTTP SERVER SUPPORT #
#######################

class MochitestServer:
  "Web server used to serve Mochitests, for closer fidelity to the real web."

  def __init__(self, options):
    self._closeWhenDone = options.closeWhenDone

  def start(self):
    "Run the Mochitest server, returning the process ID of the server."
    
    env = dict(os.environ)
    if automation.UNIXISH:
      env["LD_LIBRARY_PATH"] = automation.DIST_BIN
      env["MOZILLA_FIVE_HOME"] = automation.DIST_BIN
      env["XPCOM_DEBUG_BREAK"] = "warn"

    args = ["-v", "170",
            "-f", "./" + "httpd.js",
            "-f", "./" + "server.js"]

    xpcshell = automation.DIST_BIN + "/" + "xpcshell";
    self._process = automation.Process(xpcshell, args, env = env)
    pid = self._process.pid
    if pid < 0:
      print "Error starting server."
      sys.exit(2)
    log.info("Server pid: %d", pid)
    

  def ensureReady(self, timeout):
    assert timeout >= 0

    aliveFile = PROFILE_DIRECTORY + "/" + "server_alive.txt"
    i = 0
    while i < timeout:
      if os.path.exists(aliveFile):
        break
      time.sleep(1)
      i += 1
    else:
      print "Timed out while waiting for server startup."
      self.stop()
      sys.exit(1)

  def stop(self):
    try:
      c = urllib2.urlopen(SERVER_SHUTDOWN_URL)
      c.read()
      c.close()
      self._process.wait()
    except:
      self._process.kill()


#################
# MAIN FUNCTION #
#################

def main():
  parser = MochitestOptions()
  options, args = parser.parse_args()

  # If the leak threshold wasn't explicitly set, we override the default of
  # infinity when the set of tests we're running are known to leak only a
  # particular amount.  If for some reason you don't want a new leak threshold
  # enforced, just pass an explicit --leak-threshold=N to prevent the override.
  maybeForceLeakThreshold(options)

  if not os.path.exists(options.app):
    msg = """\
Error: Path %(app)s doesn't exist.
Are you executing $objdir/_tests/testing/mochitest/runtests.py?"""
    print msg % {"app": options.app}
    sys.exit(1)

  # browser environment
  browserEnv = dict(os.environ)

  # These variables are necessary for correct application startup; change
  # via the commandline at your own risk.
  browserEnv["NO_EM_RESTART"] = "1"
  browserEnv["XPCOM_DEBUG_BREAK"] = "warn"
  if automation.UNIXISH:
    browserEnv["LD_LIBRARY_PATH"] = automation.DIST_BIN 
    browserEnv["MOZILLA_FIVE_HOME"] = automation.DIST_BIN 
    browserEnv["GNOME_DISABLE_CRASH_DIALOG"] = "1"

  for v in options.environment:
    ix = v.find("=")
    if ix <= 0:
      print "Error: syntax error in --setenv=" + v
      sys.exit(1)
    browserEnv[v[:ix]] = v[ix + 1:]

  automation.initializeProfile(PROFILE_DIRECTORY)
  manifest = addChromeToProfile(options)
  server = MochitestServer(options)
  server.start()

  # If we're lucky, the server has fully started by now, and all paths are
  # ready, etc.  However, xpcshell cold start times suck, at least for debug
  # builds.  We'll try to connect to the server for awhile, and if we fail,
  # we'll try to kill the server and exit with an error.
  server.ensureReady(SERVER_STARTUP_TIMEOUT)


  # URL parameters to test URL:
  #
  # autorun -- kick off tests automatically
  # closeWhenDone -- runs quit.js after tests
  # logFile -- logs test run to an absolute path
  #
  
  # consoleLevel, fileLevel: set the logging level of the console and
  # file logs, if activated.
  # <http://mochikit.com/doc/html/MochiKit/Logging.html>

  testURL = TESTS_URL + options.testPath
  urlOpts = []
  if options.chrome:
    testURL = CHROMETESTS_URL
    if options.testPath:
      urlOpts.append("testPath=" + encodeURIComponent(options.testPath))
  elif options.a11y:
    testURL = A11YTESTS_URL
    if options.testPath:
      urlOpts.append("testPath=" + encodeURIComponent(options.testPath))
  elif options.browserChrome:
    testURL = "about:blank"

  if options.browserChrome:
    makeTestConfig(options)
  else:
    if options.autorun:
      urlOpts.append("autorun=1")
    if options.closeWhenDone:
      urlOpts.append("closeWhenDone=1")
    if options.logFile:
      urlOpts.append("logFile=" + encodeURIComponent(os.path.normpath(os.path.join(oldcwd, options.logFile))))
      urlOpts.append("fileLevel=" + encodeURIComponent(options.fileLevel))
    if options.consoleLevel:
      urlOpts.append("consoleLevel=" + encodeURIComponent(options.consoleLevel))
    if len(urlOpts) > 0:
      testURL += "?" + "&".join(urlOpts)

  browserEnv["XPCOM_MEM_BLOAT_LOG"] = LEAK_REPORT_FILE

  if options.fatalAssertions:
    browserEnv["XPCOM_DEBUG_BREAK"] = "stack-and-abort"

  start = automation.runApp(testURL, browserEnv, options.app, PROFILE_DIRECTORY,
                            options.browserArgs)

  # Server's no longer needed, and perhaps more importantly, anything it might
  # spew to console shouldn't disrupt the leak information table we print next.
  server.stop()

  if not os.path.exists(LEAK_REPORT_FILE):
    log.info("WARNING refcount logging is off, so leaks can't be detected!")
  else:
    leaks = open(LEAK_REPORT_FILE, "r")
    for line in leaks:
      log.info(line.rstrip())
    leaks.close()

    threshold = options.leakThreshold
    leaks = open(LEAK_REPORT_FILE, "r")
    #                  Per-Inst  Leaked      Total  Rem ...
    #   0 TOTAL              17     192  419115886    2 ...
    # 833 nsTimerImpl        60     120      24726    2 ...
    lineRe = re.compile(r"^\s*\d+\s+(?P<name>\S+)\s+"
                        r"(?P<size>\d+)\s+(?P<bytesLeaked>\d+)\s+"
                        r"\d+\s+(?P<numLeaked>\d+)")
    thresholdExceeded = False
    seenTotal = False
    prefix = "TEST-PASS"
    for line in leaks:
      matches = lineRe.match(line)
      if not matches:
        continue
      name = matches.group("name")
      if "TOTAL" == name:
        seenTotal = True
        bytesLeaked = int(matches.group("bytesLeaked"))
        if bytesLeaked > threshold:
          thresholdExceeded = True
          prefix = "TEST-UNEXPECTED-FAIL"
          log.info("TEST-UNEXPECTED-FAIL | runtests-leaks | leaked %d bytes during test execution (should "
                   "have leaked no more than %d bytes)",
                   bytesLeaked, threshold)
        elif bytesLeaked > 0:
          log.info("TEST-PASS | runtests-leaks | WARNING leaked %d bytes during test execution",
                   bytesLeaked)
        else:
          log.info("TEST-PASS | runtests-leaks | no leaks detected!")
      else:
        numLeaked = int(matches.group("numLeaked"))
        if numLeaked != 0:
          if numLeaked > 1:
            instance = "instances"
            rest = " each (%s bytes total)" % matches.group("bytesLeaked")
          else:
            instance = "instance"
            rest = ""
          log.info("%(prefix)s | runtests-leaks | leaked %(numLeaked)d %(instance)s of %(name)s "
                   "with size %(size)s bytes%(rest)s" %
                   { "prefix": prefix,
                     "numLeaked": numLeaked,
                     "instance": instance,
                     "name": name,
                     "size": matches.group("size"),
                     "rest": rest })
    if not seenTotal:
      log.info("TEST-UNEXPECTED-FAIL | runtests-leaks | missing output line for total leaks!")
    leaks.close()


  # print test run times
  finish = datetime.now()
  log.info(" started: %s", str(start))
  log.info("finished: %s", str(finish))

  # delete the profile and manifest
  os.remove(manifest)

  # hanging due to non-halting threads is no fun; assume we hit the errors we
  # were going to hit already and exit with a success code
  sys.exit(0)



#######################
# CONFIGURATION SETUP #
#######################

def maybeForceLeakThreshold(options):
  """
  Modifies an unset leak threshold if it is known that a particular leak
  threshold can be successfully forced for the particular Mochitest type
  and platform in use.
  """
  if options.leakThreshold == INFINITY:
    if options.chrome:
      # We don't leak running the --chrome tests.
      options.leakThreshold = 0
    elif options.browserChrome:
      # We still leak a nondeterministic amount running browser-chrome tests.
      pass
    else:
      # We leak nothing on Windows or OS X running normal Mochitests.
      if automation.IS_MAC or automation.IS_WIN32:
        options.leakThreshold = 0

def makeTestConfig(options):
  "Creates a test configuration file for customizing test execution."
  def boolString(b):
    if b:
      return "true"
    return "false"

  logFile = options.logFile.replace("\\", "\\\\")
  testPath = options.testPath.replace("\\", "\\\\")
  content = """\
({
  autoRun: %(autorun)s,
  closeWhenDone: %(closeWhenDone)s,
  logPath: "%(logPath)s",
  testPath: "%(testPath)s"
})""" % {"autorun": boolString(options.autorun),
         "closeWhenDone": boolString(options.closeWhenDone),
         "logPath": logFile,
         "testPath": testPath}

  config = open(PROFILE_DIRECTORY + "/" + "testConfig.js", "w")
  config.write(content)
  config.close() 


def addChromeToProfile(options):
  "Adds MochiKit chrome tests to the profile."

  chromedir = PROFILE_DIRECTORY + "/" + "chrome"
  os.mkdir(chromedir)

  chrome = []

  part = """
@namespace url("http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"); /* set default namespace to XUL */
toolbar,
toolbarpalette {
  background-color: rgb(235, 235, 235) !important;
}
toolbar#nav-bar {
  background-image: none !important;
}
"""
  chrome.append(part)



  # write userChrome.css
  chromeFile = open(PROFILE_DIRECTORY + "/" + "userChrome.css", "a")
  chromeFile.write("".join(chrome))
  chromeFile.close()


  # register our chrome dir
  chrometestDir = os.path.abspath(".") + "/"
  if automation.IS_WIN32:
    chrometestDir = "file:///" + chrometestDir.replace("\\", "/")


  (path, leaf) = os.path.split(options.app)
  manifest = path + "/" + "chrome/mochikit.manifest"
  manifestFile = open(manifest, "w")
  manifestFile.write("content mochikit " + chrometestDir + " contentaccessible=yes\n")
  if options.browserChrome:
    overlayLine = "overlay " + BROWSER_CHROME_URL + " " \
                  "chrome://mochikit/content/browser-test-overlay.xul\n"
    manifestFile.write(overlayLine)
  manifestFile.close()

  return manifest

#########
# DO IT #
#########

if __name__ == "__main__":
  main()