From 96f807a4546bf7317ffba63587088f6d4eab7c6d Mon Sep 17 00:00:00 2001 From: Joel Maher Date: Tue, 19 Jan 2010 11:45:04 -0800 Subject: [PATCH] Bug 530475 - convert test harness python code to classes additional refactoring for automation.py and runtests.py p=jmaher r=ted --- build/automation.py.in | 188 +++++++++++++------------ testing/mochitest/runtests.py.in | 231 ++++++++++++++++++------------- 2 files changed, 236 insertions(+), 183 deletions(-) diff --git a/build/automation.py.in b/build/automation.py.in index 4c3da741939..f6dd751c856 100644 --- a/build/automation.py.in +++ b/build/automation.py.in @@ -577,6 +577,103 @@ user_pref("camino.use_system_proxy_settings", false); // Camino-only, harmless t self.log.info("Can't trigger Breakpad, just killing process") proc.kill() + def waitForFinish(self, proc, utilityPath, timeout, maxTime): + """ Look for timeout or crashes and return the status after the process terminates """ + stackFixerProcess = None + didTimeout = False + if proc.stdout is None: + self.log.info("TEST-INFO: Not logging stdout or stderr due to debugger connection") + else: + logsource = proc.stdout + if self.IS_DEBUG_BUILD: + stackFixerCommand = None + if self.IS_MAC: + stackFixerCommand = "fix-macosx-stack.pl" + elif self.IS_LINUX: + stackFixerCommand = "fix-linux-stack.pl" + if stackFixerCommand is not None: + stackFixerProcess = self.Process([self.PERL, os.path.join(utilityPath, stackFixerCommand)], + stdin=logsource, + stdout=subprocess.PIPE) + logsource = stackFixerProcess.stdout + + (line, didTimeout) = self.readWithTimeout(logsource, timeout) + hitMaxTime = False + while line != "" and not didTimeout: + self.log.info(line.rstrip()) + (line, didTimeout) = self.readWithTimeout(logsource, timeout) + if not hitMaxTime and maxTime and datetime.now() - startTime > timedelta(seconds = maxTime): + # Kill the application, but continue reading from stack fixer so as not to deadlock on stackFixerProcess.wait(). + hitMaxTime = True + self.log.info("TEST-UNEXPECTED-FAIL | automation.py | application ran for longer than allowed maximum time of %d seconds", int(maxTime)) + self.triggerBreakpad(proc, utilityPath) + if didTimeout: + self.log.info("TEST-UNEXPECTED-FAIL | automation.py | application timed out after %d seconds with no output", int(timeout)) + self.triggerBreakpad(proc, utilityPath) + + status = proc.wait() + if status != 0 and not didTimeout and not hitMaxTime: + self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Exited with code %d during test run", status) + if stackFixerProcess is not None: + fixerStatus = stackFixerProcess.wait() + if fixerStatus != 0 and not didTimeout and not hitMaxTime: + self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Stack fixer process exited with code %d during test run", fixerStatus) + return status + + def buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs): + """ build the application command line """ + + cmd = app + if self.IS_MAC and not self.IS_CAMINO and not cmd.endswith("-bin"): + cmd += "-bin" + cmd = os.path.abspath(cmd) + + args = [] + + if debuggerInfo: + args.extend(debuggerInfo["args"]) + args.append(cmd) + cmd = os.path.abspath(debuggerInfo["path"]) + + if self.IS_MAC: + args.append("-foreground") + + if self.IS_CYGWIN: + profileDirectory = commands.getoutput("cygpath -w \"" + profileDir + "/\"") + else: + profileDirectory = profileDir + "/" + + args.extend(("-no-remote", "-profile", profileDirectory)) + if testURL is not None: + if self.IS_CAMINO: + args.extend(("-url", testURL)) + else: + args.append((testURL)) + args.extend(extraArgs) + return cmd, args + + def checkForZombies(self, processLog): + """ Look for hung processes """ + if not os.path.exists(processLog): + self.log.info('INFO | automation.py | PID log not found: %s', processLog) + else: + self.log.info('INFO | automation.py | Reading PID log: %s', processLog) + processList = [] + pidRE = re.compile(r'launched child process (\d+)$') + processLogFD = open(processLog) + for line in processLogFD: + self.log.info(line.rstrip()) + m = pidRE.search(line) + if m: + processList.append(int(m.group(1))) + processLogFD.close() + + for processPID in processList: + self.log.info("INFO | automation.py | Checking for orphan process with PID: %d", processPID) + if self.isPidAlive(processPID): + self.log.info("TEST-UNEXPECTED-FAIL | automation.py | child process %d still alive after shutdown", processPID) + self.killPid(processPID) + def runApp(self, testURL, env, app, profileDir, extraArgs, runSSLTunnel = False, utilityPath = None, xrePath = None, certPath = None, @@ -617,35 +714,7 @@ user_pref("camino.use_system_proxy_settings", false); // Camino-only, harmless t env = self.environment(xrePath = xrePath)) self.log.info("INFO | automation.py | SSL tunnel pid: %d", ssltunnelProcess.pid) - # now run with the profile we created - cmd = app - if self.IS_MAC and not self.IS_CAMINO and not cmd.endswith("-bin"): - cmd += "-bin" - cmd = os.path.abspath(cmd) - - args = [] - - if debuggerInfo: - args.extend(debuggerInfo["args"]) - args.append(cmd) - cmd = os.path.abspath(debuggerInfo["path"]) - - if self.IS_MAC: - args.append("-foreground") - - if self.IS_CYGWIN: - profileDirectory = commands.getoutput("cygpath -w \"" + profileDir + "/\"") - else: - profileDirectory = profileDir + "/" - - args.extend(("-no-remote", "-profile", profileDirectory)) - if testURL is not None: - if self.IS_CAMINO: - args.extend(("-url", testURL)) - else: - args.append((testURL)) - args.extend(extraArgs) - + cmd, args = self.buildCommandLine(app, debuggerInfo, profileDir, testURL, extraArgs) startTime = datetime.now() # Don't redirect stdout and stderr if an interactive debugger is attached @@ -661,68 +730,11 @@ user_pref("camino.use_system_proxy_settings", false); // Camino-only, harmless t stderr = subprocess.STDOUT) self.log.info("INFO | automation.py | Application pid: %d", proc.pid) - stackFixerProcess = None - didTimeout = False - if outputPipe is None: - self.log.info("TEST-INFO: Not logging stdout or stderr due to debugger connection") - else: - logsource = proc.stdout - if self.IS_DEBUG_BUILD: - stackFixerCommand = None - if self.IS_MAC: - stackFixerCommand = "fix-macosx-stack.pl" - elif self.IS_LINUX: - stackFixerCommand = "fix-linux-stack.pl" - if stackFixerCommand is not None: - stackFixerProcess = self.Process([self.PERL, os.path.join(utilityPath, stackFixerCommand)], - stdin=logsource, - stdout=subprocess.PIPE) - logsource = stackFixerProcess.stdout - - (line, didTimeout) = self.readWithTimeout(logsource, timeout) - hitMaxTime = False - while line != "" and not didTimeout: - self.log.info(line.rstrip()) - (line, didTimeout) = self.readWithTimeout(logsource, timeout) - if not hitMaxTime and maxTime and datetime.now() - startTime > timedelta(seconds = maxTime): - # Kill the application, but continue reading from stack fixer so as not to deadlock on stackFixerProcess.wait(). - hitMaxTime = True - self.log.info("TEST-UNEXPECTED-FAIL | automation.py | application ran for longer than allowed maximum time of %d seconds", int(maxTime)) - self.triggerBreakpad(proc, utilityPath) - if didTimeout: - self.log.info("TEST-UNEXPECTED-FAIL | automation.py | application timed out after %d seconds with no output", int(timeout)) - self.triggerBreakpad(proc, utilityPath) - - status = proc.wait() - if status != 0 and not didTimeout and not hitMaxTime: - self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Exited with code %d during test run", status) - if stackFixerProcess is not None: - fixerStatus = stackFixerProcess.wait() - if fixerStatus != 0 and not didTimeout and not hitMaxTime: - self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Stack fixer process exited with code %d during test run", fixerStatus) + status = self.waitForFinish(proc, utilityPath, timeout, maxTime) self.log.info("INFO | automation.py | Application ran for: %s", str(datetime.now() - startTime)) # Do a final check for zombie child processes. - if not os.path.exists(processLog): - self.log.info('INFO | automation.py | PID log not found: %s', processLog) - else: - self.log.info('INFO | automation.py | Reading PID log: %s', processLog) - processList = [] - pidRE = re.compile(r'launched child process (\d+)$') - processLogFD = open(processLog) - for line in processLogFD: - self.log.info(line.rstrip()) - m = pidRE.search(line) - if m: - processList.append(int(m.group(1))) - processLogFD.close() - - for processPID in processList: - self.log.info("INFO | automation.py | Checking for orphan process with PID: %d", processPID) - if self.isPidAlive(processPID): - self.log.info("TEST-UNEXPECTED-FAIL | automation.py | child process %d still alive after shutdown", processPID) - self.killPid(processPID) - + self.checkForZombies(processLog) self.automationutils.checkForCrashes(os.path.join(profileDir, "minidumps"), symbolsPath) if os.path.exists(processLog): diff --git a/testing/mochitest/runtests.py.in b/testing/mochitest/runtests.py.in index dc740922410..8a7c0fd98a4 100644 --- a/testing/mochitest/runtests.py.in +++ b/testing/mochitest/runtests.py.in @@ -54,7 +54,7 @@ import urllib2 import commands from automation import Automation from automationutils import * - +import tempfile ####################### # COMMANDLINE OPTIONS # @@ -212,7 +212,6 @@ See for details on the logg self.set_usage(usage) - ####################### # HTTP SERVER SUPPORT # ####################### @@ -220,12 +219,13 @@ See for details on the logg class MochitestServer: "Web server used to serve Mochitests, for closer fidelity to the real web." - def __init__(self, automation, options, profileDir): + def __init__(self, automation, options, profileDir, shutdownURL): self._automation = automation self._closeWhenDone = options.closeWhenDone self._utilityPath = options.utilityPath self._xrePath = options.xrePath self._profileDir = profileDir + self.shutdownURL = shutdownURL def start(self): "Run the Mochitest server, returning the process ID of the server." @@ -266,7 +266,7 @@ class MochitestServer: def stop(self): try: - c = urllib2.urlopen(SERVER_SHUTDOWN_URL) + c = urllib2.urlopen(self.shutdownURL) c.read() c.close() self._process.wait() @@ -280,10 +280,8 @@ class Mochitest(object): TEST_PATH = "/tests/" CHROME_PATH = "/redirect.html"; A11Y_PATH = "/redirect-a11y.html" - TESTS_URL = "http://" + TEST_SERVER_HOST + TEST_PATH - CHROMETESTS_URL = "http://" + TEST_SERVER_HOST + CHROME_PATH - A11YTESTS_URL = "http://" + TEST_SERVER_HOST + A11Y_PATH - SERVER_SHUTDOWN_URL = "http://" + TEST_SERVER_HOST + "/server/shutdown" + urlOpts = [] + runSSLTunnel = True oldcwd = os.getcwd() @@ -309,11 +307,55 @@ class Mochitest(object): "Get an absolute path relative to self.oldcwd." return os.path.normpath(os.path.join(self.oldcwd, os.path.expanduser(path))) - def runTests(self, options): - debuggerInfo = getDebuggerInfo(self.oldcwd, options.debugger, options.debuggerArgs, - options.debuggerInteractive); + def buildTestPath(self, options): + """ build the url path to the specific test harness and test file or directory """ + testHost = "http://" + self.TEST_SERVER_HOST + testURL = testHost + self.TEST_PATH + options.testPath + if options.chrome: + testURL = testHost + self.CHROME_PATH + if options.testPath: + self.urlOpts.append("testPath=" + encodeURIComponent(options.testPath)) + elif options.a11y: + testURL = testHost + self.A11Y_PATH + if options.testPath: + self.urlOpts.append("testPath=" + encodeURIComponent(options.testPath)) + elif options.browserChrome: + testURL = "about:blank" + return testURL - # browser environment + def startWebServer(self, options): + """ create the webserver and start it up """ + shutdownURL = "http://" + self.TEST_SERVER_HOST + "/server/shutdown" + self.server = MochitestServer(self.automation, options, self.PROFILE_DIRECTORY, shutdownURL) + self.server.start() + + # If we're lucky, the server has fully started by now, and all paths are + # ready, etc. However, xpcshell cold start times suck, at least for debug + # builds. We'll try to connect to the server for awhile, and if we fail, + # we'll try to kill the server and exit with an error. + self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT) + + def stopWebServer(self): + """ Server's no longer needed, and perhaps more importantly, anything it might + spew to console shouldn't disrupt the leak information table we print next. + """ + self.server.stop() + + def getLogFilePath(self, logFile): + """ return the log file path relative to the device we are testing on, in most cases + it will be the full path on the local system + """ + return self.getFullPath(logFile) + + def buildProfile(self, options): + """ create the profile and add optional chrome bits and files if requested """ + self.automation.initializeProfile(self.PROFILE_DIRECTORY, options.extraPrefs) + manifest = self.addChromeToProfile(options) + self.copyExtraFilesToProfile(options) + return manifest + + def buildBrowserEnv(self, options): + """ build the environment variables for the specific test and operating system """ browserEnv = self.automation.environment(xrePath = options.xrePath) # These variables are necessary for correct application startup; change @@ -324,82 +366,20 @@ class Mochitest(object): ix = v.find("=") if ix <= 0: print "Error: syntax error in --setenv=" + v - return 1 + return None browserEnv[v[:ix]] = v[ix + 1:] - self.automation.initializeProfile(self.PROFILE_DIRECTORY, options.extraPrefs) - manifest = self.addChromeToProfile(options) - self.copyExtraFilesToProfile(options) - server = MochitestServer(self.automation, options, self.PROFILE_DIRECTORY) - server.start() - - # If we're lucky, the server has fully started by now, and all paths are - # ready, etc. However, xpcshell cold start times suck, at least for debug - # builds. We'll try to connect to the server for awhile, and if we fail, - # we'll try to kill the server and exit with an error. - server.ensureReady(self.SERVER_STARTUP_TIMEOUT) - - # URL parameters to test URL: - # - # autorun -- kick off tests automatically - # closeWhenDone -- runs quit.js after tests - # logFile -- logs test run to an absolute path - # totalChunks -- how many chunks to split tests into - # thisChunk -- which chunk to run - # timeout -- per-test timeout in seconds - # - - # consoleLevel, fileLevel: set the logging level of the console and - # file logs, if activated. - # - - testURL = self.TESTS_URL + options.testPath - urlOpts = [] - if options.chrome: - testURL = self.CHROMETESTS_URL - if options.testPath: - urlOpts.append("testPath=" + encodeURIComponent(options.testPath)) - elif options.a11y: - testURL = self.A11YTESTS_URL - if options.testPath: - urlOpts.append("testPath=" + encodeURIComponent(options.testPath)) - elif options.browserChrome: - testURL = "about:blank" - - # allow relative paths for logFile - if options.logFile: - options.logFile = self.getFullPath(options.logFile) - if options.browserChrome: - self.makeTestConfig(options) - else: - if options.autorun: - urlOpts.append("autorun=1") - if options.timeout: - urlOpts.append("timeout=%d" % options.timeout) - if options.closeWhenDone: - urlOpts.append("closeWhenDone=1") - if options.logFile: - urlOpts.append("logFile=" + encodeURIComponent(options.logFile)) - urlOpts.append("fileLevel=" + encodeURIComponent(options.fileLevel)) - if options.consoleLevel: - urlOpts.append("consoleLevel=" + encodeURIComponent(options.consoleLevel)) - if options.totalChunks: - urlOpts.append("totalChunks=%d" % options.totalChunks) - urlOpts.append("thisChunk=%d" % options.thisChunk) - if options.chunkByDir: - urlOpts.append("chunkByDir=%d" % options.chunkByDir) - if options.shuffle: - urlOpts.append("shuffle=1") - if len(urlOpts) > 0: - testURL += "?" + "&".join(urlOpts) - browserEnv["XPCOM_MEM_BLOAT_LOG"] = self.LEAK_REPORT_FILE if options.fatalAssertions: browserEnv["XPCOM_DEBUG_BREAK"] = "stack-and-abort" - # run once with -silent to let the extension manager do its thing - # and then exit the app + return browserEnv + + def runExtensionRegistration(self, options, browserEnv): + """ run once with -silent to let the extension manager do its thing + and then exit the app + """ self.automation.log.info("INFO | runtests.py | Performing extension manager registration: start.\n") # Don't care about this |status|: |runApp()| reporting it should be enough. status = self.automation.runApp(None, browserEnv, options.app, @@ -410,6 +390,69 @@ class Mochitest(object): # We don't care to call |processLeakLog()| for this step. self.automation.log.info("\nINFO | runtests.py | Performing extension manager registration: end.") + def buildURLOptions(self, options): + """ Add test control options from the command line to the url + + URL parameters to test URL: + + autorun -- kick off tests automatically + closeWhenDone -- runs quit.js after tests + logFile -- logs test run to an absolute path + totalChunks -- how many chunks to split tests into + thisChunk -- which chunk to run + timeout -- per-test timeout in seconds + """ + + # allow relative paths for logFile + if options.logFile: + options.logFile = self.getFullPath(options.logFile) + if options.browserChrome: + self.makeTestConfig(options) + else: + if options.autorun: + self.urlOpts.append("autorun=1") + if options.timeout: + self.urlOpts.append("timeout=%d" % options.timeout) + if options.closeWhenDone: + self.urlOpts.append("closeWhenDone=1") + if options.logFile: + self.urlOpts.append("logFile=" + encodeURIComponent(options.logFile)) + self.urlOpts.append("fileLevel=" + encodeURIComponent(options.fileLevel)) + if options.consoleLevel: + self.urlOpts.append("consoleLevel=" + encodeURIComponent(options.consoleLevel)) + if options.totalChunks: + self.urlOpts.append("totalChunks=%d" % options.totalChunks) + self.urlOpts.append("thisChunk=%d" % options.thisChunk) + if options.chunkByDir: + self.urlOpts.append("chunkByDir=%d" % options.chunkByDir) + if options.shuffle: + self.urlOpts.append("shuffle=1") + + def cleanup(self, manifest): + """ remove temporary files and profile """ + os.remove(manifest) + shutil.rmtree(self.PROFILE_DIRECTORY) + + def runTests(self, options): + """ Prepare, configure, run tests and cleanup """ + debuggerInfo = getDebuggerInfo(self.oldcwd, options.debugger, options.debuggerArgs, + options.debuggerInteractive); + + browserEnv = self.buildBrowserEnv(options) + if (browserEnv == None): + return 1 + + manifest = self.buildProfile(options) + self.startWebServer(options) + + + testURL = self.buildTestPath(options) + self.buildURLOptions(options) + if (len(self.urlOpts) > 0): + testURL += "?" + "&".join(self.urlOpts) + + self.runExtensionRegistration(options, browserEnv) + # Remove the leak detection file so it can't "leak" to the tests run. # The file is not there if leak logging was not enabled in the application build. if os.path.exists(self.LEAK_REPORT_FILE): @@ -425,7 +468,7 @@ class Mochitest(object): self.automation.log.info("INFO | runtests.py | Running tests: start.\n") status = self.automation.runApp(testURL, browserEnv, options.app, self.PROFILE_DIRECTORY, options.browserArgs, - runSSLTunnel = True, + runSSLTunnel = self.runSSLTunnel, utilityPath = options.utilityPath, xrePath = options.xrePath, certPath=options.certPath, @@ -433,18 +476,11 @@ class Mochitest(object): symbolsPath=options.symbolsPath, timeout = timeout) - # Server's no longer needed, and perhaps more importantly, anything it might - # spew to console shouldn't disrupt the leak information table we print next. - server.stop() - + self.stopWebServer() processLeakLog(self.LEAK_REPORT_FILE, options.leakThreshold) self.automation.log.info("\nINFO | runtests.py | Running tests: end.") - # delete the profile and manifest - os.remove(manifest) - - # hanging due to non-halting threads is no fun; assume we hit the errors we - # were going to hit already and exit. + self.cleanup(manifest) return status def makeTestConfig(self, options): @@ -500,10 +536,8 @@ toolbar#nav-bar { if self.automation.IS_WIN32: chrometestDir = "file:///" + chrometestDir.replace("\\", "/") - - (path, leaf) = os.path.split(options.app) - manifest = os.path.join(path, "chrome", "mochikit.manifest") - manifestFile = open(manifest, "w") + temp_file = os.path.join(tempfile.mkdtemp(), "mochikit.manifest") + manifestFile = open(temp_file, "w") manifestFile.write("content mochikit " + chrometestDir + " contentaccessible=yes\n") if options.browserChrome: @@ -512,6 +546,13 @@ overlay chrome://browser/content/browser.xul chrome://mochikit/content/browser-t """) manifestFile.close() + return self.installChromeFile(temp_file, options) + + def installChromeFile(self, filename, options): + (p, file) = os.path.split(filename) + (path, leaf) = os.path.split(options.app) + manifest = os.path.join(path, "chrome", file) + shutil.copy(filename, manifest) return manifest def copyExtraFilesToProfile(self, options):