diff --git a/build/automation.py.in b/build/automation.py.in index 008458dce09..ce30cd9a644 100644 --- a/build/automation.py.in +++ b/build/automation.py.in @@ -52,6 +52,7 @@ import sys import threading import tempfile import sqlite3 +import zipfile SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0]))) sys.path.insert(0, SCRIPT_DIR) @@ -97,6 +98,69 @@ _log.setLevel(logging.INFO) _log.addHandler(handler) +class ZipFileReader(object): + """ + Class to read zip files in Python 2.5 and later. Limited to only what we + actually use. + """ + + def __init__(self, filename): + self._zipfile = zipfile.ZipFile(filename, "r") + + def __del__(self): + self._zipfile.close() + + def _getnormalizedpath(self, path): + """ + Gets a normalized path from 'path' (or the current working directory if + 'path' is None). Also asserts that the path exists. + """ + if path is None: + path = os.curdir + path = os.path.normpath(os.path.expanduser(path)) + assert os.path.isdir(path) + return path + + def _extractname(self, name, path): + """ + Extracts a file with the given name from the zip file to the given path. + Also creates any directories needed along the way. + """ + filename = os.path.normpath(os.path.join(path, name)) + if name.endswith("/"): + os.makedirs(filename) + else: + path = os.path.split(filename)[0] + if not os.path.isdir(path): + os.makedirs(path) + with open(filename, "wb") as dest: + dest.write(self._zipfile.read(name)) + + def namelist(self): + return self._zipfile.namelist() + + def read(self, name): + return self._zipfile.read(name) + + def extract(self, name, path = None): + if hasattr(self._zipfile, "extract"): + return self._zipfile.extract(name, path) + + # This will throw if name is not part of the zip file. + self._zipfile.getinfo(name) + + self._extractname(name, self._getnormalizedpath(path)) + + def extractall(self, path = None): + if hasattr(self._zipfile, "extractall"): + return self._zipfile.extractall(path) + + path = self._getnormalizedpath(path) + + for name in self._zipfile.namelist(): + self._extractname(name, path) + + ################# # PROFILE SETUP # ################# @@ -988,7 +1052,7 @@ user_pref("camino.use_system_proxy_settings", false); // Camino-only, harmless t os.makedirs(extensionsRootDir) if os.path.isfile(extensionSource): - reader = automationutils.ZipFileReader(extensionSource) + reader = ZipFileReader(extensionSource) for filename in reader.namelist(): # Sanity check the zip file. diff --git a/build/automationutils.py b/build/automationutils.py index be7f34794df..3f52516af8d 100644 --- a/build/automationutils.py +++ b/build/automationutils.py @@ -36,12 +36,11 @@ # # ***** END LICENSE BLOCK ***** */ -import glob, logging, os, platform, shutil, subprocess, sys, tempfile, urllib2, zipfile +import glob, logging, os, platform, shutil, subprocess, sys import re from urlparse import urlparse __all__ = [ - "ZipFileReader", "addCommonOptions", "checkForCrashes", "dumpLeakLog", @@ -71,68 +70,6 @@ DEBUGGER_INFO = { } } -class ZipFileReader(object): - """ - Class to read zip files in Python 2.5 and later. Limited to only what we - actually use. - """ - - def __init__(self, filename): - self._zipfile = zipfile.ZipFile(filename, "r") - - def __del__(self): - self._zipfile.close() - - def _getnormalizedpath(self, path): - """ - Gets a normalized path from 'path' (or the current working directory if - 'path' is None). Also asserts that the path exists. - """ - if path is None: - path = os.curdir - path = os.path.normpath(os.path.expanduser(path)) - assert os.path.isdir(path) - return path - - def _extractname(self, name, path): - """ - Extracts a file with the given name from the zip file to the given path. - Also creates any directories needed along the way. - """ - filename = os.path.normpath(os.path.join(path, name)) - if name.endswith("/"): - os.makedirs(filename) - else: - path = os.path.split(filename)[0] - if not os.path.isdir(path): - os.makedirs(path) - with open(filename, "wb") as dest: - dest.write(self._zipfile.read(name)) - - def namelist(self): - return self._zipfile.namelist() - - def read(self, name): - return self._zipfile.read(name) - - def extract(self, name, path = None): - if hasattr(self._zipfile, "extract"): - return self._zipfile.extract(name, path) - - # This will throw if name is not part of the zip file. - self._zipfile.getinfo(name) - - self._extractname(name, self._getnormalizedpath(path)) - - def extractall(self, path = None): - if hasattr(self._zipfile, "extractall"): - return self._zipfile.extractall(path) - - path = self._getnormalizedpath(path) - - for name in self._zipfile.namelist(): - self._extractname(name, path) - log = logging.getLogger() def isURL(thing): @@ -165,6 +102,7 @@ def addCommonOptions(parser, defaults={}): def checkForCrashes(dumpDir, symbolsPath, testName=None): stackwalkPath = os.environ.get('MINIDUMP_STACKWALK', None) + stackwalkCGI = os.environ.get('MINIDUMP_STACKWALK_CGI', None) # try to get the caller's filename if no test name is given if testName is None: try: @@ -172,70 +110,70 @@ def checkForCrashes(dumpDir, symbolsPath, testName=None): except: testName = "unknown" - # Check preconditions - dumps = glob.glob(os.path.join(dumpDir, '*.dmp')) - if len(dumps) == 0: - return False - foundCrash = False - removeSymbolsPath = False - - # If our symbols are at a remote URL, download them now - if isURL(symbolsPath): - print "Downloading symbols from: " + symbolsPath - removeSymbolsPath = True - # Get the symbols and write them to a temporary zipfile - data = urllib2.urlopen(symbolsPath) - symbolsFile = tempfile.TemporaryFile() - symbolsFile.write(data.read()) - # extract symbols to a temporary directory (which we'll delete after - # processing all crashes) - symbolsPath = tempfile.mkdtemp() - zfile = ZipFileReader(symbolsFile) - zfile.extractall(symbolsPath) - - try: - for d in dumps: - log.info("PROCESS-CRASH | %s | application crashed (minidump found)", testName) - print "Crash dump filename: " + d - if symbolsPath and stackwalkPath and os.path.exists(stackwalkPath): - # run minidump stackwalk - p = subprocess.Popen([stackwalkPath, d, symbolsPath], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - (out, err) = p.communicate() - if len(out) > 3: - # minidump_stackwalk is chatty, so ignore stderr when it succeeds. - print out + dumps = glob.glob(os.path.join(dumpDir, '*.dmp')) + for d in dumps: + log.info("PROCESS-CRASH | %s | application crashed (minidump found)", testName) + print "Crash dump filename: " + d + if symbolsPath and stackwalkPath and os.path.exists(stackwalkPath): + p = subprocess.Popen([stackwalkPath, d, symbolsPath], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + (out, err) = p.communicate() + if len(out) > 3: + # minidump_stackwalk is chatty, so ignore stderr when it succeeds. + print out + else: + print "stderr from minidump_stackwalk:" + print err + if p.returncode != 0: + print "minidump_stackwalk exited with return code %d" % p.returncode + elif stackwalkCGI and symbolsPath and isURL(symbolsPath): + f = None + try: + f = open(d, "rb") + sys.path.append(os.path.join(os.path.dirname(__file__), "poster.zip")) + from poster.encode import multipart_encode + from poster.streaminghttp import register_openers + import urllib2 + register_openers() + datagen, headers = multipart_encode({"minidump": f, + "symbols": symbolsPath}) + request = urllib2.Request(stackwalkCGI, datagen, headers) + result = urllib2.urlopen(request).read() + if len(result) > 3: + print result else: - print "stderr from minidump_stackwalk:" - print err - if p.returncode != 0: - print "minidump_stackwalk exited with return code %d" % p.returncode + print "stackwalkCGI returned nothing." + finally: + if f: + f.close() + else: + if not symbolsPath: + print "No symbols path given, can't process dump." + if not stackwalkPath and not stackwalkCGI: + print "Neither MINIDUMP_STACKWALK nor MINIDUMP_STACKWALK_CGI is set, can't process dump." else: - if not symbolsPath: - print "No symbols path given, can't process dump." - if not stackwalkPath: - print "MINIDUMP_STACKWALK not set, can't process dump." - elif stackwalkPath and not os.path.exists(stackwalkPath): + if stackwalkPath and not os.path.exists(stackwalkPath): print "MINIDUMP_STACKWALK binary not found: %s" % stackwalkPath - dumpSavePath = os.environ.get('MINIDUMP_SAVE_PATH', None) - if dumpSavePath: - shutil.move(d, dumpSavePath) - print "Saved dump as %s" % os.path.join(dumpSavePath, - os.path.basename(d)) - else: - os.remove(d) - extra = os.path.splitext(d)[0] + ".extra" - if os.path.exists(extra): - os.remove(extra) - foundCrash = True - finally: - if removeSymbolsPath: - shutil.rmtree(symbolsPath) + elif stackwalkCGI and not isURL(stackwalkCGI): + print "MINIDUMP_STACKWALK_CGI is not a URL: %s" % stackwalkCGI + elif symbolsPath and not isURL(symbolsPath): + print "symbolsPath is not a URL: %s" % symbolsPath + dumpSavePath = os.environ.get('MINIDUMP_SAVE_PATH', None) + if dumpSavePath: + shutil.move(d, dumpSavePath) + print "Saved dump as %s" % os.path.join(dumpSavePath, + os.path.basename(d)) + else: + os.remove(d) + extra = os.path.splitext(d)[0] + ".extra" + if os.path.exists(extra): + os.remove(extra) + foundCrash = True return foundCrash - + def getFullPath(directory, path): "Get an absolute path relative to 'directory'." return os.path.normpath(os.path.join(directory, os.path.expanduser(path))) diff --git a/layout/tools/reftest/Makefile.in b/layout/tools/reftest/Makefile.in index 94b0ddba4a2..7d589259262 100644 --- a/layout/tools/reftest/Makefile.in +++ b/layout/tools/reftest/Makefile.in @@ -84,6 +84,7 @@ _HARNESS_FILES = \ $(topsrcdir)/build/mobile/devicemanagerADB.py \ $(topsrcdir)/build/mobile/devicemanagerSUT.py \ $(topsrcdir)/build/automationutils.py \ + $(topsrcdir)/build/poster.zip \ $(topsrcdir)/build/mobile/remoteautomation.py \ $(topsrcdir)/testing/mochitest/server.js \ $(topsrcdir)/build/pgo/server-locations.txt \ diff --git a/testing/mochitest/Makefile.in b/testing/mochitest/Makefile.in index 6e38bc9c433..2a8eb7dd21a 100644 --- a/testing/mochitest/Makefile.in +++ b/testing/mochitest/Makefile.in @@ -82,6 +82,7 @@ _SERV_FILES = \ $(topsrcdir)/build/mobile/devicemanagerADB.py \ $(topsrcdir)/build/mobile/devicemanagerSUT.py \ $(topsrcdir)/build/automationutils.py \ + $(topsrcdir)/build/poster.zip \ $(topsrcdir)/build/mobile/remoteautomation.py \ gen_template.pl \ server.js \ diff --git a/testing/xpcshell/Makefile.in b/testing/xpcshell/Makefile.in index d1a37405526..8528c3dba1b 100644 --- a/testing/xpcshell/Makefile.in +++ b/testing/xpcshell/Makefile.in @@ -65,6 +65,7 @@ EXTRA_BUILD_FILES := \ automationutils.py \ manifestparser.py \ mozinfo.py \ + poster.zip \ $(NULL) # And files for running xpcshell remotely from $(topsrcdir)/build/mobile